0ct0pu5/wesnoth
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

[units.wesnoth.org] Converted the script to Python 3.

Browse source 
Basically ran all .py files through 2to3. I made a copy of wmlparser2.py
called wmlparser3.py for the Python3 version, so as to not inconvenience anyone
who may be using it in their old Python2 scripts.
This commit is contained in:
Elias Pschernig 2015-09-09 22:49:41 -04:00
parent 480a2a746c
commit 3ab3db3166
8 changed files with 919 additions and 121 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

12
data/tools/unit_tree/animations.py
View file

@ -119,13 +119,9 @@ def put_units(f, us):
f.write("</tr>\n")
def by_race(u1, u2):
r1 = u1.rid
r2 = u2.rid
r = cmp(r1, r2)
if r == 0: r = cmp(u1.id, u2.id)
return r
us.sort(by_race)
def by_race(u):
return u.rid + u.id
us.sort(key = by_race)
race = None
color = 0
for u in us:
@ -156,7 +152,7 @@ td.none {border: solid 1px; background-color: #ffffff;}
f.write("<i>total frames (number of animations)</i>\n")
f.write("<h2>Mainline</h2>\n")
us = [x for x in wesnoth.unit_lookup.values() if x.campaigns[0] == "mainline"]
us = [x for x in list(wesnoth.unit_lookup.values()) if x.campaigns[0] == "mainline"]
put_units(f, us)
#f.write("<h2>Campaigns and Addons</h2>\n")

29
data/tools/unit_tree/helpers.py
View file

@ -1,10 +1,9 @@
"""
Various helpers for use by the wmlunits tool.
"""
import sys, os, re, glob, shutil, copy, urllib2, subprocess
import sys, os, re, glob, shutil, copy, urllib.request, urllib.error, urllib.parse, subprocess
import wesnoth.wmlparser2 as wmlparser2
import wesnoth.wmltools as wmltools
import wesnoth.wmlparser3 as wmlparser3
def get_datadir(wesnoth_exe):
p = subprocess.Popen([wesnoth_exe, "--path"],
@ -115,7 +114,7 @@ class ImageCollector:
return image.id_name
def copy_and_color_images(self, target_path):
for image in self.images_by_ipath.values():
for image in list(self.images_by_ipath.values()):
opath = os.path.join(target_path, "pics", image.id_name)
try:
os.makedirs(os.path.dirname(opath))
@ -161,7 +160,7 @@ class WesnothList:
self.movetype_lookup = {}
self.era_lookup = {}
self.campaign_lookup = {}
self.parser = wmlparser2.Parser(wesnoth_exe, config_dir,
self.parser = wmlparser3.Parser(wesnoth_exe, config_dir,
data_dir, no_preprocess = False)
@ -184,7 +183,7 @@ class WesnothList:
"""
self.languages_found = {}
parser = wmlparser2.Parser(options.wesnoth, options.config_dir,
parser = wmlparser3.Parser(options.wesnoth, options.config_dir,
options.data_dir, no_preprocess = False)
parser.parse_text("{languages}")
@ -346,7 +345,7 @@ class WesnothList:
"""
# handle advancefrom tags
for uid, unit in self.unit_lookup.items():
for uid, unit in list(self.unit_lookup.items()):
for advancefrom in unit.get_all(tag = "advancefrom"):
fromid = advancefrom.get_text_val("unit")
if fromid:
@ -361,12 +360,12 @@ class WesnothList:
fromunit.advance.append(uid)
def find_unit_factions(self):
for unit in self.unit_lookup.values():
for unit in list(self.unit_lookup.values()):
unit.factions = []
unit.eras = []
for eid, era in self.era_lookup.items():
for fid, multiplayer_side in era.faction_lookup.items():
for eid, era in list(self.era_lookup.items()):
for fid, multiplayer_side in list(era.faction_lookup.items()):
for uid in multiplayer_side.units:
try:
unit = self.unit_lookup[uid]
@ -383,7 +382,7 @@ class WesnothList:
unit.factions.append((eid, fid))
# as a special case, add units from this addon but with no faction
for unit in self.unit_lookup.values():
for unit in list(self.unit_lookup.values()):
if unit.campaigns[0] == self.cid:
if not unit.factions:
if not eid in unit.eras:
@ -436,7 +435,7 @@ class UnitForest:
"""
# Complete the network
for uid, u in self.lookup.items():
for uid, u in list(self.lookup.items()):
for cid in u.child_ids:
c = self.lookup.get(cid, None)
if not c: continue
@ -445,7 +444,7 @@ class UnitForest:
c.parent_ids.append(uid)
# Put all roots into the forest
for uid, u in self.lookup.items():
for uid, u in list(self.lookup.items()):
if not u.parent_ids:
self.trees[uid] = u
@ -464,14 +463,14 @@ class UnitForest:
u.children.remove(c)
for c in u.children:
recurse(c, already2)
for u in self.trees.values():
for u in list(self.trees.values()):
already = {u.id : True}
recurse(u, already)
def update(self):
self.create_network()
self.breadth = sum([x.update_breadth() for x in self.trees.values()])
self.breadth = sum([x.update_breadth() for x in list(self.trees.values())])
return self.breadth
def get_children(self, uid):

108
data/tools/unit_tree/html_output.py
View file

@ -3,7 +3,7 @@
import os, gettext, time, copy, sys, re
import traceback
import unit_tree.helpers as helpers
import wesnoth.wmlparser2 as wmlparser2
import wesnoth.wmlparser3 as wmlparser3
pics_location = "../../pics"
@ -62,10 +62,9 @@ class MyFile:
"""
def __init__(self, filename, mode):
self.filename = filename
self.f = open(filename, mode)
self.f = open(filename, mode + "b")
def write(self, x):
x = x.encode("utf8")
self.f.write(x)
self.f.write(x.encode("utf8"))
def close(self):
self.f.close()
@ -76,7 +75,7 @@ class Translation:
self.localedir = localedir
self.langcode = langcode
class Dummy:
def ugettext(self, x):
def gettext(self, x):
if not x: return ""
caret = x.find("^")
if caret < 0: return x
@ -98,7 +97,7 @@ class Translation:
# gettext.translation call sometimes
self.catalog[textdomain] = self.dummy
r = self.catalog[textdomain].ugettext(string)
r = self.catalog[textdomain].gettext(string)
return r
@ -186,7 +185,7 @@ class HTMLOutput:
# Build an advancement tree forest of all units.
forest = self.forest = helpers.UnitForest()
units_added = {}
for uid, u in self.wesnoth.unit_lookup.items():
for uid, u in list(self.wesnoth.unit_lookup.items()):
if u.hidden: continue
if grouper.unitfilter(u):
forest.add_node(helpers.UnitNode(u))
@ -197,7 +196,7 @@ class HTMLOutput:
# Always add any child units, even if they have been filtered out..
while units_added:
new_units_added = {}
for uid, u in units_added.items():
for uid, u in list(units_added.items()):
for auid in u.advance:
if not auid in units_added:
try:
@ -216,7 +215,7 @@ class HTMLOutput:
added = True
while added:
added = False
for uid, u in self.wesnoth.unit_lookup.items():
for uid, u in list(self.wesnoth.unit_lookup.items()):
if uid in forest.lookup: continue
for auid in u.advance:
if auid in forest.lookup:
@ -230,7 +229,7 @@ class HTMLOutput:
groups = {}
breadth = 0
for tree in forest.trees.values():
for tree in list(forest.trees.values()):
u = tree.unit
ugroups = grouper.groups(u)
@ -238,35 +237,32 @@ class HTMLOutput:
groups[group] = groups.get(group, []) + [tree]
breadth += tree.breadth
thelist = groups.keys()
thelist = list(groups.keys())
thelist.sort(key = lambda x: grouper.group_name(x))
rows_count = breadth + len(thelist)
# Create empty grid.
rows = []
for j in xrange(rows_count):
for j in range(rows_count):
column = []
for i in xrange(6):
for i in range(6):
column.append((1, 1, None))
rows.append(column)
# Sort advancement trees by name of first unit and place into the grid.
def by_name(t1, t2):
u1 = t1.unit
u2 = t2.unit
u1name = T(u1, "name")
u2name = T(u2, "name")
return cmp(u1name, u2name)
def by_name(t):
x = T(t.unit, "name")
if x is None: return ""
return x
def grid_place(nodes, x):
nodes.sort(by_name)
nodes.sort(key = by_name)
for node in nodes:
level = node.unit.level
if level < 0: level = 0
if level > 5: level = 5
rows[x][level] = (1, node.breadth, node)
for i in xrange(1, node.breadth):
for i in range(1, node.breadth):
rows[x + i][level] = (0, 0, node)
grid_place(node.children, x)
x += node.breadth
@ -278,7 +274,7 @@ class HTMLOutput:
node.name = grouper.group_name(group)
rows[x][0] = (6, 1, node)
for i in xrange(1, 6):
for i in range(1, 6):
rows[x][i] = (0, 0, None)
nodes = groups[group]
x += 1
@ -294,7 +290,7 @@ class HTMLOutput:
all_written_html_files.append((self.isocode, self.output.filename))
languages = self.wesnoth.languages_found
langlist = languages.keys()
langlist = list(languages.keys())
langlist.sort()
write(top_bar % {"path" : "../../"})
@ -310,7 +306,7 @@ class HTMLOutput:
def abbrev(name):
abbrev = name[0]
word_seperators = [" ", "_", "+", "(", ")"]
for i in xrange(1, len(name)):
for i in range(1, len(name)):
if name[i] in ["+", "(", ")"] or name[i - 1] in word_seperators and name[i] not in word_seperators:
abbrev += name[i]
return abbrev
@ -354,7 +350,7 @@ class HTMLOutput:
self.translate("all", "wesnoth-editor")))
r = {}, {}
for u in self.wesnoth.unit_lookup.values():
for u in list(self.wesnoth.unit_lookup.values()):
race = u.race
racename = T(race, "plural_name")
@ -365,7 +361,7 @@ class HTMLOutput:
r[m][racename] = race.get_text_val("id") if race else "none"
racenames = sorted(r[0].items())
if r[1].items():
if list(r[1].items()):
racenames += [("-", "-")] + sorted(r[1].items())
for racename, rid in racenames:
@ -381,7 +377,7 @@ class HTMLOutput:
add_menu("races_menu", x)
for row in self.unitgrid:
for column in xrange(6):
for column in range(6):
hspan, vspan, un = row[column]
if not un: continue
if isinstance(un, helpers.GroupNode):
@ -397,7 +393,7 @@ class HTMLOutput:
class Entry: pass
races = {}
for uid, u in self.wesnoth.unit_lookup.items():
for uid, u in list(self.wesnoth.unit_lookup.items()):
if self.campaign != "units":
if self.campaign not in u.campaigns: continue
if u.race:
@ -488,7 +484,7 @@ class HTMLOutput:
if portrait:
portrait = portrait[0].get_text_val("image")
if not image:
if x.name == "female":
if x.name == b"female":
baseunit = self.wesnoth.get_base_unit(u)
if baseunit:
female = baseunit.get_all(tag="female")
@ -531,7 +527,7 @@ class HTMLOutput:
def copy_attributes(copy_from, copy_to):
for c in copy_from.data:
if isinstance(c, wmlparser2.AttributeNode):
if isinstance(c, wmlparser3.AttributeNode):
copy_to.data.append(c)
# Use attacks of base_units as base, if we have one.
@ -556,7 +552,7 @@ class HTMLOutput:
rows = self.unitgrid
write("<table class=\"units\">\n")
write("<colgroup>")
for i in xrange(6):
for i in range(6):
write("<col class=\"col%d\" />" % i)
write("</colgroup>")
@ -564,9 +560,9 @@ class HTMLOutput:
"../../../images/misc/leader-crown.png", no_tc=True)
crownimage = os.path.join(pics_location, pic)
ms = None
for row in xrange(len(rows)):
for row in range(len(rows)):
write("<tr>\n")
for column in xrange(6):
for column in range(6):
hspan, vspan, un = rows[row][column]
if vspan:
attributes = ""
@ -613,15 +609,15 @@ class HTMLOutput:
crown = ""
if ms:
if un.id in ms.units:
crown = u" ♟"
crown = " ♟"
if un.id in ms.is_leader:
crown = u" ♚"
crown = " ♚"
uaddon = "mainline"
if "mainline" not in u.campaigns: uaddon = self.addon
link = "../../%s/%s/%s.html" % (uaddon, self.isocode, uid)
write("<div class=\"i\"><a href=\"%s\" title=\"id=%s\">%s</a>" % (
link, uid, u"i"))
link, uid, "i"))
write("</div>")
write("<div class=\"l\">L%s%s</div>" % (level, crown))
write("<a href=\"%s\">%s</a><br/>" % (link, name))
@ -631,7 +627,7 @@ class HTMLOutput:
write('<a href=\"%s\">' % link)
if crown == u" ♚":
if crown == " ♚":
write('<div style="background: url(%s)">' % image)
write('<img src="%s" alt="(image)" />' % crownimage)
write("</div>")
@ -949,7 +945,7 @@ class HTMLOutput:
write('</div>')
write('<div class="unit-column-right">')
for si in xrange(2):
for si in range(2):
if si and not female: break
if si:
sportrait = fportrait
@ -979,7 +975,7 @@ class HTMLOutput:
terrains = self.wesnoth.terrain_lookup
terrainlist = []
already = {}
for tstring, t in terrains.items():
for tstring, t in list(terrains.items()):
tid = t.get_text_val("id")
if tid in ["off_map", "off_map2", "fog", "shroud", "impassable",
"void", "rails"]: continue
@ -1064,7 +1060,7 @@ def generate_campaign_report(addon, isocode, campaign, wesnoth):
cid = "mainline"
if not cid: cid = addon + "_" + campaign.get_text_val("define")
print("campaign " + addon + " " + cid + " " + isocode)
print(("campaign " + addon + " " + cid + " " + isocode))
path = os.path.join(options.output, addon, isocode)
if not os.path.isdir(path): os.mkdir(path)
@ -1089,7 +1085,7 @@ def generate_campaign_report(addon, isocode, campaign, wesnoth):
def generate_era_report(addon, isocode, era, wesnoth):
eid = era.get_text_val("id")
print("era " + addon + " " + eid + " " + isocode)
print(("era " + addon + " " + eid + " " + isocode))
path = os.path.join(options.output, addon, isocode)
if not os.path.isdir(path): os.mkdir(path)
@ -1116,13 +1112,13 @@ def generate_single_unit_reports(addon, isocode, wesnoth):
grouper = GroupByNothing()
html.analyze_units(grouper, True)
for uid, unit in wesnoth.unit_lookup.items():
for uid, unit in list(wesnoth.unit_lookup.items()):
if unit.hidden: continue
if "mainline" in unit.campaigns and addon != "mainline": continue
try:
htmlname = u"%s.html" % uid
filename = os.path.join(path, htmlname).encode("utf8")
htmlname = "%s.html" % uid
filename = os.path.join(path, htmlname)
# We probably can come up with something better.
if os.path.exists(filename):
@ -1142,10 +1138,10 @@ def generate_single_unit_reports(addon, isocode, wesnoth):
def html_postprocess_file(filename, isocode, batchlist):
print(u"postprocessing " + repr(filename))
print(("postprocessing " + repr(filename)))
chtml = u""
ehtml = u""
chtml = ""
ehtml = ""
cids = [[], []]
for addon in batchlist:
@ -1161,7 +1157,7 @@ def html_postprocess_file(filename, isocode, batchlist):
else:
cids[1].append(c)
for i in xrange(2):
for i in range(2):
campaigns = cids[i]
campaigns.sort(key = lambda x: "A" if x[1] == "mainline" else "B" + x[2])
@ -1169,10 +1165,10 @@ def html_postprocess_file(filename, isocode, batchlist):
for campaign in campaigns:
addon, cname, campname, lang = campaign
chtml += u" <a title=\"%s\" href=\"../../%s/%s/%s.html\">%s</a><br/>\n" % (
chtml += " <a title=\"%s\" href=\"../../%s/%s/%s.html\">%s</a><br/>\n" % (
campname, addon, lang, cname, campname)
if i == 0 and cids[1]:
chtml += u"-<br/>\n"
chtml += "-<br/>\n"
eids = [[], []]
for addon in batchlist:
@ -1188,22 +1184,22 @@ def html_postprocess_file(filename, isocode, batchlist):
else:
eids[1].append(e)
for i in xrange(2):
for i in range(2):
eras = eids[i]
eras.sort(key = lambda x: x[2])
for era in eras:
addon, eid, eraname, lang = era
ehtml += u" <a title=\"%s\" href=\"../../%s/%s/%s.html\">%s</a><br/>" % (
ehtml += " <a title=\"%s\" href=\"../../%s/%s/%s.html\">%s</a><br/>" % (
eraname, addon, lang, eid, eraname)
if i == 0 and eids[1]:
ehtml += u"-<br/>\n"
ehtml += "-<br/>\n"
f = open(filename, "r+b")
html = f.read().decode("utf8")
html = html.replace(u"PLACE CAMPAIGNS HERE\n", chtml)
html = html.replace(u"PLACE ERAS HERE\n", ehtml)
html = html.replace("PLACE CAMPAIGNS HERE\n", chtml)
html = html.replace("PLACE ERAS HERE\n", ehtml)
f.seek(0)
f.write(html.encode("utf8"))
f.close()

10
data/tools/unit_tree/overview.py
View file

@ -1,11 +1,11 @@
#!/usr/bin/env python
import glob, os, sys, time, re
sys.path.append(os.path.join(os.path.dirname(__file__), ".."))
import html_output
from . import html_output
def write_addon_overview(folder, addon):
out = open(os.path.join(folder, "index.html"), "wb")
def w(x): out.write(x.encode("utf8") + "\n")
out = open(os.path.join(folder, "index.html"), "w")
def w(x): out.write(x + "\n")
name = addon["name"]
@ -50,8 +50,8 @@ def write_addon_overview(folder, addon):
def main(folder):
out = open(os.path.join(folder, "overview.html"), "wb")
def w(x): out.write(x.encode("utf8") + "\n")
out = open(os.path.join(folder, "overview.html"), "w")
def w(x): out.write(x + "\n")
path = ""
title = "Wesnoth Unit Database Overview"

23
data/tools/unit_tree/wiki_output.py
View file

@ -1,5 +1,6 @@
import helpers, sys, helpers
from html_output import Translation
import sys
from . import helpers
from .html_output import Translation
def main():
wesnoth = helpers.WesnothList(
@ -40,14 +41,14 @@ def main():
# Get all defined races.
races = {}
for campaign, unitslists in punits.items():
for campaign, unitslists in list(punits.items()):
for unitlist in unitslists:
for race in unitlist.get_all(tag = "race"):
races[race.get_text_val("id")] = race
# Go through all units and put them into a dictionary.
all_units = {}
for campaign, unitslists in punits.items():
for campaign, unitslists in list(punits.items()):
for unitlist in unitslists:
for unit in unitlist.get_all(tag = "unit_type"):
if unit.get_text_val("do_not_list") in ["yes", "true"]: continue
@ -68,7 +69,7 @@ def main():
return None
# Handle unit attributes
for unit in all_units.values():
for unit in list(all_units.values()):
unit.name = base_val(unit, "name", translation = translated.translate)
unit.orig_name = base_val(unit, "name", translation = original.translate)
@ -85,7 +86,7 @@ def main():
else: unit.advances_to = [x.strip() for x in a.split(",")]
# Find children and parents of all units.
for unit in all_units.values():
for unit in list(all_units.values()):
for aid in unit.advances_to:
unit.children.append(all_units[aid])
all_units[aid].parents.append(unit)
@ -100,13 +101,13 @@ def main():
# Group by race/campaign
units_per_race = {}
for unit in all_units.values():
for unit in list(all_units.values()):
x = race_key(unit)
if x not in units_per_race: units_per_race[x] = set()
units_per_race[x].add(unit)
# Recursively add all related units of a units to the same race as well.
for race in units_per_race.keys():
for race in list(units_per_race.keys()):
while True:
add = []
for unit in units_per_race[race]:
@ -153,14 +154,14 @@ def main():
# Create grid.
grid = []
for j in xrange(n + 1):
for j in range(n + 1):
grid.append([None] * 6)
for unit in units:
grid[unit.y][unit.level] = unit
# Output it.
for y in xrange(n + 1):
for x in xrange(6):
for y in range(n + 1):
for x in range(6):
unit = grid[y][x]
if unit:
w("|'''" + unit.name + "'''<br />" + unit.orig_name)

3
data/tools/wesnoth/wmlparser2.py
View file

@ -4,6 +4,9 @@
"""
This parser uses the --preprocess option of wesnoth so a working
wesnoth executable must be available at runtime.
If you are using this you shold instead use wmlparser3.py and upgrade
your code to Python 3.
"""
import os, glob, sys, re, subprocess, argparse, tempfile, shutil

803
data/tools/wesnoth/wmlparser3.py Executable file
View file

@ -0,0 +1,803 @@
#!/usr/bin/env python3
# encoding: utf8
"""
This parser uses the --preprocess option of wesnoth so a working
wesnoth executable must be available at runtime.
"""
import os, glob, sys, re, subprocess, argparse, tempfile, shutil
import atexit
tempdirs_to_clean = []
@atexit.register
def cleaner():
for temp_dir in tempdirs_to_clean:
shutil.rmtree(temp_dir, ignore_errors=True)
class WMLError(Exception):
"""
Catch this exception to retrieve the first error message from
the parser.
"""
def __init__(self, parser=None, message=None):
if parser:
self.line = parser.parser_line
self.wml_line = parser.last_wml_line
self.message = message
self.preprocessed = parser.preprocessed
def __str__(self):
return """WMLError:
%s %s
%s
%s
""" % (str(self.line), self.preprocessed, self.wml_line, self.message)
class StringNode:
"""
One part of an attribute's value. Because a single WML string
can be made from multiple translatable strings we need to model
it this way (as a list of several StringNode).
"""
def __init__(self, data):
self.textdomain = None # non-translatable by default
self.data = data
def debug(self):
if self.textdomain:
return "_<%s>'%s'" % (self.textdomain,
self.data.decode("utf8", "ignore"))
else:
return "'%s'" % self.data.decode("utf8", "ignore")
class AttributeNode:
"""
A WML attribute. For example the "id=Elfish Archer" in:
[unit]
id=Elfish Archer
[/unit]
"""
def __init__(self, name, location=None):
self.name = name
self.location = location
self.value = [] # List of StringNode
def debug(self):
return self.name.decode("utf8") + "=" + " .. ".join(
[v.debug() for v in self.value])
def get_text(self, translation=None):
r = ""
for s in self.value:
ustr = s.data.decode("utf8", "ignore")
if translation:
r += translation(ustr, s.textdomain)
else:
r += ustr
return r
class TagNode:
"""
A WML tag. For example the "unit" in this example:
[unit]
id=Elfish Archer
[/unit]
"""
def __init__(self, name, location=None):
self.name = name
self.location = location
# List of child elements, which are either of type TagNode or
# AttributeNode.
self.data = []
self.speedy_tags = {}
def debug(self):
s = "[%s]\n" % self.name.decode("utf8")
for sub in self.data:
for subline in sub.debug().splitlines():
s += " %s\n" % subline
s += "[/%s]\n" % self.name.decode("utf8")
return s
def get_all(self, **kw):
"""
This gets all child tags or child attributes of the tag.
For example:
[unit]
name=A
name=B
[attack]
[/attack]
[attack]
[/attack]
[/unit]
unit.get_all(att = "name")
will return two nodes for "name=A" and "name=B"
unit.get_all(tag = "attack")
will return two nodes for the two [attack] tags.
unit.get_all()
will return 4 nodes for all 4 sub-elements.
unit.get_all(att = "")
Will return the two attribute nodes.
unit.get_all(tag = "")
Will return the two tag nodes.
If no elements are found an empty list is returned.
"""
if len(kw) == 1 and "tag" in kw and kw["tag"]:
return self.speedy_tags.get(kw["tag"].encode("utf8"), [])
r = []
for sub in self.data:
ok = True
for k, v in list(kw.items()):
v = v.encode("utf8")
if k == "tag":
if not isinstance(sub, TagNode): ok = False
elif v != b"" and sub.name != v: ok = False
elif k == "att":
if not isinstance(sub, AttributeNode): ok = False
elif v != b"" and sub.name != v: ok = False
if ok:
r.append(sub)
return r
def get_text_val(self, name, default=None, translation=None, val=-1):
"""
Returns the value of the specified attribute. If the attribute
is given multiple times, the value number val is returned (default
behaviour being to return the last value). If the
attribute is not found, the default parameter is returned.
If a translation is specified, it should be a function which
when passed a unicode string and text-domain returns a
translation of the unicode string. The easiest way is to pass
it to gettext.translation if you have the binary message
catalogues loaded.
"""
x = self.get_all(att=name)
if not x: return default
return x[val].get_text(translation)
def append(self, node):
self.data.append(node)
if isinstance(node, TagNode):
if node.name not in self.speedy_tags:
self.speedy_tags[node.name] = []
self.speedy_tags[node.name].append(node)
class RootNode(TagNode):
"""
The root node. There is exactly one such node.
"""
def __init__(self):
TagNode.__init__(self, None)
def debug(self):
s = ""
for sub in self.data:
for subline in sub.debug().splitlines():
s += subline + "\n"
return s
class Parser:
def __init__(self, wesnoth_exe, config_dir, data_dir,
no_preprocess):
"""
path - Path to the file to parse.
wesnoth_exe - Wesnoth executable to use. This should have been
configured to use the desired data and config directories.
"""
self.wesnoth_exe = wesnoth_exe
self.config_dir = None
if config_dir: self.config_dir = os.path.abspath(config_dir)
self.data_dir = None
if data_dir: self.data_dir = os.path.abspath(data_dir)
self.keep_temp_dir = None
self.temp_dir = None
self.no_preprocess = no_preprocess
self.preprocessed = None
self.verbose = False
self.last_wml_line = "?"
self.parser_line = 0
self.line_in_file = 42424242
self.chunk_start = "?"
def parse_file(self, path, defines=""):
self.path = path
if not self.no_preprocess:
self.preprocess(defines)
self.parse()
def parse_text(self, text, defines=""):
temp = tempfile.NamedTemporaryFile(prefix="wmlparser_",
suffix=".cfg")
temp.write(text.encode("utf8"))
temp.flush()
self.path = temp.name
if not self.no_preprocess:
self.preprocess(defines)
self.parse()
def preprocess(self, defines):
"""
Call wesnoth --preprocess to get preprocessed WML which we
can subsequently parse.
If this is not called then the .parse method will assume the
WML is already preprocessed.
"""
if self.keep_temp_dir:
output = self.keep_temp_dir
else:
output = tempfile.mkdtemp(prefix="wmlparser_")
tempdirs_to_clean.append(output)
self.temp_dir = output
commandline = [self.wesnoth_exe]
if self.data_dir:
commandline += ["--data-dir", self.data_dir]
if self.config_dir:
commandline += ["--config-dir", self.config_dir]
commandline += ["--preprocess", self.path, output]
if defines:
commandline += ["--preprocess-defines", defines]
if self.verbose:
print((" ".join(commandline)))
p = subprocess.Popen(commandline,
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
out, err = p.communicate()
if self.verbose:
print((out + err))
self.preprocessed = output + "/" + os.path.basename(self.path) +\
".plain"
if not os.path.exists(self.preprocessed):
first_line = open(self.path).readline().strip()
raise WMLError(self, "Preprocessor error:\n" +
" ".join(commandline) + "\n" +
"First line: " + first_line + "\n" +
out +
err)
def parse_line_without_commands(self, line):
"""
Once the .plain commands are handled WML lines are passed to
this.
"""
if not line: return
if line.strip():
self.skip_newlines_after_plus = False
if self.in_tag:
self.handle_tag(line)
return
if self.in_arrows:
arrows = line.find(b'>>')
if arrows >= 0:
self.in_arrows = False
self.temp_string += line[:arrows]
self.temp_string_node = StringNode(self.temp_string)
self.temp_string = b""
self.temp_key_nodes[self.commas].value.append(
self.temp_string_node)
self.in_arrows = False
self.parse_line_without_commands(line[arrows + 2:])
else:
self.temp_string += line
return
quote = line.find(b'"')
if not self.in_string:
arrows = line.find(b'<<')
if arrows >= 0 and (quote < 0 or quote > arrows):
self.parse_line_without_commands(line[:arrows])
self.in_arrows = True
self.parse_line_without_commands(line[arrows + 2:])
return
if quote >= 0:
if self.in_string:
# double quote
if quote < len(line) - 1 and line[quote + 1] == b'"'[0]:
self.temp_string += line[:quote + 1]
self.parse_line_without_commands(line[quote + 2:])
return
self.temp_string += line[:quote]
self.temp_string_node = StringNode(self.temp_string)
if self.translatable:
self.temp_string_node.textdomain = self.textdomain
self.translatable = False
self.temp_string = b""
if not self.temp_key_nodes:
raise WMLError(self, "Unexpected string value.")
self.temp_key_nodes[self.commas].value.append(
self.temp_string_node)
self.in_string = False
self.parse_line_without_commands(line[quote + 1:])
else:
self.parse_outside_strings(line[:quote])
self.in_string = True
self.parse_line_without_commands(line[quote + 1:])
else:
if self.in_string:
self.temp_string += line
else:
self.parse_outside_strings(line)
def parse_outside_strings(self, line):
"""
Parse a WML fragment outside of strings.
"""
if not line: return
if not self.temp_key_nodes:
line = line.lstrip()
if not line: return
if line.startswith(b"#textdomain "):
self.textdomain = line[12:].strip().decode("utf8")
return
# Is it a tag?
if line.startswith(b"["):
self.handle_tag(line)
# No tag, must be an attribute.
else:
self.handle_attribute(line)
else:
for i, segment in enumerate(line.split(b"+")):
segment = segment.lstrip(b" ")
if i > 0:
# If the last segment is empty (there was a plus sign
# at the end) we need to skip newlines.
self.skip_newlines_after_plus = not segment.strip()
if not segment: continue
if segment.startswith(b"_"):
self.translatable = True
segment = segment[1:].lstrip(b" ")
if not segment: continue
self.handle_value(segment)
def handle_tag(self, line):
end = line.find(b"]")
if end < 0:
if line.endswith(b"\n"):
raise WMLError(self, "Expected closing bracket.")
self.in_tag += line
return
tag = (self.in_tag + line[:end])[1:]
self.in_tag = b""
if tag.startswith(b"/"):
self.parent_node = self.parent_node[:-1]
else:
node = TagNode(tag, location=(self.line_in_file, self.chunk_start))
if self.parent_node:
self.parent_node[-1].append(node)
self.parent_node.append(node)
self.parse_outside_strings(line[end + 1:])
def handle_attribute(self, line):
assign = line.find(b"=")
remainder = None
if assign >= 0:
remainder = line[assign + 1:]
line = line[:assign]
self.commas = 0
self.temp_key_nodes = []
for att in line.split(b","):
att = att.strip()
node = AttributeNode(att, location=(self.line_in_file, self.chunk_start))
self.temp_key_nodes.append(node)
if self.parent_node:
self.parent_node[-1].append(node)
if remainder:
self.parse_outside_strings(remainder)
def handle_value(self, segment):
def add_text(segment):
segment = segment.rstrip()
if not segment: return
n = len(self.temp_key_nodes)
maxsplit = n - self.commas - 1
if maxsplit < 0: maxsplit = 0
for subsegment in segment.split(b",", maxsplit):
self.temp_string += subsegment.strip()
self.temp_string_node = StringNode(self.temp_string)
self.temp_string = b""
self.temp_key_nodes[self.commas].value.append(
self.temp_string_node)
if self.commas < n - 1:
self.commas += 1
# Finish assignment on newline, except if there is a
# plus sign before the newline.
add_text(segment)
if segment.endswith(b"\n") and not self.skip_newlines_after_plus:
self.temp_key_nodes = []
def parse(self):
"""
Parse preprocessed WML into a tree of tags and attributes.
"""
# parsing state
self.temp_string = b""
self.temp_string_node = None
self.commas = 0
self.temp_key_nodes = []
self.in_string = False
self.in_arrows = False
self.textdomain = "wesnoth"
self.translatable = False
self.root = RootNode()
self.parent_node = [self.root]
self.skip_newlines_after_plus = False
self.in_tag = b""
command_marker_byte = bytes([254])
input = self.preprocessed
if not input: input = self.path
for rawline in open(input, "rb"):
compos = rawline.find(command_marker_byte)
self.parser_line += 1
# Everything from chr(254) to newline is the command.
if compos != 0:
self.line_in_file += 1
if compos >= 0:
self.parse_line_without_commands(rawline[:compos])
self.handle_command(rawline[compos + 1:-1])
else:
self.parse_line_without_commands(rawline)
if self.keep_temp_dir is None and self.temp_dir:
if self.verbose:
print(("removing " + self.temp_dir))
shutil.rmtree(self.temp_dir, ignore_errors=True)
def handle_command(self, com):
if com.startswith(b"line "):
self.last_wml_line = com[5:]
_ = self.last_wml_line.split(b" ")
self.chunk_start = [(_[i+1], int(_[i])) for i in range(0, len(_), 2)]
self.line_in_file = self.chunk_start[0][1]
elif com.startswith(b"textdomain "):
self.textdomain = com[11:].decode("utf8")
else:
raise WMLError(self, "Unknown parser command: " + com)
def get_all(self, **kw):
return self.root.get_all(**kw)
def get_text_val(self, name, default=None, translation=None):
return self.root.get_text_val(name, default, translation)
import json
def jsonify(tree, verbose=False, depth=0):
"""
Convert a DataSub into JSON
If verbose, insert a linebreak after every brace and comma (put every item on its own line), otherwise, condense everything into a single line.
"""
print("{", end=' ')
first = True
sdepth1 = "\n" + " " * depth
sdepth2 = sdepth1 + " "
for pair in tree.speedy_tags.items():
if first:
first = False
else:
sys.stdout.write(",")
if verbose:
sys.stdout.write(sdepth2)
print('"%s":' % pair[0], end=' ')
if verbose:
sys.stdout.write(sdepth1)
print('[', end=' ')
first_tag = True
for tag in pair[1]:
if first_tag:
first_tag = False
else:
sys.stdout.write(",")
if verbose:
sys.stdout.write(sdepth2)
jsonify(tag, verbose, depth + 2)
if verbose:
sys.stdout.write(sdepth2)
sys.stdout.write("]")
for child in tree.data:
if isinstance(child, TagNode):
continue
if first:
first = False
else:
sys.stdout.write(",")
if verbose:
sys.stdout.write(sdepth2)
print('"%s":' % child.name, end=' ')
print(json.dumps(child.get_text()), end=' ')
if verbose:
sys.stdout.write(sdepth1)
sys.stdout.write("}")
from xml.sax.saxutils import escape
def xmlify(tree, verbose=False, depth=0):
sdepth = ""
if verbose:
sdepth = " " * depth
for child in tree.data:
if isinstance(child, TagNode):
print('%s<%s>' % (sdepth, child.name))
xmlify(child, verbose, depth + 1)
print('%s</%s>' % (sdepth, child.name))
else:
if "\n" in child.get_text() or "\r" in child.get_text():
print(sdepth + '<' + child.name + '>' + \
'<![CDATA[' + child.get_text() + ']]>' + '</' + child.name + '>')
else:
print(sdepth + '<' + child.name + '>' + \
escape(child.get_text()) + '</' + child.name + '>')
if __name__ == "__main__":
arg = argparse.ArgumentParser()
arg.add_argument("-a", "--data-dir", help="directly passed on to wesnoth.exe")
arg.add_argument("-c", "--config-dir", help="directly passed on to wesnoth.exe")
arg.add_argument("-i", "--input", help="a WML file to parse")
arg.add_argument("-k", "--keep-temp", help="specify directory where to keep temp files")
arg.add_argument("-t", "--text", help="WML text to parse")
arg.add_argument("-w", "--wesnoth", help="path to wesnoth.exe")
arg.add_argument("-d", "--defines", help="comma separated list of WML defines")
arg.add_argument("-T", "--test", action="store_true")
arg.add_argument("-j", "--to-json", action="store_true")
arg.add_argument("-n", "--no-preprocess", action="store_true")
arg.add_argument("-v", "--verbose", action="store_true")
arg.add_argument("-x", "--to-xml", action="store_true")
args = arg.parse_args()
if not args.input and not args.text and not args.test:
sys.stderr.write("No input given. Use -h for help.\n")
sys.exit(1)
if not args.no_preprocess and (not args.wesnoth or not
os.path.exists(args.wesnoth)):
sys.stderr.write("Wesnoth executable not found.\n")
sys.exit(1)
if args.test:
print("Running tests")
p = Parser(args.wesnoth, args.config_dir,
args.data_dir, args.no_preprocess)
if args.keep_temp:
p.keep_temp_dir = args.keep_temp
if args.verbose: p.verbose = True
only = None
def test2(input, expected, note, function):
if only and note != only: return
input = input.strip()
expected = expected.strip()
p.parse_text(input)
output = function(p).strip()
if output != expected:
print("__________")
print(("FAILED " + note))
print("INPUT:")
print(input)
print("OUTPUT:")
print(output)
print("EXPECTED:")
print(expected)
print("__________")
else:
print(("PASSED " + note))
def test(input, expected, note):
test2(input, expected, note, lambda p: p.root.debug())
test(
"""
[test]
a=1
[/test]
""", """
[test]
a='1'
[/test]
""", "simple")
test(
"""
[test]
a, b, c = 1, 2, 3
[/test]
""", """
[test]
a='1'
b='2'
c='3'
[/test]
""", "multi assign")
test(
"""
[test]
a, b = 1, 2, 3
[/test]
""", """
[test]
a='1'
b='2, 3'
[/test]
""", "multi assign 2")
test(
"""
[test]
a, b, c = 1, 2
[/test]
""", """
[test]
a='1'
b='2'
c=
[/test]
""", "multi assign 3")
test(
"""
#textdomain A
#define X
_ "abc"
#enddef
#textdomain B
[test]
x = _ "abc" + {X}
[/test]
""", """
[test]
x=_<B>'abc' .. _<A>'abc'
[/test]
""", "textdomain")
test(
"""
[test]
a = "a ""quoted"" word"
[/test]
""",
"""
[test]
a='a "quoted" word'
[/test]
""", "quoted")
test(
"""
[test]
code = <<
"quotes" here
""blah""
>>
[/test]
""",
"""
[test]
code='
"quotes" here
""blah""
'
[/test]
""", "quoted2")
test(
"""
foo="bar"+
"baz"
""",
"""
foo='bar' .. 'baz'
""", "multi line string")
test(
"""
#define baz
"baz"
#enddef
foo="bar"+{baz}
""",
"""
foo='bar' .. 'baz'
""", "defined multi line string")
test(
"""
foo="bar" + "baz" # blah
""",
"""
foo='bar' .. 'baz'
""", "comment after +")
test(
"""
#define baz
"baz"
#enddef
foo="bar" {baz}
""",
"""
foo='bar' .. 'baz'
""", "defined string concatenation")
test(
"""
#define A BLOCK
[{BLOCK}]
[/{BLOCK}]
#enddef
{A blah}
""",
"""
[blah]
[/blah]
""", "defined tag")
test2(
"""
[test]
a=1
b=2
a=3
b=4
[/test]
""", "3, 4", "multiatt",
lambda p:
p.get_all(tag = "test")[0].get_text_val("a") + ", " +
p.get_all(tag = "test")[0].get_text_val("b"))
sys.exit(0)
p = Parser(args.wesnoth, args.config_dir, args.data_dir,
args.no_preprocess)
if args.keep_temp:
p.keep_temp_dir = args.keep_temp
if args.verbose: p.verbose = True
if args.input: p.parse_file(args.input, args.defines)
elif args.text: p.parse_text(args.text, args.defines)
if args.to_json:
jsonify(p.root, True)
print()
elif args.to_xml:
print('<?xml version="1.0" encoding="UTF-8" ?>')
print('<root>')
xmlify(p.root, True, 1)
print('</root>')
else:
print((p.root.debug()))

52
data/tools/wmlunits
View file

@ -1,4 +1,4 @@
#!/usr/bin/env python
#!/usr/bin/env python3
#encoding: utf8
"""
wmlunits -- tool to output information on all units in HTML
@ -10,11 +10,11 @@ Run without arguments to see usage.
try: import psyco; psyco.full()
except ImportError: pass
import sys, os, glob, shutil, urllib2, argparse, traceback
import sys, os, glob, shutil, urllib.request, urllib.error, urllib.parse, argparse, traceback
import subprocess, yaml
import multiprocessing, Queue
import multiprocessing, queue
import wesnoth.wmlparser2 as wmlparser2
import wesnoth.wmlparser3 as wmlparser3
import unit_tree.helpers as helpers
import unit_tree.animations as animations
import unit_tree.html_output as html_output
@ -36,9 +36,9 @@ def copy_images():
"http://www.wesnoth.org/mw/skins/glamdrol/navbg.png"]:
local = os.path.join(options.output, grab[grab.rfind("/") + 1:])
if not os.path.exists(local):
print "Fetching", grab
url = urllib2.urlopen(grab)
file(local, "w").write(url.read())
print("Fetching", grab)
url = urllib.request.urlopen(grab)
open(local, "wb").write(url.read())
def shell(com):
#print(com)
@ -77,13 +77,13 @@ def get_info(addon):
try:
path = options.addons + "/" + addon + "/_info.cfg"
if os.path.exists(path):
parser = wmlparser2.Parser(options.wesnoth, options.config_dir,
parser = wmlparser3.Parser(options.wesnoth, options.config_dir,
options.data_dir, no_preprocess = False)
parser.parse_file(path)
_info[addon] = parser
else:
print("Cannot find " + path)
except wmlparser2.WMLError as e:
print(("Cannot find " + path))
except wmlparser3.WMLError as e:
print(e)
return _info[addon]
@ -106,7 +106,7 @@ def get_dependencies(addon):
if d in global_addons:
_deps[addon].append(d)
else:
print("Missing dependency for " + addon + ": " + d)
print(("Missing dependency for " + addon + ": " + d))
except Exception as e:
print(e)
return _deps[addon]
@ -140,7 +140,7 @@ def sorted_by_dependencies(addons):
n += 1
continue
if n == 0:
print("Cannot sort dependencies for these addons: " + str(unsorted))
print(("Cannot sort dependencies for these addons: " + str(unsorted)))
sorted += unsorted
break
return sorted
@ -238,7 +238,7 @@ def list_contents():
p.start()
try:
s, local.wesnoth = q.get(timeout = TIMEOUT)
except Queue.Empty:
except queue.Empty:
p.terminate()
raise
#print("local", s, local.wesnoth)
@ -322,14 +322,14 @@ def list_contents():
info["campaigns"] = list_campaigns(batchlist, addon)
info["version"] = version
sys.stdout.write("ok\n")
except wmlparser2.WMLError as e:
except wmlparser3.WMLError as e:
ef = open(logname, "w")
ef.write("<PARSE ERROR>\n")
ef.write(str(e))
ef.write("</PARSE ERROR>\n")
ef.close()
sys.stdout.write("failed\n")
except Queue.Empty as e:
except queue.Empty as e:
ef = open(logname, "w")
ef.write("<TIMEOUT ERROR>\n")
ef.write("Failed to parse the WML within " + str(TIMEOUT) + " seconds.")
@ -355,7 +355,7 @@ def list_contents():
def process_campaign_or_era(addon, cid, define, batchlist):
n = 0
print(addon + ": " + cid + " " + define)
print((addon + ": " + cid + " " + define))
wesnoth = helpers.WesnothList(
options.wesnoth,
@ -404,7 +404,7 @@ def process_campaign_or_era(addon, cid, define, batchlist):
if addon != "mainline" and isocode != "en_US": continue
if define == "MULTIPLAYER":
for era in wesnoth.era_lookup.values():
for era in list(wesnoth.era_lookup.values()):
if era.get_text_val("id") == cid:
n = html_output.generate_era_report(addon, isocode, era, wesnoth)
break
@ -412,7 +412,7 @@ def process_campaign_or_era(addon, cid, define, batchlist):
if cid == "mainline":
n = html_output.generate_campaign_report(addon, isocode, None, wesnoth)
for campaign in wesnoth.campaign_lookup.values():
for campaign in list(wesnoth.campaign_lookup.values()):
if campaign.get_text_val("id") == cid:
n = html_output.generate_campaign_report(addon, isocode, campaign, wesnoth)
break
@ -453,7 +453,7 @@ def batch_process():
try:
if not worked:
print(name + " not found")
print((name + " not found"))
continue
for era in addon.get("eras", []):
@ -468,16 +468,16 @@ def batch_process():
n = process_campaign_or_era(name, cid, campaign["define"], batchlist)
campaign["units"] = n
except wmlparser2.WMLError as e:
except wmlparser3.WMLError as e:
ef = open(logname, "a")
ef.write("<WML ERROR>\n")
ef.write(str(e))
ef.write("</WML ERROR>\n")
ef.close()
print(" " + name + " failed")
print((" " + name + " failed"))
except Exception as e:
traceback.print_exc()
print(" " + name + " failed")
print((" " + name + " failed"))
ef = open(logname, "a")
ef.write("<INTERNAL ERROR>\n")
ef.write("please report as bug\n")
@ -505,7 +505,7 @@ def batch_process():
def write_unit_ids_UNUSED():
# Write a list with all unit ids, just for fun.
uids = wesnoth.unit_lookup.keys()
uids = list(wesnoth.unit_lookup.keys())
def by_race(u1, u2):
r = cmp(wesnoth.unit_lookup[u1].rid,
wesnoth.unit_lookup[u2].rid)
@ -604,11 +604,11 @@ if __name__ == '__main__':
if not options.data_dir:
options.data_dir = shell_out([options.wesnoth, "--path"]).strip()
print("Using " + options.data_dir + " as data dir.")
print(("Using " + options.data_dir + " as data dir."))
if not options.config_dir:
options.config_dir = shell_out([options.wesnoth, "--config-path"]).strip()
print("Using " + options.config_dir + " as config dir.")
print(("Using " + options.config_dir + " as config dir."))
if not options.transdir:
options.transdir = os.getcwd()
@ -623,7 +623,7 @@ if __name__ == '__main__':
if options.language == "all":
languages = []
parser = wmlparser2.Parser(options.wesnoth, options.config_dir,
parser = wmlparser3.Parser(options.wesnoth, options.config_dir,
options.data_dir, no_preprocess = False)
parser.parse_text("{languages}")