Upgraded wesnoth_addon_manager to Python 3

This includes campaignserver_client.py which is also used by wescamp.py
- however wescamp.py is still Python 2 at this point and will need to be
  upgraded at a later point in time.
This commit is contained in:
Elias Pschernig 2015-09-22 16:31:28 -04:00
parent d5eea2abbf
commit 900c84657b
4 changed files with 119 additions and 157 deletions

View file

@ -56,7 +56,7 @@ Select the add-on you want to install from the list and click "OK". The download
"asc.gif", "bg.gif", "desc.gif"]:
Popen(["cp", "-u", am_dir + name, path])
campaigns = data.get_or_create_sub("campaigns")
campaigns = data.get_all(tag = "campaigns")[0]
w("<table class=\"tablesorter\" id=\"campaigns\">")
w("<thead>")
w("<tr>")
@ -67,9 +67,9 @@ Select the add-on you want to install from the list and click "OK". The download
w("<tbody>")
root_dir = am_dir + "../../../"
images_to_tc = []
for campaign in campaigns.get_all("campaign"):
for campaign in campaigns.get_all(tag = "campaign"):
v = campaign.get_text_val
translations = campaign.get_all("translation")
translations = campaign.get_all(tag = "translation")
languages = [x.get_text_val("language") for x in translations]
w("<tr>")
icon = v("icon", "")

View file

@ -1,12 +1,26 @@
import gzip, zlib, StringIO
import gzip, zlib, io
import socket, struct, glob, sys, shutil, threading, os, fnmatch
import wesnoth.wmldata as wmldata
import wesnoth.wmlparser as wmlparser
import wesnoth.wmlparser3 as wmlparser
# See the following files (among others):
# src/addon/*
# src/network.cpp
def append_attributes(tag, **attributes):
for k, v in attributes.items():
if isinstance(k, str): k = k.encode("utf8")
if isinstance(v, str): v = v.encode("utf8")
kn = wmlparser.AttributeNode(k)
vn = wmlparser.StringNode(v)
kn.value.append(vn)
tag.append(kn)
def append_tag(tag : wmlparser.TagNode, sub : str) -> wmlparser.TagNode:
subtag = wmlparser.TagNode(sub.encode("utf8"))
if tag:
tag.append(subtag)
return subtag
dumpi = 0
class CampaignClient:
# First port listed will be used as default.
@ -95,20 +109,18 @@ class CampaignClient:
def make_packet(self, doc):
root = wmldata.DataSub("WML")
root.insert(doc)
return root.make_string()
return doc.wml()
def send_packet(self, packet):
"""
Send binary data to the server.
"""
# Compress the packet before we send it
io = StringIO.StringIO()
z = gzip.GzipFile(mode = "w", fileobj = io)
fio = io.BytesIO()
z = gzip.GzipFile(mode = "wb", fileobj = fio)
z.write(packet)
z.close()
zdata = io.getvalue()
zdata = fio.getvalue()
zpacket = struct.pack("!I", len(zdata)) + zdata
self.sock.sendall(zpacket)
@ -117,7 +129,7 @@ class CampaignClient:
"""
Read binary data from the server.
"""
packet = ""
packet = b""
while len(packet) < 4 and not self.canceled:
r = self.sock.recv(4 - len(packet))
if not r:
@ -131,7 +143,7 @@ class CampaignClient:
if not self.quiet:
sys.stderr.write("Receiving %d bytes.\n" % self.length)
packet = ""
packet = b""
while len(packet) < l and not self.canceled:
r = self.sock.recv(l - len(packet))
if not r:
@ -144,16 +156,16 @@ class CampaignClient:
if not self.quiet:
sys.stderr.write("Received %d bytes.\n" % len(packet))
if packet.startswith("\x1F\x8B"):
if packet.startswith(b"\x1F\x8B"):
if self.verbose:
sys.stderr.write("GZIP compression found...\n")
io = StringIO.StringIO(packet)
z = gzip.GzipFile(fileobj = io)
bio = io.BytesIO(packet)
z = gzip.GzipFile("rb", fileobj = bio)
unzip = z.read()
z.close()
packet = unzip
elif packet.startswith( '\x78\x9C' ):
elif packet.startswith(b'\x78\x9C' ):
if self.verbose:
sys.stderr.write("ZLIB compression found...\n")
packet = zlib.decompres( packet )
@ -169,77 +181,22 @@ class CampaignClient:
def unescape(self, data):
# 01 is used as escape character
data2 = ""
data2 = b""
escape = False
for c in data:
if escape:
data2 += chr(ord(c) - 1)
data2 += bytes([c - 1])
escape = False
elif c == "\01":
elif c == 1:
escape = True
else:
data2 += c
data2 += bytes([c])
return data2
def decode_WML(self, data):
p = wmlparser.Parser( None, no_macros_in_string=True )
p.verbose = False
p.do_preprocessor_logic = True
p.no_macros = True
p.parse_text(data, binary=True)
doc = wmldata.DataSub( "WML" )
p.parse_top(doc)
return doc
def done():
return pos[0] >= len(data)
def next():
c = data[pos[0]]
pos[0] += 1
return c
def literal():
s = pos[0]
e = data.find("\00", s)
pack = data[s:e]
pack = pack.replace("\01\01", "\00")
pack = pack.replace("\01\02", "\01")
pos[0] = e + 1
return pack
while not done():
code = ord(next())
if code == 0: # open element (name code follows)
open_element = True
elif code == 1: # close current element
tag.pop()
elif code == 2: # add code
self.words[self.wordcount] = literal()
self.wordcount += 1
else:
if code == 3:
word = literal() # literal word
else:
word = self.words[code] # code
if open_element: # we handle opening an element
element = wmldata.DataSub(word)
tag[-1].insert(element) # add it to the current one
tag.append(element) # put to our stack to keep track
elif word == "contents": # detect any binary attributes
binary = wmldata.DataBinary(word, literal())
tag[-1].insert(binary)
else: # others are text attributes
text = wmldata.DataText(word, literal())
tag[-1].insert(text)
open_element = False
return WML
p = wmlparser.Parser()
p.parse_binary(data)
return p.root
def encode_WML( self, data ):
"""
@ -259,9 +216,9 @@ class CampaignClient:
"""
if self.error:
return None
request = wmldata.DataSub("request_campaign_list")
request = append_tag(None, "request_campaign_list")
if addon:
request.insert_text("name", addon)
append_attributes(request, name = addon)
self.send_packet(self.make_packet(request))
return self.decode(self.read_packet())
@ -270,9 +227,9 @@ class CampaignClient:
"""
Deletes the named campaign on the server.
"""
request = wmldata.DataSub("delete")
request.set_text_val("name", name)
request.set_text_val("passphrase", passphrase)
request = append_tag(None, "delete")
append_attributes(request, name = name,
passphrase = passphrase)
self.send_packet(self.make_packet(request))
return self.decode(self.read_packet())
@ -281,10 +238,9 @@ class CampaignClient:
"""
Changes the passphrase of a campaign on the server.
"""
request = wmldata.DataSub("change_passphrase")
request.set_text_val("name", name)
request.set_text_val("passphrase", old)
request.set_text_val("new_passphrase", new)
request = append_tag(None, "change_passphrase")
append_attributes(request, name = name,
passphrase = old, new_passphrase = new)
self.send_packet(self.make_packet(request))
return self.decode(self.read_packet())
@ -293,8 +249,8 @@ class CampaignClient:
"""
Downloads the named campaign and returns it as a raw binary WML packet.
"""
request = wmldata.DataSub("request_campaign")
request.insert(wmldata.DataText("name", name))
request = append_tag(None, "request_campaign")
append_attributes(request, name = name)
self.send_packet(self.make_packet(request))
raw_packet = self.read_packet()
@ -323,47 +279,45 @@ class CampaignClient:
The directory is the name of the campaign's directory.
"""
request = pbl.copy()
request.name = "upload"
request.set_text_val("name", name)
data = wmldata.DataSub("data")
request.insert(data)
request = pbl
request.name = b"upload"
append_attributes(request, name = name)
data = append_tag(request, "data")
def put_file(name, f):
for ig in ign:
if ig and ig[-1] != "/" and fnmatch.fnmatch(name, ig):
print("Ignored file", name)
print(("Ignored file", name))
return None
fileNode = wmldata.DataSub("file")
fileNode = append_tag(None, "file")
# Order in which we apply escape sequences matters.
contents = f.read()
contents = contents.replace("\x01", "\x01\x02" )
contents = contents.replace("\x00", "\x01\x01")
contents = contents.replace("\x0d", "\x01\x0e")
contents = contents.replace("\xfe", "\x01\xff")
contents = contents.replace(b"\x01", b"\x01\x02" )
contents = contents.replace(b"\x00", b"\x01\x01")
contents = contents.replace(b"\x0d", b"\x01\x0e")
contents = contents.replace(b"\xfe", b"\x01\xff")
fileContents = wmldata.DataText("contents", contents)
fileNode.insert(fileContents)
fileNode.set_text_val("name", name)
append_attributes(fileNode, name = name)
append_attributes(fileNode, contents = contents)
return fileNode
def put_dir(name, path):
for ig in ign:
if ig and ig[-1] == "/" and fnmatch.fnmatch(name, ig[:-1]):
print("Ignored dir", name)
print(("Ignored dir", name))
return None
dataNode = wmldata.DataSub("dir")
dataNode.set_text_val("name", name)
dataNode = append_tag("dir")
append_attributes(dataNode, name = name)
for fn in glob.glob(path + "/*"):
if os.path.isdir(fn):
sub = put_dir(os.path.basename(fn), fn)
else:
sub = put_file(os.path.basename(fn), open(fn, 'rb'))
if sub: dataNode.insert(sub)
if sub:
dataNode.append(sub)
return dataNode
# Only used if it's an old-style campaign directory
@ -373,13 +327,16 @@ class CampaignClient:
if not self.quiet:
sys.stderr.write("Adding directory %s as %s.\n" % (directory, name))
data.insert(put_dir(name, directory))
data.append(put_dir(name, directory))
packet = self.make_packet(request)
open("packet.dump", "wb").write(packet)
self.send_packet(packet)
return self.decode(self.read_packet())
response = self.read_packet()
if not response:
response = b""
return self.decode(response)
def get_campaign_raw_async(self, name):
"""
@ -450,20 +407,18 @@ class CampaignClient:
os.mkdir(path)
except OSError:
pass
for f in data.get_all("file"):
for f in data.get_all(tag = "file"):
name = f.get_text_val("name", "?")
contents = f.get_text("contents")
contents = f.get_binary("contents")
if not contents:
contents = ""
contents = b""
if not self.quiet:
sys.stderr.write("File %s is empty.\n" % name)
sys.stderr.write(f.debug(write = False) + "\n")
if contents:
contents = contents.get_value()
if verbose:
sys.stderr.write(i * " " + name + " (" +
str(len(contents)) + ")\n")
save = file( os.path.join(path, name), "wb")
save = open( os.path.join(path, name), "wb")
# We MUST un-escape our data
# Order we apply escape sequences matter here
@ -471,7 +426,7 @@ class CampaignClient:
save.write(contents)
save.close()
for dir in data.get_all("dir"):
for dir in data.get_all(tag = "dir"):
name = dir.get_text_val("name", "?")
shutil.rmtree(os.path.join(path, name), True)
os.mkdir(os.path.join(path, name))

View file

@ -23,6 +23,14 @@ import sys, os.path, argparse, tempfile, shutil, logging, socket
# in case the wesnoth python package has not been installed
sys.path.append("data/tools")
print("""
Note: campaignserver_client has since been moved to Python 3 - the
easiest way to run this script is to use campaginserver_client from
an earlier Wesnoth version. And then in the long run convert this
script to Python 3 as well.
""")
sys.exit(1)
#import CampaignClient as libwml
import wesnoth.campaignserver_client as libwml

View file

@ -1,4 +1,4 @@
#!/usr/bin/env python
#!/usr/bin/env python3
# encoding: utf8
"""
add-on_manager.py -- a command-line client for the Wesnoth add-on server
@ -11,8 +11,7 @@ add-ons.
import sys, os.path, re, time, glob, shutil
from subprocess import Popen
import wesnoth.wmldata as wmldata
import wesnoth.wmlparser as wmlparser
import wesnoth.wmlparser3 as wmlparser
from wesnoth.campaignserver_client import CampaignClient
if __name__ == "__main__":
@ -32,7 +31,7 @@ if __name__ == "__main__":
help="Output a HTML overview into the given directory.",)
argumentparser.add_argument("-p", "--port",
help="specify server port or BfW version (%s)" % " or ".join(
map(lambda x: x[1], CampaignClient.portmap)),
[x[1] for x in CampaignClient.portmap]),
default=CampaignClient.portmap[0][0])
argumentparser.add_argument("-l", "--list", help="list available add-ons",
action="store_true",)
@ -112,7 +111,7 @@ if __name__ == "__main__":
while not mythread.event.isSet():
mythread.event.wait(1)
if pcounter != cs.counter:
print "%s: %d/%d" % (name, cs.counter, cs.length)
print("%s: %d/%d" % (name, cs.counter, cs.length))
pcounter = cs.counter
if args.raw_download:
@ -127,13 +126,13 @@ if __name__ == "__main__":
try: os.remove(oldcfg_path)
except OSError: pass
print "Unpacking %s..." % name
print("Unpacking %s..." % name)
cs.unpackdir(decoded, cdir, verbose=args.verbose)
info = os.path.join(dirname, "_info.cfg")
try:
f = file(info, "w")
f = open(info, "w")
infowml = """#
# File automatically generated by Wesnoth to keep track
# of version information on installed add-ons. DO NOT EDIT!
@ -151,8 +150,7 @@ if __name__ == "__main__":
f.close()
except OSError:
pass
for message in decoded.find_all("message", "error"):
print message.get_text_val("message")
print_messages(decoded)
if args.tar:
try: os.mkdir(args.tar)
@ -171,6 +169,15 @@ if __name__ == "__main__":
"cjf %(tarname)s -C %(cdir)s %(name)s\n" % locals())
Popen(["tar", "cjf", tarname, "-C", cdir, name])
def print_messages(data):
for message in data.get_all(tag = "message") + data.get_all(tag = "error"):
print(message.get_text_val("message"))
def parse_wml_file(name):
p = wmlparser.Parser()
p.parse_file(name)
return p.root
def get_info(name):
"""
Get info for a locally installed add-on. It expects a direct path
@ -179,12 +186,9 @@ if __name__ == "__main__":
if not os.path.exists(name):
return None, None
p = wmlparser.Parser(None)
p.parse_file(name)
info = wmldata.DataSub("WML")
p.parse_top(info)
uploads = info.get_or_create_sub("info").get_text_val("uploads", "")
version = info.get_or_create_sub("info").get_text_val("version", "")
info = parse_wml_file(name)
uploads = info.get_all(tag = "info")[0].get_text_val("uploads", "")
version = info.get_all(tag = "info")[0].get_text_val("version", "")
return uploads, version
campaign_list = None
@ -193,17 +197,16 @@ if __name__ == "__main__":
cs = CampaignClient(address)
campaign_list = data = cs.list_campaigns()
if data:
campaigns = data.get_or_create_sub("campaigns")
campaigns = data.get_all(tag = "campaigns")[0]
if args.wml:
for campaign in campaigns.get_all("campaign"):
campaign.debug(show_contents=True,
use_color=args.color)
for campaign in campaigns.get_all(tag = "campaign"):
print(campaign.debug())
else:
column_sizes = [10, 5, 10, 7, 8, 8, 10, 5, 10, 13]
columns = [["type", "name", "title", "author",
"version", "uploads", "downloads",
"size", "timestamp", "translate"]]
for campaign in campaigns.get_all("campaign"):
for campaign in campaigns.get_all(tag = "campaign"):
column = [
campaign.get_text_val("type", "?"),
campaign.get_text_val("name", "?"),
@ -223,8 +226,7 @@ if __name__ == "__main__":
for i, f in enumerate(c):
sys.stdout.write(f.ljust(column_sizes[i]))
sys.stdout.write("\n")
for message in data.find_all("message", "error"):
print message.get_text_val("message")
print_messages(data)
else:
sys.stderr.write("Could not connect.\n")
@ -233,8 +235,8 @@ if __name__ == "__main__":
fetchlist = []
campaign_list = data = cs.list_campaigns()
if data:
campaigns = data.get_or_create_sub("campaigns")
for campaign in campaigns.get_all("campaign"):
campaigns = data.get_all(tag = "campaigns")[0]
for campaign in campaigns.get_all(tag = "campaign"):
name = campaign.get_text_val("name", "?")
title = campaign.get_text_val("title")
type = campaign.get_text_val("type", "")
@ -256,31 +258,29 @@ if __name__ == "__main__":
if version != local_version or uploads > local_uploads:
get(name, title, version, type, uploads, dependencies, args.campaigns_dir)
else:
print "Not downloading", name, \
print("Not downloading", name, \
"as the version already is", local_version, \
"(The add-on got re-uploaded.)"
"(The add-on got re-uploaded.)")
else:
if args.verbose:
print "Not downloading", name, \
"because it is already up-to-date."
print("Not downloading", name, \
"because it is already up-to-date.")
elif args.unpack:
cs = CampaignClient(address)
data = file(args.unpack).read()
decoded = cs.decode(data)
print "Unpacking %s..." % args.unpack
print("Unpacking %s..." % args.unpack)
cs.unpackdir(decoded, args.campaigns_dir, verbose=True)
elif args.remove:
cs = CampaignClient(address)
data = cs.delete_campaign(args.remove, args.password)
for message in data.find_all("message", "error"):
print message.get_text_val("message")
print_messages(data)
elif args.change_passphrase:
cs = CampaignClient(address)
data = cs.change_passphrase(*args.change_passphrase)
for message in data.find_all("message", "error"):
print message.get_text_val("message")
print_messages(data)
elif args.upload:
cs = CampaignClient(address)
@ -306,7 +306,7 @@ if __name__ == "__main__":
if args.pbl:
pblfile = args.pbl
pbl = wmldata.read_file(pblfile, "PBL")
pbl = parse_wml_file(pblfile)
if os.path.exists(ignfile):
ign = open(ignfile).readlines()
# strip line endings and whitespace
@ -341,11 +341,10 @@ if __name__ == "__main__":
while not mythread.event.isSet():
mythread.event.wait(1)
if cs.counter != pcounter:
print "%d/%d" % (cs.counter, cs.length)
print("%d/%d" % (cs.counter, cs.length))
pcounter = cs.counter
for message in mythread.data.find_all("message", "error"):
print message.get_text_val("message")
print_messages(mythread.data)
elif args.update or args.status:
if args.status:
@ -360,7 +359,7 @@ if __name__ == "__main__":
sys.stderr.write("Could not connect to the add-on server.\n")
sys.exit(-1)
campaigns = {}
for c in data.get_or_create_sub("campaigns").get_all("campaign"):
for c in data.get_all(tag = "campaigns")[0].get_all(tag = "campaign"):
name = c.get_text_val("name")
campaigns[name] = c
for d in dirs: