Commit wmlxgettext2, Nobun's python3 reimplementation of wmlxgettext

po/SConscript

@@ -74,7 +74,7 @@ if "pot-update" in COMMAND_LINE_TARGETS:
             wml_pot = env.Command(
                 join(domain, domain + ".wml.pot"),
                 cfgs,
-                "utils/wmlxgettext --directory=. --domain=%s $SOURCES > $TARGET" % domain
+                "utils/wmlxgettext --directory=. --domain=%s $SOURCES -o $TARGET" % domain
                 )
 
         new_pot = str(pot) + ".new"

utils/pywmlx/__init__.py

@@ -0,0 +1,13 @@
+from pywmlx.wmlerr import ansi_setEnabled
+from pywmlx.wmlerr import wmlerr
+from pywmlx.wmlerr import wmlwarn
+from pywmlx.wmlerr import set_warnall
+
+from pywmlx.postring import PoCommentedString
+from pywmlx.postring import WmlNodeSentence
+from pywmlx.postring import WmlNode
+
+import pywmlx.autof
+
+import pywmlx.state as statemachine
+

utils/pywmlx/autof.py

@@ -0,0 +1,30 @@
+import os
+import sys
+import re
+
+
+def autoscan(pathdir):
+    filelist = None
+    parentdir = os.path.realpath(os.path.join(pathdir, '..'))
+    for root, dirs, files in os.walk(pathdir, topdown=False):
+        for name in files:
+            rx = re.compile(r'\.(cfg|lua)$', re.I)
+            m = re.search(rx, name)
+            if m:
+                value = os.path.join(root, name)
+                value = value [ len(parentdir) +1 : ]
+                if os.name == "posix":
+                    value = re.sub(r'^\/', '', value)
+                else:
+                    value = re.sub(r'^(?:[A-Za-z]\:)?\\', '', value)
+                if filelist is None: 
+                    filelist = [ value ]
+                else:
+                    filelist.append(value)
+                # end if m
+            # end for name
+        # end for root
+    # end for scandir
+    return (parentdir, filelist)
+# end autoscan
+

utils/pywmlx/nodemanip.py

@@ -0,0 +1,121 @@
+import re
+
+import pywmlx.postring as pos
+from pywmlx.wmlerr import wmlerr
+from pywmlx.wmlerr import wmlwarn
+
+
+fileref = None
+fileno = None
+nodes = None
+onDefineMacro = False
+
+
+
+def _closenode_update_dict(podict):
+    if nodes[-1].sentences is not None:
+        for i in nodes[-1].sentences:
+            posentence = podict.get(i.sentence)
+            if posentence is None:
+                podict[i.sentence] = ( 
+                       nodes[-1].nodesentence_to_posentence(i) )
+            else:
+                posentence.update_with_commented_string(
+                       nodes[-1].nodesentence_to_posentence(i) ) 
+
+
+
+def newfile(file_ref, file_no):
+    global fileref
+    global fileno
+    global nodes
+    global onDefineMacro
+    onDefineMacro = False
+    fileref = file_ref
+    fileno = file_no
+    if nodes is not None:
+        del nodes[:]
+    nodes = None
+
+
+
+def closefile(mydict, lineno):
+    if nodes is not None:
+        if len(nodes) > 1:
+            err_message = ("End of WML file reached, but some tags were " + 
+                            "not properly closed.\n"
+                            "(nearest unclosed tag is: " + 
+                            nodes[-1].tagname + ")" )
+            finfo = fileref + ":" + str(lineno)
+            wmlerr(finfo, err_message)
+        else:
+            _closenode_update_dict(mydict)
+
+
+def newnode(tagname):
+    global fileref
+    global fileno
+    global nodes
+    if nodes is None:
+        nodes = [pos.WmlNode(fileref, fileno, "", autowml=False)]
+    if tagname == "[lua]":
+        nodes.append( pos.WmlNode(fileref, fileno, 
+                                  tagname, autowml=False) )
+    # elif tagname == "":
+    # self.nodes.append( WmlNode(self.fileref, self.fileno, 
+    #                            "[unknown]", False) )
+    else:
+        nodes.append( pos.WmlNode(fileref, fileno, 
+                                  tagname, autowml=True) )
+
+
+
+def closenode(closetag, mydict, lineno):
+    global fileref
+    global fileno
+    global nodes
+    if nodes is None:
+        err_message = ("unexpected closing tag '" + 
+                            closetag + "' outside any scope.")
+        finfo = fileref + ":" + str(lineno)
+        wmlerr(finfo, err_message)
+    else:
+        # node to close is the LAST element in self.nodes list
+        mytag = nodes[-1].tagname
+        mynode = nodes[-1]
+        expected_closetag = re.sub(r'\[', r'[/', mytag)
+        finfo = fileref + ":" + str(lineno)
+        if mynode.tagname == "": 
+            err_message = ("unexpected closing tag '" + 
+                            closetag + "' outside any scope.")
+            wmlerr(finfo, err_message)
+        else:
+            if closetag != expected_closetag:
+                err_message = ("expected closing tag '" + 
+                                expected_closetag + "' but '" +
+                                closetag + "' found.")
+                wmlerr(finfo, err_message)
+        _closenode_update_dict(mydict)
+        nodes.pop()
+
+
+def addNodeSentence(sentence, *, ismultiline, lineno, lineno_sub,
+                    override, addition):
+    global nodes
+    if nodes is None:
+        nodes = [pos.WmlNode(fileref=fileref, fileno=fileno, 
+                              tagname="", autowml=False)]
+    nodes[-1].add_sentence(sentence, ismultiline=ismultiline, 
+                           lineno=lineno, lineno_sub=lineno_sub,
+                           override=override, addition=addition)
+
+
+def addWmlInfo(info):
+    global nodes
+    if nodes is None:
+        nodes = [pos.WmlNode(fileref=fileref, fileno=fileno, 
+                              tagname="", autowml=False)]
+    if nodes[-1].wmlinfos is None:
+        nodes[-1].wmlinfos = []
+    nodes[-1].wmlinfos.append(info)
+

utils/pywmlx/postring.py

@@ -0,0 +1,209 @@
+import re
+
+class PoCommentedString:
+    def __init__(self, sentence, *, orderid, ismultiline, 
+                 wmlinfos, finfos, addedinfos):
+        self.sentence = sentence
+        self.wmlinfos = wmlinfos
+        self.addedinfos = addedinfos
+        self.finfos = finfos
+        self.orderid = orderid
+        self.ismultiline = ismultiline
+        
+    def update_orderid(self, orderid):
+        if orderid < self.orderid:
+            self.orderid = orderid
+            
+    def update_with_commented_string(self, commented_string):
+        if commented_string.wmlinfos:
+            if commented_string.wmlinfos[0] not in self.wmlinfos:
+                self.wmlinfos.append(commented_string.wmlinfos[0])
+        if commented_string.addedinfos:
+            self.addedinfos += commented_string.addedinfos
+        self.finfos += commented_string.finfos
+        self.update_orderid(commented_string.orderid)
+        # if commented_string.orderid < self.orderid:
+        #     self.orderid = commented_string.orderid
+        #     self.sentence = commented_string.sentence
+            
+    def write(self, filebuf, fuzzy):
+        if self.wmlinfos is not None:
+            for i in self.wmlinfos:
+                print('#.', i, file=filebuf)
+        if self.addedinfos is not None:
+            for i in self.addedinfos:
+                print('#.', i, file=filebuf)
+        if self.finfos is not None:
+            for i in self.finfos:
+                print('#:', i, file=filebuf)
+        if fuzzy:
+            print('#, fuzzy', file=filebuf)
+        self.sentence = '"' + self.sentence + '"'
+        if self.ismultiline:
+            lf = r'\\n"' + '\n"'
+            self.sentence = re.sub(r'(\n\r|\r\n|[\n\r])', lf, self.sentence)
+            self.sentence = '""\n' + self.sentence
+        print('msgid', self.sentence, file=filebuf)
+        print('msgstr ""', file=filebuf)
+
+
+
+class WmlNodeSentence:
+    def __init__(self, sentence, *, ismultiline, lineno, lineno_sub=0,
+                 override=None, addition=None):
+        self.sentence = sentence
+        # Say if it is multiline or not.
+        self.ismultiline = ismultiline
+        self.lineno = lineno
+        # lineno_sub:
+        # used only in WmlNodeSentence. This parameter is actually used 
+        # only when multiple translatable strings were found on the same line.
+        # On PoCommentedString translation a lineno_sub != 0 
+        # will translated as a floated lineno value 
+        #      (ex. lineno = 10.02 means "lineno = 10, lineno_sub = 2)
+        self.lineno_sub = lineno_sub
+        # overrideinfo is usually None.
+        # overrideinfo will be used when an override wml info 
+        # was found into lua or Wml code -> 
+        #                   (overrideinfo will be a non-empty string)
+        # every translated string can have only ONE override info.
+        self.overrideinfo = override
+        # addedinfo is usually an empty list.
+        # list will contains custom comment that will be added at the
+        # end of translator's comment list.
+        self.addedinfo = addition
+        if addition is None: self.addedinfo = []
+
+
+
+class WmlNode:
+    def __init__(self, fileref, fileno, tagname, *, autowml=True):
+        self.tagname = tagname
+        self.fileref = fileref + ":"
+        self.fileno = fileno
+        self.sentences = None
+        self.wmlinfos = None
+        self.autowml = autowml
+    
+    def add_sentence(self, sentence, *, ismultiline, lineno, 
+                     lineno_sub=0, override=None, addition=None):
+        if self.sentences is None:
+            self.sentences = []
+        self.sentences.append( WmlNodeSentence(sentence, 
+                                          ismultiline=ismultiline, 
+                                          lineno=lineno, 
+                                          lineno_sub=lineno_sub, 
+                                          override=override, 
+                                          addition=addition) ) 
+        
+    def assemble_wmlinfo(self):
+        if self.tagname == "":
+            value = []
+            return value
+        else:
+            mystr = self.tagname
+            intstr = 0
+            if self.wmlinfos is not None:
+                if len(self.wmlinfos) > 0:
+                    mystr += ": "
+                    for k in self.wmlinfos:
+                        intstr += 1
+                        if intstr > 1:
+                            mystr += ", "
+                        mystr += k
+            return mystr
+    
+    def assemble_orderid(self, nodesentence):
+        return (self.fileno, nodesentence.lineno,
+                nodesentence.lineno_sub)
+            
+    def nodesentence_to_posentence(self, nodesentence):
+        # if nodesentence has overrideinfo, overrideinfo will be written 
+        # instead of the automatic wml infos.
+        # if it has an EMPTY overrideinfo, no wml infos will be added 
+        #             (this happens if a macro call encountered)
+        # also no wml infos will be added if "autowml" is false 
+        #             (this happens: on root node, on [lua] tags.
+        #              on #define nodes)
+        # but if "autowml" is false AND an overrideinfo is given, than 
+        # overridden wmlinfo is added.
+        # added_infos are also infos for translators, but it will be written 
+        # after wml_infos, in addition to them
+        if nodesentence.overrideinfo is not None:
+            if nodesentence.overrideinfo == "":
+                if(nodesentence.addedinfo is not None and 
+                              nodesentence.addedinfo != ""):
+                    return PoCommentedString(nodesentence.sentence, 
+                               ismultiline=nodesentence.ismultiline, 
+                               orderid=self.assemble_orderid(nodesentence),
+                               wmlinfos=[],
+                               finfos=[self.fileref + 
+                                        str(nodesentence.lineno)],
+                               addedinfos=nodesentence.addedinfo)
+                else:
+                    return PoCommentedString(nodesentence.sentence, 
+                               ismultiline=nodesentence.ismultiline, 
+                               orderid=self.assemble_orderid(nodesentence),
+                               wmlinfos=[],
+                               finfos=[self.fileref + 
+                                        str(nodesentence.lineno)],
+                               addedinfos=[])
+            else: # having a non-empty override
+                if(nodesentence.addedinfo is not None and 
+                              nodesentence.addedinfo != ""):
+                    return PoCommentedString(nodesentence.sentence, 
+                               ismultiline=nodesentence.ismultiline, 
+                               orderid=self.assemble_orderid(nodesentence),
+                               wmlinfos=[nodesentence.overrideinfo],
+                               finfos=[self.fileref + 
+                                        str(nodesentence.lineno)],
+                               addedinfos=nodesentence.addedinfo)
+                else:
+                    return PoCommentedString(nodesentence.sentence, 
+                               ismultiline=nodesentence.ismultiline, 
+                               orderid=self.assemble_orderid(nodesentence),
+                               wmlinfos=[nodesentence.overrideinfo],
+                               finfos=[self.fileref + 
+                                        str(nodesentence.lineno)],
+                               addedinfos=[])
+        # if you don't have override and autowml is true 
+        #  --> wmlinfos will be always added
+        elif self.autowml == True:
+            if(nodesentence.addedinfo is not None and 
+                          nodesentence.addedinfo != ""):
+                return PoCommentedString(nodesentence.sentence, 
+                               ismultiline=nodesentence.ismultiline, 
+                               orderid=self.assemble_orderid(nodesentence),
+                               wmlinfos=[self.assemble_wmlinfo()],
+                               finfos=[self.fileref + 
+                                        str(nodesentence.lineno)],
+                               addedinfos=nodesentence.addedinfo)
+            else:
+                return PoCommentedString(nodesentence.sentence, 
+                               ismultiline=nodesentence.ismultiline, 
+                               orderid=self.assemble_orderid(nodesentence),
+                               wmlinfos=[self.assemble_wmlinfo()], 
+                               finfos=[self.fileref + 
+                                        str(nodesentence.lineno)],
+                               addedinfos=[])
+        # if you don't have override and autowml is false 
+        #  --> wmlinfos will never added
+        else:
+            if(nodesentence.addedinfo is not None and 
+                          nodesentence.addedinfo != ""):
+                return PoCommentedString(nodesentence.sentence, 
+                               ismultiline=nodesentence.ismultiline, 
+                               orderid=self.assemble_orderid(nodesentence),
+                               wmlinfos=[],
+                               finfos=[self.fileref + 
+                                        str(nodesentence.lineno)],
+                               addedinfos=nodesentence.addedinfo)
+            else:
+                return PoCommentedString(nodesentence.sentence, 
+                               ismultiline=nodesentence.ismultiline, 
+                               orderid=self.assemble_orderid(nodesentence),
+                               wml_infos=[],
+                               finfos=[self.fileref + 
+                                        str(nodesentence.lineno)],
+                               addedinfos=[])
+

utils/pywmlx/state/__init__.py

@@ -0,0 +1,3 @@
+from pywmlx.state.machine import setup
+from pywmlx.state.machine import run
+

utils/pywmlx/state/lua_states.py

@@ -0,0 +1,301 @@
+import re
+import pywmlx.state.machine
+from pywmlx.state.state import State
+from pywmlx.wmlerr import wmlwarn
+
+
+
+class LuaIdleState:
+    def __init__(self):
+        self.regex = None
+        self.iffail = None
+    
+    def run(self, xline, lineno, match):
+        _nextstate = 'lua_checkdom'
+        if pywmlx.state.machine._pending_luastring is not None:
+            pywmlx.state.machine._pending_luastring.store()
+            pywmlx.state.machine._pending_luastring = None
+        m = re.match(r'\s*$', xline)
+        if m:
+            xline = None
+            _nextstate = 'lua_idle'
+        return (xline, _nextstate)
+
+
+
+class LuaCheckdomState:
+    def __init__(self):
+        rx = (   r'\s*(local\s+)?_\s*=\s*wesnoth\s*\.\s*textdomain\s*'
+               r'''(?:\(\s*)?(["'])(.*?)\2''')
+        self.regex = re.compile(rx, re.I)
+        self.iffail = 'lua_checkpo'
+    
+    def run(self, xline, lineno, match):
+        pywmlx.state.machine._currentdomain = match.group(3)
+        xline = None
+        if match.group(1) is None and pywmlx.state.machine._warnall:
+            finfo = pywmlx.nodemanip.fileref + ":" + str(lineno)
+            wmlwarn(finfo, "function '_', in lua code, should be local.")
+        return (xline, 'lua_idle')
+
+
+
+class LuaCheckpoState:
+    def __init__(self):
+        self.regex = re.compile(r'\s*--\s*(?:#)?\s*(po-override|po):\s+(.+)', 
+                                re.I)
+        self.iffail = 'lua_comment'
+    
+    def run(self, xline, lineno, match):
+        # on -- #po: addedinfo
+        if match.group(1) == "po":
+            if pywmlx.state.machine._pending_addedinfo is None:
+                pywmlx.state.machine._pending_addedinfo = [ match.group(2) ]
+            else:
+                pywmlx.state.machine._pending_addedinfo.append(match.group(2))
+        # on -- #po-override: overrideinfo
+        elif pywmlx.state.machine._pending_overrideinfo is None:
+            pywmlx.state.machine._pending_overrideinfo = [ match.group(2) ]
+        else:
+            pywmlx.state.machine._pending_overrideinfo.append(match.group(2))
+        xline = None
+        return (xline, 'lua_idle')
+
+
+
+class LuaCommentState:
+    def __init__(self):
+        self.regex = re.compile(r'\s*--.+')
+        self.iffail = 'lua_str01'
+    
+    def run(self, xline, lineno, match):
+        xline = None
+        return (xline, 'lua_idle')
+
+
+
+class LuaStr01:
+    def __init__(self):
+        rx = r'''(?:[^["']*?)(_?)\s*"((?:\\"|[^"])*)("?)'''
+        self.regex = re.compile(rx)
+        self.iffail = 'lua_str02'
+    
+    def run(self, xline, lineno, match):
+        _nextstate = 'lua_idle'
+        loc_translatable = True
+        if match.group(1) == "":
+            loc_translatable = False
+        loc_multiline = False
+        if match.group(3) == "":
+            xline = None
+            loc_multiline = True
+            _nextstate = 'lua_str10'
+        else:
+            xline = xline [ match.end(): ]    
+        pywmlx.state.machine._pending_luastring = (
+            pywmlx.state.machine.PendingLuaString( 
+                lineno, 'luastr1', match.group(2), loc_multiline, 
+                loc_translatable 
+            )
+        ) 
+        return (xline, _nextstate)
+
+
+
+class LuaStr02:
+    def __init__(self):
+        rx = r'''(?:[^["']*?)(_?)\s*'((?:\\'|[^'])*)('?)'''
+        self.regex = re.compile(rx)
+        self.iffail = 'lua_str03' # 'lua_gowml' #'lua_str03'
+    
+    def run(self, xline, lineno, match):
+        _nextstate = 'lua_idle'
+        loc_translatable = True
+        if match.group(1) == "":
+            loc_translatable = False
+        loc_multiline = False
+        if match.group(3) == "":
+            xline = None
+            loc_multiline = True
+            _nextstate = 'lua_str20'
+        else:
+            xline = xline [ match.end(): ]
+        pywmlx.state.machine._pending_luastring = (
+            pywmlx.state.machine.PendingLuaString( 
+                lineno, 'luastr2', match.group(2), loc_multiline,
+                loc_translatable
+            )
+        )
+        return (xline, _nextstate)
+
+
+
+class LuaStr03:
+    def __init__(self):
+        rx = r'''(?:[^["']*?)(_?)\s*\[(=*)\[(.*?)]\2]'''
+        self.regex = re.compile(rx)
+        self.iffail = 'lua_str03o'
+    
+    def run(self, xline, lineno, match):
+        xline = xline [ match.end(): ]
+        loc_translatable = True
+        if match.group(1) == "":
+            loc_translatable = False
+        loc_multiline = False
+        pywmlx.state.machine._pending_luastring = (
+            pywmlx.state.machine.PendingLuaString( 
+                lineno, 'luastr3', match.group(3), loc_multiline,
+                loc_translatable
+            )
+        )
+        return (xline, 'lua_idle')
+
+
+
+class LuaStr03o:
+    def __init__(self):
+        rx = r'''(?:[^["']*?)(_?)\s*\[(=*)\[(.*)'''
+        self.regex = re.compile(rx)
+        self.iffail = 'lua_gowml'
+    
+    def run(self, xline, lineno, match):
+        xline = None
+        loc_translatable = True
+        if match.group(1) == "":
+            loc_translatable = False
+        loc_multiline = True
+        pywmlx.state.machine._pending_luastring = (
+            pywmlx.state.machine.PendingLuaString( 
+                lineno, 'luastr3', match.group(3), loc_multiline,
+                loc_translatable, len(match.group(2))
+            )
+        )
+        return (xline, 'lua_str30')
+
+
+
+# well... the regex will always be true on this state, so iffail will never
+# be executed
+class LuaStr10:
+    def __init__(self):
+        self.regex = re.compile(r'((?:\\"|[^"])*)("?)')
+        self.iffail = 'lua_str10'
+        
+    def run(self, xline, lineno, match):
+        _nextstate = None
+        pywmlx.state.machine._pending_luastring.addline( match.group(1) )
+        if match.group(2) == "":
+            _nextstate = 'lua_str10'
+            xline = None
+        else:
+            _nextstate = 'lua_idle'
+            xline = xline [ match.end(): ]
+        return (xline, _nextstate)
+
+
+
+# well... the regex will always be true on this state, so iffail will never
+# be executed
+class LuaStr20:
+    def __init__(self):
+        self.regex = re.compile(r"((?:\\'|[^'])*)('?)")
+        self.iffail = 'lua_str20'
+        
+    def run(self, xline, lineno, match):
+        _nextstate = None
+        pywmlx.state.machine._pending_luastring.addline( match.group(1) )
+        if match.group(2) == "":
+            _nextstate = 'lua_str20'
+            xline = None
+        else:
+            _nextstate = 'lua_idle'
+            xline = xline [ match.end(): ]
+        return (xline, _nextstate)
+
+
+
+class LuaStr30:
+    def __init__(self):
+        self.regex = None
+        self.iffail = None
+    
+    def run(self, xline, lineno, match):
+        rx = ( r'(.*?)]={' +  
+              str(pywmlx.state.machine._pending_luastring.numequals) +
+              '}]' )
+        realmatch = re.match(rx, xline)
+        _nextstate = 'lua_str30'
+        if realmatch:
+            pywmlx.state.machine._pending_luastring.addline( 
+                realmatch.group(1) )
+            xline = xline [ realmatch.end(): ]
+            _nextstate = 'lua_idle'
+        else:
+            pywmlx.state.machine._pending_luastring.addline(xline)
+            xline = None
+            _nextstate = 'lua_str30'
+        return (xline, _nextstate)
+
+
+
+class LuaGowmlState:
+    def __init__(self):
+        self.regex = re.compile(r'.*?>>\s*')
+        self.iffail = 'lua_final'
+    
+    def run(self, xline, lineno, match):
+        _nextstate = 'lua_idle'
+        if pywmlx.state.machine._waitwml:
+            xline = None
+            _nextstate = 'wml_idle'
+        else:
+            xline = xline [ match.end(): ]
+        return (xline, _nextstate)
+
+
+
+class LuaFinalState:
+    def __init__(self):
+        self.regex = None
+        self.iffail = None
+    
+    def run(self, xline, lineno, match):
+        rx_str = ( r'function\s+([a-zA-Z0-9_.]+)|' +
+                   r'([a-zA-Z0-9_.]+)\s*=\s*function' # +
+                 )  # r'(local)\s+(?!function).*?=' )
+        rx = re.compile(rx_str, re.I)
+        m = re.search(rx, xline)
+        if m:
+            if m.group(1):
+                pywmlx.state.machine._pending_luafuncname = m.group(1)
+            elif m.group(2):
+                pywmlx.state.machine._pending_luafuncname = m.group(2)
+            elif m.group(3):
+                pywmlx.state.machine._pending_luafuncname = None
+        xline = None
+        if pywmlx.state.machine._pending_wmlstring is not None:
+            pywmlx.state.machine._pending_wmlstring.store()
+            pywmlx.state.machine._pending_wmlstring = None
+        return (xline, 'lua_idle')
+
+
+
+def setup_luastates():
+    for statename, stateclass in [ ('lua_idle', LuaIdleState),
+                                   ('lua_checkdom', LuaCheckdomState),
+                                   ('lua_checkpo', LuaCheckpoState),
+                                   ('lua_comment', LuaCommentState),
+                                   ('lua_str01', LuaStr01),
+                                   ('lua_str02', LuaStr02),
+                                   ('lua_str03', LuaStr03),
+                                   ('lua_str03o', LuaStr03o),
+                                   ('lua_str10', LuaStr10),
+                                   ('lua_str20', LuaStr20),
+                                   ('lua_str30', LuaStr30),
+                                   ('lua_gowml', LuaGowmlState),
+                                   ('lua_final', LuaFinalState)]:
+        st = stateclass()
+        pywmlx.state.machine.addstate(statename, 
+            State(st.regex, st.run, st.iffail) )
+
+

utils/pywmlx/state/machine.py

@@ -0,0 +1,311 @@
+import re
+# import os
+
+from pywmlx.wmlerr import wmlerr
+from pywmlx.wmlerr import wmlwarn
+from pywmlx.wmlerr import warnall
+from pywmlx.postring import PoCommentedString
+from pywmlx.state.state import State
+from pywmlx.state.lua_states import setup_luastates
+from pywmlx.state.wml_states import setup_wmlstates
+
+import pywmlx.nodemanip
+
+
+
+# --------------------------------------------------------------------
+#  PART 1: machine.py global variables
+# --------------------------------------------------------------------
+
+
+# True if --warnall option is used
+_warnall = False
+# dictionary of pot sentences
+_dictionary = None
+# dictionary containing lua and WML states
+_states = None
+# initialdomain value (setted with --initialdomain command line option)
+_initialdomain = None
+# the current domain value when parsing file (changed by #textdomain text)
+_currentdomain = None
+# the domain value (setted with --domain command line option)
+_domain = None
+# this boolean value will be usually:
+#   True (when the file is a WML .cfg file)
+#   False (when the file is a .lua file)
+_waitwml = True
+# this boolean value is very useful to avoid a possible bug
+# verified in a special case 
+# (see WmlGoluaState on wml_states.py for more details)
+_on_luatag = False
+
+# ---------
+
+# pending additional infos for translators (# po: addedinfo)
+_pending_addedinfo = None
+# pending override wmlinfo for translators (# po-override: overrideinfo)
+_pending_overrideinfo = None
+# type of pending wmlinfo: 
+# it can be None or it can have an actual value.
+# Possible actual values are: 'speaker', 'id', 'role', 'description', 
+#                             'condition', 'type', or 'race'
+_pending_winfotype = None
+
+# ----------
+
+# the last function name encountered in a lua code (if any).
+# If no lua functions already encountered, this var will be None
+_pending_luafuncname = None
+
+# ----------
+
+# pending lua/wml string (they will be evaluated, and if translatable it will
+# be added in _dictionary
+_pending_luastring = None
+_pending_wmlstring = None
+
+# ----------
+
+# counting line number
+_current_lineno = 0
+# lineno_sub helps to set the right orderid of the future PoCommentedString
+_linenosub = 0
+
+
+
+# --------------------------------------------------------------------
+#  PART 2: machine.py functions and classes
+# --------------------------------------------------------------------
+
+
+
+def checkdomain():
+    global _currentdomain
+    global _domain
+    global _pending_addedinfo
+    global _pending_overrideinfo
+    if _currentdomain == _domain:
+        return True
+    else:
+        _pending_addedinfo = None
+        _pending_overrideinfo = None
+        return False
+
+
+
+def checksentence(mystring, finfo, *, islua=False):
+    m = re.match(r'\s*$', mystring)
+    if m:
+        wmlwarn(finfo, "found an empty translatable message")
+        return 1
+    elif warnall() and not islua:
+        if "}" in mystring:
+            wmsg = ("found a translatable string containing a WML macro. "
+                    " Translation for this string will NEVER work")
+            wmlwarn(finfo, wmsg)
+            return 2
+        else:
+            return 0
+    else:
+        return 0
+
+
+
+class PendingLuaString:
+    def __init__(self, lineno, luatype, luastring, ismultiline, 
+                 istranslatable, numequals=0):
+        self.lineno = lineno
+        self.luatype = luatype
+        self.luastring = ''
+        self.ismultiline = ismultiline
+        self.istranslatable = istranslatable
+        self.numequals = numequals
+        self.addline(luastring, True)
+    
+    def addline(self, value, isfirstline=False):
+        if self.luatype != 'luastr3':
+            value = re.sub('\\\s*$', '', value)
+        else:
+            value = value.replace('\\', r'\\')
+            # nope
+        if isfirstline:
+            self.luastring = value
+        else:
+            self.luastring = self.luastring + '\n' + value
+    
+    def store(self):
+        global _pending_addedinfo 
+        global _pending_overrideinfo
+        global _linenosub
+        if checkdomain() and self.istranslatable:
+            _linenosub += 1
+            finfo = pywmlx.nodemanip.fileref + ":" + str(self.lineno)
+            fileno = pywmlx.nodemanip.fileno
+            errcode = checksentence(self.luastring, finfo, islua=True)
+            if errcode != 1:
+                # when errcode is equal to 1, the translatable string is empty
+                # so, using "if errcode != 1" 
+                # we will add the translatable string ONLY if it is NOT empty
+                if self.luatype == 'luastr2':
+                    self.luastring = re.sub(r"\\\'", r"'", self.luastring)
+                if self.luatype != 'luastr3':
+                    self.luastring = re.sub(r'(?<!\\)"', r'\"', self.luastring)
+                if self.luatype == 'luastr3':
+                    self.luastring = self.luastring.replace('"', r'\"')
+                loc_wmlinfos = []
+                loc_addedinfos = None
+                if _pending_overrideinfo is not None:
+                    loc_wmlinfos.append(_pending_overrideinfo)
+                if (_pending_luafuncname is not None and 
+                        _pending_overrideinfo is None):
+                    winf = '[lua]: ' + _pending_luafuncname
+                    loc_wmlinfos.append(winf)
+                if _pending_addedinfo is None:
+                    loc_addedinfos = []
+                if _pending_addedinfo is not None:
+                    loc_addedinfos = _pending_addedinfo
+                loc_posentence = _dictionary.get(self.luastring)
+                if loc_posentence is None:
+                    _dictionary[self.luastring] = PoCommentedString(
+                                self.luastring, 
+                                orderid=(fileno, self.lineno, _linenosub),
+                                ismultiline=self.ismultiline,
+                                wmlinfos=loc_wmlinfos, finfos=[finfo],
+                                addedinfos=loc_addedinfos )
+                else:
+                    loc_posentence.update_with_commented_string(
+                           PoCommentedString(
+                                self.luastring, 
+                                orderid=(fileno, self.lineno, _linenosub),
+                                ismultiline=self.ismultiline,
+                                wmlinfos=loc_wmlinfos, finfos=[finfo],
+                                addedinfos=loc_addedinfos
+                    ) )
+        # finally PendingLuaString.store() will clear pendinginfos,
+        # in any case (even if the pending string is not translatable)
+        _pending_overrideinfo = None
+        _pending_addedinfo = None
+
+
+
+class PendingWmlString:
+    def __init__(self, lineno, wmlstring, ismultiline, istranslatable):
+        self.lineno = lineno
+        self.wmlstring = wmlstring.replace('\\', r'\\')
+        self.ismultiline = ismultiline
+        self.istranslatable = istranslatable
+    
+    def addline(self, value):
+        self.wmlstring = self.wmlstring + '\n' + value.replace('\\', r'\\')
+    
+    def store(self):
+        global _pending_addedinfo 
+        global _pending_overrideinfo
+        global _linenosub
+        global _pending_winfotype
+        if _pending_winfotype is not None:
+            if self.ismultiline is False and self.istranslatable is False:
+                winf = _pending_winfotype + '=' + self.wmlstring
+                pywmlx.nodemanip.addWmlInfo(winf)
+            _pending_winfotype = None
+        if checkdomain() and self.istranslatable:
+            finfo = pywmlx.nodemanip.fileref + ":" + str(self.lineno)
+            errcode = checksentence(self.wmlstring, finfo, islua=False)
+            if errcode != 1:
+                # when errcode is equal to 1, the translatable string is empty
+                # so, using "if errcode != 1" 
+                # we will add the translatable string ONLY if it is NOT empty
+                _linenosub += 1
+                self.wmlstring = re.sub('""', r'\"', self.wmlstring)
+                pywmlx.nodemanip.addNodeSentence(self.wmlstring, 
+                                             ismultiline=self.ismultiline, 
+                                             lineno=self.lineno, 
+                                             lineno_sub=_linenosub,
+                                             override=_pending_overrideinfo, 
+                                             addition=_pending_addedinfo)
+        _pending_overrideinfo = None
+        _pending_addedinfo = None
+
+
+
+def addstate(name, value):
+    global _states
+    if _states is None:
+        _states = {}
+    _states[name.lower()] = value
+
+
+
+def setup(dictionary, initialdomain, domain, wall):
+    global _dictionary
+    global _initialdomain
+    global _domain
+    global _warnall
+    _dictionary = dictionary
+    _initialdomain = initialdomain
+    _domain = domain
+    _warnall = wall
+    setup_luastates()
+    setup_wmlstates()
+
+
+
+def run(*, filebuf, fileref, fileno, startstate, waitwml=True):
+    global _states
+    global _current_lineno
+    global _linenosub
+    global _waitwml
+    global _currentdomain
+    global _dictionary
+    global _pending_luafuncname
+    global _on_luatag
+    _pending_luafuncname = None
+    _on_luatag = False
+    # cs is "current state"
+    cs = _states.get(startstate)
+    _current_lineno = 0
+    _linenosub = 0
+    _waitwml = waitwml
+    _currentdomain = _initialdomain
+    pywmlx.nodemanip.newfile(fileref, fileno)
+    # debug_cs = startstate
+    for xline in filebuf:
+        xline = xline.strip('\n\r')
+        _current_lineno += 1
+        while xline is not None:
+            # action number is used to know what function we should run
+            # debug_file0 = open(os.path.realpath('./debug.txt'), 'a')
+            # print("!!!", xline, file=debug_file0)
+            # debug_file0.close()
+            action = 0
+            v = None
+            m = None
+            if cs.regex is None:
+                # action = 1 --> execute state.run
+                action = 1 
+            else:
+                # m is match
+                m = re.match(cs.regex, xline)
+                if m:
+                    # action = 1 --> execute state.run
+                    action = 1
+                else:
+                    # action = 2 --> change to the state pointed by 
+                    #                state.iffail
+                    action = 2
+            if action == 1:
+                # xline, ns: xline --> override xline with new value
+                #            ns --> value of next state
+                xline, ns = cs.run(xline, _current_lineno, m)
+                # debug_cs = ns
+                cs = _states.get(ns)
+            else:
+                # debug_cs = cs.iffail
+                cs = _states.get(cs.iffail)
+            # debug_file = open(os.path.realpath('./debug.txt'), 'a')
+            # print(debug_cs, file=debug_file)
+            # debug_file.close()
+        # end while xline
+    # end for xline
+    pywmlx.nodemanip.closefile(_dictionary, _current_lineno)
+

utils/pywmlx/state/state.py

@@ -0,0 +1,6 @@
+class State:
+    def __init__(self, regex, run, iffail):
+        self.regex = regex
+        self.run = run
+        self.iffail = iffail
+

utils/pywmlx/state/wml_states.py

@@ -0,0 +1,280 @@
+import re
+import pywmlx.state.machine
+from pywmlx.state.state import State
+import pywmlx.nodemanip
+from pywmlx.wmlerr import wmlerr
+
+
+
+class WmlIdleState:
+    def __init__(self):
+        self.regex = None
+        self.iffail = None
+    
+    def run(self, xline, lineno, match):
+        _nextstate = 'wml_checkdom'
+        if pywmlx.state.machine._pending_wmlstring is not None:
+            pywmlx.state.machine._pending_wmlstring.store()
+            pywmlx.state.machine._pending_wmlstring = None
+        m = re.match(r'\s*$', xline)
+        if m:
+            xline = None
+            _nextstate = 'wml_idle'
+        return (xline, _nextstate) # 'wml_define'
+
+
+
+'''
+class WmlDefineState:
+    def __init__(self):
+        self.regex = re.compile('\s*#(define|enddef|\s+wmlxgettext:\s+)', re.I)
+        self.iffail = 'wml_checkdom'
+    
+    def run(self, xline, lineno, match):
+        if match.group(1).lower() == 'define':
+            # define
+            xline = None
+            if pywmlx.nodemanip.onDefineMacro is False:
+                pywmlx.nodemanip.onDefineMacro = True
+            else:
+                err_message = ("expected an #enddef before opening ANOTHER " + 
+                               "macro definition with #define")
+                finfo = pywmlx.nodemanip.fileref + ":" + str(lineno)
+                wmlerr(finfo, err_message)
+        elif match.group(1).lower() == 'enddef':
+            # enddef
+            xline = None
+            if pywmlx.nodemanip.onDefineMacro is True:
+                pywmlx.nodemanip.onDefineMacro = False
+            else:
+                err_message = ("found an #enddef, but no macro definition " +
+                               "is pending. Perhaps you forgot to put a " +
+                               "#define somewhere?")
+                finfo = pywmlx.nodemanip.fileref + ":" + str(lineno)
+                wmlerr(finfo, err_message)
+        else:
+            # wmlxgettext: {WML CODE}
+            xline = xline [ match.end(): ]
+        return (xline, 'wml_idle')
+'''
+
+
+
+class WmlCheckdomState:
+    def __init__(self):
+        self.regex = re.compile(r'\s*#textdomain\s+(\S+)', re.I)
+        self.iffail = 'wml_checkpo'
+    
+    def run(self, xline, lineno, match):
+        pywmlx.state.machine._currentdomain = match.group(1)
+        xline = None
+        return (xline, 'wml_idle')
+
+
+
+class WmlCheckpoState:
+    def __init__(self):
+        rx = r'\s*#\s*(wmlxgettext|po-override|po):\s+(.+)'
+        self.regex = re.compile(rx, re.I)
+        self.iffail = 'wml_comment'
+    
+    def run(self, xline, lineno, match):
+        if match.group(1) == 'wmlxgettext':
+            xline = match.group(2)
+        # on  #po: addedinfo
+        elif match.group(1) == "po":
+            xline = None
+            if pywmlx.state.machine._pending_addedinfo is None:
+                pywmlx.state.machine._pending_addedinfo = [ match.group(2) ]
+            else:
+                pywmlx.state.machine._pending_addedinfo.append(match.group(2))
+        # on -- #po-override: overrideinfo
+        elif pywmlx.state.machine._pending_overrideinfo is None:
+            pywmlx.state.machine._pending_overrideinfo = [ match.group(2) ]
+            xline = None
+        else:
+            pywmlx.state.machine._pending_overrideinfo.append(match.group(2))
+            xline = None
+        return (xline, 'wml_idle')
+
+
+
+class WmlCommentState:
+    def __init__(self):
+        self.regex = re.compile(r'\s*#.+')
+        self.iffail = 'wml_tag'
+    
+    def run(self, xline, lineno, match):
+        xline = None
+        return (xline, 'wml_idle')
+
+
+
+class WmlTagState:
+    def __init__(self):
+        # this regexp is deeply discussed in Source Documentation, chapter 6
+        rx = r'\s*(?:[^"]+\(\s*)?\[\s*([\/+-]?)\s*([A-Za-z0-9_]+)\s*\]'
+        self.regex = re.compile(rx)
+        self.iffail = 'wml_getinf'
+    
+    def run(self, xline, lineno, match):
+        # xdebug = open('./debug.txt', 'a')
+        # xdebug_str = None
+        if match.group(1) == '/':
+            closetag = '[/' + match.group(2) + ']'
+            pywmlx.nodemanip.closenode(closetag, 
+                                       pywmlx.state.machine._dictionary,
+                                       lineno)
+            if closetag == '[/lua]':
+                pywmlx.state.machine._pending_luafuncname = None
+                pywmlx.state.machine._on_luatag = False
+            # xdebug_str = closetag + ': ' + str(lineno)
+        else:
+            opentag = '[' + match.group(2) + ']'
+            pywmlx.nodemanip.newnode(opentag)
+            if(opentag == '[lua]'):
+                pywmlx.state.machine._on_luatag = True
+            # xdebug_str = opentag + ': ' + str(lineno)
+        # print(xdebug_str, file=xdebug)
+        # xdebug.close()
+        pywmlx.state.machine._pending_addedinfo = None
+        pywmlx.state.machine._pending_overrideinfo = None
+        xline = xline [ match.end(): ]
+        return (xline, 'wml_idle')
+
+
+
+class WmlGetinfState:
+    def __init__(self):
+        rx = ( r'\s*(speaker|id|role|description|condition|type|race)' +
+               r'\s*=\s*(.*)' )
+        self.regex = re.compile(rx, re.I)
+        self.iffail = 'wml_str01'
+    def run(self, xline, lineno, match):
+        _nextstate = 'wml_idle'
+        if '"' in match.group(2):
+            _nextstate = 'wml_str01'
+            pywmlx.state.machine._pending_winfotype = match.group(1)
+        else:
+            loc_wmlinfo = match.group(1) + '=' + match.group(2)
+            xline = None
+            pywmlx.nodemanip.addWmlInfo(loc_wmlinfo)
+        return (xline, _nextstate)
+
+
+
+class WmlStr01:
+    def __init__(self):
+        rx = r'(?:[^"]*?)\s*(_?)\s*"((?:""|[^"])*)("?)'
+        self.regex = re.compile(rx)
+        self.iffail = 'wml_golua'
+    
+    def run(self, xline, lineno, match):
+        _nextstate = 'wml_idle'
+        loc_translatable = True
+        if match.group(1) == "":
+            loc_translatable = False
+        loc_multiline = False
+        if match.group(3) == "":
+            xline = None
+            loc_multiline = True
+            _nextstate = 'wml_str10'
+        else:
+            xline = xline [ match.end(): ]    
+        pywmlx.state.machine._pending_wmlstring = (
+            pywmlx.state.machine.PendingWmlString( 
+                lineno, match.group(2), loc_multiline, loc_translatable
+            )
+        ) 
+        return (xline, _nextstate)
+
+
+
+# well... the regex will always be true on this state, so iffail will never
+# be executed
+class WmlStr10:
+    def __init__(self):
+        self.regex = re.compile(r'((?:""|[^"])*)("?)')
+        self.iffail = 'wml_str10'
+        
+    def run(self, xline, lineno, match):
+        _nextstate = None
+        pywmlx.state.machine._pending_wmlstring.addline( match.group(1) )
+        if match.group(2) == "":
+            _nextstate = 'wml_str10'
+            xline = None
+        else:
+            _nextstate = 'wml_idle'
+            xline = xline [ match.end(): ]
+        return (xline, _nextstate)
+
+
+
+# Only if the symbol '<<' is found inside a [lua] tag, than it means we are
+# actually starting a lua code.
+# It can happen that WML use the '<<' symbol in a very different context
+# wich has nothing to do with lua, so switching to the lua states in that
+# case can lead to problems.
+# This happened on the file data/gui/default/widget/toggle_button_orb.cfg
+# on wesnoth 1.13.x, where there is this line inside the first [image] tag:
+#
+#     name = "('buttons/misc/orb{STATE}.png" + <<~RC(magenta>{icon})')>>
+#
+# In that case, after 'name' there is a WML string 
+# "('buttons/misc/orb{STATE}.png"
+# And after that you find a cuncatenation with a literal string 
+# <<~RC(magenta>{icon})')>>
+#
+# That second string has nothing to do with lua, and, most importantly, if 
+# it is parsed with lua states, it return an error... why?
+# Semply becouse of the final ' symbol, wich is valid symbol, in lua, for
+# opening a new string; but, in that case, there is not an opening string,
+# but a ' symbol that must be used literally.
+#
+# This is why we use a global var _on_luatag in state.py wich is usually False.
+# it will be setted True only when opening a lua tag (see WmlTagState)
+# it will be setted to False again when the lua tag is closed (see WmlTagState)
+class WmlGoluaState:
+    def __init__(self):
+        self.regex = re.compile(r'.*?<<\s*')
+        self.iffail = 'wml_final'
+    
+    def run(self, xline, lineno, match):
+        if pywmlx.state.machine._on_luatag:
+            xline = xline [ match.end(): ]
+            return (xline, 'lua_idle')
+        else:
+            return (xline, 'wml_final')
+
+
+
+class WmlFinalState:
+    def __init__(self):
+        self.regex = None
+        self.iffail = None
+    
+    def run(self, xline, lineno, match):
+        xline = None
+        if pywmlx.state.machine._pending_wmlstring is not None:
+            pywmlx.state.machine._pending_wmlstring.store()
+            pywmlx.state.machine._pending_wmlstring = None
+        return (xline, 'wml_idle')
+
+
+
+def setup_wmlstates():
+    for statename, stateclass in [ ('wml_idle', WmlIdleState),
+                                   # ('wml_define', WmlDefineState),
+                                   ('wml_checkdom', WmlCheckdomState),
+                                   ('wml_checkpo', WmlCheckpoState),
+                                   ('wml_comment', WmlCommentState),
+                                   ('wml_tag', WmlTagState),                    
+                                   ('wml_getinf', WmlGetinfState),
+                                   ('wml_str01', WmlStr01),
+                                   ('wml_str10', WmlStr10),
+                                   ('wml_golua', WmlGoluaState),
+                                   ('wml_final', WmlFinalState)]:
+        st = stateclass()
+        pywmlx.state.machine.addstate(statename, 
+            State(st.regex, st.run, st.iffail) )
+

utils/pywmlx/wmlerr.py

@@ -0,0 +1,96 @@
+import os
+import sys
+import warnings
+
+enabled_ansi_col = True
+is_utest = False
+_warnall = False
+
+
+def wmlerr_debug():
+    global is_utest
+    is_utest = True
+
+
+
+def ansi_setEnabled(value):
+    global enabled_ansi_col
+    enabled_ansi_col = value
+ 
+
+
+def warnall():
+    return _warnall
+
+
+
+def set_warnall(value):
+    _warnall = value
+
+
+
+class WmlError(ValueError):
+    pass
+
+class WmlWarning(UserWarning):
+    pass
+
+
+
+def print_wmlerr(message, iserr):
+    ansi_color = '\033[91;1m'  #red
+    errtype = "error:"
+    if iserr is False:
+        ansi_color = '\033[94m'  #blue
+        errtype = "warning:"
+    # now we have ascii_color and errtype values
+    # here we print the error/warning. 
+    # 1) On posix we write "error" in red and "warning" in blue
+    if os.name == "posix" and enabled_ansi_col is True:
+        msg = ansi_color + errtype + ' ' + message
+    # 2) On non-posix systems (ex. windows) we don't use colors
+    else:
+        msg = errtype + ' ' + message
+    print(msg, file=sys.stderr)
+    
+    
+    
+def dualcol_message(finfo, message):
+    if os.name == "posix" and enabled_ansi_col is True:
+        msg = '\033[0m\033[93m' + finfo + ':\033[0m ' + message
+    else:
+        msg = finfo + ': ' + message
+    return msg
+    
+
+
+def my_showwarning(message, category, filename, lineno, file=None, line=None):
+    try:
+        print_wmlerr(message.args[0], False)
+    except OSError:
+        pass # the file (probably stderr) is invalid - this warning gets lost.
+
+
+
+warnings.showwarning = my_showwarning
+
+
+
+def wmlerr(finfo, message, errtype=WmlError):
+    if not is_utest:
+        try:
+            msg = dualcol_message(finfo, message)
+            raise errtype(msg)
+        except errtype as err:
+            print_wmlerr(err.args[0], True)
+            sys.exit(1)
+    else:
+        msg = dualcol_message(finfo, message)
+        raise errtype(msg)
+
+
+
+def wmlwarn(finfo, message):
+    msg = dualcol_message(finfo, message)
+    warnings.warn(msg, WmlWarning)
+

utils/wmlxgettext

@@ -1,397 +1,261 @@
-#!/usr/bin/perl -w
+#!/usr/bin/env python3
 
-# FIXME:
-# - maybe restrict "ability" matching to unit defs (not yet necessary)
 
-use strict;
-use File::Basename;
-use POSIX qw(strftime);
-use Getopt::Long;
-
-# use utf8 for stdout and file reading
-use utf8;
-use open IO => ':utf8';
-binmode(STDOUT, ':utf8');
-
-our $toplevel = '.';
-our $initialdomain = 'wesnoth';
-our $domain = undef;
-GetOptions ('directory=s' => \$toplevel,
-	    'initialdomain=s' => \$initialdomain,
-	    'domain=s' => \$domain);
-
-$domain = $initialdomain unless defined $domain;
-
-sub wml2postring {
-  my $str = shift;
-
-  $str =~ s/\"\"/\\\"/mg; # "" -> \"
-  $str =~ s/^(.*)$/"$1\\n"/mg; # Surround with doublequotes and add final escaped newline
-  $str =~ s/\n$/\n"\\n"/mg; # Split the string around newlines: "foo\nbar" -> "foo\n" LF "bar"
-  $str =~ s/\\n\"$/\"\n/g; # Remove final escaped newline and add newline after last line
-
-  return $str;
-}
-
-sub lua2postring {
-  my $str = shift;
-  my $quote = shift;
-
-  if ($quote eq "'") {
-    $str =~ s/"/\\"/g;
-  }
-  $str =~ s/^(.*)$/"$1\\n"/mg; # Surround with doublequotes and add final escaped newline
-  $str =~ s/\n$/\n"\\n"/mg; # Split the string around newlines: "foo\nbar" -> "foo\n" LF "bar"
-  $str =~ s/\\n\"$/\"\n/g; # Remove final escaped newline and add newline after last line
-
-  return $str;
-}
-
-## extract strings with their refs and node info into %messages
-
-sub possible_node_info {
-  my ($nodeinfostack, $field, $value) = @_;
-  if ($field =~ m/\b(speaker|id|role|description|condition|type|race)\b/) {
-    push @{$nodeinfostack->[-1][2]}, "$field=$value";
-  }
-}
-
-sub read_wml_file {
-  my ($file) = @_;
-  our (%messages,%nodeinfo);
-  my ($str, $translatable, $line);
-  my $readingattack = 0;
-  my @nodeinfostack = (["top", [], [], []]); # dummy top node
-  my @domainstack = ($initialdomain);
-  my $valid_wml = 1;
-  open (FILE, "<$file") or die "cannot read from $file";
- LINE: while (<FILE>) {
-    # record a #define scope
-    if (m/\#define/) {
-      unshift @domainstack, $domainstack[0];
-      next LINE;
-    } elsif (m/^\s*\#enddef/) {
-      shift @domainstack;
-    }
-
-    # change the current textdomain when hitting the directive
-    if (m/\#textdomain\s+(\S+)/) {
-      $domainstack[0] = $1;
-      next LINE;
-    }
-
-    if (m/\#\s*po:\s+(.+)/) {
-      push @{$nodeinfostack[-1][3]}, $1;
-    }
-
-    # skip other # lines as comments
-    next LINE if m/^\s*\#/ and !defined $str and not m/#\s*wmlxgettext/;
-
-    if (!defined $str and m/^(?:[^\"]*?)((?:_\s*)?)\"([^\"]*(?:\"\"[^\"]*)*)\"(.*)/) {
-      # single-line quoted string
-
-      $translatable = ($1 ne '');
-      my $rest = $3;
-
-      # if translatable and in the requested domain
-      if ($translatable and $domainstack[0] eq $domain) {
-        my $msg = wml2postring($2);
-        push @{$messages{$msg}}, [$file, $.];
-        push @{$nodeinfostack[-1][1]}, $msg if $valid_wml;
-
-      } elsif (not $translatable and m/(\S+)\s*=\s*\"([^\"]*)\"/) {
-        # may be a piece of node info to extract
-        possible_node_info(\@nodeinfostack, $1, $2) if $valid_wml;
-      }
-
-      # process remaining of the line
-      $_ = $rest . "\n";
-      redo LINE;
-
-    } elsif (!defined $str and m/^(?:[^\"]*?)((?:_\s*)?)\s*\"([^\"]*(?:\"\"[^\"]*)*)/) {
-      # start of multi-line
-
-      $translatable = ($1 ne '');
-      $_ = $2;
-      if (m/(.*)\r/) { $_ = "$1\n"; }
-      $str = $_;
-      $line = $.;
-
-    } elsif (m/([^\"]*(?:\"\"[^\"]*)*)\"(.*)/) {
-      # end of multi-line
-      die "end of string without a start in $file" if !defined $str;
-
-      $str .= $1;
-
-      if ($translatable and $domainstack[0] eq $domain) {
-        my $msg = "\"\"\n" . wml2postring($str);
-        push @{$messages{$msg}}, [$file, $line];
-        push @{$nodeinfostack[-1][1]}, $msg if $valid_wml;
-      }
-      $str = undef;
-
-      # process remaining of the line
-      $_ = $2 . "\n";
-      redo LINE;
-
-    } elsif (defined $str) {
-      # part of multi-line
-      if (m/(.*)\r/) { $_ = "$1\n"; }
-      $str .= $_;
-
-    } elsif (m/(\S+)\s*=\s*(.*?)\s*$/) {
-      # single-line non-quoted string
-      die "nested string in $file" if defined $str;
-      possible_node_info(\@nodeinfostack, $1, $2) if $valid_wml;
-
-### probably not needed ###
-#      # magic handling of weapon descriptions
-#      push @{$messages{wml2postring($2)}},  "$file:$."
-#	if $readingattack and
-#	  ($1 eq 'name' or $1 eq 'type' or $1 eq 'special');
+# encoding: utf8
+#
+# wmlxgettext -- generate a blank .po file for official campaigns translations 
+#                    (build tool for wesnoth core)
+#             -- if you are a UMC developer, you could use umcpo, instead
+#                
+#
+# By Nobun, october 2015
+#
+#                              PURPOSE
+#
+# wmlxgettext is a python3 tool that replace the old (but very good) 
+# perl script with the same name.
+# Replacing perl with python3 will ensure more portability.
+#
+# wmlxgettext is a tool that is directly used during wesnoth build process 
+# to generate the pot files for the core campaigns.
+#
+# If you are an UMC developer you might want to use umcpo instead of using
+# wmlxgettext directly (but using wmlxgettext directly is however possible).
+#
+#                              BASIC PROCEDURE
+#
+# todo
+#
+#                              MAGIC COMMENTS
+#
+# todo
+#
+#                              DEVELOPER INFORMATION
+#
+# todo.
 #
-#      # magic handling of unit abilities
-#      push @{$messages{wml2postring($2)}},  "$file:$."
-#	if $1 eq 'ability';
-
-    } elsif (m,\[attack\],) {
-      $readingattack = 1;
-    } elsif (m,\[/attack\],) {
-      $readingattack = 0;
-    }
-
-    # check for node opening/closing to handle message metadata
-    next LINE if not $valid_wml;
-    next LINE if defined $str; # skip lookup if inside multi-line
-    next LINE if m/^ *\{.*\} *$/; # skip lookup if a statement line
-    next LINE if m/^[^#]*=/; # skip lookup if a field line
-    while (m,\[ *([a-z/+].*?) *\],g) {
-      my $nodename = $1;
-      #my $ind = "  " x (@nodeinfostack + ($nodename =~ m,/, ? 0 : 1));
-      if ($nodename =~ s,/ *,,) { # closing node
-        if (@nodeinfostack == 0) {
-          warn "empty node stack on closed node at $file:$.";
-          $valid_wml = 0;
-          last;
-        }
-        my ($openname, $nodemessages, $metadata, $comments) = @{pop @nodeinfostack};
-        if ($nodename ne $openname) {
-          warn "expected closed node \'$openname\' ".
-               "got \'$nodename\' at $file:$.";
-          $valid_wml = 0;
-          last;
-        }
-        # some nodes should inherit parent metadata
-        if ($nodename =~ m/option/) {
-          $metadata = $nodeinfostack[-1][2];
-        }
-        #print STDERR "$ind<<< $.: $nodename\n";
-        #print STDERR "==> $file:$.: $nodename: @{$metadata}\n" if @{$nodemessages};
-        for my $msg (@{$nodemessages}) {
-          push @{$nodeinfo{$msg}}, [$nodename, $metadata, $comments];
-        }
-      } else { # opening node
-        #print STDERR "$ind>>> $.: $nodename\n";
-        $nodename =~ s/\+//;
-        push @nodeinfostack, [$nodename, [], [], []];
-      }
-    }
-    # do not add anything here, beware of the next's before the loop
-  }
-  pop @nodeinfostack if @nodeinfostack; # dummy top node
-  if (@nodeinfostack) {
-    warn "non-empty node stack at end of $file";
-    $valid_wml = 0;
-  }
-
-  close FILE;
-
-  if (not $valid_wml) {
-    warn "WML seems invalid for $file, node info extraction forfeited ".
-         "past the error point";
-  }
-}
-
-sub read_lua_file {
-  my ($file) = @_;
-  our (%messages,%nodeinfo);
-  my ($str, $translatable, $line, $quote);
-  my $curdomain = $initialdomain;
-  my $metadata = undef;
-  open (FILE, "<$file") or die "cannot read from $file";
-  LINE: while (<FILE>) {
-    if (defined $metadata and m/--\s*po:\s*(.*)$/) {
-      push @{@{$metadata}[2]}, $1;
-    }
-
-    # skip comments unless told not to
-    next LINE if m/^\s*--/ and not defined $str and not m/--\s*wmlxgettext/;
-
-    # change the textdomain
-    if (m/textdomain\s*\(?\s*(["'])([^"]+)\g1(.*)/) {
-      $curdomain = $2;
-
-      # process rest of the line
-      $_ = $3 . "\n";
-      redo LINE;
-    }
-
-    # Single-line quoted string
-    # TODO: support [[long strings]]
-    # TODO: get 'singlequotes' to work
-    if (not defined $str and m/^(?:[^"]*?)((?:_\s*)?)\s*(")([^"]*(?:\\"[^"]*)*)"(.*)/) {
-      # $1 = underscore or not
-      # $2 = quote (double or single)
-      # $3 = string
-      # $4 = rest
-      my $translatable = ($1 ne '');
-      my $rest = $4;
-
-      if ($translatable and $curdomain eq $domain) {
-        my $msg = lua2postring($3, $2);
-        push @{$messages{$msg}}, [$file, $.];
-        push @{$nodeinfo{$msg}}, $metadata if defined $metadata;
-      }
-
-      # process rest of the line
-      $_ = $rest . "\n";
-      redo LINE;
-    }
-    # Begin multiline quoted string
-    elsif (not defined $str and m/^(?:[^"]*?)((?:_\s*)?)\s*(")([^"]*(?:\\"[^"]*)*)/) {
-      # $1 = underscore or not
-      # $2 = quote (double or single)
-      # $3 = string
-      $translatable = ($1 ne '');
-      $quote = $2;
-      $str = $3;
-      $line = $.;
-    }
-    # End multiline quoted string
-    elsif (m/([^"]*(?:\\"[^"]*)*)(")(.*)/) {
-      # $1 = string
-      # $2 = quote (double or single)
-      # $3 = rest
-      die "end of string without a start in $file" if not defined $str;
-
-      $str .= $1;
-      my $rest = $3;
-
-      # TODO: ensure $quote eq $2 when this matches
-      if ($translatable and $curdomain eq $domain) {
-        my $msg = "\"\"\n" . lua2postring($str, $quote);
-        push @{$messages{$msg}}, [$file, $line];
-        push @{$nodeinfo{$msg}}, $metadata if defined $metadata;
-      }
-      $str = undef;
-      $translatable = undef;
-      $line = undef;
-      $quote = undef;
-
-      # process rest of the line
-      $_ = $rest . "\n";
-      redo LINE;
-    }
-    # Middle of multiline quoted string
-    elsif (defined $str) {
-      $str .= $_;
-    }
-    # Function or WML tag
-    elsif (m/function\s+([a-zA-Z0-9_.]+)\s*\(/ or m/([a-zA-Z0-9_.]+)\s*=\s*function/) {
-      $metadata = ["lua", [$1], []];
-    }
-  }
-}
-
-our (%messages,%nodeinfo);
-chdir $toplevel;
-foreach my $file (@ARGV) {
-  if ($file =~ m/\.lua$/i) {
-    read_lua_file($file);
-  } else {
-    read_wml_file($file);
-  }
-}
-
-## index strings by their location in the source so we can sort them
-
-our @revmessages;
-foreach my $key (keys %messages) {
-  foreach my $line (@{$messages{$key}}) {
-    my ($file, $lineno) = @{$line};
-    push @revmessages, [ $file, $lineno, $key ];
-  }
-}
-
-# sort them
-@revmessages = sort { $a->[0] cmp $b->[0] or $a->[1] <=> $b->[1] } @revmessages;
 
 
-## output
+import os
+import re
+import sys
+import warnings
+import argparse
+from datetime import datetime
+import pywmlx
 
-my $date = strftime "%F %R%z", localtime();
 
-print <<EOH
-msgid ""
-msgstr ""
-"Project-Id-Version: PACKAGE VERSION\\n"
-"Report-Msgid-Bugs-To: http://bugs.wesnoth.org/\\n"
-"POT-Creation-Date: $date\\n"
-EOH
-;
-# we must break this string to avoid triggering a bug in some po-mode
-# installations, at save-time for this file
-print "\"PO-Revision-Date: YEAR-MO-DA ", "HO:MI+ZONE\\n\"\n";
-print <<EOH
-"Last-Translator: FULL NAME <EMAIL\@ADDRESS>\\n"
-"Language-Team: LANGUAGE <LL\@li.org>\\n"
-"MIME-Version: 1.0\\n"
-"Content-Type: text/plain; charset=UTF-8\\n"
-"Content-Transfer-Encoding: 8bit\\n"
 
-EOH
-;
+def commandline(args):
+    parser = argparse.ArgumentParser(
+        description='Generate .po from WML/lua file list.',
+        usage='''wmlxgettext --domain=DOMAIN [--directory=START_PATH]
+                   [--recursive] [--initialdomain=INITIALDOMAIN]
+                   [--package-version=PACKAGE_VERSION]
+                   [--no-ansi-colors] [--fuzzy] [--warnall] [-o OUTPUT_FILE]
+                   FILE1 FILE2 ... FILEN'''
+    )
+    parser.add_argument(
+        '-o',
+        default=None,
+        dest='outfile',
+        help= ('Destination file. By default the output '
+               'will be printed on stdout')
+    )
+    parser.add_argument(
+        '--domain', 
+        default='wmlxgettext',
+        required=True,
+        dest='domain',
+        help= ('The textdomain (on WML/lua file) wich contains the '
+               'strings that will be actually translated '
+               '[**REQUIRED ARGUMENT**]')
+    )
+    parser.add_argument(
+        '--directory', 
+        default='.',
+        dest='start_path',
+        help=('Complete path of your "start directory". '
+              'The path to every source file should start from this '
+              'directory.')
+    )
+    parser.add_argument(
+        '--initialdomain', 
+        default='wesnoth',
+        dest='initdom',
+        help=('Initial domain value on WML/lua file when no textdomain '
+              'setted in that WML/lua file.\nBy default it is equal to '
+              '"wesnoth" and usually you don\'t need to change this value')
+    )
+    parser.add_argument(
+        '--package-version',
+        default='PACKAGE VERSION',
+        dest='package_version',
+        help=('Version number of your wesnoth add-on. You don\'t actually '
+              'require to set this option since you can directly edit the '
+              'po file produced by wmlxgettext. However this option could '
+              'help you to save a bit of time')
+    )
+    parser.add_argument(
+        '--no-ansi-colors',
+        action='store_false',
+        default=True,
+        dest='ansi_col',
+        help=("By default warnings are displayed with colored text. You can "
+              "disable this feature using this flag.\n"
+              "This option doesn't have any effect on windows, since it "
+              "doesn't support ansi colors (on windows colors are ALWAYS"
+              ' disabled).')
+    )
+    parser.add_argument(
+        '--warnall',
+        action='store_true',
+        default=False,
+        dest='warnall',
+        help="Show all warnings. By default some warnings are hided"
+    )
+    parser.add_argument(
+        '--fuzzy',
+        action='store_true',
+        default=False,
+        dest='fuzzy',
+        help=("If you specify this flag, all sentences contained on the POT "
+              "file created by wmlxgettext will be setted as fuzzy.\n"
+              "By default sentences are NOT setted as fuzzy")
+    )
+    parser.add_argument(
+        '--recursive',
+        action='store_true',
+        default=False,
+        help=("If this option is used, wmlxgettext will scan recursively the"
+              " directory setted on the '--directory' parameter. "
+              "If this option is used, EXPLICIT LIST of files will be "
+              "ignored.") 
+    )
+    parser.add_argument(
+        'filelist',
+        help='List of WML/lua files of your UMC (source files)',
+        nargs='*'
+    )
+    return parser.parse_args(args)
 
-foreach my $occurence (@revmessages) {
-  my $key = $occurence->[2];
-  if (defined $messages{$key}) {
-    if (defined $nodeinfo{$key}) {
-      my %added;
-      my @lines;
-      for my $info (@{$nodeinfo{$key}}) {
-        my ($name, $data, $comments) = @{$info};
-        my $desc = join(", ", @{$data});
-        my $nodestr = $desc ? "[$name]: $desc" : "[$name]";
-        # Add only unique node info strings.
-        if (not defined $added{$nodestr}) {
-            $added{$nodestr} = 1;
-            push @lines, [sprintf("#. %s\n", $nodestr), []];
-        }
-        foreach my $comment (@{$comments}) {
-            push @{@{$lines[-1]}[1]}, sprintf("#. $comment\n");
-        }
-      }
-      for my $line (sort { @{$a}[0] cmp @{$b}[0] } @lines) {
-        my ($tag, $comments) = @{$line};
-        print $tag;
-        foreach my $comment (@{$comments}) {
-          print $comment;
-        }
-      }
-    }
-    my @msgs = sort { $a->[0] cmp $b->[0] or $a->[1] <=> $b->[1] } @{$messages{$key}};
-    my @msgsfmt;
-    foreach my $msg (@msgs) {
-      my ($file, $line) = @{$msg};
-      push @msgsfmt, "$file:$line";
-    }
-    my $locationdata = join(" ", @msgsfmt);
-    die "Translatable empty string at $locationdata" if $key =~ /^""\n$/;
-    printf "#: %s\n", $locationdata;
-    print "msgid $key";
-    print "msgstr \"\"\n\n";
 
-    # be sure we don't output the same message twice
-    delete $messages{$key};
-  }
-}
+
+def main():
+    # nota: rimane scoperto il controllo dell' eventualità che l'ultimo #define
+    # di un file WML non sia stato correttamente chiuso da un #enddef entro
+    # la fine del file
+    args = commandline(sys.argv[1:])
+    pywmlx.ansi_setEnabled(args.ansi_col)
+    pywmlx.set_warnall(args.warnall)
+    startPath = os.path.realpath(os.path.normpath(args.start_path))
+    sentlist = dict()
+    fileno = 0
+    pywmlx.statemachine.setup(sentlist, args.initdom, args.domain, 
+                              args.warnall)
+    if args.warnall is True and args.outfile is None:
+        pywmlx.wmlwarn('command line warning', 'Writing the output to stdout '
+                       '(and then eventually redirect that output to a file) '
+                       'is a deprecated usage. Please, consider to use the '
+                       '"-o <outfile.po>" option, instead of using the '
+                       'output redirection')
+    filelist = None
+    if args.recursive is False and args.filelist is None:
+        pywmlx.wmlerr("bad command line", "FILELIST must not be empty. "
+               "Please, run wmlxgettext again and, this time, add some file "
+               "in FILELIST or use the --recursive option.")
+    elif args.recursive is False and len(args.filelist) <= 0:
+        pywmlx.wmlerr("bad command line", "FILELIST must not be empty. " 
+               "Please, run wmlxgettext again and, this time, add some file "
+               "in FILELIST or use the --recursive option.")
+    elif args.recursive is False:
+        filelist = args.filelist
+    # the following elif cases implicitly expexcts that args.recursive is True
+    elif args.filelist is not None:
+        if len(args.filelist) > 0:
+            pywmlx.wmlwarn("command line warning", "Option --recursive was "
+                "used, but FILELIST is not empty. All extra file listed in "
+                "FILELIST will be ignored.")
+        # if we use the --recursive option we recursively scan the add-on
+        # directory. 
+        #    But we want that the file reference informations placed 
+        # into the .po file will remember the (relative) root name of the
+        # addon. 
+        #    This is why the autof.autoscan function returns a tuple of 
+        # values: 
+        #   the first one is the parent directory of the original startPath
+        #   the second one is the filelist (with the "fixed" file references)
+        # In this way we will override startPath to the parent directory
+        # containing the main directory of the wesnoth add-on, without 
+        # introducing bugs.
+        startPath, filelist = pywmlx.autof.autoscan(startPath)
+    # this last case is equal to: 
+    # if args.recursive is True and args.filelist is None:
+    else:
+        startPath, filelist = pywmlx.autof.autoscan(startPath)
+    for fileno, fx in enumerate(filelist):
+        fname = os.path.join(startPath, os.path.normpath(fx))
+        is_file = os.path.isfile(fname)
+        if is_file:
+            infile = None
+            try: 
+                infile = open(fname, 'r', encoding="utf-8")
+            except OSError as e:
+                errmsg = 'cannot read file: ' + e.args[1]
+                pywmlx.wmlerr(e.filename, errmsg, OSError)
+            if fname.lower().endswith('.cfg'):
+                pywmlx.statemachine.run(filebuf=infile, fileref=fx, 
+                            fileno=fileno, startstate='wml_idle', waitwml=True)
+            if fname.lower().endswith('.lua'):
+                pywmlx.statemachine.run(filebuf=infile, fileref=fx, 
+                            fileno=fileno, startstate='lua_idle', waitwml=False)
+            infile.close()
+    outfile = None
+    if args.outfile is None:
+        outfile = sys.stdout
+    else:
+        outfile_name = os.path.realpath(os.path.normpath(args.outfile))
+        try:
+            outfile = open(outfile_name, 'w', encoding="utf-8")
+        except OSError as e:
+            errmsg = 'cannot write file: ' + e.args[1]
+            pywmlx.wmlerr(e.filename, errmsg, OSError)
+    pkgversion = args.package_version + '\\n"'
+    print('msgid ""\nmsgstr ""', file=outfile)
+    print('"Project-Id-Version:', pkgversion, file=outfile)
+    print('"Report-Msgid-Bugs-To: http://bugs.wesnoth.org/\\n"', file=outfile)
+    now = datetime.utcnow()
+    cdate = '{:04d}-{:02d}-{:02d} {:02d}:{:02d} UTC\\n"'.format(now.year,
+                                                                now.month,
+                                                                now.day,
+                                                                now.hour,
+                                                                now.minute)
+
+    print('"POT-Creation-Date:', cdate, file=outfile)
+    print('"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\\n"', file=outfile)
+    print('"Last-Translator: FULL NAME <EMAIL@ADDRESS>\\n"', file=outfile)
+    print('"Language-Team: LANGUAGE <LL@li.org>\\n"', file=outfile)
+    print('"MIME-Version: 1.0\\n"', file=outfile)
+    print('"Content-Type: text/plain; charset=UTF-8\\n"', file=outfile)
+    print('"Content-Transfer-Encoding: 8bit\\n"\n', file=outfile)
+    for posentence in sorted(sentlist.values(), key=lambda x: x.orderid):
+        posentence.write(outfile, args.fuzzy)
+        print('', file=outfile)
+    if args.outfile is not None:
+        outfile.close()
+
+
+
+if __name__ == "__main__":
+    main()
+
+
+
+# def test_wmlerr_internal():
+#     wmlwarn("file1:5", "testing warning...")
+#     wmlerr("file2:7", "testing error...")
+#     print("this string should not be printed")
+
+
+# if __name__ == "__main__":   test_wmlerr_internal()
+

utils/wmlxgettext_perl

@@ -0,0 +1,397 @@
+#!/usr/bin/perl -w
+
+# FIXME:
+# - maybe restrict "ability" matching to unit defs (not yet necessary)
+
+use strict;
+use File::Basename;
+use POSIX qw(strftime);
+use Getopt::Long;
+
+# use utf8 for stdout and file reading
+use utf8;
+use open IO => ':utf8';
+binmode(STDOUT, ':utf8');
+
+our $toplevel = '.';
+our $initialdomain = 'wesnoth';
+our $domain = undef;
+GetOptions ('directory=s' => \$toplevel,
+	    'initialdomain=s' => \$initialdomain,
+	    'domain=s' => \$domain);
+
+$domain = $initialdomain unless defined $domain;
+
+sub wml2postring {
+  my $str = shift;
+
+  $str =~ s/\"\"/\\\"/mg; # "" -> \"
+  $str =~ s/^(.*)$/"$1\\n"/mg; # Surround with doublequotes and add final escaped newline
+  $str =~ s/\n$/\n"\\n"/mg; # Split the string around newlines: "foo\nbar" -> "foo\n" LF "bar"
+  $str =~ s/\\n\"$/\"\n/g; # Remove final escaped newline and add newline after last line
+
+  return $str;
+}
+
+sub lua2postring {
+  my $str = shift;
+  my $quote = shift;
+
+  if ($quote eq "'") {
+    $str =~ s/"/\\"/g;
+  }
+  $str =~ s/^(.*)$/"$1\\n"/mg; # Surround with doublequotes and add final escaped newline
+  $str =~ s/\n$/\n"\\n"/mg; # Split the string around newlines: "foo\nbar" -> "foo\n" LF "bar"
+  $str =~ s/\\n\"$/\"\n/g; # Remove final escaped newline and add newline after last line
+
+  return $str;
+}
+
+## extract strings with their refs and node info into %messages
+
+sub possible_node_info {
+  my ($nodeinfostack, $field, $value) = @_;
+  if ($field =~ m/\b(speaker|id|role|description|condition|type|race)\b/) {
+    push @{$nodeinfostack->[-1][2]}, "$field=$value";
+  }
+}
+
+sub read_wml_file {
+  my ($file) = @_;
+  our (%messages,%nodeinfo);
+  my ($str, $translatable, $line);
+  my $readingattack = 0;
+  my @nodeinfostack = (["top", [], [], []]); # dummy top node
+  my @domainstack = ($initialdomain);
+  my $valid_wml = 1;
+  open (FILE, "<$file") or die "cannot read from $file";
+ LINE: while (<FILE>) {
+    # record a #define scope
+    if (m/\#define/) {
+      unshift @domainstack, $domainstack[0];
+      next LINE;
+    } elsif (m/^\s*\#enddef/) {
+      shift @domainstack;
+    }
+
+    # change the current textdomain when hitting the directive
+    if (m/\#textdomain\s+(\S+)/) {
+      $domainstack[0] = $1;
+      next LINE;
+    }
+
+    if (m/\#\s*po:\s+(.+)/) {
+      push @{$nodeinfostack[-1][3]}, $1;
+    }
+
+    # skip other # lines as comments
+    next LINE if m/^\s*\#/ and !defined $str and not m/#\s*wmlxgettext/;
+
+    if (!defined $str and m/^(?:[^\"]*?)((?:_\s*)?)\"([^\"]*(?:\"\"[^\"]*)*)\"(.*)/) {
+      # single-line quoted string
+
+      $translatable = ($1 ne '');
+      my $rest = $3;
+
+      # if translatable and in the requested domain
+      if ($translatable and $domainstack[0] eq $domain) {
+        my $msg = wml2postring($2);
+        push @{$messages{$msg}}, [$file, $.];
+        push @{$nodeinfostack[-1][1]}, $msg if $valid_wml;
+
+      } elsif (not $translatable and m/(\S+)\s*=\s*\"([^\"]*)\"/) {
+        # may be a piece of node info to extract
+        possible_node_info(\@nodeinfostack, $1, $2) if $valid_wml;
+      }
+
+      # process remaining of the line
+      $_ = $rest . "\n";
+      redo LINE;
+
+    } elsif (!defined $str and m/^(?:[^\"]*?)((?:_\s*)?)\s*\"([^\"]*(?:\"\"[^\"]*)*)/) {
+      # start of multi-line
+
+      $translatable = ($1 ne '');
+      $_ = $2;
+      if (m/(.*)\r/) { $_ = "$1\n"; }
+      $str = $_;
+      $line = $.;
+
+    } elsif (m/([^\"]*(?:\"\"[^\"]*)*)\"(.*)/) {
+      # end of multi-line
+      die "end of string without a start in $file" if !defined $str;
+
+      $str .= $1;
+
+      if ($translatable and $domainstack[0] eq $domain) {
+        my $msg = "\"\"\n" . wml2postring($str);
+        push @{$messages{$msg}}, [$file, $line];
+        push @{$nodeinfostack[-1][1]}, $msg if $valid_wml;
+      }
+      $str = undef;
+
+      # process remaining of the line
+      $_ = $2 . "\n";
+      redo LINE;
+
+    } elsif (defined $str) {
+      # part of multi-line
+      if (m/(.*)\r/) { $_ = "$1\n"; }
+      $str .= $_;
+
+    } elsif (m/(\S+)\s*=\s*(.*?)\s*$/) {
+      # single-line non-quoted string
+      die "nested string in $file" if defined $str;
+      possible_node_info(\@nodeinfostack, $1, $2) if $valid_wml;
+
+### probably not needed ###
+#      # magic handling of weapon descriptions
+#      push @{$messages{wml2postring($2)}},  "$file:$."
+#	if $readingattack and
+#	  ($1 eq 'name' or $1 eq 'type' or $1 eq 'special');
+#
+#      # magic handling of unit abilities
+#      push @{$messages{wml2postring($2)}},  "$file:$."
+#	if $1 eq 'ability';
+
+    } elsif (m,\[attack\],) {
+      $readingattack = 1;
+    } elsif (m,\[/attack\],) {
+      $readingattack = 0;
+    }
+
+    # check for node opening/closing to handle message metadata
+    next LINE if not $valid_wml;
+    next LINE if defined $str; # skip lookup if inside multi-line
+    next LINE if m/^ *\{.*\} *$/; # skip lookup if a statement line
+    next LINE if m/^[^#]*=/; # skip lookup if a field line
+    while (m,\[ *([a-z/+].*?) *\],g) {
+      my $nodename = $1;
+      #my $ind = "  " x (@nodeinfostack + ($nodename =~ m,/, ? 0 : 1));
+      if ($nodename =~ s,/ *,,) { # closing node
+        if (@nodeinfostack == 0) {
+          warn "empty node stack on closed node at $file:$.";
+          $valid_wml = 0;
+          last;
+        }
+        my ($openname, $nodemessages, $metadata, $comments) = @{pop @nodeinfostack};
+        if ($nodename ne $openname) {
+          warn "expected closed node \'$openname\' ".
+               "got \'$nodename\' at $file:$.";
+          $valid_wml = 0;
+          last;
+        }
+        # some nodes should inherit parent metadata
+        if ($nodename =~ m/option/) {
+          $metadata = $nodeinfostack[-1][2];
+        }
+        #print STDERR "$ind<<< $.: $nodename\n";
+        #print STDERR "==> $file:$.: $nodename: @{$metadata}\n" if @{$nodemessages};
+        for my $msg (@{$nodemessages}) {
+          push @{$nodeinfo{$msg}}, [$nodename, $metadata, $comments];
+        }
+      } else { # opening node
+        #print STDERR "$ind>>> $.: $nodename\n";
+        $nodename =~ s/\+//;
+        push @nodeinfostack, [$nodename, [], [], []];
+      }
+    }
+    # do not add anything here, beware of the next's before the loop
+  }
+  pop @nodeinfostack if @nodeinfostack; # dummy top node
+  if (@nodeinfostack) {
+    warn "non-empty node stack at end of $file";
+    $valid_wml = 0;
+  }
+
+  close FILE;
+
+  if (not $valid_wml) {
+    warn "WML seems invalid for $file, node info extraction forfeited ".
+         "past the error point";
+  }
+}
+
+sub read_lua_file {
+  my ($file) = @_;
+  our (%messages,%nodeinfo);
+  my ($str, $translatable, $line, $quote);
+  my $curdomain = $initialdomain;
+  my $metadata = undef;
+  open (FILE, "<$file") or die "cannot read from $file";
+  LINE: while (<FILE>) {
+    if (defined $metadata and m/--\s*po:\s*(.*)$/) {
+      push @{@{$metadata}[2]}, $1;
+    }
+
+    # skip comments unless told not to
+    next LINE if m/^\s*--/ and not defined $str and not m/--\s*wmlxgettext/;
+
+    # change the textdomain
+    if (m/textdomain\s*\(?\s*(["'])([^"]+)\g1(.*)/) {
+      $curdomain = $2;
+
+      # process rest of the line
+      $_ = $3 . "\n";
+      redo LINE;
+    }
+
+    # Single-line quoted string
+    # TODO: support [[long strings]]
+    # TODO: get 'singlequotes' to work
+    if (not defined $str and m/^(?:[^"]*?)((?:_\s*)?)\s*(")([^"]*(?:\\"[^"]*)*)"(.*)/) {
+      # $1 = underscore or not
+      # $2 = quote (double or single)
+      # $3 = string
+      # $4 = rest
+      my $translatable = ($1 ne '');
+      my $rest = $4;
+
+      if ($translatable and $curdomain eq $domain) {
+        my $msg = lua2postring($3, $2);
+        push @{$messages{$msg}}, [$file, $.];
+        push @{$nodeinfo{$msg}}, $metadata if defined $metadata;
+      }
+
+      # process rest of the line
+      $_ = $rest . "\n";
+      redo LINE;
+    }
+    # Begin multiline quoted string
+    elsif (not defined $str and m/^(?:[^"]*?)((?:_\s*)?)\s*(")([^"]*(?:\\"[^"]*)*)/) {
+      # $1 = underscore or not
+      # $2 = quote (double or single)
+      # $3 = string
+      $translatable = ($1 ne '');
+      $quote = $2;
+      $str = $3;
+      $line = $.;
+    }
+    # End multiline quoted string
+    elsif (m/([^"]*(?:\\"[^"]*)*)(")(.*)/) {
+      # $1 = string
+      # $2 = quote (double or single)
+      # $3 = rest
+      die "end of string without a start in $file" if not defined $str;
+
+      $str .= $1;
+      my $rest = $3;
+
+      # TODO: ensure $quote eq $2 when this matches
+      if ($translatable and $curdomain eq $domain) {
+        my $msg = "\"\"\n" . lua2postring($str, $quote);
+        push @{$messages{$msg}}, [$file, $line];
+        push @{$nodeinfo{$msg}}, $metadata if defined $metadata;
+      }
+      $str = undef;
+      $translatable = undef;
+      $line = undef;
+      $quote = undef;
+
+      # process rest of the line
+      $_ = $rest . "\n";
+      redo LINE;
+    }
+    # Middle of multiline quoted string
+    elsif (defined $str) {
+      $str .= $_;
+    }
+    # Function or WML tag
+    elsif (m/function\s+([a-zA-Z0-9_.]+)\s*\(/ or m/([a-zA-Z0-9_.]+)\s*=\s*function/) {
+      $metadata = ["lua", [$1], []];
+    }
+  }
+}
+
+our (%messages,%nodeinfo);
+chdir $toplevel;
+foreach my $file (@ARGV) {
+  if ($file =~ m/\.lua$/i) {
+    read_lua_file($file);
+  } else {
+    read_wml_file($file);
+  }
+}
+
+## index strings by their location in the source so we can sort them
+
+our @revmessages;
+foreach my $key (keys %messages) {
+  foreach my $line (@{$messages{$key}}) {
+    my ($file, $lineno) = @{$line};
+    push @revmessages, [ $file, $lineno, $key ];
+  }
+}
+
+# sort them
+@revmessages = sort { $a->[0] cmp $b->[0] or $a->[1] <=> $b->[1] } @revmessages;
+
+
+## output
+
+my $date = strftime "%F %R%z", localtime();
+
+print <<EOH
+msgid ""
+msgstr ""
+"Project-Id-Version: PACKAGE VERSION\\n"
+"Report-Msgid-Bugs-To: http://bugs.wesnoth.org/\\n"
+"POT-Creation-Date: $date\\n"
+EOH
+;
+# we must break this string to avoid triggering a bug in some po-mode
+# installations, at save-time for this file
+print "\"PO-Revision-Date: YEAR-MO-DA ", "HO:MI+ZONE\\n\"\n";
+print <<EOH
+"Last-Translator: FULL NAME <EMAIL\@ADDRESS>\\n"
+"Language-Team: LANGUAGE <LL\@li.org>\\n"
+"MIME-Version: 1.0\\n"
+"Content-Type: text/plain; charset=UTF-8\\n"
+"Content-Transfer-Encoding: 8bit\\n"
+
+EOH
+;
+
+foreach my $occurence (@revmessages) {
+  my $key = $occurence->[2];
+  if (defined $messages{$key}) {
+    if (defined $nodeinfo{$key}) {
+      my %added;
+      my @lines;
+      for my $info (@{$nodeinfo{$key}}) {
+        my ($name, $data, $comments) = @{$info};
+        my $desc = join(", ", @{$data});
+        my $nodestr = $desc ? "[$name]: $desc" : "[$name]";
+        # Add only unique node info strings.
+        if (not defined $added{$nodestr}) {
+            $added{$nodestr} = 1;
+            push @lines, [sprintf("#. %s\n", $nodestr), []];
+        }
+        foreach my $comment (@{$comments}) {
+            push @{@{$lines[-1]}[1]}, sprintf("#. $comment\n");
+        }
+      }
+      for my $line (sort { @{$a}[0] cmp @{$b}[0] } @lines) {
+        my ($tag, $comments) = @{$line};
+        print $tag;
+        foreach my $comment (@{$comments}) {
+          print $comment;
+        }
+      }
+    }
+    my @msgs = sort { $a->[0] cmp $b->[0] or $a->[1] <=> $b->[1] } @{$messages{$key}};
+    my @msgsfmt;
+    foreach my $msg (@msgs) {
+      my ($file, $line) = @{$msg};
+      push @msgsfmt, "$file:$line";
+    }
+    my $locationdata = join(" ", @msgsfmt);
+    die "Translatable empty string at $locationdata" if $key =~ /^""\n$/;
+    printf "#: %s\n", $locationdata;
+    print "msgid $key";
+    print "msgstr \"\"\n\n";
+
+    # be sure we don't output the same message twice
+    delete $messages{$key};
+  }
+}