Minor updates:

*Made branches' po update multi-process!
*0.8 looks a bit too much a "similarity cutoff" value for messages, changed to 0.75.

release/scripts/modules/bl_i18n_utils/update_po.py

@@ -22,7 +22,7 @@
 
 # Update po’s in the branches from blender.pot in /trunk/po dir.
 
-import subprocess
+import concurrent.futures
 import os
 import sys
 from codecs import open
@@ -41,69 +41,22 @@ TRUNK_PO_DIR = settings.TRUNK_PO_DIR
 FILE_NAME_POT = settings.FILE_NAME_POT
 
 
-# XXX Not updated, not sure it's that much useful...
+def process_po(data):
-def pproc_newcontext_po(po, pot_messages, pot_stats):
+    po, lang, pot_msgs = data
-    print("Adding new contexts to {}...".format(po))
-    messages, state, stats = utils.parse_messages(po)
-    known_ctxt = stats["contexts"]
-    print("Already known (present) context(s): {}".format(str(known_ctxt)))
-
-    new_ctxt = set()
-    added = 0
-    # Only use valid already translated messages!
-    allowed_keys = state["trans_msg"] - state["fuzzy_msg"] - state["comm_msg"]
-    for key in pot_messages.keys():
-        ctxt, msgid = key
-        if ctxt in known_ctxt:
-            continue
-        new_ctxt.add(ctxt)
-        for t_ctxt in known_ctxt:
-            # XXX The first match will win, this might not be optimal...
-            t_key = (t_ctxt, msgid)
-            if t_key in allowed_keys:
-                # Wrong comments (sources) will be removed by msgmerge...
-                messages[key] = messages[t_key]
-                messages[key]["msgctxt_lines"] = [ctxt]
-                added += 1
-
-    utils.write_messages(po, messages, state["comm_msg"], state["fuzzy_msg"])
-    print("Finished!\n    {} new context(s) was/were added {}, adding {} new "
-          "messages.\n".format(len(new_ctxt), str(new_ctxt), added))
-    return 0
-
-
-def process_po(po, lang):
     # update po file
-    #cmd = (GETTEXT_MSGMERGE_EXECUTABLE,
-           #"--update",
-           #"-w", "1",  # XXX Ugly hack to prevent msgmerge merging short source comments together!
-           #"--no-wrap",
-           #"--backup=none",
-           #"--lang={}".format(lang),
-           #po,
-           #FILE_NAME_POT,
-           #)
-
-    pot = utils.I18nMessages(kind='PO', src=FILE_NAME_POT)
     msg = utils.I18nMessages(iso=lang, kind='PO', src=po)
     print("Updating {}...".format(po))
-    msg.update(pot)
+    msg.update(pot_msgs)
     msg.write(kind='PO', dest=po)
-    #print("Running ", " ".join(cmd))
+    print("Finished updating {}!\n".format(po))
-    #ret = subprocess.call(cmd)
-    print("Finished!\n")
     return 0
 
 
 def main():
     import argparse
-    parser = argparse.ArgumentParser(description="Write out messages.txt "
+    parser = argparse.ArgumentParser(description="Write out messages.txt from Blender.")
-                                                 "from Blender.")
     parser.add_argument('-t', '--trunk', action="store_true", help="Update po’s in /trunk/po rather than /branches.")
     parser.add_argument('-i', '--input', metavar="File", help="Input pot file path.")
-    #parser.add_argument('--pproc-contexts', action="store_true",
-                        #help="Pre-process po’s to avoid having plenty of fuzzy msgids just because a context was "
-                             #"added/changed!")
     parser.add_argument('-a', '--add', action="store_true",
                         help="Add missing po’s (useful only when one or more languages are given!).")
     parser.add_argument('langs', metavar='ISO_code', nargs='*', help="Restrict processed languages to those.")
@@ -114,14 +67,8 @@ def main():
         FILE_NAME_POT = args.input
     ret = 0
 
-    #if args.pproc_contexts:
+    pot_msgs = utils.I18nMessages(kind='PO', src=FILE_NAME_POT)
-        #_ctxt_proc = pproc_newcontext_po
+    pool_data = []
-        #pot_messages, _a, pot_stats = utils.parse_messages(FILE_NAME_POT)
-    if 0:
-        pass
-    else:
-        _ctxt_proc = lambda a, b, c: 0
-        pot_messages, pot_stats = None, None
 
     if args.langs:
         for lang in args.langs:
@@ -137,37 +84,27 @@ def main():
                 if not os.path.exists(po):
                     shutil.copy(FILE_NAME_POT, po)
             if args.add or os.path.exists(po):
-                t = _ctxt_proc(po, pot_messages, pot_stats)
+                pool_data.append((po, lang, pot_msgs))
-                if t:
-                    ret = t
-                t = process_po(po, lang)
-                if t:
-                    ret = t
     elif args.trunk:
         for po in os.listdir(TRUNK_PO_DIR):
             if po.endswith(".po"):
                 lang = os.path.basename(po)[:-3]
                 po = os.path.join(TRUNK_PO_DIR, po)
-                t = _ctxt_proc(po, pot_messages, pot_stats)
+                pool_data.append((po, lang, pot_msgs))
-                if t:
-                    ret = t
-                t = process_po(po, lang)
-                if t:
-                    ret = t
     else:
         for lang in os.listdir(BRANCHES_DIR):
             po = os.path.join(BRANCHES_DIR, lang, ".".join((lang, "po")))
             if os.path.exists(po):
-                t = _ctxt_proc(po, pot_messages, pot_stats)
+                pool_data.append((po, lang, pot_msgs))
-                if t:
+
-                    ret = t
+    with concurrent.futures.ProcessPoolExecutor() as executor:
-                t = process_po(po, lang)
+        for r in executor.map(process_po, pool_data, timeout=600):
-                if t:
+            if r != 0:
-                    ret = t
+                ret = r
 
     return ret
 
 
 if __name__ == "__main__":
     print("\n\n *** Running {} *** \n".format(__file__))
-    sys.exit(main())
+    sys.exit(main())

release/scripts/modules/bl_i18n_utils/utils.py

@@ -226,7 +226,7 @@ class I18nMessages:
     def merge(self, replace=False, *args):
         pass
 
-    def update(self, ref, use_similar=0.8, keep_old_commented=True):
+    def update(self, ref, use_similar=0.75, keep_old_commented=True):
         """
         Update this I18nMessage with the ref one. Translations from ref are never used. Source comments from ref
         completely replace current ones. If use_similar is not 0.0, it will try to match new messages in ref with an
@@ -493,7 +493,6 @@ class I18nMessages:
                     msgstr_lines.append(line)
                 else:
                     self.parsing_errors.append((line_nr, "regular string outside msgctxt, msgid or msgstr scope"))
-                    print(line)
 
         # If no final empty line, last message is not finalized!
         if reading_msgstr:
@@ -534,7 +533,6 @@ class I18nMessages:
                     else:
                         chunks += ["\n" + _pmsgctxt + "\"" + msg.msgctxt + "\""]
                 if len(msg.msgid_lines) > 1:
-                    print(msg.msgid_lines)
                     chunks += [
                         "\n" + _pmsgid + "\"\"\n" + _p + "\"",
                         ("\"\n" + _p + "\"").join(msg.msgid_lines),