Merge #3300

3300: Check translations in github, check formatting strings, fix r=Jupeyy a=def-

## Checklist

- [ ] Tested the change ingame
- [ ] Provided screenshots if it is a visual change
- [ ] Tested in combination with possibly related configuration options
- [ ] Written a unit test if it works standalone, system.c especially
- [ ] Considered possible null pointers and out of bounds array indexing
- [ ] Changed no physics that affect existing maps
- [ ] Tested the change with [ASan+UBSan or valgrind's memcheck](https://github.com/ddnet/ddnet/#using-addresssanitizer--undefinedbehavioursanitizer-or-valgrinds-memcheck) (optional)


Co-authored-by: def <dennis@felsin9.de>

.github/workflows/style.yml

@@ -23,3 +23,4 @@ jobs:
         clang-format -version
         scripts/fix_style.py --dry-run
         scripts/check_header_guards.py
+        scripts/languages/update_all.py

data/languages/dutch.txt

@@ -1045,7 +1045,7 @@ Save the best demo of each race
 == Sla de beste demo op van elke race
 
 Default length: %d
-== Standaart lengte
+== Standaart lengte: %d
 
 Enable replays
 == Replays aan zetten

data/languages/persian.txt

@@ -79,7 +79,7 @@ Delete demo
 Demo details
 == ﻮﻣﺩ ﺕﺎﯿﺋﺰﺟ
 Demofile: %s
-== s% :ﻮﻣﺩ ﻞﯾﺎﻓ
+== %s :ﻮﻣﺩ ﻞﯾﺎﻓ
 Demos
 == ﺎﻫ ﻮﻣﺩ
 Disconnect

data/languages/polish.txt

@@ -28,7 +28,7 @@
 == Pozostało sekund: %i
 
 %s wins!
-== Wygrał %d!
+== Wygrał %s!
 
 -Page %d-
 == -Strona %d-

data/languages/simplified_chinese.txt

@@ -1233,7 +1233,7 @@ Existing Player
 == 玩家已存在
 
 Your nickname '%s' is already used (%d points). Do you still want to use it?
-== 你使用的昵称在DDNet中有%d分的记录，这可能代表改昵称已经被其他人使用过。确认要使用这个名字吗？
+== 你使用的昵称"%s"在DDNet中有%d分的记录，这可能代表该昵称已经被其他人使用过。确认要使用这个名字吗？
 
 Checking for existing player with your name
 == 正在查找该昵称的DDNet记录

data/languages/traditional_chinese.txt

@@ -1227,7 +1227,7 @@ Existing Player
 == 玩家已存在
 
 Your nickname '%s' is already used (%d points). Do you still want to use it?
-== 你使用的暱稱在DDNet中有%d分的記錄，這可能代表改暱稱已經被其他人使用過。確認要使用這個名字嗎？
+== 你使用的暱稱"%s"在DDNet中有%d分的記錄，這可能代表這個暱稱已經被其他人使用過。確認要使用這個名字嗎？
 
 Checking for existing player with your name
 == 正在查詢該暱稱的DDNet記錄

data/languages/turkish.txt

@@ -127,7 +127,7 @@ Demo details
 == Demo ayrıntıları
 
 Demofile: %s
-== Demo dosyası
+== Demo dosyası: %s
 
 Demos
 == Demolar

scripts/languages/copy_fix.py

@@ -3,28 +3,7 @@ import twlang
 import os
 import sys
 
-os.chdir(os.path.dirname(__file__) + "/../..")
+def copy_fix(infile, delete_unused, append_missing, delete_empty):
-
-if len(sys.argv) < 3:
-    print("usage: python copy_fix.py <infile> <outfile> [--delete-unused] [--append-missing] [--delete-empty]")
-    sys.exit()
-infile = sys.argv[1]
-outfile = sys.argv[2]
-args = sys.argv[3:]
-delete_unused = False
-append_missing = False
-delete_empty = False
-for arg in args:
-    if arg == "--delete-unused":
-        delete_unused = True
-    elif arg == "--append-missing":
-        append_missing = True
-    elif arg == "--delete-empty":
-        delete_empty = True
-    else:
-        print("No such argument '"+arg+"'.")
-        sys.exit()
-
     content = open(infile).readlines()
     trans = twlang.translations(infile)
     if delete_unused or append_missing:
@@ -52,6 +31,32 @@ if append_missing:
                     content.append("["+local[miss][1]+"]\n")
                 content.append(local[miss][0]+"\n== \n\n")
             content[-1] = content[-1][:-1]
+    return "".join(content)
+
+if __name__ == '__main__':
+    os.chdir(os.path.dirname(__file__) + "/../..")
+
+    if len(sys.argv) < 3:
+        print("usage: python copy_fix.py <infile> <outfile> [--delete-unused] [--append-missing] [--delete-empty]")
+        sys.exit()
+    infile = sys.argv[1]
+    outfile = sys.argv[2]
+    args = sys.argv[3:]
+    delete_unused = False
+    append_missing = False
+    delete_empty = False
+    for arg in args:
+        if arg == "--delete-unused":
+            delete_unused = True
+        elif arg == "--append-missing":
+            append_missing = True
+        elif arg == "--delete-empty":
+            delete_empty = True
+        else:
+            print("No such argument '"+arg+"'.")
+            sys.exit()
+
+    content = copy_fix(infile, delete_unused, append_missing, delete_empty)
 
     open(outfile, "w").write("".join(content))
     print("Successfully created '" + outfile + "'.")

scripts/languages/readme.txt

@@ -38,7 +38,7 @@ $ ./find_unchanged.py ../spanish.txt
 
 To update all languages:
 
-$ for i in data/languages/*.txt; do [ "${i:t}" != "license.txt" ] && [ "${i:t}" != "index.txt" ] && scripts/languages/copy_fix.py $i $i.$$.tmp --delete-unused --append-missing && mv $i.$$.tmp $i; done
+$ ./update_all.py
 
 To get a statistic of how complete the translation is:
 

scripts/languages/twlang.py

@@ -8,6 +8,21 @@ class LanguageDecodeError(Exception):
         super(LanguageDecodeError, self).__init__(error)
 
 
+# Taken from https://stackoverflow.com/questions/30011379/how-can-i-parse-a-c-format-string-in-python
+cfmt = '''\
+(                                  # start of capture group 1
+%                                  # literal "%"
+(?:                                # first option
+(?:[-+0 #]{0,5})                   # optional flags
+(?:\d+|\*)?                        # width
+(?:\.(?:\d+|\*))?                  # precision
+(?:h|l|ll|w|I|I32|I64)?            # size
+[cCdiouxXeEfgGaAnpsSZ]             # type
+) |                                # OR
+%%)                                # literal "%%"
+'''
+
+
 def decode(fileobj, elements_per_key):
     data = {}
     current_context = ""
@@ -30,7 +45,11 @@ def decode(fileobj, elements_per_key):
             if len(data[current_key]) >= 1+elements_per_key:
                 raise LanguageDecodeError("Wrong number of elements per key", fileobj.name, index)
             if current_key:
-                data[current_key].extend([line[3:]])
+                original = current_key[0]
+                translation = line[3:]
+                if translation and [m.group(1) for m in re.finditer(cfmt, original, flags=re.X)] != [m.group(1) for m in re.finditer(cfmt, translation, flags=re.X)]:
+                    raise LanguageDecodeError("Non-matching formatting string", fileobj.name, index)
+                data[current_key].extend([translation])
             else:
                 raise LanguageDecodeError("Element before key given", fileobj.name, index)
         else:
@@ -68,7 +87,7 @@ def check_folder(path):
 
 def languages():
     index = decode(open("data/languages/index.txt"), 2)
-    langs = {"data/languages/"+key+".txt" : [key]+elements for key, elements in index.items()}
+    langs = {"data/languages/"+key[0]+".txt" : [key[0]]+elements for key, elements in index.items()}
     return langs
 
 

scripts/languages/update_all.py

@@ -0,0 +1,10 @@
+#!/usr/bin/env python3
+import os
+from copy_fix import copy_fix
+import twlang
+
+os.chdir(os.path.dirname(__file__) + "/../..")
+
+for lang in twlang.languages():
+    content = copy_fix(lang, delete_unused=True, append_missing=True, delete_empty=False)
+    open(lang, "w").write(content)