Add RFC 3066 language tags to language index

Add RFC 3066 language tags to language index in order to identify the language to select when first starting the game based on the user's locale.

Multiple language tags can be specified per entry separated with a semicolon, as some languages have different tags that do not share a unique prefix (e.g., Simplified and Traditional Chinese).
This commit is contained in:
Robert Müller 2023-03-20 21:45:43 +01:00
parent 481698c7d0
commit cddbc78592
4 changed files with 106 additions and 24 deletions

View file

@ -1,142 +1,181 @@
##### language indices #####
# Format for each language entry:
# 1. filename (in english)
# 2. name (in native language)
# 3. country code (ISO 3166-1 numeric)
# 4. language tags (RFC 3066) (multiple can be separated with semicolon)
arabic
== ﻲﺑﺮﻋ
== 682
== ar
belarusian
== Беларуская
== 112
== be
bosnian
== Bosanski
== 70
== bs-Latn
brazilian_portuguese
== Português brasileiro
== 76
== pt-BR
bulgarian
== Български
== 100
== bg
catalan
== Català
== 906
== ca
chuvash
== Чăвашла
== 643
== cv
czech
== Česky
== 203
== cs
danish
== Dansk
== 208
== da
dutch
== Nederlands
== 528
== nl
esperanto
== Esperanto
== -1
== eo
finnish
== Suomi
== 246
== fi
french
== Français
== 250
== fr
german
== Deutsch
== 276
== de
greek
== Ελληνικά
== 300
== el
hungarian
== Magyar
== 348
== hu
italian
== Italiano
== 380
== it
japanese
== 日本語
== 392
== ja
korean
== 한국어
== 410
== ko
kyrgyz
== Кыргызча
== 417
== ky
norwegian
== Norsk
== 578
== no;nb
persian
== Persian
== 364
== fa
polish
== Polski
== 616
== pl
portuguese
== Português
== 620
== pt
romanian
== Română
== 642
== ro
russian
== Русский
== 643
== ru
serbian
== Srpski
== 688
== sr-Latn
serbian_cyrillic
== Српски
== 688
== sr-Cyrl
simplified_chinese
== 简体中文
== 156
== zh-Hans;zh-CN;zh-SG
slovak
== Slovensky
== 703
== sk
spanish
== Español
== 724
== es
swedish
== Svenska
== 752
== sv
traditional_chinese
== 繁體中文
== -1
== zh-Hant;zh-HK;zh-MO;zh-TW
turkish
== Türkçe
== 792
== tr
ukrainian
== Українська
== 804
== uk

View file

@ -91,7 +91,7 @@ def check_folder(path):
def languages():
with open("data/languages/index.txt", encoding="utf-8") as f:
index = decode(f, 2)
index = decode(f, 3)
langs = {"data/languages/"+key[0]+".txt" : [key[0]]+elements for key, elements in index.items()}
return langs

View file

@ -38,65 +38,107 @@ CLocalizationDatabase::CLocalizationDatabase()
void CLocalizationDatabase::LoadIndexfile(IStorage *pStorage, IConsole *pConsole)
{
m_vLanguages.clear();
m_vLanguages.emplace_back("English", "", 826);
const std::vector<std::string> vEnglishLanguageCodes = {"en"};
m_vLanguages.emplace_back("English", "", 826, vEnglishLanguageCodes);
const char *pFilename = "languages/index.txt";
IOHANDLE File = pStorage->OpenFile(pFilename, IOFLAG_READ | IOFLAG_SKIP_BOM, IStorage::TYPE_ALL);
if(!File)
{
char aBuf[128];
str_format(aBuf, sizeof(aBuf), "couldn't open index file '%s'", pFilename);
char aBuf[64 + IO_MAX_PATH_LENGTH];
str_format(aBuf, sizeof(aBuf), "Couldn't open index file '%s'", pFilename);
pConsole->Print(IConsole::OUTPUT_LEVEL_ADDINFO, "localization", aBuf);
return;
}
char aOrigin[128];
char aReplacement[128];
CLineReader LineReader;
LineReader.Init(File);
char *pLine;
const char *pLine;
while((pLine = LineReader.Get()))
{
if(!str_length(pLine) || pLine[0] == '#') // skip empty lines and comments
continue;
str_copy(aOrigin, pLine);
char aEnglishName[128];
str_copy(aEnglishName, pLine);
pLine = LineReader.Get();
if(!pLine)
{
pConsole->Print(IConsole::OUTPUT_LEVEL_ADDINFO, "localization", "unexpected end of index file");
char aBuf[256];
str_format(aBuf, sizeof(aBuf), "Unexpected end of index file after language '%s'", aEnglishName);
pConsole->Print(IConsole::OUTPUT_LEVEL_ADDINFO, "localization", aBuf);
break;
}
if(pLine[0] != '=' || pLine[1] != '=' || pLine[2] != ' ')
if(!str_startswith(pLine, "== "))
{
char aBuf[128];
str_format(aBuf, sizeof(aBuf), "malform replacement for index '%s'", aOrigin);
char aBuf[256];
str_format(aBuf, sizeof(aBuf), "Missing native name for language '%s'", aEnglishName);
pConsole->Print(IConsole::OUTPUT_LEVEL_ADDINFO, "localization", aBuf);
(void)LineReader.Get();
(void)LineReader.Get();
continue;
}
char aNativeName[128];
str_copy(aNativeName, pLine + 3);
pLine = LineReader.Get();
if(!pLine)
{
char aBuf[256];
str_format(aBuf, sizeof(aBuf), "Unexpected end of index file after language '%s'", aEnglishName);
pConsole->Print(IConsole::OUTPUT_LEVEL_ADDINFO, "localization", aBuf);
break;
}
if(!str_startswith(pLine, "== "))
{
char aBuf[256];
str_format(aBuf, sizeof(aBuf), "Missing country code for language '%s'", aEnglishName);
pConsole->Print(IConsole::OUTPUT_LEVEL_ADDINFO, "localization", aBuf);
(void)LineReader.Get();
continue;
}
str_copy(aReplacement, pLine + 3);
char aCountryCode[128];
str_copy(aCountryCode, pLine + 3);
pLine = LineReader.Get();
if(!pLine)
{
pConsole->Print(IConsole::OUTPUT_LEVEL_ADDINFO, "localization", "unexpected end of index file");
char aBuf[256];
str_format(aBuf, sizeof(aBuf), "Unexpected end of index file after language '%s'", aEnglishName);
pConsole->Print(IConsole::OUTPUT_LEVEL_ADDINFO, "localization", aBuf);
break;
}
if(pLine[0] != '=' || pLine[1] != '=' || pLine[2] != ' ')
if(!str_startswith(pLine, "== "))
{
char aBuf[128];
str_format(aBuf, sizeof(aBuf), "malform replacement for index '%s'", aOrigin);
char aBuf[256];
str_format(aBuf, sizeof(aBuf), "Missing language codes for language '%s'", aEnglishName);
pConsole->Print(IConsole::OUTPUT_LEVEL_ADDINFO, "localization", aBuf);
continue;
}
const char *pLanguageCodes = pLine + 3;
char aLanguageCode[256];
std::vector<std::string> vLanguageCodes;
while((pLanguageCodes = str_next_token(pLanguageCodes, ";", aLanguageCode, sizeof(aLanguageCode))))
{
if(aLanguageCode[0])
{
vLanguageCodes.emplace_back(aLanguageCode);
}
}
if(vLanguageCodes.empty())
{
char aBuf[256];
str_format(aBuf, sizeof(aBuf), "At least one language code required for language '%s'", aEnglishName);
pConsole->Print(IConsole::OUTPUT_LEVEL_ADDINFO, "localization", aBuf);
continue;
}
char aFileName[IO_MAX_PATH_LENGTH];
str_format(aFileName, sizeof(aFileName), "languages/%s.txt", aOrigin);
m_vLanguages.emplace_back(aReplacement, aFileName, str_toint(pLine + 3));
str_format(aFileName, sizeof(aFileName), "languages/%s.txt", aEnglishName);
m_vLanguages.emplace_back(aNativeName, aFileName, str_toint(aCountryCode), vLanguageCodes);
}
io_close(File);

View file

@ -14,12 +14,13 @@ class CLanguage
{
public:
CLanguage() = default;
CLanguage(const char *pName, const char *pFileName, int Code) :
m_Name(pName), m_FileName(pFileName), m_CountryCode(Code) {}
CLanguage(const char *pName, const char *pFileName, int Code, const std::vector<std::string> &vLanguageCodes) :
m_Name(pName), m_FileName(pFileName), m_CountryCode(Code), m_vLanguageCodes(vLanguageCodes) {}
std::string m_Name;
std::string m_FileName;
int m_CountryCode;
std::vector<std::string> m_vLanguageCodes;
bool operator<(const CLanguage &Other) const { return m_Name < Other.m_Name; }
};