6449: Select language on first start based on user locale r=heinrich5991 a=Robyt3

Closes #2459.

## Checklist

- [X] Tested the change ingame
- [ ] Provided screenshots if it is a visual change
- [X] Tested in combination with possibly related configuration options
- [X] Written a unit test (especially base/) or added coverage to integration test
- [X] Considered possible null pointers and out of bounds array indexing
- [X] Changed no physics that affect existing maps
- [ ] Tested the change with [ASan+UBSan or valgrind's memcheck](https://github.com/ddnet/ddnet/#using-addresssanitizer--undefinedbehavioursanitizer-or-valgrinds-memcheck) (optional)


Co-authored-by: Robert Müller <robytemueller@gmail.com>
This commit is contained in:
bors[bot] 2023-03-21 20:52:04 +00:00 committed by GitHub
commit 8057d591db
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
9 changed files with 343 additions and 94 deletions

View file

@ -1,142 +1,181 @@
##### language indices #####
# Format for each language entry:
# 1. filename (in english)
# 2. name (in native language)
# 3. country code (ISO 3166-1 numeric)
# 4. language tags (RFC 3066) (multiple can be separated with semicolon)
arabic
== ﻲﺑﺮﻋ
== 682
== ar
belarusian
== Беларуская
== 112
== be
bosnian
== Bosanski
== 70
== bs-Latn
brazilian_portuguese
== Português brasileiro
== 76
== pt-BR
bulgarian
== Български
== 100
== bg
catalan
== Català
== 906
== ca
chuvash
== Чăвашла
== 643
== cv
czech
== Česky
== 203
== cs
danish
== Dansk
== 208
== da
dutch
== Nederlands
== 528
== nl
esperanto
== Esperanto
== -1
== eo
finnish
== Suomi
== 246
== fi
french
== Français
== 250
== fr
german
== Deutsch
== 276
== de
greek
== Ελληνικά
== 300
== el
hungarian
== Magyar
== 348
== hu
italian
== Italiano
== 380
== it
japanese
== 日本語
== 392
== ja
korean
== 한국어
== 410
== ko
kyrgyz
== Кыргызча
== 417
== ky
norwegian
== Norsk
== 578
== no;nb
persian
== Persian
== 364
== fa
polish
== Polski
== 616
== pl
portuguese
== Português
== 620
== pt
romanian
== Română
== 642
== ro
russian
== Русский
== 643
== ru
serbian
== Srpski
== 688
== sr-Latn
serbian_cyrillic
== Српски
== 688
== sr-Cyrl
simplified_chinese
== 简体中文
== 156
== zh-Hans;zh-CN;zh-SG
slovak
== Slovensky
== 703
== sk
spanish
== Español
== 724
== es
swedish
== Svenska
== 752
== sv
traditional_chinese
== 繁體中文
== -1
== zh-Hant;zh-HK;zh-MO;zh-TW
turkish
== Türkçe
== 792
== tr
ukrainian
== Українська
== 804
== uk

View file

@ -91,7 +91,7 @@ def check_folder(path):
def languages():
with open("data/languages/index.txt", encoding="utf-8") as f:
index = decode(f, 2)
index = decode(f, 3)
langs = {"data/languages/"+key[0]+".txt" : [key[0]]+elements for key, elements in index.items()}
return langs

View file

@ -26,6 +26,7 @@
#if defined(CONF_FAMILY_UNIX)
#include <csignal>
#include <locale>
#include <sys/stat.h>
#include <sys/time.h>
#include <sys/utsname.h>
@ -51,6 +52,7 @@
#define _task_user_
#include <Carbon/Carbon.h>
#include <CoreFoundation/CoreFoundation.h>
#include <mach-o/dyld.h>
#include <mach/mach_time.h>
@ -4266,6 +4268,72 @@ int os_version_str(char *version, int length)
#endif
}
void os_locale_str(char *locale, size_t length)
{
#if defined(CONF_FAMILY_WINDOWS)
wchar_t wide_buffer[LOCALE_NAME_MAX_LENGTH];
dbg_assert(GetUserDefaultLocaleName(wide_buffer, std::size(wide_buffer)) > 0, "GetUserDefaultLocaleName failure");
// Assume maximum possible length for encoding as UTF-8.
char buffer[UTF8_BYTE_LENGTH * LOCALE_NAME_MAX_LENGTH + 1];
dbg_assert(WideCharToMultiByte(CP_UTF8, 0, wide_buffer, -1, buffer, sizeof(buffer), NULL, NULL) > 0, "WideCharToMultiByte failure");
str_copy(locale, buffer, length);
#elif defined(CONF_PLATFORM_MACOS)
CFLocaleRef locale_ref = CFLocaleCopyCurrent();
CFStringRef locale_identifier_ref = static_cast<CFStringRef>(CFLocaleGetValue(locale_ref, kCFLocaleIdentifier));
// Count number of UTF16 codepoints, +1 for zero-termination.
// Assume maximum possible length for encoding as UTF-8.
CFIndex locale_identifier_size = (UTF8_BYTE_LENGTH * CFStringGetLength(locale_identifier_ref) + 1) * sizeof(char);
char *locale_identifier = (char *)malloc(locale_identifier_size);
dbg_assert(CFStringGetCString(locale_identifier_ref, locale_identifier, locale_identifier_size, kCFStringEncodingUTF8), "CFStringGetCString failure");
str_copy(locale, locale_identifier, length);
free(locale_identifier);
CFRelease(locale_ref);
#else
static const char *ENV_VARIABLES[] = {
"LC_ALL",
"LC_MESSAGES",
"LANG",
};
locale[0] = '\0';
for(const char *env_variable : ENV_VARIABLES)
{
const char *env_value = getenv(env_variable);
if(env_value)
{
str_copy(locale, env_value, length);
break;
}
}
#endif
// Ensure RFC 3066 format:
// - use hyphens instead of underscores
// - truncate locale string after first non-standard letter
for(int i = 0; i < str_length(locale); ++i)
{
if(locale[i] == '_')
{
locale[i] = '-';
}
else if(locale[i] != '-' && !(locale[i] >= 'a' && locale[i] <= 'z') && !(locale[i] >= 'A' && locale[i] <= 'Z') && !(locale[i] >= '0' && locale[i] <= '9'))
{
locale[i] = '\0';
break;
}
}
// Use default if we could not determine the locale,
// i.e. if only the C or POSIX locale is available.
if(locale[0] == '\0' || str_comp(locale, "C") == 0 || str_comp(locale, "POSIX") == 0)
str_copy(locale, "en-US", length);
}
#if defined(CONF_EXCEPTION_HANDLING)
#if defined(CONF_FAMILY_WINDOWS)
static HMODULE exception_handling_module = nullptr;

View file

@ -2143,6 +2143,14 @@ char str_uppercase(char c);
int str_isallnum(const char *str);
unsigned str_quickhash(const char *str);
enum
{
/**
* The maximum bytes necessary to encode one Unicode codepoint with UTF-8.
*/
UTF8_BYTE_LENGTH = 4,
};
int str_utf8_to_skeleton(const char *str, int *buf, int buf_len);
/*
@ -2586,6 +2594,20 @@ int secure_rand_below(int below);
*/
int os_version_str(char *version, int length);
/**
* Returns a string of the preferred locale of the user / operating system.
* The string conforms to [RFC 3066](https://www.ietf.org/rfc/rfc3066.txt)
* and only contains the characters `a`-`z`, `A`-`Z`, `0`-`9` and `-`.
* If the preferred locale could not be determined this function
* falls back to the locale `"en-US"`.
*
* @param locale Buffer to use for the output.
* @param length Length of the output buffer.
*
* @remark The destination buffer will be zero-terminated.
*/
void os_locale_str(char *locale, size_t length);
#if defined(CONF_EXCEPTION_HANDLING)
void init_exception_handler();
void set_exception_handler_log_file(const char *log_file_path);

View file

@ -2026,109 +2026,32 @@ void CMenus::RenderSettingsSound(CUIRect MainView)
}
}
class CLanguage
{
public:
CLanguage() = default;
CLanguage(const char *pName, const char *pFileName, int Code) :
m_Name(pName), m_FileName(pFileName), m_CountryCode(Code) {}
std::string m_Name;
std::string m_FileName;
int m_CountryCode;
bool operator<(const CLanguage &Other) const { return m_Name < Other.m_Name; }
};
void LoadLanguageIndexfile(IStorage *pStorage, IConsole *pConsole, std::vector<CLanguage> &vLanguages)
{
const char *pFilename = "languages/index.txt";
IOHANDLE File = pStorage->OpenFile(pFilename, IOFLAG_READ | IOFLAG_SKIP_BOM, IStorage::TYPE_ALL);
if(!File)
{
char aBuf[128];
str_format(aBuf, sizeof(aBuf), "couldn't open index file '%s'", pFilename);
pConsole->Print(IConsole::OUTPUT_LEVEL_ADDINFO, "localization", aBuf);
return;
}
char aOrigin[128];
char aReplacement[128];
CLineReader LineReader;
LineReader.Init(File);
char *pLine;
while((pLine = LineReader.Get()))
{
if(!str_length(pLine) || pLine[0] == '#') // skip empty lines and comments
continue;
str_copy(aOrigin, pLine);
pLine = LineReader.Get();
if(!pLine)
{
pConsole->Print(IConsole::OUTPUT_LEVEL_ADDINFO, "localization", "unexpected end of index file");
break;
}
if(pLine[0] != '=' || pLine[1] != '=' || pLine[2] != ' ')
{
char aBuf[128];
str_format(aBuf, sizeof(aBuf), "malform replacement for index '%s'", aOrigin);
pConsole->Print(IConsole::OUTPUT_LEVEL_ADDINFO, "localization", aBuf);
(void)LineReader.Get();
continue;
}
str_copy(aReplacement, pLine + 3);
pLine = LineReader.Get();
if(!pLine)
{
pConsole->Print(IConsole::OUTPUT_LEVEL_ADDINFO, "localization", "unexpected end of index file");
break;
}
if(pLine[0] != '=' || pLine[1] != '=' || pLine[2] != ' ')
{
char aBuf[128];
str_format(aBuf, sizeof(aBuf), "malform replacement for index '%s'", aOrigin);
pConsole->Print(IConsole::OUTPUT_LEVEL_ADDINFO, "localization", aBuf);
continue;
}
char aFileName[IO_MAX_PATH_LENGTH];
str_format(aFileName, sizeof(aFileName), "languages/%s.txt", aOrigin);
vLanguages.emplace_back(aReplacement, aFileName, str_toint(pLine + 3));
}
io_close(File);
}
bool CMenus::RenderLanguageSelection(CUIRect MainView)
{
static int s_SelectedLanguage = -1;
static std::vector<CLanguage> s_vLanguages;
static int s_SelectedLanguage = -2; // -2 = unloaded, -1 = unset
static CListBox s_ListBox;
if(s_vLanguages.empty())
if(s_SelectedLanguage == -2)
{
s_vLanguages.emplace_back("English", "", 826);
LoadLanguageIndexfile(Storage(), Console(), s_vLanguages);
std::sort(s_vLanguages.begin(), s_vLanguages.end());
for(size_t i = 0; i < s_vLanguages.size(); i++)
if(str_comp(s_vLanguages[i].m_FileName.c_str(), g_Config.m_ClLanguagefile) == 0)
s_SelectedLanguage = -1;
for(size_t i = 0; i < g_Localization.Languages().size(); i++)
{
if(str_comp(g_Localization.Languages()[i].m_FileName.c_str(), g_Config.m_ClLanguagefile) == 0)
{
s_SelectedLanguage = i;
s_ListBox.ScrollToSelected();
break;
}
}
}
const int OldSelected = s_SelectedLanguage;
s_ListBox.DoStart(24.0f, s_vLanguages.size(), 1, 3, s_SelectedLanguage, &MainView, true);
s_ListBox.DoStart(24.0f, g_Localization.Languages().size(), 1, 3, s_SelectedLanguage, &MainView, true);
for(auto &Language : s_vLanguages)
for(const auto &Language : g_Localization.Languages())
{
const CListboxItem Item = s_ListBox.DoNextItem(&Language.m_Name, s_SelectedLanguage != -1 && !str_comp(s_vLanguages[s_SelectedLanguage].m_Name.c_str(), Language.m_Name.c_str()));
const CListboxItem Item = s_ListBox.DoNextItem(&Language.m_Name, s_SelectedLanguage != -1 && !str_comp(g_Localization.Languages()[s_SelectedLanguage].m_Name.c_str(), Language.m_Name.c_str()));
if(!Item.m_Visible)
continue;
@ -2148,8 +2071,8 @@ bool CMenus::RenderLanguageSelection(CUIRect MainView)
if(OldSelected != s_SelectedLanguage)
{
str_copy(g_Config.m_ClLanguagefile, s_vLanguages[s_SelectedLanguage].m_FileName.c_str());
g_Localization.Load(s_vLanguages[s_SelectedLanguage].m_FileName.c_str(), Storage(), Console());
str_copy(g_Config.m_ClLanguagefile, g_Localization.Languages()[s_SelectedLanguage].m_FileName.c_str());
g_Localization.Load(g_Localization.Languages()[s_SelectedLanguage].m_FileName.c_str(), Storage(), Console());
GameClient()->OnLanguageChange();
}

View file

@ -243,6 +243,9 @@ void CGameClient::OnInit()
}
// set the language
g_Localization.LoadIndexfile(Storage(), Console());
if(g_Config.m_ClShowWelcome)
g_Localization.SelectDefaultLanguage(Console(), g_Config.m_ClLanguagefile, sizeof(g_Config.m_ClLanguagefile));
g_Localization.Load(g_Config.m_ClLanguagefile, Storage(), Console());
// TODO: this should be different

View file

@ -35,9 +35,168 @@ CLocalizationDatabase::CLocalizationDatabase()
m_CurrentVersion = 0;
}
void CLocalizationDatabase::AddString(const char *pOrgStr, const char *pNewStr, const char *pContext)
void CLocalizationDatabase::LoadIndexfile(IStorage *pStorage, IConsole *pConsole)
{
m_vStrings.emplace_back(str_quickhash(pOrgStr), str_quickhash(pContext), m_StringsHeap.StoreString(*pNewStr ? pNewStr : pOrgStr));
m_vLanguages.clear();
const std::vector<std::string> vEnglishLanguageCodes = {"en"};
m_vLanguages.emplace_back("English", "", 826, vEnglishLanguageCodes);
const char *pFilename = "languages/index.txt";
IOHANDLE File = pStorage->OpenFile(pFilename, IOFLAG_READ | IOFLAG_SKIP_BOM, IStorage::TYPE_ALL);
if(!File)
{
char aBuf[64 + IO_MAX_PATH_LENGTH];
str_format(aBuf, sizeof(aBuf), "Couldn't open index file '%s'", pFilename);
pConsole->Print(IConsole::OUTPUT_LEVEL_ADDINFO, "localization", aBuf);
return;
}
CLineReader LineReader;
LineReader.Init(File);
const char *pLine;
while((pLine = LineReader.Get()))
{
if(!str_length(pLine) || pLine[0] == '#') // skip empty lines and comments
continue;
char aEnglishName[128];
str_copy(aEnglishName, pLine);
pLine = LineReader.Get();
if(!pLine)
{
char aBuf[256];
str_format(aBuf, sizeof(aBuf), "Unexpected end of index file after language '%s'", aEnglishName);
pConsole->Print(IConsole::OUTPUT_LEVEL_ADDINFO, "localization", aBuf);
break;
}
if(!str_startswith(pLine, "== "))
{
char aBuf[256];
str_format(aBuf, sizeof(aBuf), "Missing native name for language '%s'", aEnglishName);
pConsole->Print(IConsole::OUTPUT_LEVEL_ADDINFO, "localization", aBuf);
(void)LineReader.Get();
(void)LineReader.Get();
continue;
}
char aNativeName[128];
str_copy(aNativeName, pLine + 3);
pLine = LineReader.Get();
if(!pLine)
{
char aBuf[256];
str_format(aBuf, sizeof(aBuf), "Unexpected end of index file after language '%s'", aEnglishName);
pConsole->Print(IConsole::OUTPUT_LEVEL_ADDINFO, "localization", aBuf);
break;
}
if(!str_startswith(pLine, "== "))
{
char aBuf[256];
str_format(aBuf, sizeof(aBuf), "Missing country code for language '%s'", aEnglishName);
pConsole->Print(IConsole::OUTPUT_LEVEL_ADDINFO, "localization", aBuf);
(void)LineReader.Get();
continue;
}
char aCountryCode[128];
str_copy(aCountryCode, pLine + 3);
pLine = LineReader.Get();
if(!pLine)
{
char aBuf[256];
str_format(aBuf, sizeof(aBuf), "Unexpected end of index file after language '%s'", aEnglishName);
pConsole->Print(IConsole::OUTPUT_LEVEL_ADDINFO, "localization", aBuf);
break;
}
if(!str_startswith(pLine, "== "))
{
char aBuf[256];
str_format(aBuf, sizeof(aBuf), "Missing language codes for language '%s'", aEnglishName);
pConsole->Print(IConsole::OUTPUT_LEVEL_ADDINFO, "localization", aBuf);
continue;
}
const char *pLanguageCodes = pLine + 3;
char aLanguageCode[256];
std::vector<std::string> vLanguageCodes;
while((pLanguageCodes = str_next_token(pLanguageCodes, ";", aLanguageCode, sizeof(aLanguageCode))))
{
if(aLanguageCode[0])
{
vLanguageCodes.emplace_back(aLanguageCode);
}
}
if(vLanguageCodes.empty())
{
char aBuf[256];
str_format(aBuf, sizeof(aBuf), "At least one language code required for language '%s'", aEnglishName);
pConsole->Print(IConsole::OUTPUT_LEVEL_ADDINFO, "localization", aBuf);
continue;
}
char aFileName[IO_MAX_PATH_LENGTH];
str_format(aFileName, sizeof(aFileName), "languages/%s.txt", aEnglishName);
m_vLanguages.emplace_back(aNativeName, aFileName, str_toint(aCountryCode), vLanguageCodes);
}
io_close(File);
std::sort(m_vLanguages.begin(), m_vLanguages.end());
}
void CLocalizationDatabase::SelectDefaultLanguage(IConsole *pConsole, char *pFilename, size_t Length) const
{
char aLocaleStr[128];
os_locale_str(aLocaleStr, sizeof(aLocaleStr));
char aBuf[256];
str_format(aBuf, sizeof(aBuf), "Choosing default language based on user locale '%s'", aLocaleStr);
pConsole->Print(IConsole::OUTPUT_LEVEL_ADDINFO, "localization", aBuf);
while(true)
{
const CLanguage *pPrefixMatch = nullptr;
for(const auto &Language : Languages())
{
for(const auto &LanguageCode : Language.m_vLanguageCodes)
{
if(LanguageCode == aLocaleStr)
{
// Exact match found, use it immediately
str_copy(pFilename, Language.m_FileName.c_str(), Length);
return;
}
else if(LanguageCode.rfind(aLocaleStr, 0) == 0)
{
// Locale is prefix of language code, e.g. locale is "en" and current language is "en-US"
pPrefixMatch = &Language;
}
}
}
// Use prefix match if no exact match was found
if(pPrefixMatch)
{
str_copy(pFilename, pPrefixMatch->m_FileName.c_str(), Length);
return;
}
// Remove last segment of locale string and try again with more generic locale, e.g. "en-US" -> "en"
int i = str_length(aLocaleStr) - 1;
for(; i >= 0; --i)
{
if(aLocaleStr[i] == '-')
{
aLocaleStr[i] = '\0';
break;
}
}
// Stop if no more locale segments are left
if(i == 0)
break;
}
}
bool CLocalizationDatabase::Load(const char *pFilename, IStorage *pStorage, IConsole *pConsole)
@ -119,6 +278,11 @@ bool CLocalizationDatabase::Load(const char *pFilename, IStorage *pStorage, ICon
return true;
}
void CLocalizationDatabase::AddString(const char *pOrgStr, const char *pNewStr, const char *pContext)
{
m_vStrings.emplace_back(str_quickhash(pOrgStr), str_quickhash(pContext), m_StringsHeap.StoreString(*pNewStr ? pNewStr : pOrgStr));
}
const char *CLocalizationDatabase::FindString(unsigned Hash, unsigned ContextHash) const
{
CString String;

View file

@ -4,9 +4,27 @@
#define GAME_LOCALIZATION_H
#include <base/system.h> // GNUC_ATTRIBUTE
#include <engine/shared/memheap.h>
#include <string>
#include <vector>
class CLanguage
{
public:
CLanguage() = default;
CLanguage(const char *pName, const char *pFileName, int Code, const std::vector<std::string> &vLanguageCodes) :
m_Name(pName), m_FileName(pFileName), m_CountryCode(Code), m_vLanguageCodes(vLanguageCodes) {}
std::string m_Name;
std::string m_FileName;
int m_CountryCode;
std::vector<std::string> m_vLanguageCodes;
bool operator<(const CLanguage &Other) const { return m_Name < Other.m_Name; }
};
class CLocalizationDatabase
{
class CString
@ -27,6 +45,7 @@ class CLocalizationDatabase
bool operator==(const CString &Other) const { return m_Hash == Other.m_Hash && m_ContextHash == Other.m_ContextHash; }
};
std::vector<CLanguage> m_vLanguages;
std::vector<CString> m_vStrings;
CHeap m_StringsHeap;
int m_VersionCounter;
@ -35,6 +54,10 @@ class CLocalizationDatabase
public:
CLocalizationDatabase();
void LoadIndexfile(class IStorage *pStorage, class IConsole *pConsole);
const std::vector<CLanguage> &Languages() const { return m_vLanguages; }
void SelectDefaultLanguage(class IConsole *pConsole, char *pFilename, size_t Length) const;
bool Load(const char *pFilename, class IStorage *pStorage, class IConsole *pConsole);
int Version() const { return m_CurrentVersion; }

View file

@ -9,3 +9,10 @@ TEST(Os, VersionStr)
EXPECT_FALSE(os_version_str(aVersion, sizeof(aVersion)));
EXPECT_STRNE(aVersion, "");
}
TEST(Os, LocaleStr)
{
char aLocale[128];
os_locale_str(aLocale, sizeof(aLocale));
EXPECT_STRNE(aLocale, "");
}