Merge #6555

6555: Add convenience functions to convert between UTF-8 encoded strings and wide character strings on Windows r=def- a=Robyt3



## Checklist

- [X] Tested the change ingame
- [ ] Provided screenshots if it is a visual change
- [ ] Tested in combination with possibly related configuration options
- [X] Written a unit test (especially base/) or added coverage to integration test
- [X] Considered possible null pointers and out of bounds array indexing
- [X] Changed no physics that affect existing maps
- [ ] Tested the change with [ASan+UBSan or valgrind's memcheck](https://github.com/ddnet/ddnet/#using-addresssanitizer--undefinedbehavioursanitizer-or-valgrinds-memcheck) (optional)


Co-authored-by: Robert Müller <robytemueller@gmail.com>

src/base/log.cpp

@@ -328,12 +328,7 @@ public:
 	}
 	void Log(const CLogMessage *pMessage) override
 	{
-		int WLen = MultiByteToWideChar(CP_UTF8, 0, pMessage->m_aLine, pMessage->m_LineLength, NULL, 0);
-		dbg_assert(WLen > 0, "MultiByteToWideChar failure");
-		WCHAR *pWide = (WCHAR *)malloc((WLen + 2) * sizeof(*pWide));
-		dbg_assert(MultiByteToWideChar(CP_UTF8, 0, pMessage->m_aLine, pMessage->m_LineLength, pWide, WLen) == WLen, "MultiByteToWideChar failure");
-		pWide[WLen++] = '\r';
-		pWide[WLen++] = '\n';
+		const std::wstring WideMessage = windows_utf8_to_wide(pMessage->m_aLine) + L"\r\n";
 
 		int Color = m_BackgroundColor;
 		if(pMessage->m_HaveColor)
@@ -351,10 +346,9 @@ public:
 		if(!m_Finished)
 		{
 			SetConsoleTextAttribute(m_pConsole, Color);
-			WriteConsoleW(m_pConsole, pWide, WLen, NULL, NULL);
+			WriteConsoleW(m_pConsole, WideMessage.c_str(), WideMessage.length(), NULL, NULL);
 		}
 		m_OutputLock.unlock();
-		free(pWide);
 	}
 	void GlobalFinish() override
 	{
@@ -413,12 +407,8 @@ class CLoggerWindowsDebugger : public ILogger
 public:
 	void Log(const CLogMessage *pMessage) override
 	{
-		int WLen = MultiByteToWideChar(CP_UTF8, 0, pMessage->m_aLine, -1, NULL, 0);
-		dbg_assert(WLen > 0, "MultiByteToWideChar failure");
-		WCHAR *pWide = (WCHAR *)malloc(WLen * sizeof(*pWide));
-		dbg_assert(MultiByteToWideChar(CP_UTF8, 0, pMessage->m_aLine, -1, pWide, WLen) == WLen, "MultiByteToWideChar failure");
-		OutputDebugStringW(pWide);
-		free(pWide);
+		const std::wstring WideMessage = windows_utf8_to_wide(pMessage->m_aLine);
+		OutputDebugStringW(WideMessage.c_str());
 	}
 };
 std::unique_ptr<ILogger> log_logger_windows_debugger()

src/base/system.cpp

@@ -100,18 +100,13 @@ IOHANDLE io_current_exe()
 {
 	// From https://stackoverflow.com/a/1024937.
 #if defined(CONF_FAMILY_WINDOWS)
-	wchar_t wpath[IO_MAX_PATH_LENGTH];
-	char path[IO_MAX_PATH_LENGTH];
-	if(GetModuleFileNameW(NULL, wpath, std::size(wpath)) == 0 || GetLastError() != ERROR_SUCCESS)
+	wchar_t wide_path[IO_MAX_PATH_LENGTH];
+	if(GetModuleFileNameW(NULL, wide_path, std::size(wide_path)) == 0 || GetLastError() != ERROR_SUCCESS)
 	{
 		return 0;
 	}
-	if(WideCharToMultiByte(CP_UTF8, 0, wpath, -1, path, sizeof(path), NULL, NULL) == 0)
-	{
-		dbg_assert(GetLastError() == ERROR_INSUFFICIENT_BUFFER, "WideCharToMultiByte failure");
-		return 0;
-	}
-	return io_open(path, IOFLAG_READ);
+	const std::string path = windows_wide_to_utf8(wide_path);
+	return io_open(path.c_str(), IOFLAG_READ);
 #elif defined(CONF_PLATFORM_MACOS)
 	char path[IO_MAX_PATH_LENGTH];
 	uint32_t path_size = sizeof(path);
@@ -247,14 +242,13 @@ IOHANDLE io_open_impl(const char *filename, int flags)
 {
 	dbg_assert(flags == (IOFLAG_READ | IOFLAG_SKIP_BOM) || flags == IOFLAG_READ || flags == IOFLAG_WRITE || flags == IOFLAG_APPEND, "flags must be read, read+skipbom, write or append");
 #if defined(CONF_FAMILY_WINDOWS)
-	WCHAR wBuffer[IO_MAX_PATH_LENGTH];
-	dbg_assert(MultiByteToWideChar(CP_UTF8, 0, filename, -1, wBuffer, std::size(wBuffer)) > 0, "MultiByteToWideChar failure");
+	const std::wstring wide_filename = windows_utf8_to_wide(filename);
 	if((flags & IOFLAG_READ) != 0)
-		return (IOHANDLE)_wfsopen(wBuffer, L"rb", _SH_DENYNO);
+		return (IOHANDLE)_wfsopen(wide_filename.c_str(), L"rb", _SH_DENYNO);
 	if(flags == IOFLAG_WRITE)
-		return (IOHANDLE)_wfsopen(wBuffer, L"wb", _SH_DENYNO);
+		return (IOHANDLE)_wfsopen(wide_filename.c_str(), L"wb", _SH_DENYNO);
 	if(flags == IOFLAG_APPEND)
-		return (IOHANDLE)_wfsopen(wBuffer, L"ab", _SH_DENYNO);
+		return (IOHANDLE)_wfsopen(wide_filename.c_str(), L"ab", _SH_DENYNO);
 	return 0x0;
 #else
 	if((flags & IOFLAG_READ) != 0)
@@ -1461,18 +1455,14 @@ static int priv_net_close_all_sockets(NETSOCKET sock)
 }
 
 #if defined(CONF_FAMILY_WINDOWS)
-char *windows_format_system_message(unsigned long error)
+std::string windows_format_system_message(unsigned long error)
 {
 	WCHAR *wide_message;
 	const DWORD flags = FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS | FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_MAX_WIDTH_MASK;
 	if(FormatMessageW(flags, NULL, error, 0, (LPWSTR)&wide_message, 0, NULL) == 0)
-	{
-		return nullptr;
-	}
-	int len = WideCharToMultiByte(CP_UTF8, 0, wide_message, -1, NULL, 0, NULL, NULL);
-	dbg_assert(len > 0, "WideCharToMultiByte failure");
-	char *message = (char *)malloc(len * sizeof(*message));
-	dbg_assert(WideCharToMultiByte(CP_UTF8, 0, wide_message, -1, message, len, NULL, NULL) == len, "WideCharToMultiByte failure");
+		return "unknown error";
+
+	const std::string message = windows_wide_to_utf8(wide_message);
 	LocalFree(wide_message);
 	return message;
 }
@@ -1488,9 +1478,8 @@ static int priv_net_create_socket(int domain, int type, struct sockaddr *addr, i
 	{
 #if defined(CONF_FAMILY_WINDOWS)
 		int error = WSAGetLastError();
-		char *message = windows_format_system_message(error);
-		dbg_msg("net", "failed to create socket with domain %d and type %d (%d '%s')", domain, type, error, message == nullptr ? "unknown error" : message);
-		free(message);
+		const std::string message = windows_format_system_message(error);
+		dbg_msg("net", "failed to create socket with domain %d and type %d (%d '%s')", domain, type, error, message.c_str());
 #else
 		dbg_msg("net", "failed to create socket with domain %d and type %d (%d '%s')", domain, type, errno, strerror(errno));
 #endif
@@ -1524,9 +1513,8 @@ static int priv_net_create_socket(int domain, int type, struct sockaddr *addr, i
 	{
 #if defined(CONF_FAMILY_WINDOWS)
 		int error = WSAGetLastError();
-		char *message = windows_format_system_message(error);
-		dbg_msg("net", "failed to bind socket with domain %d and type %d (%d '%s')", domain, type, error, message == nullptr ? "unknown error" : message);
-		free(message);
+		const std::string message = windows_format_system_message(error);
+		dbg_msg("net", "failed to bind socket with domain %d and type %d (%d '%s')", domain, type, error, message.c_str());
 #else
 		dbg_msg("net", "failed to bind socket with domain %d and type %d (%d '%s')", domain, type, errno, strerror(errno));
 #endif
@@ -2119,27 +2107,20 @@ static inline time_t filetime_to_unixtime(LPFILETIME filetime)
 void fs_listdir(const char *dir, FS_LISTDIR_CALLBACK cb, int type, void *user)
 {
 #if defined(CONF_FAMILY_WINDOWS)
-	WIN32_FIND_DATAW finddata;
-	HANDLE handle;
 	char buffer[IO_MAX_PATH_LENGTH];
-	WCHAR wBuffer[IO_MAX_PATH_LENGTH];
-
 	str_format(buffer, sizeof(buffer), "%s/*", dir);
-	dbg_assert(MultiByteToWideChar(CP_UTF8, 0, buffer, -1, wBuffer, std::size(wBuffer)) > 0, "MultiByteToWideChar failure");
+	const std::wstring wide_buffer = windows_utf8_to_wide(buffer);
 
-	handle = FindFirstFileW(wBuffer, &finddata);
+	WIN32_FIND_DATAW finddata;
+	HANDLE handle = FindFirstFileW(wide_buffer.c_str(), &finddata);
 	if(handle == INVALID_HANDLE_VALUE)
 		return;
 
 	/* add all the entries */
 	do
 	{
-		if(WideCharToMultiByte(CP_UTF8, 0, finddata.cFileName, -1, buffer, sizeof(buffer), NULL, NULL) == 0)
-		{
-			dbg_assert(GetLastError() == ERROR_INSUFFICIENT_BUFFER, "WideCharToMultiByte failure");
-			continue;
-		}
-		if(cb(buffer, (finddata.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) != 0, type, user))
+		const std::string current_entry = windows_wide_to_utf8(finddata.cFileName);
+		if(cb(current_entry.c_str(), (finddata.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) != 0, type, user))
 			break;
 	} while(FindNextFileW(handle, &finddata));
 
@@ -2171,29 +2152,22 @@ void fs_listdir(const char *dir, FS_LISTDIR_CALLBACK cb, int type, void *user)
 void fs_listdir_fileinfo(const char *dir, FS_LISTDIR_CALLBACK_FILEINFO cb, int type, void *user)
 {
 #if defined(CONF_FAMILY_WINDOWS)
-	WIN32_FIND_DATAW finddata;
-	HANDLE handle;
 	char buffer[IO_MAX_PATH_LENGTH];
-	WCHAR wBuffer[IO_MAX_PATH_LENGTH];
-
 	str_format(buffer, sizeof(buffer), "%s/*", dir);
-	dbg_assert(MultiByteToWideChar(CP_UTF8, 0, buffer, -1, wBuffer, std::size(wBuffer)) > 0, "MultiByteToWideChar failure");
+	const std::wstring wide_buffer = windows_utf8_to_wide(buffer);
 
-	handle = FindFirstFileW(wBuffer, &finddata);
+	WIN32_FIND_DATAW finddata;
+	HANDLE handle = FindFirstFileW(wide_buffer.c_str(), &finddata);
 	if(handle == INVALID_HANDLE_VALUE)
 		return;
 
 	/* add all the entries */
 	do
 	{
-		if(WideCharToMultiByte(CP_UTF8, 0, finddata.cFileName, -1, buffer, sizeof(buffer), NULL, NULL) == 0)
-		{
-			dbg_assert(GetLastError() == ERROR_INSUFFICIENT_BUFFER, "WideCharToMultiByte failure");
-			continue;
-		}
+		const std::string current_entry = windows_wide_to_utf8(finddata.cFileName);
 
 		CFsFileInfo info;
-		info.m_pName = buffer;
+		info.m_pName = current_entry.c_str();
 		info.m_TimeCreated = filetime_to_unixtime(&finddata.ftCreationTime);
 		info.m_TimeModified = filetime_to_unixtime(&finddata.ftLastWriteTime);
 
@@ -2238,16 +2212,11 @@ void fs_listdir_fileinfo(const char *dir, FS_LISTDIR_CALLBACK_FILEINFO cb, int t
 int fs_storage_path(const char *appname, char *path, int max)
 {
 #if defined(CONF_FAMILY_WINDOWS)
-	WCHAR *home = _wgetenv(L"APPDATA");
-	if(!home)
+	WCHAR *wide_home = _wgetenv(L"APPDATA");
+	if(!wide_home)
 		return -1;
-	char buffer[IO_MAX_PATH_LENGTH];
-	if(WideCharToMultiByte(CP_UTF8, 0, home, -1, buffer, sizeof(buffer), NULL, NULL) == 0)
-	{
-		dbg_assert(GetLastError() == ERROR_INSUFFICIENT_BUFFER, "WideCharToMultiByte failure");
-		return -1;
-	}
-	str_format(path, max, "%s/%s", buffer, appname);
+	const std::string home = windows_wide_to_utf8(wide_home);
+	str_format(path, max, "%s/%s", home.c_str(), appname);
 	return 0;
 #elif defined(CONF_PLATFORM_ANDROID)
 	// just use the data directory
@@ -2304,9 +2273,8 @@ int fs_makedir_rec_for(const char *path)
 int fs_makedir(const char *path)
 {
 #if defined(CONF_FAMILY_WINDOWS)
-	WCHAR wBuffer[IO_MAX_PATH_LENGTH];
-	dbg_assert(MultiByteToWideChar(CP_UTF8, 0, path, -1, wBuffer, std::size(wBuffer)) > 0, "MultiByteToWideChar failure");
-	if(CreateDirectoryW(wBuffer, NULL) != 0)
+	const std::wstring wide_path = windows_utf8_to_wide(path);
+	if(CreateDirectoryW(wide_path.c_str(), NULL) != 0)
 		return 0;
 	if(GetLastError() == ERROR_ALREADY_EXISTS)
 		return 0;
@@ -2328,9 +2296,8 @@ int fs_makedir(const char *path)
 int fs_removedir(const char *path)
 {
 #if defined(CONF_FAMILY_WINDOWS)
-	WCHAR wPath[IO_MAX_PATH_LENGTH];
-	dbg_assert(MultiByteToWideChar(CP_UTF8, 0, path, -1, wPath, std::size(wPath)) > 0, "MultiByteToWideChar failure");
-	if(RemoveDirectoryW(wPath) != 0)
+	const std::wstring wide_path = windows_utf8_to_wide(path);
+	if(RemoveDirectoryW(wide_path.c_str()) != 0)
 		return 0;
 	return -1;
 #else
@@ -2343,9 +2310,8 @@ int fs_removedir(const char *path)
 int fs_is_file(const char *path)
 {
 #if defined(CONF_FAMILY_WINDOWS)
-	WCHAR wPath[IO_MAX_PATH_LENGTH];
-	dbg_assert(MultiByteToWideChar(CP_UTF8, 0, path, -1, wPath, std::size(wPath)) > 0, "MultiByteToWideChar failure");
-	DWORD attributes = GetFileAttributesW(wPath);
+	const std::wstring wide_path = windows_utf8_to_wide(path);
+	DWORD attributes = GetFileAttributesW(wide_path.c_str());
 	return attributes != INVALID_FILE_ATTRIBUTES && !(attributes & FILE_ATTRIBUTE_DIRECTORY) ? 1 : 0;
 #else
 	struct stat sb;
@@ -2358,9 +2324,8 @@ int fs_is_file(const char *path)
 int fs_is_dir(const char *path)
 {
 #if defined(CONF_FAMILY_WINDOWS)
-	WCHAR wPath[IO_MAX_PATH_LENGTH];
-	dbg_assert(MultiByteToWideChar(CP_UTF8, 0, path, -1, wPath, std::size(wPath)) > 0, "MultiByteToWideChar failure");
-	DWORD attributes = GetFileAttributesW(wPath);
+	const std::wstring wide_path = windows_utf8_to_wide(path);
+	DWORD attributes = GetFileAttributesW(wide_path.c_str());
 	return attributes != INVALID_FILE_ATTRIBUTES && (attributes & FILE_ATTRIBUTE_DIRECTORY) ? 1 : 0;
 #else
 	struct stat sb;
@@ -2373,9 +2338,8 @@ int fs_is_dir(const char *path)
 int fs_is_relative_path(const char *path)
 {
 #if defined(CONF_FAMILY_WINDOWS)
-	WCHAR wPath[IO_MAX_PATH_LENGTH];
-	dbg_assert(MultiByteToWideChar(CP_UTF8, 0, path, -1, wPath, std::size(wPath)) > 0, "MultiByteToWideChar failure");
-	return PathIsRelativeW(wPath) ? 1 : 0;
+	const std::wstring wide_path = windows_utf8_to_wide(path);
+	return PathIsRelativeW(wide_path.c_str()) ? 1 : 0;
 #else
 	return path[0] == '/' ? 0 : 1; // yes, it's that simple
 #endif
@@ -2386,14 +2350,10 @@ int fs_chdir(const char *path)
 	if(fs_is_dir(path))
 	{
 #if defined(CONF_FAMILY_WINDOWS)
-		WCHAR wBuffer[IO_MAX_PATH_LENGTH];
-		dbg_assert(MultiByteToWideChar(CP_UTF8, 0, path, -1, wBuffer, std::size(wBuffer)) > 0, "MultiByteToWideChar failure");
-		return SetCurrentDirectoryW(wBuffer) != 0 ? 0 : 1;
+		const std::wstring wide_path = windows_utf8_to_wide(path);
+		return SetCurrentDirectoryW(wide_path.c_str()) != 0 ? 0 : 1;
 #else
-		if(chdir(path))
-			return 1;
-		else
-			return 0;
+		return chdir(path) ? 1 : 0;
 #endif
 	}
 	else
@@ -2403,15 +2363,18 @@ int fs_chdir(const char *path)
 char *fs_getcwd(char *buffer, int buffer_size)
 {
 #if defined(CONF_FAMILY_WINDOWS)
-	WCHAR wBuffer[IO_MAX_PATH_LENGTH];
-	DWORD result = GetCurrentDirectoryW(std::size(wBuffer), wBuffer);
-	if(result == 0 || result > std::size(wBuffer))
-		return nullptr;
-	if(WideCharToMultiByte(CP_UTF8, 0, wBuffer, -1, buffer, buffer_size, NULL, NULL) == 0)
+	const DWORD size_needed = GetCurrentDirectoryW(0, nullptr);
+	std::wstring wide_current_dir(size_needed, L'0');
+	DWORD result = GetCurrentDirectoryW(size_needed, &wide_current_dir[0]);
+	if(result == 0)
 	{
-		dbg_assert(GetLastError() == ERROR_INSUFFICIENT_BUFFER, "WideCharToMultiByte failure");
+		const DWORD LastError = GetLastError();
+		const std::string ErrorMsg = windows_format_system_message(LastError);
+		dbg_msg("filesystem", "GetCurrentDirectoryW failed: %ld %s", LastError, ErrorMsg.c_str());
 		return nullptr;
 	}
+	const std::string current_dir = windows_wide_to_utf8(wide_current_dir.c_str());
+	str_copy(buffer, current_dir.c_str(), buffer_size);
 	return buffer;
 #else
 	return getcwd(buffer, buffer_size);
@@ -2438,9 +2401,8 @@ int fs_parent_dir(char *path)
 int fs_remove(const char *filename)
 {
 #if defined(CONF_FAMILY_WINDOWS)
-	WCHAR wFilename[IO_MAX_PATH_LENGTH];
-	dbg_assert(MultiByteToWideChar(CP_UTF8, 0, filename, -1, wFilename, std::size(wFilename)) > 0, "MultiByteToWideChar failure");
-	return DeleteFileW(wFilename) == 0;
+	const std::wstring wide_filename = windows_utf8_to_wide(filename);
+	return DeleteFileW(wide_filename.c_str()) == 0;
 #else
 	return unlink(filename) != 0;
 #endif
@@ -2449,11 +2411,9 @@ int fs_remove(const char *filename)
 int fs_rename(const char *oldname, const char *newname)
 {
 #if defined(CONF_FAMILY_WINDOWS)
-	WCHAR wOldname[IO_MAX_PATH_LENGTH];
-	WCHAR wNewname[IO_MAX_PATH_LENGTH];
-	dbg_assert(MultiByteToWideChar(CP_UTF8, 0, oldname, -1, wOldname, std::size(wOldname)) > 0, "MultiByteToWideChar failure");
-	dbg_assert(MultiByteToWideChar(CP_UTF8, 0, newname, -1, wNewname, std::size(wNewname)) > 0, "MultiByteToWideChar failure");
-	if(MoveFileExW(wOldname, wNewname, MOVEFILE_REPLACE_EXISTING | MOVEFILE_COPY_ALLOWED) == 0)
+	const std::wstring wide_oldname = windows_utf8_to_wide(oldname);
+	const std::wstring wide_newname = windows_utf8_to_wide(newname);
+	if(MoveFileExW(wide_oldname.c_str(), wide_newname.c_str(), MOVEFILE_REPLACE_EXISTING | MOVEFILE_COPY_ALLOWED) == 0)
 		return 1;
 #else
 	if(rename(oldname, newname) != 0)
@@ -2466,11 +2426,8 @@ int fs_file_time(const char *name, time_t *created, time_t *modified)
 {
 #if defined(CONF_FAMILY_WINDOWS)
 	WIN32_FIND_DATAW finddata;
-	HANDLE handle;
-	WCHAR wBuffer[IO_MAX_PATH_LENGTH];
-
-	dbg_assert(MultiByteToWideChar(CP_UTF8, 0, name, -1, wBuffer, std::size(wBuffer)) > 0, "MultiByteToWideChar failure");
-	handle = FindFirstFileW(wBuffer, &finddata);
+	const std::wstring wide_name = windows_utf8_to_wide(name);
+	HANDLE handle = FindFirstFileW(wide_name.c_str(), &finddata);
 	if(handle == INVALID_HANDLE_VALUE)
 		return 1;
 
@@ -3895,13 +3852,13 @@ void cmdline_free(int argc, const char **argv)
 PROCESS shell_execute(const char *file)
 {
 #if defined(CONF_FAMILY_WINDOWS)
-	WCHAR wBuffer[IO_MAX_PATH_LENGTH];
-	dbg_assert(MultiByteToWideChar(CP_UTF8, 0, file, -1, wBuffer, std::size(wBuffer)) > 0, "MultiByteToWideChar failure");
+	const std::wstring wide_file = windows_utf8_to_wide(file);
+
 	SHELLEXECUTEINFOW info;
 	mem_zero(&info, sizeof(SHELLEXECUTEINFOW));
 	info.cbSize = sizeof(SHELLEXECUTEINFOW);
 	info.lpVerb = L"open";
-	info.lpFile = wBuffer;
+	info.lpFile = wide_file.c_str();
 	info.nShow = SW_SHOWMINNOACTIVE;
 	info.fMask = SEE_MASK_NOCLOSEPROCESS;
 	// Save and restore the FPU control word because ShellExecute might change it
@@ -3949,13 +3906,13 @@ int kill_process(PROCESS process)
 int open_link(const char *link)
 {
 #if defined(CONF_FAMILY_WINDOWS)
-	WCHAR wBuffer[IO_MAX_PATH_LENGTH];
-	dbg_assert(MultiByteToWideChar(CP_UTF8, 0, link, -1, wBuffer, std::size(wBuffer)) > 0, "MultiByteToWideChar failure");
+	const std::wstring wide_link = windows_utf8_to_wide(link);
+
 	SHELLEXECUTEINFOW info;
 	mem_zero(&info, sizeof(SHELLEXECUTEINFOW));
 	info.cbSize = sizeof(SHELLEXECUTEINFOW);
 	info.lpVerb = NULL; // NULL to use the default verb, as "open" may not be available
-	info.lpFile = wBuffer;
+	info.lpFile = wide_link.c_str();
 	info.nShow = SW_SHOWNORMAL;
 	// The SEE_MASK_NOASYNC flag ensures that the ShellExecuteEx function
 	// finishes its DDE conversation before it returns, so it's not necessary
@@ -4125,7 +4082,9 @@ void secure_random_fill(void *bytes, unsigned length)
 #if defined(CONF_FAMILY_WINDOWS)
 	if(!CryptGenRandom(secure_random_data.provider, length, (unsigned char *)bytes))
 	{
-		dbg_msg("secure", "CryptGenRandom failed, last_error=%ld", GetLastError());
+		const DWORD LastError = GetLastError();
+		const std::string ErrorMsg = windows_format_system_message(LastError);
+		dbg_msg("secure", "CryptGenRandom failed: %ld %s", LastError, ErrorMsg.c_str());
 		dbg_break();
 	}
 #else
@@ -4257,11 +4216,8 @@ void os_locale_str(char *locale, size_t length)
 	wchar_t wide_buffer[LOCALE_NAME_MAX_LENGTH];
 	dbg_assert(GetUserDefaultLocaleName(wide_buffer, std::size(wide_buffer)) > 0, "GetUserDefaultLocaleName failure");
 
-	// Assume maximum possible length for encoding as UTF-8.
-	char buffer[UTF8_BYTE_LENGTH * LOCALE_NAME_MAX_LENGTH + 1];
-	dbg_assert(WideCharToMultiByte(CP_UTF8, 0, wide_buffer, -1, buffer, sizeof(buffer), NULL, NULL) > 0, "WideCharToMultiByte failure");
-
-	str_copy(locale, buffer, length);
+	const std::string buffer = windows_wide_to_utf8(wide_buffer);
+	str_copy(locale, buffer.c_str(), length);
 #elif defined(CONF_PLATFORM_MACOS)
 	CFLocaleRef locale_ref = CFLocaleCopyCurrent();
 	CFStringRef locale_identifier_ref = static_cast<CFStringRef>(CFLocaleGetValue(locale_ref, kCFLocaleIdentifier));
@@ -4329,7 +4285,9 @@ void init_exception_handler()
 	exception_handling_module = LoadLibraryA(module_name);
 	if(exception_handling_module == nullptr)
 	{
-		dbg_msg("exception_handling", "failed to load exception handling library '%s' (error %ld)", module_name, GetLastError());
+		const DWORD LastError = GetLastError();
+		const std::string ErrorMsg = windows_format_system_message(LastError);
+		dbg_msg("exception_handling", "failed to load exception handling library '%s' (error %ld %s)", module_name, LastError, ErrorMsg.c_str());
 	}
 #else
 #error exception handling not implemented
@@ -4341,8 +4299,7 @@ void set_exception_handler_log_file(const char *log_file_path)
 #if defined(CONF_FAMILY_WINDOWS)
 	if(exception_handling_module != nullptr)
 	{
-		WCHAR wBuffer[IO_MAX_PATH_LENGTH];
-		dbg_assert(MultiByteToWideChar(CP_UTF8, 0, log_file_path, -1, wBuffer, std::size(wBuffer)) > 0, "MultiByteToWideChar failure");
+		const std::wstring wide_log_file_path = windows_utf8_to_wide(log_file_path);
 		// Intentional
 #ifdef __MINGW32__
 #pragma GCC diagnostic push
@@ -4354,9 +4311,13 @@ void set_exception_handler_log_file(const char *log_file_path)
 #pragma GCC diagnostic pop
 #endif
 		if(exception_log_file_path_func == nullptr)
-			dbg_msg("exception_handling", "could not find function '%s' in exception handling library (error %ld)", function_name, GetLastError());
+		{
+			const DWORD LastError = GetLastError();
+			const std::string ErrorMsg = windows_format_system_message(LastError);
+			dbg_msg("exception_handling", "could not find function '%s' in exception handling library (error %ld %s)", function_name, LastError, ErrorMsg.c_str());
+		}
 		else
-			exception_log_file_path_func(wBuffer);
+			exception_log_file_path_func(wide_log_file_path.c_str());
 	}
 #else
 #error exception handling not implemented
@@ -4376,6 +4337,28 @@ int net_socket_read_wait(NETSOCKET sock, std::chrono::nanoseconds nanoseconds)
 }
 
 #if defined(CONF_FAMILY_WINDOWS)
+std::wstring windows_utf8_to_wide(const char *str)
+{
+	const int orig_length = str_length(str);
+	if(orig_length == 0)
+		return L"";
+	const int size_needed = MultiByteToWideChar(CP_UTF8, 0, str, orig_length, nullptr, 0);
+	std::wstring wide_string(size_needed, L'\0');
+	dbg_assert(MultiByteToWideChar(CP_UTF8, 0, str, orig_length, &wide_string[0], size_needed) == size_needed, "MultiByteToWideChar failure");
+	return wide_string;
+}
+
+std::string windows_wide_to_utf8(const wchar_t *wide_str)
+{
+	const int orig_length = wcslen(wide_str);
+	if(orig_length == 0)
+		return "";
+	const int size_needed = WideCharToMultiByte(CP_UTF8, 0, wide_str, orig_length, nullptr, 0, nullptr, nullptr);
+	std::string string(size_needed, '\0');
+	dbg_assert(WideCharToMultiByte(CP_UTF8, 0, wide_str, orig_length, &string[0], size_needed, nullptr, nullptr) == size_needed, "WideCharToMultiByte failure");
+	return string;
+}
+
 // See https://learn.microsoft.com/en-us/windows/win32/learnwin32/initializing-the-com-library
 CWindowsComLifecycle::CWindowsComLifecycle(bool HasWindow)
 {
@@ -4390,18 +4373,8 @@ CWindowsComLifecycle::~CWindowsComLifecycle()
 
 static void windows_print_error(const char *system, const char *prefix, HRESULT error)
 {
-	char *message = windows_format_system_message(error);
-	dbg_msg(system, "%s: %s", prefix, message == nullptr ? "unknown error" : message);
-	free(message);
-}
-
-static std::wstring utf8_to_wstring(const char *str)
-{
-	const int orig_length = str_length(str);
-	int size_needed = MultiByteToWideChar(CP_UTF8, 0, str, orig_length, NULL, 0);
-	std::wstring wide_string(size_needed, '\0');
-	dbg_assert(MultiByteToWideChar(CP_UTF8, 0, str, orig_length, &wide_string[0], size_needed) == size_needed, "MultiByteToWideChar failure");
-	return wide_string;
+	const std::string message = windows_format_system_message(error);
+	dbg_msg(system, "%s: %s", prefix, message.c_str());
 }
 
 static std::wstring filename_from_path(const std::wstring &path)
@@ -4412,8 +4385,8 @@ static std::wstring filename_from_path(const std::wstring &path)
 
 bool shell_register_protocol(const char *protocol_name, const char *executable, bool *updated)
 {
-	const std::wstring protocol_name_wide = utf8_to_wstring(protocol_name);
-	const std::wstring executable_wide = utf8_to_wstring(executable);
+	const std::wstring protocol_name_wide = windows_utf8_to_wide(protocol_name);
+	const std::wstring executable_wide = windows_utf8_to_wide(executable);
 
 	// Open registry key for protocol associations of the current user
 	HKEY handle_subkey_classes;
@@ -4512,11 +4485,11 @@ bool shell_register_protocol(const char *protocol_name, const char *executable,
 
 bool shell_register_extension(const char *extension, const char *description, const char *executable_name, const char *executable, bool *updated)
 {
-	const std::wstring extension_wide = utf8_to_wstring(extension);
-	const std::wstring executable_name_wide = utf8_to_wstring(executable_name);
-	const std::wstring description_wide = executable_name_wide + L" " + utf8_to_wstring(description);
+	const std::wstring extension_wide = windows_utf8_to_wide(extension);
+	const std::wstring executable_name_wide = windows_utf8_to_wide(executable_name);
+	const std::wstring description_wide = executable_name_wide + L" " + windows_utf8_to_wide(description);
 	const std::wstring program_id_wide = executable_name_wide + extension_wide;
-	const std::wstring executable_wide = utf8_to_wstring(executable);
+	const std::wstring executable_wide = windows_utf8_to_wide(executable);
 
 	// Open registry key for file associations of the current user
 	HKEY handle_subkey_classes;
@@ -4648,8 +4621,8 @@ bool shell_register_extension(const char *extension, const char *description, co
 
 bool shell_register_application(const char *name, const char *executable, bool *updated)
 {
-	const std::wstring name_wide = utf8_to_wstring(name);
-	const std::wstring executable_filename = filename_from_path(utf8_to_wstring(executable));
+	const std::wstring name_wide = windows_utf8_to_wide(name);
+	const std::wstring executable_filename = filename_from_path(windows_utf8_to_wide(executable));
 
 	// Open registry key for application registrations
 	HKEY handle_subkey_applications;
@@ -4697,7 +4670,7 @@ bool shell_register_application(const char *name, const char *executable, bool *
 
 bool shell_unregister_class(const char *shell_class, bool *updated)
 {
-	const std::wstring class_wide = utf8_to_wstring(shell_class);
+	const std::wstring class_wide = windows_utf8_to_wide(shell_class);
 
 	// Open registry key for protocol and file associations of the current user
 	HKEY handle_subkey_classes;
@@ -4726,7 +4699,7 @@ bool shell_unregister_class(const char *shell_class, bool *updated)
 
 bool shell_unregister_application(const char *executable, bool *updated)
 {
-	const std::wstring executable_filename = filename_from_path(utf8_to_wstring(executable));
+	const std::wstring executable_filename = filename_from_path(windows_utf8_to_wide(executable));
 
 	// Open registry key for application registrations
 	HKEY handle_subkey_applications;

src/base/system.h

@@ -18,6 +18,7 @@
 #include <cstdarg>
 #include <cstdint>
 #include <ctime>
+#include <string>
 
 #ifdef __MINGW32__
 #undef PRId64
@@ -1162,12 +1163,9 @@ void net_unix_close(UNIXSOCKET sock);
  *
  * @param error The Windows error code.
  *
- * @return A new string representing the error code.
- *
- * @remark Guarantees that result will contain zero-termination.
- * @remark The result must be freed after it has been used.
+ * @return A new std::string representing the error code.
  */
-char *windows_format_system_message(unsigned long error);
+std::string windows_format_system_message(unsigned long error);
 
 #endif
 
@@ -2655,6 +2653,30 @@ public:
 };
 
 #if defined(CONF_FAMILY_WINDOWS)
+/**
+ * Converts a utf8 encoded string to a wide character string
+ * for use with the Windows API.
+ *
+ * @param str The utf8 encoded string to convert.
+ *
+ * @return The argument as a wide character string.
+ *
+ * @remark The argument string must be zero-terminated.
+ */
+std::wstring windows_utf8_to_wide(const char *str);
+
+/**
+ * Converts a wide character string obtained from the Windows API
+ * to a utf8 encoded string.
+ *
+ * @param wide_str The wide character string to convert.
+ *
+ * @return The argument as a utf8 encoded string.
+ *
+ * @remark The argument string must be zero-terminated.
+ */
+std::string windows_wide_to_utf8(const wchar_t *wide_str);
+
 /**
  * This is a RAII wrapper to initialize/uninitialize the Windows COM library,
  * which may be necessary for using the open_file and open_link functions.

src/engine/client/input.cpp

@@ -806,11 +806,7 @@ void CInput::ProcessSystemMessage(SDL_SysWMmsg *pMsg)
 				for(DWORD i = pCandidateList->dwPageStart; i < pCandidateList->dwCount && (int)m_vCandidates.size() < (int)pCandidateList->dwPageSize; i++)
 				{
 					LPCWSTR pCandidate = (LPCWSTR)((DWORD_PTR)pCandidateList + pCandidateList->dwOffset[i]);
-					const int Len = WideCharToMultiByte(CP_UTF8, 0, pCandidate, -1, nullptr, 0, nullptr, nullptr);
-					dbg_assert(Len > 0, "WideCharToMultiByte failure");
-					m_vCandidates.emplace_back();
-					m_vCandidates.back().resize(Len, '\0');
-					dbg_assert(WideCharToMultiByte(CP_UTF8, 0, pCandidate, -1, &m_vCandidates.back()[0], Len, nullptr, nullptr) == Len, "WideCharToMultiByte failure");
+					m_vCandidates.push_back(std::move(windows_wide_to_utf8(pCandidate)));
 				}
 				m_CandidateSelectedIndex = pCandidateList->dwSelection - pCandidateList->dwPageStart;
 			}

src/engine/shared/fifo.cpp

@@ -95,11 +95,8 @@ void CFifo::Init(IConsole *pConsole, char *pFifoFile, int Flag)
 	str_append(m_aFilename, pFifoFile, sizeof(m_aFilename));
 	m_Flag = Flag;
 
-	const int WLen = MultiByteToWideChar(CP_UTF8, 0, m_aFilename, -1, NULL, 0);
-	dbg_assert(WLen > 0, "MultiByteToWideChar failure");
-	wchar_t *pWide = static_cast<wchar_t *>(malloc(WLen * sizeof(*pWide)));
-	dbg_assert(MultiByteToWideChar(CP_UTF8, 0, m_aFilename, -1, pWide, WLen) == WLen, "MultiByteToWideChar failure");
-	m_pPipe = CreateNamedPipeW(pWide,
+	const std::wstring WideFilename = windows_utf8_to_wide(m_aFilename);
+	m_pPipe = CreateNamedPipeW(WideFilename.c_str(),
 		PIPE_ACCESS_DUPLEX,
 		PIPE_TYPE_MESSAGE | PIPE_READMODE_MESSAGE | PIPE_NOWAIT | PIPE_REJECT_REMOTE_CLIENTS,
 		PIPE_UNLIMITED_INSTANCES,
@@ -107,13 +104,11 @@ void CFifo::Init(IConsole *pConsole, char *pFifoFile, int Flag)
 		8192,
 		NMPWAIT_USE_DEFAULT_WAIT,
 		NULL);
-	free(pWide);
 	if(m_pPipe == INVALID_HANDLE_VALUE)
 	{
 		const DWORD LastError = GetLastError();
-		char *pErrorMsg = windows_format_system_message(LastError);
-		dbg_msg("fifo", "failed to create named pipe '%s' (%ld %s)", m_aFilename, LastError, pErrorMsg);
-		free(pErrorMsg);
+		const std::string ErrorMsg = windows_format_system_message(LastError);
+		dbg_msg("fifo", "failed to create named pipe '%s' (%ld %s)", m_aFilename, LastError, ErrorMsg.c_str());
 	}
 	else
 		dbg_msg("fifo", "created named pipe '%s'", m_aFilename);
@@ -147,9 +142,8 @@ void CFifo::Update()
 		}
 		if(LastError != ERROR_PIPE_CONNECTED) // pipe already connected, not an error
 		{
-			char *pErrorMsg = windows_format_system_message(LastError);
-			dbg_msg("fifo", "failed to connect named pipe '%s' (%ld %s)", m_aFilename, LastError, pErrorMsg);
-			free(pErrorMsg);
+			const std::string ErrorMsg = windows_format_system_message(LastError);
+			dbg_msg("fifo", "failed to connect named pipe '%s' (%ld %s)", m_aFilename, LastError, ErrorMsg.c_str());
 			return;
 		}
 	}
@@ -162,9 +156,8 @@ void CFifo::Update()
 			const DWORD LastError = GetLastError();
 			if(LastError != ERROR_BAD_PIPE) // pipe not connected, not an error
 			{
-				char *pErrorMsg = windows_format_system_message(LastError);
-				dbg_msg("fifo", "failed to peek at pipe '%s' (%ld %s)", m_aFilename, LastError, pErrorMsg);
-				free(pErrorMsg);
+				const std::string ErrorMsg = windows_format_system_message(LastError);
+				dbg_msg("fifo", "failed to peek at pipe '%s' (%ld %s)", m_aFilename, LastError, ErrorMsg.c_str());
 			}
 			return;
 		}
@@ -176,9 +169,8 @@ void CFifo::Update()
 		if(!ReadFile(m_pPipe, pBuf, BytesAvailable, &Length, NULL))
 		{
 			const DWORD LastError = GetLastError();
-			char *pErrorMsg = windows_format_system_message(LastError);
-			dbg_msg("fifo", "failed to read from pipe '%s' (%ld %s)", m_aFilename, LastError, pErrorMsg);
-			free(pErrorMsg);
+			const std::string ErrorMsg = windows_format_system_message(LastError);
+			dbg_msg("fifo", "failed to read from pipe '%s' (%ld %s)", m_aFilename, LastError, ErrorMsg.c_str());
 			free(pBuf);
 			return;
 		}

src/test/str.cpp

@@ -950,3 +950,35 @@ TEST(Str, CountChar)
 	EXPECT_EQ(str_countchr(pStr, '\0'), 0);
 	EXPECT_EQ(str_countchr(pStr, 'y'), 0);
 }
+
+#if defined(CONF_FAMILY_WINDOWS)
+TEST(Str, WindowsUtf8WideConversion)
+{
+	const char *apUtf8Strings[] = {
+		"",
+		"abc",
+		"a bb ccc dddd       eeeee",
+		"öüä",
+		"привет Наташа",
+		"ąçęįǫų",
+		"DDNet最好了",
+		"aβい🐘"};
+	const wchar_t *apWideStrings[] = {
+		L"",
+		L"abc",
+		L"a bb ccc dddd       eeeee",
+		L"öüä",
+		L"привет Наташа",
+		L"ąçęįǫų",
+		L"DDNet最好了",
+		L"aβい🐘"};
+	static_assert(std::size(apUtf8Strings) == std::size(apWideStrings));
+	for(size_t i = 0; i < std::size(apUtf8Strings); i++)
+	{
+		const std::string ConvertedUtf8 = windows_wide_to_utf8(apWideStrings[i]);
+		const std::wstring ConvertedWide = windows_utf8_to_wide(apUtf8Strings[i]);
+		EXPECT_STREQ(ConvertedUtf8.c_str(), apUtf8Strings[i]);
+		EXPECT_STREQ(ConvertedWide.c_str(), apWideStrings[i]);
+	}
+}
+#endif