From 8d04e7e5e1cdec6ee7b3d98cdabaca44de864c9c Mon Sep 17 00:00:00 2001 From: heinrich5991 Date: Wed, 11 Jul 2018 20:17:21 +0200 Subject: [PATCH] Share libcurl resources across requests Use the libcurl-share interface to share DNS cache and connections between different requests. If compiled with OpenSSL, libcurl can only be safely used from multiple threads for OpenSSL >= 1.1.0, but this problem is not newly introduced by this commit: According to libcurl-thread(3): >OpenSSL <= 1.0.2 the user must set callbacks. > >https://www.openssl.org/docs/man1.0.2/crypto/threads.html#DESCRIPTION > >https://curl.haxx.se/libcurl/c/opensslthreadlock.html --- CMakeLists.txt | 30 +-- src/engine/client/client.cpp | 20 +- src/engine/client/client.h | 2 +- src/engine/client/updater.cpp | 2 +- src/engine/client/updater.h | 2 +- src/engine/server/server.cpp | 4 +- src/engine/shared/config_variables.h | 1 + src/engine/shared/fetcher.cpp | 253 ------------------ src/engine/shared/fetcher.h | 74 ----- src/engine/shared/http.cpp | 385 +++++++++++++++++++++++++++ src/engine/shared/http.h | 109 ++++++++ src/game/server/ddracechat.cpp | 4 +- src/game/server/player.h | 2 +- src/game/server/teehistorian.h | 2 +- src/test/json.cpp | 2 +- 15 files changed, 524 insertions(+), 368 deletions(-) delete mode 100644 src/engine/shared/fetcher.cpp delete mode 100644 src/engine/shared/fetcher.h create mode 100644 src/engine/shared/http.cpp create mode 100644 src/engine/shared/http.h diff --git a/CMakeLists.txt b/CMakeLists.txt index 5a5bf84fe..646285e8d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -477,14 +477,16 @@ endif() # DEPENDENCY COMPILATION ######################################################################## -if(CLIENT) - # Static dependencies - set_glob(JSON_SRC GLOB src/engine/external/json-parser json.c json.h) - add_library(json EXCLUDE_FROM_ALL OBJECT ${JSON_SRC}) +# Static dependencies +set_glob(DEP_JSON_SRC GLOB src/engine/external/json-parser json.c json.h) +add_library(json EXCLUDE_FROM_ALL OBJECT ${DEP_JSON_SRC}) - list(APPEND TARGETS_DEP json) - set(JSON_DEP $) -endif() +set_glob(DEP_MD5_SRC GLOB src/engine/external/md5 md5.c md5.h) +add_library(md5 EXCLUDE_FROM_ALL OBJECT ${DEP_MD5_SRC}) + +list(APPEND TARGETS_DEP json md5) +set(DEP_JSON $) +set(DEP_MD5 $) ######################################################################## # COPY DATA AND DLLS @@ -613,8 +615,6 @@ set_glob(ENGINE_SHARED GLOB src/engine/shared econ.cpp econ.h engine.cpp - fetcher.cpp - fetcher.h fifo.cpp fifo.h filecollection.cpp @@ -622,6 +622,8 @@ set_glob(ENGINE_SHARED GLOB src/engine/shared ghost.cpp ghost.h global_uuid_manager.cpp + http.cpp + http.h huffman.cpp huffman.h jobs.cpp @@ -693,13 +695,7 @@ set(GAME_GENERATED_SHARED src/game/generated/protocol.h ) -# Static dependencies -set_glob(DEP_MD5_SRC GLOB src/engine/external/md5 md5.c md5.h) -add_library(md5 EXCLUDE_FROM_ALL OBJECT ${DEP_MD5_SRC}) -set(DEP_MD5 $) -list(APPEND TARGETS_DEP md5) - -set(DEPS ${DEP_MD5} ${ZLIB_DEP}) +set(DEPS ${DEP_JSON} ${DEP_MD5} ${ZLIB_DEP}) # Libraries set(LIBS ${CMAKE_THREAD_LIBS_INIT} ${CURL_LIBRARIES} ${CRYPTO_LIBRARIES} ${WEBSOCKETS_LIBRARIES} ${ZLIB_LIBRARIES} ${PLATFORM_LIBS}) @@ -842,7 +838,7 @@ if(CLIENT) ) set(CLIENT_SRC ${ENGINE_CLIENT} ${PLATFORM_CLIENT} ${GAME_CLIENT} ${GAME_EDITOR} ${GAME_GENERATED_CLIENT}) - set(DEPS_CLIENT ${DEPS} ${GLEW_DEP} ${JSON_DEP} ${PNGLITE_DEP} ${WAVPACK_DEP}) + set(DEPS_CLIENT ${DEPS} ${GLEW_DEP} ${PNGLITE_DEP} ${WAVPACK_DEP}) # Libraries set(LIBS_CLIENT diff --git a/src/engine/client/client.cpp b/src/engine/client/client.cpp index c631c709a..2b7a57298 100644 --- a/src/engine/client/client.cpp +++ b/src/engine/client/client.cpp @@ -40,9 +40,9 @@ #include #include #include -#include #include #include +#include #include #include #include @@ -1598,15 +1598,7 @@ void CClient::ProcessServerPacket(CNetChunk *pPacket) EscapeUrl(aEscaped, sizeof(aEscaped), aFilename); str_format(aUrl, sizeof(aUrl), "%s/%s", g_Config.m_ClDDNetMapDownloadUrl, aEscaped); - // We only trust our own custom-selected CAs for our own servers. - // Other servers can use any CA trusted by the system. - bool UseDDNetCA = - str_comp_nocase_num("maps.ddnet.tw/", aUrl, 14) == 0 || - str_comp_nocase_num("http://maps.ddnet.tw/", aUrl, 21) == 0 || - str_comp_nocase_num("https://maps.ddnet.tw/", aUrl, 22) == 0; - - - m_pMapdownloadTask = std::make_shared(Storage(), aUrl, m_aMapdownloadFilename, IStorage::TYPE_SAVE, UseDDNetCA, true); + m_pMapdownloadTask = std::make_shared(Storage(), aUrl, m_aMapdownloadFilename, IStorage::TYPE_SAVE, true); Engine()->AddJob(m_pMapdownloadTask); } else @@ -2250,7 +2242,7 @@ void CClient::LoadDDNetInfo() return; const json_value *pVersion = json_object_get(pDDNetInfo, "version"); - if(pVersion && pVersion->type == json_string) + if(pVersion->type == json_string) { const char *pVersionString = json_string_get(pVersion); if(str_comp(pVersionString, GAME_RELEASE_VERSION)) @@ -2265,7 +2257,7 @@ void CClient::LoadDDNetInfo() } const json_value *pNews = json_object_get(pDDNetInfo, "news"); - if(pNews && pNews->type == json_string) + if(pNews->type == json_string) { const char *pNewsString = json_string_get(pNews); @@ -2652,7 +2644,7 @@ void CClient::InitInterfaces() m_ServerBrowser.SetBaseInfo(&m_NetClient[2], m_pGameClient->NetVersion()); - FetcherInit(); + HttpInit(m_pStorage); #if !defined(CONF_PLATFORM_MACOSX) && !defined(__ANDROID__) m_Updater.Init(); @@ -3777,7 +3769,7 @@ void CClient::RequestDDNetInfo() str_append(aUrl, aEscaped, sizeof(aUrl)); } - m_pDDNetInfoTask = std::make_shared(Storage(), aUrl, "ddnet-info.json.tmp", IStorage::TYPE_SAVE, true, true); + m_pDDNetInfoTask = std::make_shared(Storage(), aUrl, "ddnet-info.json.tmp", IStorage::TYPE_SAVE, true); Engine()->AddJob(m_pDDNetInfoTask); } diff --git a/src/engine/client/client.h b/src/engine/client/client.h index 544ab3489..2997d8760 100644 --- a/src/engine/client/client.h +++ b/src/engine/client/client.h @@ -6,7 +6,7 @@ #include #include -#include +#include class CGraph { diff --git a/src/engine/client/updater.cpp b/src/engine/client/updater.cpp index b7f05b07b..68ccd8cb3 100644 --- a/src/engine/client/updater.cpp +++ b/src/engine/client/updater.cpp @@ -41,7 +41,7 @@ static const char *GetUpdaterDestPath(char *pBuf, int BufSize, const char *pFile } CUpdaterFetchTask::CUpdaterFetchTask(CUpdater *pUpdater, const char *pFile, const char *pDestPath) : - CGetFile(pUpdater->m_pStorage, GetUpdaterUrl(m_aBuf, sizeof(m_aBuf), pFile), GetUpdaterDestPath(m_aBuf2, sizeof(m_aBuf), pFile, pDestPath), -2, true, false), + CGetFile(pUpdater->m_pStorage, GetUpdaterUrl(m_aBuf, sizeof(m_aBuf), pFile), GetUpdaterDestPath(m_aBuf2, sizeof(m_aBuf), pFile, pDestPath), -2, false), m_pUpdater(pUpdater) { } diff --git a/src/engine/client/updater.h b/src/engine/client/updater.h index 6be13f854..2dab900ae 100644 --- a/src/engine/client/updater.h +++ b/src/engine/client/updater.h @@ -2,7 +2,7 @@ #define ENGINE_CLIENT_UPDATER_H #include -#include +#include #include #include diff --git a/src/engine/server/server.cpp b/src/engine/server/server.cpp index 88c3809fa..a4acf9c20 100644 --- a/src/engine/server/server.cpp +++ b/src/engine/server/server.cpp @@ -19,8 +19,8 @@ #include #include #include -#include #include +#include #include #include #include @@ -2895,7 +2895,7 @@ int main(int argc, const char **argv) // ignore_convention pEngineMasterServer->Init(); pEngineMasterServer->Load(); - FetcherInit(); + HttpInit(pStorage); // register all console commands pServer->RegisterCommands(); diff --git a/src/engine/shared/config_variables.h b/src/engine/shared/config_variables.h index cc1dff0f6..7d146dad6 100644 --- a/src/engine/shared/config_variables.h +++ b/src/engine/shared/config_variables.h @@ -170,6 +170,7 @@ MACRO_CONFIG_INT(EcAuthTimeout, ec_auth_timeout, 30, 1, 120, CFGFLAG_ECON, "Time MACRO_CONFIG_INT(EcOutputLevel, ec_output_level, 1, 0, 2, CFGFLAG_ECON, "Adjusts the amount of information in the external console") MACRO_CONFIG_INT(Debug, debug, 0, 0, 1, CFGFLAG_CLIENT|CFGFLAG_SERVER, "Debug mode") +MACRO_CONFIG_INT(DbgCurl, dbg_curl, 0, 0, 1, CFGFLAG_CLIENT|CFGFLAG_SERVER, "Debug curl") MACRO_CONFIG_INT(DbgPref, dbg_pref, 0, 0, 1, CFGFLAG_SERVER, "Performance outputs") MACRO_CONFIG_INT(DbgGraphs, dbg_graphs, 0, 0, 1, CFGFLAG_CLIENT, "Performance graphs") MACRO_CONFIG_INT(DbgHitch, dbg_hitch, 0, 0, 0, CFGFLAG_SERVER, "Hitch warnings") diff --git a/src/engine/shared/fetcher.cpp b/src/engine/shared/fetcher.cpp deleted file mode 100644 index f613733e9..000000000 --- a/src/engine/shared/fetcher.cpp +++ /dev/null @@ -1,253 +0,0 @@ -#include -#include -#include -#include -#include - -#define WIN32_LEAN_AND_MEAN -#include "curl/curl.h" -#include "curl/easy.h" - -#include "fetcher.h" - -double CGetFile::Current() const { return m_Current; } -double CGetFile::Size() const { return m_Size; } -int CGetFile::Progress() const { return m_Progress; } -int CGetFile::State() const { return m_State; } -const char *CGetFile::Dest() const { return m_aDest; } - -void CGetFile::Abort() { m_Abort = true; }; - -CGetFile::CGetFile(IStorage *pStorage, const char *pUrl, const char *pDest, int StorageType, bool UseDDNetCA, bool CanTimeout) : - m_pStorage(pStorage), - m_StorageType(StorageType), - m_UseDDNetCA(UseDDNetCA), - m_CanTimeout(CanTimeout), - m_Size(0), - m_Progress(0), - m_State(HTTP_QUEUED), - m_Abort(false) -{ - str_copy(m_aUrl, pUrl, sizeof(m_aUrl)); - str_copy(m_aDest, pDest, sizeof(m_aDest)); -} - - -bool FetcherInit() -{ - return !curl_global_init(CURL_GLOBAL_DEFAULT); -} - -void EscapeUrl(char *pBuf, int Size, const char *pStr) -{ - char *pEsc = curl_easy_escape(0, pStr, 0); - str_copy(pBuf, pEsc, Size); - curl_free(pEsc); -} - -static void SetCurlOptions(CURL *pHandle, char *pErrorBuffer, const char *pUrl, bool CanTimeout) -{ - //curl_easy_setopt(pHandle, CURLOPT_VERBOSE, 1L); - curl_easy_setopt(pHandle, CURLOPT_ERRORBUFFER, pErrorBuffer); - - if(CanTimeout) - { - curl_easy_setopt(pHandle, CURLOPT_CONNECTTIMEOUT_MS, (long)g_Config.m_ClHTTPConnectTimeoutMs); - curl_easy_setopt(pHandle, CURLOPT_LOW_SPEED_LIMIT, (long)g_Config.m_ClHTTPLowSpeedLimit); - curl_easy_setopt(pHandle, CURLOPT_LOW_SPEED_TIME, (long)g_Config.m_ClHTTPLowSpeedTime); - } - else - { - curl_easy_setopt(pHandle, CURLOPT_CONNECTTIMEOUT_MS, 0L); - curl_easy_setopt(pHandle, CURLOPT_LOW_SPEED_LIMIT, 0L); - curl_easy_setopt(pHandle, CURLOPT_LOW_SPEED_TIME, 0L); - } - curl_easy_setopt(pHandle, CURLOPT_FOLLOWLOCATION, 1L); - curl_easy_setopt(pHandle, CURLOPT_MAXREDIRS, 4L); - curl_easy_setopt(pHandle, CURLOPT_FAILONERROR, 1L); - curl_easy_setopt(pHandle, CURLOPT_URL, pUrl); - curl_easy_setopt(pHandle, CURLOPT_NOSIGNAL, 1L); - curl_easy_setopt(pHandle, CURLOPT_USERAGENT, "DDNet " GAME_RELEASE_VERSION " (" CONF_PLATFORM_STRING "; " CONF_ARCH_STRING ")"); -} - -void CGetFile::Run() -{ - CURL *pHandle = curl_easy_init(); - if(!pHandle) - { - m_State = HTTP_ERROR; - return; - } - - char aPath[512]; - if(m_StorageType == -2) - m_pStorage->GetBinaryPath(m_aDest, aPath, sizeof(aPath)); - else - m_pStorage->GetCompletePath(m_StorageType, m_aDest, aPath, sizeof(aPath)); - - if(fs_makedir_rec_for(aPath) < 0) - dbg_msg("fetcher", "i/o error, cannot create folder for: %s", aPath); - - IOHANDLE File = io_open(aPath, IOFLAG_WRITE); - - if(!File) - { - dbg_msg("fetcher", "i/o error, cannot open file: %s", m_aDest); - m_State = HTTP_ERROR; - return; - } - - char aErr[CURL_ERROR_SIZE]; - SetCurlOptions(pHandle, aErr, m_aUrl, m_CanTimeout); - if(m_UseDDNetCA) - { - char aCAFile[512]; - m_pStorage->GetBinaryPath("data/ca-ddnet.pem", aCAFile, sizeof aCAFile); - curl_easy_setopt(pHandle, CURLOPT_CAINFO, aCAFile); - } - curl_easy_setopt(pHandle, CURLOPT_WRITEDATA, File); - curl_easy_setopt(pHandle, CURLOPT_WRITEFUNCTION, WriteCallback); - curl_easy_setopt(pHandle, CURLOPT_NOPROGRESS, 0L); - curl_easy_setopt(pHandle, CURLOPT_PROGRESSDATA, this); - curl_easy_setopt(pHandle, CURLOPT_PROGRESSFUNCTION, ProgressCallback); - - dbg_msg("fetcher", "downloading %s", m_aDest); - m_State = HTTP_RUNNING; - int Ret = curl_easy_perform(pHandle); - io_close(File); - if(Ret != CURLE_OK) - { - dbg_msg("fetcher", "task failed. libcurl error: %s", aErr); - m_State = (Ret == CURLE_ABORTED_BY_CALLBACK) ? HTTP_ABORTED : HTTP_ERROR; - } - else - { - dbg_msg("fetcher", "task done %s", m_aDest); - m_State = HTTP_DONE; - } - - curl_easy_cleanup(pHandle); - - OnCompletion(); -} - -size_t CGetFile::WriteCallback(char *pData, size_t Size, size_t Number, void *pFile) -{ - return io_write((IOHANDLE)pFile, pData, Size * Number); -} - -int CGetFile::ProgressCallback(void *pUser, double DlTotal, double DlCurr, double UlTotal, double UlCurr) -{ - CGetFile *pTask = (CGetFile *)pUser; - pTask->m_Current = DlCurr; - pTask->m_Size = DlTotal; - pTask->m_Progress = (100 * DlCurr) / (DlTotal ? DlTotal : 1); - pTask->OnProgress(); - return pTask->m_Abort ? -1 : 0; -} - -int CPostJson::State() const { return m_State; } - -CPostJson::CPostJson(const char *pUrl, const char *pJson) - : m_State(HTTP_QUEUED) -{ - str_copy(m_aUrl, pUrl, sizeof(m_aUrl)); - str_copy(m_aJson, pJson, sizeof(m_aJson)); -} - -void CPostJson::Run() -{ - CURL *pHandle = curl_easy_init(); - if(!pHandle) - { - m_State = HTTP_ERROR; - return; - } - - char aErr[CURL_ERROR_SIZE]; - SetCurlOptions(pHandle, aErr, m_aUrl, true); - - curl_slist *pHeaders = NULL; - pHeaders = curl_slist_append(pHeaders, "Content-Type: application/json"); - curl_easy_setopt(pHandle, CURLOPT_HTTPHEADER, pHeaders); - curl_easy_setopt(pHandle, CURLOPT_POSTFIELDS, m_aJson); - - dbg_msg("fetcher", "posting to %s", m_aUrl); - m_State = HTTP_RUNNING; - int Result = curl_easy_perform(pHandle); - - curl_slist_free_all(pHeaders); - if(Result != CURLE_OK) - { - dbg_msg("fetcher", "task failed. libcurl error: %s", aErr); - m_State = HTTP_ERROR; - } - else - { - dbg_msg("fetcher", "posting to %s done", m_aUrl); - m_State = HTTP_DONE; - } - - curl_easy_cleanup(pHandle); - - OnCompletion(); -} - -static char EscapeJsonChar(char c) -{ - switch(c) - { - case '\"': return '\"'; - case '\\': return '\\'; - case '\b': return 'b'; - case '\n': return 'n'; - case '\r': return 'r'; - case '\t': return 't'; - // Don't escape '\f', who uses that. :) - default: return 0; - } -} - -char *EscapeJson(char *pBuffer, int BufferSize, const char *pString) -{ - dbg_assert(BufferSize > 0, "can't null-terminate the string"); - // Subtract the space for null termination early. - BufferSize--; - - char *pResult = pBuffer; - while(BufferSize && *pString) - { - char c = *pString; - pString++; - char Escaped = EscapeJsonChar(c); - if(Escaped) - { - if(BufferSize < 2) - { - break; - } - *pBuffer++ = '\\'; - *pBuffer++ = Escaped; - BufferSize -= 2; - } - // Assuming ASCII/UTF-8, "if control character". - else if((unsigned char)c < 0x20) - { - // \uXXXX - if(BufferSize < 6) - { - break; - } - str_format(pBuffer, BufferSize, "\\u%04x", c); - pBuffer += 6; - BufferSize -= 6; - } - else - { - *pBuffer++ = c; - BufferSize--; - } - } - *pBuffer = 0; - return pResult; -} diff --git a/src/engine/shared/fetcher.h b/src/engine/shared/fetcher.h deleted file mode 100644 index fdc441c86..000000000 --- a/src/engine/shared/fetcher.h +++ /dev/null @@ -1,74 +0,0 @@ -#ifndef ENGINE_SHARED_FETCHER_H -#define ENGINE_SHARED_FETCHER_H - -#include -#include -#include - -enum -{ - HTTP_ERROR = -1, - HTTP_QUEUED, - HTTP_RUNNING, - HTTP_DONE, - HTTP_ABORTED, -}; - -class CGetFile : public IJob -{ -private: - IStorage *m_pStorage; - - char m_aUrl[256]; - char m_aDest[256]; - int m_StorageType; - bool m_UseDDNetCA; - bool m_CanTimeout; - - double m_Size; - double m_Current; - int m_Progress; - std::atomic m_State; - - std::atomic m_Abort; - - virtual void OnProgress() { } - virtual void OnCompletion() { } - - static int ProgressCallback(void *pUser, double DlTotal, double DlCurr, double UlTotal, double UlCurr); - static size_t WriteCallback(char *pData, size_t Size, size_t Number, void *pFile); - - void Run(); - -public: - CGetFile(IStorage *pStorage, const char *pUrl, const char *pDest, int StorageType = -2, bool UseDDNetCA = false, bool CanTimeout = true); - - double Current() const; - double Size() const; - int Progress() const; - int State() const; - const char *Dest() const; - void Abort(); -}; - - -class CPostJson : public IJob -{ -private: - char m_aUrl[256]; - char m_aJson[1024]; - std::atomic m_State; - - void Run(); - - virtual void OnCompletion() { } - -public: - CPostJson(const char *pUrl, const char *pJson); - int State() const; -}; - -bool FetcherInit(); -void EscapeUrl(char *pBuf, int Size, const char *pStr); -char *EscapeJson(char *pBuffer, int BufferSize, const char *pString); -#endif // ENGINE_SHARED_FETCHER_H diff --git a/src/engine/shared/http.cpp b/src/engine/shared/http.cpp new file mode 100644 index 000000000..3cc2ed519 --- /dev/null +++ b/src/engine/shared/http.cpp @@ -0,0 +1,385 @@ +#include "http.h" + +#include +#include +#include +#include +#include +#include + +#define WIN32_LEAN_AND_MEAN +#include "curl/curl.h" +#include "curl/easy.h" + +static char CA_FILE_PATH[512]; +// TODO: Non-global pls? +static CURLSH *gs_Share; +static LOCK gs_aLocks[CURL_LOCK_DATA_LAST+1]; + +static int GetLockIndex(int Data) +{ + if(!(0 <= Data && Data < CURL_LOCK_DATA_LAST)) + { + Data = CURL_LOCK_DATA_LAST; + } + return Data; +} + +static void CurlLock(CURL *pHandle, curl_lock_data Data, curl_lock_access Access, void *pUser) +{ + (void)pHandle; + (void)Access; + (void)pUser; + lock_wait(gs_aLocks[GetLockIndex(Data)]); +} + +static void CurlUnlock(CURL *pHandle, curl_lock_data Data, void *pUser) +{ + (void)pHandle; + (void)pUser; + lock_unlock(gs_aLocks[GetLockIndex(Data)]); +} + +bool HttpInit(IStorage *pStorage) +{ + pStorage->GetBinaryPath("data/ca-ddnet.pem", CA_FILE_PATH, sizeof(CA_FILE_PATH)); + if(curl_global_init(CURL_GLOBAL_DEFAULT)) + { + return true; + } + gs_Share = curl_share_init(); + if(!gs_Share) + { + return true; + } + for(unsigned int i = 0; i < sizeof(gs_aLocks) / sizeof(gs_aLocks[0]); i++) + { + gs_aLocks[i] = lock_create(); + } + curl_share_setopt(gs_Share, CURLSHOPT_SHARE, CURL_LOCK_DATA_DNS); + curl_share_setopt(gs_Share, CURLSHOPT_SHARE, CURL_LOCK_DATA_SSL_SESSION); + curl_share_setopt(gs_Share, CURLSHOPT_SHARE, CURL_LOCK_DATA_CONNECT); + curl_share_setopt(gs_Share, CURLSHOPT_LOCKFUNC, CurlLock); + curl_share_setopt(gs_Share, CURLSHOPT_UNLOCKFUNC, CurlUnlock); + return false; +} + +void EscapeUrl(char *pBuf, int Size, const char *pStr) +{ + char *pEsc = curl_easy_escape(0, pStr, 0); + str_copy(pBuf, pEsc, Size); + curl_free(pEsc); +} + +CRequest::CRequest(const char *pUrl, bool CanTimeout) : + m_CanTimeout(CanTimeout), + m_Size(0), + m_Progress(0), + m_State(HTTP_QUEUED), + m_Abort(false) +{ + str_copy(m_aUrl, pUrl, sizeof(m_aUrl)); +} + +void CRequest::Run() +{ + if(BeforeInit()) + { + m_State = HTTP_ERROR; + return; + } + + CURL *pHandle = curl_easy_init(); + if(!pHandle) + { + m_State = HTTP_ERROR; + return; + } + + if(g_Config.m_DbgCurl) + { + curl_easy_setopt(pHandle, CURLOPT_VERBOSE, 1L); + } + char aErr[CURL_ERROR_SIZE]; + curl_easy_setopt(pHandle, CURLOPT_ERRORBUFFER, aErr); + + if(m_CanTimeout) + { + curl_easy_setopt(pHandle, CURLOPT_CONNECTTIMEOUT_MS, (long)g_Config.m_ClHTTPConnectTimeoutMs); + curl_easy_setopt(pHandle, CURLOPT_LOW_SPEED_LIMIT, (long)g_Config.m_ClHTTPLowSpeedLimit); + curl_easy_setopt(pHandle, CURLOPT_LOW_SPEED_TIME, (long)g_Config.m_ClHTTPLowSpeedTime); + } + else + { + curl_easy_setopt(pHandle, CURLOPT_CONNECTTIMEOUT_MS, 0L); + curl_easy_setopt(pHandle, CURLOPT_LOW_SPEED_LIMIT, 0L); + curl_easy_setopt(pHandle, CURLOPT_LOW_SPEED_TIME, 0L); + } + curl_easy_setopt(pHandle, CURLOPT_SHARE, gs_Share); + curl_easy_setopt(pHandle, CURLOPT_PROTOCOLS, CURLPROTO_HTTP | CURLPROTO_HTTPS); + curl_easy_setopt(pHandle, CURLOPT_FOLLOWLOCATION, 1L); + curl_easy_setopt(pHandle, CURLOPT_MAXREDIRS, 4L); + curl_easy_setopt(pHandle, CURLOPT_FAILONERROR, 1L); + curl_easy_setopt(pHandle, CURLOPT_URL, m_aUrl); + curl_easy_setopt(pHandle, CURLOPT_NOSIGNAL, 1L); + curl_easy_setopt(pHandle, CURLOPT_USERAGENT, "DDNet " GAME_RELEASE_VERSION " (" CONF_PLATFORM_STRING "; " CONF_ARCH_STRING ")"); + + // We only trust our own custom-selected CAs for our own servers. + // Other servers can use any CA trusted by the system. + if(false + || str_comp_nocase_num("maps.ddnet.tw/", m_aUrl, 14) == 0 + || str_comp_nocase_num("http://maps.ddnet.tw/", m_aUrl, 21) == 0 + || str_comp_nocase_num("https://maps.ddnet.tw/", m_aUrl, 22) == 0 + || str_comp_nocase_num("http://info.ddnet.tw/", m_aUrl, 21) == 0 + || str_comp_nocase_num("https://info.ddnet.tw/", m_aUrl, 22) == 0 + || str_comp_nocase_num("https://update4.ddnet.tw/", m_aUrl, 25) == 0) + { + curl_easy_setopt(pHandle, CURLOPT_CAINFO, CA_FILE_PATH); + } + curl_easy_setopt(pHandle, CURLOPT_WRITEDATA, this); + curl_easy_setopt(pHandle, CURLOPT_WRITEFUNCTION, WriteCallback); + curl_easy_setopt(pHandle, CURLOPT_NOPROGRESS, 0L); + curl_easy_setopt(pHandle, CURLOPT_PROGRESSDATA, this); + curl_easy_setopt(pHandle, CURLOPT_PROGRESSFUNCTION, ProgressCallback); + + if(AfterInit(pHandle)) + { + curl_easy_cleanup(pHandle); + m_State = HTTP_ERROR; + return; + } + + dbg_msg("http", "http %s", m_aUrl); + m_State = HTTP_RUNNING; + int Ret = curl_easy_perform(pHandle); + if(Ret != CURLE_OK) + { + dbg_msg("http", "task failed. libcurl error: %s", aErr); + m_State = (Ret == CURLE_ABORTED_BY_CALLBACK) ? HTTP_ABORTED : HTTP_ERROR; + } + else + { + dbg_msg("http", "task done %s", m_aUrl); + m_State = HTTP_DONE; + } + + curl_easy_cleanup(pHandle); + + BeforeCompletion(); + OnCompletion(); +} + +size_t CRequest::WriteCallback(char *pData, size_t Size, size_t Number, void *pUser) +{ + return ((CRequest *)pUser)->OnData(pData, Size * Number); +} + +int CRequest::ProgressCallback(void *pUser, double DlTotal, double DlCurr, double UlTotal, double UlCurr) +{ + CGetFile *pTask = (CGetFile *)pUser; + pTask->m_Current = DlCurr; + pTask->m_Size = DlTotal; + pTask->m_Progress = (100 * DlCurr) / (DlTotal ? DlTotal : 1); + pTask->OnProgress(); + return pTask->m_Abort ? -1 : 0; +} + +CGet::CGet(const char *pUrl, bool CanTimeout) : + CRequest(pUrl, CanTimeout), + m_BufferSize(0), + m_BufferLength(0), + m_pBuffer(NULL) +{ +} + +CGet::~CGet() +{ + m_BufferSize = 0; + m_BufferLength = 0; + free(m_pBuffer); + m_pBuffer = NULL; +} + +unsigned char *CGet::Result() const +{ + if(State() != HTTP_DONE) + { + return NULL; + } + return m_pBuffer; +} + +unsigned char *CGet::TakeResult() +{ + unsigned char *pResult = Result(); + if(pResult) + { + m_BufferSize = 0; + m_BufferLength = 0; + m_pBuffer = NULL; + } + return pResult; +} + +json_value *CGet::ResultJson() const +{ + unsigned char *pResult = Result(); + if(!pResult) + { + return NULL; + } + return json_parse((char *)pResult, m_BufferLength); +} + +size_t CGet::OnData(char *pData, size_t DataSize) +{ + if(DataSize == 0) + { + return DataSize; + } + bool Reallocate = false; + if(m_BufferSize == 0) + { + m_BufferSize = 1024; + Reallocate = true; + } + while(m_BufferLength + DataSize > m_BufferSize) + { + m_BufferSize *= 2; + Reallocate = true; + } + if(Reallocate) + { + m_pBuffer = (unsigned char *)realloc(m_pBuffer, m_BufferSize); + } + mem_copy(m_pBuffer + m_BufferLength, pData, DataSize); + m_BufferLength += DataSize; + return DataSize; +} + +CGetFile::CGetFile(IStorage *pStorage, const char *pUrl, const char *pDest, int StorageType, bool CanTimeout) : + CRequest(pUrl, CanTimeout), + m_pStorage(pStorage), + m_StorageType(StorageType) +{ + str_copy(m_aDest, pDest, sizeof(m_aDest)); +} + +int CGetFile::BeforeInit() +{ + char aPath[512]; + if(m_StorageType == -2) + m_pStorage->GetBinaryPath(m_aDest, aPath, sizeof(aPath)); + else + m_pStorage->GetCompletePath(m_StorageType, m_aDest, aPath, sizeof(aPath)); + + if(fs_makedir_rec_for(aPath) < 0) + dbg_msg("http", "i/o error, cannot create folder for: %s", aPath); + + m_File = io_open(aPath, IOFLAG_WRITE); + if(!m_File) + { + dbg_msg("http", "i/o error, cannot open file: %s", m_aDest); + return 1; + } + return 0; +} + +size_t CGetFile::OnData(char *pData, size_t DataSize) +{ + return io_write(m_File, pData, DataSize); +} + +void CGetFile::BeforeCompletion() +{ + io_close(m_File); +} + +CPostJson::CPostJson(const char *pUrl, bool CanTimeout, const char *pJson) + : CRequest(pUrl, CanTimeout) +{ + str_copy(m_aJson, pJson, sizeof(m_aJson)); +} + +int CPostJson::AfterInit(void *pCurl) +{ + CURL *pHandle = (CURL *)pCurl; + + curl_slist *pHeaders = NULL; + pHeaders = curl_slist_append(pHeaders, "Content-Type: application/json"); + curl_easy_setopt(pHandle, CURLOPT_HTTPHEADER, pHeaders); + curl_easy_setopt(pHandle, CURLOPT_POSTFIELDS, m_aJson); + + return 0; +} + +static char EscapeJsonChar(char c) +{ + switch(c) + { + case '\"': return '\"'; + case '\\': return '\\'; + case '\b': return 'b'; + case '\n': return 'n'; + case '\r': return 'r'; + case '\t': return 't'; + // Don't escape '\f', who uses that. :) + default: return 0; + } +} + +char *EscapeJson(char *pBuffer, int BufferSize, const char *pString) +{ + dbg_assert(BufferSize > 0, "can't null-terminate the string"); + // Subtract the space for null termination early. + BufferSize--; + + char *pResult = pBuffer; + while(BufferSize && *pString) + { + char c = *pString; + pString++; + char Escaped = EscapeJsonChar(c); + if(Escaped) + { + if(BufferSize < 2) + { + break; + } + *pBuffer++ = '\\'; + *pBuffer++ = Escaped; + BufferSize -= 2; + } + // Assuming ASCII/UTF-8, "if control character". + else if((unsigned char)c < 0x20) + { + // \uXXXX + if(BufferSize < 6) + { + break; + } + str_format(pBuffer, BufferSize, "\\u%04x", c); + pBuffer += 6; + BufferSize -= 6; + } + else + { + *pBuffer++ = c; + BufferSize--; + } + } + *pBuffer = 0; + return pResult; +} + +const char *JsonBool(bool Bool) +{ + if(Bool) + { + return "true"; + } + else + { + return "false"; + } +} diff --git a/src/engine/shared/http.h b/src/engine/shared/http.h new file mode 100644 index 000000000..3c21921fb --- /dev/null +++ b/src/engine/shared/http.h @@ -0,0 +1,109 @@ +#ifndef ENGINE_SHARED_HTTP_H +#define ENGINE_SHARED_HTTP_H + +#include +#include +#include + +typedef struct _json_value json_value; + +enum +{ + HTTP_ERROR = -1, + HTTP_QUEUED, + HTTP_RUNNING, + HTTP_DONE, + HTTP_ABORTED, +}; + +class CRequest : public IJob +{ + // Abort the request with an error if `BeforeInit()` or `AfterInit()` + // returns something nonzero. Also abort the request if `OnData()` + // returns something other than `DataSize`. + virtual int BeforeInit() { return 0; } + virtual int AfterInit(void *pCurl) { return 0; } + virtual size_t OnData(char *pData, size_t DataSize) = 0; + + virtual void OnProgress() { } + virtual void BeforeCompletion() { } + virtual void OnCompletion() { } + + char m_aUrl[256]; + bool m_CanTimeout; + + double m_Size; + double m_Current; + int m_Progress; + + std::atomic m_State; + std::atomic m_Abort; + + static int ProgressCallback(void *pUser, double DlTotal, double DlCurr, double UlTotal, double UlCurr); + static size_t WriteCallback(char *pData, size_t Size, size_t Number, void *pUser); + + void Run(); + +public: + CRequest(const char *pUrl, bool CanTimeout); + + double Current() const { return m_Current; } + double Size() const { return m_Size; } + int Progress() const { return m_Progress; } + int State() const { return m_State; } + void Abort() { m_Abort = true; } +}; + +class CGet : public CRequest +{ + virtual size_t OnData(char *pData, size_t DataSize); + + size_t m_BufferSize; + size_t m_BufferLength; + unsigned char *m_pBuffer; + +public: + CGet(const char *pUrl, bool CanTimeout); + ~CGet(); + + size_t ResultSize() const { if(!Result()) { return 0; } else { return m_BufferSize; } } + unsigned char *Result() const; + unsigned char *TakeResult(); + json_value *ResultJson() const; +}; + +class CGetFile : public CRequest +{ + virtual size_t OnData(char *pData, size_t DataSize); + virtual int BeforeInit(); + virtual void BeforeCompletion(); + + IStorage *m_pStorage; + + char m_aDest[256]; + int m_StorageType; + IOHANDLE m_File; + +public: + CGetFile(IStorage *pStorage, const char *pUrl, const char *pDest, int StorageType = -2, bool CanTimeout = true); + + const char *Dest() const { return m_aDest; } +}; + + +class CPostJson : public CRequest +{ + virtual size_t OnData(char *pData, size_t DataSize) { return DataSize; } + virtual int AfterInit(void *pCurl); + + char m_aJson[1024]; + +public: + CPostJson(const char *pUrl, bool CanTimeout, const char *pJson); +}; + +bool HttpInit(IStorage *pStorage); +void EscapeUrl(char *pBuf, int Size, const char *pStr); +char *EscapeJson(char *pBuffer, int BufferSize, const char *pString); +const char *JsonBool(bool Bool); +#endif // ENGINE_SHARED_HTTP_H diff --git a/src/game/server/ddracechat.cpp b/src/game/server/ddracechat.cpp index 789675139..cd1a30de5 100644 --- a/src/game/server/ddracechat.cpp +++ b/src/game/server/ddracechat.cpp @@ -1458,11 +1458,11 @@ void CGameContext::ConModhelp(IConsole::IResult *pResult, void *pUserData) char aMessage[128]; str_format(aJson, sizeof(aJson), "{\"port\":%d,\"moderator_present\":%s,\"player_id\":%d,\"player_name\":\"%s\",\"message\":\"%s\"}", g_Config.m_SvPort, - ModeratorPresent ? "true" : "false", + JsonBool(ModeratorPresent), pResult->m_ClientID, EscapeJson(aPlayerName, sizeof(aPlayerName), pSelf->Server()->ClientName(pResult->m_ClientID)), EscapeJson(aMessage, sizeof(aMessage), pResult->GetString(0))); - pSelf->Engine()->AddJob(pPlayer->m_pPostJson = std::make_shared(g_Config.m_SvModhelpUrl, aJson)); + pSelf->Engine()->AddJob(pPlayer->m_pPostJson = std::make_shared(g_Config.m_SvModhelpUrl, false, aJson)); } } diff --git a/src/game/server/player.h b/src/game/server/player.h index 5b832022f..91ae139e6 100644 --- a/src/game/server/player.h +++ b/src/game/server/player.h @@ -6,7 +6,7 @@ // this include should perhaps be removed #include "entities/character.h" #include "gamecontext.h" -#include +#include // player object class CPlayer diff --git a/src/game/server/teehistorian.h b/src/game/server/teehistorian.h index fd09c63d7..8f3be6a0f 100644 --- a/src/game/server/teehistorian.h +++ b/src/game/server/teehistorian.h @@ -3,7 +3,7 @@ #include #include -#include +#include #include #include #include diff --git a/src/test/json.cpp b/src/test/json.cpp index b46f6db1c..4dbc6d8a3 100644 --- a/src/test/json.cpp +++ b/src/test/json.cpp @@ -1,6 +1,6 @@ #include -#include +#include TEST(Json, Escape) {