edg-l/ddnet
1
0
Fork 0
mirror of https://github.com/ddnet/ddnet.git synced 2024-11-10 01:58:19 +00:00
Code Issues Packages Projects Releases Wiki Activity

Write DDNet info file only when it changed

Browse source 
Avoid many writes to disk each time the DDNet info is downloaded, i.e. each time the refresh button is pressed in the server browser, by first loading the DDNet info into memory and only writing it to disk when it differs from the current DDNet info based on the SHA256 hash.

The SHA256 is now also used to track whether the DDNet info was modified for the community filters and icons instead of using the current time for this purpose.

Closes #3941.
This commit is contained in:
Robert Müller 2024-02-04 17:09:37 +01:00
parent 535400d8a8
commit f0deb129c0
9 changed files with 72 additions and 84 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

103
src/engine/client/client.cpp
View file

@ -84,7 +84,6 @@ CClient::CClient() :
for(auto &DemoRecorder : m_aDemoRecorder)
DemoRecorder = CDemoRecorder(&m_SnapshotDelta);
m_LastRenderTime = time_get();
IStorage::FormatTmpPath(m_aDDNetInfoTmp, sizeof(m_aDDNetInfoTmp), DDNET_INFO_FILE);
mem_zero(m_aInputs, sizeof(m_aInputs));
mem_zero(m_aapSnapshots, sizeof(m_aapSnapshots));
for(auto &SnapshotStorage : m_aSnapshotStorage)
@ -2054,7 +2053,7 @@ void CClient::FinishMapDownload()
}
}
void CClient::ResetDDNetInfo()
void CClient::ResetDDNetInfoTask()
{
if(m_pDDNetInfoTask)
{
@ -2063,56 +2062,50 @@ void CClient::ResetDDNetInfo()
}
}
bool CClient::IsDDNetInfoChanged()
{
IOHANDLE OldFile = m_pStorage->OpenFile(DDNET_INFO_FILE, IOFLAG_READ | IOFLAG_SKIP_BOM, IStorage::TYPE_SAVE);
if(!OldFile)
return true;
IOHANDLE NewFile = m_pStorage->OpenFile(m_aDDNetInfoTmp, IOFLAG_READ | IOFLAG_SKIP_BOM, IStorage::TYPE_SAVE);
if(NewFile)
{
char aOldData[4096];
char aNewData[4096];
unsigned OldBytes;
unsigned NewBytes;
do
{
OldBytes = io_read(OldFile, aOldData, sizeof(aOldData));
NewBytes = io_read(NewFile, aNewData, sizeof(aNewData));
if(OldBytes != NewBytes || mem_comp(aOldData, aNewData, OldBytes) != 0)
{
io_close(NewFile);
io_close(OldFile);
return true;
}
} while(OldBytes > 0);
io_close(NewFile);
}
io_close(OldFile);
return false;
}
void CClient::FinishDDNetInfo()
{
ResetDDNetInfo();
if(IsDDNetInfoChanged())
if(m_ServerBrowser.DDNetInfoSha256() == m_pDDNetInfoTask->Sha256())
{
m_pStorage->RenameFile(m_aDDNetInfoTmp, DDNET_INFO_FILE, IStorage::TYPE_SAVE);
LoadDDNetInfo();
if(m_ServerBrowser.GetCurrentType() == IServerBrowser::TYPE_INTERNET || m_ServerBrowser.GetCurrentType() == IServerBrowser::TYPE_FAVORITES)
m_ServerBrowser.Refresh(m_ServerBrowser.GetCurrentType());
log_debug("client/info", "DDNet info already up-to-date");
return;
}
else
char aTempFilename[IO_MAX_PATH_LENGTH];
IStorage::FormatTmpPath(aTempFilename, sizeof(aTempFilename), DDNET_INFO_FILE);
IOHANDLE File = Storage()->OpenFile(aTempFilename, IOFLAG_WRITE, IStorage::TYPE_SAVE);
if(!File)
{
m_pStorage->RemoveFile(m_aDDNetInfoTmp, IStorage::TYPE_SAVE);
log_error("client/info", "Failed to open temporary DDNet info '%s' for writing", aTempFilename);
return;
}
unsigned char *pResult;
size_t ResultLength;
m_pDDNetInfoTask->Result(&pResult, &ResultLength);
dbg_assert(pResult != nullptr, "Invalid info task state");
bool Error = io_write(File, pResult, ResultLength) != ResultLength;
Error |= io_close(File) != 0;
if(Error)
{
log_error("client/info", "Error writing temporary DDNet info to file '%s'", aTempFilename);
return;
}
if(Storage()->FileExists(DDNET_INFO_FILE, IStorage::TYPE_SAVE) && !Storage()->RemoveFile(DDNET_INFO_FILE, IStorage::TYPE_SAVE))
{
log_error("client/info", "Failed to remove old DDNet info '%s'", DDNET_INFO_FILE);
Storage()->RemoveFile(aTempFilename, IStorage::TYPE_SAVE);
return;
}
if(!Storage()->RenameFile(aTempFilename, DDNET_INFO_FILE, IStorage::TYPE_SAVE))
{
log_error("client/info", "Failed to rename temporary DDNet info '%s' to '%s'", aTempFilename, DDNET_INFO_FILE);
Storage()->RemoveFile(aTempFilename, IStorage::TYPE_SAVE);
return;
}
log_debug("client/info", "Loading new DDNet info");
LoadDDNetInfo();
}
typedef std::tuple<int, int, int> TVersion;
@ -2590,16 +2583,13 @@ void CClient::Update()
if(m_pDDNetInfoTask)
{
if(m_pDDNetInfoTask->State() == EHttpState::DONE)
{
FinishDDNetInfo();
else if(m_pDDNetInfoTask->State() == EHttpState::ERROR)
{
Storage()->RemoveFile(m_aDDNetInfoTmp, IStorage::TYPE_SAVE);
ResetDDNetInfo();
ResetDDNetInfoTask();
}
else if(m_pDDNetInfoTask->State() == EHttpState::ABORTED)
else if(m_pDDNetInfoTask->State() == EHttpState::ERROR || m_pDDNetInfoTask->State() == EHttpState::ABORTED)
{
Storage()->RemoveFile(m_aDDNetInfoTmp, IStorage::TYPE_SAVE);
m_pDDNetInfoTask = NULL;
ResetDDNetInfoTask();
}
}
@ -3008,11 +2998,6 @@ void CClient::Run()
s_SavedConfig = true;
}
if(m_pStorage->FileExists(m_aDDNetInfoTmp, IStorage::TYPE_SAVE))
{
m_pStorage->RemoveFile(m_aDDNetInfoTmp, IStorage::TYPE_SAVE);
}
if(m_vWarnings.empty() && !GameClient()->IsDisplayingWarning())
break;
}
@ -4578,7 +4563,7 @@ void CClient::RequestDDNetInfo()
}
// Use ipv4 so we can know the ingame ip addresses of players before they join game servers
m_pDDNetInfoTask = HttpGetFile(aUrl, Storage(), m_aDDNetInfoTmp, IStorage::TYPE_SAVE);
m_pDDNetInfoTask = HttpGet(aUrl);
m_pDDNetInfoTask->Timeout(CTimeout{10000, 0, 500, 10});
m_pDDNetInfoTask->IpResolve(IPRESOLVE::V4);
Http()->Run(m_pDDNetInfoTask);

4
src/engine/client/client.h
View file

@ -157,7 +157,6 @@ class CClient : public IClient, public CDemoPlayer::IListener
SHA256_DIGEST m_MapDetailsSha256 = SHA256_ZEROED;
char m_aMapDetailsUrl[256] = "";
char m_aDDNetInfoTmp[64];
std::shared_ptr<CHttpRequest> m_pDDNetInfoTask = nullptr;
// time
@ -353,8 +352,7 @@ public:
void FinishMapDownload();
void RequestDDNetInfo() override;
void ResetDDNetInfo();
bool IsDDNetInfoChanged();
void ResetDDNetInfoTask();
void FinishDDNetInfo();
void LoadDDNetInfo();

9
src/engine/client/serverbrowser.cpp
View file

@ -66,9 +66,6 @@ CServerBrowser::CServerBrowser() :
m_BroadcastTime = 0;
secure_random_fill(m_aTokenSeed, sizeof(m_aTokenSeed));
m_pDDNetInfo = nullptr;
m_DDNetInfoUpdateTime = 0;
CleanUp();
}
@ -1272,7 +1269,6 @@ const json_value *CServerBrowser::LoadDDNetInfo()
UpdateServerCommunity(&m_ppServerlist[i]->m_Info);
UpdateServerRank(&m_ppServerlist[i]->m_Info);
}
m_DDNetInfoUpdateTime = time_get();
return m_pDDNetInfo;
}
@ -1281,7 +1277,12 @@ void CServerBrowser::LoadDDNetInfoJson()
void *pBuf;
unsigned Length;
if(!m_pStorage->ReadFile(DDNET_INFO_FILE, IStorage::TYPE_SAVE, &pBuf, &Length))
{
m_DDNetInfoSha256 = SHA256_ZEROED;
return;
}
m_DDNetInfoSha256 = sha256(pBuf, Length);
json_value_free(m_pDDNetInfo);
json_settings JsonSettings{};

7
src/engine/client/serverbrowser.h
View file

@ -3,6 +3,7 @@
#ifndef ENGINE_CLIENT_SERVERBROWSER_H
#define ENGINE_CLIENT_SERVERBROWSER_H
#include <base/hash.h>
#include <base/system.h>
#include <engine/console.h>
@ -252,7 +253,7 @@ public:
unsigned CurrentCommunitiesHash() const override;
bool DDNetInfoAvailable() const override { return m_pDDNetInfo != nullptr; }
int64_t DDNetInfoUpdateTime() const override { return m_DDNetInfoUpdateTime; }
SHA256_DIGEST DDNetInfoSha256() const override { return m_DDNetInfoSha256; }
CFavoriteCommunityFilterList &FavoriteCommunitiesFilter() override { return m_FavoriteCommunitiesFilter; }
CExcludedCommunityFilterList &CommunitiesFilter() override { return m_CommunitiesFilter; }
@ -311,8 +312,8 @@ private:
CExcludedCommunityCountryFilterList m_CountriesFilter;
CExcludedCommunityTypeFilterList m_TypesFilter;
json_value *m_pDDNetInfo;
int64_t m_DDNetInfoUpdateTime;
json_value *m_pDDNetInfo = nullptr;
SHA256_DIGEST m_DDNetInfoSha256 = SHA256_ZEROED;
CServerEntry *m_pFirstReqServer; // request list
CServerEntry *m_pLastReqServer;

2
src/engine/serverbrowser.h
View file

@ -306,7 +306,7 @@ public:
virtual unsigned CurrentCommunitiesHash() const = 0;
virtual bool DDNetInfoAvailable() const = 0;
virtual int64_t DDNetInfoUpdateTime() const = 0;
virtual SHA256_DIGEST DDNetInfoSha256() const = 0;
virtual IFilterList &FavoriteCommunitiesFilter() = 0;
virtual IFilterList &CommunitiesFilter() = 0;

12
src/engine/shared/http.cpp
View file

@ -67,7 +67,7 @@ bool HttpHasIpresolveBug()
CHttpRequest::CHttpRequest(const char *pUrl)
{
str_copy(m_aUrl, pUrl);
sha256_init(&m_ActualSha256);
sha256_init(&m_ActualSha256Ctx);
}
CHttpRequest::~CHttpRequest()
@ -213,7 +213,7 @@ size_t CHttpRequest::OnData(char *pData, size_t DataSize)
return 0;
}
sha256_update(&m_ActualSha256, pData, DataSize);
sha256_update(&m_ActualSha256Ctx, pData, DataSize);
if(!m_WriteToFile)
{
@ -286,15 +286,15 @@ void CHttpRequest::OnCompletionInternal(std::optional<unsigned int> Result)
State = EHttpState::ERROR;
}
if(State == EHttpState::DONE && m_ExpectedSha256 != SHA256_ZEROED)
if(State == EHttpState::DONE)
{
const SHA256_DIGEST ActualSha256 = sha256_finish(&m_ActualSha256);
if(ActualSha256 != m_ExpectedSha256)
m_ActualSha256 = sha256_finish(&m_ActualSha256Ctx);
if(m_ExpectedSha256 != SHA256_ZEROED && m_ActualSha256 != m_ExpectedSha256)
{
if(g_Config.m_DbgCurl || m_LogProgress >= HTTPLOG::FAILURE)
{
char aActualSha256[SHA256_MAXSTRSIZE];
sha256_str(ActualSha256, aActualSha256, sizeof(aActualSha256));
sha256_str(m_ActualSha256, aActualSha256, sizeof(aActualSha256));
char aExpectedSha256[SHA256_MAXSTRSIZE];
sha256_str(m_ExpectedSha256, aExpectedSha256, sizeof(aExpectedSha256));
log_error("http", "SHA256 mismatch: got=%s, expected=%s, url=%s", aActualSha256, aExpectedSha256, m_aUrl);

5
src/engine/shared/http.h
View file

@ -88,7 +88,8 @@ class CHttpRequest : public IHttpRequest
int64_t m_MaxResponseSize = -1;
REQUEST m_Type = REQUEST::GET;
SHA256_CTX m_ActualSha256;
SHA256_DIGEST m_ActualSha256 = SHA256_ZEROED;
SHA256_CTX m_ActualSha256Ctx;
SHA256_DIGEST m_ExpectedSha256 = SHA256_ZEROED;
bool m_WriteToFile = false;
@ -197,6 +198,8 @@ public:
void Result(unsigned char **ppResult, size_t *pResultLength) const;
json_value *ResultJson() const;
const SHA256_DIGEST &Sha256() const { return m_ActualSha256; }
};
inline std::unique_ptr<CHttpRequest> HttpHead(const char *pUrl)

4
src/game/client/components/menus.h
View file

@ -506,7 +506,7 @@ protected:
static void ConchainUiPageUpdate(IConsole::IResult *pResult, void *pUserData, IConsole::FCommandCallback pfnCallback, void *pCallbackUserData);
struct SCommunityCache
{
int64_t m_UpdateTime = 0;
SHA256_DIGEST m_InfoSha256 = SHA256_ZEROED;
int m_LastPage = 0;
unsigned m_SelectedCommunitiesHash;
std::vector<const CCommunity *> m_vpSelectedCommunities;
@ -560,7 +560,7 @@ protected:
std::vector<SCommunityIcon> m_vCommunityIcons;
std::deque<std::shared_ptr<CCommunityIconLoadJob>> m_CommunityIconLoadJobs;
std::deque<std::shared_ptr<CCommunityIconDownloadJob>> m_CommunityIconDownloadJobs;
int64_t m_CommunityIconsUpdateTime = 0;
SHA256_DIGEST m_CommunityIconsInfoSha256 = SHA256_ZEROED;
static int CommunityIconScan(const char *pName, int IsDir, int DirType, void *pUser);
const SCommunityIcon *FindCommunityIcon(const char *pCommunityId);
bool LoadCommunityIconFile(const char *pPath, int DirType, CImageInfo &Info, SHA256_DIGEST &Sha256);

10
src/game/client/components/menus_browser.cpp
View file

@ -1840,8 +1840,8 @@ void CMenus::UpdateCommunityCache(bool Force)
ServerBrowser()->Refresh(g_Config.m_UiPage - PAGE_FAVORITE_COMMUNITY_1 + IServerBrowser::TYPE_FAVORITE_COMMUNITY_1, true);
}
if(!Force && m_CommunityCache.m_UpdateTime != 0 &&
m_CommunityCache.m_UpdateTime == ServerBrowser()->DDNetInfoUpdateTime() &&
if(!Force && m_CommunityCache.m_InfoSha256 != SHA256_ZEROED &&
m_CommunityCache.m_InfoSha256 == ServerBrowser()->DDNetInfoSha256() &&
!CurrentCommunitiesChanged && !PageChanged)
{
return;
@ -1849,7 +1849,7 @@ void CMenus::UpdateCommunityCache(bool Force)
ServerBrowser()->CleanFilters();
m_CommunityCache.m_UpdateTime = ServerBrowser()->DDNetInfoUpdateTime();
m_CommunityCache.m_InfoSha256 = ServerBrowser()->DDNetInfoSha256();
m_CommunityCache.m_LastPage = g_Config.m_UiPage;
m_CommunityCache.m_SelectedCommunitiesHash = CommunitiesHash;
m_CommunityCache.m_vpSelectedCommunities = ServerBrowser()->CurrentCommunities();
@ -2055,9 +2055,9 @@ void CMenus::UpdateCommunityIcons()
}
// Rescan for changed communities only when necessary
if(!ServerBrowser()->DDNetInfoAvailable() || (m_CommunityIconsUpdateTime != 0 && m_CommunityIconsUpdateTime == ServerBrowser()->DDNetInfoUpdateTime()))
if(!ServerBrowser()->DDNetInfoAvailable() || (m_CommunityIconsInfoSha256 != SHA256_ZEROED && m_CommunityIconsInfoSha256 == ServerBrowser()->DDNetInfoSha256()))
return;
m_CommunityIconsUpdateTime = ServerBrowser()->DDNetInfoUpdateTime();
m_CommunityIconsInfoSha256 = ServerBrowser()->DDNetInfoSha256();
// Remove icons for removed communities
auto RemovalIterator = m_vCommunityIcons.begin();