Merge pull request #8099 from heinrich5991/pr_ddnet_http_age

Take serverlist age into account when choosing master

ddnet-libs

@@ -1 +1 @@
-Subproject commit a4fbe0e52338a129bc3ac2e3a1de54de1d175279
+Subproject commit 87fbb57839080f40d16cd77d8ad11ae91bb9c849

src/engine/client/serverbrowser.cpp

@@ -102,7 +102,7 @@ void CServerBrowser::SetBaseInfo(class CNetClient *pClient, const char *pNetVers
 
 void CServerBrowser::OnInit()
 {
-	m_pHttp = CreateServerBrowserHttp(m_pEngine, m_pConsole, m_pStorage, m_pHttpClient, g_Config.m_BrCachedBestServerinfoUrl);
+	m_pHttp = CreateServerBrowserHttp(m_pEngine, m_pStorage, m_pHttpClient, g_Config.m_BrCachedBestServerinfoUrl);
 }
 
 void CServerBrowser::RegisterCommands()

src/engine/client/serverbrowser_http.cpp

@@ -11,6 +11,7 @@
 #include <engine/storage.h>
 
 #include <base/lock.h>
+#include <base/log.h>
 #include <base/system.h>
 
 #include <memory>
@@ -20,6 +21,28 @@
 
 using namespace std::chrono_literals;
 
+static int SanitizeAge(std::optional<int64_t> Age)
+{
+	// A year is of course pi*10**7 seconds.
+	if(!(Age && 0 <= *Age && *Age < 31415927))
+	{
+		return 31415927;
+	}
+	return *Age;
+}
+
+// Classify HTTP responses into buckets, treat 15 seconds as fresh, 1 minute as
+// less fresh, etc. This ensures that differences in the order of seconds do
+// not affect master choice.
+static int ClassifyAge(int AgeSeconds)
+{
+	return 0 //
+	       + (AgeSeconds >= 15) // 15 seconds
+	       + (AgeSeconds >= 60) // 1 minute
+	       + (AgeSeconds >= 300) // 5 minutes
+	       + (AgeSeconds / 3600); // 1 hour
+}
+
 class CChooseMaster
 {
 public:
@@ -184,9 +207,11 @@ void CChooseMaster::CJob::Run()
 	// fail.
 	CTimeout Timeout{10000, 0, 8000, 10};
 	int aTimeMs[MAX_URLS];
+	int aAgeS[MAX_URLS];
 	for(int i = 0; i < m_pData->m_NumUrls; i++)
 	{
 		aTimeMs[i] = -1;
+		aAgeS[i] = SanitizeAge({});
 		const char *pUrl = m_pData->m_aaUrls[aRandomized[i]];
 		std::shared_ptr<CHttpRequest> pHead = HttpHead(pUrl);
 		pHead->Timeout(Timeout);
@@ -200,7 +225,7 @@ void CChooseMaster::CJob::Run()
 		pHead->Wait();
 		if(pHead->State() == EHttpState::ABORTED || State() == IJob::STATE_ABORTED)
 		{
-			dbg_msg("serverbrowse_http", "master chooser aborted");
+			log_debug("serverbrowse_http", "master chooser aborted");
 			return;
 		}
 		if(pHead->State() != EHttpState::DONE)
@@ -223,7 +248,7 @@ void CChooseMaster::CJob::Run()
 		auto Time = std::chrono::duration_cast<std::chrono::milliseconds>(time_get_nanoseconds() - StartTime);
 		if(pGet->State() == EHttpState::ABORTED || State() == IJob::STATE_ABORTED)
 		{
-			dbg_msg("serverbrowse_http", "master chooser aborted");
+			log_debug("serverbrowse_http", "master chooser aborted");
 			return;
 		}
 		if(pGet->State() != EHttpState::DONE)
@@ -242,39 +267,44 @@ void CChooseMaster::CJob::Run()
 		{
 			continue;
 		}
-		dbg_msg("serverbrowse_http", "found master, url='%s' time=%dms", pUrl, (int)Time.count());
+		int AgeS = SanitizeAge(pGet->ResultAgeSeconds());
+		log_info("serverbrowse_http", "found master, url='%s' time=%dms age=%ds", pUrl, (int)Time.count(), AgeS);
+
 		aTimeMs[i] = Time.count();
+		aAgeS[i] = AgeS;
 	}
 
 	// Determine index of the minimum time.
 	int BestIndex = -1;
 	int BestTime = 0;
+	int BestAge = 0;
 	for(int i = 0; i < m_pData->m_NumUrls; i++)
 	{
 		if(aTimeMs[i] < 0)
 		{
 			continue;
 		}
-		if(BestIndex == -1 || aTimeMs[i] < BestTime)
+		if(BestIndex == -1 || std::tuple(ClassifyAge(aAgeS[i]), aTimeMs[i]) < std::tuple(ClassifyAge(BestAge), BestTime))
 		{
 			BestTime = aTimeMs[i];
+			BestAge = aAgeS[i];
 			BestIndex = aRandomized[i];
 		}
 	}
 	if(BestIndex == -1)
 	{
-		dbg_msg("serverbrowse_http", "WARNING: no usable masters found");
+		log_error("serverbrowse_http", "WARNING: no usable masters found");
 		return;
 	}
 
-	dbg_msg("serverbrowse_http", "determined best master, url='%s' time=%dms", m_pData->m_aaUrls[BestIndex], BestTime);
+	log_info("serverbrowse_http", "determined best master, url='%s' time=%dms age=%ds", m_pData->m_aaUrls[BestIndex], BestTime, BestAge);
 	m_pData->m_BestIndex.store(BestIndex);
 }
 
 class CServerBrowserHttp : public IServerBrowserHttp
 {
 public:
-	CServerBrowserHttp(IEngine *pEngine, IConsole *pConsole, IHttp *pHttp, const char **ppUrls, int NumUrls, int PreviousBestIndex);
+	CServerBrowserHttp(IEngine *pEngine, IHttp *pHttp, const char **ppUrls, int NumUrls, int PreviousBestIndex);
 	~CServerBrowserHttp() override;
 	void Update() override;
 	bool IsRefreshing() override { return m_State != STATE_DONE; }
@@ -310,7 +340,6 @@ private:
 	static bool Validate(json_value *pJson);
 	static bool Parse(json_value *pJson, std::vector<CServerInfo> *pvServers, std::vector<NETADDR> *pvLegacyServers);
 
-	IConsole *m_pConsole;
 	IHttp *m_pHttp;
 
 	int m_State = STATE_DONE;
@@ -321,8 +350,7 @@ private:
 	std::vector<NETADDR> m_vLegacyServers;
 };
 
-CServerBrowserHttp::CServerBrowserHttp(IEngine *pEngine, IConsole *pConsole, IHttp *pHttp, const char **ppUrls, int NumUrls, int PreviousBestIndex) :
-	m_pConsole(pConsole),
+CServerBrowserHttp::CServerBrowserHttp(IEngine *pEngine, IHttp *pHttp, const char **ppUrls, int NumUrls, int PreviousBestIndex) :
 	m_pHttp(pHttp),
 	m_pChooseMaster(new CChooseMaster(pEngine, pHttp, Validate, ppUrls, NumUrls, PreviousBestIndex))
 {
@@ -346,7 +374,7 @@ void CServerBrowserHttp::Update()
 		{
 			if(!m_pChooseMaster->IsRefreshing())
 			{
-				m_pConsole->Print(IConsole::OUTPUT_LEVEL_STANDARD, "serverbrowse_http", "no working serverlist URL found");
+				log_error("serverbrowse_http", "no working serverlist URL found");
 				m_State = STATE_NO_MASTER;
 			}
 			return;
@@ -372,12 +400,20 @@ void CServerBrowserHttp::Update()
 		Success = Success && pJson;
 		Success = Success && !Parse(pJson, &m_vServers, &m_vLegacyServers);
 		json_value_free(pJson);
+		int Age = SanitizeAge(pGetServers->ResultAgeSeconds());
 		if(!Success)
 		{
-			m_pConsole->Print(IConsole::OUTPUT_LEVEL_STANDARD, "serverbrowse_http", "failed getting serverlist, trying to find best URL");
+			log_error("serverbrowse_http", "failed getting serverlist, trying to find best URL");
 			m_pChooseMaster->Reset();
 			m_pChooseMaster->Refresh();
 		}
+		// Try to find new master if the current one returns results
+		// that are 5 minutes old.
+		else if(Age > 300)
+		{
+			log_info("serverbrowse_http", "got stale serverlist, age=%ds, trying to find best URL", Age);
+			m_pChooseMaster->Refresh();
+		}
 	}
 }
 void CServerBrowserHttp::Refresh()
@@ -435,7 +471,7 @@ bool CServerBrowserHttp::Parse(json_value *pJson, std::vector<CServerInfo> *pvSe
 		}
 		if(CServerInfo2::FromJson(&ParsedInfo, &Info))
 		{
-			//dbg_msg("dbg/serverbrowser", "skipped due to info, i=%d", i);
+			log_debug("serverbrowser_http", "skipped due to info, i=%d", i);
 			// Only skip the current server on parsing
 			// failure; the server info is "user input" by
 			// the game server and can be set to arbitrary
@@ -455,7 +491,7 @@ bool CServerBrowserHttp::Parse(json_value *pJson, std::vector<CServerInfo> *pvSe
 			NETADDR ParsedAddr;
 			if(ServerbrowserParseUrl(&ParsedAddr, Addresses[a]))
 			{
-				//dbg_msg("dbg/serverbrowser", "unknown address, i=%d a=%d", i, a);
+				log_debug("dbg/serverbrowser", "unknown address, i=%d a=%d", i, a);
 				// Skip unknown addresses.
 				continue;
 			}
@@ -495,7 +531,7 @@ static const char *DEFAULT_SERVERLIST_URLS[] = {
 	"https://master4.ddnet.org/ddnet/15/servers.json",
 };
 
-IServerBrowserHttp *CreateServerBrowserHttp(IEngine *pEngine, IConsole *pConsole, IStorage *pStorage, IHttp *pHttp, const char *pPreviousBestUrl)
+IServerBrowserHttp *CreateServerBrowserHttp(IEngine *pEngine, IStorage *pStorage, IHttp *pHttp, const char *pPreviousBestUrl)
 {
 	char aaUrls[CChooseMaster::MAX_URLS][256];
 	const char *apUrls[CChooseMaster::MAX_URLS] = {0};
@@ -532,5 +568,5 @@ IServerBrowserHttp *CreateServerBrowserHttp(IEngine *pEngine, IConsole *pConsole
 			break;
 		}
 	}
-	return new CServerBrowserHttp(pEngine, pConsole, pHttp, ppUrls, NumUrls, PreviousBestIndex);
+	return new CServerBrowserHttp(pEngine, pHttp, ppUrls, NumUrls, PreviousBestIndex);
 }

src/engine/client/serverbrowser_http.h

@@ -3,7 +3,6 @@
 #include <base/types.h>
 
 class CServerInfo;
-class IConsole;
 class IEngine;
 class IStorage;
 class IHttp;
@@ -26,5 +25,5 @@ public:
 	virtual const NETADDR &LegacyServer(int Index) const = 0;
 };
 
-IServerBrowserHttp *CreateServerBrowserHttp(IEngine *pEngine, IConsole *pConsole, IStorage *pStorage, IHttp *pHttp, const char *pPreviousBestUrl);
+IServerBrowserHttp *CreateServerBrowserHttp(IEngine *pEngine, IStorage *pStorage, IHttp *pHttp, const char *pPreviousBestUrl);
 #endif // ENGINE_CLIENT_SERVERBROWSER_HTTP_H

src/engine/shared/http.cpp

@@ -148,6 +148,8 @@ bool CHttpRequest::ConfigureHandle(void *pHandle)
 	curl_easy_setopt(pH, CURLOPT_USERAGENT, GAME_NAME " " GAME_RELEASE_VERSION " (" CONF_PLATFORM_STRING "; " CONF_ARCH_STRING ")");
 	curl_easy_setopt(pH, CURLOPT_ACCEPT_ENCODING, ""); // Use any compression algorithm supported by libcurl.
 
+	curl_easy_setopt(pH, CURLOPT_HEADERDATA, this);
+	curl_easy_setopt(pH, CURLOPT_HEADERFUNCTION, HeaderCallback);
 	curl_easy_setopt(pH, CURLOPT_WRITEDATA, this);
 	curl_easy_setopt(pH, CURLOPT_WRITEFUNCTION, WriteCallback);
 	curl_easy_setopt(pH, CURLOPT_NOPROGRESS, 0L);
@@ -206,6 +208,52 @@ bool CHttpRequest::ConfigureHandle(void *pHandle)
 	return true;
 }
 
+size_t CHttpRequest::OnHeader(char *pHeader, size_t HeaderSize)
+{
+	// `pHeader` is NOT null-terminated.
+	// `pHeader` has a trailing newline.
+
+	if(HeaderSize <= 1)
+	{
+		m_HeadersEnded = true;
+		return HeaderSize;
+	}
+	if(m_HeadersEnded)
+	{
+		// redirect, clear old headers
+		m_HeadersEnded = false;
+		m_ResultDate = {};
+		m_ResultLastModified = {};
+	}
+
+	static const char DATE[] = "Date: ";
+	static const char LAST_MODIFIED[] = "Last-Modified: ";
+
+	// Trailing newline and null termination evens out.
+	if(HeaderSize - 1 >= sizeof(DATE) - 1 && str_startswith_nocase(pHeader, DATE))
+	{
+		char aValue[128];
+		str_truncate(aValue, sizeof(aValue), pHeader + (sizeof(DATE) - 1), HeaderSize - (sizeof(DATE) - 1) - 1);
+		int64_t Value = curl_getdate(aValue, nullptr);
+		if(Value != -1)
+		{
+			m_ResultDate = Value;
+		}
+	}
+	if(HeaderSize - 1 >= sizeof(LAST_MODIFIED) - 1 && str_startswith_nocase(pHeader, LAST_MODIFIED))
+	{
+		char aValue[128];
+		str_truncate(aValue, sizeof(aValue), pHeader + (sizeof(LAST_MODIFIED) - 1), HeaderSize - (sizeof(LAST_MODIFIED) - 1) - 1);
+		int64_t Value = curl_getdate(aValue, nullptr);
+		if(Value != -1)
+		{
+			m_ResultLastModified = Value;
+		}
+	}
+
+	return HeaderSize;
+}
+
 size_t CHttpRequest::OnData(char *pData, size_t DataSize)
 {
 	// Need to check for the maximum response size here as curl can only
@@ -244,6 +292,12 @@ size_t CHttpRequest::OnData(char *pData, size_t DataSize)
 	}
 }
 
+size_t CHttpRequest::HeaderCallback(char *pData, size_t Size, size_t Number, void *pUser)
+{
+	dbg_assert(Size == 1, "invalid size parameter passed to header callback");
+	return ((CHttpRequest *)pUser)->OnHeader(pData, Number);
+}
+
 size_t CHttpRequest::WriteCallback(char *pData, size_t Size, size_t Number, void *pUser)
 {
 	return ((CHttpRequest *)pUser)->OnData(pData, Size * Number);
@@ -390,6 +444,22 @@ int CHttpRequest::StatusCode() const
 	return m_StatusCode;
 }
 
+std::optional<int64_t> CHttpRequest::ResultAgeSeconds() const
+{
+	dbg_assert(State() == EHttpState::DONE, "Request not done");
+	if(!m_ResultDate || !m_ResultLastModified)
+	{
+		return {};
+	}
+	return *m_ResultDate - *m_ResultLastModified;
+}
+
+std::optional<int64_t> CHttpRequest::ResultLastModified() const
+{
+	dbg_assert(State() == EHttpState::DONE, "Request not done");
+	return m_ResultLastModified;
+}
+
 bool CHttp::Init(std::chrono::milliseconds ShutdownDelay)
 {
 	m_ShutdownDelay = ShutdownDelay;

src/engine/shared/http.h

@@ -118,6 +118,9 @@ class CHttpRequest : public IHttpRequest
 	std::atomic<bool> m_Abort{false};
 
 	int m_StatusCode = 0;
+	bool m_HeadersEnded = false;
+	std::optional<int64_t> m_ResultDate = {};
+	std::optional<int64_t> m_ResultLastModified = {};
 
 	// Abort the request with an error if `BeforeInit()` returns false.
 	bool BeforeInit();
@@ -125,11 +128,15 @@ class CHttpRequest : public IHttpRequest
 	// `pHandle` can be nullptr if no handle was ever created for this request.
 	void OnCompletionInternal(void *pHandle, unsigned int Result); // void * == CURL *, unsigned int == CURLcode
 
+	// Abort the request if `OnHeader()` returns something other than
+	// `DataSize`. `pHeader` is NOT null-terminated.
+	size_t OnHeader(char *pHeader, size_t HeaderSize);
 	// Abort the request if `OnData()` returns something other than
 	// `DataSize`.
 	size_t OnData(char *pData, size_t DataSize);
 
 	static int ProgressCallback(void *pUser, double DlTotal, double DlCurr, double UlTotal, double UlCurr);
+	static size_t HeaderCallback(char *pData, size_t Size, size_t Number, void *pUser);
 	static size_t WriteCallback(char *pData, size_t Size, size_t Number, void *pUser);
 
 protected:
@@ -207,6 +214,8 @@ public:
 	const SHA256_DIGEST &ResultSha256() const;
 
 	int StatusCode() const;
+	std::optional<int64_t> ResultAgeSeconds() const;
+	std::optional<int64_t> ResultLastModified() const;
 };
 
 inline std::unique_ptr<CHttpRequest> HttpHead(const char *pUrl)