Merge #5829

5829: Add HTTPS map download URL field for game servers r=def- a=heinrich5991

This allows every game server to provide its own HTTPS server for map
downloads. Since the ingame protocol for downloading map data is very
inefficient, this is desirable. Previously, only servers hosted by DDNet
could benefit from this.

Security concerns:
- Attackers can find out whether a given HTTPS GET request matches a
  known answer.

  This isn't deemed to be problematic as no cookies for authentication
  are sent and only the whole response can be matched.

- Sending requests to honeypot URLs to get people in legal trouble.

  This seems to be already possible with HTML image embeds, so it can't
  be that bad™.

- Downloading huge files, filling up a player's disk. The players might
  cancel when seeing huge files.

  There's a generous limit of 1 GiB per map file.

- Downloading huge files transparently compressed with gzip. See above.

Fixes #5812.

## Checklist

- [x] Tested the change ingame
- [ ] Provided screenshots if it is a visual change
- [ ] Tested in combination with possibly related configuration options
- [ ] Written a unit test (especially base/) or added coverage to integration test
- [ ] Considered possible null pointers and out of bounds array indexing
- [ ] Changed no physics that affect existing maps
- [ ] Tested the change with [ASan+UBSan or valgrind's memcheck](https://github.com/ddnet/ddnet/#using-addresssanitizer--undefinedbehavioursanitizer-or-valgrinds-memcheck) (optional)


Co-authored-by: heinrich5991 <heinrich5991@gmail.com>

src/engine/client/client.cpp

@@ -359,6 +359,7 @@ CClient::CClient() :
 	m_aMapDetailsName[0] = 0;
 	m_MapDetailsSha256 = SHA256_ZEROED;
 	m_MapDetailsCrc = 0;
+	m_aMapDetailsUrl[0] = 0;
 
 	IStorage::FormatTmpPath(m_aDDNetInfoTmp, sizeof(m_aDDNetInfoTmp), DDNET_INFO);
 	m_pDDNetInfoTask = NULL;
@@ -1658,16 +1659,25 @@ void CClient::ProcessServerPacket(CNetChunk *pPacket, int Conn, bool Dummy)
 			const char *pMap = Unpacker.GetString(CUnpacker::SANITIZE_CC | CUnpacker::SKIP_START_WHITESPACES);
 			SHA256_DIGEST *pMapSha256 = (SHA256_DIGEST *)Unpacker.GetRaw(sizeof(*pMapSha256));
 			int MapCrc = Unpacker.GetInt();
+			int MapSize = Unpacker.GetInt();
 
 			if(Unpacker.Error())
 			{
 				return;
 			}
 
+			const char *pMapUrl = Unpacker.GetString(CUnpacker::SANITIZE_CC);
+			if(Unpacker.Error())
+			{
+				pMapUrl = "";
+			}
+
 			m_MapDetailsPresent = true;
+			(void)MapSize;
 			str_copy(m_aMapDetailsName, pMap);
 			m_MapDetailsSha256 = *pMapSha256;
 			m_MapDetailsCrc = MapCrc;
+			str_copy(m_aMapDetailsUrl, pMapUrl);
 		}
 		else if(Conn == CONN_MAIN && (pPacket->m_Flags & NET_CHUNKFLAG_VITAL) != 0 && Msg == NETMSG_CAPABILITIES)
 		{
@@ -1720,9 +1730,11 @@ void CClient::ProcessServerPacket(CNetChunk *pPacket, int Conn, bool Dummy)
 			else
 			{
 				SHA256_DIGEST *pMapSha256 = 0;
+				const char *pMapUrl = nullptr;
 				if(MapDetailsWerePresent && str_comp(m_aMapDetailsName, pMap) == 0 && m_MapDetailsCrc == MapCrc)
 				{
 					pMapSha256 = &m_MapDetailsSha256;
+					pMapUrl = m_aMapDetailsUrl[0] ? m_aMapDetailsUrl : nullptr;
 				}
 				pError = LoadMapSearch(pMap, pMapSha256, MapCrc);
 
@@ -1767,8 +1779,9 @@ void CClient::ProcessServerPacket(CNetChunk *pPacket, int Conn, bool Dummy)
 						bool UseConfigUrl = str_comp(g_Config.m_ClMapDownloadUrl, "https://maps.ddnet.org") != 0 || m_aMapDownloadUrl[0] == '\0';
 						str_format(aUrl, sizeof(aUrl), "%s/%s", UseConfigUrl ? g_Config.m_ClMapDownloadUrl : m_aMapDownloadUrl, aEscaped);
 
-						m_pMapdownloadTask = HttpGetFile(aUrl, Storage(), m_aMapdownloadFilenameTemp, IStorage::TYPE_SAVE);
+						m_pMapdownloadTask = HttpGetFile(pMapUrl ? pMapUrl : aUrl, Storage(), m_aMapdownloadFilenameTemp, IStorage::TYPE_SAVE);
 						m_pMapdownloadTask->Timeout(CTimeout{g_Config.m_ClMapDownloadConnectTimeoutMs, 0, g_Config.m_ClMapDownloadLowSpeedLimit, g_Config.m_ClMapDownloadLowSpeedTime});
+						m_pMapdownloadTask->MaxResponseSize(1024 * 1024 * 1024); // 1 GiB
 						Engine()->AddJob(m_pMapdownloadTask);
 					}
 					else
@@ -2891,16 +2904,12 @@ void CClient::Update()
 	{
 		if(m_pMapdownloadTask->State() == HTTP_DONE)
 			FinishMapDownload();
-		else if(m_pMapdownloadTask->State() == HTTP_ERROR)
+		else if(m_pMapdownloadTask->State() == HTTP_ERROR || m_pMapdownloadTask->State() == HTTP_ABORTED)
 		{
 			dbg_msg("webdl", "http failed, falling back to gameserver");
 			ResetMapDownload();
 			SendMapRequest();
 		}
-		else if(m_pMapdownloadTask->State() == HTTP_ABORTED)
-		{
-			m_pMapdownloadTask = NULL;
-		}
 	}
 
 	if(m_pDDNetInfoTask)

src/engine/client/client.h

@@ -203,6 +203,7 @@ class CClient : public IClient, public CDemoPlayer::IListener
 	char m_aMapDetailsName[256];
 	int m_MapDetailsCrc;
 	SHA256_DIGEST m_MapDetailsSha256;
+	char m_aMapDetailsUrl[256];
 
 	char m_aDDNetInfoTmp[64];
 	std::shared_ptr<CHttpRequest> m_pDDNetInfoTask;

src/engine/server/server.cpp

@@ -1200,6 +1200,7 @@ void CServer::SendMap(int ClientID)
 		Msg.AddRaw(&m_aCurrentMapSha256[MapType].data, sizeof(m_aCurrentMapSha256[MapType].data));
 		Msg.AddInt(m_aCurrentMapCrc[MapType]);
 		Msg.AddInt(m_aCurrentMapSize[MapType]);
+		Msg.AddString("", 0); // HTTPS map download URL
 		SendMsg(&Msg, MSGFLAG_VITAL, ClientID);
 	}
 	{

src/engine/shared/config_variables.h

@@ -201,6 +201,8 @@ MACRO_CONFIG_INT(DbgStressNetwork, dbg_stress_network, 0, 0, 0, CFGFLAG_CLIENT |
 MACRO_CONFIG_STR(DbgStressServer, dbg_stress_server, 32, "localhost", CFGFLAG_CLIENT, "Server to stress (Debug build only)")
 #endif
 
+MACRO_CONFIG_INT(HttpAllowInsecure, http_allow_insecure, 0, 0, 1, CFGFLAG_CLIENT | CFGFLAG_SERVER, "Allow insecure HTTP protocol in addition to the secure HTTPS one. Mostly useful for testing.")
+
 // DDRace
 MACRO_CONFIG_STR(SvWelcome, sv_welcome, 64, "", CFGFLAG_SERVER, "Message that will be displayed to players who join the server")
 MACRO_CONFIG_INT(SvReservedSlots, sv_reserved_slots, 0, 0, MAX_CLIENTS, CFGFLAG_SERVER, "The number of slots that are reserved for special players")

src/engine/shared/http.cpp

@@ -123,10 +123,10 @@ CHttpRequest::CHttpRequest(const char *pUrl)
 
 CHttpRequest::~CHttpRequest()
 {
+	m_ResponseLength = 0;
 	if(!m_WriteToFile)
 	{
 		m_BufferSize = 0;
-		m_BufferLength = 0;
 		free(m_pBuffer);
 		m_pBuffer = nullptr;
 	}
@@ -191,6 +191,11 @@ int CHttpRequest::RunImpl(CURL *pUser)
 		curl_easy_setopt(pHandle, CURLOPT_VERBOSE, 1L);
 		curl_easy_setopt(pHandle, CURLOPT_DEBUGFUNCTION, CurlDebug);
 	}
+	long Protocols = CURLPROTO_HTTPS;
+	if(g_Config.m_HttpAllowInsecure)
+	{
+		Protocols |= CURLPROTO_HTTP;
+	}
 	char aErr[CURL_ERROR_SIZE];
 	curl_easy_setopt(pHandle, CURLOPT_ERRORBUFFER, aErr);
 
@@ -198,9 +203,13 @@ int CHttpRequest::RunImpl(CURL *pUser)
 	curl_easy_setopt(pHandle, CURLOPT_TIMEOUT_MS, m_Timeout.TimeoutMs);
 	curl_easy_setopt(pHandle, CURLOPT_LOW_SPEED_LIMIT, m_Timeout.LowSpeedLimit);
 	curl_easy_setopt(pHandle, CURLOPT_LOW_SPEED_TIME, m_Timeout.LowSpeedTime);
+	if(m_MaxResponseSize >= 0)
+	{
+		curl_easy_setopt(pHandle, CURLOPT_MAXFILESIZE_LARGE, (curl_off_t)m_MaxResponseSize);
+	}
 
 	curl_easy_setopt(pHandle, CURLOPT_SHARE, gs_pShare);
-	curl_easy_setopt(pHandle, CURLOPT_PROTOCOLS, CURLPROTO_HTTP | CURLPROTO_HTTPS);
+	curl_easy_setopt(pHandle, CURLOPT_PROTOCOLS, Protocols);
 	curl_easy_setopt(pHandle, CURLOPT_FOLLOWLOCATION, 1L);
 	curl_easy_setopt(pHandle, CURLOPT_MAXREDIRS, 4L);
 	curl_easy_setopt(pHandle, CURLOPT_FAILONERROR, 1L);
@@ -275,6 +284,12 @@ int CHttpRequest::RunImpl(CURL *pUser)
 
 size_t CHttpRequest::OnData(char *pData, size_t DataSize)
 {
+	// Need to check for the maximum response size here as curl can only
+	// guarantee it if the server sets a Content-Length header.
+	if(m_MaxResponseSize >= 0 && m_ResponseLength + DataSize > (uint64_t)m_MaxResponseSize)
+	{
+		return 0;
+	}
 	if(!m_WriteToFile)
 	{
 		if(DataSize == 0)
@@ -282,7 +297,7 @@ size_t CHttpRequest::OnData(char *pData, size_t DataSize)
 			return DataSize;
 		}
 		size_t NewBufferSize = maximum((size_t)1024, m_BufferSize);
-		while(m_BufferLength + DataSize > NewBufferSize)
+		while(m_ResponseLength + DataSize > NewBufferSize)
 		{
 			NewBufferSize *= 2;
 		}
@@ -291,12 +306,13 @@ size_t CHttpRequest::OnData(char *pData, size_t DataSize)
 			m_pBuffer = (unsigned char *)realloc(m_pBuffer, NewBufferSize);
 			m_BufferSize = NewBufferSize;
 		}
-		mem_copy(m_pBuffer + m_BufferLength, pData, DataSize);
-		m_BufferLength += DataSize;
+		mem_copy(m_pBuffer + m_ResponseLength, pData, DataSize);
+		m_ResponseLength += DataSize;
 		return DataSize;
 	}
 	else
 	{
+		m_ResponseLength += DataSize;
 		return io_write(m_File, pData, DataSize);
 	}
 }
@@ -362,7 +378,7 @@ void CHttpRequest::Result(unsigned char **ppResult, size_t *pResultLength) const
 		return;
 	}
 	*ppResult = m_pBuffer;
-	*pResultLength = m_BufferLength;
+	*pResultLength = m_ResponseLength;
 }
 
 json_value *CHttpRequest::ResultJson() const

src/engine/shared/http.h

@@ -55,13 +55,15 @@ class CHttpRequest : public IJob
 	size_t m_BodyLength = 0;
 
 	CTimeout m_Timeout = CTimeout{0, 0, 0, 0};
+	int64_t m_MaxResponseSize = -1;
 	REQUEST m_Type = REQUEST::GET;
 
 	bool m_WriteToFile = false;
 
+	uint64_t m_ResponseLength = 0;
+
 	// If `m_WriteToFile` is false.
 	size_t m_BufferSize = 0;
-	size_t m_BufferLength = 0;
 	unsigned char *m_pBuffer = nullptr;
 
 	// If `m_WriteToFile` is true.
@@ -99,6 +101,7 @@ public:
 	~CHttpRequest();
 
 	void Timeout(CTimeout Timeout) { m_Timeout = Timeout; }
+	void MaxResponseSize(int64_t MaxResponseSize) { m_MaxResponseSize = MaxResponseSize; }
 	void LogProgress(HTTPLOG LogProgress) { m_LogProgress = LogProgress; }
 	void IpResolve(IPRESOLVE IpResolve) { m_IpResolve = IpResolve; }
 	void WriteToFile(IStorage *pStorage, const char *pDest, int StorageType);