2018-07-11 20:46:04 +00:00
|
|
|
#include "serverbrowser_http.h"
|
|
|
|
|
2021-05-13 00:46:28 +00:00
|
|
|
#include <engine/console.h>
|
2018-07-11 20:46:04 +00:00
|
|
|
#include <engine/engine.h>
|
|
|
|
#include <engine/external/json-parser/json.h>
|
|
|
|
#include <engine/serverbrowser.h>
|
2021-12-24 11:17:46 +00:00
|
|
|
#include <engine/shared/http.h>
|
2021-06-23 07:50:23 +00:00
|
|
|
#include <engine/shared/jobs.h>
|
2021-05-13 00:46:28 +00:00
|
|
|
#include <engine/shared/linereader.h>
|
2018-07-11 20:46:04 +00:00
|
|
|
#include <engine/shared/serverinfo.h>
|
2021-05-13 00:46:28 +00:00
|
|
|
#include <engine/storage.h>
|
2018-07-11 20:46:04 +00:00
|
|
|
|
2023-11-10 20:15:51 +00:00
|
|
|
#include <base/lock.h>
|
2024-03-14 10:19:32 +00:00
|
|
|
#include <base/log.h>
|
2022-05-18 16:00:05 +00:00
|
|
|
#include <base/system.h>
|
|
|
|
|
2018-07-11 20:46:04 +00:00
|
|
|
#include <memory>
|
2022-01-02 03:27:29 +00:00
|
|
|
#include <vector>
|
2018-07-11 20:46:04 +00:00
|
|
|
|
2022-05-18 16:00:05 +00:00
|
|
|
#include <chrono>
|
|
|
|
|
|
|
|
using namespace std::chrono_literals;
|
|
|
|
|
2024-03-14 11:19:18 +00:00
|
|
|
static int SanitizeAge(std::optional<int64_t> Age)
|
|
|
|
{
|
|
|
|
// A year is of course pi*10**7 seconds.
|
|
|
|
if(!(Age && 0 <= *Age && *Age < 31415927))
|
|
|
|
{
|
|
|
|
return 31415927;
|
|
|
|
}
|
|
|
|
return *Age;
|
|
|
|
}
|
|
|
|
|
|
|
|
// Classify HTTP responses into buckets, treat 15 seconds as fresh, 1 minute as
|
|
|
|
// less fresh, etc. This ensures that differences in the order of seconds do
|
|
|
|
// not affect master choice.
|
|
|
|
static int ClassifyAge(int AgeSeconds)
|
|
|
|
{
|
|
|
|
return 0 //
|
|
|
|
+ (AgeSeconds >= 15) // 15 seconds
|
|
|
|
+ (AgeSeconds >= 60) // 1 minute
|
|
|
|
+ (AgeSeconds >= 300) // 5 minutes
|
|
|
|
+ (AgeSeconds / 3600); // 1 hour
|
|
|
|
}
|
|
|
|
|
2018-07-11 20:46:04 +00:00
|
|
|
class CChooseMaster
|
|
|
|
{
|
|
|
|
public:
|
2021-05-12 17:09:16 +00:00
|
|
|
typedef bool (*VALIDATOR)(json_value *pJson);
|
|
|
|
|
2018-07-11 20:46:04 +00:00
|
|
|
enum
|
|
|
|
{
|
|
|
|
MAX_URLS = 16,
|
|
|
|
};
|
2023-12-18 19:01:26 +00:00
|
|
|
CChooseMaster(IEngine *pEngine, IHttp *pHttp, VALIDATOR pfnValidator, const char **ppUrls, int NumUrls, int PreviousBestIndex);
|
2022-01-08 08:58:11 +00:00
|
|
|
virtual ~CChooseMaster();
|
2018-07-11 20:46:04 +00:00
|
|
|
|
2021-05-13 00:46:28 +00:00
|
|
|
bool GetBestUrl(const char **pBestUrl) const;
|
|
|
|
void Reset();
|
Allow background jobs to be aborted, refactoring
Add `IJob::Abortable(bool)` function which jobs can call to specify whether they can be aborted. Jobs are not abortable per default. Abortable jobs may have their state set to `IJob::STATE_ABORTED` at any point if the job was aborted. The job state should be checked periodically in the `IJob::Run` function and the job should terminate at the earliest, safe opportunity when aborted. Scheduled jobs which are not abortable are guaranteed to fully complete before the job pool is shut down. However, if the job pool is already shutting down, no additional jobs will be enqueue anymore and abortable jobs will immediately be aborted.
In particular, the sound loading, community icon loading, master chooser and host lookup jobs are specified as being abortable. Conversely, the jobs saving replay demos, editor maps and screenshots are expected to finish before the client is shut down.
When the client is quitting/restarting, it will now disconnect from the current server first, before saving the config, to ensure that any actions that happen on disconnect (demo recorders being stopped etc.) happen first. The shutdown message is rendered before disconnecting and waiting for background jobs to finish.
The HTTP client is now initialized later during server launch, after the network initialization. Error handling is added and the server stops if the HTTP client could not be initialized, same as the client.
The `RunBlocking` functions are removed, as they are not used anymore after curl-multi was added.
The function `IJob::Status` is renamed to `State` and `IJob::STATE_PENDING` is renamed to `STATE_QUEUED` for consistency with naming of the HTTP client.
The member variables of the engine interface are encapsulated and the `jobs.h` include is removed from `engine.h`, which removes transitive includes of `system.h`.
Documentation for all job and job pool API is added.
2024-02-16 19:44:33 +00:00
|
|
|
bool IsRefreshing() const { return m_pJob && !m_pJob->Done(); }
|
2018-07-11 20:46:04 +00:00
|
|
|
void Refresh();
|
|
|
|
|
|
|
|
private:
|
2021-05-13 00:46:28 +00:00
|
|
|
int GetBestIndex() const;
|
|
|
|
|
2018-07-11 20:46:04 +00:00
|
|
|
class CData
|
|
|
|
{
|
|
|
|
public:
|
|
|
|
std::atomic_int m_BestIndex{-1};
|
|
|
|
// Constant after construction.
|
2021-05-12 17:09:16 +00:00
|
|
|
VALIDATOR m_pfnValidator;
|
2018-07-11 20:46:04 +00:00
|
|
|
int m_NumUrls;
|
|
|
|
char m_aaUrls[MAX_URLS][256];
|
|
|
|
};
|
|
|
|
class CJob : public IJob
|
|
|
|
{
|
2023-12-18 19:01:26 +00:00
|
|
|
CChooseMaster *m_pParent;
|
2023-11-10 20:15:51 +00:00
|
|
|
CLock m_Lock;
|
2018-07-11 20:46:04 +00:00
|
|
|
std::shared_ptr<CData> m_pData;
|
2024-01-12 20:17:16 +00:00
|
|
|
std::shared_ptr<CHttpRequest> m_pHead;
|
|
|
|
std::shared_ptr<CHttpRequest> m_pGet;
|
2022-05-28 00:14:46 +00:00
|
|
|
void Run() override REQUIRES(!m_Lock);
|
2018-07-11 20:46:04 +00:00
|
|
|
|
|
|
|
public:
|
2023-12-18 19:01:26 +00:00
|
|
|
CJob(CChooseMaster *pParent, std::shared_ptr<CData> pData) :
|
Allow background jobs to be aborted, refactoring
Add `IJob::Abortable(bool)` function which jobs can call to specify whether they can be aborted. Jobs are not abortable per default. Abortable jobs may have their state set to `IJob::STATE_ABORTED` at any point if the job was aborted. The job state should be checked periodically in the `IJob::Run` function and the job should terminate at the earliest, safe opportunity when aborted. Scheduled jobs which are not abortable are guaranteed to fully complete before the job pool is shut down. However, if the job pool is already shutting down, no additional jobs will be enqueue anymore and abortable jobs will immediately be aborted.
In particular, the sound loading, community icon loading, master chooser and host lookup jobs are specified as being abortable. Conversely, the jobs saving replay demos, editor maps and screenshots are expected to finish before the client is shut down.
When the client is quitting/restarting, it will now disconnect from the current server first, before saving the config, to ensure that any actions that happen on disconnect (demo recorders being stopped etc.) happen first. The shutdown message is rendered before disconnecting and waiting for background jobs to finish.
The HTTP client is now initialized later during server launch, after the network initialization. Error handling is added and the server stops if the HTTP client could not be initialized, same as the client.
The `RunBlocking` functions are removed, as they are not used anymore after curl-multi was added.
The function `IJob::Status` is renamed to `State` and `IJob::STATE_PENDING` is renamed to `STATE_QUEUED` for consistency with naming of the HTTP client.
The member variables of the engine interface are encapsulated and the `jobs.h` include is removed from `engine.h`, which removes transitive includes of `system.h`.
Documentation for all job and job pool API is added.
2024-02-16 19:44:33 +00:00
|
|
|
m_pParent(pParent),
|
|
|
|
m_pData(std::move(pData))
|
|
|
|
{
|
|
|
|
Abortable(true);
|
|
|
|
}
|
|
|
|
bool Abort() override REQUIRES(!m_Lock);
|
2018-07-11 20:46:04 +00:00
|
|
|
};
|
|
|
|
|
|
|
|
IEngine *m_pEngine;
|
2023-12-18 19:01:26 +00:00
|
|
|
IHttp *m_pHttp;
|
2018-07-11 20:46:04 +00:00
|
|
|
int m_PreviousBestIndex;
|
|
|
|
std::shared_ptr<CData> m_pData;
|
2021-05-13 00:46:28 +00:00
|
|
|
std::shared_ptr<CJob> m_pJob;
|
2018-07-11 20:46:04 +00:00
|
|
|
};
|
|
|
|
|
2023-12-18 19:01:26 +00:00
|
|
|
CChooseMaster::CChooseMaster(IEngine *pEngine, IHttp *pHttp, VALIDATOR pfnValidator, const char **ppUrls, int NumUrls, int PreviousBestIndex) :
|
2018-07-11 20:46:04 +00:00
|
|
|
m_pEngine(pEngine),
|
2023-12-18 19:01:26 +00:00
|
|
|
m_pHttp(pHttp),
|
2018-07-11 20:46:04 +00:00
|
|
|
m_PreviousBestIndex(PreviousBestIndex)
|
|
|
|
{
|
|
|
|
dbg_assert(NumUrls >= 0, "no master URLs");
|
|
|
|
dbg_assert(NumUrls <= MAX_URLS, "too many master URLs");
|
|
|
|
dbg_assert(PreviousBestIndex >= -1, "previous best index negative and not -1");
|
|
|
|
dbg_assert(PreviousBestIndex < NumUrls, "previous best index too high");
|
|
|
|
m_pData = std::make_shared<CData>();
|
2021-05-12 17:09:16 +00:00
|
|
|
m_pData->m_pfnValidator = pfnValidator;
|
2018-07-11 20:46:04 +00:00
|
|
|
m_pData->m_NumUrls = NumUrls;
|
|
|
|
for(int i = 0; i < m_pData->m_NumUrls; i++)
|
|
|
|
{
|
2022-07-09 16:14:56 +00:00
|
|
|
str_copy(m_pData->m_aaUrls[i], ppUrls[i]);
|
2018-07-11 20:46:04 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2022-01-08 08:58:11 +00:00
|
|
|
CChooseMaster::~CChooseMaster()
|
|
|
|
{
|
|
|
|
if(m_pJob)
|
|
|
|
{
|
|
|
|
m_pJob->Abort();
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2018-07-11 20:46:04 +00:00
|
|
|
int CChooseMaster::GetBestIndex() const
|
|
|
|
{
|
|
|
|
int BestIndex = m_pData->m_BestIndex.load();
|
|
|
|
if(BestIndex >= 0)
|
|
|
|
{
|
|
|
|
return BestIndex;
|
|
|
|
}
|
|
|
|
else
|
|
|
|
{
|
|
|
|
return m_PreviousBestIndex;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2021-05-13 00:46:28 +00:00
|
|
|
bool CChooseMaster::GetBestUrl(const char **ppBestUrl) const
|
|
|
|
{
|
|
|
|
int Index = GetBestIndex();
|
|
|
|
if(Index < 0)
|
|
|
|
{
|
|
|
|
*ppBestUrl = nullptr;
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
*ppBestUrl = m_pData->m_aaUrls[Index];
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
|
|
|
|
void CChooseMaster::Reset()
|
2018-07-11 20:46:04 +00:00
|
|
|
{
|
2021-05-13 00:46:28 +00:00
|
|
|
m_PreviousBestIndex = -1;
|
|
|
|
m_pData->m_BestIndex.store(-1);
|
2018-07-11 20:46:04 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
void CChooseMaster::Refresh()
|
|
|
|
{
|
Allow background jobs to be aborted, refactoring
Add `IJob::Abortable(bool)` function which jobs can call to specify whether they can be aborted. Jobs are not abortable per default. Abortable jobs may have their state set to `IJob::STATE_ABORTED` at any point if the job was aborted. The job state should be checked periodically in the `IJob::Run` function and the job should terminate at the earliest, safe opportunity when aborted. Scheduled jobs which are not abortable are guaranteed to fully complete before the job pool is shut down. However, if the job pool is already shutting down, no additional jobs will be enqueue anymore and abortable jobs will immediately be aborted.
In particular, the sound loading, community icon loading, master chooser and host lookup jobs are specified as being abortable. Conversely, the jobs saving replay demos, editor maps and screenshots are expected to finish before the client is shut down.
When the client is quitting/restarting, it will now disconnect from the current server first, before saving the config, to ensure that any actions that happen on disconnect (demo recorders being stopped etc.) happen first. The shutdown message is rendered before disconnecting and waiting for background jobs to finish.
The HTTP client is now initialized later during server launch, after the network initialization. Error handling is added and the server stops if the HTTP client could not be initialized, same as the client.
The `RunBlocking` functions are removed, as they are not used anymore after curl-multi was added.
The function `IJob::Status` is renamed to `State` and `IJob::STATE_PENDING` is renamed to `STATE_QUEUED` for consistency with naming of the HTTP client.
The member variables of the engine interface are encapsulated and the `jobs.h` include is removed from `engine.h`, which removes transitive includes of `system.h`.
Documentation for all job and job pool API is added.
2024-02-16 19:44:33 +00:00
|
|
|
if(m_pJob == nullptr || m_pJob->State() == IJob::STATE_DONE)
|
|
|
|
{
|
|
|
|
m_pJob = std::make_shared<CJob>(this, m_pData);
|
|
|
|
m_pEngine->AddJob(m_pJob);
|
|
|
|
}
|
2018-07-11 20:46:04 +00:00
|
|
|
}
|
|
|
|
|
Allow background jobs to be aborted, refactoring
Add `IJob::Abortable(bool)` function which jobs can call to specify whether they can be aborted. Jobs are not abortable per default. Abortable jobs may have their state set to `IJob::STATE_ABORTED` at any point if the job was aborted. The job state should be checked periodically in the `IJob::Run` function and the job should terminate at the earliest, safe opportunity when aborted. Scheduled jobs which are not abortable are guaranteed to fully complete before the job pool is shut down. However, if the job pool is already shutting down, no additional jobs will be enqueue anymore and abortable jobs will immediately be aborted.
In particular, the sound loading, community icon loading, master chooser and host lookup jobs are specified as being abortable. Conversely, the jobs saving replay demos, editor maps and screenshots are expected to finish before the client is shut down.
When the client is quitting/restarting, it will now disconnect from the current server first, before saving the config, to ensure that any actions that happen on disconnect (demo recorders being stopped etc.) happen first. The shutdown message is rendered before disconnecting and waiting for background jobs to finish.
The HTTP client is now initialized later during server launch, after the network initialization. Error handling is added and the server stops if the HTTP client could not be initialized, same as the client.
The `RunBlocking` functions are removed, as they are not used anymore after curl-multi was added.
The function `IJob::Status` is renamed to `State` and `IJob::STATE_PENDING` is renamed to `STATE_QUEUED` for consistency with naming of the HTTP client.
The member variables of the engine interface are encapsulated and the `jobs.h` include is removed from `engine.h`, which removes transitive includes of `system.h`.
Documentation for all job and job pool API is added.
2024-02-16 19:44:33 +00:00
|
|
|
bool CChooseMaster::CJob::Abort()
|
2022-01-08 08:58:11 +00:00
|
|
|
{
|
Allow background jobs to be aborted, refactoring
Add `IJob::Abortable(bool)` function which jobs can call to specify whether they can be aborted. Jobs are not abortable per default. Abortable jobs may have their state set to `IJob::STATE_ABORTED` at any point if the job was aborted. The job state should be checked periodically in the `IJob::Run` function and the job should terminate at the earliest, safe opportunity when aborted. Scheduled jobs which are not abortable are guaranteed to fully complete before the job pool is shut down. However, if the job pool is already shutting down, no additional jobs will be enqueue anymore and abortable jobs will immediately be aborted.
In particular, the sound loading, community icon loading, master chooser and host lookup jobs are specified as being abortable. Conversely, the jobs saving replay demos, editor maps and screenshots are expected to finish before the client is shut down.
When the client is quitting/restarting, it will now disconnect from the current server first, before saving the config, to ensure that any actions that happen on disconnect (demo recorders being stopped etc.) happen first. The shutdown message is rendered before disconnecting and waiting for background jobs to finish.
The HTTP client is now initialized later during server launch, after the network initialization. Error handling is added and the server stops if the HTTP client could not be initialized, same as the client.
The `RunBlocking` functions are removed, as they are not used anymore after curl-multi was added.
The function `IJob::Status` is renamed to `State` and `IJob::STATE_PENDING` is renamed to `STATE_QUEUED` for consistency with naming of the HTTP client.
The member variables of the engine interface are encapsulated and the `jobs.h` include is removed from `engine.h`, which removes transitive includes of `system.h`.
Documentation for all job and job pool API is added.
2024-02-16 19:44:33 +00:00
|
|
|
if(!IJob::Abort())
|
|
|
|
{
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
|
2022-06-03 10:10:05 +00:00
|
|
|
CLockScope ls(m_Lock);
|
2022-01-08 08:58:11 +00:00
|
|
|
if(m_pHead != nullptr)
|
|
|
|
{
|
|
|
|
m_pHead->Abort();
|
|
|
|
}
|
|
|
|
|
|
|
|
if(m_pGet != nullptr)
|
|
|
|
{
|
|
|
|
m_pGet->Abort();
|
|
|
|
}
|
Allow background jobs to be aborted, refactoring
Add `IJob::Abortable(bool)` function which jobs can call to specify whether they can be aborted. Jobs are not abortable per default. Abortable jobs may have their state set to `IJob::STATE_ABORTED` at any point if the job was aborted. The job state should be checked periodically in the `IJob::Run` function and the job should terminate at the earliest, safe opportunity when aborted. Scheduled jobs which are not abortable are guaranteed to fully complete before the job pool is shut down. However, if the job pool is already shutting down, no additional jobs will be enqueue anymore and abortable jobs will immediately be aborted.
In particular, the sound loading, community icon loading, master chooser and host lookup jobs are specified as being abortable. Conversely, the jobs saving replay demos, editor maps and screenshots are expected to finish before the client is shut down.
When the client is quitting/restarting, it will now disconnect from the current server first, before saving the config, to ensure that any actions that happen on disconnect (demo recorders being stopped etc.) happen first. The shutdown message is rendered before disconnecting and waiting for background jobs to finish.
The HTTP client is now initialized later during server launch, after the network initialization. Error handling is added and the server stops if the HTTP client could not be initialized, same as the client.
The `RunBlocking` functions are removed, as they are not used anymore after curl-multi was added.
The function `IJob::Status` is renamed to `State` and `IJob::STATE_PENDING` is renamed to `STATE_QUEUED` for consistency with naming of the HTTP client.
The member variables of the engine interface are encapsulated and the `jobs.h` include is removed from `engine.h`, which removes transitive includes of `system.h`.
Documentation for all job and job pool API is added.
2024-02-16 19:44:33 +00:00
|
|
|
|
|
|
|
return true;
|
2022-01-08 08:58:11 +00:00
|
|
|
}
|
|
|
|
|
2018-07-11 20:46:04 +00:00
|
|
|
void CChooseMaster::CJob::Run()
|
|
|
|
{
|
|
|
|
// Check masters in a random order.
|
2021-05-13 23:25:59 +00:00
|
|
|
int aRandomized[MAX_URLS] = {0};
|
2018-07-11 20:46:04 +00:00
|
|
|
for(int i = 0; i < m_pData->m_NumUrls; i++)
|
|
|
|
{
|
|
|
|
aRandomized[i] = i;
|
|
|
|
}
|
|
|
|
// https://en.wikipedia.org/w/index.php?title=Fisher%E2%80%93Yates_shuffle&oldid=1002922479#The_modern_algorithm
|
|
|
|
// The equivalent version.
|
|
|
|
for(int i = 0; i <= m_pData->m_NumUrls - 2; i++)
|
|
|
|
{
|
|
|
|
int j = i + secure_rand_below(m_pData->m_NumUrls - i);
|
|
|
|
std::swap(aRandomized[i], aRandomized[j]);
|
|
|
|
}
|
|
|
|
// Do a HEAD request to ensure that a connection is established and
|
|
|
|
// then do a GET request to check how fast we can get the server list.
|
|
|
|
//
|
|
|
|
// 10 seconds connection timeout, lower than 8KB/s for 10 seconds to
|
|
|
|
// fail.
|
2022-05-28 22:13:59 +00:00
|
|
|
CTimeout Timeout{10000, 0, 8000, 10};
|
2018-07-11 20:46:04 +00:00
|
|
|
int aTimeMs[MAX_URLS];
|
2024-03-14 11:19:18 +00:00
|
|
|
int aAgeS[MAX_URLS];
|
2018-07-11 20:46:04 +00:00
|
|
|
for(int i = 0; i < m_pData->m_NumUrls; i++)
|
|
|
|
{
|
|
|
|
aTimeMs[i] = -1;
|
2024-03-14 11:19:18 +00:00
|
|
|
aAgeS[i] = SanitizeAge({});
|
2018-07-11 20:46:04 +00:00
|
|
|
const char *pUrl = m_pData->m_aaUrls[aRandomized[i]];
|
2024-01-12 20:17:16 +00:00
|
|
|
std::shared_ptr<CHttpRequest> pHead = HttpHead(pUrl);
|
2022-05-05 12:49:25 +00:00
|
|
|
pHead->Timeout(Timeout);
|
|
|
|
pHead->LogProgress(HTTPLOG::FAILURE);
|
2022-06-03 10:10:05 +00:00
|
|
|
{
|
|
|
|
CLockScope ls(m_Lock);
|
2023-12-18 19:01:26 +00:00
|
|
|
m_pHead = pHead;
|
2022-06-03 10:10:05 +00:00
|
|
|
}
|
2023-12-18 19:01:26 +00:00
|
|
|
|
|
|
|
m_pParent->m_pHttp->Run(pHead);
|
|
|
|
pHead->Wait();
|
Allow background jobs to be aborted, refactoring
Add `IJob::Abortable(bool)` function which jobs can call to specify whether they can be aborted. Jobs are not abortable per default. Abortable jobs may have their state set to `IJob::STATE_ABORTED` at any point if the job was aborted. The job state should be checked periodically in the `IJob::Run` function and the job should terminate at the earliest, safe opportunity when aborted. Scheduled jobs which are not abortable are guaranteed to fully complete before the job pool is shut down. However, if the job pool is already shutting down, no additional jobs will be enqueue anymore and abortable jobs will immediately be aborted.
In particular, the sound loading, community icon loading, master chooser and host lookup jobs are specified as being abortable. Conversely, the jobs saving replay demos, editor maps and screenshots are expected to finish before the client is shut down.
When the client is quitting/restarting, it will now disconnect from the current server first, before saving the config, to ensure that any actions that happen on disconnect (demo recorders being stopped etc.) happen first. The shutdown message is rendered before disconnecting and waiting for background jobs to finish.
The HTTP client is now initialized later during server launch, after the network initialization. Error handling is added and the server stops if the HTTP client could not be initialized, same as the client.
The `RunBlocking` functions are removed, as they are not used anymore after curl-multi was added.
The function `IJob::Status` is renamed to `State` and `IJob::STATE_PENDING` is renamed to `STATE_QUEUED` for consistency with naming of the HTTP client.
The member variables of the engine interface are encapsulated and the `jobs.h` include is removed from `engine.h`, which removes transitive includes of `system.h`.
Documentation for all job and job pool API is added.
2024-02-16 19:44:33 +00:00
|
|
|
if(pHead->State() == EHttpState::ABORTED || State() == IJob::STATE_ABORTED)
|
2022-01-08 08:58:11 +00:00
|
|
|
{
|
2024-03-15 17:15:51 +00:00
|
|
|
log_debug("serverbrowser_http", "master chooser aborted");
|
2022-01-08 08:58:11 +00:00
|
|
|
return;
|
|
|
|
}
|
2024-01-21 21:56:42 +00:00
|
|
|
if(pHead->State() != EHttpState::DONE)
|
2018-07-11 20:46:04 +00:00
|
|
|
{
|
|
|
|
continue;
|
|
|
|
}
|
2023-12-18 19:01:26 +00:00
|
|
|
|
2022-06-13 16:07:29 +00:00
|
|
|
auto StartTime = time_get_nanoseconds();
|
2024-01-12 20:17:16 +00:00
|
|
|
std::shared_ptr<CHttpRequest> pGet = HttpGet(pUrl);
|
2022-05-05 12:49:25 +00:00
|
|
|
pGet->Timeout(Timeout);
|
|
|
|
pGet->LogProgress(HTTPLOG::FAILURE);
|
2022-06-03 10:10:05 +00:00
|
|
|
{
|
|
|
|
CLockScope ls(m_Lock);
|
2023-12-18 19:01:26 +00:00
|
|
|
m_pGet = pGet;
|
2022-06-03 10:10:05 +00:00
|
|
|
}
|
2023-12-18 19:01:26 +00:00
|
|
|
|
|
|
|
m_pParent->m_pHttp->Run(pGet);
|
|
|
|
pGet->Wait();
|
|
|
|
|
2022-06-13 16:07:29 +00:00
|
|
|
auto Time = std::chrono::duration_cast<std::chrono::milliseconds>(time_get_nanoseconds() - StartTime);
|
Allow background jobs to be aborted, refactoring
Add `IJob::Abortable(bool)` function which jobs can call to specify whether they can be aborted. Jobs are not abortable per default. Abortable jobs may have their state set to `IJob::STATE_ABORTED` at any point if the job was aborted. The job state should be checked periodically in the `IJob::Run` function and the job should terminate at the earliest, safe opportunity when aborted. Scheduled jobs which are not abortable are guaranteed to fully complete before the job pool is shut down. However, if the job pool is already shutting down, no additional jobs will be enqueue anymore and abortable jobs will immediately be aborted.
In particular, the sound loading, community icon loading, master chooser and host lookup jobs are specified as being abortable. Conversely, the jobs saving replay demos, editor maps and screenshots are expected to finish before the client is shut down.
When the client is quitting/restarting, it will now disconnect from the current server first, before saving the config, to ensure that any actions that happen on disconnect (demo recorders being stopped etc.) happen first. The shutdown message is rendered before disconnecting and waiting for background jobs to finish.
The HTTP client is now initialized later during server launch, after the network initialization. Error handling is added and the server stops if the HTTP client could not be initialized, same as the client.
The `RunBlocking` functions are removed, as they are not used anymore after curl-multi was added.
The function `IJob::Status` is renamed to `State` and `IJob::STATE_PENDING` is renamed to `STATE_QUEUED` for consistency with naming of the HTTP client.
The member variables of the engine interface are encapsulated and the `jobs.h` include is removed from `engine.h`, which removes transitive includes of `system.h`.
Documentation for all job and job pool API is added.
2024-02-16 19:44:33 +00:00
|
|
|
if(pGet->State() == EHttpState::ABORTED || State() == IJob::STATE_ABORTED)
|
2022-01-08 08:58:11 +00:00
|
|
|
{
|
2024-03-15 17:15:51 +00:00
|
|
|
log_debug("serverbrowser_http", "master chooser aborted");
|
2022-01-08 08:58:11 +00:00
|
|
|
return;
|
|
|
|
}
|
2024-01-21 21:56:42 +00:00
|
|
|
if(pGet->State() != EHttpState::DONE)
|
2018-07-11 20:46:04 +00:00
|
|
|
{
|
|
|
|
continue;
|
|
|
|
}
|
2022-01-08 08:58:11 +00:00
|
|
|
json_value *pJson = pGet->ResultJson();
|
2021-05-12 17:09:16 +00:00
|
|
|
if(!pJson)
|
|
|
|
{
|
|
|
|
continue;
|
|
|
|
}
|
2023-12-18 19:01:26 +00:00
|
|
|
|
2021-05-12 17:09:16 +00:00
|
|
|
bool ParseFailure = m_pData->m_pfnValidator(pJson);
|
|
|
|
json_value_free(pJson);
|
|
|
|
if(ParseFailure)
|
|
|
|
{
|
|
|
|
continue;
|
|
|
|
}
|
2024-03-14 11:19:18 +00:00
|
|
|
int AgeS = SanitizeAge(pGet->ResultAgeSeconds());
|
2024-03-15 17:15:51 +00:00
|
|
|
log_info("serverbrowser_http", "found master, url='%s' time=%dms age=%ds", pUrl, (int)Time.count(), AgeS);
|
2024-03-14 11:19:18 +00:00
|
|
|
|
2022-05-18 16:00:05 +00:00
|
|
|
aTimeMs[i] = Time.count();
|
2024-03-14 11:19:18 +00:00
|
|
|
aAgeS[i] = AgeS;
|
2018-07-11 20:46:04 +00:00
|
|
|
}
|
2023-12-18 19:01:26 +00:00
|
|
|
|
2018-07-11 20:46:04 +00:00
|
|
|
// Determine index of the minimum time.
|
|
|
|
int BestIndex = -1;
|
|
|
|
int BestTime = 0;
|
2024-03-14 11:19:18 +00:00
|
|
|
int BestAge = 0;
|
2018-07-11 20:46:04 +00:00
|
|
|
for(int i = 0; i < m_pData->m_NumUrls; i++)
|
|
|
|
{
|
|
|
|
if(aTimeMs[i] < 0)
|
|
|
|
{
|
|
|
|
continue;
|
|
|
|
}
|
2024-03-14 11:19:18 +00:00
|
|
|
if(BestIndex == -1 || std::tuple(ClassifyAge(aAgeS[i]), aTimeMs[i]) < std::tuple(ClassifyAge(BestAge), BestTime))
|
2018-07-11 20:46:04 +00:00
|
|
|
{
|
|
|
|
BestTime = aTimeMs[i];
|
2024-03-14 11:19:18 +00:00
|
|
|
BestAge = aAgeS[i];
|
2018-07-11 20:46:04 +00:00
|
|
|
BestIndex = aRandomized[i];
|
|
|
|
}
|
|
|
|
}
|
|
|
|
if(BestIndex == -1)
|
|
|
|
{
|
2024-03-15 17:15:51 +00:00
|
|
|
log_error("serverbrowser_http", "WARNING: no usable masters found");
|
2018-07-11 20:46:04 +00:00
|
|
|
return;
|
|
|
|
}
|
2023-12-18 19:01:26 +00:00
|
|
|
|
2024-03-15 17:15:51 +00:00
|
|
|
log_info("serverbrowser_http", "determined best master, url='%s' time=%dms age=%ds", m_pData->m_aaUrls[BestIndex], BestTime, BestAge);
|
2018-07-11 20:46:04 +00:00
|
|
|
m_pData->m_BestIndex.store(BestIndex);
|
|
|
|
}
|
|
|
|
|
|
|
|
class CServerBrowserHttp : public IServerBrowserHttp
|
|
|
|
{
|
|
|
|
public:
|
2024-03-14 10:19:32 +00:00
|
|
|
CServerBrowserHttp(IEngine *pEngine, IHttp *pHttp, const char **ppUrls, int NumUrls, int PreviousBestIndex);
|
2023-10-06 09:20:25 +00:00
|
|
|
~CServerBrowserHttp() override;
|
2022-05-17 18:33:27 +00:00
|
|
|
void Update() override;
|
|
|
|
bool IsRefreshing() override { return m_State != STATE_DONE; }
|
|
|
|
void Refresh() override;
|
|
|
|
bool GetBestUrl(const char **pBestUrl) const override { return m_pChooseMaster->GetBestUrl(pBestUrl); }
|
2018-07-11 20:46:04 +00:00
|
|
|
|
2022-05-17 18:33:27 +00:00
|
|
|
int NumServers() const override
|
2018-07-11 20:46:04 +00:00
|
|
|
{
|
2022-06-11 19:38:18 +00:00
|
|
|
return m_vServers.size();
|
2018-07-11 20:46:04 +00:00
|
|
|
}
|
2022-05-23 18:16:18 +00:00
|
|
|
const CServerInfo &Server(int Index) const override
|
2018-07-11 20:46:04 +00:00
|
|
|
{
|
2022-05-23 18:16:18 +00:00
|
|
|
return m_vServers[Index];
|
2018-07-11 20:46:04 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
private:
|
2021-05-13 00:46:28 +00:00
|
|
|
enum
|
|
|
|
{
|
|
|
|
STATE_DONE,
|
|
|
|
STATE_WANTREFRESH,
|
|
|
|
STATE_REFRESHING,
|
2021-07-24 22:58:30 +00:00
|
|
|
STATE_NO_MASTER,
|
2021-05-13 00:46:28 +00:00
|
|
|
};
|
|
|
|
|
2021-05-12 17:09:16 +00:00
|
|
|
static bool Validate(json_value *pJson);
|
2024-07-25 15:24:39 +00:00
|
|
|
static bool Parse(json_value *pJson, std::vector<CServerInfo> *pvServers);
|
2021-05-12 17:09:16 +00:00
|
|
|
|
2023-12-18 19:01:26 +00:00
|
|
|
IHttp *m_pHttp;
|
2021-05-13 00:46:28 +00:00
|
|
|
|
|
|
|
int m_State = STATE_DONE;
|
2022-05-05 12:49:25 +00:00
|
|
|
std::shared_ptr<CHttpRequest> m_pGetServers;
|
2018-07-11 20:46:04 +00:00
|
|
|
std::unique_ptr<CChooseMaster> m_pChooseMaster;
|
|
|
|
|
2022-05-23 18:16:18 +00:00
|
|
|
std::vector<CServerInfo> m_vServers;
|
2018-07-11 20:46:04 +00:00
|
|
|
};
|
|
|
|
|
2024-03-14 10:19:32 +00:00
|
|
|
CServerBrowserHttp::CServerBrowserHttp(IEngine *pEngine, IHttp *pHttp, const char **ppUrls, int NumUrls, int PreviousBestIndex) :
|
2023-12-18 19:01:26 +00:00
|
|
|
m_pHttp(pHttp),
|
|
|
|
m_pChooseMaster(new CChooseMaster(pEngine, pHttp, Validate, ppUrls, NumUrls, PreviousBestIndex))
|
2018-07-11 20:46:04 +00:00
|
|
|
{
|
|
|
|
m_pChooseMaster->Refresh();
|
|
|
|
}
|
2022-01-08 08:58:11 +00:00
|
|
|
|
|
|
|
CServerBrowserHttp::~CServerBrowserHttp()
|
|
|
|
{
|
|
|
|
if(m_pGetServers != nullptr)
|
|
|
|
{
|
|
|
|
m_pGetServers->Abort();
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2018-07-11 20:46:04 +00:00
|
|
|
void CServerBrowserHttp::Update()
|
|
|
|
{
|
2021-05-13 00:46:28 +00:00
|
|
|
if(m_State == STATE_WANTREFRESH)
|
2018-07-11 20:46:04 +00:00
|
|
|
{
|
2021-05-13 00:46:28 +00:00
|
|
|
const char *pBestUrl;
|
|
|
|
if(m_pChooseMaster->GetBestUrl(&pBestUrl))
|
|
|
|
{
|
|
|
|
if(!m_pChooseMaster->IsRefreshing())
|
|
|
|
{
|
2024-03-15 17:15:51 +00:00
|
|
|
log_error("serverbrowser_http", "no working serverlist URL found");
|
2021-07-24 22:58:30 +00:00
|
|
|
m_State = STATE_NO_MASTER;
|
2021-05-13 00:46:28 +00:00
|
|
|
}
|
|
|
|
return;
|
|
|
|
}
|
2022-05-05 12:49:25 +00:00
|
|
|
m_pGetServers = HttpGet(pBestUrl);
|
2021-06-09 13:24:43 +00:00
|
|
|
// 10 seconds connection timeout, lower than 8KB/s for 10 seconds to fail.
|
2022-05-28 22:13:59 +00:00
|
|
|
m_pGetServers->Timeout(CTimeout{10000, 0, 8000, 10});
|
2023-12-18 19:01:26 +00:00
|
|
|
m_pHttp->Run(m_pGetServers);
|
2021-05-13 00:46:28 +00:00
|
|
|
m_State = STATE_REFRESHING;
|
|
|
|
}
|
|
|
|
else if(m_State == STATE_REFRESHING)
|
|
|
|
{
|
2024-02-10 21:13:34 +00:00
|
|
|
if(!m_pGetServers->Done())
|
2021-05-13 00:46:28 +00:00
|
|
|
{
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
m_State = STATE_DONE;
|
2022-05-05 12:49:25 +00:00
|
|
|
std::shared_ptr<CHttpRequest> pGetServers = nullptr;
|
2021-05-12 17:09:16 +00:00
|
|
|
std::swap(m_pGetServers, pGetServers);
|
|
|
|
|
2021-05-13 00:46:28 +00:00
|
|
|
bool Success = true;
|
2024-02-10 21:13:34 +00:00
|
|
|
json_value *pJson = pGetServers->State() == EHttpState::DONE ? pGetServers->ResultJson() : nullptr;
|
2021-05-13 00:46:28 +00:00
|
|
|
Success = Success && pJson;
|
2024-07-25 15:24:39 +00:00
|
|
|
Success = Success && !Parse(pJson, &m_vServers);
|
2021-05-12 17:09:16 +00:00
|
|
|
json_value_free(pJson);
|
2021-05-13 00:46:28 +00:00
|
|
|
if(!Success)
|
2018-07-11 20:46:04 +00:00
|
|
|
{
|
2024-03-15 17:15:51 +00:00
|
|
|
log_error("serverbrowser_http", "failed getting serverlist, trying to find best URL");
|
2021-05-13 00:46:28 +00:00
|
|
|
m_pChooseMaster->Reset();
|
|
|
|
m_pChooseMaster->Refresh();
|
2018-07-11 20:46:04 +00:00
|
|
|
}
|
2024-03-16 14:39:47 +00:00
|
|
|
else
|
2024-03-14 11:19:18 +00:00
|
|
|
{
|
2024-03-16 14:39:47 +00:00
|
|
|
// Try to find new master if the current one returns
|
|
|
|
// results that are 5 minutes old.
|
|
|
|
int Age = SanitizeAge(pGetServers->ResultAgeSeconds());
|
|
|
|
if(Age > 300)
|
|
|
|
{
|
2024-03-16 22:23:35 +00:00
|
|
|
log_info("serverbrowser_http", "got stale serverlist, age=%ds, trying to find best URL", Age);
|
2024-03-16 14:39:47 +00:00
|
|
|
m_pChooseMaster->Refresh();
|
|
|
|
}
|
2024-03-14 11:19:18 +00:00
|
|
|
}
|
2018-07-11 20:46:04 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
void CServerBrowserHttp::Refresh()
|
|
|
|
{
|
2021-07-24 22:58:30 +00:00
|
|
|
if(m_State == STATE_WANTREFRESH || m_State == STATE_REFRESHING || m_State == STATE_NO_MASTER)
|
2021-05-13 00:46:28 +00:00
|
|
|
{
|
2021-07-24 22:58:30 +00:00
|
|
|
if(m_State == STATE_NO_MASTER)
|
|
|
|
m_State = STATE_WANTREFRESH;
|
2021-05-13 00:46:28 +00:00
|
|
|
m_pChooseMaster->Refresh();
|
|
|
|
}
|
2021-06-09 03:36:32 +00:00
|
|
|
if(m_State == STATE_DONE)
|
|
|
|
m_State = STATE_WANTREFRESH;
|
2021-05-13 00:46:28 +00:00
|
|
|
Update();
|
2018-07-11 20:46:04 +00:00
|
|
|
}
|
|
|
|
bool ServerbrowserParseUrl(NETADDR *pOut, const char *pUrl)
|
|
|
|
{
|
2023-07-16 12:04:58 +00:00
|
|
|
return net_addr_from_url(pOut, pUrl, nullptr, 0) != 0;
|
2018-07-11 20:46:04 +00:00
|
|
|
}
|
2021-05-12 17:09:16 +00:00
|
|
|
bool CServerBrowserHttp::Validate(json_value *pJson)
|
|
|
|
{
|
2022-05-23 18:16:18 +00:00
|
|
|
std::vector<CServerInfo> vServers;
|
2024-07-25 15:24:39 +00:00
|
|
|
return Parse(pJson, &vServers);
|
2021-05-12 17:09:16 +00:00
|
|
|
}
|
2024-07-25 15:24:39 +00:00
|
|
|
bool CServerBrowserHttp::Parse(json_value *pJson, std::vector<CServerInfo> *pvServers)
|
2018-07-11 20:46:04 +00:00
|
|
|
{
|
2022-05-23 18:16:18 +00:00
|
|
|
std::vector<CServerInfo> vServers;
|
2018-07-11 20:46:04 +00:00
|
|
|
|
|
|
|
const json_value &Json = *pJson;
|
|
|
|
const json_value &Servers = Json["servers"];
|
2024-07-25 15:24:39 +00:00
|
|
|
if(Servers.type != json_array)
|
2018-07-11 20:46:04 +00:00
|
|
|
{
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
for(unsigned int i = 0; i < Servers.u.array.length; i++)
|
|
|
|
{
|
|
|
|
const json_value &Server = Servers[i];
|
|
|
|
const json_value &Addresses = Server["addresses"];
|
|
|
|
const json_value &Info = Server["info"];
|
|
|
|
const json_value &Location = Server["location"];
|
|
|
|
int ParsedLocation = CServerInfo::LOC_UNKNOWN;
|
|
|
|
CServerInfo2 ParsedInfo;
|
|
|
|
if(Addresses.type != json_array || (Location.type != json_string && Location.type != json_none))
|
|
|
|
{
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
if(Location.type == json_string)
|
|
|
|
{
|
|
|
|
if(CServerInfo::ParseLocation(&ParsedLocation, Location))
|
|
|
|
{
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
if(CServerInfo2::FromJson(&ParsedInfo, &Info))
|
|
|
|
{
|
2024-03-14 10:19:32 +00:00
|
|
|
log_debug("serverbrowser_http", "skipped due to info, i=%d", i);
|
2018-07-11 20:46:04 +00:00
|
|
|
// Only skip the current server on parsing
|
|
|
|
// failure; the server info is "user input" by
|
|
|
|
// the game server and can be set to arbitrary
|
|
|
|
// values.
|
|
|
|
continue;
|
|
|
|
}
|
|
|
|
CServerInfo SetInfo = ParsedInfo;
|
|
|
|
SetInfo.m_Location = ParsedLocation;
|
2022-05-23 18:16:18 +00:00
|
|
|
SetInfo.m_NumAddresses = 0;
|
2018-07-11 20:46:04 +00:00
|
|
|
for(unsigned int a = 0; a < Addresses.u.array.length; a++)
|
|
|
|
{
|
|
|
|
const json_value &Address = Addresses[a];
|
|
|
|
if(Address.type != json_string)
|
|
|
|
{
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
NETADDR ParsedAddr;
|
|
|
|
if(ServerbrowserParseUrl(&ParsedAddr, Addresses[a]))
|
|
|
|
{
|
2024-03-15 17:18:50 +00:00
|
|
|
// log_debug("serverbrowser_http", "unknown address, i=%d a=%d", i, a);
|
2018-07-11 20:46:04 +00:00
|
|
|
// Skip unknown addresses.
|
|
|
|
continue;
|
|
|
|
}
|
2022-05-23 18:16:18 +00:00
|
|
|
if(SetInfo.m_NumAddresses < (int)std::size(SetInfo.m_aAddresses))
|
|
|
|
{
|
|
|
|
SetInfo.m_aAddresses[SetInfo.m_NumAddresses] = ParsedAddr;
|
|
|
|
SetInfo.m_NumAddresses += 1;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
if(SetInfo.m_NumAddresses > 0)
|
|
|
|
{
|
|
|
|
vServers.push_back(SetInfo);
|
2018-07-11 20:46:04 +00:00
|
|
|
}
|
|
|
|
}
|
2022-06-15 17:34:41 +00:00
|
|
|
*pvServers = vServers;
|
2018-07-11 20:46:04 +00:00
|
|
|
return false;
|
|
|
|
}
|
2021-05-13 00:46:28 +00:00
|
|
|
|
|
|
|
static const char *DEFAULT_SERVERLIST_URLS[] = {
|
2022-08-30 08:09:06 +00:00
|
|
|
"https://master1.ddnet.org/ddnet/15/servers.json",
|
|
|
|
"https://master2.ddnet.org/ddnet/15/servers.json",
|
|
|
|
"https://master3.ddnet.org/ddnet/15/servers.json",
|
|
|
|
"https://master4.ddnet.org/ddnet/15/servers.json",
|
2021-05-13 00:46:28 +00:00
|
|
|
};
|
|
|
|
|
2024-03-14 10:19:32 +00:00
|
|
|
IServerBrowserHttp *CreateServerBrowserHttp(IEngine *pEngine, IStorage *pStorage, IHttp *pHttp, const char *pPreviousBestUrl)
|
2018-07-11 20:46:04 +00:00
|
|
|
{
|
2021-05-13 00:46:28 +00:00
|
|
|
char aaUrls[CChooseMaster::MAX_URLS][256];
|
2021-05-13 23:25:59 +00:00
|
|
|
const char *apUrls[CChooseMaster::MAX_URLS] = {0};
|
2021-05-13 00:46:28 +00:00
|
|
|
const char **ppUrls = apUrls;
|
|
|
|
int NumUrls = 0;
|
Rewrite and fix line reader, read entire file into memory
Read the entire file into memory immediately when the line reader is initialized instead of using a fixed size buffer of size 32769, which leads to broken line reading for larger files (closes #8431).
Replace the `CLineReader::Init` function with the `CLineReader::OpenFile` function, which additionally checks whether the file contains any null bytes (indicates that the file is likely not a text file).
As the file will be read into memory entirely in the `OpenFile` function, is can also be closed immediately, since using the `io_read_all_str` function should ensure that nothing more can be read from the file. This also simplifies the usage of the `CLineReader` class, as manually closing the file is inconvenient and error-prone. In fact, the file handle for the `ddnet-serverlist-urls.cfg` file was previously not closed properly.
Benchmarking on Windows did not show any noticeable performance impact of this change both for smaller files (config and language files) and larger files (synthetic ~10 MiB files).
Let the `CLineReader::Get` function return `const char *` instead of `char *`, since users of this class should not modify the internal buffer. Also, since the entire file is read into memory now, the returned strings are now valid until the respective line reader object is destructed, instead of only until the `Get` function is called again. This simplifies the usage in some cases where all lines are handled, since additional temporary buffers are now unnecessary.
Remove the `IOFLAG_SKIP_BOM` flag from the `io_open` function, as this flag was only used together with the line reader. This was inconvenient to use, as any use of the `io_open` function specifically for line readers had to be used with `IOFLAG_SKIP_BOM`. Now, the line reader transparently skips the UTF-8 BOM internally. Skipping the UTF-8 BOM never worked with the `IStorage::ReadFileStr` function, because `io_length` is used in the `io_read_all` function, which rewinds the file position to before the UTF-8 BOM that was skipped by using `IOFLAG_SKIP_BOM`. In any case, as the `ReadFileStr` function is currently unused, this has/had no further effect. The respective test cases for `IOFLAG_SKIP_BOM` are removed.
Add more test cases for the `CLineReader` class. All test cases are checked both with and without the UTF-8 BOM. Additional tests with mixed new lines, empty lines with different new lines, and internal null bytes are added.
Consistently use the construct `while(const char *pLine = LineReader.Get())` to iterate over all lines of a line reader. In this case, the assignment inside the `while`-loop seems acceptable over the alternatives, e.g. `while(true)` with `break` or adding a function `CLineReader::ForAll(std::function<const char *> Consumer)`.
2024-06-03 22:00:04 +00:00
|
|
|
CLineReader LineReader;
|
|
|
|
if(LineReader.OpenFile(pStorage->OpenFile("ddnet-serverlist-urls.cfg", IOFLAG_READ, IStorage::TYPE_ALL)))
|
2021-05-13 00:46:28 +00:00
|
|
|
{
|
Rewrite and fix line reader, read entire file into memory
Read the entire file into memory immediately when the line reader is initialized instead of using a fixed size buffer of size 32769, which leads to broken line reading for larger files (closes #8431).
Replace the `CLineReader::Init` function with the `CLineReader::OpenFile` function, which additionally checks whether the file contains any null bytes (indicates that the file is likely not a text file).
As the file will be read into memory entirely in the `OpenFile` function, is can also be closed immediately, since using the `io_read_all_str` function should ensure that nothing more can be read from the file. This also simplifies the usage of the `CLineReader` class, as manually closing the file is inconvenient and error-prone. In fact, the file handle for the `ddnet-serverlist-urls.cfg` file was previously not closed properly.
Benchmarking on Windows did not show any noticeable performance impact of this change both for smaller files (config and language files) and larger files (synthetic ~10 MiB files).
Let the `CLineReader::Get` function return `const char *` instead of `char *`, since users of this class should not modify the internal buffer. Also, since the entire file is read into memory now, the returned strings are now valid until the respective line reader object is destructed, instead of only until the `Get` function is called again. This simplifies the usage in some cases where all lines are handled, since additional temporary buffers are now unnecessary.
Remove the `IOFLAG_SKIP_BOM` flag from the `io_open` function, as this flag was only used together with the line reader. This was inconvenient to use, as any use of the `io_open` function specifically for line readers had to be used with `IOFLAG_SKIP_BOM`. Now, the line reader transparently skips the UTF-8 BOM internally. Skipping the UTF-8 BOM never worked with the `IStorage::ReadFileStr` function, because `io_length` is used in the `io_read_all` function, which rewinds the file position to before the UTF-8 BOM that was skipped by using `IOFLAG_SKIP_BOM`. In any case, as the `ReadFileStr` function is currently unused, this has/had no further effect. The respective test cases for `IOFLAG_SKIP_BOM` are removed.
Add more test cases for the `CLineReader` class. All test cases are checked both with and without the UTF-8 BOM. Additional tests with mixed new lines, empty lines with different new lines, and internal null bytes are added.
Consistently use the construct `while(const char *pLine = LineReader.Get())` to iterate over all lines of a line reader. In this case, the assignment inside the `while`-loop seems acceptable over the alternatives, e.g. `while(true)` with `break` or adding a function `CLineReader::ForAll(std::function<const char *> Consumer)`.
2024-06-03 22:00:04 +00:00
|
|
|
while(const char *pLine = LineReader.Get())
|
2021-05-13 00:46:28 +00:00
|
|
|
{
|
Rewrite and fix line reader, read entire file into memory
Read the entire file into memory immediately when the line reader is initialized instead of using a fixed size buffer of size 32769, which leads to broken line reading for larger files (closes #8431).
Replace the `CLineReader::Init` function with the `CLineReader::OpenFile` function, which additionally checks whether the file contains any null bytes (indicates that the file is likely not a text file).
As the file will be read into memory entirely in the `OpenFile` function, is can also be closed immediately, since using the `io_read_all_str` function should ensure that nothing more can be read from the file. This also simplifies the usage of the `CLineReader` class, as manually closing the file is inconvenient and error-prone. In fact, the file handle for the `ddnet-serverlist-urls.cfg` file was previously not closed properly.
Benchmarking on Windows did not show any noticeable performance impact of this change both for smaller files (config and language files) and larger files (synthetic ~10 MiB files).
Let the `CLineReader::Get` function return `const char *` instead of `char *`, since users of this class should not modify the internal buffer. Also, since the entire file is read into memory now, the returned strings are now valid until the respective line reader object is destructed, instead of only until the `Get` function is called again. This simplifies the usage in some cases where all lines are handled, since additional temporary buffers are now unnecessary.
Remove the `IOFLAG_SKIP_BOM` flag from the `io_open` function, as this flag was only used together with the line reader. This was inconvenient to use, as any use of the `io_open` function specifically for line readers had to be used with `IOFLAG_SKIP_BOM`. Now, the line reader transparently skips the UTF-8 BOM internally. Skipping the UTF-8 BOM never worked with the `IStorage::ReadFileStr` function, because `io_length` is used in the `io_read_all` function, which rewinds the file position to before the UTF-8 BOM that was skipped by using `IOFLAG_SKIP_BOM`. In any case, as the `ReadFileStr` function is currently unused, this has/had no further effect. The respective test cases for `IOFLAG_SKIP_BOM` are removed.
Add more test cases for the `CLineReader` class. All test cases are checked both with and without the UTF-8 BOM. Additional tests with mixed new lines, empty lines with different new lines, and internal null bytes are added.
Consistently use the construct `while(const char *pLine = LineReader.Get())` to iterate over all lines of a line reader. In this case, the assignment inside the `while`-loop seems acceptable over the alternatives, e.g. `while(true)` with `break` or adding a function `CLineReader::ForAll(std::function<const char *> Consumer)`.
2024-06-03 22:00:04 +00:00
|
|
|
if(NumUrls == CChooseMaster::MAX_URLS)
|
2021-05-13 00:46:28 +00:00
|
|
|
{
|
|
|
|
break;
|
|
|
|
}
|
2022-07-09 16:14:56 +00:00
|
|
|
str_copy(aaUrls[NumUrls], pLine);
|
2021-05-13 00:46:28 +00:00
|
|
|
apUrls[NumUrls] = aaUrls[NumUrls];
|
|
|
|
NumUrls += 1;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
if(NumUrls == 0)
|
|
|
|
{
|
|
|
|
ppUrls = DEFAULT_SERVERLIST_URLS;
|
2022-03-30 13:16:19 +00:00
|
|
|
NumUrls = std::size(DEFAULT_SERVERLIST_URLS);
|
2021-05-13 00:46:28 +00:00
|
|
|
}
|
|
|
|
int PreviousBestIndex = -1;
|
|
|
|
for(int i = 0; i < NumUrls; i++)
|
|
|
|
{
|
|
|
|
if(str_comp(ppUrls[i], pPreviousBestUrl) == 0)
|
|
|
|
{
|
|
|
|
PreviousBestIndex = i;
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
}
|
2024-03-14 10:19:32 +00:00
|
|
|
return new CServerBrowserHttp(pEngine, pHttp, ppUrls, NumUrls, PreviousBestIndex);
|
2018-07-11 20:46:04 +00:00
|
|
|
}
|