Merge pull request #8530 from Robyt3/Engine-LineReader-io_read_all

Rewrite and fix line reader, read entire file into memory

src/android/android_main.cpp

@@ -120,17 +120,14 @@ public:
 
 static std::vector<CIntegrityFileLine> ReadIntegrityFile(const char *pFilename)
 {
-	IOHANDLE IntegrityFile = io_open(pFilename, IOFLAG_READ);
+	CLineReader LineReader;
-	if(!IntegrityFile)
+	if(!LineReader.OpenFile(io_open(pFilename, IOFLAG_READ)))
 	{
 		return {};
 	}
 
-	CLineReader LineReader;
-	LineReader.Init(IntegrityFile);
-	const char *pReadLine;
 	std::vector<CIntegrityFileLine> vLines;
-	while((pReadLine = LineReader.Get()))
+	while(const char *pReadLine = LineReader.Get())
 	{
 		const char *pSpaceInLine = str_rchr(pReadLine, ' ');
 		CIntegrityFileLine Line;
@@ -159,7 +156,6 @@ static std::vector<CIntegrityFileLine> ReadIntegrityFile(const char *pFilename)
 		vLines.emplace_back(std::move(Line));
 	}
 
-	io_close(IntegrityFile);
 	return vLines;
 }
 

src/base/system.cpp

@@ -251,9 +251,9 @@ bool mem_has_null(const void *block, size_t size)
 	return false;
 }
 
-IOHANDLE io_open_impl(const char *filename, int flags)
+IOHANDLE io_open(const char *filename, int flags)
 {
-	dbg_assert(flags == (IOFLAG_READ | IOFLAG_SKIP_BOM) || flags == IOFLAG_READ || flags == IOFLAG_WRITE || flags == IOFLAG_APPEND, "flags must be read, read+skipbom, write or append");
+	dbg_assert(flags == IOFLAG_READ || flags == IOFLAG_WRITE || flags == IOFLAG_APPEND, "flags must be read, write or append");
 #if defined(CONF_FAMILY_WINDOWS)
 	const std::wstring wide_filename = windows_utf8_to_wide(filename);
 	DWORD desired_access;
@@ -313,21 +313,6 @@ IOHANDLE io_open_impl(const char *filename, int flags)
 #endif
 }
 
-IOHANDLE io_open(const char *filename, int flags)
-{
-	IOHANDLE result = io_open_impl(filename, flags);
-	unsigned char buf[3];
-	if((flags & IOFLAG_SKIP_BOM) == 0 || !result)
-	{
-		return result;
-	}
-	if(io_read(result, buf, sizeof(buf)) != 3 || buf[0] != 0xef || buf[1] != 0xbb || buf[2] != 0xbf)
-	{
-		io_seek(result, 0, IOSEEK_START);
-	}
-	return result;
-}
-
 unsigned io_read(IOHANDLE io, void *buffer, unsigned size)
 {
 	return fread(buffer, 1, size, (FILE *)io);

src/base/system.h

@@ -222,7 +222,6 @@ enum
 	IOFLAG_READ = 1,
 	IOFLAG_WRITE = 2,
 	IOFLAG_APPEND = 4,
-	IOFLAG_SKIP_BOM = 8,
 
 	IOSEEK_START = 0,
 	IOSEEK_CUR = 1,
@@ -237,7 +236,7 @@ enum
  * @param File to open.
  * @param flags A set of IOFLAG flags.
  *
- * @sa IOFLAG_READ, IOFLAG_WRITE, IOFLAG_APPEND, IOFLAG_SKIP_BOM.
+ * @sa IOFLAG_READ, IOFLAG_WRITE, IOFLAG_APPEND.
  *
  * @return A handle to the file on success and 0 on failure.
  *

src/engine/client/backend/opengl/opengl_sl.cpp

@@ -23,10 +23,10 @@ bool CGLSL::LoadShader(CGLSLCompiler *pCompiler, IStorage *pStorage, const char
 {
 	if(m_IsLoaded)
 		return true;
-	IOHANDLE f = pStorage->OpenFile(pFile, IOFLAG_READ | IOFLAG_SKIP_BOM, IStorage::TYPE_ALL);
 
+	CLineReader LineReader;
 	std::vector<std::string> vLines;
-	if(f)
+	if(LineReader.OpenFile(pStorage->OpenFile(pFile, IOFLAG_READ, IStorage::TYPE_ALL)))
 	{
 		EBackendType BackendType = pCompiler->m_IsOpenGLES ? BACKEND_TYPE_OPENGL_ES : BACKEND_TYPE_OPENGL;
 		bool IsNewOpenGL = (BackendType == BACKEND_TYPE_OPENGL ? (pCompiler->m_OpenGLVersionMajor >= 4 || (pCompiler->m_OpenGLVersionMajor == 3 && pCompiler->m_OpenGLVersionMinor == 3)) : pCompiler->m_OpenGLVersionMajor >= 3);
@@ -81,17 +81,13 @@ bool CGLSL::LoadShader(CGLSLCompiler *pCompiler, IStorage *pStorage, const char
 			vLines.emplace_back("#extension GL_EXT_texture_array : enable\r\n");
 		}
 
-		CLineReader LineReader;
+		while(const char *pReadLine = LineReader.Get())
-		LineReader.Init(f);
-		char *pReadLine = NULL;
-		while((pReadLine = LineReader.Get()))
 		{
 			std::string Line;
 			pCompiler->ParseLine(Line, pReadLine, Type == GL_FRAGMENT_SHADER ? GLSL_SHADER_COMPILER_TYPE_FRAGMENT : GLSL_SHADER_COMPILER_TYPE_VERTEX);
 			Line.append("\r\n");
 			vLines.push_back(Line);
 		}
-		io_close(f);
 
 		const char **ShaderCode = new const char *[vLines.size()];
 

src/engine/client/serverbrowser_http.cpp

@@ -540,15 +540,12 @@ IServerBrowserHttp *CreateServerBrowserHttp(IEngine *pEngine, IStorage *pStorage
 	const char *apUrls[CChooseMaster::MAX_URLS] = {0};
 	const char **ppUrls = apUrls;
 	int NumUrls = 0;
-	IOHANDLE File = pStorage->OpenFile("ddnet-serverlist-urls.cfg", IOFLAG_READ | IOFLAG_SKIP_BOM, IStorage::TYPE_ALL);
+	CLineReader LineReader;
-	if(File)
+	if(LineReader.OpenFile(pStorage->OpenFile("ddnet-serverlist-urls.cfg", IOFLAG_READ, IStorage::TYPE_ALL)))
 	{
-		CLineReader Lines;
+		while(const char *pLine = LineReader.Get())
-		Lines.Init(File);
-		while(NumUrls < CChooseMaster::MAX_URLS)
 		{
-			const char *pLine = Lines.Get();
+			if(NumUrls == CChooseMaster::MAX_URLS)
-			if(!pLine)
 			{
 				break;
 			}

src/engine/server/server.cpp

@@ -3860,18 +3860,18 @@ const char *CServer::GetAnnouncementLine(const char *pFileName)
 		str_copy(m_aAnnouncementFile, pFileName);
 		m_vAnnouncements.clear();
 
-		IOHANDLE File = m_pStorage->OpenFile(pFileName, IOFLAG_READ | IOFLAG_SKIP_BOM, IStorage::TYPE_ALL);
+		CLineReader LineReader;
-		if(!File)
+		if(!LineReader.OpenFile(m_pStorage->OpenFile(pFileName, IOFLAG_READ, IStorage::TYPE_ALL)))
+		{
 			return 0;
-
+		}
-		char *pLine;
+		while(const char *pLine = LineReader.Get())
-		CLineReader Reader;
+		{
-		Reader.Init(File);
-		while((pLine = Reader.Get()))
 			if(str_length(pLine) && pLine[0] != '#')
+			{
 				m_vAnnouncements.emplace_back(pLine);
-
+			}
-		io_close(File);
+		}
 	}
 
 	if(m_vAnnouncements.empty())

src/engine/shared/console.cpp

@@ -622,23 +622,19 @@ bool CConsole::ExecuteFile(const char *pFilename, int ClientId, bool LogFailure,
 	m_pFirstExec = &ThisFile;
 
 	// exec the file
-	IOHANDLE File = m_pStorage->OpenFile(pFilename, IOFLAG_READ | IOFLAG_SKIP_BOM, StorageType);
+	CLineReader LineReader;
-
 	bool Success = false;
 	char aBuf[32 + IO_MAX_PATH_LENGTH];
-	if(File)
+	if(LineReader.OpenFile(m_pStorage->OpenFile(pFilename, IOFLAG_READ, StorageType)))
 	{
 		str_format(aBuf, sizeof(aBuf), "executing '%s'", pFilename);
 		Print(IConsole::OUTPUT_LEVEL_STANDARD, "console", aBuf);
 
-		CLineReader Reader;
+		while(const char *pLine = LineReader.Get())
-		Reader.Init(File);
+		{
-
-		char *pLine;
-		while((pLine = Reader.Get()))
 			ExecuteLine(pLine, ClientId);
+		}
 
-		io_close(File);
 		Success = true;
 	}
 	else if(LogFailure)

src/engine/shared/linereader.cpp

@@ -4,97 +4,92 @@
 
 #include <base/system.h>
 
-void CLineReader::Init(IOHANDLE File)
+CLineReader::CLineReader()
 {
-	m_BufferMaxSize = sizeof(m_aBuffer) - 1;
+	m_pBuffer = nullptr;
-	m_BufferSize = 0;
-	m_BufferPos = 0;
-	m_File = File;
 }
 
-char *CLineReader::Get()
+CLineReader::~CLineReader()
 {
-	unsigned LineStart = m_BufferPos;
+	free(m_pBuffer);
-	bool CRLFBreak = false;
+}
 
-	while(true)
+bool CLineReader::OpenFile(IOHANDLE File)
+{
+	if(!File)
 	{
-		if(m_BufferPos >= m_BufferSize)
+		return false;
-		{
+	}
-			// fetch more
+	char *pBuffer = io_read_all_str(File);
+	io_close(File);
+	if(pBuffer == nullptr)
+	{
+		return false;
+	}
+	OpenBuffer(pBuffer);
+	return true;
+}
 
-			// move the remaining part to the front
+void CLineReader::OpenBuffer(char *pBuffer)
-			unsigned Left = m_BufferSize - LineStart;
+{
+	dbg_assert(pBuffer != nullptr, "Line reader initialized without valid buffer");
 
-			if(LineStart > m_BufferSize)
+	m_pBuffer = pBuffer;
-				Left = 0;
+	m_BufferPos = 0;
-			if(Left)
+	m_ReadLastLine = false;
-				mem_move(m_aBuffer, &m_aBuffer[LineStart], Left);
-			m_BufferPos = Left;
 
-			// fill the buffer
+	// Skip UTF-8 BOM
-			unsigned Read = io_read(m_File, &m_aBuffer[m_BufferPos], m_BufferMaxSize - m_BufferPos);
+	if(m_pBuffer[0] == '\xEF' && m_pBuffer[1] == '\xBB' && m_pBuffer[2] == '\xBF')
-			m_BufferSize = Left + Read;
+	{
-			LineStart = 0;
+		m_BufferPos += 3;
-
+	}
-			if(!Read)
+}
-			{
+
-				if(Left)
+const char *CLineReader::Get()
-				{
+{
-					m_aBuffer[Left] = '\0'; // return the last line
+	dbg_assert(m_pBuffer != nullptr, "Line reader not initialized");
-					m_BufferPos = Left;
+	if(m_ReadLastLine)
-					m_BufferSize = Left;
+	{
-					if(!str_utf8_check(m_aBuffer))
+		return nullptr;
-					{
+	}
-						LineStart = m_BufferPos;
+
-						CRLFBreak = false;
+	unsigned LineStart = m_BufferPos;
-						continue; // skip lines containing invalid UTF-8
+	while(true)
-					}
+	{
-					return m_aBuffer;
+		if(m_pBuffer[m_BufferPos] == '\0' || m_pBuffer[m_BufferPos] == '\n' || (m_pBuffer[m_BufferPos] == '\r' && m_pBuffer[m_BufferPos + 1] == '\n'))
-				}
+		{
-				return nullptr; // we are done!
+			if(m_pBuffer[m_BufferPos] == '\0')
-			}
+			{
-		}
+				m_ReadLastLine = true;
-		else
+			}
-		{
+			else
-			if(m_aBuffer[m_BufferPos] == '\n' || m_aBuffer[m_BufferPos] == '\r')
+			{
-			{
+				if(m_pBuffer[m_BufferPos] == '\r')
-				// line found
+				{
-				if(m_aBuffer[m_BufferPos] == '\r')
+					m_pBuffer[m_BufferPos] = '\0';
-				{
+					++m_BufferPos;
-					if(m_BufferPos + 1 >= m_BufferSize)
+				}
-					{
+				m_pBuffer[m_BufferPos] = '\0';
-						// read more to get the connected '\n'
+				++m_BufferPos;
-						CRLFBreak = true;
+			}
-						++m_BufferPos;
+
-						continue;
+			if(!str_utf8_check(&m_pBuffer[LineStart]))
-					}
+			{
-					else if(m_aBuffer[m_BufferPos + 1] == '\n')
+				// Skip lines containing invalid UTF-8
-						m_aBuffer[m_BufferPos++] = '\0';
+				if(m_ReadLastLine)
-				}
+				{
-				m_aBuffer[m_BufferPos++] = '\0';
+					return nullptr;
-				if(!str_utf8_check(&m_aBuffer[LineStart]))
+				}
-				{
+				LineStart = m_BufferPos;
-					LineStart = m_BufferPos;
+				continue;
-					CRLFBreak = false;
+			}
-					continue; // skip lines containing invalid UTF-8
+			// Skip trailing empty line
-				}
+			if(m_ReadLastLine && m_pBuffer[LineStart] == '\0')
-				return &m_aBuffer[LineStart];
+			{
-			}
+				return nullptr;
-			else if(CRLFBreak)
+			}
-			{
+			return &m_pBuffer[LineStart];
-				if(m_aBuffer[m_BufferPos] == '\n')
+		}
-					m_aBuffer[m_BufferPos++] = '\0';
+		++m_BufferPos;
-				if(!str_utf8_check(&m_aBuffer[LineStart]))
-				{
-					LineStart = m_BufferPos;
-					CRLFBreak = false;
-					continue; // skip lines containing invalid UTF-8
-				}
-				return &m_aBuffer[LineStart];
-			}
-			else
-				m_BufferPos++;
-		}
 	}
 }

src/engine/shared/linereader.h

@@ -4,17 +4,20 @@
 #define ENGINE_SHARED_LINEREADER_H
 #include <base/types.h>
 
-// buffered stream for reading lines, should perhaps be something smaller
+// buffered stream for reading lines
 class CLineReader
 {
-	char m_aBuffer[4 * 8192 + 1]; // 1 additional byte for null termination
+	char *m_pBuffer;
 	unsigned m_BufferPos;
-	unsigned m_BufferSize;
+	bool m_ReadLastLine;
-	unsigned m_BufferMaxSize;
-	IOHANDLE m_File;
 
 public:
-	void Init(IOHANDLE File);
+	CLineReader();
-	char *Get(); // Returned string is only valid until next Get() call
+	~CLineReader();
+
+	bool OpenFile(IOHANDLE File);
+	void OpenBuffer(char *pBuffer); // Buffer must have been allocated with malloc, will be freed by the line reader
+
+	const char *Get(); // Returned string is valid until the line reader is destroyed
 };
 #endif

src/engine/shared/storage.cpp

@@ -127,7 +127,7 @@ public:
 	void LoadPaths(const char *pArgv0)
 	{
 		// check current directory
-		IOHANDLE File = io_open("storage.cfg", IOFLAG_READ | IOFLAG_SKIP_BOM);
+		IOHANDLE File = io_open("storage.cfg", IOFLAG_READ);
 		if(!File)
 		{
 			// check usable path in argv[0]
@@ -140,7 +140,7 @@ public:
 				char aBuffer[IO_MAX_PATH_LENGTH];
 				str_copy(aBuffer, pArgv0, Pos + 1);
 				str_append(aBuffer, "/storage.cfg");
-				File = io_open(aBuffer, IOFLAG_READ | IOFLAG_SKIP_BOM);
+				File = io_open(aBuffer, IOFLAG_READ);
 			}
 
 			if(Pos >= IO_MAX_PATH_LENGTH || !File)
@@ -151,10 +151,12 @@ public:
 		}
 
 		CLineReader LineReader;
-		LineReader.Init(File);
+		if(!LineReader.OpenFile(File))
-
+		{
-		char *pLine;
+			dbg_msg("storage", "couldn't open storage.cfg");
-		while((pLine = LineReader.Get()))
+			return;
+		}
+		while(const char *pLine = LineReader.Get())
 		{
 			const char *pLineWithoutPrefix = str_startswith(pLine, "add_path ");
 			if(pLineWithoutPrefix)
@@ -163,8 +165,6 @@ public:
 			}
 		}
 
-		io_close(File);
-
 		if(!m_NumPaths)
 			dbg_msg("storage", "no paths found in storage.cfg");
 	}
@@ -565,7 +565,7 @@ public:
 
 	char *ReadFileStr(const char *pFilename, int Type) override
 	{
-		IOHANDLE File = OpenFile(pFilename, IOFLAG_READ | IOFLAG_SKIP_BOM, Type);
+		IOHANDLE File = OpenFile(pFilename, IOFLAG_READ, Type);
 		if(!File)
 			return nullptr;
 		char *pResult = io_read_all_str(File);

src/game/client/components/countryflags.cpp

@@ -16,8 +16,8 @@
 void CCountryFlags::LoadCountryflagsIndexfile()
 {
 	const char *pFilename = "countryflags/index.txt";
-	IOHANDLE File = Storage()->OpenFile(pFilename, IOFLAG_READ | IOFLAG_SKIP_BOM, IStorage::TYPE_ALL);
+	CLineReader LineReader;
-	if(!File)
+	if(!LineReader.OpenFile(Storage()->OpenFile(pFilename, IOFLAG_READ, IStorage::TYPE_ALL)))
 	{
 		char aBuf[128];
 		str_format(aBuf, sizeof(aBuf), "couldn't open index file '%s'", pFilename);
@@ -26,16 +26,13 @@ void CCountryFlags::LoadCountryflagsIndexfile()
 	}
 
 	char aOrigin[128];
-	CLineReader LineReader;
+	while(const char *pLine = LineReader.Get())
-	LineReader.Init(File);
-	char *pLine;
-	while((pLine = LineReader.Get()))
 	{
 		if(!str_length(pLine) || pLine[0] == '#') // skip empty lines and comments
 			continue;
 
 		str_copy(aOrigin, pLine);
-		char *pReplacement = LineReader.Get();
+		const char *pReplacement = LineReader.Get();
 		if(!pReplacement)
 		{
 			Console()->Print(IConsole::OUTPUT_LEVEL_ADDINFO, "countryflags", "unexpected end of index file");
@@ -84,7 +81,7 @@ void CCountryFlags::LoadCountryflagsIndexfile()
 		}
 		m_vCountryFlags.push_back(CountryFlag);
 	}
-	io_close(File);
+
 	std::sort(m_vCountryFlags.begin(), m_vCountryFlags.end());
 
 	// find index of default item

src/game/editor/auto_map.cpp

@@ -49,8 +49,8 @@ void CAutoMapper::Load(const char *pTileName)
 {
 	char aPath[IO_MAX_PATH_LENGTH];
 	str_format(aPath, sizeof(aPath), "editor/automap/%s.rules", pTileName);
-	IOHANDLE RulesFile = Storage()->OpenFile(aPath, IOFLAG_READ | IOFLAG_SKIP_BOM, IStorage::TYPE_ALL);
+	CLineReader LineReader;
-	if(!RulesFile)
+	if(!LineReader.OpenFile(Storage()->OpenFile(aPath, IOFLAG_READ, IStorage::TYPE_ALL)))
 	{
 		char aBuf[IO_MAX_PATH_LENGTH + 32];
 		str_format(aBuf, sizeof(aBuf), "failed to load %s", aPath);
@@ -58,15 +58,12 @@ void CAutoMapper::Load(const char *pTileName)
 		return;
 	}
 
-	CLineReader LineReader;
-	LineReader.Init(RulesFile);
-
 	CConfiguration *pCurrentConf = nullptr;
 	CRun *pCurrentRun = nullptr;
 	CIndexRule *pCurrentIndex = nullptr;
 
 	// read each line
-	while(char *pLine = LineReader.Get())
+	while(const char *pLine = LineReader.Get())
 	{
 		// skip blank/empty lines as well as comments
 		if(str_length(pLine) > 0 && pLine[0] != '#' && pLine[0] != '\n' && pLine[0] != '\r' && pLine[0] != '\t' && pLine[0] != '\v' && pLine[0] != ' ')
@@ -365,8 +362,6 @@ void CAutoMapper::Load(const char *pTileName)
 		}
 	}
 
-	io_close(RulesFile);
-
 	char aBuf[IO_MAX_PATH_LENGTH + 16];
 	str_format(aBuf, sizeof(aBuf), "loaded %s", aPath);
 	Console()->Print(IConsole::OUTPUT_LEVEL_DEBUG, "editor/automap", aBuf);

src/game/localization.cpp

@@ -23,18 +23,14 @@ void CLocalizationDatabase::LoadIndexfile(IStorage *pStorage, IConsole *pConsole
 	m_vLanguages.emplace_back("English", "", 826, vEnglishLanguageCodes);
 
 	const char *pFilename = "languages/index.txt";
-	IOHANDLE File = pStorage->OpenFile(pFilename, IOFLAG_READ | IOFLAG_SKIP_BOM, IStorage::TYPE_ALL);
+	CLineReader LineReader;
-	if(!File)
+	if(!LineReader.OpenFile(pStorage->OpenFile(pFilename, IOFLAG_READ, IStorage::TYPE_ALL)))
 	{
 		log_error("localization", "Couldn't open index file '%s'", pFilename);
 		return;
 	}
 
-	CLineReader LineReader;
+	while(const char *pLine = LineReader.Get())
-	LineReader.Init(File);
-
-	const char *pLine;
-	while((pLine = LineReader.Get()))
 	{
 		if(!str_length(pLine) || pLine[0] == '#') // skip empty lines and comments
 			continue;
@@ -105,8 +101,6 @@ void CLocalizationDatabase::LoadIndexfile(IStorage *pStorage, IConsole *pConsole
 		m_vLanguages.emplace_back(aNativeName, aFileName, str_toint(aCountryCode), vLanguageCodes);
 	}
 
-	io_close(File);
-
 	std::sort(m_vLanguages.begin(), m_vLanguages.end());
 }
 
@@ -179,8 +173,8 @@ bool CLocalizationDatabase::Load(const char *pFilename, IStorage *pStorage, ICon
 		return true;
 	}
 
-	IOHANDLE IoHandle = pStorage->OpenFile(pFilename, IOFLAG_READ | IOFLAG_SKIP_BOM, IStorage::TYPE_ALL);
+	CLineReader LineReader;
-	if(!IoHandle)
+	if(!LineReader.OpenFile(pStorage->OpenFile(pFilename, IOFLAG_READ, IStorage::TYPE_ALL)))
 		return false;
 
 	log_info("localization", "loaded '%s'", pFilename);
@@ -189,11 +183,8 @@ bool CLocalizationDatabase::Load(const char *pFilename, IStorage *pStorage, ICon
 
 	char aContext[512];
 	char aOrigin[512];
-	CLineReader LineReader;
-	LineReader.Init(IoHandle);
-	char *pLine;
 	int Line = 0;
-	while((pLine = LineReader.Get()))
+	while(const char *pLine = LineReader.Get())
 	{
 		Line++;
 		if(!str_length(pLine))
@@ -225,7 +216,7 @@ bool CLocalizationDatabase::Load(const char *pFilename, IStorage *pStorage, ICon
 		}
 
 		str_copy(aOrigin, pLine);
-		char *pReplacement = LineReader.Get();
+		const char *pReplacement = LineReader.Get();
 		if(!pReplacement)
 		{
 			log_error("localization", "unexpected end of file after original '%s' on line %d", aOrigin, Line);
@@ -242,7 +233,6 @@ bool CLocalizationDatabase::Load(const char *pFilename, IStorage *pStorage, ICon
 		pReplacement += 3;
 		AddString(aOrigin, pReplacement, aContext);
 	}
-	io_close(IoHandle);
 	std::sort(m_vStrings.begin(), m_vStrings.end());
 	return true;
 }

src/game/server/gamecontext.cpp

@@ -3846,21 +3846,17 @@ void CGameContext::OnInit(const void *pPersistentData)
 		m_pController = new CGameControllerDDRace(this);
 
 	const char *pCensorFilename = "censorlist.txt";
-	IOHANDLE File = Storage()->OpenFile(pCensorFilename, IOFLAG_READ | IOFLAG_SKIP_BOM, IStorage::TYPE_ALL);
+	CLineReader LineReader;
-	if(!File)
+	if(LineReader.OpenFile(Storage()->OpenFile(pCensorFilename, IOFLAG_READ, IStorage::TYPE_ALL)))
 	{
-		dbg_msg("censorlist", "failed to open '%s'", pCensorFilename);
+		while(const char *pLine = LineReader.Get())
-	}
-	else
-	{
-		CLineReader LineReader;
-		LineReader.Init(File);
-		char *pLine;
-		while((pLine = LineReader.Get()))
 		{
 			m_vCensorlist.emplace_back(pLine);
 		}
-		io_close(File);
+	}
+	else
+	{
+		dbg_msg("censorlist", "failed to open '%s'", pCensorFilename);
 	}
 
 	m_TeeHistorianActive = g_Config.m_SvTeeHistorian;
@@ -4064,36 +4060,28 @@ void CGameContext::OnMapChange(char *pNewMapName, int MapNameSize)
 	char aConfig[IO_MAX_PATH_LENGTH];
 	str_format(aConfig, sizeof(aConfig), "maps/%s.cfg", g_Config.m_SvMap);
 
-	IOHANDLE File = Storage()->OpenFile(aConfig, IOFLAG_READ | IOFLAG_SKIP_BOM, IStorage::TYPE_ALL);
+	CLineReader LineReader;
-	if(!File)
+	if(!LineReader.OpenFile(Storage()->OpenFile(aConfig, IOFLAG_READ, IStorage::TYPE_ALL)))
 	{
 		// No map-specific config, just return.
 		return;
 	}
-	CLineReader LineReader;
-	LineReader.Init(File);
 
-	std::vector<char *> vLines;
+	std::vector<const char *> vpLines;
-	char *pLine;
 	int TotalLength = 0;
-	while((pLine = LineReader.Get()))
+	while(const char *pLine = LineReader.Get())
 	{
-		int Length = str_length(pLine) + 1;
+		vpLines.push_back(pLine);
-		char *pCopy = (char *)malloc(Length);
+		TotalLength += str_length(pLine) + 1;
-		str_copy(pCopy, pLine, Length);
-		vLines.push_back(pCopy);
-		TotalLength += Length;
 	}
-	io_close(File);
 
 	char *pSettings = (char *)malloc(maximum(1, TotalLength));
 	int Offset = 0;
-	for(auto &Line : vLines)
+	for(const char *pLine : vpLines)
 	{
-		int Length = str_length(Line) + 1;
+		int Length = str_length(pLine) + 1;
-		mem_copy(pSettings + Offset, Line, Length);
+		mem_copy(pSettings + Offset, pLine, Length);
 		Offset += Length;
-		free(Line);
 	}
 
 	CDataFileReader Reader;

src/game/server/score.cpp

@@ -79,20 +79,16 @@ CScore::CScore(CGameContext *pGameServer, CDbConnectionPool *pPool) :
 	secure_random_fill(aSeed, sizeof(aSeed));
 	m_Prng.Seed(aSeed);
 
-	IOHANDLE File = GameServer()->Storage()->OpenFile("wordlist.txt", IOFLAG_READ | IOFLAG_SKIP_BOM, IStorage::TYPE_ALL);
+	CLineReader LineReader;
-	if(File)
+	if(LineReader.OpenFile(GameServer()->Storage()->OpenFile("wordlist.txt", IOFLAG_READ, IStorage::TYPE_ALL)))
 	{
-		CLineReader LineReader;
+		while(const char *pLine = LineReader.Get())
-		LineReader.Init(File);
-		char *pLine;
-		while((pLine = LineReader.Get()))
 		{
 			char aWord[32] = {0};
 			sscanf(pLine, "%*s %31s", aWord);
 			aWord[31] = 0;
 			m_vWordlist.emplace_back(aWord);
 		}
-		io_close(File);
 	}
 	else
 	{

src/test/io.cpp

@@ -3,18 +3,19 @@
 
 #include <base/system.h>
 
-void TestFileRead(const char *pWritten, bool SkipBom, const char *pRead)
+void TestFileRead(const char *pWritten)
 {
 	CTestInfo Info;
 	char aBuf[512] = {0};
 	IOHANDLE File = io_open(Info.m_aFilename, IOFLAG_WRITE);
+	const int WrittenLength = str_length(pWritten);
 	ASSERT_TRUE(File);
-	EXPECT_EQ(io_write(File, pWritten, str_length(pWritten)), str_length(pWritten));
+	EXPECT_EQ(io_write(File, pWritten, WrittenLength), WrittenLength);
 	EXPECT_FALSE(io_close(File));
-	File = io_open(Info.m_aFilename, IOFLAG_READ | (SkipBom ? IOFLAG_SKIP_BOM : 0));
+	File = io_open(Info.m_aFilename, IOFLAG_READ);
 	ASSERT_TRUE(File);
-	EXPECT_EQ(io_read(File, aBuf, sizeof(aBuf)), str_length(pRead));
+	EXPECT_EQ(io_read(File, aBuf, sizeof(aBuf)), WrittenLength);
-	EXPECT_TRUE(mem_comp(aBuf, pRead, str_length(pRead)) == 0);
+	EXPECT_TRUE(mem_comp(aBuf, pWritten, WrittenLength) == 0);
 	EXPECT_FALSE(io_close(File));
 
 	fs_remove(Info.m_aFilename);
@@ -22,45 +23,21 @@ void TestFileRead(const char *pWritten, bool SkipBom, const char *pRead)
 
 TEST(Io, Read1)
 {
-	TestFileRead("", false, "");
+	TestFileRead("");
 }
 TEST(Io, Read2)
 {
-	TestFileRead("abc", false, "abc");
+	TestFileRead("abc");
 }
 TEST(Io, Read3)
 {
-	TestFileRead("\xef\xbb\xbf", false, "\xef\xbb\xbf");
+	TestFileRead("\xef\xbb\xbf");
 }
 TEST(Io, Read4)
 {
-	TestFileRead("\xef\xbb\xbfxyz", false, "\xef\xbb\xbfxyz");
+	TestFileRead("\xef\xbb\xbfxyz");
 }
 
-TEST(Io, ReadBom1)
-{
-	TestFileRead("", true, "");
-}
-TEST(Io, ReadBom2)
-{
-	TestFileRead("abc", true, "abc");
-}
-TEST(Io, ReadBom3)
-{
-	TestFileRead("\xef\xbb\xbf", true, "");
-}
-TEST(Io, ReadBom4)
-{
-	TestFileRead("\xef\xbb\xbfxyz", true, "xyz");
-}
-TEST(Io, ReadBom5)
-{
-	TestFileRead("\xef\xbb\xbf\xef\xbb\xbf", true, "\xef\xbb\xbf");
-}
-TEST(Io, ReadBom6)
-{
-	TestFileRead("\xef\xbb\xbfxyz\xef\xbb\xbf", true, "xyz\xef\xbb\xbf");
-}
 TEST(Io, CurrentExe)
 {
 	IOHANDLE CurrentExe = io_current_exe();

src/test/linereader.cpp

@@ -4,45 +4,89 @@
 #include <base/system.h>
 #include <engine/shared/linereader.h>
 
-void TestFileLineReader(const char *pWritten, bool SkipBom, std::initializer_list<const char *> pReads)
+void TestFileLineReaderRaw(const char *pWritten, unsigned WrittenLength, std::initializer_list<const char *> pReads, bool ExpectSuccess, bool WriteBom)
 {
 	CTestInfo Info;
 	IOHANDLE File = io_open(Info.m_aFilename, IOFLAG_WRITE);
 	ASSERT_TRUE(File);
-	EXPECT_EQ(io_write(File, pWritten, str_length(pWritten)), str_length(pWritten));
+	if(WriteBom)
-	EXPECT_FALSE(io_close(File));
-	File = io_open(Info.m_aFilename, IOFLAG_READ | (SkipBom ? IOFLAG_SKIP_BOM : 0));
-	ASSERT_TRUE(File);
-	CLineReader LineReader;
-	LineReader.Init(File);
-	for(const char *pRead : pReads)
 	{
-		const char *pReadLine = LineReader.Get();
+		constexpr const unsigned char UTF8_BOM[] = {0xEF, 0xBB, 0xBF};
-		ASSERT_TRUE(pReadLine);
+		EXPECT_EQ(io_write(File, UTF8_BOM, sizeof(UTF8_BOM)), sizeof(UTF8_BOM));
-		EXPECT_STREQ(pReadLine, pRead);
 	}
-	EXPECT_FALSE(LineReader.Get());
+	EXPECT_EQ(io_write(File, pWritten, WrittenLength), WrittenLength);
 	EXPECT_FALSE(io_close(File));
 
+	CLineReader LineReader;
+	const bool ActualSuccess = LineReader.OpenFile(io_open(Info.m_aFilename, IOFLAG_READ));
+	ASSERT_EQ(ActualSuccess, ExpectSuccess);
+	if(ActualSuccess)
+	{
+		for(const char *pRead : pReads)
+		{
+			const char *pReadLine = LineReader.Get();
+			ASSERT_TRUE(pReadLine) << "Line reader returned less lines than expected";
+			EXPECT_STREQ(pReadLine, pRead) << "Line reader returned unexpected line";
+		}
+		EXPECT_FALSE(LineReader.Get()) << "Line reader returned more lines than expected";
+	}
+
 	fs_remove(Info.m_aFilename);
 }
 
+void TestFileLineReaderRaw(const char *pWritten, unsigned WrittenLength, std::initializer_list<const char *> pReads, bool ExpectSuccess)
+{
+	TestFileLineReaderRaw(pWritten, WrittenLength, pReads, ExpectSuccess, false);
+	TestFileLineReaderRaw(pWritten, WrittenLength, pReads, ExpectSuccess, true);
+}
+
+void TestFileLineReader(const char *pWritten, std::initializer_list<const char *> pReads)
+{
+	TestFileLineReaderRaw(pWritten, str_length(pWritten), pReads, true);
+}
+
 TEST(LineReader, NormalNewline)
 {
-	TestFileLineReader("foo\nbar\nbaz\n", false, {"foo", "bar", "baz"});
+	TestFileLineReader("foo\nbar\nbaz", {"foo", "bar", "baz"});
+	TestFileLineReader("foo\nbar\nbaz\n", {"foo", "bar", "baz"});
 }
 
 TEST(LineReader, CRLFNewline)
 {
-	TestFileLineReader("foo\r\nbar\r\nbaz", true, {"foo", "bar", "baz"});
+	TestFileLineReader("foo\r\nbar\r\nbaz", {"foo", "bar", "baz"});
+	TestFileLineReader("foo\r\nbar\r\nbaz\r\n", {"foo", "bar", "baz"});
+}
+
+TEST(LineReader, MixedNewline)
+{
+	TestFileLineReader("1\n2\r\n3\n4\n5\r\n6", {"1", "2", "3", "4", "5", "6"});
+	TestFileLineReader("1\n2\r\n3\n4\n5\r\n6\n", {"1", "2", "3", "4", "5", "6"});
+	TestFileLineReader("1\n2\r\n3\n4\n5\r\n6\r\n", {"1", "2", "3", "4", "5", "6"});
+	TestFileLineReader("1\n2\r\n3\n4\n5\r\n6\r", {"1", "2", "3", "4", "5", "6\r"});
+}
+
+TEST(LineReader, EmptyLines)
+{
+	TestFileLineReader("\n\r\n\n\n\r\n", {"", "", "", "", ""});
+	TestFileLineReader("\n\r\n\n\n\r\n\n", {"", "", "", "", "", ""});
+	TestFileLineReader("\n\r\n\n\n\r\n\r\n", {"", "", "", "", "", ""});
+	TestFileLineReader("\n\r\n\n\n\r\n\r", {"", "", "", "", "", "\r"});
 }
 
 TEST(LineReader, Invalid)
 {
 	// Lines containing invalid UTF-8 are skipped
-	TestFileLineReader("foo\xff\nbar\xff\nbaz\xff\n", false, {});
+	TestFileLineReader("foo\xff\nbar\xff\nbaz\xff", {});
-	TestFileLineReader("foo\xff\nbar\nbaz\n", false, {"bar", "baz"});
+	TestFileLineReader("foo\xff\nbar\nbaz", {"bar", "baz"});
-	TestFileLineReader("foo\nbar\xff\nbaz\n", false, {"foo", "baz"});
+	TestFileLineReader("foo\nbar\xff\nbaz", {"foo", "baz"});
-	TestFileLineReader("foo\nbar\nbaz\xff\n", false, {"foo", "bar"});
+	TestFileLineReader("foo\nbar\nbaz\xff", {"foo", "bar"});
-	TestFileLineReader("foo\nbar1\xff\nbar2\xff\nfoobar\nbar3\xff\nbaz\n", false, {"foo", "foobar", "baz"});
+	TestFileLineReader("foo\nbar1\xff\nbar2\xff\nfoobar\nbar3\xff\nbaz", {"foo", "foobar", "baz"});
+}
+
+TEST(LineReader, NullBytes)
+{
+	// Line reader does not read any lines if the file contains null bytes
+	TestFileLineReaderRaw("foo\0\nbar\nbaz", 12, {}, false);
+	TestFileLineReaderRaw("foo\nbar\0\nbaz", 12, {}, false);
+	TestFileLineReaderRaw("foo\nbar\nbaz\0", 12, {}, false);
 }

src/tools/config_store.cpp

@@ -8,37 +8,28 @@
 
 void Process(IStorage *pStorage, const char *pMapName, const char *pConfigName)
 {
-	IOHANDLE File = pStorage->OpenFile(pConfigName, IOFLAG_READ | IOFLAG_SKIP_BOM, IStorage::TYPE_ABSOLUTE);
+	CLineReader LineReader;
-	if(!File)
+	if(!LineReader.OpenFile(pStorage->OpenFile(pConfigName, IOFLAG_READ, IStorage::TYPE_ABSOLUTE)))
 	{
 		dbg_msg("config_store", "config '%s' not found", pConfigName);
 		return;
 	}
 
-	CLineReader LineReader;
+	std::vector<const char *> vpLines;
-	LineReader.Init(File);
-
-	char *pLine;
 	int TotalLength = 0;
-	std::vector<char *> vLines;
+	while(const char *pLine = LineReader.Get())
-	while((pLine = LineReader.Get()))
 	{
-		int Length = str_length(pLine) + 1;
+		vpLines.push_back(pLine);
-		char *pCopy = (char *)malloc(Length);
+		TotalLength += str_length(pLine) + 1;
-		mem_copy(pCopy, pLine, Length);
-		vLines.push_back(pCopy);
-		TotalLength += Length;
 	}
-	io_close(File);
 
 	char *pSettings = (char *)malloc(maximum(1, TotalLength));
 	int Offset = 0;
-	for(auto &Line : vLines)
+	for(const char *pLine : vpLines)
 	{
-		int Length = str_length(Line) + 1;
+		int Length = str_length(pLine) + 1;
-		mem_copy(pSettings + Offset, Line, Length);
+		mem_copy(pSettings + Offset, pLine, Length);
 		Offset += Length;
-		free(Line);
 	}
 
 	CDataFileReader Reader;