Use separate thread to finish saving maps, add saving indicator

Compressing the data with zlib takes the majority of the time when saving a datafile. Therefore, compressing is now delayed until the `CDataFileWriter::Finish` function is called. This function is then off-loaded to another thread to make saving maps in the editor not block the rendering.

A message "Saving…" is shown in the bottom right of the editor view while a job to save a map is running in the background.

While a map is being finished in a background thread another save for the same filename cannot be initiated to prevent multiples accesses to the same file.

Closes #6762.

src/engine/shared/datafile.cpp

@@ -525,16 +525,35 @@ CDataFileWriter::CDataFileWriter()
 
 CDataFileWriter::~CDataFileWriter()
 {
+	if(m_File)
+	{
+		io_close(m_File);
+		m_File = 0;
+	}
+
 	free(m_pItemTypes);
 	m_pItemTypes = nullptr;
-	for(int i = 0; i < m_NumItems; i++)
-		free(m_pItems[i].m_pData);
-	for(int i = 0; i < m_NumDatas; ++i)
-		free(m_pDatas[i].m_pCompressedData);
-	free(m_pItems);
-	m_pItems = nullptr;
-	free(m_pDatas);
-	m_pDatas = nullptr;
+
+	if(m_pItems)
+	{
+		for(int i = 0; i < m_NumItems; i++)
+		{
+			free(m_pItems[i].m_pData);
+		}
+		free(m_pItems);
+		m_pItems = nullptr;
+	}
+
+	if(m_pDatas)
+	{
+		for(int i = 0; i < m_NumDatas; ++i)
+		{
+			free(m_pDatas[i].m_pUncompressedData);
+			free(m_pDatas[i].m_pCompressedData);
+		}
+		free(m_pDatas);
+		m_pDatas = nullptr;
+	}
 }
 
 bool CDataFileWriter::OpenFile(class IStorage *pStorage, const char *pFilename, int StorageType)
@@ -636,21 +655,12 @@ int CDataFileWriter::AddData(int Size, void *pData, int CompressionLevel)
 	dbg_assert(m_NumDatas < 1024, "too much data");
 
 	CDataInfo *pInfo = &m_pDatas[m_NumDatas];
-	unsigned long s = compressBound(Size);
-	void *pCompData = malloc(s); // temporary buffer that we use during compression
-
-	int Result = compress2((Bytef *)pCompData, &s, (Bytef *)pData, Size, CompressionLevel);
-	if(Result != Z_OK)
-	{
-		dbg_msg("datafile", "compression error %d", Result);
-		dbg_assert(0, "zlib error");
-	}
-
+	pInfo->m_pUncompressedData = malloc(Size);
+	mem_copy(pInfo->m_pUncompressedData, pData, Size);
 	pInfo->m_UncompressedSize = Size;
-	pInfo->m_CompressedSize = (int)s;
-	pInfo->m_pCompressedData = malloc(pInfo->m_CompressedSize);
-	mem_copy(pInfo->m_pCompressedData, pCompData, pInfo->m_CompressedSize);
-	free(pCompData);
+	pInfo->m_pCompressedData = nullptr;
+	pInfo->m_CompressedSize = 0;
+	pInfo->m_CompressionLevel = CompressionLevel;
 
 	m_NumDatas++;
 	return m_NumDatas - 1;
@@ -672,15 +682,31 @@ int CDataFileWriter::AddDataSwapped(int Size, void *pData)
 #endif
 }
 
-int CDataFileWriter::Finish()
+void CDataFileWriter::Finish()
 {
-	if(!m_File)
-		return 1;
+	dbg_assert((bool)m_File, "file not open");
 
 	// we should now write this file!
 	if(DEBUG)
 		dbg_msg("datafile", "writing");
 
+	// Compress data. This takes the majority of the time when saving a datafile,
+	// so it's delayed until the end so it can be off-loaded to another thread.
+	for(int i = 0; i < m_NumDatas; i++)
+	{
+		unsigned long CompressedSize = compressBound(m_pDatas[i].m_UncompressedSize);
+		m_pDatas[i].m_pCompressedData = malloc(CompressedSize);
+		const int Result = compress2((Bytef *)m_pDatas[i].m_pCompressedData, &CompressedSize, (Bytef *)m_pDatas[i].m_pUncompressedData, m_pDatas[i].m_UncompressedSize, m_pDatas[i].m_CompressionLevel);
+		m_pDatas[i].m_CompressedSize = CompressedSize;
+		free(m_pDatas[i].m_pUncompressedData);
+		m_pDatas[i].m_pUncompressedData = nullptr;
+		if(Result != Z_OK)
+		{
+			dbg_msg("datafile", "compression error %d", Result);
+			dbg_assert(false, "zlib error");
+		}
+	}
+
 	// calculate sizes
 	int ItemSize = 0;
 	for(int i = 0; i < m_NumItems; i++)
@@ -851,5 +877,4 @@ int CDataFileWriter::Finish()
 
 	if(DEBUG)
 		dbg_msg("datafile", "done");
-	return 0;
 }

src/engine/shared/datafile.h

@@ -58,9 +58,11 @@ class CDataFileWriter
 {
 	struct CDataInfo
 	{
+		void *m_pUncompressedData;
 		int m_UncompressedSize;
-		int m_CompressedSize;
 		void *m_pCompressedData;
+		int m_CompressedSize;
+		int m_CompressionLevel;
 	};
 
 	struct CItemInfo
@@ -103,14 +105,31 @@ class CDataFileWriter
 
 public:
 	CDataFileWriter();
+	CDataFileWriter(CDataFileWriter &&Other) :
+		m_NumItems(Other.m_NumItems),
+		m_NumDatas(Other.m_NumDatas),
+		m_NumItemTypes(Other.m_NumItemTypes),
+		m_NumExtendedItemTypes(Other.m_NumExtendedItemTypes)
+	{
+		m_File = Other.m_File;
+		Other.m_File = 0;
+		m_pItemTypes = Other.m_pItemTypes;
+		Other.m_pItemTypes = nullptr;
+		m_pItems = Other.m_pItems;
+		Other.m_pItems = nullptr;
+		m_pDatas = Other.m_pDatas;
+		Other.m_pDatas = nullptr;
+		mem_copy(m_aExtendedItemTypes, Other.m_aExtendedItemTypes, sizeof(m_aExtendedItemTypes));
+	}
 	~CDataFileWriter();
+
 	void Init();
 	bool OpenFile(class IStorage *pStorage, const char *pFilename, int StorageType = IStorage::TYPE_SAVE);
 	bool Open(class IStorage *pStorage, const char *pFilename, int StorageType = IStorage::TYPE_SAVE);
 	int AddData(int Size, void *pData, int CompressionLevel = Z_DEFAULT_COMPRESSION);
 	int AddDataSwapped(int Size, void *pData);
 	int AddItem(int Type, int ID, int Size, void *pData);
-	int Finish();
+	void Finish();
 };
 
 #endif

src/game/editor/editor.cpp

@@ -6441,6 +6441,7 @@ void CEditor::Render()
 		RenderStatusbar(StatusBar);
 
 	RenderPressedKeys(View);
+	RenderSavingIndicator(View);
 	RenderMousePointer();
 }
 
@@ -6466,6 +6467,17 @@ void CEditor::RenderPressedKeys(CUIRect View)
 	}
 }
 
+void CEditor::RenderSavingIndicator(CUIRect View)
+{
+	if(m_lpWriterFinishJobs.empty())
+		return;
+
+	UI()->MapScreen();
+	CUIRect Label;
+	View.Margin(20.0f, &Label);
+	UI()->DoLabel(&Label, "Saving…", 24.0f, TEXTALIGN_BR);
+}
+
 void CEditor::RenderMousePointer()
 {
 	if(!m_ShowMousePointer)
@@ -6857,6 +6869,7 @@ void CEditor::Init()
 	m_pClient = Kernel()->RequestInterface<IClient>();
 	m_pConfig = Kernel()->RequestInterface<IConfigManager>()->Values();
 	m_pConsole = Kernel()->RequestInterface<IConsole>();
+	m_pEngine = Kernel()->RequestInterface<IEngine>();
 	m_pGraphics = Kernel()->RequestInterface<IGraphics>();
 	m_pTextRender = Kernel()->RequestInterface<ITextRender>();
 	m_pStorage = Kernel()->RequestInterface<IStorage>();
@@ -7060,6 +7073,41 @@ bool CEditor::PerformAutosave()
 	}
 }
 
+void CEditor::HandleWriterFinishJobs()
+{
+	if(m_lpWriterFinishJobs.empty())
+		return;
+
+	std::shared_ptr<CDataFileWriterFinishJob> pJob = m_lpWriterFinishJobs.front();
+	if(pJob->Status() != IJob::STATE_DONE)
+		return;
+
+	char aBuf[IO_MAX_PATH_LENGTH + 32];
+	str_format(aBuf, sizeof(aBuf), "saving '%s' done", pJob->GetFileName());
+	Console()->Print(IConsole::OUTPUT_LEVEL_ADDINFO, "editor", aBuf);
+
+	// send rcon.. if we can
+	if(Client()->RconAuthed())
+	{
+		CServerInfo CurrentServerInfo;
+		Client()->GetServerInfo(&CurrentServerInfo);
+		NETADDR ServerAddr = Client()->ServerAddress();
+		const unsigned char aIpv4Localhost[16] = {127, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
+		const unsigned char aIpv6Localhost[16] = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1};
+
+		// and if we're on localhost
+		if(!mem_comp(ServerAddr.ip, aIpv4Localhost, sizeof(aIpv4Localhost)) || !mem_comp(ServerAddr.ip, aIpv6Localhost, sizeof(aIpv6Localhost)))
+		{
+			char aMapName[128];
+			IStorage::StripPathAndExtension(pJob->GetFileName(), aMapName, sizeof(aMapName));
+			if(!str_comp(aMapName, CurrentServerInfo.m_aMap))
+				Client()->Rcon("reload");
+		}
+	}
+
+	m_lpWriterFinishJobs.pop_front();
+}
+
 void CEditor::OnUpdate()
 {
 	CUIElementBase::Init(UI()); // update static pointer because game and editor use separate UI
@@ -7072,6 +7120,7 @@ void CEditor::OnUpdate()
 
 	HandleCursorMovement();
 	HandleAutosave();
+	HandleWriterFinishJobs();
 }
 
 void CEditor::OnRender()

src/game/editor/editor.h

@@ -12,11 +12,15 @@
 #include <game/mapitems_ex.h>
 
 #include <engine/editor.h>
+#include <engine/engine.h>
 #include <engine/graphics.h>
+#include <engine/shared/datafile.h>
+#include <engine/shared/jobs.h>
 
 #include "auto_map.h"
 
 #include <chrono>
+#include <list>
 #include <string>
 #include <vector>
 
@@ -691,16 +695,37 @@ public:
 	CUI::EPopupMenuFunctionResult RenderProperties(CUIRect *pToolbox) override;
 };
 
+class CDataFileWriterFinishJob : public IJob
+{
+	char m_aFileName[IO_MAX_PATH_LENGTH];
+	CDataFileWriter m_Writer;
+
+	void Run() override
+	{
+		m_Writer.Finish();
+	}
+
+public:
+	CDataFileWriterFinishJob(const char *pFileName, CDataFileWriter &&Writer) :
+		m_Writer(std::move(Writer))
+	{
+		str_copy(m_aFileName, pFileName);
+	}
+
+	const char *GetFileName() const { return m_aFileName; }
+};
+
 class CEditor : public IEditor
 {
-	class IInput *m_pInput;
-	class IClient *m_pClient;
-	class CConfig *m_pConfig;
-	class IConsole *m_pConsole;
-	class IGraphics *m_pGraphics;
-	class ITextRender *m_pTextRender;
-	class ISound *m_pSound;
-	class IStorage *m_pStorage;
+	class IInput *m_pInput = nullptr;
+	class IClient *m_pClient = nullptr;
+	class CConfig *m_pConfig = nullptr;
+	class IConsole *m_pConsole = nullptr;
+	class IEngine *m_pEngine = nullptr;
+	class IGraphics *m_pGraphics = nullptr;
+	class ITextRender *m_pTextRender = nullptr;
+	class ISound *m_pSound = nullptr;
+	class IStorage *m_pStorage = nullptr;
 	CRenderTools m_RenderTools;
 	CUI m_UI;
 
@@ -728,6 +753,7 @@ public:
 	class IClient *Client() { return m_pClient; }
 	class CConfig *Config() { return m_pConfig; }
 	class IConsole *Console() { return m_pConsole; }
+	class IEngine *Engine() { return m_pEngine; }
 	class IGraphics *Graphics() { return m_pGraphics; }
 	class ISound *Sound() { return m_pSound; }
 	class ITextRender *TextRender() { return m_pTextRender; }
@@ -738,12 +764,6 @@ public:
 	CEditor() :
 		m_TilesetPicker(16, 16)
 	{
-		m_pInput = nullptr;
-		m_pClient = nullptr;
-		m_pGraphics = nullptr;
-		m_pTextRender = nullptr;
-		m_pSound = nullptr;
-
 		m_EntitiesTexture.Invalidate();
 		m_FrontTexture.Invalidate();
 		m_TeleTexture.Invalidate();
@@ -858,6 +878,7 @@ public:
 	void HandleCursorMovement();
 	void HandleAutosave();
 	bool PerformAutosave();
+	void HandleWriterFinishJobs();
 
 	CLayerGroup *m_apSavedBrushes[10];
 
@@ -877,6 +898,7 @@ public:
 	void Render();
 
 	void RenderPressedKeys(CUIRect View);
+	void RenderSavingIndicator(CUIRect View);
 	void RenderMousePointer();
 
 	void ResetMenuBackgroundPositions();
@@ -1131,6 +1153,8 @@ public:
 	static const void *ms_pUiGotContext;
 
 	CEditorMap m_Map;
+	std::list<std::shared_ptr<CDataFileWriterFinishJob>> m_lpWriterFinishJobs;
+
 	int m_ShiftBy;
 
 	static void EnvelopeEval(int TimeOffsetMillis, int Env, ColorRGBA &Channels, void *pUser);

src/game/editor/io.cpp

@@ -33,6 +33,10 @@ struct CSoundSource_DEPRECATED
 
 bool CEditor::Save(const char *pFilename)
 {
+	// Check if file with this name is already being saved at the moment
+	if(std::any_of(std::begin(m_lpWriterFinishJobs), std::end(m_lpWriterFinishJobs), [pFilename](const std::shared_ptr<CDataFileWriterFinishJob> &Job) { return str_comp(pFilename, Job->GetFileName()) == 0; }))
+		return false;
+
 	return m_Map.Save(pFilename);
 }
 
@@ -372,27 +376,9 @@ bool CEditorMap::Save(const char *pFileName)
 	free(pPoints);
 
 	// finish the data file
-	df.Finish();
-	m_pEditor->Console()->Print(IConsole::OUTPUT_LEVEL_ADDINFO, "editor", "saving done");
-
-	// send rcon.. if we can
-	if(m_pEditor->Client()->RconAuthed())
-	{
-		CServerInfo CurrentServerInfo;
-		m_pEditor->Client()->GetServerInfo(&CurrentServerInfo);
-		NETADDR ServerAddr = m_pEditor->Client()->ServerAddress();
-		const unsigned char aIpv4Localhost[16] = {127, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
-		const unsigned char aIpv6Localhost[16] = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1};
-
-		// and if we're on localhost
-		if(!mem_comp(ServerAddr.ip, aIpv4Localhost, sizeof(aIpv4Localhost)) || !mem_comp(ServerAddr.ip, aIpv6Localhost, sizeof(aIpv6Localhost)))
-		{
-			char aMapName[128];
-			IStorage::StripPathAndExtension(pFileName, aMapName, sizeof(aMapName));
-			if(!str_comp(aMapName, CurrentServerInfo.m_aMap))
-				m_pEditor->Client()->Rcon("reload");
-		}
-	}
+	std::shared_ptr<CDataFileWriterFinishJob> pWriterFinishJob = std::make_shared<CDataFileWriterFinishJob>(pFileName, std::move(df));
+	m_pEditor->Engine()->AddJob(pWriterFinishJob);
+	m_pEditor->m_lpWriterFinishJobs.push_back(pWriterFinishJob);
 
 	return true;
 }