Merge #947

947: Add support for extra chunks in teehistorian r=Learath2 a=heinrich5991

This allows to add rarely-used chunks without increasing the file format
version.

CMakeLists.txt

@@ -688,6 +688,9 @@ set_glob(ENGINE_SHARED GLOB src/engine/shared
   snapshot.cpp
   snapshot.h
   storage.cpp
+  teehistorian_ex.cpp
+  teehistorian_ex.h
+  teehistorian_ex_chunks.h
   uuid_manager.cpp
   uuid_manager.h
   websockets.cpp

datasrc/network.py

@@ -12,7 +12,7 @@ Powerups = ["HEALTH", "ARMOR", "WEAPON", "NINJA"]
 RawHeader = '''
 
 #include <engine/message.h>
-#include <engine/shared/protocol_ex.h>
+#include <engine/shared/teehistorian_ex.h>
 
 enum
 {

src/engine/shared/global_uuid_manager.cpp

@@ -1,4 +1,5 @@
 #include "protocol_ex.h"
+#include "teehistorian_ex.h"
 #include "uuid_manager.h"
 
 #include <engine/uuid.h>
@@ -7,6 +8,7 @@ static CUuidManager CreateGlobalUuidManager()
 {
 	CUuidManager Manager;
 	RegisterUuids(&Manager);
+	RegisterTeehistorianUuids(&Manager);
 	RegisterGameUuids(&Manager);
 	return Manager;
 }

src/engine/shared/protocol_ex.cpp

@@ -6,7 +6,7 @@
 
 #include <new>
 
-void RegisterUuids(class CUuidManager *pManager)
+void RegisterUuids(CUuidManager *pManager)
 {
 	#define UUID(id, name) pManager->RegisterName(id, name);
 	#include "protocol_ex_msgs.h"

src/engine/shared/protocol_ex.h

@@ -14,7 +14,7 @@ enum
 	#define UUID(id, name) id,
 	#include "protocol_ex_msgs.h"
 	#undef UUID
-	OFFSET_GAME_UUID,
+	OFFSET_TEEHISTORIAN_UUID,
 
 	UNPACKMESSAGE_ERROR=0,
 	UNPACKMESSAGE_OK,

src/engine/shared/teehistorian_ex.cpp

@@ -0,0 +1,10 @@
+#include "teehistorian_ex.h"
+
+#include <engine/shared/uuid_manager.h>
+
+void RegisterTeehistorianUuids(CUuidManager *pManager)
+{
+	#define UUID(id, name) pManager->RegisterName(id, name);
+	#include "teehistorian_ex_chunks.h"
+	#undef UUID
+}

src/engine/shared/teehistorian_ex.h

@@ -0,0 +1,15 @@
+#ifndef GAME_TEEHISTORIAN_EX_H
+#define GAME_TEEHISTORIAN_EX_H
+#include "protocol_ex.h"
+
+enum
+{
+	__TEEHISTORIAN_UUID_HELPER=OFFSET_TEEHISTORIAN_UUID-1,
+	#define UUID(id, name) id,
+	#include "teehistorian_ex_chunks.h"
+	#undef UUID
+	OFFSET_GAME_UUID
+};
+
+void RegisterTeehistorianUuids(class CUuidManager *pManager);
+#endif // GAME_TEEHISTORIAN_EX_H

src/engine/shared/teehistorian_ex_chunks.h

@@ -0,0 +1 @@
+UUID(TEEHISTORIAN_TEST,         "teehistorian-test@ddnet.tw")

src/engine/shared/uuid_manager.cpp

@@ -110,6 +110,11 @@ int CUuidManager::LookupUuid(CUuid Uuid) const
 	return UUID_UNKNOWN;
 }
 
+int CUuidManager::NumUuids() const
+{
+	return m_aNames.size();
+}
+
 int CUuidManager::UnpackUuid(CUnpacker *pUnpacker) const
 {
 	CUuid Temp;

src/engine/shared/uuid_manager.h

@@ -42,6 +42,7 @@ public:
 	CUuid GetUuid(int ID) const;
 	const char *GetName(int ID) const;
 	int LookupUuid(CUuid Uuid) const;
+	int NumUuids() const;
 
 	int UnpackUuid(CUnpacker *pUnpacker) const;
 	int UnpackUuid(CUnpacker *pUnpacker, CUuid *pOut) const;

src/game/server/gamecontext.cpp

@@ -2592,6 +2592,7 @@ void CGameContext::OnInit(/*class IKernel *pKernel*/)
 
 		GameInfo.m_pConfig = &g_Config;
 		GameInfo.m_pTuning = Tuning();
+		GameInfo.m_pUuids = &g_UuidManager;
 
 		char aMapName[128];
 		Server()->GetMapInfo(aMapName, sizeof(aMapName), &GameInfo.m_MapSize, &GameInfo.m_MapCrc);

src/game/server/teehistorian.cpp

@@ -6,7 +6,11 @@
 
 static const char TEEHISTORIAN_NAME[] = "teehistorian@ddnet.tw";
 static const CUuid TEEHISTORIAN_UUID = CalculateUuid(TEEHISTORIAN_NAME);
-static const char TEEHISTORIAN_VERSION[] = "1";
+static const char TEEHISTORIAN_VERSION[] = "2";
+
+#define UUID(id, name) static const CUuid UUID_ ## id = CalculateUuid(name);
+#include <engine/shared/teehistorian_ex_chunks.h>
+#undef UUID
 
 enum
 {
@@ -21,6 +25,7 @@ enum
 	TEEHISTORIAN_JOIN,
 	TEEHISTORIAN_DROP,
 	TEEHISTORIAN_CONSOLE_COMMAND,
+	TEEHISTORIAN_EX,
 };
 
 static char EscapeJsonChar(char c)
@@ -119,7 +124,7 @@ void CTeeHistorian::WriteHeader(const CGameInfo *pGameInfo)
 	char aStartTime[128];
 
 	FormatUuid(pGameInfo->m_GameUuid, aGameUuid, sizeof(aGameUuid));
-	str_timestamp_ex(pGameInfo->m_StartTime, aStartTime, sizeof(aStartTime), "%Y-%m-%d %H:%M:%S %z");
+	str_timestamp_ex(pGameInfo->m_StartTime, aStartTime, sizeof(aStartTime), "%Y-%m-%dT%H:%M:%S%z");
 
 	char aCommentBuffer[128];
 	char aServerVersionBuffer[128];
@@ -196,11 +201,35 @@ void CTeeHistorian::WriteHeader(const CGameInfo *pGameInfo)
 	#include <game/tuning.h>
 	#undef MACRO_TUNING_PARAM
 
-	str_format(aJson, sizeof(aJson), "}}");
+	str_format(aJson, sizeof(aJson), "},\"uuids\":[");
+	Write(aJson, str_length(aJson));
+
+	for(int i = 0; i < pGameInfo->m_pUuids->NumUuids(); i++)
+	{
+		str_format(aJson, sizeof(aJson), "%s\"%s\"",
+			i == 0 ? "" : ",",
+			E(aBuffer1, pGameInfo->m_pUuids->GetName(OFFSET_UUID + i)));
+		Write(aJson, str_length(aJson));
+	}
+
+	str_format(aJson, sizeof(aJson), "]}");
 	Write(aJson, str_length(aJson));
 	Write("", 1); // Null termination.
 }
 
+void CTeeHistorian::WriteExtra(CUuid Uuid, const void *pData, int DataSize)
+{
+	EnsureTickWritten();
+
+	CPacker Ex;
+	Ex.Reset();
+	Ex.AddInt(-TEEHISTORIAN_EX);
+	Ex.AddRaw(&Uuid, sizeof(Uuid));
+	Ex.AddInt(DataSize);
+	Write(Ex.Data(), Ex.Size());
+	Write(pData, DataSize);
+}
+
 
 void CTeeHistorian::BeginTick(int Tick)
 {
@@ -484,6 +513,11 @@ void CTeeHistorian::RecordConsoleCommand(int ClientID, int FlagMask, const char
 	Write(Buffer.Data(), Buffer.Size());
 }
 
+void CTeeHistorian::RecordTestExtra()
+{
+	WriteExtra(UUID_TEEHISTORIAN_TEST, "", 0);
+}
+
 void CTeeHistorian::EndInputs()
 {
 	dbg_assert(m_State == STATE_INPUTS, "invalid teehistorian state");

src/game/server/teehistorian.h

@@ -7,6 +7,7 @@
 
 struct CConfiguration;
 class CTuningParams;
+class CUuidManager;
 
 class CTeeHistorian
 {
@@ -29,6 +30,7 @@ public:
 
 		CConfiguration *m_pConfig;
 		CTuningParams *m_pTuning;
+		CUuidManager *m_pUuids;
 	};
 
 	CTeeHistorian();
@@ -51,6 +53,7 @@ public:
 	void RecordPlayerJoin(int ClientID);
 	void RecordPlayerDrop(int ClientID, const char *pReason);
 	void RecordConsoleCommand(int ClientID, int FlagMask, const char *pCmd, IConsole::IResult *pResult);
+	void RecordTestExtra();
 	void EndInputs();
 
 	void EndTick();
@@ -59,6 +62,7 @@ public:
 
 private:
 	void WriteHeader(const CGameInfo *pGameInfo);
+	void WriteExtra(CUuid Uuid, const void *pData, int DataSize);
 	void EnsureTickWrittenPlayerData(int ClientID);
 	void EnsureTickWritten();
 	void WriteTick();

src/test/teehistorian.cpp

@@ -5,12 +5,15 @@
 #include <game/gamecore.h>
 #include <game/server/teehistorian.h>
 
+void RegisterGameUuids(CUuidManager *pManager);
+
 class TeeHistorian : public ::testing::Test
 {
 protected:
 	CTeeHistorian m_TH;
 	CConfiguration m_Config;
 	CTuningParams m_Tuning;
+	CUuidManager m_UuidManager;
 	CTeeHistorian::CGameInfo m_GameInfo;
 
 	CPacker m_Buffer;
@@ -35,6 +38,10 @@ protected:
 		#undef MACRO_CONFIG_STR
 		#undef MACRO_CONFIG_INT
 
+		RegisterUuids(&m_UuidManager);
+		RegisterTeehistorianUuids(&m_UuidManager);
+		RegisterGameUuids(&m_UuidManager);
+
 		mem_zero(&m_GameInfo, sizeof(m_GameInfo));
 
 		m_GameInfo.m_GameUuid = CalculateUuid("test@ddnet.tw");
@@ -51,6 +58,7 @@ protected:
 
 		m_GameInfo.m_pConfig = &m_Config;
 		m_GameInfo.m_pTuning = &m_Tuning;
+		m_GameInfo.m_pUuids = &m_UuidManager;
 
 		Reset(&m_GameInfo);
 	}
@@ -71,11 +79,12 @@ protected:
 	void Expect(const unsigned char *pOutput, int OutputSize)
 	{
 		static CUuid TEEHISTORIAN_UUID = CalculateUuid("teehistorian@ddnet.tw");
-		static const char PREFIX1[] = "{\"comment\":\"teehistorian@ddnet.tw\",\"version\":\"1\",\"game_uuid\":\"a1eb7182-796e-3b3e-941d-38ca71b2a4a8\",\"server_version\":\"DDNet test\",\"start_time\":\"";
+		static const char PREFIX1[] = "{\"comment\":\"teehistorian@ddnet.tw\",\"version\":\"2\",\"game_uuid\":\"a1eb7182-796e-3b3e-941d-38ca71b2a4a8\",\"server_version\":\"DDNet test\",\"start_time\":\"";
-		static const char PREFIX2[] = "\",\"server_name\":\"server name\",\"server_port\":\"8303\",\"game_type\":\"game type\",\"map_name\":\"Kobra 3 Solo\",\"map_size\":\"903514\",\"map_crc\":\"eceaf25c\",\"config\":{},\"tuning\":{}}";
+		static const char PREFIX2[] = "\",\"server_name\":\"server name\",\"server_port\":\"8303\",\"game_type\":\"game type\",\"map_name\":\"Kobra 3 Solo\",\"map_size\":\"903514\",\"map_crc\":\"eceaf25c\",\"config\":{},\"tuning\":{},\"uuids\":[";
+		static const char PREFIX3[] = "]}";
 
 		char aTimeBuf[64];
-		str_timestamp_ex(m_GameInfo.m_StartTime, aTimeBuf, sizeof(aTimeBuf), "%Y-%m-%d %H:%M:%S %z");
+		str_timestamp_ex(m_GameInfo.m_StartTime, aTimeBuf, sizeof(aTimeBuf), "%Y-%m-%dT%H:%M:%S%z");
 
 		CPacker Buffer;
 		Buffer.Reset();
@@ -83,6 +92,15 @@ protected:
 		Buffer.AddRaw(PREFIX1, str_length(PREFIX1));
 		Buffer.AddRaw(aTimeBuf, str_length(aTimeBuf));
 		Buffer.AddRaw(PREFIX2, str_length(PREFIX2));
+		for(int i = 0; i < m_UuidManager.NumUuids(); i++)
+		{
+			char aBuf[64];
+			str_format(aBuf, sizeof(aBuf), "%s\"%s\"",
+				i == 0 ? "" : ",",
+				m_UuidManager.GetName(OFFSET_UUID + i));
+			Buffer.AddRaw(aBuf, str_length(aBuf));
+		}
+		Buffer.AddRaw(PREFIX3, str_length(PREFIX3));
 		Buffer.AddRaw("", 1);
 		Buffer.AddRaw(pOutput, OutputSize);
 
@@ -93,13 +111,12 @@ protected:
 
 	void ExpectFull(const unsigned char *pOutput, int OutputSize)
 	{
-		ASSERT_FALSE(m_Buffer.Error());
-
 		const ::testing::TestInfo *pTestInfo =
 			::testing::UnitTest::GetInstance()->current_test_info();
 		const char *pTestName = pTestInfo->name();
 
-		if(m_Buffer.Size() != OutputSize
+		if(m_Buffer.Error()
+			|| m_Buffer.Size() != OutputSize
 			|| mem_comp(m_Buffer.Data(), pOutput, OutputSize) != 0)
 		{
 			char aFilename[64];
@@ -118,6 +135,7 @@ protected:
 			io_close(File);
 		}
 
+		ASSERT_FALSE(m_Buffer.Error());
 		ASSERT_EQ(m_Buffer.Size(), OutputSize);
 		ASSERT_TRUE(mem_comp(m_Buffer.Data(), pOutput, OutputSize) == 0);
 	}
@@ -273,3 +291,19 @@ TEST_F(TeeHistorian, TickExplicitPlayerMessage)
 	Finish();
 	Expect(EXPECTED, sizeof(EXPECTED));
 }
+
+TEST_F(TeeHistorian, ExtraMessage)
+{
+	const unsigned char EXPECTED[] = {
+		0x41, 0x00, // TICK_SKIP dt=0
+		// EX uuid=6bb8ba88-0f0b-382e-8dae-dbf4052b8b7d data=""
+		0x4a,
+		0x6b, 0xb8, 0xba, 0x88, 0x0f, 0x0b, 0x38, 0x2e,
+		0x8d, 0xae, 0xdb, 0xf4, 0x05, 0x2b, 0x8b, 0x7d,
+		0x00,
+		0x40, // FINISH
+	};
+	Tick(1); Inputs(); m_TH.RecordTestExtra();
+	Finish();
+	Expect(EXPECTED, sizeof(EXPECTED));
+}