Merge #1329

1329: Add support for extra map items in datafiles r=Learath2 a=heinrich5991

This works by utilizing the good old UUIDs – this way we can make sure
that we don't clash with other people extending the map format.

Co-authored-by: heinrich5991 <heinrich5991@gmail.com>

.gitignore

@@ -95,6 +95,7 @@ cscope.out
 *.tar.gz
 *.tar.xz
 *.teehistorian
+*.tmp
 *.user
 *.vcxproj
 *.vs

CMakeLists.txt

@@ -675,7 +675,6 @@ set_glob(ENGINE_SHARED GLOB src/engine/shared
   websockets.cpp
   websockets.h
 )
-set(ENGINE_GENERATED_SHARED src/game/generated/protocol.cpp src/game/generated/protocol.h)
 set_glob(GAME_SHARED GLOB src/game
   collision.cpp
   collision.h
@@ -693,6 +692,9 @@ set_glob(GAME_SHARED GLOB src/game
   mapbugs_list.h
   mapitems.cpp
   mapitems.h
+  mapitems_ex.cpp
+  mapitems_ex.h
+  mapitems_ex_types.h
   teamscore.cpp
   teamscore.h
   tuning.h
@@ -700,6 +702,21 @@ set_glob(GAME_SHARED GLOB src/game
   version.h
   voting.h
 )
+# A bit hacky, but these are needed to register all the UUIDs, even for stuff
+# that doesn't link game.
+set(ENGINE_UUID_SHARED
+  src/game/generated/protocol.cpp
+  src/game/generated/protocol.h
+  src/game/mapitems_ex.cpp
+  src/game/mapitems_ex.h
+  src/game/mapitems_ex_types.h
+)
+foreach(s ${GAME_SHARED})
+  if(s MATCHES "mapitems_(ex.cpp|ex.h|ex_types.h)$")
+    list(REMOVE_ITEM GAME_SHARED ${s})
+  endif()
+endforeach()
+list(REMOVE_ITEM GAME_SHARED ${ENGINE_UUID_SHARED})
 set(GAME_GENERATED_SHARED
   src/game/generated/git_revision.cpp
   src/game/generated/protocol.h
@@ -714,7 +731,7 @@ set(LIBS ${CURL_LIBRARIES} ${CRYPTO_LIBRARIES} ${WEBSOCKETS_LIBRARIES} ${ZLIB_LI
 list(APPEND LIBS ${CMAKE_THREAD_LIBS_INIT})
 
 # Targets
-add_library(engine-shared EXCLUDE_FROM_ALL OBJECT ${ENGINE_INTERFACE} ${ENGINE_SHARED} ${ENGINE_GENERATED_SHARED} ${BASE})
+add_library(engine-shared EXCLUDE_FROM_ALL OBJECT ${ENGINE_INTERFACE} ${ENGINE_SHARED} ${ENGINE_UUID_SHARED} ${BASE})
 add_library(game-shared EXCLUDE_FROM_ALL OBJECT ${GAME_SHARED} ${GAME_GENERATED_SHARED})
 list(APPEND TARGETS_OWN engine-shared game-shared)
 
@@ -1117,6 +1134,7 @@ add_custom_target(everything DEPENDS ${TARGETS_OWN})
 if(GTEST_FOUND OR DOWNLOAD_GTEST)
   set_glob(TESTS GLOB src/test
     aio.cpp
+    datafile.cpp
     fs.cpp
     git_revision.cpp
     hash.cpp

datasrc/compile.py

@@ -115,7 +115,7 @@ if gen_network_header:
 	extended = [o for o in network.Messages if o.ex is not None]
 	for l in create_enum_table(["NETMSGTYPE_EX"]+[o.enum_name for o in non_extended], "NUM_NETMSGTYPES"): print(l)
 	print("")
-	for l in create_enum_table(["__NETMSGTYPE_UUID_HELPER=OFFSET_NETMSGTYPE_UUID-1"]+[o.enum_name for o in extended], "END_NETMSGTYPE_UUID"): print(l)
+	for l in create_enum_table(["__NETMSGTYPE_UUID_HELPER=OFFSET_NETMSGTYPE_UUID-1"]+[o.enum_name for o in extended], "OFFSET_MAPITEMTYPE_UUID"): print(l)
 	print("")
 
 	for item in network.Objects + network.Messages:
@@ -167,6 +167,7 @@ if gen_network_source:
 	lines += ['#include <engine/shared/protocol.h>']
 	lines += ['#include <engine/message.h>']
 	lines += ['#include "protocol.h"']
+	lines += ['#include <game/mapitems_ex.h>']
 
 	lines += ['CNetObjHandler::CNetObjHandler()']
 	lines += ['{']
@@ -348,6 +349,7 @@ if gen_network_source:
 	for item in network.Objects + network.Messages:
 		if item.ex is not None:
 			lines += ['\tpManager->RegisterName(%s, "%s");' % (item.enum_name, item.ex)]
+	lines += ['\tRegisterMapItemTypeUuids(pManager);']
 	lines += ['}']
 
 	for l in lines:

src/engine/shared/datafile.cpp

@@ -1,14 +1,63 @@
 /* (c) Magnus Auvinen. See licence.txt in the root of the distribution for more information. */
 /* If you are missing that file, acquire a complete release at teeworlds.com.                */
+
+#include "datafile.h"
+
 #include <base/math.h>
 #include <base/hash_ctxt.h>
 #include <base/system.h>
 #include <engine/storage.h>
-#include "datafile.h"
+
+#include "uuid_manager.h"
+
 #include <zlib.h>
 
 static const int DEBUG=0;
 
+enum
+{
+	OFFSET_UUID_TYPE=0x8000,
+	ITEMTYPE_EX=0xffff,
+};
+
+struct CItemEx
+{
+	int m_aUuid[sizeof(CUuid) / 4];
+
+	static CItemEx FromUuid(CUuid Uuid)
+	{
+		CItemEx Result;
+		for(int i = 0; i < (int)sizeof(CUuid) / 4; i++)
+		{
+			Result.m_aUuid[i] =
+				(Uuid.m_aData[i * 4 + 0] << 24) |
+				(Uuid.m_aData[i * 4 + 1] << 16) |
+				(Uuid.m_aData[i * 4 + 2] << 8) |
+				(Uuid.m_aData[i * 4 + 3]);
+		}
+		return Result;
+	}
+
+	CUuid ToUuid() const
+	{
+		CUuid Result;
+		for(int i = 0; i < (int)sizeof(CUuid) / 4; i++)
+		{
+			Result.m_aData[i * 4 + 0] = m_aUuid[i] >> 24;
+			Result.m_aData[i * 4 + 1] = m_aUuid[i] >> 16;
+			Result.m_aData[i * 4 + 2] = m_aUuid[i] >> 8;
+			Result.m_aData[i * 4 + 3] = m_aUuid[i];
+		}
+		return Result;
+	}
+};
+
+static int GetTypeFromIndex(int Index)
+{
+	return ITEMTYPE_EX - Index - 1;
+}
+
+
 struct CDatafileItemType
 {
 	int m_Type;
@@ -345,15 +394,60 @@ int CDataFileReader::GetItemSize(int Index)
 	return m_pDataFile->m_Info.m_pItemOffsets[Index+1]-m_pDataFile->m_Info.m_pItemOffsets[Index] - sizeof(CDatafileItem);
 }
 
+int CDataFileReader::GetExternalItemType(int InternalType)
+{
+	if(InternalType <= OFFSET_UUID_TYPE || InternalType == ITEMTYPE_EX)
+	{
+		return InternalType;
+	}
+	int TypeIndex = FindItemIndex(ITEMTYPE_EX, InternalType);
+	if(TypeIndex < 0 || GetItemSize(TypeIndex) < (int)sizeof(CItemEx))
+	{
+		return InternalType;
+	}
+	const CItemEx *pItemEx = (const CItemEx *)GetItem(TypeIndex, 0, 0);
+	// Propagate UUID_UNKNOWN, it doesn't hurt.
+	return g_UuidManager.LookupUuid(pItemEx->ToUuid());
+}
+
+int CDataFileReader::GetInternalItemType(int ExternalType)
+{
+	if(ExternalType < OFFSET_UUID)
+	{
+		return ExternalType;
+	}
+	CUuid Uuid = g_UuidManager.GetUuid(ExternalType);
+	int Start, Num;
+	GetType(ITEMTYPE_EX, &Start, &Num);
+	for(int i = Start; i < Start + Num; i++)
+	{
+		if(GetItemSize(i) < (int)sizeof(CItemEx))
+		{
+			continue;
+		}
+		int ID;
+		if(Uuid == ((const CItemEx *)GetItem(i, 0, &ID))->ToUuid())
+		{
+			return ID;
+		}
+	}
+	return -1;
+}
+
 void *CDataFileReader::GetItem(int Index, int *pType, int *pID)
 {
 	if(!m_pDataFile) { if(pType) *pType = 0; if(pID) *pID = 0; return 0; }
 
 	CDatafileItem *i = (CDatafileItem *)(m_pDataFile->m_Info.m_pItemStart+m_pDataFile->m_Info.m_pItemOffsets[Index]);
 	if(pType)
-		*pType = (i->m_TypeAndID>>16)&0xffff; // remove sign extension
+	{
+		// remove sign extension
+		*pType = GetExternalItemType((i->m_TypeAndID>>16)&0xffff);
+	}
 	if(pID)
+	{
 		*pID = i->m_TypeAndID&0xffff;
+	}
 	return (void *)(i+1);
 }
 
@@ -365,6 +459,7 @@ void CDataFileReader::GetType(int Type, int *pStart, int *pNum)
 	if(!m_pDataFile)
 		return;
 
+	Type = GetInternalItemType(Type);
 	for(int i = 0; i < m_pDataFile->m_Header.m_NumItemTypes; i++)
 	{
 		if(m_pDataFile->m_Info.m_pItemTypes[i].m_Type == Type)
@@ -376,20 +471,35 @@ void CDataFileReader::GetType(int Type, int *pStart, int *pNum)
 	}
 }
 
-void *CDataFileReader::FindItem(int Type, int ID)
+int CDataFileReader::FindItemIndex(int Type, int ID)
 {
-	if(!m_pDataFile) return 0;
+	if(!m_pDataFile)
+	{
+		return -1;
+	}
 
 	int Start, Num;
 	GetType(Type, &Start, &Num);
 	for(int i = 0; i < Num; i++)
 	{
 		int ItemID;
-		void *pItem = GetItem(Start+i,0, &ItemID);
+		GetItem(Start + i, 0, &ItemID);
 		if(ID == ItemID)
-			return pItem;
+		{
+			return Start + i;
 		}
+	}
+	return -1;
+}
+
+void *CDataFileReader::FindItem(int Type, int ID)
+{
+	int Index = FindItemIndex(Type, ID);
+	if(Index < 0)
+	{
 		return 0;
+	}
+	return GetItem(Index, 0, 0);
 }
 
 int CDataFileReader::NumItems()
@@ -478,7 +588,9 @@ void CDataFileWriter::Init()
 	m_NumItems = 0;
 	m_NumDatas = 0;
 	m_NumItemTypes = 0;
+	m_NumExtendedItemTypes = 0;
 	mem_zero(m_pItemTypes, sizeof(CItemTypeInfo) * MAX_ITEM_TYPES);
+	mem_zero(m_aExtendedItemTypes, sizeof(m_aExtendedItemTypes));
 
 	for(int i = 0; i < MAX_ITEM_TYPES; i++)
 	{
@@ -493,12 +605,37 @@ bool CDataFileWriter::Open(class IStorage *pStorage, const char *pFilename)
 	return OpenFile(pStorage, pFilename);
 }
 
+int CDataFileWriter::GetExtendedItemTypeIndex(int Type)
+{
+	for(int i = 0; i < m_NumExtendedItemTypes; i++)
+	{
+		if(m_aExtendedItemTypes[i] == Type)
+		{
+			return i;
+		}
+	}
+
+	// Type not found, add it.
+	dbg_assert(m_NumExtendedItemTypes < MAX_EXTENDED_ITEM_TYPES, "too many extended item types");
+	int Index = m_NumExtendedItemTypes++;
+	m_aExtendedItemTypes[Index] = Type;
+
+	CItemEx ExtendedType = CItemEx::FromUuid(g_UuidManager.GetUuid(Type));
+	AddItem(ITEMTYPE_EX, GetTypeFromIndex(Index), sizeof(ExtendedType), &ExtendedType);
+	return Index;
+}
+
 int CDataFileWriter::AddItem(int Type, int ID, int Size, void *pData)
 {
-	dbg_assert(Type >= 0 && Type < 0xFFFF, "incorrect type");
+	dbg_assert((Type >= 0 && Type < MAX_ITEM_TYPES) || Type >= OFFSET_UUID, "incorrect type");
 	dbg_assert(m_NumItems < 1024, "too many items");
 	dbg_assert(Size%sizeof(int) == 0, "incorrect boundary");
 
+	if(Type >= OFFSET_UUID)
+	{
+		Type = GetTypeFromIndex(GetExtendedItemTypeIndex(Type));
+	}
+
 	m_pItems[m_NumItems].m_Type = Type;
 	m_pItems[m_NumItems].m_ID = ID;
 	m_pItems[m_NumItems].m_Size = Size;
@@ -631,7 +768,7 @@ int CDataFileWriter::Finish()
 	}
 
 	// write types
-	for(int i = 0, Count = 0; i < 0xffff; i++)
+	for(int i = 0, Count = 0; i < MAX_ITEM_TYPES; i++)
 	{
 		if(m_pItemTypes[i].m_Num)
 		{
@@ -651,7 +788,7 @@ int CDataFileWriter::Finish()
 	}
 
 	// write item offsets
-	for(int i = 0, Offset = 0; i < 0xffff; i++)
+	for(int i = 0, Offset = 0; i < MAX_ITEM_TYPES; i++)
 	{
 		if(m_pItemTypes[i].m_Num)
 		{
@@ -700,7 +837,7 @@ int CDataFileWriter::Finish()
 	}
 
 	// write m_pItems
-	for(int i = 0; i < 0xffff; i++)
+	for(int i = 0; i < MAX_ITEM_TYPES; i++)
 	{
 		if(m_pItemTypes[i].m_Num)
 		{

src/engine/shared/datafile.h

@@ -3,6 +3,7 @@
 #ifndef ENGINE_SHARED_DATAFILE_H
 #define ENGINE_SHARED_DATAFILE_H
 
+#include <base/system.h>
 #include <base/hash.h>
 
 // raw datafile access
@@ -11,6 +12,10 @@ class CDataFileReader
 	struct CDatafile *m_pDataFile;
 	void *GetDataImpl(int Index, int Swap);
 	int GetFileDataSize(int Index);
+
+	int GetExternalItemType(int InternalType);
+	int GetInternalItemType(int ExternalType);
+
 public:
 	CDataFileReader() : m_pDataFile(0) {}
 	~CDataFileReader() { Close(); }
@@ -27,6 +32,7 @@ public:
 	void *GetItem(int Index, int *pType, int *pID);
 	int GetItemSize(int Index);
 	void GetType(int Type, int *pStart, int *pNum);
+	int FindItemIndex(int Type, int ID);
 	void *FindItem(int Type, int ID);
 	int NumItems();
 	int NumData();
@@ -67,18 +73,23 @@ class CDataFileWriter
 
 	enum
 	{
-		MAX_ITEM_TYPES=0xffff,
+		MAX_ITEM_TYPES=0x10000,
 		MAX_ITEMS=1024,
 		MAX_DATAS=1024,
+		MAX_EXTENDED_ITEM_TYPES=64,
 	};
 
 	IOHANDLE m_File;
 	int m_NumItems;
 	int m_NumDatas;
 	int m_NumItemTypes;
+	int m_NumExtendedItemTypes;
 	CItemTypeInfo *m_pItemTypes;
 	CItemInfo *m_pItems;
 	CDataInfo *m_pDatas;
+	int m_aExtendedItemTypes[MAX_EXTENDED_ITEM_TYPES];
+
+	int GetExtendedItemTypeIndex(int Type);
 
 public:
 	CDataFileWriter();

src/game/mapitems.h

@@ -29,6 +29,9 @@ enum
 	MAPITEMTYPE_LAYER,
 	MAPITEMTYPE_ENVPOINTS,
 	MAPITEMTYPE_SOUND,
+	// High map item type numbers suggest that they use the alternate
+	// format with UUIDs. See src/engine/shared/datafile.cpp for some of
+	// the implementation.
 
 
 	CURVETYPE_STEP=0,
@@ -419,7 +422,6 @@ struct CMapItemSound
 	int m_SoundDataSize;
 } ;
 
-
 // DDRace
 
 class CTeleTile

src/game/mapitems_ex.cpp

@@ -0,0 +1,10 @@
+#include "mapitems_ex.h"
+
+#include <engine/shared/uuid_manager.h>
+
+void RegisterMapItemTypeUuids(CUuidManager *pManager)
+{
+	#define UUID(id, name) pManager->RegisterName(id, name);
+	#include "mapitems_ex_types.h"
+	#undef UUID
+}

src/game/mapitems_ex.h

@@ -0,0 +1,26 @@
+#ifndef GAME_MAPITEMS_EX_H
+#define GAME_MAPITEMS_EX_H
+#include <game/generated/protocol.h>
+
+enum
+{
+	__MAPITEMTYPE_UUID_HELPER=OFFSET_MAPITEMTYPE_UUID-1,
+	#define UUID(id, name) id,
+	#include "mapitems_ex_types.h"
+	#undef UUID
+	END_MAPITEMTYPES_UUID,
+};
+
+struct CMapItemTest
+{
+	enum { CURRENT_VERSION=1 };
+
+	int m_Version;
+	int m_aFields[2];
+	int m_Field3;
+	int m_Field4;
+} ;
+
+
+void RegisterMapItemTypeUuids(class CUuidManager *pManager);
+#endif // GAME_MAPITEMS_EX_H

src/game/mapitems_ex_types.h

@@ -0,0 +1,3 @@
+// This file can be included several times.
+
+UUID(MAPITEMTYPE_TEST,          "mapitemtype-test@ddnet.tw")

src/test/datafile.cpp

@@ -0,0 +1,61 @@
+#include "test.h"
+#include <gtest/gtest.h>
+
+#include <engine/shared/datafile.h>
+#include <engine/storage.h>
+#include <game/mapitems_ex.h>
+
+TEST(Datafile, ExtendedType)
+{
+	IStorage *pStorage = CreateLocalStorage();
+	CTestInfo Info;
+
+	CMapItemTest Test;
+	Test.m_Version = CMapItemTest::CURRENT_VERSION;
+	Test.m_aFields[0] = 1234;
+	Test.m_aFields[1] = 5678;
+	Test.m_Field3 = 9876;
+	Test.m_Field4 = 5432;
+
+	{
+		CDataFileWriter Writer;
+		Writer.Open(pStorage, Info.m_aFilename);
+
+		Writer.AddItem(MAPITEMTYPE_TEST, 0x8000, sizeof(Test), &Test);
+
+		Writer.Finish();
+	}
+
+	{
+		CDataFileReader Reader;
+		Reader.Open(pStorage, Info.m_aFilename, IStorage::TYPE_ALL);
+
+		int Start, Num;
+		Reader.GetType(MAPITEMTYPE_TEST, &Start, &Num);
+		EXPECT_EQ(Num, 1);
+
+		int Index = Reader.FindItemIndex(MAPITEMTYPE_TEST, 0x8000);
+		EXPECT_EQ(Start, Index);
+		ASSERT_GE(Index, 0);
+		ASSERT_EQ(Reader.GetItemSize(Index), (int)sizeof(Test));
+
+		int Type, ID;
+		const CMapItemTest *pTest = (const CMapItemTest *)Reader.GetItem(Index, &Type, &ID);
+		EXPECT_EQ(pTest, Reader.FindItem(MAPITEMTYPE_TEST, 0x8000));
+		EXPECT_EQ(Type, MAPITEMTYPE_TEST);
+		EXPECT_EQ(ID, 0x8000);
+
+		EXPECT_EQ(pTest->m_Version, Test.m_Version);
+		EXPECT_EQ(pTest->m_aFields[0], Test.m_aFields[0]);
+		EXPECT_EQ(pTest->m_aFields[1], Test.m_aFields[1]);
+		EXPECT_EQ(pTest->m_Field3, Test.m_Field3);
+		EXPECT_EQ(pTest->m_Field4, Test.m_Field4);
+	}
+
+	if(!HasFailure())
+	{
+		pStorage->RemoveFile(Info.m_aFilename, IStorage::TYPE_SAVE);
+	}
+
+	delete pStorage;
+}