edg-l/ddnet
1
0
Fork 0
mirror of https://github.com/ddnet/ddnet.git synced 2024-11-17 13:38:18 +00:00
Code Issues Packages Projects Releases Wiki Activity

Substantially reduce stack memory usage of CDataFileWriter

Browse source 
Use an `std::map` to sparsely store only the used `CItemTypeInfo`s on the heap instead of using an `std::array` to store all 65536 entries (`MAX_ITEM_TYPES`) on the stack. This reduces the (stack) memory usage of `CDataFileWriter`s from 786512 to 128 bytes.
This commit is contained in:
Robert Müller 2024-11-14 20:20:27 +01:00
parent 4f53dd8048
commit 1cb3a6bf7d
2 changed files with 36 additions and 47 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

70
src/engine/shared/datafile.cpp
View file

@ -596,12 +596,6 @@ int CDataFileReader::MapSize() const
CDataFileWriter::CDataFileWriter() CDataFileWriter::CDataFileWriter()
{ {
m_File = 0; m_File = 0;
for(CItemTypeInfo &ItemTypeInfo : m_aItemTypes)
{
ItemTypeInfo.m_Num = 0;
ItemTypeInfo.m_First = -1;
ItemTypeInfo.m_Last = -1;
}
} }
CDataFileWriter::~CDataFileWriter() CDataFileWriter::~CDataFileWriter()
@ -700,17 +694,18 @@ int CDataFileWriter::AddItem(int Type, int Id, size_t Size, const void *pData, c
Info.m_pData = nullptr; Info.m_pData = nullptr;
// link // link
Info.m_Prev = m_aItemTypes[Type].m_Last; CItemTypeInfo &ItemType = m_ItemTypes[Type];
Info.m_Prev = ItemType.m_Last;
Info.m_Next = -1; Info.m_Next = -1;
if(m_aItemTypes[Type].m_Last != -1) if(ItemType.m_Last != -1)
m_vItems[m_aItemTypes[Type].m_Last].m_Next = NumItems; m_vItems[ItemType.m_Last].m_Next = NumItems;
m_aItemTypes[Type].m_Last = NumItems; ItemType.m_Last = NumItems;
if(m_aItemTypes[Type].m_First == -1) if(ItemType.m_First == -1)
m_aItemTypes[Type].m_First = NumItems; ItemType.m_First = NumItems;
m_aItemTypes[Type].m_Num++; ItemType.m_Num++;
return NumItems; return NumItems;
} }
@ -807,23 +802,15 @@ void CDataFileWriter::Finish()
for(const CDataInfo &DataInfo : m_vDatas) for(const CDataInfo &DataInfo : m_vDatas)
DataSize += DataInfo.m_CompressedSize; DataSize += DataInfo.m_CompressedSize;
// Count number of item types
int NumItemTypes = 0;
for(const CItemTypeInfo &ItemType : m_aItemTypes)
{
if(ItemType.m_Num > 0)
++NumItemTypes;
}
// Calculate complete file size // Calculate complete file size
const size_t TypesSize = NumItemTypes * sizeof(CDatafileItemType); const size_t TypesSize = m_ItemTypes.size() * sizeof(CDatafileItemType);
const size_t HeaderSize = sizeof(CDatafileHeader); const size_t HeaderSize = sizeof(CDatafileHeader);
const size_t OffsetSize = (m_vItems.size() + m_vDatas.size() * 2) * sizeof(int); // ItemOffsets, DataOffsets, DataUncompressedSizes const size_t OffsetSize = (m_vItems.size() + m_vDatas.size() * 2) * sizeof(int); // ItemOffsets, DataOffsets, DataUncompressedSizes
const size_t SwapSize = HeaderSize + TypesSize + OffsetSize + ItemSize; const size_t SwapSize = HeaderSize + TypesSize + OffsetSize + ItemSize;
const size_t FileSize = SwapSize + DataSize; const size_t FileSize = SwapSize + DataSize;
if(DEBUG) if(DEBUG)
dbg_msg("datafile", "NumItemTypes=%d TypesSize=%" PRIzu " ItemSize=%" PRIzu " DataSize=%" PRIzu, NumItemTypes, TypesSize, ItemSize, DataSize); dbg_msg("datafile", "NumItemTypes=" PRIzu " TypesSize=%" PRIzu " ItemSize=%" PRIzu " DataSize=%" PRIzu, m_ItemTypes.size(), TypesSize, ItemSize, DataSize);
// This also ensures that SwapSize, ItemSize and DataSize are valid. // This also ensures that SwapSize, ItemSize and DataSize are valid.
dbg_assert(FileSize <= (size_t)std::numeric_limits<int>::max(), "File size too large"); dbg_assert(FileSize <= (size_t)std::numeric_limits<int>::max(), "File size too large");
@ -838,7 +825,7 @@ void CDataFileWriter::Finish()
Header.m_Version = 4; Header.m_Version = 4;
Header.m_Size = FileSize - Header.SizeOffset(); Header.m_Size = FileSize - Header.SizeOffset();
Header.m_Swaplen = SwapSize - Header.SizeOffset(); Header.m_Swaplen = SwapSize - Header.SizeOffset();
Header.m_NumItemTypes = NumItemTypes; Header.m_NumItemTypes = m_ItemTypes.size();
Header.m_NumItems = m_vItems.size(); Header.m_NumItems = m_vItems.size();
Header.m_NumRawData = m_vDatas.size(); Header.m_NumRawData = m_vDatas.size();
Header.m_ItemSize = ItemSize; Header.m_ItemSize = ItemSize;
@ -851,15 +838,15 @@ void CDataFileWriter::Finish()
} }
// Write item types // Write item types
for(int Type = 0, Count = 0; Type < (int)m_aItemTypes.size(); ++Type) int ItemCount = 0;
for(const auto &[Type, ItemType] : m_ItemTypes)
{ {
if(!m_aItemTypes[Type].m_Num) dbg_assert(ItemType.m_Num > 0, "Invalid item type entry");
continue;
CDatafileItemType Info; CDatafileItemType Info;
Info.m_Type = Type; Info.m_Type = Type;
Info.m_Start = Count; Info.m_Start = ItemCount;
Info.m_Num = m_aItemTypes[Type].m_Num; Info.m_Num = ItemType.m_Num;
if(DEBUG) if(DEBUG)
dbg_msg("datafile", "writing item type. Type=%x Start=%d Num=%d", Info.m_Type, Info.m_Start, Info.m_Num); dbg_msg("datafile", "writing item type. Type=%x Start=%d Num=%d", Info.m_Type, Info.m_Start, Info.m_Num);
@ -868,40 +855,41 @@ void CDataFileWriter::Finish()
swap_endian(&Info, sizeof(int), sizeof(CDatafileItemType) / sizeof(int)); swap_endian(&Info, sizeof(int), sizeof(CDatafileItemType) / sizeof(int));
#endif #endif
io_write(m_File, &Info, sizeof(Info)); io_write(m_File, &Info, sizeof(Info));
Count += m_aItemTypes[Type].m_Num; ItemCount += ItemType.m_Num;
} }
// Write item offsets sorted by type // Write item offsets sorted by type
for(int Type = 0, Offset = 0; Type < (int)m_aItemTypes.size(); Type++) int ItemOffset = 0;
for(const auto &[Type, ItemType] : m_ItemTypes)
{ {
// Write all items offsets of this type // Write all items offsets of this type
for(int ItemIndex = m_aItemTypes[Type].m_First; ItemIndex != -1; ItemIndex = m_vItems[ItemIndex].m_Next) for(int ItemIndex = ItemType.m_First; ItemIndex != -1; ItemIndex = m_vItems[ItemIndex].m_Next)
{ {
if(DEBUG) if(DEBUG)
dbg_msg("datafile", "writing item offset. Type=%d ItemIndex=%d Offset=%d", Type, ItemIndex, Offset); dbg_msg("datafile", "writing item offset. Type=%d ItemIndex=%d ItemOffset=%d", Type, ItemIndex, ItemOffset);
int Temp = Offset; int Temp = ItemOffset;
#if defined(CONF_ARCH_ENDIAN_BIG) #if defined(CONF_ARCH_ENDIAN_BIG)
swap_endian(&Temp, sizeof(int), sizeof(Temp) / sizeof(int)); swap_endian(&Temp, sizeof(int), sizeof(Temp) / sizeof(int));
#endif #endif
io_write(m_File, &Temp, sizeof(Temp)); io_write(m_File, &Temp, sizeof(Temp));
Offset += m_vItems[ItemIndex].m_Size + sizeof(CDatafileItem); ItemOffset += m_vItems[ItemIndex].m_Size + sizeof(CDatafileItem);
} }
} }
// Write data offsets // Write data offsets
int Offset = 0, DataIndex = 0; int DataOffset = 0, DataIndex = 0;
for(const CDataInfo &DataInfo : m_vDatas) for(const CDataInfo &DataInfo : m_vDatas)
{ {
if(DEBUG) if(DEBUG)
dbg_msg("datafile", "writing data offset. DataIndex=%d Offset=%d", DataIndex, Offset); dbg_msg("datafile", "writing data offset. DataIndex=%d DataOffset=%d", DataIndex, DataOffset);
int Temp = Offset; int Temp = DataOffset;
#if defined(CONF_ARCH_ENDIAN_BIG) #if defined(CONF_ARCH_ENDIAN_BIG)
swap_endian(&Temp, sizeof(int), sizeof(Temp) / sizeof(int)); swap_endian(&Temp, sizeof(int), sizeof(Temp) / sizeof(int));
#endif #endif
io_write(m_File, &Temp, sizeof(Temp)); io_write(m_File, &Temp, sizeof(Temp));
Offset += DataInfo.m_CompressedSize; DataOffset += DataInfo.m_CompressedSize;
++DataIndex; ++DataIndex;
} }
@ -921,10 +909,10 @@ void CDataFileWriter::Finish()
} }
// Write items sorted by type // Write items sorted by type
for(int Type = 0; Type < (int)m_aItemTypes.size(); ++Type) for(const auto &[Type, ItemType] : m_ItemTypes)
{ {
// Write all items of this type // Write all items of this type
for(int ItemIndex = m_aItemTypes[Type].m_First; ItemIndex != -1; ItemIndex = m_vItems[ItemIndex].m_Next) for(int ItemIndex = ItemType.m_First; ItemIndex != -1; ItemIndex = m_vItems[ItemIndex].m_Next)
{ {
CDatafileItem Item; CDatafileItem Item;
Item.m_TypeAndId = (Type << 16) | m_vItems[ItemIndex].m_Id; Item.m_TypeAndId = (Type << 16) | m_vItems[ItemIndex].m_Id;

13
src/engine/shared/datafile.h
View file

@ -10,7 +10,8 @@
#include "uuid_manager.h" #include "uuid_manager.h"
#include <array> #include <cstdint>
#include <map>
#include <vector> #include <vector>
enum enum
@ -97,9 +98,9 @@ private:
struct CItemTypeInfo struct CItemTypeInfo
{ {
int m_Num; int m_Num = 0;
int m_First; int m_First = -1;
int m_Last; int m_Last = -1;
}; };
struct CExtendedItemType struct CExtendedItemType
@ -114,7 +115,7 @@ private:
}; };
IOHANDLE m_File; IOHANDLE m_File;
std::array<CItemTypeInfo, MAX_ITEM_TYPES> m_aItemTypes; std::map<uint16_t, CItemTypeInfo, std::less<>> m_ItemTypes; // item types must be sorted in ascending order
std::vector<CItemInfo> m_vItems; std::vector<CItemInfo> m_vItems;
std::vector<CDataInfo> m_vDatas; std::vector<CDataInfo> m_vDatas;
std::vector<CExtendedItemType> m_vExtendedItemTypes; std::vector<CExtendedItemType> m_vExtendedItemTypes;
@ -128,7 +129,7 @@ public:
{ {
m_File = Other.m_File; m_File = Other.m_File;
Other.m_File = 0; Other.m_File = 0;
m_aItemTypes = std::move(Other.m_aItemTypes); m_ItemTypes = std::move(Other.m_ItemTypes);
m_vItems = std::move(Other.m_vItems); m_vItems = std::move(Other.m_vItems);
m_vDatas = std::move(Other.m_vDatas); m_vDatas = std::move(Other.m_vDatas);
m_vExtendedItemTypes = std::move(Other.m_vExtendedItemTypes); m_vExtendedItemTypes = std::move(Other.m_vExtendedItemTypes);