2010-11-20 10:37:14 +00:00
|
|
|
/* (c) Magnus Auvinen. See licence.txt in the root of the distribution for more information. */
|
|
|
|
/* If you are missing that file, acquire a complete release at teeworlds.com. */
|
2018-10-05 09:20:30 +00:00
|
|
|
|
|
|
|
#include "datafile.h"
|
|
|
|
|
2018-06-05 19:22:40 +00:00
|
|
|
#include <base/hash_ctxt.h>
|
2022-10-01 20:45:10 +00:00
|
|
|
#include <base/log.h>
|
2023-01-07 02:38:01 +00:00
|
|
|
#include <base/math.h>
|
2010-05-29 07:25:38 +00:00
|
|
|
#include <base/system.h>
|
|
|
|
#include <engine/storage.h>
|
2018-10-05 09:20:30 +00:00
|
|
|
|
|
|
|
#include "uuid_manager.h"
|
|
|
|
|
2022-06-16 15:23:36 +00:00
|
|
|
#include <cstdlib>
|
2023-11-01 11:17:02 +00:00
|
|
|
#include <limits>
|
2022-06-16 15:23:36 +00:00
|
|
|
|
2020-09-26 19:41:58 +00:00
|
|
|
static const int DEBUG = 0;
|
2010-05-29 07:25:38 +00:00
|
|
|
|
2018-10-05 09:20:30 +00:00
|
|
|
enum
|
|
|
|
{
|
2020-09-26 19:41:58 +00:00
|
|
|
OFFSET_UUID_TYPE = 0x8000,
|
2018-10-05 09:20:30 +00:00
|
|
|
};
|
|
|
|
|
|
|
|
struct CItemEx
|
|
|
|
{
|
2023-02-04 00:22:49 +00:00
|
|
|
int m_aUuid[sizeof(CUuid) / sizeof(int32_t)];
|
2018-10-05 09:20:30 +00:00
|
|
|
|
|
|
|
static CItemEx FromUuid(CUuid Uuid)
|
|
|
|
{
|
|
|
|
CItemEx Result;
|
2023-02-04 00:22:49 +00:00
|
|
|
for(size_t i = 0; i < sizeof(CUuid) / sizeof(int32_t); i++)
|
|
|
|
Result.m_aUuid[i] = bytes_be_to_uint(&Uuid.m_aData[i * sizeof(int32_t)]);
|
2018-10-05 09:20:30 +00:00
|
|
|
return Result;
|
|
|
|
}
|
|
|
|
|
|
|
|
CUuid ToUuid() const
|
|
|
|
{
|
|
|
|
CUuid Result;
|
2023-02-04 00:22:49 +00:00
|
|
|
for(size_t i = 0; i < sizeof(CUuid) / sizeof(int32_t); i++)
|
|
|
|
uint_to_bytes_be(&Result.m_aData[i * sizeof(int32_t)], m_aUuid[i]);
|
2018-10-05 09:20:30 +00:00
|
|
|
return Result;
|
|
|
|
}
|
|
|
|
};
|
|
|
|
|
2010-05-29 07:25:38 +00:00
|
|
|
struct CDatafileItemType
|
|
|
|
{
|
|
|
|
int m_Type;
|
|
|
|
int m_Start;
|
|
|
|
int m_Num;
|
2020-09-26 19:41:58 +00:00
|
|
|
};
|
2010-05-29 07:25:38 +00:00
|
|
|
|
|
|
|
struct CDatafileItem
|
|
|
|
{
|
2011-02-12 10:40:36 +00:00
|
|
|
int m_TypeAndID;
|
2010-05-29 07:25:38 +00:00
|
|
|
int m_Size;
|
|
|
|
};
|
|
|
|
|
|
|
|
struct CDatafileHeader
|
|
|
|
{
|
2011-02-12 10:40:36 +00:00
|
|
|
char m_aID[4];
|
2010-05-29 07:25:38 +00:00
|
|
|
int m_Version;
|
|
|
|
int m_Size;
|
|
|
|
int m_Swaplen;
|
|
|
|
int m_NumItemTypes;
|
|
|
|
int m_NumItems;
|
|
|
|
int m_NumRawData;
|
|
|
|
int m_ItemSize;
|
|
|
|
int m_DataSize;
|
2023-11-01 11:45:45 +00:00
|
|
|
|
|
|
|
constexpr size_t SizeOffset()
|
|
|
|
{
|
|
|
|
// The size of these members is not included in m_Size and m_Swaplen
|
|
|
|
return sizeof(m_aID) + sizeof(m_Version) + sizeof(m_Size) + sizeof(m_Swaplen);
|
|
|
|
}
|
2010-05-29 07:25:38 +00:00
|
|
|
};
|
|
|
|
|
|
|
|
struct CDatafileInfo
|
|
|
|
{
|
|
|
|
CDatafileItemType *m_pItemTypes;
|
|
|
|
int *m_pItemOffsets;
|
|
|
|
int *m_pDataOffsets;
|
|
|
|
int *m_pDataSizes;
|
|
|
|
|
|
|
|
char *m_pItemStart;
|
|
|
|
char *m_pDataStart;
|
|
|
|
};
|
|
|
|
|
|
|
|
struct CDatafile
|
|
|
|
{
|
|
|
|
IOHANDLE m_File;
|
2018-06-05 19:22:40 +00:00
|
|
|
SHA256_DIGEST m_Sha256;
|
2010-05-29 07:25:38 +00:00
|
|
|
unsigned m_Crc;
|
|
|
|
CDatafileInfo m_Info;
|
|
|
|
CDatafileHeader m_Header;
|
|
|
|
int m_DataStartOffset;
|
|
|
|
char **m_ppDataPtrs;
|
2022-08-14 16:00:52 +00:00
|
|
|
int *m_pDataSizes;
|
2010-05-29 07:25:38 +00:00
|
|
|
char *m_pData;
|
|
|
|
};
|
|
|
|
|
2010-10-06 21:07:35 +00:00
|
|
|
bool CDataFileReader::Open(class IStorage *pStorage, const char *pFilename, int StorageType)
|
2010-05-29 07:25:38 +00:00
|
|
|
{
|
2022-10-01 20:45:10 +00:00
|
|
|
log_trace("datafile", "loading. filename='%s'", pFilename);
|
2010-05-29 07:25:38 +00:00
|
|
|
|
2010-10-06 21:07:35 +00:00
|
|
|
IOHANDLE File = pStorage->OpenFile(pFilename, IOFLAG_READ, StorageType);
|
2010-05-29 07:25:38 +00:00
|
|
|
if(!File)
|
|
|
|
{
|
|
|
|
dbg_msg("datafile", "could not open '%s'", pFilename);
|
|
|
|
return false;
|
2011-04-13 18:37:12 +00:00
|
|
|
}
|
|
|
|
|
2010-05-29 07:25:38 +00:00
|
|
|
// take the CRC of the file and store it
|
|
|
|
unsigned Crc = 0;
|
2018-06-05 19:22:40 +00:00
|
|
|
SHA256_DIGEST Sha256;
|
2010-05-29 07:25:38 +00:00
|
|
|
{
|
|
|
|
enum
|
|
|
|
{
|
2020-09-26 19:41:58 +00:00
|
|
|
BUFFER_SIZE = 64 * 1024
|
2010-05-29 07:25:38 +00:00
|
|
|
};
|
2011-04-13 18:37:12 +00:00
|
|
|
|
2018-06-05 19:22:40 +00:00
|
|
|
SHA256_CTX Sha256Ctxt;
|
|
|
|
sha256_init(&Sha256Ctxt);
|
2010-05-29 07:25:38 +00:00
|
|
|
unsigned char aBuffer[BUFFER_SIZE];
|
2011-04-13 18:37:12 +00:00
|
|
|
|
2022-02-14 23:12:52 +00:00
|
|
|
while(true)
|
2010-05-29 07:25:38 +00:00
|
|
|
{
|
|
|
|
unsigned Bytes = io_read(File, aBuffer, BUFFER_SIZE);
|
2022-10-30 13:01:47 +00:00
|
|
|
if(Bytes == 0)
|
2010-05-29 07:25:38 +00:00
|
|
|
break;
|
2022-02-16 19:54:11 +00:00
|
|
|
Crc = crc32(Crc, aBuffer, Bytes);
|
2018-06-05 19:22:40 +00:00
|
|
|
sha256_update(&Sha256Ctxt, aBuffer, Bytes);
|
2010-05-29 07:25:38 +00:00
|
|
|
}
|
2018-06-05 19:22:40 +00:00
|
|
|
Sha256 = sha256_finish(&Sha256Ctxt);
|
2011-04-13 18:37:12 +00:00
|
|
|
|
2010-05-29 07:25:38 +00:00
|
|
|
io_seek(File, 0, IOSEEK_START);
|
|
|
|
}
|
2011-04-13 18:37:12 +00:00
|
|
|
|
2010-05-29 07:25:38 +00:00
|
|
|
// TODO: change this header
|
|
|
|
CDatafileHeader Header;
|
2020-09-26 19:41:58 +00:00
|
|
|
if(sizeof(Header) != io_read(File, &Header, sizeof(Header)))
|
2016-01-18 16:20:47 +00:00
|
|
|
{
|
2016-01-17 08:48:21 +00:00
|
|
|
dbg_msg("datafile", "couldn't load header");
|
2022-02-14 23:12:52 +00:00
|
|
|
return false;
|
2016-01-17 08:48:21 +00:00
|
|
|
}
|
2011-02-12 10:40:36 +00:00
|
|
|
if(Header.m_aID[0] != 'A' || Header.m_aID[1] != 'T' || Header.m_aID[2] != 'A' || Header.m_aID[3] != 'D')
|
2010-05-29 07:25:38 +00:00
|
|
|
{
|
2011-02-12 10:40:36 +00:00
|
|
|
if(Header.m_aID[0] != 'D' || Header.m_aID[1] != 'A' || Header.m_aID[2] != 'T' || Header.m_aID[3] != 'A')
|
2010-05-29 07:25:38 +00:00
|
|
|
{
|
2011-02-12 10:40:36 +00:00
|
|
|
dbg_msg("datafile", "wrong signature. %x %x %x %x", Header.m_aID[0], Header.m_aID[1], Header.m_aID[2], Header.m_aID[3]);
|
2022-02-14 23:12:52 +00:00
|
|
|
return false;
|
2010-05-29 07:25:38 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
#if defined(CONF_ARCH_ENDIAN_BIG)
|
2020-09-26 19:41:58 +00:00
|
|
|
swap_endian(&Header, sizeof(int), sizeof(Header) / sizeof(int));
|
2010-05-29 07:25:38 +00:00
|
|
|
#endif
|
|
|
|
if(Header.m_Version != 3 && Header.m_Version != 4)
|
|
|
|
{
|
|
|
|
dbg_msg("datafile", "wrong version. version=%x", Header.m_Version);
|
2022-02-14 23:12:52 +00:00
|
|
|
return false;
|
2010-05-29 07:25:38 +00:00
|
|
|
}
|
2011-04-13 18:37:12 +00:00
|
|
|
|
2010-05-29 07:25:38 +00:00
|
|
|
// read in the rest except the data
|
|
|
|
unsigned Size = 0;
|
2020-09-26 19:41:58 +00:00
|
|
|
Size += Header.m_NumItemTypes * sizeof(CDatafileItemType);
|
|
|
|
Size += (Header.m_NumItems + Header.m_NumRawData) * sizeof(int);
|
2010-05-29 07:25:38 +00:00
|
|
|
if(Header.m_Version == 4)
|
2020-09-26 19:41:58 +00:00
|
|
|
Size += Header.m_NumRawData * sizeof(int); // v4 has uncompressed data sizes as well
|
2010-05-29 07:25:38 +00:00
|
|
|
Size += Header.m_ItemSize;
|
2011-04-13 18:37:12 +00:00
|
|
|
|
2010-05-29 07:25:38 +00:00
|
|
|
unsigned AllocSize = Size;
|
|
|
|
AllocSize += sizeof(CDatafile); // add space for info structure
|
2020-09-26 19:41:58 +00:00
|
|
|
AllocSize += Header.m_NumRawData * sizeof(void *); // add space for data pointers
|
2022-08-14 16:00:52 +00:00
|
|
|
AllocSize += Header.m_NumRawData * sizeof(int); // add space for data sizes
|
|
|
|
if(Size > (((int64_t)1) << 31) || Header.m_NumItemTypes < 0 || Header.m_NumItems < 0 || Header.m_NumRawData < 0 || Header.m_ItemSize < 0)
|
|
|
|
{
|
|
|
|
io_close(File);
|
|
|
|
dbg_msg("datafile", "unable to load file, invalid file information");
|
|
|
|
return false;
|
|
|
|
}
|
2010-05-29 07:25:38 +00:00
|
|
|
|
2018-04-09 09:56:39 +00:00
|
|
|
CDatafile *pTmpDataFile = (CDatafile *)malloc(AllocSize);
|
2010-09-28 10:46:15 +00:00
|
|
|
pTmpDataFile->m_Header = Header;
|
|
|
|
pTmpDataFile->m_DataStartOffset = sizeof(CDatafileHeader) + Size;
|
2020-09-26 19:41:58 +00:00
|
|
|
pTmpDataFile->m_ppDataPtrs = (char **)(pTmpDataFile + 1);
|
2022-08-14 16:00:52 +00:00
|
|
|
pTmpDataFile->m_pDataSizes = (int *)(pTmpDataFile->m_ppDataPtrs + Header.m_NumRawData);
|
|
|
|
pTmpDataFile->m_pData = (char *)(pTmpDataFile->m_pDataSizes + Header.m_NumRawData);
|
2010-09-28 10:46:15 +00:00
|
|
|
pTmpDataFile->m_File = File;
|
2018-06-05 19:22:40 +00:00
|
|
|
pTmpDataFile->m_Sha256 = Sha256;
|
2010-09-28 10:46:15 +00:00
|
|
|
pTmpDataFile->m_Crc = Crc;
|
2011-04-13 18:37:12 +00:00
|
|
|
|
2022-08-14 16:00:52 +00:00
|
|
|
// clear the data pointers and sizes
|
2020-09-26 19:41:58 +00:00
|
|
|
mem_zero(pTmpDataFile->m_ppDataPtrs, Header.m_NumRawData * sizeof(void *));
|
2022-08-14 16:00:52 +00:00
|
|
|
mem_zero(pTmpDataFile->m_pDataSizes, Header.m_NumRawData * sizeof(int));
|
2011-04-13 18:37:12 +00:00
|
|
|
|
2010-05-29 07:25:38 +00:00
|
|
|
// read types, offsets, sizes and item data
|
2010-09-28 10:46:15 +00:00
|
|
|
unsigned ReadSize = io_read(File, pTmpDataFile->m_pData, Size);
|
2010-05-29 07:25:38 +00:00
|
|
|
if(ReadSize != Size)
|
|
|
|
{
|
2010-09-28 10:46:15 +00:00
|
|
|
io_close(pTmpDataFile->m_File);
|
2018-04-09 09:56:39 +00:00
|
|
|
free(pTmpDataFile);
|
2010-05-29 07:25:38 +00:00
|
|
|
dbg_msg("datafile", "couldn't load the whole thing, wanted=%d got=%d", Size, ReadSize);
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
|
2010-09-28 10:46:15 +00:00
|
|
|
Close();
|
|
|
|
m_pDataFile = pTmpDataFile;
|
|
|
|
|
2010-05-29 07:25:38 +00:00
|
|
|
#if defined(CONF_ARCH_ENDIAN_BIG)
|
2019-04-26 19:36:49 +00:00
|
|
|
swap_endian(m_pDataFile->m_pData, sizeof(int), minimum(static_cast<unsigned>(Header.m_Swaplen), Size) / sizeof(int));
|
2010-05-29 07:25:38 +00:00
|
|
|
#endif
|
|
|
|
|
2022-05-30 20:43:06 +00:00
|
|
|
if(DEBUG)
|
2010-05-29 07:25:38 +00:00
|
|
|
{
|
|
|
|
dbg_msg("datafile", "allocsize=%d", AllocSize);
|
|
|
|
dbg_msg("datafile", "readsize=%d", ReadSize);
|
|
|
|
dbg_msg("datafile", "swaplen=%d", Header.m_Swaplen);
|
|
|
|
dbg_msg("datafile", "item_size=%d", m_pDataFile->m_Header.m_ItemSize);
|
|
|
|
}
|
2011-04-13 18:37:12 +00:00
|
|
|
|
2010-05-29 07:25:38 +00:00
|
|
|
m_pDataFile->m_Info.m_pItemTypes = (CDatafileItemType *)m_pDataFile->m_pData;
|
|
|
|
m_pDataFile->m_Info.m_pItemOffsets = (int *)&m_pDataFile->m_Info.m_pItemTypes[m_pDataFile->m_Header.m_NumItemTypes];
|
2019-04-11 10:21:42 +00:00
|
|
|
m_pDataFile->m_Info.m_pDataOffsets = &m_pDataFile->m_Info.m_pItemOffsets[m_pDataFile->m_Header.m_NumItems];
|
|
|
|
m_pDataFile->m_Info.m_pDataSizes = &m_pDataFile->m_Info.m_pDataOffsets[m_pDataFile->m_Header.m_NumRawData];
|
2011-04-13 18:37:12 +00:00
|
|
|
|
2010-05-29 07:25:38 +00:00
|
|
|
if(Header.m_Version == 4)
|
|
|
|
m_pDataFile->m_Info.m_pItemStart = (char *)&m_pDataFile->m_Info.m_pDataSizes[m_pDataFile->m_Header.m_NumRawData];
|
|
|
|
else
|
|
|
|
m_pDataFile->m_Info.m_pItemStart = (char *)&m_pDataFile->m_Info.m_pDataOffsets[m_pDataFile->m_Header.m_NumRawData];
|
|
|
|
m_pDataFile->m_Info.m_pDataStart = m_pDataFile->m_Info.m_pItemStart + m_pDataFile->m_Header.m_ItemSize;
|
|
|
|
|
2022-10-01 20:45:10 +00:00
|
|
|
log_trace("datafile", "loading done. datafile='%s'", pFilename);
|
2010-05-29 07:25:38 +00:00
|
|
|
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
2023-05-15 16:04:58 +00:00
|
|
|
bool CDataFileReader::Close()
|
|
|
|
{
|
|
|
|
if(!m_pDataFile)
|
|
|
|
return true;
|
|
|
|
|
|
|
|
// free the data that is loaded
|
|
|
|
for(int i = 0; i < m_pDataFile->m_Header.m_NumRawData; i++)
|
2022-08-14 16:00:52 +00:00
|
|
|
{
|
2023-05-15 16:04:58 +00:00
|
|
|
free(m_pDataFile->m_ppDataPtrs[i]);
|
2022-08-14 16:00:52 +00:00
|
|
|
m_pDataFile->m_ppDataPtrs[i] = nullptr;
|
|
|
|
m_pDataFile->m_pDataSizes[i] = 0;
|
|
|
|
}
|
2023-05-15 16:04:58 +00:00
|
|
|
|
|
|
|
io_close(m_pDataFile->m_File);
|
|
|
|
free(m_pDataFile);
|
|
|
|
m_pDataFile = nullptr;
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
2023-05-15 16:15:17 +00:00
|
|
|
IOHANDLE CDataFileReader::File() const
|
2023-05-15 16:04:58 +00:00
|
|
|
{
|
|
|
|
if(!m_pDataFile)
|
|
|
|
return 0;
|
|
|
|
return m_pDataFile->m_File;
|
|
|
|
}
|
|
|
|
|
2021-02-08 21:26:26 +00:00
|
|
|
int CDataFileReader::NumData() const
|
2010-05-29 07:25:38 +00:00
|
|
|
{
|
2020-09-26 19:41:58 +00:00
|
|
|
if(!m_pDataFile)
|
|
|
|
{
|
|
|
|
return 0;
|
|
|
|
}
|
2010-05-29 07:25:38 +00:00
|
|
|
return m_pDataFile->m_Header.m_NumRawData;
|
|
|
|
}
|
|
|
|
|
2017-08-30 06:36:17 +00:00
|
|
|
// returns the size in the file
|
2023-05-15 16:15:17 +00:00
|
|
|
int CDataFileReader::GetFileDataSize(int Index) const
|
2010-05-29 07:25:38 +00:00
|
|
|
{
|
2020-09-26 19:41:58 +00:00
|
|
|
if(!m_pDataFile)
|
|
|
|
{
|
|
|
|
return 0;
|
|
|
|
}
|
2011-04-13 18:37:12 +00:00
|
|
|
|
2020-09-26 19:41:58 +00:00
|
|
|
if(Index == m_pDataFile->m_Header.m_NumRawData - 1)
|
|
|
|
return m_pDataFile->m_Header.m_DataSize - m_pDataFile->m_Info.m_pDataOffsets[Index];
|
2022-08-14 16:00:52 +00:00
|
|
|
|
2020-09-26 19:41:58 +00:00
|
|
|
return m_pDataFile->m_Info.m_pDataOffsets[Index + 1] - m_pDataFile->m_Info.m_pDataOffsets[Index];
|
2010-05-29 07:25:38 +00:00
|
|
|
}
|
|
|
|
|
2017-08-30 06:36:17 +00:00
|
|
|
// returns the size of the resulting data
|
2023-05-15 16:15:17 +00:00
|
|
|
int CDataFileReader::GetDataSize(int Index) const
|
2014-01-10 23:46:32 +00:00
|
|
|
{
|
2022-08-14 16:00:52 +00:00
|
|
|
if(!m_pDataFile || Index < 0 || Index >= m_pDataFile->m_Header.m_NumRawData)
|
2020-09-26 19:41:58 +00:00
|
|
|
{
|
|
|
|
return 0;
|
|
|
|
}
|
2014-01-10 23:46:32 +00:00
|
|
|
|
2022-08-14 16:00:52 +00:00
|
|
|
if(!m_pDataFile->m_ppDataPtrs[Index])
|
|
|
|
{
|
|
|
|
if(m_pDataFile->m_Header.m_Version >= 4)
|
|
|
|
{
|
|
|
|
return m_pDataFile->m_Info.m_pDataSizes[Index];
|
|
|
|
}
|
|
|
|
else
|
|
|
|
{
|
|
|
|
return GetFileDataSize(Index);
|
|
|
|
}
|
|
|
|
}
|
2023-07-26 14:31:21 +00:00
|
|
|
const int Size = m_pDataFile->m_pDataSizes[Index];
|
|
|
|
if(Size < 0)
|
|
|
|
return 0; // summarize all errors as zero size
|
|
|
|
return Size;
|
2014-01-10 23:46:32 +00:00
|
|
|
}
|
|
|
|
|
2023-08-13 09:46:36 +00:00
|
|
|
void *CDataFileReader::GetDataImpl(int Index, bool Swap)
|
2010-05-29 07:25:38 +00:00
|
|
|
{
|
2020-09-26 19:41:58 +00:00
|
|
|
if(!m_pDataFile)
|
|
|
|
{
|
2023-05-15 18:35:17 +00:00
|
|
|
return nullptr;
|
2020-09-26 19:41:58 +00:00
|
|
|
}
|
2011-04-13 18:37:12 +00:00
|
|
|
|
2019-03-25 15:28:54 +00:00
|
|
|
if(Index < 0 || Index >= m_pDataFile->m_Header.m_NumRawData)
|
2023-05-15 18:35:17 +00:00
|
|
|
return nullptr;
|
2019-03-25 15:28:54 +00:00
|
|
|
|
2010-05-29 07:25:38 +00:00
|
|
|
// load it if needed
|
|
|
|
if(!m_pDataFile->m_ppDataPtrs[Index])
|
|
|
|
{
|
2023-07-26 14:31:21 +00:00
|
|
|
// don't try to load again if it previously failed
|
|
|
|
if(m_pDataFile->m_pDataSizes[Index] < 0)
|
|
|
|
return nullptr;
|
|
|
|
|
2010-05-29 07:25:38 +00:00
|
|
|
// fetch the data size
|
2023-07-26 14:31:21 +00:00
|
|
|
unsigned DataSize = GetFileDataSize(Index);
|
2011-05-04 23:43:27 +00:00
|
|
|
#if defined(CONF_ARCH_ENDIAN_BIG)
|
2023-07-26 14:31:21 +00:00
|
|
|
unsigned SwapSize = DataSize;
|
2011-05-04 23:43:27 +00:00
|
|
|
#endif
|
2011-04-13 18:37:12 +00:00
|
|
|
|
2010-05-29 07:25:38 +00:00
|
|
|
if(m_pDataFile->m_Header.m_Version == 4)
|
|
|
|
{
|
|
|
|
// v4 has compressed data
|
2023-07-26 14:31:21 +00:00
|
|
|
const unsigned OriginalUncompressedSize = m_pDataFile->m_Info.m_pDataSizes[Index];
|
|
|
|
unsigned long UncompressedSize = OriginalUncompressedSize;
|
2010-05-29 07:25:38 +00:00
|
|
|
|
2023-07-26 14:31:21 +00:00
|
|
|
log_trace("datafile", "loading data. index=%d size=%u uncompressed=%u", Index, DataSize, OriginalUncompressedSize);
|
2011-04-13 18:37:12 +00:00
|
|
|
|
2010-05-29 07:25:38 +00:00
|
|
|
// read the compressed data
|
2023-07-26 14:31:21 +00:00
|
|
|
void *pCompressedData = malloc(DataSize);
|
|
|
|
unsigned ActualDataSize = 0;
|
|
|
|
if(io_seek(m_pDataFile->m_File, m_pDataFile->m_DataStartOffset + m_pDataFile->m_Info.m_pDataOffsets[Index], IOSEEK_START) == 0)
|
|
|
|
ActualDataSize = io_read(m_pDataFile->m_File, pCompressedData, DataSize);
|
|
|
|
if(DataSize != ActualDataSize)
|
|
|
|
{
|
|
|
|
log_error("datafile", "truncation error, could not read all data. index=%d wanted=%u got=%u", Index, DataSize, ActualDataSize);
|
|
|
|
free(pCompressedData);
|
|
|
|
m_pDataFile->m_ppDataPtrs[Index] = nullptr;
|
|
|
|
m_pDataFile->m_pDataSizes[Index] = -1;
|
|
|
|
return nullptr;
|
|
|
|
}
|
|
|
|
|
|
|
|
// decompress the data
|
|
|
|
m_pDataFile->m_ppDataPtrs[Index] = (char *)malloc(UncompressedSize);
|
|
|
|
m_pDataFile->m_pDataSizes[Index] = UncompressedSize;
|
|
|
|
const int Result = uncompress((Bytef *)m_pDataFile->m_ppDataPtrs[Index], &UncompressedSize, (Bytef *)pCompressedData, DataSize);
|
|
|
|
free(pCompressedData);
|
|
|
|
if(Result != Z_OK || UncompressedSize != OriginalUncompressedSize)
|
|
|
|
{
|
|
|
|
log_error("datafile", "uncompress error. result=%d wanted=%u got=%lu", Result, OriginalUncompressedSize, UncompressedSize);
|
|
|
|
free(m_pDataFile->m_ppDataPtrs[Index]);
|
|
|
|
m_pDataFile->m_ppDataPtrs[Index] = nullptr;
|
|
|
|
m_pDataFile->m_pDataSizes[Index] = -1;
|
|
|
|
return nullptr;
|
|
|
|
}
|
2010-05-29 07:25:38 +00:00
|
|
|
|
2011-05-04 23:43:27 +00:00
|
|
|
#if defined(CONF_ARCH_ENDIAN_BIG)
|
2023-07-26 14:31:21 +00:00
|
|
|
SwapSize = UncompressedSize;
|
2011-05-04 23:43:27 +00:00
|
|
|
#endif
|
2010-05-29 07:25:38 +00:00
|
|
|
}
|
|
|
|
else
|
|
|
|
{
|
|
|
|
// load the data
|
2023-07-26 14:31:21 +00:00
|
|
|
log_trace("datafile", "loading data. index=%d size=%d", Index, DataSize);
|
|
|
|
m_pDataFile->m_ppDataPtrs[Index] = static_cast<char *>(malloc(DataSize));
|
2022-08-14 16:00:52 +00:00
|
|
|
m_pDataFile->m_pDataSizes[Index] = DataSize;
|
2023-07-26 14:31:21 +00:00
|
|
|
unsigned ActualDataSize = 0;
|
|
|
|
if(io_seek(m_pDataFile->m_File, m_pDataFile->m_DataStartOffset + m_pDataFile->m_Info.m_pDataOffsets[Index], IOSEEK_START) == 0)
|
|
|
|
ActualDataSize = io_read(m_pDataFile->m_File, m_pDataFile->m_ppDataPtrs[Index], DataSize);
|
|
|
|
if(DataSize != ActualDataSize)
|
|
|
|
{
|
|
|
|
log_error("datafile", "truncation error, could not read all data. index=%d wanted=%u got=%u", Index, DataSize, ActualDataSize);
|
|
|
|
free(m_pDataFile->m_ppDataPtrs[Index]);
|
|
|
|
m_pDataFile->m_ppDataPtrs[Index] = nullptr;
|
|
|
|
m_pDataFile->m_pDataSizes[Index] = -1;
|
|
|
|
return nullptr;
|
|
|
|
}
|
2010-05-29 07:25:38 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
#if defined(CONF_ARCH_ENDIAN_BIG)
|
|
|
|
if(Swap && SwapSize)
|
2020-09-26 19:41:58 +00:00
|
|
|
swap_endian(m_pDataFile->m_ppDataPtrs[Index], sizeof(int), SwapSize / sizeof(int));
|
2010-05-29 07:25:38 +00:00
|
|
|
#endif
|
|
|
|
}
|
2011-04-13 18:37:12 +00:00
|
|
|
|
2010-05-29 07:25:38 +00:00
|
|
|
return m_pDataFile->m_ppDataPtrs[Index];
|
|
|
|
}
|
|
|
|
|
|
|
|
void *CDataFileReader::GetData(int Index)
|
|
|
|
{
|
2023-08-13 09:46:36 +00:00
|
|
|
return GetDataImpl(Index, false);
|
2010-05-29 07:25:38 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
void *CDataFileReader::GetDataSwapped(int Index)
|
|
|
|
{
|
2023-08-13 09:46:36 +00:00
|
|
|
return GetDataImpl(Index, true);
|
2010-05-29 07:25:38 +00:00
|
|
|
}
|
|
|
|
|
2023-08-13 09:50:35 +00:00
|
|
|
const char *CDataFileReader::GetDataString(int Index)
|
|
|
|
{
|
|
|
|
if(Index == -1)
|
|
|
|
return "";
|
|
|
|
const int DataSize = GetDataSize(Index);
|
|
|
|
if(!DataSize)
|
|
|
|
return nullptr;
|
|
|
|
const char *pData = static_cast<char *>(GetData(Index));
|
|
|
|
if(pData == nullptr || mem_has_null(pData, DataSize - 1) || pData[DataSize - 1] != '\0' || !str_utf8_check(pData))
|
|
|
|
return nullptr;
|
|
|
|
return pData;
|
|
|
|
}
|
|
|
|
|
2022-08-14 16:00:52 +00:00
|
|
|
void CDataFileReader::ReplaceData(int Index, char *pData, size_t Size)
|
2017-12-11 08:50:15 +00:00
|
|
|
{
|
2023-08-13 10:24:35 +00:00
|
|
|
dbg_assert(Index >= 0 && Index < m_pDataFile->m_Header.m_NumRawData, "Index invalid");
|
|
|
|
|
2023-07-26 14:31:21 +00:00
|
|
|
free(m_pDataFile->m_ppDataPtrs[Index]);
|
2017-12-11 08:50:15 +00:00
|
|
|
m_pDataFile->m_ppDataPtrs[Index] = pData;
|
2022-08-14 16:00:52 +00:00
|
|
|
m_pDataFile->m_pDataSizes[Index] = Size;
|
2017-12-11 08:50:15 +00:00
|
|
|
}
|
|
|
|
|
2010-05-29 07:25:38 +00:00
|
|
|
void CDataFileReader::UnloadData(int Index)
|
|
|
|
{
|
2019-03-25 15:28:54 +00:00
|
|
|
if(Index < 0 || Index >= m_pDataFile->m_Header.m_NumRawData)
|
2010-05-29 07:25:38 +00:00
|
|
|
return;
|
2011-04-13 18:37:12 +00:00
|
|
|
|
2018-04-09 09:56:39 +00:00
|
|
|
free(m_pDataFile->m_ppDataPtrs[Index]);
|
2023-05-15 18:35:17 +00:00
|
|
|
m_pDataFile->m_ppDataPtrs[Index] = nullptr;
|
2022-08-14 16:00:52 +00:00
|
|
|
m_pDataFile->m_pDataSizes[Index] = 0;
|
2010-05-29 07:25:38 +00:00
|
|
|
}
|
|
|
|
|
2021-02-08 21:26:26 +00:00
|
|
|
int CDataFileReader::GetItemSize(int Index) const
|
2010-05-29 07:25:38 +00:00
|
|
|
{
|
2017-08-28 16:06:19 +00:00
|
|
|
if(!m_pDataFile)
|
|
|
|
return 0;
|
2020-09-26 19:41:58 +00:00
|
|
|
if(Index == m_pDataFile->m_Header.m_NumItems - 1)
|
|
|
|
return m_pDataFile->m_Header.m_ItemSize - m_pDataFile->m_Info.m_pItemOffsets[Index] - sizeof(CDatafileItem);
|
|
|
|
return m_pDataFile->m_Info.m_pItemOffsets[Index + 1] - m_pDataFile->m_Info.m_pItemOffsets[Index] - sizeof(CDatafileItem);
|
2010-05-29 07:25:38 +00:00
|
|
|
}
|
|
|
|
|
2018-10-05 09:20:30 +00:00
|
|
|
int CDataFileReader::GetExternalItemType(int InternalType)
|
|
|
|
{
|
|
|
|
if(InternalType <= OFFSET_UUID_TYPE || InternalType == ITEMTYPE_EX)
|
|
|
|
{
|
|
|
|
return InternalType;
|
|
|
|
}
|
|
|
|
int TypeIndex = FindItemIndex(ITEMTYPE_EX, InternalType);
|
|
|
|
if(TypeIndex < 0 || GetItemSize(TypeIndex) < (int)sizeof(CItemEx))
|
|
|
|
{
|
|
|
|
return InternalType;
|
|
|
|
}
|
2023-02-20 22:38:54 +00:00
|
|
|
const CItemEx *pItemEx = (const CItemEx *)GetItem(TypeIndex);
|
2018-10-05 09:20:30 +00:00
|
|
|
// Propagate UUID_UNKNOWN, it doesn't hurt.
|
|
|
|
return g_UuidManager.LookupUuid(pItemEx->ToUuid());
|
|
|
|
}
|
|
|
|
|
|
|
|
int CDataFileReader::GetInternalItemType(int ExternalType)
|
|
|
|
{
|
|
|
|
if(ExternalType < OFFSET_UUID)
|
|
|
|
{
|
|
|
|
return ExternalType;
|
|
|
|
}
|
|
|
|
CUuid Uuid = g_UuidManager.GetUuid(ExternalType);
|
|
|
|
int Start, Num;
|
|
|
|
GetType(ITEMTYPE_EX, &Start, &Num);
|
|
|
|
for(int i = Start; i < Start + Num; i++)
|
|
|
|
{
|
|
|
|
if(GetItemSize(i) < (int)sizeof(CItemEx))
|
|
|
|
{
|
|
|
|
continue;
|
|
|
|
}
|
|
|
|
int ID;
|
2023-02-20 22:38:54 +00:00
|
|
|
if(Uuid == ((const CItemEx *)GetItem(i, nullptr, &ID))->ToUuid())
|
2018-10-05 09:20:30 +00:00
|
|
|
{
|
|
|
|
return ID;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return -1;
|
|
|
|
}
|
|
|
|
|
2011-02-12 10:40:36 +00:00
|
|
|
void *CDataFileReader::GetItem(int Index, int *pType, int *pID)
|
2010-05-29 07:25:38 +00:00
|
|
|
{
|
2020-09-26 19:41:58 +00:00
|
|
|
if(!m_pDataFile)
|
|
|
|
{
|
|
|
|
if(pType)
|
|
|
|
*pType = 0;
|
|
|
|
if(pID)
|
|
|
|
*pID = 0;
|
2023-05-15 18:35:17 +00:00
|
|
|
return nullptr;
|
2020-09-26 19:41:58 +00:00
|
|
|
}
|
2011-04-13 18:37:12 +00:00
|
|
|
|
2022-06-30 22:36:32 +00:00
|
|
|
CDatafileItem *pItem = (CDatafileItem *)(m_pDataFile->m_Info.m_pItemStart + m_pDataFile->m_Info.m_pItemOffsets[Index]);
|
2010-05-29 07:25:38 +00:00
|
|
|
if(pType)
|
2018-10-05 09:20:30 +00:00
|
|
|
{
|
|
|
|
// remove sign extension
|
2022-06-30 22:36:32 +00:00
|
|
|
*pType = GetExternalItemType((pItem->m_TypeAndID >> 16) & 0xffff);
|
2018-10-05 09:20:30 +00:00
|
|
|
}
|
2011-02-12 10:40:36 +00:00
|
|
|
if(pID)
|
2018-10-05 09:20:30 +00:00
|
|
|
{
|
2022-06-30 22:36:32 +00:00
|
|
|
*pID = pItem->m_TypeAndID & 0xffff;
|
2018-10-05 09:20:30 +00:00
|
|
|
}
|
2022-06-30 22:36:32 +00:00
|
|
|
return (void *)(pItem + 1);
|
2010-05-29 07:25:38 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
void CDataFileReader::GetType(int Type, int *pStart, int *pNum)
|
|
|
|
{
|
|
|
|
*pStart = 0;
|
|
|
|
*pNum = 0;
|
|
|
|
|
|
|
|
if(!m_pDataFile)
|
|
|
|
return;
|
2011-04-13 18:37:12 +00:00
|
|
|
|
2018-10-05 09:20:30 +00:00
|
|
|
Type = GetInternalItemType(Type);
|
2010-05-29 07:25:38 +00:00
|
|
|
for(int i = 0; i < m_pDataFile->m_Header.m_NumItemTypes; i++)
|
|
|
|
{
|
|
|
|
if(m_pDataFile->m_Info.m_pItemTypes[i].m_Type == Type)
|
|
|
|
{
|
|
|
|
*pStart = m_pDataFile->m_Info.m_pItemTypes[i].m_Start;
|
|
|
|
*pNum = m_pDataFile->m_Info.m_pItemTypes[i].m_Num;
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2018-10-05 09:20:30 +00:00
|
|
|
int CDataFileReader::FindItemIndex(int Type, int ID)
|
2010-05-29 07:25:38 +00:00
|
|
|
{
|
2018-10-05 09:20:30 +00:00
|
|
|
if(!m_pDataFile)
|
|
|
|
{
|
|
|
|
return -1;
|
|
|
|
}
|
2011-04-13 18:37:12 +00:00
|
|
|
|
2010-05-29 07:25:38 +00:00
|
|
|
int Start, Num;
|
|
|
|
GetType(Type, &Start, &Num);
|
|
|
|
for(int i = 0; i < Num; i++)
|
|
|
|
{
|
2011-02-12 10:40:36 +00:00
|
|
|
int ItemID;
|
2023-02-20 22:38:54 +00:00
|
|
|
GetItem(Start + i, nullptr, &ItemID);
|
2011-02-12 10:40:36 +00:00
|
|
|
if(ID == ItemID)
|
2018-10-05 09:20:30 +00:00
|
|
|
{
|
|
|
|
return Start + i;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return -1;
|
|
|
|
}
|
|
|
|
|
|
|
|
void *CDataFileReader::FindItem(int Type, int ID)
|
|
|
|
{
|
|
|
|
int Index = FindItemIndex(Type, ID);
|
|
|
|
if(Index < 0)
|
|
|
|
{
|
2023-05-15 18:35:17 +00:00
|
|
|
return nullptr;
|
2010-05-29 07:25:38 +00:00
|
|
|
}
|
2023-02-20 22:38:54 +00:00
|
|
|
return GetItem(Index);
|
2010-05-29 07:25:38 +00:00
|
|
|
}
|
|
|
|
|
2021-02-08 21:26:26 +00:00
|
|
|
int CDataFileReader::NumItems() const
|
2010-05-29 07:25:38 +00:00
|
|
|
{
|
2020-09-26 19:41:58 +00:00
|
|
|
if(!m_pDataFile)
|
|
|
|
return 0;
|
2010-05-29 07:25:38 +00:00
|
|
|
return m_pDataFile->m_Header.m_NumItems;
|
|
|
|
}
|
|
|
|
|
2021-02-08 21:26:26 +00:00
|
|
|
SHA256_DIGEST CDataFileReader::Sha256() const
|
2018-06-05 19:22:40 +00:00
|
|
|
{
|
|
|
|
if(!m_pDataFile)
|
|
|
|
{
|
|
|
|
SHA256_DIGEST Result;
|
2020-10-26 14:14:07 +00:00
|
|
|
for(unsigned char &d : Result.data)
|
2018-06-05 19:22:40 +00:00
|
|
|
{
|
2020-10-26 14:14:07 +00:00
|
|
|
d = 0xff;
|
2018-06-05 19:22:40 +00:00
|
|
|
}
|
|
|
|
return Result;
|
|
|
|
}
|
|
|
|
return m_pDataFile->m_Sha256;
|
|
|
|
}
|
|
|
|
|
2021-02-08 21:26:26 +00:00
|
|
|
unsigned CDataFileReader::Crc() const
|
2010-05-29 07:25:38 +00:00
|
|
|
{
|
2020-09-26 19:41:58 +00:00
|
|
|
if(!m_pDataFile)
|
|
|
|
return 0xFFFFFFFF;
|
2010-05-29 07:25:38 +00:00
|
|
|
return m_pDataFile->m_Crc;
|
|
|
|
}
|
|
|
|
|
2021-02-08 21:26:26 +00:00
|
|
|
int CDataFileReader::MapSize() const
|
2017-07-08 11:38:27 +00:00
|
|
|
{
|
2020-09-26 19:41:58 +00:00
|
|
|
if(!m_pDataFile)
|
|
|
|
return 0;
|
2023-11-01 11:45:45 +00:00
|
|
|
return m_pDataFile->m_Header.m_Size + m_pDataFile->m_Header.SizeOffset();
|
2017-07-08 11:38:27 +00:00
|
|
|
}
|
|
|
|
|
2011-07-30 23:40:28 +00:00
|
|
|
CDataFileWriter::CDataFileWriter()
|
|
|
|
{
|
|
|
|
m_File = 0;
|
2023-10-14 09:40:18 +00:00
|
|
|
|
2018-04-09 09:56:39 +00:00
|
|
|
m_pItemTypes = static_cast<CItemTypeInfo *>(calloc(MAX_ITEM_TYPES, sizeof(CItemTypeInfo)));
|
2023-10-14 09:40:18 +00:00
|
|
|
m_NumItemTypes = 0;
|
|
|
|
mem_zero(m_pItemTypes, sizeof(CItemTypeInfo) * MAX_ITEM_TYPES);
|
|
|
|
for(int i = 0; i < MAX_ITEM_TYPES; i++)
|
|
|
|
{
|
|
|
|
m_pItemTypes[i].m_First = -1;
|
|
|
|
m_pItemTypes[i].m_Last = -1;
|
|
|
|
}
|
|
|
|
|
2018-04-09 09:56:39 +00:00
|
|
|
m_pItems = static_cast<CItemInfo *>(calloc(MAX_ITEMS, sizeof(CItemInfo)));
|
2023-10-14 09:40:18 +00:00
|
|
|
m_NumItems = 0;
|
|
|
|
|
2018-04-09 09:56:39 +00:00
|
|
|
m_pDatas = static_cast<CDataInfo *>(calloc(MAX_DATAS, sizeof(CDataInfo)));
|
2023-10-14 09:40:18 +00:00
|
|
|
m_NumDatas = 0;
|
|
|
|
|
|
|
|
mem_zero(m_aExtendedItemTypes, sizeof(m_aExtendedItemTypes));
|
|
|
|
m_NumExtendedItemTypes = 0;
|
2011-07-30 23:40:28 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
CDataFileWriter::~CDataFileWriter()
|
|
|
|
{
|
2023-06-26 18:48:16 +00:00
|
|
|
if(m_File)
|
|
|
|
{
|
|
|
|
io_close(m_File);
|
|
|
|
m_File = 0;
|
|
|
|
}
|
|
|
|
|
2018-04-09 09:56:39 +00:00
|
|
|
free(m_pItemTypes);
|
2023-05-15 18:35:17 +00:00
|
|
|
m_pItemTypes = nullptr;
|
2023-06-26 18:48:16 +00:00
|
|
|
|
|
|
|
if(m_pItems)
|
|
|
|
{
|
|
|
|
for(int i = 0; i < m_NumItems; i++)
|
|
|
|
{
|
|
|
|
free(m_pItems[i].m_pData);
|
|
|
|
}
|
|
|
|
free(m_pItems);
|
|
|
|
m_pItems = nullptr;
|
|
|
|
}
|
|
|
|
|
|
|
|
if(m_pDatas)
|
|
|
|
{
|
|
|
|
for(int i = 0; i < m_NumDatas; ++i)
|
|
|
|
{
|
|
|
|
free(m_pDatas[i].m_pUncompressedData);
|
|
|
|
free(m_pDatas[i].m_pCompressedData);
|
|
|
|
}
|
|
|
|
free(m_pDatas);
|
|
|
|
m_pDatas = nullptr;
|
|
|
|
}
|
2011-07-30 23:40:28 +00:00
|
|
|
}
|
|
|
|
|
2023-10-14 09:40:18 +00:00
|
|
|
bool CDataFileWriter::Open(class IStorage *pStorage, const char *pFilename, int StorageType)
|
2010-05-29 07:25:38 +00:00
|
|
|
{
|
2023-10-14 09:40:18 +00:00
|
|
|
dbg_assert(!m_File, "File already open");
|
2019-12-08 22:14:56 +00:00
|
|
|
m_File = pStorage->OpenFile(pFilename, IOFLAG_WRITE, StorageType);
|
2015-08-21 21:02:50 +00:00
|
|
|
return m_File != 0;
|
|
|
|
}
|
2011-04-13 18:37:12 +00:00
|
|
|
|
2023-05-15 16:15:17 +00:00
|
|
|
int CDataFileWriter::GetTypeFromIndex(int Index) const
|
2022-07-02 12:25:16 +00:00
|
|
|
{
|
|
|
|
return ITEMTYPE_EX - Index - 1;
|
|
|
|
}
|
|
|
|
|
2018-10-05 09:20:30 +00:00
|
|
|
int CDataFileWriter::GetExtendedItemTypeIndex(int Type)
|
|
|
|
{
|
|
|
|
for(int i = 0; i < m_NumExtendedItemTypes; i++)
|
|
|
|
{
|
|
|
|
if(m_aExtendedItemTypes[i] == Type)
|
|
|
|
{
|
|
|
|
return i;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// Type not found, add it.
|
|
|
|
dbg_assert(m_NumExtendedItemTypes < MAX_EXTENDED_ITEM_TYPES, "too many extended item types");
|
|
|
|
int Index = m_NumExtendedItemTypes++;
|
|
|
|
m_aExtendedItemTypes[Index] = Type;
|
|
|
|
|
|
|
|
CItemEx ExtendedType = CItemEx::FromUuid(g_UuidManager.GetUuid(Type));
|
|
|
|
AddItem(ITEMTYPE_EX, GetTypeFromIndex(Index), sizeof(ExtendedType), &ExtendedType);
|
|
|
|
return Index;
|
|
|
|
}
|
|
|
|
|
2023-11-01 11:17:02 +00:00
|
|
|
int CDataFileWriter::AddItem(int Type, int ID, size_t Size, const void *pData)
|
2010-05-29 07:25:38 +00:00
|
|
|
{
|
2023-11-01 11:45:02 +00:00
|
|
|
dbg_assert((Type >= 0 && Type < MAX_ITEM_TYPES) || Type >= OFFSET_UUID, "Invalid type");
|
|
|
|
dbg_assert(ID >= 0 && ID <= ITEMTYPE_EX, "Invalid ID");
|
2010-09-27 20:35:57 +00:00
|
|
|
dbg_assert(m_NumItems < 1024, "too many items");
|
2023-11-01 11:17:02 +00:00
|
|
|
dbg_assert(Size == 0 || pData != nullptr, "Data missing"); // Items without data are allowed
|
|
|
|
dbg_assert(Size <= (size_t)std::numeric_limits<int>::max(), "Data too large");
|
|
|
|
dbg_assert(Size % sizeof(int) == 0, "Invalid data boundary");
|
2010-09-27 20:35:57 +00:00
|
|
|
|
2018-10-05 09:20:30 +00:00
|
|
|
if(Type >= OFFSET_UUID)
|
|
|
|
{
|
|
|
|
Type = GetTypeFromIndex(GetExtendedItemTypeIndex(Type));
|
|
|
|
}
|
|
|
|
|
2011-07-30 23:40:28 +00:00
|
|
|
m_pItems[m_NumItems].m_Type = Type;
|
|
|
|
m_pItems[m_NumItems].m_ID = ID;
|
|
|
|
m_pItems[m_NumItems].m_Size = Size;
|
2011-04-13 18:37:12 +00:00
|
|
|
|
2010-05-29 07:25:38 +00:00
|
|
|
// copy data
|
2023-11-01 11:17:02 +00:00
|
|
|
if(Size > 0)
|
|
|
|
{
|
|
|
|
m_pItems[m_NumItems].m_pData = malloc(Size);
|
|
|
|
mem_copy(m_pItems[m_NumItems].m_pData, pData, Size);
|
|
|
|
}
|
|
|
|
else
|
|
|
|
m_pItems[m_NumItems].m_pData = nullptr;
|
2010-05-29 07:25:38 +00:00
|
|
|
|
2011-07-30 23:40:28 +00:00
|
|
|
if(!m_pItemTypes[Type].m_Num) // count item types
|
2010-05-29 07:25:38 +00:00
|
|
|
m_NumItemTypes++;
|
|
|
|
|
|
|
|
// link
|
2011-07-30 23:40:28 +00:00
|
|
|
m_pItems[m_NumItems].m_Prev = m_pItemTypes[Type].m_Last;
|
|
|
|
m_pItems[m_NumItems].m_Next = -1;
|
2011-04-13 18:37:12 +00:00
|
|
|
|
2011-07-30 23:40:28 +00:00
|
|
|
if(m_pItemTypes[Type].m_Last != -1)
|
|
|
|
m_pItems[m_pItemTypes[Type].m_Last].m_Next = m_NumItems;
|
|
|
|
m_pItemTypes[Type].m_Last = m_NumItems;
|
2011-04-13 18:37:12 +00:00
|
|
|
|
2011-07-30 23:40:28 +00:00
|
|
|
if(m_pItemTypes[Type].m_First == -1)
|
|
|
|
m_pItemTypes[Type].m_First = m_NumItems;
|
2011-04-13 18:37:12 +00:00
|
|
|
|
2011-07-30 23:40:28 +00:00
|
|
|
m_pItemTypes[Type].m_Num++;
|
2011-04-13 18:37:12 +00:00
|
|
|
|
2010-05-29 07:25:38 +00:00
|
|
|
m_NumItems++;
|
2020-09-26 19:41:58 +00:00
|
|
|
return m_NumItems - 1;
|
2010-05-29 07:25:38 +00:00
|
|
|
}
|
|
|
|
|
2023-11-01 11:17:02 +00:00
|
|
|
int CDataFileWriter::AddData(size_t Size, const void *pData, int CompressionLevel)
|
2010-05-29 07:25:38 +00:00
|
|
|
{
|
2010-09-27 20:35:57 +00:00
|
|
|
dbg_assert(m_NumDatas < 1024, "too much data");
|
2023-11-01 11:17:02 +00:00
|
|
|
dbg_assert(Size > 0 && pData != nullptr, "Data missing");
|
|
|
|
dbg_assert(Size <= (size_t)std::numeric_limits<int>::max(), "Data too large");
|
2010-09-27 20:35:57 +00:00
|
|
|
|
2011-07-30 23:40:28 +00:00
|
|
|
CDataInfo *pInfo = &m_pDatas[m_NumDatas];
|
2023-06-26 18:48:16 +00:00
|
|
|
pInfo->m_pUncompressedData = malloc(Size);
|
|
|
|
mem_copy(pInfo->m_pUncompressedData, pData, Size);
|
2010-05-29 07:25:38 +00:00
|
|
|
pInfo->m_UncompressedSize = Size;
|
2023-06-26 18:48:16 +00:00
|
|
|
pInfo->m_pCompressedData = nullptr;
|
|
|
|
pInfo->m_CompressedSize = 0;
|
|
|
|
pInfo->m_CompressionLevel = CompressionLevel;
|
2010-05-29 07:25:38 +00:00
|
|
|
|
|
|
|
m_NumDatas++;
|
2020-09-26 19:41:58 +00:00
|
|
|
return m_NumDatas - 1;
|
2010-05-29 07:25:38 +00:00
|
|
|
}
|
|
|
|
|
2023-11-01 11:17:02 +00:00
|
|
|
int CDataFileWriter::AddDataSwapped(size_t Size, const void *pData)
|
2010-05-29 07:25:38 +00:00
|
|
|
{
|
2023-11-01 11:17:02 +00:00
|
|
|
dbg_assert(Size > 0 && pData != nullptr, "Data missing");
|
|
|
|
dbg_assert(Size <= (size_t)std::numeric_limits<int>::max(), "Data too large");
|
|
|
|
dbg_assert(Size % sizeof(int) == 0, "Invalid data boundary");
|
2010-09-27 20:35:57 +00:00
|
|
|
|
2010-05-29 07:25:38 +00:00
|
|
|
#if defined(CONF_ARCH_ENDIAN_BIG)
|
2018-04-09 09:56:39 +00:00
|
|
|
void *pSwapped = malloc(Size); // temporary buffer that we use during compression
|
2010-05-29 07:25:38 +00:00
|
|
|
mem_copy(pSwapped, pData, Size);
|
2020-09-26 19:41:58 +00:00
|
|
|
swap_endian(pSwapped, sizeof(int), Size / sizeof(int));
|
2010-09-27 20:35:57 +00:00
|
|
|
int Index = AddData(Size, pSwapped);
|
2018-04-09 09:56:39 +00:00
|
|
|
free(pSwapped);
|
2010-05-29 07:25:38 +00:00
|
|
|
return Index;
|
|
|
|
#else
|
|
|
|
return AddData(Size, pData);
|
|
|
|
#endif
|
|
|
|
}
|
|
|
|
|
2023-08-13 09:50:35 +00:00
|
|
|
int CDataFileWriter::AddDataString(const char *pStr)
|
|
|
|
{
|
2023-11-01 11:17:02 +00:00
|
|
|
dbg_assert(pStr != nullptr, "Data missing");
|
|
|
|
|
2023-08-13 09:50:35 +00:00
|
|
|
if(pStr[0] == '\0')
|
|
|
|
return -1;
|
|
|
|
return AddData(str_length(pStr) + 1, pStr);
|
|
|
|
}
|
|
|
|
|
2023-06-26 18:48:16 +00:00
|
|
|
void CDataFileWriter::Finish()
|
2010-05-29 07:25:38 +00:00
|
|
|
{
|
2023-06-26 18:48:16 +00:00
|
|
|
dbg_assert((bool)m_File, "file not open");
|
2010-09-27 20:35:57 +00:00
|
|
|
|
2010-05-29 07:25:38 +00:00
|
|
|
// we should now write this file!
|
|
|
|
if(DEBUG)
|
|
|
|
dbg_msg("datafile", "writing");
|
|
|
|
|
2023-06-26 18:48:16 +00:00
|
|
|
// Compress data. This takes the majority of the time when saving a datafile,
|
|
|
|
// so it's delayed until the end so it can be off-loaded to another thread.
|
|
|
|
for(int i = 0; i < m_NumDatas; i++)
|
|
|
|
{
|
|
|
|
unsigned long CompressedSize = compressBound(m_pDatas[i].m_UncompressedSize);
|
|
|
|
m_pDatas[i].m_pCompressedData = malloc(CompressedSize);
|
|
|
|
const int Result = compress2((Bytef *)m_pDatas[i].m_pCompressedData, &CompressedSize, (Bytef *)m_pDatas[i].m_pUncompressedData, m_pDatas[i].m_UncompressedSize, m_pDatas[i].m_CompressionLevel);
|
|
|
|
m_pDatas[i].m_CompressedSize = CompressedSize;
|
|
|
|
free(m_pDatas[i].m_pUncompressedData);
|
|
|
|
m_pDatas[i].m_pUncompressedData = nullptr;
|
|
|
|
if(Result != Z_OK)
|
|
|
|
{
|
|
|
|
dbg_msg("datafile", "compression error %d", Result);
|
|
|
|
dbg_assert(false, "zlib error");
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2010-05-29 07:25:38 +00:00
|
|
|
// calculate sizes
|
2023-11-01 11:17:02 +00:00
|
|
|
size_t ItemSize = 0;
|
2010-05-29 07:25:38 +00:00
|
|
|
for(int i = 0; i < m_NumItems; i++)
|
|
|
|
{
|
|
|
|
if(DEBUG)
|
2017-07-26 01:58:00 +00:00
|
|
|
dbg_msg("datafile", "item=%d size=%d (%d)", i, m_pItems[i].m_Size, m_pItems[i].m_Size + (int)sizeof(CDatafileItem));
|
2023-11-01 11:17:02 +00:00
|
|
|
ItemSize += m_pItems[i].m_Size;
|
|
|
|
ItemSize += sizeof(CDatafileItem);
|
2010-05-29 07:25:38 +00:00
|
|
|
}
|
2011-04-13 18:37:12 +00:00
|
|
|
|
2023-11-01 11:17:02 +00:00
|
|
|
size_t DataSize = 0;
|
2010-05-29 07:25:38 +00:00
|
|
|
for(int i = 0; i < m_NumDatas; i++)
|
2011-07-30 23:40:28 +00:00
|
|
|
DataSize += m_pDatas[i].m_CompressedSize;
|
2011-04-13 18:37:12 +00:00
|
|
|
|
2010-05-29 07:25:38 +00:00
|
|
|
// calculate the complete size
|
2023-11-01 11:17:02 +00:00
|
|
|
const size_t TypesSize = m_NumItemTypes * sizeof(CDatafileItemType);
|
|
|
|
const size_t HeaderSize = sizeof(CDatafileHeader);
|
|
|
|
const size_t OffsetSize = ((size_t)m_NumItems + m_NumDatas + m_NumDatas) * sizeof(int); // ItemOffsets, DataOffsets, DataUncompressedSizes
|
|
|
|
const size_t SwapSize = HeaderSize + TypesSize + OffsetSize + ItemSize;
|
|
|
|
const size_t FileSize = SwapSize + DataSize;
|
2011-04-13 18:37:12 +00:00
|
|
|
|
2010-05-29 07:25:38 +00:00
|
|
|
if(DEBUG)
|
2023-11-01 11:17:02 +00:00
|
|
|
dbg_msg("datafile", "m_NumItemTypes=%d TypesSize=%" PRIzu " ItemSize=%" PRIzu " DataSize=%" PRIzu, m_NumItemTypes, TypesSize, ItemSize, DataSize);
|
|
|
|
|
|
|
|
// This also ensures that SwapSize, ItemSize and DataSize are valid.
|
|
|
|
dbg_assert(FileSize <= (size_t)std::numeric_limits<int>::max(), "File size too large");
|
2011-04-13 18:37:12 +00:00
|
|
|
|
2010-05-29 07:25:38 +00:00
|
|
|
// construct Header
|
|
|
|
{
|
2023-05-15 18:43:28 +00:00
|
|
|
CDatafileHeader Header;
|
2011-02-12 10:40:36 +00:00
|
|
|
Header.m_aID[0] = 'D';
|
|
|
|
Header.m_aID[1] = 'A';
|
|
|
|
Header.m_aID[2] = 'T';
|
|
|
|
Header.m_aID[3] = 'A';
|
2010-05-29 07:25:38 +00:00
|
|
|
Header.m_Version = 4;
|
2023-11-01 11:45:45 +00:00
|
|
|
Header.m_Size = FileSize - Header.SizeOffset();
|
|
|
|
Header.m_Swaplen = SwapSize - Header.SizeOffset();
|
2010-05-29 07:25:38 +00:00
|
|
|
Header.m_NumItemTypes = m_NumItemTypes;
|
|
|
|
Header.m_NumItems = m_NumItems;
|
|
|
|
Header.m_NumRawData = m_NumDatas;
|
|
|
|
Header.m_ItemSize = ItemSize;
|
|
|
|
Header.m_DataSize = DataSize;
|
2011-04-13 18:37:12 +00:00
|
|
|
|
2010-05-29 07:25:38 +00:00
|
|
|
// write Header
|
|
|
|
if(DEBUG)
|
2017-07-26 01:58:00 +00:00
|
|
|
dbg_msg("datafile", "HeaderSize=%d", (int)sizeof(Header));
|
2010-09-27 20:35:57 +00:00
|
|
|
#if defined(CONF_ARCH_ENDIAN_BIG)
|
2020-09-26 19:41:58 +00:00
|
|
|
swap_endian(&Header, sizeof(int), sizeof(Header) / sizeof(int));
|
2010-09-27 20:35:57 +00:00
|
|
|
#endif
|
2010-05-29 07:25:38 +00:00
|
|
|
io_write(m_File, &Header, sizeof(Header));
|
|
|
|
}
|
2011-04-13 18:37:12 +00:00
|
|
|
|
2010-05-29 07:25:38 +00:00
|
|
|
// write types
|
2018-10-05 09:20:30 +00:00
|
|
|
for(int i = 0, Count = 0; i < MAX_ITEM_TYPES; i++)
|
2010-05-29 07:25:38 +00:00
|
|
|
{
|
2011-07-30 23:40:28 +00:00
|
|
|
if(m_pItemTypes[i].m_Num)
|
2010-05-29 07:25:38 +00:00
|
|
|
{
|
|
|
|
// write info
|
|
|
|
CDatafileItemType Info;
|
|
|
|
Info.m_Type = i;
|
|
|
|
Info.m_Start = Count;
|
2011-07-30 23:40:28 +00:00
|
|
|
Info.m_Num = m_pItemTypes[i].m_Num;
|
2010-05-29 07:25:38 +00:00
|
|
|
if(DEBUG)
|
|
|
|
dbg_msg("datafile", "writing type=%x start=%d num=%d", Info.m_Type, Info.m_Start, Info.m_Num);
|
2010-09-27 20:35:57 +00:00
|
|
|
#if defined(CONF_ARCH_ENDIAN_BIG)
|
2020-09-26 19:41:58 +00:00
|
|
|
swap_endian(&Info, sizeof(int), sizeof(CDatafileItemType) / sizeof(int));
|
2010-09-27 20:35:57 +00:00
|
|
|
#endif
|
2010-05-29 07:25:38 +00:00
|
|
|
io_write(m_File, &Info, sizeof(Info));
|
2011-07-30 23:40:28 +00:00
|
|
|
Count += m_pItemTypes[i].m_Num;
|
2010-05-29 07:25:38 +00:00
|
|
|
}
|
|
|
|
}
|
2011-04-13 18:37:12 +00:00
|
|
|
|
2010-05-29 07:25:38 +00:00
|
|
|
// write item offsets
|
2018-10-05 09:20:30 +00:00
|
|
|
for(int i = 0, Offset = 0; i < MAX_ITEM_TYPES; i++)
|
2010-05-29 07:25:38 +00:00
|
|
|
{
|
2011-07-30 23:40:28 +00:00
|
|
|
if(m_pItemTypes[i].m_Num)
|
2010-05-29 07:25:38 +00:00
|
|
|
{
|
2011-07-30 23:40:28 +00:00
|
|
|
// write all m_pItems in of this type
|
|
|
|
int k = m_pItemTypes[i].m_First;
|
2010-05-29 07:25:38 +00:00
|
|
|
while(k != -1)
|
|
|
|
{
|
|
|
|
if(DEBUG)
|
|
|
|
dbg_msg("datafile", "writing item offset num=%d offset=%d", k, Offset);
|
2010-09-27 20:35:57 +00:00
|
|
|
int Temp = Offset;
|
|
|
|
#if defined(CONF_ARCH_ENDIAN_BIG)
|
2020-09-26 19:41:58 +00:00
|
|
|
swap_endian(&Temp, sizeof(int), sizeof(Temp) / sizeof(int));
|
2010-09-27 20:35:57 +00:00
|
|
|
#endif
|
|
|
|
io_write(m_File, &Temp, sizeof(Temp));
|
2011-07-30 23:40:28 +00:00
|
|
|
Offset += m_pItems[k].m_Size + sizeof(CDatafileItem);
|
2011-04-13 18:37:12 +00:00
|
|
|
|
2010-05-29 07:25:38 +00:00
|
|
|
// next
|
2011-07-30 23:40:28 +00:00
|
|
|
k = m_pItems[k].m_Next;
|
2010-05-29 07:25:38 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
2011-04-13 18:37:12 +00:00
|
|
|
|
2010-05-29 07:25:38 +00:00
|
|
|
// write data offsets
|
|
|
|
for(int i = 0, Offset = 0; i < m_NumDatas; i++)
|
|
|
|
{
|
|
|
|
if(DEBUG)
|
|
|
|
dbg_msg("datafile", "writing data offset num=%d offset=%d", i, Offset);
|
2010-09-27 20:35:57 +00:00
|
|
|
int Temp = Offset;
|
|
|
|
#if defined(CONF_ARCH_ENDIAN_BIG)
|
2020-09-26 19:41:58 +00:00
|
|
|
swap_endian(&Temp, sizeof(int), sizeof(Temp) / sizeof(int));
|
2010-09-27 20:35:57 +00:00
|
|
|
#endif
|
|
|
|
io_write(m_File, &Temp, sizeof(Temp));
|
2011-07-30 23:40:28 +00:00
|
|
|
Offset += m_pDatas[i].m_CompressedSize;
|
2010-05-29 07:25:38 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
// write data uncompressed sizes
|
|
|
|
for(int i = 0; i < m_NumDatas; i++)
|
|
|
|
{
|
|
|
|
if(DEBUG)
|
2011-07-30 23:40:28 +00:00
|
|
|
dbg_msg("datafile", "writing data uncompressed size num=%d size=%d", i, m_pDatas[i].m_UncompressedSize);
|
|
|
|
int UncompressedSize = m_pDatas[i].m_UncompressedSize;
|
2010-09-27 20:35:57 +00:00
|
|
|
#if defined(CONF_ARCH_ENDIAN_BIG)
|
2020-09-26 19:41:58 +00:00
|
|
|
swap_endian(&UncompressedSize, sizeof(int), sizeof(UncompressedSize) / sizeof(int));
|
2010-09-27 20:35:57 +00:00
|
|
|
#endif
|
|
|
|
io_write(m_File, &UncompressedSize, sizeof(UncompressedSize));
|
2010-05-29 07:25:38 +00:00
|
|
|
}
|
2011-04-13 18:37:12 +00:00
|
|
|
|
2011-07-30 23:40:28 +00:00
|
|
|
// write m_pItems
|
2018-10-05 09:20:30 +00:00
|
|
|
for(int i = 0; i < MAX_ITEM_TYPES; i++)
|
2010-05-29 07:25:38 +00:00
|
|
|
{
|
2011-07-30 23:40:28 +00:00
|
|
|
if(m_pItemTypes[i].m_Num)
|
2010-05-29 07:25:38 +00:00
|
|
|
{
|
2011-07-30 23:40:28 +00:00
|
|
|
// write all m_pItems in of this type
|
|
|
|
int k = m_pItemTypes[i].m_First;
|
2010-05-29 07:25:38 +00:00
|
|
|
while(k != -1)
|
|
|
|
{
|
|
|
|
CDatafileItem Item;
|
2020-09-26 19:41:58 +00:00
|
|
|
Item.m_TypeAndID = (i << 16) | m_pItems[k].m_ID;
|
2011-07-30 23:40:28 +00:00
|
|
|
Item.m_Size = m_pItems[k].m_Size;
|
2010-05-29 07:25:38 +00:00
|
|
|
if(DEBUG)
|
2011-07-30 23:40:28 +00:00
|
|
|
dbg_msg("datafile", "writing item type=%x idx=%d id=%d size=%d", i, k, m_pItems[k].m_ID, m_pItems[k].m_Size);
|
2011-04-13 18:37:12 +00:00
|
|
|
|
2010-09-27 20:35:57 +00:00
|
|
|
#if defined(CONF_ARCH_ENDIAN_BIG)
|
2020-09-26 19:41:58 +00:00
|
|
|
swap_endian(&Item, sizeof(int), sizeof(Item) / sizeof(int));
|
2023-11-01 11:17:02 +00:00
|
|
|
if(m_pItems[k].m_pData != nullptr)
|
|
|
|
swap_endian(m_pItems[k].m_pData, sizeof(int), m_pItems[k].m_Size / sizeof(int));
|
2010-09-27 20:35:57 +00:00
|
|
|
#endif
|
2010-05-29 07:25:38 +00:00
|
|
|
io_write(m_File, &Item, sizeof(Item));
|
2023-11-01 11:17:02 +00:00
|
|
|
if(m_pItems[k].m_pData != nullptr)
|
|
|
|
io_write(m_File, m_pItems[k].m_pData, m_pItems[k].m_Size);
|
2011-04-13 18:37:12 +00:00
|
|
|
|
2010-05-29 07:25:38 +00:00
|
|
|
// next
|
2011-07-30 23:40:28 +00:00
|
|
|
k = m_pItems[k].m_Next;
|
2010-05-29 07:25:38 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
2011-04-13 18:37:12 +00:00
|
|
|
|
2010-05-29 07:25:38 +00:00
|
|
|
// write data
|
|
|
|
for(int i = 0; i < m_NumDatas; i++)
|
|
|
|
{
|
|
|
|
if(DEBUG)
|
2011-07-30 23:40:28 +00:00
|
|
|
dbg_msg("datafile", "writing data id=%d size=%d", i, m_pDatas[i].m_CompressedSize);
|
|
|
|
io_write(m_File, m_pDatas[i].m_pCompressedData, m_pDatas[i].m_CompressedSize);
|
2010-05-29 07:25:38 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
// free data
|
|
|
|
for(int i = 0; i < m_NumItems; i++)
|
2020-10-18 10:57:55 +00:00
|
|
|
{
|
2018-04-09 09:56:39 +00:00
|
|
|
free(m_pItems[i].m_pData);
|
2023-05-15 18:35:17 +00:00
|
|
|
m_pItems[i].m_pData = nullptr;
|
2020-10-18 10:57:55 +00:00
|
|
|
}
|
2010-09-27 20:35:57 +00:00
|
|
|
for(int i = 0; i < m_NumDatas; ++i)
|
2020-10-18 10:57:55 +00:00
|
|
|
{
|
2018-04-09 09:56:39 +00:00
|
|
|
free(m_pDatas[i].m_pCompressedData);
|
2023-05-15 18:35:17 +00:00
|
|
|
m_pDatas[i].m_pCompressedData = nullptr;
|
2020-10-18 10:57:55 +00:00
|
|
|
}
|
2011-04-13 18:37:12 +00:00
|
|
|
|
2010-05-29 07:25:38 +00:00
|
|
|
io_close(m_File);
|
2010-09-27 20:35:57 +00:00
|
|
|
m_File = 0;
|
2011-04-13 18:37:12 +00:00
|
|
|
|
2010-05-29 07:25:38 +00:00
|
|
|
if(DEBUG)
|
|
|
|
dbg_msg("datafile", "done");
|
|
|
|
}
|