ddnet/src/engine/e_snapshot.c

538 lines
12 KiB
C
Raw Normal View History

2007-11-25 19:42:40 +00:00
/* copyright (c) 2007 magnus auvinen, see licence.txt for more info */
2007-12-16 19:10:18 +00:00
#include <stdlib.h>
2007-12-15 10:24:49 +00:00
#include "e_snapshot.h"
#include "e_compression.h"
#include "e_common_interface.h"
int *snapitem_data(SNAPSHOT_ITEM *item) { return (int *)(item+1); }
int snapitem_type(SNAPSHOT_ITEM *item) { return item->type_and_id>>16; }
int snapitem_id(SNAPSHOT_ITEM *item) { return item->type_and_id&0xffff; }
int snapitem_key(SNAPSHOT_ITEM *item) { return item->type_and_id; }
int *snapshot_offsets(SNAPSHOT *snap) { return (int *)(snap+1); }
char *snapshot_datastart(SNAPSHOT *snap) { return (char*)(snapshot_offsets(snap)+snap->num_items); }
SNAPSHOT_ITEM *snapshot_get_item(SNAPSHOT *snap, int index)
2007-10-06 17:01:06 +00:00
{ return (SNAPSHOT_ITEM *)(snapshot_datastart(snap) + snapshot_offsets(snap)[index]); }
int snapshot_get_item_datasize(SNAPSHOT *snap, int index)
{
if(index == snap->num_items-1)
return (snap->data_size - snapshot_offsets(snap)[index]) - sizeof(SNAPSHOT_ITEM);
return (snapshot_offsets(snap)[index+1] - snapshot_offsets(snap)[index]) - sizeof(SNAPSHOT_ITEM);
}
int snapshot_get_item_index(SNAPSHOT *snap, int key)
{
/* TODO: OPT: this should not be a linear search. very bad */
int i;
for(i = 0; i < snap->num_items; i++)
{
if(snapitem_key(snapshot_get_item(snap, i)) == key)
return i;
}
return -1;
}
2007-12-16 19:10:18 +00:00
typedef struct
{
int num;
int keys[64];
int index[64];
} ITEMLIST;
static ITEMLIST sorted[256];
2007-12-16 19:10:18 +00:00
static int snapshot_generate_hash(SNAPSHOT *snap)
{
int i, key, hashid;
2007-12-16 19:10:18 +00:00
for(i = 0; i < 256; i++)
sorted[i].num = 0;
for(i = 0; i < snap->num_items; i++)
{
key = snapitem_key(snapshot_get_item(snap, i));
hashid = ((key>>8)&0xf0) | (key&0xf);
if(sorted[hashid].num != 64)
{
sorted[hashid].index[sorted[hashid].num] = i;
sorted[hashid].keys[sorted[hashid].num] = key;
sorted[hashid].num++;
}
}
return 0;
}
int snapshot_get_item_index_hashed(SNAPSHOT *snap, int key)
{
int hashid = ((key>>8)&0xf0) | (key&0xf);
int i;
for(i = 0; i < sorted[hashid].num; i++)
{
if(sorted[hashid].keys[i] == key)
return sorted[hashid].index[i];
}
return -1;
}
typedef struct
{
int num_deleted_items;
int num_update_items;
int num_temp_items; /* needed? */
int data[1];
/*
char *data_start() { return (char *)&offsets[num_deleted_items+num_update_items+num_temp_items]; }
int deleted_item(int index) { return offsets[index]; }
item *update_item(int index) { return (item *)(data_start() + offsets[num_deleted_items+index]); }
item *temp_item(int index) { return (item *)(data_start() + offsets[num_deleted_items+num_update_items+index]); }
*/
} SNAPSHOT_DELTA;
static const int MAX_ITEMS = 512;
static SNAPSHOT_DELTA empty = {0,0,0,{0}};
void *snapshot_empty_delta()
{
return &empty;
}
int snapshot_crc(SNAPSHOT *snap)
{
int crc = 0;
int i, b;
SNAPSHOT_ITEM *item;
int size;
for(i = 0; i < snap->num_items; i++)
{
item = snapshot_get_item(snap, i);
size = snapshot_get_item_datasize(snap, i);
for(b = 0; b < size/4; b++)
crc += snapitem_data(item)[b];
}
return crc;
}
void snapshot_debug_dump(SNAPSHOT *snap)
{
int size, i, b;
SNAPSHOT_ITEM *item;
dbg_msg("snapshot", "data_size=%d num_items=%d", snap->data_size, snap->num_items);
for(i = 0; i < snap->num_items; i++)
{
item = snapshot_get_item(snap, i);
size = snapshot_get_item_datasize(snap, i);
dbg_msg("snapshot", "\ttype=%d id=%d", snapitem_type(item), snapitem_id(item));
for(b = 0; b < size/4; b++)
dbg_msg("snapshot", "\t\t%3d %12d\t%08x", b, snapitem_data(item)[b], snapitem_data(item)[b]);
}
}
static int diff_item(int *past, int *current, int *out, int size)
{
int needed = 0;
while(size)
{
*out = *current-*past;
needed |= *out;
out++;
past++;
current++;
size--;
}
return needed;
}
int snapshot_data_rate[0xffff] = {0};
int snapshot_data_updates[0xffff] = {0};
static int snapshot_current = 0;
static void undiff_item(int *past, int *diff, int *out, int size)
{
while(size)
{
*out = *past+*diff;
if(*diff == 0)
snapshot_data_rate[snapshot_current] += 1;
else
{
unsigned char buf[16];
unsigned char *end = vint_pack(buf, *diff);
snapshot_data_rate[snapshot_current] += (int)(end - (unsigned char*)buf) * 8;
}
out++;
past++;
diff++;
size--;
}
}
2007-10-03 21:32:02 +00:00
/* TODO: OPT: this should be made much faster */
int snapshot_create_delta(SNAPSHOT *from, SNAPSHOT *to, void *dstdata)
{
2007-12-16 19:10:18 +00:00
static PERFORMACE_INFO hash_scope = {"hash", 0};
SNAPSHOT_DELTA *delta = (SNAPSHOT_DELTA *)dstdata;
int *data = (int *)delta->data;
int i, itemsize, pastindex;
SNAPSHOT_ITEM *fromitem;
SNAPSHOT_ITEM *curitem;
SNAPSHOT_ITEM *pastitem;
int count = 0;
int size_count = 0;
delta->num_deleted_items = 0;
delta->num_update_items = 0;
delta->num_temp_items = 0;
2007-12-16 19:10:18 +00:00
perf_start(&hash_scope);
snapshot_generate_hash(to);
perf_end();
/* pack deleted stuff */
{
2007-12-16 16:14:05 +00:00
static PERFORMACE_INFO scope = {"delete", 0};
perf_start(&scope);
for(i = 0; i < from->num_items; i++)
{
2007-12-16 16:14:05 +00:00
fromitem = snapshot_get_item(from, i);
2007-12-16 19:10:18 +00:00
if(snapshot_get_item_index_hashed(to, (snapitem_key(fromitem))) == -1)
2007-12-16 16:14:05 +00:00
{
/* deleted */
delta->num_deleted_items++;
*data = snapitem_key(fromitem);
data++;
}
}
2007-12-16 16:14:05 +00:00
perf_end();
}
2007-12-16 19:10:18 +00:00
perf_start(&hash_scope);
snapshot_generate_hash(from);
perf_end();
/* pack updated stuff */
{
2007-12-16 16:14:05 +00:00
static PERFORMACE_INFO scope = {"update", 0};
2007-12-16 19:10:18 +00:00
int pastindecies[1024];
2007-12-16 16:14:05 +00:00
perf_start(&scope);
2007-12-16 19:10:18 +00:00
/* fetch previous indices */
/* we do this as a separate pass because it helps the cache */
{
static PERFORMACE_INFO scope = {"find", 0};
perf_start(&scope);
for(i = 0; i < to->num_items; i++)
{
curitem = snapshot_get_item(to, i); /* O(1) .. O(n) */
pastindecies[i] = snapshot_get_item_index_hashed(from, snapitem_key(curitem)); /* O(n) .. O(n^n)*/
}
perf_end();
}
2007-12-16 16:14:05 +00:00
for(i = 0; i < to->num_items; i++)
{
2007-12-16 16:14:05 +00:00
/* do delta */
2007-12-16 19:10:18 +00:00
itemsize = snapshot_get_item_datasize(to, i); /* O(1) .. O(n) */
curitem = snapshot_get_item(to, i); /* O(1) .. O(n) */
pastindex = pastindecies[i];
2007-12-16 16:14:05 +00:00
if(pastindex != -1)
{
2007-12-16 16:14:05 +00:00
static PERFORMACE_INFO scope = {"diff", 0};
perf_start(&scope);
pastitem = snapshot_get_item(from, pastindex);
if(diff_item((int*)snapitem_data(pastitem), (int*)snapitem_data(curitem), data+3, itemsize/4))
{
*data++ = itemsize;
*data++ = snapitem_type(curitem);
*data++ = snapitem_id(curitem);
data += itemsize/4;
delta->num_update_items++;
}
perf_end();
}
else
{
static PERFORMACE_INFO scope = {"copy", 0};
perf_start(&scope);
*data++ = itemsize;
*data++ = snapitem_type(curitem);
*data++ = snapitem_id(curitem);
2007-12-16 16:14:05 +00:00
mem_copy(data, snapitem_data(curitem), itemsize);
size_count += itemsize;
data += itemsize/4;
delta->num_update_items++;
2007-12-16 16:14:05 +00:00
count++;
perf_end();
}
}
2007-12-16 16:14:05 +00:00
perf_end();
}
if(0)
{
dbg_msg("snapshot", "%d %d %d",
delta->num_deleted_items,
delta->num_update_items,
delta->num_temp_items);
}
/*
// TODO: pack temp stuff
// finish
//mem_copy(delta->offsets, deleted, delta->num_deleted_items*sizeof(int));
//mem_copy(&(delta->offsets[delta->num_deleted_items]), update, delta->num_update_items*sizeof(int));
//mem_copy(&(delta->offsets[delta->num_deleted_items+delta->num_update_items]), temp, delta->num_temp_items*sizeof(int));
//mem_copy(delta->data_start(), data, data_size);
//delta->data_size = data_size;
* */
if(!delta->num_deleted_items && !delta->num_update_items && !delta->num_temp_items)
return 0;
return (int)((char*)data-(char*)dstdata);
}
int snapshot_unpack_delta(SNAPSHOT *from, SNAPSHOT *to, void *srcdata, int data_size)
{
SNAPBUILD builder;
SNAPSHOT_DELTA *delta = (SNAPSHOT_DELTA *)srcdata;
int *data = (int *)delta->data;
SNAPSHOT_ITEM *fromitem;
int i, d, keep, itemsize;
int *deleted;
int id, type, key;
int fromindex;
int *newdata;
snapbuild_init(&builder);
/* unpack deleted stuff */
deleted = data;
data += delta->num_deleted_items;
/* copy all non deleted stuff */
for(i = 0; i < from->num_items; i++)
{
/* dbg_assert(0, "fail!"); */
fromitem = snapshot_get_item(from, i);
itemsize = snapshot_get_item_datasize(from, i);
keep = 1;
for(d = 0; d < delta->num_deleted_items; d++)
{
if(deleted[d] == snapitem_key(fromitem))
{
keep = 0;
break;
}
}
if(keep)
{
/* keep it */
mem_copy(
snapbuild_new_item(&builder, snapitem_type(fromitem), snapitem_id(fromitem), itemsize),
snapitem_data(fromitem), itemsize);
}
}
/* unpack updated stuff */
for(i = 0; i < delta->num_update_items; i++)
{
itemsize = *data++;
type = *data++;
id = *data++;
snapshot_current = type;
key = (type<<16)|id;
/* create the item if needed */
newdata = snapbuild_get_item_data(&builder, key);
if(!newdata)
newdata = (int *)snapbuild_new_item(&builder, key>>16, key&0xffff, itemsize);
fromindex = snapshot_get_item_index(from, key);
if(fromindex != -1)
{
/* we got an update so we need to apply the diff */
undiff_item((int *)snapitem_data(snapshot_get_item(from, fromindex)), data, newdata, itemsize/4);
snapshot_data_updates[snapshot_current]++;
}
else /* no previous, just copy the data */
{
mem_copy(newdata, data, itemsize);
snapshot_data_rate[snapshot_current] += itemsize*8;
snapshot_data_updates[snapshot_current]++;
}
data += itemsize/4;
}
/* finish up */
return snapbuild_finish(&builder, to);
}
/* SNAPSTORAGE */
void snapstorage_init(SNAPSTORAGE *ss)
{
ss->first = 0;
}
void snapstorage_purge_all(SNAPSTORAGE *ss)
{
SNAPSTORAGE_HOLDER *h = ss->first;
SNAPSTORAGE_HOLDER *next;
while(h)
{
next = h->next;
mem_free(h);
h = next;
}
/* no more snapshots in storage */
ss->first = 0;
ss->last = 0;
}
void snapstorage_purge_until(SNAPSTORAGE *ss, int tick)
{
SNAPSTORAGE_HOLDER *next;
SNAPSTORAGE_HOLDER *h = ss->first;
while(h)
{
next = h->next;
if(h->tick >= tick)
return; /* no more to remove */
mem_free(h);
2007-12-25 11:56:11 +00:00
/* did we come to the end of the list? */
if (!next)
break;
ss->first = next;
next->prev = 0x0;
h = next;
}
/* no more snapshots in storage */
ss->first = 0;
ss->last = 0;
}
void snapstorage_add(SNAPSTORAGE *ss, int tick, int64 tagtime, int data_size, void *data)
{
/* allocate memory for holder + snapshot_data */
SNAPSTORAGE_HOLDER *h = (SNAPSTORAGE_HOLDER *)mem_alloc(sizeof(SNAPSTORAGE_HOLDER)+data_size, 1);
/* set data */
h->tick = tick;
h->tagtime = tagtime;
h->snap_size = data_size;
h->snap = (SNAPSHOT*)(h+1);
mem_copy(h->snap, data, data_size);
/* link */
h->next = 0;
h->prev = ss->last;
if(ss->last)
ss->last->next = h;
else
ss->first = h;
ss->last = h;
}
int snapstorage_get(SNAPSTORAGE *ss, int tick, int64 *tagtime, SNAPSHOT **data)
{
SNAPSTORAGE_HOLDER *h = ss->first;
while(h)
{
if(h->tick == tick)
{
if(tagtime)
*tagtime = h->tagtime;
if(data)
*data = h->snap;
return h->snap_size;
}
h = h->next;
}
return -1;
}
/* SNAPBUILD */
void snapbuild_init(SNAPBUILD *sb)
{
sb->data_size = 0;
sb->num_items = 0;
}
SNAPSHOT_ITEM *snapbuild_get_item(SNAPBUILD *sb, int index)
{
return (SNAPSHOT_ITEM *)&(sb->data[sb->offsets[index]]);
}
int *snapbuild_get_item_data(SNAPBUILD *sb, int key)
{
int i;
for(i = 0; i < sb->num_items; i++)
{
if(snapitem_key(snapbuild_get_item(sb, i)) == key)
return (int *)snapitem_data(snapbuild_get_item(sb, i));
}
return 0;
}
int snapbuild_finish(SNAPBUILD *sb, void *snapdata)
{
/* flattern and make the snapshot */
SNAPSHOT *snap = (SNAPSHOT *)snapdata;
2007-10-06 17:01:06 +00:00
int offset_size = sizeof(int)*sb->num_items;
snap->data_size = sb->data_size;
snap->num_items = sb->num_items;
mem_copy(snapshot_offsets(snap), sb->offsets, offset_size);
mem_copy(snapshot_datastart(snap), sb->data, sb->data_size);
return sizeof(SNAPSHOT) + offset_size + sb->data_size;
}
void *snapbuild_new_item(SNAPBUILD *sb, int type, int id, int size)
{
SNAPSHOT_ITEM *obj = (SNAPSHOT_ITEM *)(sb->data+sb->data_size);
mem_zero(obj, sizeof(SNAPSHOT_ITEM) + size);
obj->type_and_id = (type<<16)|id;
sb->offsets[sb->num_items] = sb->data_size;
sb->data_size += sizeof(SNAPSHOT_ITEM) + size;
sb->num_items++;
dbg_assert(sb->data_size < MAX_SNAPSHOT_SIZE, "too much data");
dbg_assert(sb->num_items < SNAPBUILD_MAX_ITEMS, "too many items");
return snapitem_data(obj);
}