Skip to content

Commit

Permalink
Use empty spaces and block sizes to encode freelist
Browse files Browse the repository at this point in the history
  • Loading branch information
@kriszyp
kriszyp committed Feb 18, 2024
1 parent 72a915e commit 41a1fba
Show file tree
Hide file tree
Showing 3 changed files with 162 additions and 27 deletions.
38 changes: 23 additions & 15 deletions dependencies/lmdb/libraries/liblmdb/mdb.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2748,8 +2748,7 @@ mdb_page_alloc(MDB_cursor *mc, int num, MDB_page **mp)
}
if (entry > 0) {
pgno = entry;
if (pgno + 1 == last_pgno) block_size++;
else block_size = 1;
block_size = 1;
} else {
block_size = -entry;
pgno = mop[++i];
Expand All @@ -2759,8 +2758,8 @@ mdb_page_alloc(MDB_cursor *mc, int num, MDB_page **mp)
if (block_size >= num) {
if (block_size == num) {
// we found a block of the right size
//mop[i] = 0;
//if (block_size > 1) mop[i - 1] = 0;
mop[i] = 0;
if (entry < 1) mop[i - 1] = 0;
goto search_done;
} else if (block_size < best_fit_size || best_fit_size == 0) {
best_fit_start = i - 1;
Expand Down Expand Up @@ -2856,7 +2855,19 @@ mdb_page_alloc(MDB_cursor *mc, int num, MDB_page **mp)
if (mop != env->me_pghead) env->me_pghead = mop;
mop_len = mop[0];
}
continue_best_fit:
if (best_fit_start > 0) {
mop[best_fit_start] += num; // block length is a negative, so we add to it in order to subtract the amount we are using
if (mop[best_fit_start] == -1) mop[best_fit_start] = 0;
pgno = mop[best_fit_start + 1];
mop[best_fit_start + 1] += num;
//env->me_freelist_position = best_fit_start;
fprintf(stderr, "using best fit at %u size %u of %u\n", pgno, num, best_fit_size);
//env->me_block_size_cache[best_fit_size] = 0; // clear this out of the cache (TODO: could move it)

i = 1; // indicate that we found something
goto search_done;
}
/* Use new pages from the map when nothing suitable in the freeDB */
i = 0;
pgno = txn->mt_next_pgno;
Expand Down Expand Up @@ -2890,10 +2901,11 @@ mdb_page_alloc(MDB_cursor *mc, int num, MDB_page **mp)
}
}
if (i) {
mop[0] = mop_len -= num;
//fprintf(stderr, "using %u to %u\n", pgno, pgno + num -1);
/*mop[0] = mop_len -= num;
/* Move any stragglers down */
for (j = i-num; j < mop_len; )
mop[++j] = mop[++i];
/*for (j = i-num; j < mop_len; )
mop[++j] = mop[++i];*/
} else {
txn->mt_next_pgno = pgno + num;
}
Expand Down Expand Up @@ -4187,8 +4199,10 @@ mdb_page_flush(MDB_txn *txn, int keep)
pgno_t p = dl[n].mid + dl_nump[n];
if (p > pgno)
pgno = p;
else
else {
fprintf(stderr, "Page writes are out of order\n");
assert(0);
}
}
txn->mt_flags |= MDB_TXN_DIRTYNUM;
/* <lmdb-js addition> */
Expand Down Expand Up @@ -7853,13 +7867,7 @@ mdb_ovpage_free(MDB_cursor *mc, MDB_page *mp)
freeme = mp;
release:
/* Insert in me_pghead */
mop = env->me_pghead;
j = mop[0] + ovpages;
for (i = mop[0]; i && mop[i] < pg; i--)
mop[j--] = mop[i];
while (j>i)
mop[j--] = pg++;
mop[0] += ovpages;
mdb_midl_insert(&env->me_pghead, pg, ovpages);
} else {
rc = mdb_midl_append_range(&txn->mt_free_pgs, pg, ovpages);
if (rc)
Expand Down
149 changes: 138 additions & 11 deletions dependencies/lmdb/libraries/liblmdb/midl.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -78,12 +78,11 @@ unsigned mdb_midl_search( MDB_IDL ids, MDB_ID id )
return cursor;
}

int mdb_midl_insert( MDB_IDL* idp, MDB_ID id, int insertion_count)
int mdb_midl_insert( MDB_IDL* ids_ref, MDB_ID id, int insertion_count )
{
mdb_midl_need(idp, 1);
MDB_IDL ids = *idp;
MDB_IDL ids = *ids_ref;
unsigned x, i;

int rc;
x = mdb_midl_search( ids, id );
//assert( x > 0 );

Expand All @@ -98,21 +97,149 @@ int mdb_midl_insert( MDB_IDL* idp, MDB_ID id, int insertion_count)
return -1;
}

if ( ++ids[0] >= MDB_IDL_DB_MAX ) {
if ( ids[0] >= MDB_IDL_DB_MAX ) {
/* no room */
--ids[0];
return -2;

} else {
/* insert id */
for (i=ids[0]; i>x; i--)
ids[i] = ids[i-1];
ids[x] = id;
if (x > ids[0]) {
// need to grow
if ((rc = mdb_midl_need(ids_ref, 2)) != 0)
return rc;
ids = *ids_ref;
if (insertion_count == 1) {
ids[x] = 0;
ids[0] = x;
} else {
ids[x] = 0;
ids[x + 1] = 0;
ids[0] = x + 1;
}
}
unsigned before = x; // this will end up pointing to an entry or zero right before a block of empty space
while ((ssize_t)ids[--before] <= 0 && before > 0) {
// move past empty entries (and the length entry)
}
while ((ssize_t)ids[x] <= 0 && x < ids[0]) { x++;}
ssize_t next_id = ids[x];
ssize_t next_count = ids[x - 1];
if (next_count < 0) next_count = -next_count;
else next_count = 1;
if (id - next_count <= next_id && next_id > 0) {
if (id - next_count < next_id) {
fprintf(stderr, "overlapping duplicate entry");
return -1;
}
// connected to next entry
ssize_t count = next_count + insertion_count;
// ids[x + 1] = id; // no need to adjust id, so since we are adding to the end of the block

if (before > 0) {
MDB_ID previous_id = before > 0 ? ids[before] : 0;
int previous_count = before > 1 ? -ids[before - 1] : 0;
if (previous_count < 1) previous_count = 1;
if (previous_id - insertion_count <= id) {
if (previous_id - insertion_count < id) {
fprintf(stderr, "overlapping duplicate entry");
return -1;
}
// the block we just added to can now be connected to previous entry
count += previous_count;
if (previous_count > 1) {
ids[before - 1] = 0; // remove previous length
}
ids[before] = 0; // remove previous id
if (next_count == 1) {
// we can safely add the new count to the empty space
ids[x - 1] = -count; // update the count
return 0;
}
}
}
if (next_count > 1) {
ids[x - 1] = -count; // update the count
} else if (ids[x - 1] == 0) {
ids[x - 1] = -1 - insertion_count; // we can switch to length-2 block in place
} else {
id = -1 - insertion_count; // switching a single entry to a block size of 2
goto insert_id;
}
return 0;
}
if (before > 0) {
MDB_ID previous_id = before > 0 ? ids[before] : 0;
int count = before > 1 ? -ids[before - 1] : 0;
if (count < 1) count = 1;
if (previous_id - insertion_count <= id) {
if (previous_id - insertion_count < id) {
fprintf(stderr, "overlapping duplicate entry");
return -1;
}
// connected to previous entry
ids[before] = id; // adjust the starting block to include this
if (count > 1) {
ids[before - 1] -= insertion_count; // can just update the count to include this id
return 0;
} else {
id = -1 - insertion_count; // switching a single entry to a block size of 2
x = before;
goto insert_id;
}
}
}
if (x == 1 && ids[0] > 2 && ids[1] == 0 && ids[2] == 0 && ids[3] == 0) {
// this occurs when we have an empty list

if (insertion_count > 1) {
ids[2] = -insertion_count;
ids[3] = id;
} else
ids[2] = id;
return 0;
}
if (!ids[before + 1]) {
// there is an empty slot we can use, find a place in the middle
i = before + 3 < x ? (before + 2) : (before + 1);
if (i >= ids[0]) {
mdb_midl_need(ids_ref, 1);
ids = *ids_ref;
ids[0] = i;
}
ids[i] = id;
if (insertion_count == 1)
return 0; // done
// else insert the length
x = i;
id = -insertion_count;
}
insert_id:
// move items to try to make room
ssize_t last_id = id;
if ((ssize_t)ids[x - 1] < 0) x--;
do {
i = x;
do {
next_id = ids[i];
ids[i++] = last_id;
if (i > ids[0]) { // it is full, need to expand
mdb_midl_need(ids_ref, 1);
ids = *ids_ref;
ids[0] = i;
ids[i] = next_id;
next_id = 0; // break out;
}
last_id = next_id;
} while(next_id);
} while ((ssize_t) id > 0 && insertion_count > 1 && (id = last_id = -insertion_count));
if (i > 0 && ((int) i - x > (ids[0] >> 2) + 4)) { // or too many moves. TODO: This threshold should actually be more like the square root of the length
// respread the ids (this will replace the reference too)
mdb_midl_respread(ids_ref);
}
}
if (insertion_count > 1) return mdb_midl_insert(idp, id+1, insertion_count-1);

return 0;
}

MDB_IDL mdb_midl_alloc(int num)
{
MDB_IDL ids = malloc((num+2) * sizeof(MDB_ID));
Expand Down
2 changes: 1 addition & 1 deletion test/index.test.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,7 @@ describe('lmdb-js', function () {
});
});
let testIteration = 0;
describe('Basic use', basicTests({}));
// describe('Basic use', basicTests({}));
describe(
'Basic use with overlapping sync',
basicTests({
Expand Down

0 comments on commit 41a1fba

Please sign in to comment.