Skip to content

Commit

Permalink
More fixes for freespace handling
Browse files Browse the repository at this point in the history
  • Loading branch information
@kriszyp
kriszyp committed Feb 16, 2024
1 parent 1ed38a8 commit abd1628
Show file tree
Hide file tree
Showing 4 changed files with 144 additions and 23 deletions.
55 changes: 35 additions & 20 deletions dependencies/lmdb/libraries/liblmdb/mdb.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2741,7 +2741,7 @@ mdb_page_alloc(MDB_cursor *mc, int num, MDB_page **mp)
*/
pgno_t block_start;
fprintf(stderr, "looking for block of size %u\n", num);
if (cache_size > num) {
/*if (cache_size > num) {
block_start = env->me_block_size_cache[num];
if (block_start > 0) {
fprintf(stderr, "found block %u of right size %u\n", block_start, num);
Expand All @@ -2753,7 +2753,7 @@ mdb_page_alloc(MDB_cursor *mc, int num, MDB_page **mp)
goto search_done;
}
}
}
}*/
block_start = 0;
unsigned block_size = 0;
ssize_t entry;
Expand All @@ -2779,7 +2779,7 @@ mdb_page_alloc(MDB_cursor *mc, int num, MDB_page **mp)
if (block_size == num) {
// we found a block of the right size
mop[i] = 0;
if (block_size > 1) mop[i + 1] = 0;
if (block_size > 1) mop[i - 1] = 0;
goto search_done;
} else if (block_size < best_fit_size || best_fit_size == 0) {
best_fit_start = i - 1;
Expand All @@ -2791,7 +2791,7 @@ mdb_page_alloc(MDB_cursor *mc, int num, MDB_page **mp)
goto continue_best_fit;
}
}
}
}/*
if (block_size > 0) {
// cache this block size
if (block_size >= 2<<30) block_size = (2<<30) - 1;
Expand All @@ -2805,7 +2805,7 @@ mdb_page_alloc(MDB_cursor *mc, int num, MDB_page **mp)
}
env->me_block_size_cache[block_size] = pgno;
fprintf(stderr, "cached block %u of size %u\n", pgno, block_size);
}
}*/
//if (mop[i-n2] == pgno+n2)
// goto search_done;
}
Expand Down Expand Up @@ -2888,9 +2888,13 @@ mdb_page_alloc(MDB_cursor *mc, int num, MDB_page **mp)
DPRINTF(("IDL %"Yu, idl[j]));
#endif
/* Merge in descending sorted order */
fprintf(stderr, "merge\n");
fprintf(stderr, "merge from %u\n", last);
for (unsigned i = 1; i <= idl[0]; i++) {
if ((rc = mdb_midl_insert(&mop, idl[i])) != 0)
ssize_t entry = idl[i];
if (entry < 0) {
fprintf(stderr, "Block length entry\n");
}
if ((rc = mdb_midl_insert(&mop, entry)) != 0)
goto fail;
//mdb_midl_xmerge(mop, idl);
}
Expand Down Expand Up @@ -4046,11 +4050,14 @@ mdb_freelist_save(MDB_txn *txn)
if (rc)
return rc;
pglast = head_id = *(txnid_t *)key.mv_data;
fprintf(stderr, "deleting freespace page %u\n", head_id);
total_room = head_room = 0;
mdb_tassert(txn, pglast <= env->me_pglast);
rc = mdb_cursor_del(&mc, 0);
if (rc)
return rc;
if (pglast <= env->me_pglast) {
mdb_tassert(txn, pglast <= env->me_pglast);
rc = mdb_cursor_del(&mc, 0);
if (rc)
return rc;
}
}

/* Save the IDL of pages freed by this txn, to a single record */
Expand All @@ -4063,6 +4070,8 @@ mdb_freelist_save(MDB_txn *txn)
}
free_pgs = txn->mt_free_pgs;
/* Write to last page of freeDB */
fprintf(stderr, "write freed pages %u\n", txn->mt_txnid);
mdb_midl_print(stderr, free_pgs);
key.mv_size = sizeof(txn->mt_txnid);
key.mv_data = &txn->mt_txnid;
do {
Expand All @@ -4089,7 +4098,7 @@ mdb_freelist_save(MDB_txn *txn)
}

mop = env->me_pghead;
mop_len = (mop ? mop[0] : 0) + txn->mt_loose_count;
mop_len = (mop ? mdb_midl_pack_count(mop) : 0) + txn->mt_loose_count;

/* Reserve records for me_pghead[]. Split it if multi-page,
* to avoid searching freeDB for a page range. Use keys in
Expand Down Expand Up @@ -4135,27 +4144,31 @@ mdb_freelist_save(MDB_txn *txn)
if (txn->mt_loose_pgs) {
MDB_page *mp = txn->mt_loose_pgs;
unsigned count = txn->mt_loose_count;
MDB_IDL loose;
/* Room for loose pages + temp IDL with same */
/*MDB_IDL loose;
// Room for loose pages + temp IDL with same
if ((rc = mdb_midl_need(&env->me_pghead, 2*count+1)) != 0)
return rc;
return rc;*/
lost_loose += count;
mop = env->me_pghead;
loose = mop + MDB_IDL_ALLOCLEN(mop) - count;
//loose = mop + MDB_IDL_ALLOCLEN(mop) - count;
for (count = 0; mp; mp = NEXT_LOOSE_PAGE(mp))
loose[ ++count ] = mp->mp_pgno;
mdb_midl_insert(&mop, mp->mp_pgno);
/*loose[ ++count ] = mp->mp_pgno;
loose[0] = count;
mdb_midl_sort(loose);
mdb_midl_xmerge(mop, loose);
mdb_midl_xmerge(mop, loose);*/
env->me_pghead = mop;
txn->mt_loose_pgs = NULL;
txn->mt_loose_count = 0;
mop_len = mop[0];
//mop_len = mop[0];
}

/* Fill in the reserved me_pghead records. Everything is finally
* in place, so this will not allocate or free any DB pages.
*/
rc = MDB_SUCCESS;
mop = mdb_midl_pack(mop);
mop_len = mop ? mop[0] : 0;
if (mop_len) {
MDB_val key, data;

Expand All @@ -4167,6 +4180,7 @@ mdb_freelist_save(MDB_txn *txn)
rc = mdb_cursor_first(&mc, &key, &data);
for (; !rc; rc = mdb_cursor_next(&mc, &key, &data, MDB_NEXT)) {
txnid_t id = *(txnid_t *)key.mv_data;
fprintf(stderr, "finishing save of freespace %u\n", id);
ssize_t len = (ssize_t)(data.mv_size / sizeof(MDB_ID)) - 1;
MDB_ID save;

Expand All @@ -4185,8 +4199,9 @@ mdb_freelist_save(MDB_txn *txn)
if (rc || !mop_len)
break;
}
mdb_midl_free(mop);

env->me_pghead = mop - mop_len;
//env->me_pghead = mop - mop_len;
}

/* Restore this so we can check vs. dirty_list after mdb_page_flush() */
Expand Down
81 changes: 79 additions & 2 deletions dependencies/lmdb/libraries/liblmdb/midl.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,56 @@ unsigned mdb_midl_search( MDB_IDL ids, MDB_ID id )
* if found, returns position of id
* if not found, returns first position greater than id
*/
unsigned base = 0;
unsigned cursor = 1;
int val = 0;
unsigned n = ids[0];
unsigned end = n;

binary_search:
while( 0 < n ) {
unsigned pivot = n >> 1;
cursor = base + pivot + 1;
unsigned x = cursor;
// skip past empty and block length entries
while(((ssize_t)ids[x]) <= 0) {
if (++x > end) { // reached the end, go to lower half
n = pivot;
val = 0;
end -= pivot;
goto binary_search;
}
}
val = CMP( ids[cursor], id );

if( val < 0 ) {
n = pivot;
end -= pivot;
} else if ( val > 0 ) {
base = cursor;
n -= pivot + 1;
} else {
return cursor;
}
}
if( val > 0 ) {
unsigned next = ++cursor;
end = ids[0];
// skip past empty and block length entries
while(((ssize_t)ids[cursor]) <= 0) {
if (++cursor > end) { // reached the end
return next; // go back to the one after the last value in this case
}
}
}
return cursor;
/*
* binary search of id in ids
* if found, returns position of id
* if not found, returns first position greater than id
*
unsigned base = 0;
unsigned cursor = 1;
int val = 0;
Expand All @@ -62,6 +112,7 @@ unsigned mdb_midl_search( MDB_IDL ids, MDB_ID id )
val = CMP( entry, id );
if( val < 0 ) {
while(!ids[cursor - 1]) cursor--;
if (cursor == end) return cursor;
end = cursor;
} else if ( val > 0 ) {
Expand All @@ -75,7 +126,7 @@ unsigned mdb_midl_search( MDB_IDL ids, MDB_ID id )
if( val > 0 ) {
++cursor;
}
return cursor;
return cursor;*/
}

/* superseded by append/sort */
Expand Down Expand Up @@ -144,7 +195,6 @@ int mdb_midl_insert( MDB_IDL* ids_ref, MDB_ID id )
ids[x - 1] = -2;
} else {
id = -2; // switching a single entry to a block size of 2
x--;
goto insert_id;
}
return 0;
Expand Down Expand Up @@ -236,6 +286,33 @@ static int mdb_midl_grow( MDB_IDL *idp, int num )
return 0;
}

MDB_IDL mdb_midl_pack(MDB_IDL idl) {
if (!idl) return NULL;
MDB_IDL packed = mdb_midl_alloc(idl[0]);
unsigned j = 1;
for (unsigned i = 1; i < idl[0]; i++) {
ssize_t entry = idl[i];
if (entry) packed[j++] = entry;
}
if (j == 1) {
// empty list, just treat as no list
mdb_midl_free(packed);
return NULL;
}
packed[0] = j - 1;
return packed;
}

unsigned mdb_midl_pack_count(MDB_IDL idl) {
unsigned count = 0;
if (idl) {
for (unsigned i = 1; i < idl[0]; i++) {
if (idl[i]) count++;
}
}
return count;
}

int mdb_midl_need( MDB_IDL *idp, unsigned num )
{
MDB_IDL ids = *idp;
Expand Down
3 changes: 3 additions & 0 deletions dependencies/lmdb/libraries/liblmdb/midl.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -112,6 +112,9 @@ void mdb_midl_shrink(MDB_IDL *idp);
int mdb_midl_need(MDB_IDL *idp, unsigned num);

int mdb_midl_print( FILE *fp, MDB_IDL ids );
MDB_IDL mdb_midl_pack(MDB_IDL idl);
unsigned mdb_midl_pack_count(MDB_IDL idl);

/** Insert an ID into an IDL.
* @param[in,out] idp Address of the IDL to append to.
* @param[in] id The ID to append.
Expand Down
28 changes: 27 additions & 1 deletion test/index.test.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,7 @@ describe('lmdb-js', function () {
});
});
let testIteration = 0;
describe('Basic use', basicTests({}));
describe.skip('Basic use', basicTests({}));
describe(
'Basic use with overlapping sync',
basicTests({
Expand Down Expand Up @@ -245,6 +245,32 @@ describe('lmdb-js', function () {
'0Sdts8FwTqt2Hv5j9KE7ebjsQcFbYDdL/0Sdtsud6g8YGhPwUK04fRVKhuTywhnx8',
]);
});
it.only('bigger puts, testing free space management', async function () {
let seed = 15325223;
function random() {
return randomInt() / 2038074743;
}
function randomInt() {
seed++;
let a = seed * 15485863;
return a * a * a % 2038074743;
}

let promise;
let additive = 'this is more text';
for (let i = 0; i < 7; i++) additive += additive;
for (let i = 0; i < 1000; i++) {
let text = 'this is a test';
while(random() < 0.95) text += additive;
console.log('write', i, text.length);
promise = db.put(i % 10, text);
if (i % 6 == 0) {
await promise;
}
}
await promise;
});

it('clear between puts', async function () {
db.put('key0', 'zero');
db.clearAsync();
Expand Down

0 comments on commit abd1628

Please sign in to comment.