From abd16289488b0786cecaff346f3b611f848deb35 Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Thu, 15 Feb 2024 21:56:54 -0700 Subject: [PATCH] More fixes for freespace handling --- dependencies/lmdb/libraries/liblmdb/mdb.c | 55 +++++++++------ dependencies/lmdb/libraries/liblmdb/midl.c | 81 +++++++++++++++++++++- dependencies/lmdb/libraries/liblmdb/midl.h | 3 + test/index.test.js | 28 +++++++- 4 files changed, 144 insertions(+), 23 deletions(-) diff --git a/dependencies/lmdb/libraries/liblmdb/mdb.c b/dependencies/lmdb/libraries/liblmdb/mdb.c index c808c2412..245d9ddb3 100644 --- a/dependencies/lmdb/libraries/liblmdb/mdb.c +++ b/dependencies/lmdb/libraries/liblmdb/mdb.c @@ -2741,7 +2741,7 @@ mdb_page_alloc(MDB_cursor *mc, int num, MDB_page **mp) */ pgno_t block_start; fprintf(stderr, "looking for block of size %u\n", num); - if (cache_size > num) { + /*if (cache_size > num) { block_start = env->me_block_size_cache[num]; if (block_start > 0) { fprintf(stderr, "found block %u of right size %u\n", block_start, num); @@ -2753,7 +2753,7 @@ mdb_page_alloc(MDB_cursor *mc, int num, MDB_page **mp) goto search_done; } } - } + }*/ block_start = 0; unsigned block_size = 0; ssize_t entry; @@ -2779,7 +2779,7 @@ mdb_page_alloc(MDB_cursor *mc, int num, MDB_page **mp) if (block_size == num) { // we found a block of the right size mop[i] = 0; - if (block_size > 1) mop[i + 1] = 0; + if (block_size > 1) mop[i - 1] = 0; goto search_done; } else if (block_size < best_fit_size || best_fit_size == 0) { best_fit_start = i - 1; @@ -2791,7 +2791,7 @@ mdb_page_alloc(MDB_cursor *mc, int num, MDB_page **mp) goto continue_best_fit; } } - } + }/* if (block_size > 0) { // cache this block size if (block_size >= 2<<30) block_size = (2<<30) - 1; @@ -2805,7 +2805,7 @@ mdb_page_alloc(MDB_cursor *mc, int num, MDB_page **mp) } env->me_block_size_cache[block_size] = pgno; fprintf(stderr, "cached block %u of size %u\n", pgno, block_size); - } + }*/ //if (mop[i-n2] == pgno+n2) // goto search_done; } @@ -2888,9 +2888,13 @@ mdb_page_alloc(MDB_cursor *mc, int num, MDB_page **mp) DPRINTF(("IDL %"Yu, idl[j])); #endif /* Merge in descending sorted order */ - fprintf(stderr, "merge\n"); + fprintf(stderr, "merge from %u\n", last); for (unsigned i = 1; i <= idl[0]; i++) { - if ((rc = mdb_midl_insert(&mop, idl[i])) != 0) + ssize_t entry = idl[i]; + if (entry < 0) { + fprintf(stderr, "Block length entry\n"); + } + if ((rc = mdb_midl_insert(&mop, entry)) != 0) goto fail; //mdb_midl_xmerge(mop, idl); } @@ -4046,11 +4050,14 @@ mdb_freelist_save(MDB_txn *txn) if (rc) return rc; pglast = head_id = *(txnid_t *)key.mv_data; + fprintf(stderr, "deleting freespace page %u\n", head_id); total_room = head_room = 0; - mdb_tassert(txn, pglast <= env->me_pglast); - rc = mdb_cursor_del(&mc, 0); - if (rc) - return rc; + if (pglast <= env->me_pglast) { + mdb_tassert(txn, pglast <= env->me_pglast); + rc = mdb_cursor_del(&mc, 0); + if (rc) + return rc; + } } /* Save the IDL of pages freed by this txn, to a single record */ @@ -4063,6 +4070,8 @@ mdb_freelist_save(MDB_txn *txn) } free_pgs = txn->mt_free_pgs; /* Write to last page of freeDB */ + fprintf(stderr, "write freed pages %u\n", txn->mt_txnid); + mdb_midl_print(stderr, free_pgs); key.mv_size = sizeof(txn->mt_txnid); key.mv_data = &txn->mt_txnid; do { @@ -4089,7 +4098,7 @@ mdb_freelist_save(MDB_txn *txn) } mop = env->me_pghead; - mop_len = (mop ? mop[0] : 0) + txn->mt_loose_count; + mop_len = (mop ? mdb_midl_pack_count(mop) : 0) + txn->mt_loose_count; /* Reserve records for me_pghead[]. Split it if multi-page, * to avoid searching freeDB for a page range. Use keys in @@ -4135,27 +4144,31 @@ mdb_freelist_save(MDB_txn *txn) if (txn->mt_loose_pgs) { MDB_page *mp = txn->mt_loose_pgs; unsigned count = txn->mt_loose_count; - MDB_IDL loose; - /* Room for loose pages + temp IDL with same */ + /*MDB_IDL loose; + // Room for loose pages + temp IDL with same if ((rc = mdb_midl_need(&env->me_pghead, 2*count+1)) != 0) - return rc; + return rc;*/ lost_loose += count; mop = env->me_pghead; - loose = mop + MDB_IDL_ALLOCLEN(mop) - count; + //loose = mop + MDB_IDL_ALLOCLEN(mop) - count; for (count = 0; mp; mp = NEXT_LOOSE_PAGE(mp)) - loose[ ++count ] = mp->mp_pgno; + mdb_midl_insert(&mop, mp->mp_pgno); + /*loose[ ++count ] = mp->mp_pgno; loose[0] = count; mdb_midl_sort(loose); - mdb_midl_xmerge(mop, loose); + mdb_midl_xmerge(mop, loose);*/ + env->me_pghead = mop; txn->mt_loose_pgs = NULL; txn->mt_loose_count = 0; - mop_len = mop[0]; + //mop_len = mop[0]; } /* Fill in the reserved me_pghead records. Everything is finally * in place, so this will not allocate or free any DB pages. */ rc = MDB_SUCCESS; + mop = mdb_midl_pack(mop); + mop_len = mop ? mop[0] : 0; if (mop_len) { MDB_val key, data; @@ -4167,6 +4180,7 @@ mdb_freelist_save(MDB_txn *txn) rc = mdb_cursor_first(&mc, &key, &data); for (; !rc; rc = mdb_cursor_next(&mc, &key, &data, MDB_NEXT)) { txnid_t id = *(txnid_t *)key.mv_data; + fprintf(stderr, "finishing save of freespace %u\n", id); ssize_t len = (ssize_t)(data.mv_size / sizeof(MDB_ID)) - 1; MDB_ID save; @@ -4185,8 +4199,9 @@ mdb_freelist_save(MDB_txn *txn) if (rc || !mop_len) break; } + mdb_midl_free(mop); - env->me_pghead = mop - mop_len; + //env->me_pghead = mop - mop_len; } /* Restore this so we can check vs. dirty_list after mdb_page_flush() */ diff --git a/dependencies/lmdb/libraries/liblmdb/midl.c b/dependencies/lmdb/libraries/liblmdb/midl.c index 5a85dc976..406ed402d 100644 --- a/dependencies/lmdb/libraries/liblmdb/midl.c +++ b/dependencies/lmdb/libraries/liblmdb/midl.c @@ -38,6 +38,56 @@ unsigned mdb_midl_search( MDB_IDL ids, MDB_ID id ) * if found, returns position of id * if not found, returns first position greater than id */ + unsigned base = 0; + unsigned cursor = 1; + int val = 0; + unsigned n = ids[0]; + unsigned end = n; + + binary_search: + while( 0 < n ) { + unsigned pivot = n >> 1; + cursor = base + pivot + 1; + unsigned x = cursor; + // skip past empty and block length entries + while(((ssize_t)ids[x]) <= 0) { + if (++x > end) { // reached the end, go to lower half + n = pivot; + val = 0; + end -= pivot; + goto binary_search; + } + } + val = CMP( ids[cursor], id ); + + if( val < 0 ) { + n = pivot; + end -= pivot; + } else if ( val > 0 ) { + base = cursor; + n -= pivot + 1; + } else { + return cursor; + } + } + if( val > 0 ) { + unsigned next = ++cursor; + end = ids[0]; + // skip past empty and block length entries + while(((ssize_t)ids[cursor]) <= 0) { + if (++cursor > end) { // reached the end + return next; // go back to the one after the last value in this case + } + } + } + return cursor; + /* + * binary search of id in ids + * if found, returns position of id + * if not found, returns first position greater than id + * + + unsigned base = 0; unsigned cursor = 1; int val = 0; @@ -62,6 +112,7 @@ unsigned mdb_midl_search( MDB_IDL ids, MDB_ID id ) val = CMP( entry, id ); if( val < 0 ) { + while(!ids[cursor - 1]) cursor--; if (cursor == end) return cursor; end = cursor; } else if ( val > 0 ) { @@ -75,7 +126,7 @@ unsigned mdb_midl_search( MDB_IDL ids, MDB_ID id ) if( val > 0 ) { ++cursor; } - return cursor; + return cursor;*/ } /* superseded by append/sort */ @@ -144,7 +195,6 @@ int mdb_midl_insert( MDB_IDL* ids_ref, MDB_ID id ) ids[x - 1] = -2; } else { id = -2; // switching a single entry to a block size of 2 - x--; goto insert_id; } return 0; @@ -236,6 +286,33 @@ static int mdb_midl_grow( MDB_IDL *idp, int num ) return 0; } +MDB_IDL mdb_midl_pack(MDB_IDL idl) { + if (!idl) return NULL; + MDB_IDL packed = mdb_midl_alloc(idl[0]); + unsigned j = 1; + for (unsigned i = 1; i < idl[0]; i++) { + ssize_t entry = idl[i]; + if (entry) packed[j++] = entry; + } + if (j == 1) { + // empty list, just treat as no list + mdb_midl_free(packed); + return NULL; + } + packed[0] = j - 1; + return packed; +} + +unsigned mdb_midl_pack_count(MDB_IDL idl) { + unsigned count = 0; + if (idl) { + for (unsigned i = 1; i < idl[0]; i++) { + if (idl[i]) count++; + } + } + return count; +} + int mdb_midl_need( MDB_IDL *idp, unsigned num ) { MDB_IDL ids = *idp; diff --git a/dependencies/lmdb/libraries/liblmdb/midl.h b/dependencies/lmdb/libraries/liblmdb/midl.h index 3823dd499..2e1bff3fc 100644 --- a/dependencies/lmdb/libraries/liblmdb/midl.h +++ b/dependencies/lmdb/libraries/liblmdb/midl.h @@ -112,6 +112,9 @@ void mdb_midl_shrink(MDB_IDL *idp); int mdb_midl_need(MDB_IDL *idp, unsigned num); int mdb_midl_print( FILE *fp, MDB_IDL ids ); +MDB_IDL mdb_midl_pack(MDB_IDL idl); +unsigned mdb_midl_pack_count(MDB_IDL idl); + /** Insert an ID into an IDL. * @param[in,out] idp Address of the IDL to append to. * @param[in] id The ID to append. diff --git a/test/index.test.js b/test/index.test.js index f2ac8669b..02413e578 100644 --- a/test/index.test.js +++ b/test/index.test.js @@ -59,7 +59,7 @@ describe('lmdb-js', function () { }); }); let testIteration = 0; - describe('Basic use', basicTests({})); + describe.skip('Basic use', basicTests({})); describe( 'Basic use with overlapping sync', basicTests({ @@ -245,6 +245,32 @@ describe('lmdb-js', function () { '0Sdts8FwTqt2Hv5j9KE7ebjsQcFbYDdL/0Sdtsud6g8YGhPwUK04fRVKhuTywhnx8', ]); }); + it.only('bigger puts, testing free space management', async function () { + let seed = 15325223; + function random() { + return randomInt() / 2038074743; + } + function randomInt() { + seed++; + let a = seed * 15485863; + return a * a * a % 2038074743; + } + + let promise; + let additive = 'this is more text'; + for (let i = 0; i < 7; i++) additive += additive; + for (let i = 0; i < 1000; i++) { + let text = 'this is a test'; + while(random() < 0.95) text += additive; + console.log('write', i, text.length); + promise = db.put(i % 10, text); + if (i % 6 == 0) { + await promise; + } + } + await promise; + }); + it('clear between puts', async function () { db.put('key0', 'zero'); db.clearAsync();