From 98c4deb01791e21ec383b99b5060bb8a53af7355 Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Mon, 12 Feb 2024 15:44:27 -0700 Subject: [PATCH 1/7] Improve free-list management --- dependencies/lmdb/libraries/liblmdb/mdb.c | 106 ++++++++++++++++++--- dependencies/lmdb/libraries/liblmdb/midl.c | 67 +++++++++++-- dependencies/lmdb/libraries/liblmdb/midl.h | 7 ++ 3 files changed, 160 insertions(+), 20 deletions(-) diff --git a/dependencies/lmdb/libraries/liblmdb/mdb.c b/dependencies/lmdb/libraries/liblmdb/mdb.c index bbf38f2235..23ca244cef 100644 --- a/dependencies/lmdb/libraries/liblmdb/mdb.c +++ b/dependencies/lmdb/libraries/liblmdb/mdb.c @@ -1581,7 +1581,9 @@ typedef struct MDB_xcursor { /** State of FreeDB old pages, stored in the MDB_env */ typedef struct MDB_pgstate { pgno_t *mf_pghead; /**< Reclaimed freeDB pages, or NULL before use */ + pgno_t *mf_block_size_cache; /**< Cache of contiguous blocks, by size and page_no pairs */ txnid_t mf_pglast; /**< ID of last used record, or 0 if !mf_pghead */ + unsigned mf_position; /** Position in the free page list, so that we can keep trying to write to the same block if possible */ } MDB_pgstate; /**/ struct MDB_last_map { @@ -1642,6 +1644,8 @@ struct MDB_env { MDB_pgstate me_pgstate; /**< state of old pages from freeDB */ # define me_pglast me_pgstate.mf_pglast # define me_pghead me_pgstate.mf_pghead +# define me_block_size_cache me_pgstate.mf_block_size_cache +# define me_freelist_position me_pgstate.mf_position MDB_page *me_dpages; /**< list of malloc'd blocks for re-use */ /** IDL of pages that became unused in a write txn */ MDB_IDL me_free_pgs; @@ -2718,6 +2722,13 @@ mdb_page_alloc(MDB_cursor *mc, int num, MDB_page **mp) goto fail; } + if (!env->me_block_size_cache) { + env->me_block_size_cache = calloc(32, sizeof(pgno_t)); + env->me_block_size_cache[0] = 31; + } + unsigned cache_size = env->me_block_size_cache[0]; + pgno_t best_fit_start = 0; // this is a block we will use if we don't find an exact fit + pgno_t best_fit_size = -1; for (op = MDB_FIRST;; op = MDB_NEXT) { MDB_val key, data; MDB_node *leaf; @@ -2726,13 +2737,71 @@ mdb_page_alloc(MDB_cursor *mc, int num, MDB_page **mp) /* Seek a big enough contiguous page range. Prefer * pages at the tail, just truncating the list. */ + pgno_t block_start; + fprintf(stderr, "looking for block of size %u\n", num); + if (cache_size > num) { + block_start = env->me_block_size_cache[num]; + if (block_start > 0) { + fprintf(stderr, "found block %u of right size %u\n", block_start, num); + // we found a block of the right size + env->me_block_size_cache[num] = 0; // clear it out, since it will be used (or it is invalid) + pgno = mdb_midl_search(env->me_pghead, block_start); + fprintf(stderr, "does it checkout %u == %u\n", block_start, pgno); + if (pgno == block_start && pgno + num <= mop_len) { // double check it + goto search_done; + } + } + } + block_start = 0; + unsigned block_size = 0; + ssize_t entry; + // TODO: Skip this on the first iteration, since we already checked the cache if (mop_len > n2) { i = mop_len; do { - pgno = mop[i]; - if (mop[i-n2] == pgno+n2) - goto search_done; - } while (--i > n2); + entry = i == 0 ? 0 : mop[i]; + fprintf(stderr, "pgno %u next would be %u\n", entry, block_start + block_size); + if (entry == 0) continue; + if (entry > 0) { + pgno = entry; + block_size = 1; + } else { + block_size = -entry; + pgno = mop[--i]; + } + if (pgno == block_start + block_size) { + block_size++; // count current contiguous block size + } else { + if (block_size >= num) { + if (block_size == num) { + // we found a block of the right size + pgno = block_start; + goto search_done; + } else if (block_size < best_fit_size || best_fit_size == 0) { + best_fit_start = block_start; + best_fit_size = block_size; + } + } + if (block_size > 0) { + // cache this block size + if (block_size >= 2<<30) block_size = (2<<30) - 1; + unsigned cache_size = env->me_block_size_cache[0]; + if (block_size > cache_size) { + fprintf(stderr, "expand block size cache to %u\n", block_size << 1); + env->me_block_size_cache = realloc(env->me_block_size_cache, (block_size << 1) * sizeof(pgno_t)); + env->me_block_size_cache[0] = (block_size << 1) - 1; + memset(env->me_block_size_cache + cache_size + 1, 0, (env->me_block_size_cache[0] - cache_size) * sizeof(pgno_t)); + cache_size = env->me_block_size_cache[0]; + } + env->me_block_size_cache[block_size] = block_start; + fprintf(stderr, "cached block %u of size %u\n", block_start, block_size); + } + block_start = pgno; + block_size = 1; + } + //if (mop[i-n2] == pgno+n2) + // goto search_done; + } while (--i >= 0); if (--retry < 0) break; } @@ -2813,10 +2882,23 @@ mdb_page_alloc(MDB_cursor *mc, int num, MDB_page **mp) DPRINTF(("IDL %"Yu, idl[j])); #endif /* Merge in descending sorted order */ - mdb_midl_xmerge(mop, idl); + fprintf(stderr, "merge\n"); + for (unsigned i = i; i < idl[0]; i++) { + if (mdb_midl_insert(mop, idl[i]) == -3) { + if ((rc = mdb_midl_need(&env->me_pghead, idl[0])) != 0) + goto fail; + mop = env->me_pghead; + } + //mdb_midl_xmerge(mop, idl); + } mop_len = mop[0]; } - + if (best_fit_start > 0) { + pgno = best_fit_start; + fprintf(stderr, "using best fit at %u size %u of %u\n", pgno, num, best_fit_size); + env->me_block_size_cache[best_fit_size] = 0; // clear this out of the cache (TODO: could move it) + goto search_done; + } /* Use new pages from the map when nothing suitable in the freeDB */ i = 0; pgno = txn->mt_next_pgno; @@ -2841,6 +2923,7 @@ mdb_page_alloc(MDB_cursor *mc, int num, MDB_page **mp) #endif search_done: + fprintf(stderr, "alloc pgno %u\n", pgno); if (env->me_flags & MDB_WRITEMAP) { np = (MDB_page *)(env->me_map + env->me_psize * pgno); } else { @@ -3766,8 +3849,9 @@ mdb_txn_end(MDB_txn *txn, unsigned mode) mdb_midl_shrink(&txn->mt_free_pgs); env->me_free_pgs = txn->mt_free_pgs; /* me_pgstate: */ - env->me_pghead = NULL; - env->me_pglast = 0; + fprintf(stderr, "txn_end env->me_pghead %p", env->me_pghead); + //env->me_pghead = NULL; + //env->me_pglast = 0; env->me_txn = NULL; mode = 0; /* txn == env->me_txn0, do not free() it */ @@ -3782,7 +3866,7 @@ mdb_txn_end(MDB_txn *txn, unsigned mode) mdb_midl_free(txn->mt_free_pgs); free(txn->mt_u.dirty_list); } - mdb_midl_free(pghead); + //mdb_midl_free(pghead); } #if MDB_RPAGE_CACHE if (MDB_REMAPPING(env->me_flags) && !txn->mt_parent) { @@ -4610,8 +4694,8 @@ mdb_txn_commit(MDB_txn *txn) if (rc) goto fail; - mdb_midl_free(env->me_pghead); - env->me_pghead = NULL; + //mdb_midl_free(env->me_pghead); + //env->me_pghead = NULL; mdb_midl_shrink(&txn->mt_free_pgs); #if (MDB_DEBUG) > 2 diff --git a/dependencies/lmdb/libraries/liblmdb/midl.c b/dependencies/lmdb/libraries/liblmdb/midl.c index 272e557e3f..b4eb5b5c37 100644 --- a/dependencies/lmdb/libraries/liblmdb/midl.c +++ b/dependencies/lmdb/libraries/liblmdb/midl.c @@ -66,7 +66,7 @@ unsigned mdb_midl_search( MDB_IDL ids, MDB_ID id ) return cursor; } -#if 0 /* superseded by append/sort */ + /* superseded by append/sort */ int mdb_midl_insert( MDB_IDL ids, MDB_ID id ) { unsigned x, i; @@ -91,15 +91,51 @@ int mdb_midl_insert( MDB_IDL ids, MDB_ID id ) return -2; } else { - /* insert id */ - for (i=ids[0]; i>x; i--) - ids[i] = ids[i-1]; - ids[x] = id; + if (x >= ids[0]) return -3; // at the end + MDB_ID next_id = ids[x]; + if (id < 0) next_id = ids[x + 1]; + if (id - 1 == next_id) { + // connected to next entry + ids[x]--; // increment negatively, as we have just expanded a block + ids[x + 1] = id; + return 0; + } + unsigned before = x; + while (!ids[--before] && before >= 0){} // move past empty entries + if (before >= 0) { + MDB_ID next_id = before > 0 ? ids[before] : 0; + int count = before > 1 ? -ids[before - 1] : 0; + if (count < 1) count = 1; + if (next_id - count == id) { + // connected to previous entry + if (count > 1) { + ids[before - 1]--; // can just update the count to include this id + return 0; + } else { + // TODO: need to make space for this one + } + } + } + if (before + 1 < x) { + // there is an empty slot we can use, find a place in the middle + ids[(before + x) >> 1] = id; + return 0; + } + // move items to try to make room + MDB_ID last_id = id; + i = x; + do { + MDB_ID next_id = ids[i]; + ids[i++] = last_id; + last_id = next_id; + } while(next_id); + if (x == ids[0] || // if it is full + x - i > ids[0] >> 3) // or too many moves. TODO: This threshold should actually be more like the square root of the length + return -3; // request to grow } return 0; } -#endif MDB_IDL mdb_midl_alloc(int num) { @@ -146,10 +182,23 @@ int mdb_midl_need( MDB_IDL *idp, unsigned num ) num += ids[0]; if (num > ids[-1]) { num = (num + num/4 + (256 + 2)) & -256; - if (!(ids = realloc(ids-1, num * sizeof(MDB_ID)))) + MDB_IDL new_ids; + if (!(new_ids = alloc(ids-1, num * sizeof(MDB_ID)))) return ENOMEM; - *ids++ = num - 2; - *idp = ids; + *new_ids++ = num - 2; + unsigned j = 0; + // re-spread out the entries with gaps for growth + for (unsigned i = 1; i < ids[0]; i++) { + new_ids[j++] = 0; // empty slot for growth + ssize_t entry; + while (!(entry = ids[i])) { i++; } + new_ids[j++] = entry; + if (entry < 0) new_ids[j++] = ids[i++]; // this was a block with a length + } + // now shrink (or grow) back to appropriate size + new_ids = alloc(new_ids - 1, (j + (j >> 3)) * sizeof(MDB_ID)); + new_ids++; + *idp = new_ids; } return 0; } diff --git a/dependencies/lmdb/libraries/liblmdb/midl.h b/dependencies/lmdb/libraries/liblmdb/midl.h index aa45c2e726..bf3df89f3a 100644 --- a/dependencies/lmdb/libraries/liblmdb/midl.h +++ b/dependencies/lmdb/libraries/liblmdb/midl.h @@ -109,6 +109,13 @@ void mdb_midl_shrink(MDB_IDL *idp); */ int mdb_midl_need(MDB_IDL *idp, unsigned num); +/** Insert an ID into an IDL. + * @param[in,out] idp Address of the IDL to append to. + * @param[in] id The ID to append. + * @return 0 on success, ENOMEM if the IDL is too large. + */ +int mdb_midl_insert( MDB_IDL *idp, MDB_ID id ); + /** Append an ID onto an IDL. * @param[in,out] idp Address of the IDL to append to. * @param[in] id The ID to append. From 73d1eb4c52f503435113507aaa06a707ffb50639 Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Tue, 13 Feb 2024 13:30:11 -0700 Subject: [PATCH 2/7] Pass a reference to insert so it can grow --- dependencies/lmdb/libraries/liblmdb/mdb.c | 8 +++----- dependencies/lmdb/libraries/liblmdb/midl.c | 9 ++++++--- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/dependencies/lmdb/libraries/liblmdb/mdb.c b/dependencies/lmdb/libraries/liblmdb/mdb.c index 23ca244cef..6bf9cf6ca6 100644 --- a/dependencies/lmdb/libraries/liblmdb/mdb.c +++ b/dependencies/lmdb/libraries/liblmdb/mdb.c @@ -2884,13 +2884,11 @@ mdb_page_alloc(MDB_cursor *mc, int num, MDB_page **mp) /* Merge in descending sorted order */ fprintf(stderr, "merge\n"); for (unsigned i = i; i < idl[0]; i++) { - if (mdb_midl_insert(mop, idl[i]) == -3) { - if ((rc = mdb_midl_need(&env->me_pghead, idl[0])) != 0) - goto fail; - mop = env->me_pghead; - } + if ((rc = mdb_midl_insert(&mop, idl[i])) != 0) + goto fail; //mdb_midl_xmerge(mop, idl); } + if (mop != env->me_pghead) env->me_pghead = mop; mop_len = mop[0]; } if (best_fit_start > 0) { diff --git a/dependencies/lmdb/libraries/liblmdb/midl.c b/dependencies/lmdb/libraries/liblmdb/midl.c index b4eb5b5c37..c75d344305 100644 --- a/dependencies/lmdb/libraries/liblmdb/midl.c +++ b/dependencies/lmdb/libraries/liblmdb/midl.c @@ -67,8 +67,9 @@ unsigned mdb_midl_search( MDB_IDL ids, MDB_ID id ) } /* superseded by append/sort */ -int mdb_midl_insert( MDB_IDL ids, MDB_ID id ) +int mdb_midl_insert( MDB_IDL* ids_ref, MDB_ID id ) { + MDB_IDL ids = *ids_ref; unsigned x, i; x = mdb_midl_search( ids, id ); @@ -130,8 +131,10 @@ int mdb_midl_insert( MDB_IDL ids, MDB_ID id ) last_id = next_id; } while(next_id); if (x == ids[0] || // if it is full - x - i > ids[0] >> 3) // or too many moves. TODO: This threshold should actually be more like the square root of the length - return -3; // request to grow + x - i > ids[0] >> 3) { // or too many moves. TODO: This threshold should actually be more like the square root of the length + // grow the ids (this will replace the reference too) + mdb_midl_need(ids_ref, 1); + } } return 0; From 744fb7ef87400afc15e1647d26f0bc26ea4e650d Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Wed, 14 Feb 2024 20:48:48 -0700 Subject: [PATCH 3/7] Implementing empty slots and contiguous blocks in freelist --- dependencies/lmdb/libraries/liblmdb/mdb.c | 119 +++++++++++---------- dependencies/lmdb/libraries/liblmdb/midl.c | 107 +++++++++++------- 2 files changed, 132 insertions(+), 94 deletions(-) diff --git a/dependencies/lmdb/libraries/liblmdb/mdb.c b/dependencies/lmdb/libraries/liblmdb/mdb.c index 6bf9cf6ca6..4254223e1e 100644 --- a/dependencies/lmdb/libraries/liblmdb/mdb.c +++ b/dependencies/lmdb/libraries/liblmdb/mdb.c @@ -2726,14 +2726,16 @@ mdb_page_alloc(MDB_cursor *mc, int num, MDB_page **mp) env->me_block_size_cache = calloc(32, sizeof(pgno_t)); env->me_block_size_cache[0] = 31; } + unsigned empty_entries = 0; unsigned cache_size = env->me_block_size_cache[0]; - pgno_t best_fit_start = 0; // this is a block we will use if we don't find an exact fit - pgno_t best_fit_size = -1; + pgno_t best_fit_start; // this is a block we will use if we don't find an exact fit + pgno_t best_fit_size; for (op = MDB_FIRST;; op = MDB_NEXT) { MDB_val key, data; MDB_node *leaf; pgno_t *idl; - + best_fit_start = 0; + best_fit_size = -1; /* Seek a big enough contiguous page range. Prefer * pages at the tail, just truncating the list. */ @@ -2755,56 +2757,52 @@ mdb_page_alloc(MDB_cursor *mc, int num, MDB_page **mp) block_start = 0; unsigned block_size = 0; ssize_t entry; + empty_entries = 0; // TODO: Skip this on the first iteration, since we already checked the cache - if (mop_len > n2) { - i = mop_len; - do { - entry = i == 0 ? 0 : mop[i]; - fprintf(stderr, "pgno %u next would be %u\n", entry, block_start + block_size); - if (entry == 0) continue; - if (entry > 0) { - pgno = entry; - block_size = 1; - } else { - block_size = -entry; - pgno = mop[--i]; + for (i = 1; i <= mop_len; i++) { + entry = mop[i]; + //fprintf(stderr, "pgno %u next would be %u\n", entry, block_start + block_size); + if (entry == 0) { + empty_entries++; + continue; + } + if (entry > 0) { + pgno = entry; + block_size = 1; + } else { + block_size = -entry; + pgno = mop[++i]; + } + + if (block_size >= num) { + if (block_size == num) { + // we found a block of the right size + mop[i] = 0; + if (block_size > 1) mop[i + 1] = 0; + goto search_done; + } else if (block_size < best_fit_size || best_fit_size == 0) { + best_fit_start = i - 1; + best_fit_size = block_size; } - if (pgno == block_start + block_size) { - block_size++; // count current contiguous block size - } else { - if (block_size >= num) { - if (block_size == num) { - // we found a block of the right size - pgno = block_start; - goto search_done; - } else if (block_size < best_fit_size || best_fit_size == 0) { - best_fit_start = block_start; - best_fit_size = block_size; - } - } - if (block_size > 0) { - // cache this block size - if (block_size >= 2<<30) block_size = (2<<30) - 1; - unsigned cache_size = env->me_block_size_cache[0]; - if (block_size > cache_size) { - fprintf(stderr, "expand block size cache to %u\n", block_size << 1); - env->me_block_size_cache = realloc(env->me_block_size_cache, (block_size << 1) * sizeof(pgno_t)); - env->me_block_size_cache[0] = (block_size << 1) - 1; - memset(env->me_block_size_cache + cache_size + 1, 0, (env->me_block_size_cache[0] - cache_size) * sizeof(pgno_t)); - cache_size = env->me_block_size_cache[0]; - } - env->me_block_size_cache[block_size] = block_start; - fprintf(stderr, "cached block %u of size %u\n", block_start, block_size); - } - block_start = pgno; - block_size = 1; + } + if (block_size > 0) { + // cache this block size + if (block_size >= 2<<30) block_size = (2<<30) - 1; + unsigned cache_size = env->me_block_size_cache[0]; + if (block_size > cache_size) { + fprintf(stderr, "expand block size cache to %u\n", block_size << 1); + env->me_block_size_cache = realloc(env->me_block_size_cache, (block_size << 1) * sizeof(pgno_t)); + env->me_block_size_cache[0] = (block_size << 1) - 1; + memset(env->me_block_size_cache + cache_size + 1, 0, (env->me_block_size_cache[0] - cache_size) * sizeof(pgno_t)); + cache_size = env->me_block_size_cache[0]; } - //if (mop[i-n2] == pgno+n2) - // goto search_done; - } while (--i >= 0); - if (--retry < 0) - break; + env->me_block_size_cache[block_size] = pgno; + fprintf(stderr, "cached block %u of size %u\n", pgno, block_size); + } + //if (mop[i-n2] == pgno+n2) + // goto search_done; } + i = 0; if (op == MDB_FIRST) { /* 1st iteration */ /* Prepare to fetch more and coalesce */ @@ -2869,11 +2867,11 @@ mdb_page_alloc(MDB_cursor *mc, int num, MDB_page **mp) rc = ENOMEM; goto fail; } - } else { + } /*else { if ((rc = mdb_midl_need(&env->me_pghead, i)) != 0) goto fail; mop = env->me_pghead; - } + }*/ env->me_pglast = last; #if (MDB_DEBUG) > 1 DPRINTF(("IDL read txn %"Yu" root %"Yu" num %u", @@ -2883,7 +2881,7 @@ mdb_page_alloc(MDB_cursor *mc, int num, MDB_page **mp) #endif /* Merge in descending sorted order */ fprintf(stderr, "merge\n"); - for (unsigned i = i; i < idl[0]; i++) { + for (unsigned i = 1; i <= idl[0]; i++) { if ((rc = mdb_midl_insert(&mop, idl[i])) != 0) goto fail; //mdb_midl_xmerge(mop, idl); @@ -2892,9 +2890,15 @@ mdb_page_alloc(MDB_cursor *mc, int num, MDB_page **mp) mop_len = mop[0]; } if (best_fit_start > 0) { - pgno = best_fit_start; + mop[best_fit_start] += num; // block length is a negative, so we add to it in order to subtract the amount we are using + if (mop[best_fit_start] == -1) mop[best_fit_start] = 0; + pgno = mop[best_fit_start + 1]; + mop[best_fit_start + 1] += num; + env->me_freelist_position = pgno; fprintf(stderr, "using best fit at %u size %u of %u\n", pgno, num, best_fit_size); env->me_block_size_cache[best_fit_size] = 0; // clear this out of the cache (TODO: could move it) + + i = 1; // indicate that we found something goto search_done; } /* Use new pages from the map when nothing suitable in the freeDB */ @@ -2931,10 +2935,13 @@ mdb_page_alloc(MDB_cursor *mc, int num, MDB_page **mp) } } if (i) { - mop[0] = mop_len -= num; - /* Move any stragglers down */ + if (empty_entries > (mop_len >> 1) + 200) { + fprintf(stderr, "should resize\n"); + } +/* mop[0] = mop_len -= num; + /* Move any stragglers down for (j = i-num; j < mop_len; ) - mop[++j] = mop[++i]; + mop[++j] = mop[++i];*/ } else { txn->mt_next_pgno = pgno + num; } diff --git a/dependencies/lmdb/libraries/liblmdb/midl.c b/dependencies/lmdb/libraries/liblmdb/midl.c index c75d344305..e0cc80f3e8 100644 --- a/dependencies/lmdb/libraries/liblmdb/midl.c +++ b/dependencies/lmdb/libraries/liblmdb/midl.c @@ -41,20 +41,32 @@ unsigned mdb_midl_search( MDB_IDL ids, MDB_ID id ) unsigned base = 0; unsigned cursor = 1; int val = 0; - unsigned n = ids[0]; - - while( 0 < n ) { - unsigned pivot = n >> 1; - cursor = base + pivot + 1; - val = CMP( ids[cursor], id ); + unsigned end = ids[0]; + + while( base + 1 < end ) { + cursor = (base + end + 1) >> 1; + ssize_t entry; + while((entry = ids[cursor]) == 0) { + if (++cursor > end) { + // we went past the end, search other direction + cursor = (base + end) >> 1; + while((entry = ids[cursor]) == 0) { + if (--cursor <= base) { + // completely empty section + return (base + end + 1) >> 1; + } + } + } + } + if (entry < 0) entry = ids[cursor + 1]; // block length, skip past and compare actual id + val = CMP( entry, id ); if( val < 0 ) { - n = pivot; - + if (cursor == end) return cursor; + end = cursor; } else if ( val > 0 ) { + if (cursor == base) return cursor + 1; base = cursor; - n -= pivot + 1; - } else { return cursor; } @@ -73,7 +85,7 @@ int mdb_midl_insert( MDB_IDL* ids_ref, MDB_ID id ) unsigned x, i; x = mdb_midl_search( ids, id ); - assert( x > 0 ); + //assert( x > 0 ); if( x < 1 ) { /* internal error */ @@ -82,56 +94,69 @@ int mdb_midl_insert( MDB_IDL* ids_ref, MDB_ID id ) if ( x <= ids[0] && ids[x] == id ) { /* duplicate */ - assert(0); + //assert(0); return -1; } - if ( ++ids[0] >= MDB_IDL_DB_MAX ) { + if ( ids[0] >= MDB_IDL_DB_MAX ) { /* no room */ --ids[0]; return -2; } else { - if (x >= ids[0]) return -3; // at the end - MDB_ID next_id = ids[x]; - if (id < 0) next_id = ids[x + 1]; - if (id - 1 == next_id) { + if (x > ids[0]) return -3; // at the end + ssize_t next_id = ids[x]; + if (next_id < 0) next_id = ids[x + 1]; + if (id - 1 == next_id && next_id > 0) { // connected to next entry ids[x]--; // increment negatively, as we have just expanded a block - ids[x + 1] = id; + // ids[x + 1] = id; // no need to adjust id, so since we are adding to the end of the block return 0; } - unsigned before = x; - while (!ids[--before] && before >= 0){} // move past empty entries - if (before >= 0) { + unsigned before = x; // this will end up pointing to an entry or zero right before a block of empty space + while (!ids[--before] && before > 0) { + // move past empty entries + } + if (before > 0) { MDB_ID next_id = before > 0 ? ids[before] : 0; int count = before > 1 ? -ids[before - 1] : 0; if (count < 1) count = 1; - if (next_id - count == id) { + if (next_id - 1 == id) { // connected to previous entry + ids[before]--; // adjust the starting block to include this if (count > 1) { ids[before - 1]--; // can just update the count to include this id return 0; } else { - // TODO: need to make space for this one + id = -2; // switching a single entry to a block size of 2 + x = before; + goto insert_id; } } } + if (x == 1 && ids[0] > 2 && ids[1] == 0 && ids[2] == 0 && ids[3] == 0) { + // this occurs when we have an empty list + ids[2] = id; + return 0; + } if (before + 1 < x) { // there is an empty slot we can use, find a place in the middle - ids[(before + x) >> 1] = id; - return 0; + ids[before + 3 < x ? (before + 2) : (before + 1)] = id; + i = 0; + goto check_full; } + insert_id: // move items to try to make room - MDB_ID last_id = id; + ssize_t last_id = id; i = x; do { - MDB_ID next_id = ids[i]; + next_id = ids[i]; ids[i++] = last_id; last_id = next_id; } while(next_id); + check_full: if (x == ids[0] || // if it is full - x - i > ids[0] >> 3) { // or too many moves. TODO: This threshold should actually be more like the square root of the length + i > 0 && (i - x > ids[0] >> 3)) { // or too many moves. TODO: This threshold should actually be more like the square root of the length // grow the ids (this will replace the reference too) mdb_midl_need(ids_ref, 1); } @@ -142,10 +167,10 @@ int mdb_midl_insert( MDB_IDL* ids_ref, MDB_ID id ) MDB_IDL mdb_midl_alloc(int num) { - MDB_IDL ids = malloc((num+2) * sizeof(MDB_ID)); + MDB_IDL ids = calloc((num+2), sizeof(MDB_ID)); if (ids) { *ids++ = num; - *ids = 0; + *ids = num; } return ids; } @@ -184,23 +209,29 @@ int mdb_midl_need( MDB_IDL *idp, unsigned num ) MDB_IDL ids = *idp; num += ids[0]; if (num > ids[-1]) { - num = (num + num/4 + (256 + 2)) & -256; + num = (num + num + (256 + 2)) & -256; MDB_IDL new_ids; - if (!(new_ids = alloc(ids-1, num * sizeof(MDB_ID)))) + if (!(new_ids = calloc(num, sizeof(MDB_ID)))) return ENOMEM; *new_ids++ = num - 2; - unsigned j = 0; + *new_ids = num - 2; + unsigned j = 1; // re-spread out the entries with gaps for growth - for (unsigned i = 1; i < ids[0]; i++) { + for (unsigned i = 1; i <= ids[0]; i++) { new_ids[j++] = 0; // empty slot for growth ssize_t entry; - while (!(entry = ids[i])) { i++; } + while (!(entry = ids[i])) { + if (++i > ids[0]) break; + } new_ids[j++] = entry; - if (entry < 0) new_ids[j++] = ids[i++]; // this was a block with a length + if (entry < 0) new_ids[j++] = ids[++i]; // this was a block with a length } // now shrink (or grow) back to appropriate size - new_ids = alloc(new_ids - 1, (j + (j >> 3)) * sizeof(MDB_ID)); - new_ids++; + num = (j + (j >> 3) + 22) & -16; + if (num > new_ids[0]) num = new_ids[0]; + new_ids = realloc(new_ids - 1, (num + 2) * sizeof(MDB_ID)); + *new_ids++ = num; + *new_ids = num; *idp = new_ids; } return 0; From 1ed38a88060a442a8b326790e2b6f43ce3ab9c1b Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Thu, 15 Feb 2024 05:48:07 -0700 Subject: [PATCH 4/7] Add transaction conditional for reseting and tracking last position in freelist --- dependencies/lmdb/libraries/liblmdb/mdb.c | 24 +++++++- dependencies/lmdb/libraries/liblmdb/midl.c | 72 +++++++++++++++++++--- dependencies/lmdb/libraries/liblmdb/midl.h | 3 + 3 files changed, 88 insertions(+), 11 deletions(-) diff --git a/dependencies/lmdb/libraries/liblmdb/mdb.c b/dependencies/lmdb/libraries/liblmdb/mdb.c index 4254223e1e..c808c2412c 100644 --- a/dependencies/lmdb/libraries/liblmdb/mdb.c +++ b/dependencies/lmdb/libraries/liblmdb/mdb.c @@ -2728,7 +2728,7 @@ mdb_page_alloc(MDB_cursor *mc, int num, MDB_page **mp) } unsigned empty_entries = 0; unsigned cache_size = env->me_block_size_cache[0]; - pgno_t best_fit_start; // this is a block we will use if we don't find an exact fit + unsigned best_fit_start; // this is a block we will use if we don't find an exact fit pgno_t best_fit_size; for (op = MDB_FIRST;; op = MDB_NEXT) { MDB_val key, data; @@ -2758,8 +2758,9 @@ mdb_page_alloc(MDB_cursor *mc, int num, MDB_page **mp) unsigned block_size = 0; ssize_t entry; empty_entries = 0; + mdb_midl_print(stderr, mop); // TODO: Skip this on the first iteration, since we already checked the cache - for (i = 1; i <= mop_len; i++) { + for (i = env->me_freelist_position || 1; i <= mop_len; i++) { entry = mop[i]; //fprintf(stderr, "pgno %u next would be %u\n", entry, block_start + block_size); if (entry == 0) { @@ -2783,6 +2784,12 @@ mdb_page_alloc(MDB_cursor *mc, int num, MDB_page **mp) } else if (block_size < best_fit_size || best_fit_size == 0) { best_fit_start = i - 1; best_fit_size = block_size; + if (i == env->me_freelist_position) { + // TODO: Only if we are in the same transaction + // if we just wrote to this block and we are continuing on this block, + // skip ahead to using this block + goto continue_best_fit; + } } } if (block_size > 0) { @@ -2802,6 +2809,7 @@ mdb_page_alloc(MDB_cursor *mc, int num, MDB_page **mp) //if (mop[i-n2] == pgno+n2) // goto search_done; } + env->me_freelist_position = 1; i = 0; if (op == MDB_FIRST) { /* 1st iteration */ @@ -2889,12 +2897,13 @@ mdb_page_alloc(MDB_cursor *mc, int num, MDB_page **mp) if (mop != env->me_pghead) env->me_pghead = mop; mop_len = mop[0]; } + continue_best_fit: if (best_fit_start > 0) { mop[best_fit_start] += num; // block length is a negative, so we add to it in order to subtract the amount we are using if (mop[best_fit_start] == -1) mop[best_fit_start] = 0; pgno = mop[best_fit_start + 1]; mop[best_fit_start + 1] += num; - env->me_freelist_position = pgno; + env->me_freelist_position = best_fit_start; fprintf(stderr, "using best fit at %u size %u of %u\n", pgno, num, best_fit_size); env->me_block_size_cache[best_fit_size] = 0; // clear this out of the cache (TODO: could move it) @@ -3484,6 +3493,15 @@ mdb_txn_renew0(MDB_txn *txn) if (ti) { if (LOCK_MUTEX(rc, env, env->me_wmutex)) return rc; + // TODO: This should really be equality check and mt_txnid should be decremented if nothing was written + if (txn->mt_txnid < ti->mti_txnid) { + if (env->me_pghead) mdb_midl_free(env->me_pghead); + env->me_pghead = NULL; + env->me_pglast = 0; + env->me_freelist_position = 0; + // TODO: Free it first + env->me_block_size_cache = NULL; + } txn->mt_txnid = ti->mti_txnid; meta = env->me_metas[txn->mt_txnid & 1]; } else { diff --git a/dependencies/lmdb/libraries/liblmdb/midl.c b/dependencies/lmdb/libraries/liblmdb/midl.c index e0cc80f3e8..5a85dc9768 100644 --- a/dependencies/lmdb/libraries/liblmdb/midl.c +++ b/dependencies/lmdb/libraries/liblmdb/midl.c @@ -106,22 +106,54 @@ int mdb_midl_insert( MDB_IDL* ids_ref, MDB_ID id ) } else { if (x > ids[0]) return -3; // at the end ssize_t next_id = ids[x]; - if (next_id < 0) next_id = ids[x + 1]; - if (id - 1 == next_id && next_id > 0) { - // connected to next entry - ids[x]--; // increment negatively, as we have just expanded a block - // ids[x + 1] = id; // no need to adjust id, so since we are adding to the end of the block - return 0; + unsigned next_count = 1; + if (next_id < 0) { + next_count = -next_id; + next_id = ids[x + 1]; } unsigned before = x; // this will end up pointing to an entry or zero right before a block of empty space while (!ids[--before] && before > 0) { // move past empty entries } + if (id - next_count == next_id && next_id > 0) { + // connected to next entry + ssize_t count = next_count + 1; + // ids[x + 1] = id; // no need to adjust id, so since we are adding to the end of the block + + if (before > 0) { + MDB_ID previous_id = before > 0 ? ids[before] : 0; + int previous_count = before > 1 ? -ids[before - 1] : 0; + if (previous_count < 1) previous_count = 1; + if (previous_id - 1 == id) { + // the block we just added to can now be connected to previous entry + count += previous_count; + if (previous_count > 1) { + ids[before - 1] = 0; // remove previous length + } + ids[before] = 0; // remove previous id + if (next_count == 1) { + // we can safely add the new count to the empty space + ids[x - 1] = -count; // update the count + return 0; + } + } + } + if (next_count > 1) { + ids[x] = -count; // update the count + } else if (ids[x - 1] == 0) { + ids[x - 1] = -2; + } else { + id = -2; // switching a single entry to a block size of 2 + x--; + goto insert_id; + } + return 0; + } if (before > 0) { - MDB_ID next_id = before > 0 ? ids[before] : 0; + MDB_ID previous_id = before > 0 ? ids[before] : 0; int count = before > 1 ? -ids[before - 1] : 0; if (count < 1) count = 1; - if (next_id - 1 == id) { + if (previous_id - 1 == id) { // connected to previous entry ids[before]--; // adjust the starting block to include this if (count > 1) { @@ -213,6 +245,7 @@ int mdb_midl_need( MDB_IDL *idp, unsigned num ) MDB_IDL new_ids; if (!(new_ids = calloc(num, sizeof(MDB_ID)))) return ENOMEM; + fprintf(stderr, "Created new id list of size %u\n", num); *new_ids++ = num - 2; *new_ids = num - 2; unsigned j = 1; @@ -226,6 +259,7 @@ int mdb_midl_need( MDB_IDL *idp, unsigned num ) new_ids[j++] = entry; if (entry < 0) new_ids[j++] = ids[++i]; // this was a block with a length } + mdb_midl_free(ids); // now shrink (or grow) back to appropriate size num = (j + (j >> 3) + 22) & -16; if (num > new_ids[0]) num = new_ids[0]; @@ -237,6 +271,28 @@ int mdb_midl_need( MDB_IDL *idp, unsigned num ) return 0; } +int mdb_midl_print( FILE *fp, MDB_IDL ids ) +{ + if (ids == NULL) { + fprintf(fp, "freelist: NULL\n"); + return 0; + } + unsigned i; + fprintf(fp, "freelist: %u: ", ids[0]); + for (i=1; i<=ids[0]; i++) { + ssize_t entry = ids[i]; + if (entry < 0) { + fprintf(fp, "%li-%li ", ids[i+1], ids[i+1] - entry - 1); + i++; + } else if (ids[i] == 0) { + fprintf(fp, "_"); + } else { + fprintf(fp, "%lu ", (unsigned long)ids[i]); + } + } + fprintf(fp, "\n"); + return 0; +} int mdb_midl_append( MDB_IDL *idp, MDB_ID id ) { MDB_IDL ids = *idp; diff --git a/dependencies/lmdb/libraries/liblmdb/midl.h b/dependencies/lmdb/libraries/liblmdb/midl.h index bf3df89f3a..3823dd4998 100644 --- a/dependencies/lmdb/libraries/liblmdb/midl.h +++ b/dependencies/lmdb/libraries/liblmdb/midl.h @@ -28,6 +28,8 @@ #define _MDB_MIDL_H_ #include "lmdb.h" +#include + #ifdef __cplusplus extern "C" { @@ -109,6 +111,7 @@ void mdb_midl_shrink(MDB_IDL *idp); */ int mdb_midl_need(MDB_IDL *idp, unsigned num); +int mdb_midl_print( FILE *fp, MDB_IDL ids ); /** Insert an ID into an IDL. * @param[in,out] idp Address of the IDL to append to. * @param[in] id The ID to append. From abd16289488b0786cecaff346f3b611f848deb35 Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Thu, 15 Feb 2024 21:56:54 -0700 Subject: [PATCH 5/7] More fixes for freespace handling --- dependencies/lmdb/libraries/liblmdb/mdb.c | 55 +++++++++------ dependencies/lmdb/libraries/liblmdb/midl.c | 81 +++++++++++++++++++++- dependencies/lmdb/libraries/liblmdb/midl.h | 3 + test/index.test.js | 28 +++++++- 4 files changed, 144 insertions(+), 23 deletions(-) diff --git a/dependencies/lmdb/libraries/liblmdb/mdb.c b/dependencies/lmdb/libraries/liblmdb/mdb.c index c808c2412c..245d9ddb3a 100644 --- a/dependencies/lmdb/libraries/liblmdb/mdb.c +++ b/dependencies/lmdb/libraries/liblmdb/mdb.c @@ -2741,7 +2741,7 @@ mdb_page_alloc(MDB_cursor *mc, int num, MDB_page **mp) */ pgno_t block_start; fprintf(stderr, "looking for block of size %u\n", num); - if (cache_size > num) { + /*if (cache_size > num) { block_start = env->me_block_size_cache[num]; if (block_start > 0) { fprintf(stderr, "found block %u of right size %u\n", block_start, num); @@ -2753,7 +2753,7 @@ mdb_page_alloc(MDB_cursor *mc, int num, MDB_page **mp) goto search_done; } } - } + }*/ block_start = 0; unsigned block_size = 0; ssize_t entry; @@ -2779,7 +2779,7 @@ mdb_page_alloc(MDB_cursor *mc, int num, MDB_page **mp) if (block_size == num) { // we found a block of the right size mop[i] = 0; - if (block_size > 1) mop[i + 1] = 0; + if (block_size > 1) mop[i - 1] = 0; goto search_done; } else if (block_size < best_fit_size || best_fit_size == 0) { best_fit_start = i - 1; @@ -2791,7 +2791,7 @@ mdb_page_alloc(MDB_cursor *mc, int num, MDB_page **mp) goto continue_best_fit; } } - } + }/* if (block_size > 0) { // cache this block size if (block_size >= 2<<30) block_size = (2<<30) - 1; @@ -2805,7 +2805,7 @@ mdb_page_alloc(MDB_cursor *mc, int num, MDB_page **mp) } env->me_block_size_cache[block_size] = pgno; fprintf(stderr, "cached block %u of size %u\n", pgno, block_size); - } + }*/ //if (mop[i-n2] == pgno+n2) // goto search_done; } @@ -2888,9 +2888,13 @@ mdb_page_alloc(MDB_cursor *mc, int num, MDB_page **mp) DPRINTF(("IDL %"Yu, idl[j])); #endif /* Merge in descending sorted order */ - fprintf(stderr, "merge\n"); + fprintf(stderr, "merge from %u\n", last); for (unsigned i = 1; i <= idl[0]; i++) { - if ((rc = mdb_midl_insert(&mop, idl[i])) != 0) + ssize_t entry = idl[i]; + if (entry < 0) { + fprintf(stderr, "Block length entry\n"); + } + if ((rc = mdb_midl_insert(&mop, entry)) != 0) goto fail; //mdb_midl_xmerge(mop, idl); } @@ -4046,11 +4050,14 @@ mdb_freelist_save(MDB_txn *txn) if (rc) return rc; pglast = head_id = *(txnid_t *)key.mv_data; + fprintf(stderr, "deleting freespace page %u\n", head_id); total_room = head_room = 0; - mdb_tassert(txn, pglast <= env->me_pglast); - rc = mdb_cursor_del(&mc, 0); - if (rc) - return rc; + if (pglast <= env->me_pglast) { + mdb_tassert(txn, pglast <= env->me_pglast); + rc = mdb_cursor_del(&mc, 0); + if (rc) + return rc; + } } /* Save the IDL of pages freed by this txn, to a single record */ @@ -4063,6 +4070,8 @@ mdb_freelist_save(MDB_txn *txn) } free_pgs = txn->mt_free_pgs; /* Write to last page of freeDB */ + fprintf(stderr, "write freed pages %u\n", txn->mt_txnid); + mdb_midl_print(stderr, free_pgs); key.mv_size = sizeof(txn->mt_txnid); key.mv_data = &txn->mt_txnid; do { @@ -4089,7 +4098,7 @@ mdb_freelist_save(MDB_txn *txn) } mop = env->me_pghead; - mop_len = (mop ? mop[0] : 0) + txn->mt_loose_count; + mop_len = (mop ? mdb_midl_pack_count(mop) : 0) + txn->mt_loose_count; /* Reserve records for me_pghead[]. Split it if multi-page, * to avoid searching freeDB for a page range. Use keys in @@ -4135,27 +4144,31 @@ mdb_freelist_save(MDB_txn *txn) if (txn->mt_loose_pgs) { MDB_page *mp = txn->mt_loose_pgs; unsigned count = txn->mt_loose_count; - MDB_IDL loose; - /* Room for loose pages + temp IDL with same */ + /*MDB_IDL loose; + // Room for loose pages + temp IDL with same if ((rc = mdb_midl_need(&env->me_pghead, 2*count+1)) != 0) - return rc; + return rc;*/ lost_loose += count; mop = env->me_pghead; - loose = mop + MDB_IDL_ALLOCLEN(mop) - count; + //loose = mop + MDB_IDL_ALLOCLEN(mop) - count; for (count = 0; mp; mp = NEXT_LOOSE_PAGE(mp)) - loose[ ++count ] = mp->mp_pgno; + mdb_midl_insert(&mop, mp->mp_pgno); + /*loose[ ++count ] = mp->mp_pgno; loose[0] = count; mdb_midl_sort(loose); - mdb_midl_xmerge(mop, loose); + mdb_midl_xmerge(mop, loose);*/ + env->me_pghead = mop; txn->mt_loose_pgs = NULL; txn->mt_loose_count = 0; - mop_len = mop[0]; + //mop_len = mop[0]; } /* Fill in the reserved me_pghead records. Everything is finally * in place, so this will not allocate or free any DB pages. */ rc = MDB_SUCCESS; + mop = mdb_midl_pack(mop); + mop_len = mop ? mop[0] : 0; if (mop_len) { MDB_val key, data; @@ -4167,6 +4180,7 @@ mdb_freelist_save(MDB_txn *txn) rc = mdb_cursor_first(&mc, &key, &data); for (; !rc; rc = mdb_cursor_next(&mc, &key, &data, MDB_NEXT)) { txnid_t id = *(txnid_t *)key.mv_data; + fprintf(stderr, "finishing save of freespace %u\n", id); ssize_t len = (ssize_t)(data.mv_size / sizeof(MDB_ID)) - 1; MDB_ID save; @@ -4185,8 +4199,9 @@ mdb_freelist_save(MDB_txn *txn) if (rc || !mop_len) break; } + mdb_midl_free(mop); - env->me_pghead = mop - mop_len; + //env->me_pghead = mop - mop_len; } /* Restore this so we can check vs. dirty_list after mdb_page_flush() */ diff --git a/dependencies/lmdb/libraries/liblmdb/midl.c b/dependencies/lmdb/libraries/liblmdb/midl.c index 5a85dc9768..406ed402df 100644 --- a/dependencies/lmdb/libraries/liblmdb/midl.c +++ b/dependencies/lmdb/libraries/liblmdb/midl.c @@ -38,6 +38,56 @@ unsigned mdb_midl_search( MDB_IDL ids, MDB_ID id ) * if found, returns position of id * if not found, returns first position greater than id */ + unsigned base = 0; + unsigned cursor = 1; + int val = 0; + unsigned n = ids[0]; + unsigned end = n; + + binary_search: + while( 0 < n ) { + unsigned pivot = n >> 1; + cursor = base + pivot + 1; + unsigned x = cursor; + // skip past empty and block length entries + while(((ssize_t)ids[x]) <= 0) { + if (++x > end) { // reached the end, go to lower half + n = pivot; + val = 0; + end -= pivot; + goto binary_search; + } + } + val = CMP( ids[cursor], id ); + + if( val < 0 ) { + n = pivot; + end -= pivot; + } else if ( val > 0 ) { + base = cursor; + n -= pivot + 1; + } else { + return cursor; + } + } + if( val > 0 ) { + unsigned next = ++cursor; + end = ids[0]; + // skip past empty and block length entries + while(((ssize_t)ids[cursor]) <= 0) { + if (++cursor > end) { // reached the end + return next; // go back to the one after the last value in this case + } + } + } + return cursor; + /* + * binary search of id in ids + * if found, returns position of id + * if not found, returns first position greater than id + * + + unsigned base = 0; unsigned cursor = 1; int val = 0; @@ -62,6 +112,7 @@ unsigned mdb_midl_search( MDB_IDL ids, MDB_ID id ) val = CMP( entry, id ); if( val < 0 ) { + while(!ids[cursor - 1]) cursor--; if (cursor == end) return cursor; end = cursor; } else if ( val > 0 ) { @@ -75,7 +126,7 @@ unsigned mdb_midl_search( MDB_IDL ids, MDB_ID id ) if( val > 0 ) { ++cursor; } - return cursor; + return cursor;*/ } /* superseded by append/sort */ @@ -144,7 +195,6 @@ int mdb_midl_insert( MDB_IDL* ids_ref, MDB_ID id ) ids[x - 1] = -2; } else { id = -2; // switching a single entry to a block size of 2 - x--; goto insert_id; } return 0; @@ -236,6 +286,33 @@ static int mdb_midl_grow( MDB_IDL *idp, int num ) return 0; } +MDB_IDL mdb_midl_pack(MDB_IDL idl) { + if (!idl) return NULL; + MDB_IDL packed = mdb_midl_alloc(idl[0]); + unsigned j = 1; + for (unsigned i = 1; i < idl[0]; i++) { + ssize_t entry = idl[i]; + if (entry) packed[j++] = entry; + } + if (j == 1) { + // empty list, just treat as no list + mdb_midl_free(packed); + return NULL; + } + packed[0] = j - 1; + return packed; +} + +unsigned mdb_midl_pack_count(MDB_IDL idl) { + unsigned count = 0; + if (idl) { + for (unsigned i = 1; i < idl[0]; i++) { + if (idl[i]) count++; + } + } + return count; +} + int mdb_midl_need( MDB_IDL *idp, unsigned num ) { MDB_IDL ids = *idp; diff --git a/dependencies/lmdb/libraries/liblmdb/midl.h b/dependencies/lmdb/libraries/liblmdb/midl.h index 3823dd4998..2e1bff3fcd 100644 --- a/dependencies/lmdb/libraries/liblmdb/midl.h +++ b/dependencies/lmdb/libraries/liblmdb/midl.h @@ -112,6 +112,9 @@ void mdb_midl_shrink(MDB_IDL *idp); int mdb_midl_need(MDB_IDL *idp, unsigned num); int mdb_midl_print( FILE *fp, MDB_IDL ids ); +MDB_IDL mdb_midl_pack(MDB_IDL idl); +unsigned mdb_midl_pack_count(MDB_IDL idl); + /** Insert an ID into an IDL. * @param[in,out] idp Address of the IDL to append to. * @param[in] id The ID to append. diff --git a/test/index.test.js b/test/index.test.js index f2ac8669b9..02413e578a 100644 --- a/test/index.test.js +++ b/test/index.test.js @@ -59,7 +59,7 @@ describe('lmdb-js', function () { }); }); let testIteration = 0; - describe('Basic use', basicTests({})); + describe.skip('Basic use', basicTests({})); describe( 'Basic use with overlapping sync', basicTests({ @@ -245,6 +245,32 @@ describe('lmdb-js', function () { '0Sdts8FwTqt2Hv5j9KE7ebjsQcFbYDdL/0Sdtsud6g8YGhPwUK04fRVKhuTywhnx8', ]); }); + it.only('bigger puts, testing free space management', async function () { + let seed = 15325223; + function random() { + return randomInt() / 2038074743; + } + function randomInt() { + seed++; + let a = seed * 15485863; + return a * a * a % 2038074743; + } + + let promise; + let additive = 'this is more text'; + for (let i = 0; i < 7; i++) additive += additive; + for (let i = 0; i < 1000; i++) { + let text = 'this is a test'; + while(random() < 0.95) text += additive; + console.log('write', i, text.length); + promise = db.put(i % 10, text); + if (i % 6 == 0) { + await promise; + } + } + await promise; + }); + it('clear between puts', async function () { db.put('key0', 'zero'); db.clearAsync(); From 2e6069a17c80688bfc6f01f2d6eda1c6aa3236a5 Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Thu, 15 Feb 2024 23:54:22 -0700 Subject: [PATCH 6/7] More fixes --- .../lmdb-data-v1/libraries/liblmdb/midl.c | 4 +- dependencies/lmdb/libraries/liblmdb/mdb.c | 5 ++- dependencies/lmdb/libraries/liblmdb/midl.c | 43 +++++++++---------- test/index.test.js | 41 +++++++++--------- 4 files changed, 46 insertions(+), 47 deletions(-) diff --git a/dependencies/lmdb-data-v1/libraries/liblmdb/midl.c b/dependencies/lmdb-data-v1/libraries/liblmdb/midl.c index b0ea5383b2..24fe397390 100644 --- a/dependencies/lmdb-data-v1/libraries/liblmdb/midl.c +++ b/dependencies/lmdb-data-v1/libraries/liblmdb/midl.c @@ -113,8 +113,8 @@ MDB_IDL mdb_midl_alloc(int num) void mdb_midl_free(MDB_IDL ids) { - if (ids) - free(ids-1); +// if (ids) +// free(ids-1); } void mdb_midl_shrink( MDB_IDL *idp ) diff --git a/dependencies/lmdb/libraries/liblmdb/mdb.c b/dependencies/lmdb/libraries/liblmdb/mdb.c index 245d9ddb3a..c9ab76d2fa 100644 --- a/dependencies/lmdb/libraries/liblmdb/mdb.c +++ b/dependencies/lmdb/libraries/liblmdb/mdb.c @@ -4173,6 +4173,7 @@ mdb_freelist_save(MDB_txn *txn) MDB_val key, data; /* Protect DB env from any (buggy) freelist use when saving mop */ + MDB_IDL pghead = env->me_pghead; env->me_pghead = NULL; txn->mt_dirty_room = 0; @@ -4199,9 +4200,9 @@ mdb_freelist_save(MDB_txn *txn) if (rc || !mop_len) break; } - mdb_midl_free(mop); + //mdb_midl_free(mop); - //env->me_pghead = mop - mop_len; + env->me_pghead = pghead; } /* Restore this so we can check vs. dirty_list after mdb_page_flush() */ diff --git a/dependencies/lmdb/libraries/liblmdb/midl.c b/dependencies/lmdb/libraries/liblmdb/midl.c index 406ed402df..427f65b929 100644 --- a/dependencies/lmdb/libraries/liblmdb/midl.c +++ b/dependencies/lmdb/libraries/liblmdb/midl.c @@ -54,15 +54,15 @@ unsigned mdb_midl_search( MDB_IDL ids, MDB_ID id ) if (++x > end) { // reached the end, go to lower half n = pivot; val = 0; - end -= pivot; + end = cursor; goto binary_search; } } - val = CMP( ids[cursor], id ); + val = CMP( ids[x], id ); if( val < 0 ) { n = pivot; - end -= pivot; + end = x; } else if ( val > 0 ) { base = cursor; n -= pivot + 1; @@ -70,14 +70,14 @@ unsigned mdb_midl_search( MDB_IDL ids, MDB_ID id ) return cursor; } } - if( val > 0 ) { - unsigned next = ++cursor; - end = ids[0]; - // skip past empty and block length entries - while(((ssize_t)ids[cursor]) <= 0) { - if (++cursor > end) { // reached the end - return next; // go back to the one after the last value in this case - } + unsigned next; + if( val > 0 ) ++cursor; + unsigned x = cursor; + end = ids[0]; + // skip past empty and block length entries + while(((ssize_t)ids[cursor]) <= 0) { + if (++cursor > end) { // reached the end + return x; // go back to the one after the last value in this case } } return cursor; @@ -157,14 +157,12 @@ int mdb_midl_insert( MDB_IDL* ids_ref, MDB_ID id ) } else { if (x > ids[0]) return -3; // at the end ssize_t next_id = ids[x]; - unsigned next_count = 1; - if (next_id < 0) { - next_count = -next_id; - next_id = ids[x + 1]; - } + ssize_t next_count = ids[x - 1]; + if (next_count < 0) next_count = -next_count; + else next_count = 1; unsigned before = x; // this will end up pointing to an entry or zero right before a block of empty space - while (!ids[--before] && before > 0) { - // move past empty entries + while ((ssize_t)ids[--before] <= 0 && before > 0) { + // move past empty entries (and the length entry) } if (id - next_count == next_id && next_id > 0) { // connected to next entry @@ -190,7 +188,7 @@ int mdb_midl_insert( MDB_IDL* ids_ref, MDB_ID id ) } } if (next_count > 1) { - ids[x] = -count; // update the count + ids[x - 1] = -count; // update the count } else if (ids[x - 1] == 0) { ids[x - 1] = -2; } else { @@ -221,7 +219,7 @@ int mdb_midl_insert( MDB_IDL* ids_ref, MDB_ID id ) ids[2] = id; return 0; } - if (before + 1 < x) { + if (!ids[before + 1]) { // there is an empty slot we can use, find a place in the middle ids[before + 3 < x ? (before + 2) : (before + 1)] = id; i = 0; @@ -231,6 +229,7 @@ int mdb_midl_insert( MDB_IDL* ids_ref, MDB_ID id ) // move items to try to make room ssize_t last_id = id; i = x; + if ((ssize_t)ids[i - 1] < 0) i--; do { next_id = ids[i]; ids[i++] = last_id; @@ -259,8 +258,8 @@ MDB_IDL mdb_midl_alloc(int num) void mdb_midl_free(MDB_IDL ids) { - if (ids) - free(ids-1); + //if (ids) + // free(ids-1); } void mdb_midl_shrink( MDB_IDL *idp ) diff --git a/test/index.test.js b/test/index.test.js index 02413e578a..1cbda2ab64 100644 --- a/test/index.test.js +++ b/test/index.test.js @@ -1,34 +1,33 @@ import chai from 'chai'; -import path, { dirname } from 'path'; +import path, {dirname} from 'path'; import rimraf from 'rimraf'; - -let should = chai.should(); -let expect = chai.expect; -import { spawn } from 'child_process'; -import { unlinkSync } from 'fs' -import { fileURLToPath } from 'url' -import { Worker } from 'worker_threads'; -import { encoder as orderedBinaryEncoder } from 'ordered-binary/index.js' -import inspector from 'inspector' -//inspector.open(9229, null, true); debugger -let nativeMethods, - dirName = dirname(fileURLToPath(import.meta.url)); - -import { createRequire } from 'module'; -import { createBufferForAddress, fs } from '../native.js'; +import {spawn} from 'child_process'; +import {unlinkSync} from 'fs' +import {fileURLToPath} from 'url' +import {Worker} from 'worker_threads'; +import {encoder as orderedBinaryEncoder} from 'ordered-binary/index.js' +import {createRequire} from 'module'; +import {createBufferForAddress} from '../native.js'; import { ABORT, - IF_EXISTS, asBinary, bufferToKeyValue, + DIRECT_WRITE_PLACEHOLDER, + IF_EXISTS, keyValueToBuffer, levelup, open, - TIMESTAMP_PLACEHOLDER, - DIRECT_WRITE_PLACEHOLDER + TIMESTAMP_PLACEHOLDER } from '../node-index.js'; -import { openAsClass } from '../open.js'; -import { RangeIterable } from '../util/RangeIterable.js'; +import {openAsClass} from '../open.js'; +import {RangeIterable} from '../util/RangeIterable.js'; + +let should = chai.should(); +let expect = chai.expect; +//inspector.open(9229, null, true); debugger +let nativeMethods, + dirName = dirname(fileURLToPath(import.meta.url)); + const require = createRequire(import.meta.url); // we don't always test CJS because it messes up debugging in webstorm (and I am not about to give the awesomeness // that is webstorm debugging) From 10b44dee6918797aa677c41dec4aa29d0985c785 Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Sat, 17 Feb 2024 22:45:31 -0700 Subject: [PATCH 7/7] More debugging segfault --- dependencies/lmdb/libraries/liblmdb/mdb.c | 33 ++-- dependencies/lmdb/libraries/liblmdb/midl.c | 208 ++++++++++++++------- dependencies/lmdb/libraries/liblmdb/midl.h | 4 +- test/index.test.js | 3 +- 4 files changed, 159 insertions(+), 89 deletions(-) diff --git a/dependencies/lmdb/libraries/liblmdb/mdb.c b/dependencies/lmdb/libraries/liblmdb/mdb.c index c9ab76d2fa..cb3784f52a 100644 --- a/dependencies/lmdb/libraries/liblmdb/mdb.c +++ b/dependencies/lmdb/libraries/liblmdb/mdb.c @@ -2585,7 +2585,7 @@ mdb_page_spill(MDB_cursor *m0, MDB_val *key, MDB_val *data) goto done; need--; } - mdb_midl_sort(txn->mt_spill_pgs); + //mdb_midl_sort(txn->mt_spill_pgs); /* Flush the spilled part of dirty list */ if ((rc = mdb_page_flush(txn, i)) != MDB_SUCCESS) @@ -2740,7 +2740,7 @@ mdb_page_alloc(MDB_cursor *mc, int num, MDB_page **mp) * pages at the tail, just truncating the list. */ pgno_t block_start; - fprintf(stderr, "looking for block of size %u\n", num); + //fprintf(stderr, "looking for block of size %u\n", num); /*if (cache_size > num) { block_start = env->me_block_size_cache[num]; if (block_start > 0) { @@ -2889,15 +2889,8 @@ mdb_page_alloc(MDB_cursor *mc, int num, MDB_page **mp) #endif /* Merge in descending sorted order */ fprintf(stderr, "merge from %u\n", last); - for (unsigned i = 1; i <= idl[0]; i++) { - ssize_t entry = idl[i]; - if (entry < 0) { - fprintf(stderr, "Block length entry\n"); - } - if ((rc = mdb_midl_insert(&mop, entry)) != 0) - goto fail; - //mdb_midl_xmerge(mop, idl); - } + if ((rc = mdb_midl_append_list(&mop, idl)) != 0) + goto fail; if (mop != env->me_pghead) env->me_pghead = mop; mop_len = mop[0]; } @@ -3095,14 +3088,15 @@ mdb_page_touch(MDB_cursor *mc) if (!IS_MUTABLE(txn, mp)) { /* Page from an older snapshot */ - if ((rc = mdb_midl_need(&txn->mt_free_pgs, 1)) || + if ((rc = mdb_midl_insert(&txn->mt_free_pgs, mp->mp_pgno, 1)) || (rc = mdb_page_alloc(mc, 1, &np))) goto fail; + fprintf(stderr, "Freed page %u :", mp->mp_pgno); + mdb_midl_print(stderr, txn->mt_free_pgs); pgno = np->mp_pgno; DPRINTF(("touched db %d page %"Yu" -> %"Yu, DDBI(mc), mp->mp_pgno, pgno)); mdb_cassert(mc, mp->mp_pgno != pgno); - mdb_midl_xappend(txn->mt_free_pgs, mp->mp_pgno); /* Update the parent page, if any, to point to the new page */ if (mc->mc_top) { MDB_page *parent = mc->mc_pg[mc->mc_top-1]; @@ -3995,10 +3989,9 @@ mdb_freelist_save(MDB_txn *txn) MDB_page *mp = txn->mt_loose_pgs; MDB_ID2 *dl = txn->mt_u.dirty_list; unsigned x; - if ((rc = mdb_midl_need(&txn->mt_free_pgs, txn->mt_loose_count)) != 0) - return rc; for (; mp; mp = NEXT_LOOSE_PAGE(mp)) { - mdb_midl_xappend(txn->mt_free_pgs, mp->mp_pgno); + if ((rc = mdb_midl_insert(&txn->mt_free_pgs, mp->mp_pgno, 1)) != 0) + return rc; /* must also remove from dirty list */ if (txn->mt_flags & MDB_TXN_WRITEMAP) { for (x=1; x<=dl[0].mid; x++) @@ -4083,7 +4076,7 @@ mdb_freelist_save(MDB_txn *txn) /* Retry if mt_free_pgs[] grew during the Put() */ free_pgs = txn->mt_free_pgs; } while (freecnt < free_pgs[0]); - mdb_midl_sort(free_pgs); + //mdb_midl_sort(free_pgs); memcpy(data.mv_data, free_pgs, data.mv_size); #if (MDB_DEBUG) > 1 { @@ -4152,7 +4145,7 @@ mdb_freelist_save(MDB_txn *txn) mop = env->me_pghead; //loose = mop + MDB_IDL_ALLOCLEN(mop) - count; for (count = 0; mp; mp = NEXT_LOOSE_PAGE(mp)) - mdb_midl_insert(&mop, mp->mp_pgno); + mdb_midl_insert(&mop, mp->mp_pgno, 1); /*loose[ ++count ] = mp->mp_pgno; loose[0] = count; mdb_midl_sort(loose); @@ -7941,7 +7934,7 @@ mdb_ovpage_free(MDB_cursor *mc, MDB_page *mp) mop[j--] = pg++; mop[0] += ovpages; } else { - rc = mdb_midl_append_range(&txn->mt_free_pgs, pg, ovpages); + rc = mdb_midl_insert(&txn->mt_free_pgs, pg, ovpages); if (rc) return rc; } @@ -12340,7 +12333,7 @@ mdb_drop0(MDB_cursor *mc, int subs) if (ni->mn_flags & F_BIGDATA) { MDB_ovpage ovp; memcpy(&ovp, NODEDATA(ni), sizeof(ovp)); - rc = mdb_midl_append_range(&txn->mt_free_pgs, + rc = mdb_midl_insert(&txn->mt_free_pgs, ovp.op_pgno, ovp.op_pages); if (rc) goto done; diff --git a/dependencies/lmdb/libraries/liblmdb/midl.c b/dependencies/lmdb/libraries/liblmdb/midl.c index 427f65b929..5db6a862dd 100644 --- a/dependencies/lmdb/libraries/liblmdb/midl.c +++ b/dependencies/lmdb/libraries/liblmdb/midl.c @@ -62,7 +62,7 @@ unsigned mdb_midl_search( MDB_IDL ids, MDB_ID id ) if( val < 0 ) { n = pivot; - end = x; + end = cursor; } else if ( val > 0 ) { base = cursor; n -= pivot + 1; @@ -71,15 +71,15 @@ unsigned mdb_midl_search( MDB_IDL ids, MDB_ID id ) } } unsigned next; - if( val > 0 ) ++cursor; - unsigned x = cursor; + if( val > 0 && (ssize_t)ids[cursor] > 0) ++cursor; + /*unsigned x = cursor; end = ids[0]; // skip past empty and block length entries while(((ssize_t)ids[cursor]) <= 0) { if (++cursor > end) { // reached the end return x; // go back to the one after the last value in this case } - } + }*/ return cursor; /* * binary search of id in ids @@ -130,11 +130,11 @@ unsigned mdb_midl_search( MDB_IDL ids, MDB_ID id ) } /* superseded by append/sort */ -int mdb_midl_insert( MDB_IDL* ids_ref, MDB_ID id ) +int mdb_midl_insert( MDB_IDL* ids_ref, MDB_ID id, int insertion_count ) { MDB_IDL ids = *ids_ref; unsigned x, i; - + int rc; x = mdb_midl_search( ids, id ); //assert( x > 0 ); @@ -155,25 +155,49 @@ int mdb_midl_insert( MDB_IDL* ids_ref, MDB_ID id ) return -2; } else { - if (x > ids[0]) return -3; // at the end - ssize_t next_id = ids[x]; - ssize_t next_count = ids[x - 1]; - if (next_count < 0) next_count = -next_count; - else next_count = 1; + if (x > ids[0]) { + // need to grow + if ((rc = mdb_midl_need(ids_ref, insertion_count > 1 ? 2 : 1)) != 0) + return rc; + ids = *ids_ref; + if (insertion_count == 1) { + ids[x] = id; + ids[0] = x; + } else { + ids[x] = -insertion_count; + ids[x + 1] = id; + ids[0] = x + 1; + } + + return 0; + } unsigned before = x; // this will end up pointing to an entry or zero right before a block of empty space while ((ssize_t)ids[--before] <= 0 && before > 0) { // move past empty entries (and the length entry) } - if (id - next_count == next_id && next_id > 0) { + while ((ssize_t)ids[x] <= 0 && x < ids[0]) { x++;} + ssize_t next_id = ids[x]; + ssize_t next_count = ids[x - 1]; + if (next_count < 0) next_count = -next_count; + else next_count = 1; + if (id - next_count <= next_id && next_id > 0) { + if (id - next_count < next_id) { + fprintf(stderr, "overlapping duplicate entry"); + return -1; + } // connected to next entry - ssize_t count = next_count + 1; + ssize_t count = next_count + insertion_count; // ids[x + 1] = id; // no need to adjust id, so since we are adding to the end of the block if (before > 0) { MDB_ID previous_id = before > 0 ? ids[before] : 0; int previous_count = before > 1 ? -ids[before - 1] : 0; if (previous_count < 1) previous_count = 1; - if (previous_id - 1 == id) { + if (previous_id - insertion_count <= id) { + if (previous_id - insertion_count < id) { + fprintf(stderr, "overlapping duplicate entry"); + return -1; + } // the block we just added to can now be connected to previous entry count += previous_count; if (previous_count > 1) { @@ -190,9 +214,9 @@ int mdb_midl_insert( MDB_IDL* ids_ref, MDB_ID id ) if (next_count > 1) { ids[x - 1] = -count; // update the count } else if (ids[x - 1] == 0) { - ids[x - 1] = -2; + ids[x - 1] = -1 - insertion_count; // we can switch to length-2 block in place } else { - id = -2; // switching a single entry to a block size of 2 + id = -1 - insertion_count; // switching a single entry to a block size of 2 goto insert_id; } return 0; @@ -201,14 +225,18 @@ int mdb_midl_insert( MDB_IDL* ids_ref, MDB_ID id ) MDB_ID previous_id = before > 0 ? ids[before] : 0; int count = before > 1 ? -ids[before - 1] : 0; if (count < 1) count = 1; - if (previous_id - 1 == id) { + if (previous_id - insertion_count <= id) { + if (previous_id - insertion_count < id) { + fprintf(stderr, "overlapping duplicate entry"); + return -1; + } // connected to previous entry - ids[before]--; // adjust the starting block to include this + ids[before] = id; // adjust the starting block to include this if (count > 1) { - ids[before - 1]--; // can just update the count to include this id + ids[before - 1] -= insertion_count; // can just update the count to include this id return 0; } else { - id = -2; // switching a single entry to a block size of 2 + id = -1 - insertion_count; // switching a single entry to a block size of 2 x = before; goto insert_id; } @@ -216,30 +244,51 @@ int mdb_midl_insert( MDB_IDL* ids_ref, MDB_ID id ) } if (x == 1 && ids[0] > 2 && ids[1] == 0 && ids[2] == 0 && ids[3] == 0) { // this occurs when we have an empty list - ids[2] = id; + + if (insertion_count > 1) { + ids[2] = -insertion_count; + ids[3] = id; + } else + ids[2] = id; return 0; } if (!ids[before + 1]) { // there is an empty slot we can use, find a place in the middle - ids[before + 3 < x ? (before + 2) : (before + 1)] = id; - i = 0; - goto check_full; + i = before + 3 < x ? (before + 2) : (before + 1); + if (i >= ids[0]) { + mdb_midl_need(ids_ref, 1); + ids = *ids_ref; + ids[0] = i; + } + ids[i] = id; + if (insertion_count == 1) + return 0; // done + // else insert the length + x = i; + id = -insertion_count; } insert_id: // move items to try to make room ssize_t last_id = id; - i = x; - if ((ssize_t)ids[i - 1] < 0) i--; + if ((ssize_t)ids[x - 1] < 0) x--; do { - next_id = ids[i]; - ids[i++] = last_id; - last_id = next_id; - } while(next_id); - check_full: - if (x == ids[0] || // if it is full - i > 0 && (i - x > ids[0] >> 3)) { // or too many moves. TODO: This threshold should actually be more like the square root of the length - // grow the ids (this will replace the reference too) - mdb_midl_need(ids_ref, 1); + i = x; + do { + next_id = ids[i]; + ids[i++] = last_id; + if (i > ids[0]) { // it is full, need to expand + mdb_midl_need(ids_ref, 1); + ids = *ids_ref; + ids[0] = i; + ids[i] = next_id; + next_id = 0; // break out; + } + last_id = next_id; + } while(next_id); + } while ((ssize_t) id > 0 && insertion_count > 1 && (id = last_id = -insertion_count)); + if (i > 0 && ((int) i - x > (ids[0] >> 2) + 4)) { // or too many moves. TODO: This threshold should actually be more like the square root of the length + // respread the ids (this will replace the reference too) + mdb_midl_respread(ids_ref); } } @@ -248,10 +297,11 @@ int mdb_midl_insert( MDB_IDL* ids_ref, MDB_ID id ) MDB_IDL mdb_midl_alloc(int num) { + // TODO: SHould be malloc MDB_IDL ids = calloc((num+2), sizeof(MDB_ID)); if (ids) { *ids++ = num; - *ids = num; + *ids = 0; } return ids; } @@ -268,6 +318,7 @@ void mdb_midl_shrink( MDB_IDL *idp ) if (*(--ids) > MDB_IDL_UM_MAX && (ids = realloc(ids, (MDB_IDL_UM_MAX+2) * sizeof(MDB_ID)))) { + fprintf(stderr, "shrink realloc\n"); *ids++ = MDB_IDL_UM_MAX; *idp = ids; } @@ -317,33 +368,50 @@ int mdb_midl_need( MDB_IDL *idp, unsigned num ) MDB_IDL ids = *idp; num += ids[0]; if (num > ids[-1]) { - num = (num + num + (256 + 2)) & -256; - MDB_IDL new_ids; - if (!(new_ids = calloc(num, sizeof(MDB_ID)))) + num = (num + num/4 + (256 + 2)) & -256; + fprintf(stderr, "Reallocating %u\n", num); + if (!(ids = realloc(ids-1, num * sizeof(MDB_ID)))) return ENOMEM; - fprintf(stderr, "Created new id list of size %u\n", num); - *new_ids++ = num - 2; - *new_ids = num - 2; - unsigned j = 1; - // re-spread out the entries with gaps for growth - for (unsigned i = 1; i <= ids[0]; i++) { - new_ids[j++] = 0; // empty slot for growth - ssize_t entry; - while (!(entry = ids[i])) { - if (++i > ids[0]) break; - } - new_ids[j++] = entry; - if (entry < 0) new_ids[j++] = ids[++i]; // this was a block with a length + *ids++ = num - 2; + *idp = ids; + } + return 0; +} + + +int mdb_midl_respread( MDB_IDL *idp ) +{ + MDB_IDL ids = *idp; + unsigned num = ids[0]; + num = (num + num + (256 + 2)) & -256; + MDB_IDL new_ids; + if (!(new_ids = calloc(num, sizeof(MDB_ID)))) + return ENOMEM; + fprintf(stderr, "Created new id list of size %u\n", num); + *new_ids++ = num - 2; + *new_ids = num - 2; + unsigned j = 1; + // re-spread out the entries with gaps for growth + for (unsigned i = 1; i <= ids[0]; i++) { + new_ids[j++] = 0; // empty slot for growth + ssize_t entry; + while (!(entry = ids[i])) { + if (++i > ids[0]) break; } - mdb_midl_free(ids); - // now shrink (or grow) back to appropriate size - num = (j + (j >> 3) + 22) & -16; - if (num > new_ids[0]) num = new_ids[0]; + new_ids[j++] = entry; + if (entry < 0) new_ids[j++] = ids[++i]; // this was a block with a length + } + //mdb_midl_free(ids); + // now shrink (or grow) back to appropriate size + num = (j + (j >> 3) + 22) & -16; + if (num > new_ids[0]) { + num = new_ids[0]; + fprintf(stderr, "Reallocating new id list of size %u\n", num); new_ids = realloc(new_ids - 1, (num + 2) * sizeof(MDB_ID)); *new_ids++ = num; - *new_ids = num; - *idp = new_ids; } + new_ids[0] = j - 1; + *idp = new_ids; return 0; } @@ -354,11 +422,11 @@ int mdb_midl_print( FILE *fp, MDB_IDL ids ) return 0; } unsigned i; - fprintf(fp, "freelist: %u: ", ids[0]); + fprintf(fp, "freelist: %u/%u: ", ids[0], ids[-1]); for (i=1; i<=ids[0]; i++) { ssize_t entry = ids[i]; if (entry < 0) { - fprintf(fp, "%li-%li ", ids[i+1], ids[i+1] - entry - 1); + fprintf(fp, "%li-%li ", ids[i+1] - entry - 1, ids[i+1]); i++; } else if (ids[i] == 0) { fprintf(fp, "_"); @@ -371,6 +439,7 @@ int mdb_midl_print( FILE *fp, MDB_IDL ids ) } int mdb_midl_append( MDB_IDL *idp, MDB_ID id ) { + return mdb_midl_insert(idp, id, 1); MDB_IDL ids = *idp; /* Too big? */ if (ids[0] >= ids[-1]) { @@ -386,14 +455,19 @@ int mdb_midl_append( MDB_IDL *idp, MDB_ID id ) int mdb_midl_append_list( MDB_IDL *idp, MDB_IDL app ) { MDB_IDL ids = *idp; - /* Too big? */ - if (ids[0] + app[0] >= ids[-1]) { - if (mdb_midl_grow(idp, app[0])) - return ENOMEM; - ids = *idp; + + for (unsigned i = 1; i <= app[0]; i++) { + ssize_t entry = app[i]; + int count = 1; + if (entry <= 0) { + if (entry == 0) continue; + count = -entry; + entry = app[++i]; + } + int rc; + if ((rc = mdb_midl_insert(idp, entry, count)) != 0) + return rc; } - memcpy(&ids[ids[0]+1], &app[1], app[0] * sizeof(MDB_ID)); - ids[0] += app[0]; return 0; } diff --git a/dependencies/lmdb/libraries/liblmdb/midl.h b/dependencies/lmdb/libraries/liblmdb/midl.h index 2e1bff3fcd..888e04ec7b 100644 --- a/dependencies/lmdb/libraries/liblmdb/midl.h +++ b/dependencies/lmdb/libraries/liblmdb/midl.h @@ -111,6 +111,8 @@ void mdb_midl_shrink(MDB_IDL *idp); */ int mdb_midl_need(MDB_IDL *idp, unsigned num); +int mdb_midl_respread(MDB_IDL *idp); + int mdb_midl_print( FILE *fp, MDB_IDL ids ); MDB_IDL mdb_midl_pack(MDB_IDL idl); unsigned mdb_midl_pack_count(MDB_IDL idl); @@ -120,7 +122,7 @@ unsigned mdb_midl_pack_count(MDB_IDL idl); * @param[in] id The ID to append. * @return 0 on success, ENOMEM if the IDL is too large. */ -int mdb_midl_insert( MDB_IDL *idp, MDB_ID id ); +int mdb_midl_insert( MDB_IDL *idp, MDB_ID id, int insertion_count ); /** Append an ID onto an IDL. * @param[in,out] idp Address of the IDL to append to. diff --git a/test/index.test.js b/test/index.test.js index 1cbda2ab64..3817794b04 100644 --- a/test/index.test.js +++ b/test/index.test.js @@ -254,11 +254,12 @@ describe('lmdb-js', function () { let a = seed * 15485863; return a * a * a % 2038074743; } + await new Promise((resolve) => setTimeout(resolve, 3000)); let promise; let additive = 'this is more text'; for (let i = 0; i < 7; i++) additive += additive; - for (let i = 0; i < 1000; i++) { + for (let i = 0; i < 5000; i++) { let text = 'this is a test'; while(random() < 0.95) text += additive; console.log('write', i, text.length);