In place re-spreading with re-spreading after excessive searching

dependencies/lmdb/libraries/liblmdb/mdb.c

@@ -2736,7 +2736,7 @@ mdb_page_alloc(MDB_cursor *mc, int num, MDB_page **mp)
 		unsigned block_size = 0;
 		ssize_t entry;
 		empty_entries = 0;
-		//mdb_midl_print(stderr, mop);
+		mdb_midl_print(stderr, mop);
 		// TODO: Skip this on the first iteration, since we already checked the cache
 		pgno_t last_pgno = 0; // TODO: This should be removed
 		for (i = 1; i <= mop_len; i++) {
@@ -2901,7 +2901,11 @@ mdb_page_alloc(MDB_cursor *mc, int num, MDB_page **mp)
 		}
 	}
 	if (i) {
-		//fprintf(stderr, "using %u to %u\n", pgno, pgno + num -1);
+		fprintf(stderr, "using %u to %u\n", pgno, pgno + num -1);
+		if (empty_entries > (mop_len >> 1) + 20) {
+			fprintf(stderr, "should resize\n");
+			mdb_midl_respread(&env->me_pghead);
+		}
 		/*mop[0] = mop_len -= num;
 		/* Move any stragglers down */
 		/*for (j = i-num; j < mop_len; )

dependencies/lmdb/libraries/liblmdb/midl.c

@@ -453,36 +453,35 @@ unsigned mdb_midl_pack_count(MDB_IDL idl) {
 int mdb_midl_respread( MDB_IDL *idp )
 {
 	MDB_IDL ids = *idp;
-	unsigned num = ids[0];
-	num = (num + num + (256 + 2)) & -256;
-	MDB_IDL new_ids; 
-	if (!(new_ids = calloc(num, sizeof(MDB_ID))))
-		return ENOMEM;
-	fprintf(stderr, "Created new id list of size %u\n", num);
-	*new_ids++ = num - 2;
-	*new_ids = num - 2;
 	unsigned j = 1;
-	// re-spread out the entries with gaps for growth
-	for (unsigned i = 1; i <= ids[0]; i++) {
-		new_ids[j++] = 0; // empty slot for growth
+	unsigned size = ids[0];
+	unsigned new_size = 0;
+	// first, do compaction
+	for (unsigned i = 1; i <= size; i++) {
 		ssize_t entry;
 		while (!(entry = ids[i])) {
-			if (++i > ids[0]) break;
+			if (++i > ids[0]) goto expand;
 		}
-		new_ids[j++] = entry;
-		if (entry < 0) new_ids[j++] = ids[++i]; // this was a block with a length
+		ids[j++] = entry;
+		new_size += entry < 0 ? 3 : 2; // one for empty space, one for the entry, and one for the length if it is a block
+		if (entry < 0) ids[j++] = ids[++i]; // this was a block with a length
 	}
-	//mdb_midl_free(ids);
-	// now shrink (or grow) back to appropriate size
-	num = (j + (j >> 3) + 22) & -16;
-	if (num > new_ids[0]) {
-		num = new_ids[0];
-		fprintf(stderr, "Reallocating new id list of size %u\n", num);
-		new_ids = realloc(new_ids - 1, (num + 2) * sizeof(MDB_ID));
-		*new_ids++ = num;
+	expand:
+	mdb_midl_need(idp, new_size - ids[0]);
+	ids = *idp;
+	ids[0] = new_size;
+	j--;
+	// re-spread out the entries with gaps for growth
+	for (unsigned i = new_size; i > 0;) {
+		ssize_t pgno = ids[j--];
+		ids[i--] = pgno;
+		ssize_t entry = ids[j];
+		if (entry < 0) {
+			ids[i--] = entry;
+			j--;
+		}
+		ids[i--] = 0; // empty slot for growth
 	}
-	new_ids[0] = j - 1;
-	*idp = new_ids;
 	return 0;
 }
 

test/index.test.js

@@ -265,7 +265,7 @@ describe('lmdb-js', function () {
 					while(random() < 0.95) text += additive;
 					console.log('write', i, text.length);
 					promise = db.put(i % 10, text);
-					if (i % 6 == 0) {
+					if (i % 16 == 0) {
 						await promise;
 					}
 				}