feat: change data store backend to use liblmdb directly

Dockerfile.aztec

@@ -3,7 +3,7 @@ ENV BB_WORKING_DIRECTORY=/usr/src/bb
 ENV BB_BINARY_PATH=/usr/src/barretenberg/cpp/build/bin/bb
 ENV ACVM_WORKING_DIRECTORY=/usr/src/acvm
 ENV ACVM_BINARY_PATH=/usr/src/noir/noir-repo/target/release/acvm
-RUN mkdir -p $BB_WORKING_DIRECTORY $ACVM_WORKING_DIRECTORY /usr/src/yarn-project/world-state/build
+RUN mkdir -p $BB_WORKING_DIRECTORY $ACVM_WORKING_DIRECTORY /usr/src/yarn-project/native/build
 
 COPY /usr/src /usr/src
 

Dockerfile.end-to-end

@@ -9,7 +9,7 @@ ENV BB_BINARY_PATH=/usr/src/barretenberg/cpp/build/bin/bb
 ENV ACVM_WORKING_DIRECTORY=/usr/src/acvm
 ENV ACVM_BINARY_PATH=/usr/src/noir/noir-repo/target/release/acvm
 ENV PROVER_AGENT_CONCURRENCY=8
-RUN mkdir -p $BB_WORKING_DIRECTORY $ACVM_WORKING_DIRECTORY /usr/src/yarn-project/world-state/build
+RUN mkdir -p $BB_WORKING_DIRECTORY $ACVM_WORKING_DIRECTORY /usr/src/yarn-project/native/build
 
 COPY /usr/src /usr/src
 COPY /anvil /opt/foundry/bin/anvil

barretenberg/cpp/Earthfile

@@ -59,10 +59,10 @@ test-cache-read:
     --command="exit 1"
   SAVE ARTIFACT build/bin
 
-preset-release-world-state:
+preset-release-nodejs-module:
   FROM +source
-  DO +CACHE_BUILD_BIN --prefix=preset-release-world-state \
-    --command="cmake --preset clang16-pic -Bbuild && cmake --build build --target world_state_napi && mv ./build/lib/world_state_napi.node ./build/bin"
+  DO +CACHE_BUILD_BIN --prefix=preset-release-nodejs-module \
+    --command="cmake --preset clang16-pic -Bbuild && cmake --build build --target nodejs_module && mv ./build/lib/nodejs_module.node ./build/bin"
   SAVE ARTIFACT build/bin
 
 preset-release-assert:
@@ -317,4 +317,4 @@ build:
   BUILD +preset-wasm
   BUILD +preset-wasm-threads
   BUILD +preset-release
-  BUILD +preset-release-world-state
+  BUILD +preset-release-nodejs-module

barretenberg/cpp/bootstrap.sh

@@ -17,12 +17,12 @@ function build_native {
     cache_upload barretenberg-release-$hash.tar.gz build/bin
   fi
 
-  (cd src/barretenberg/world_state_napi && yarn --frozen-lockfile --prefer-offline)
-  if ! cache_download barretenberg-release-world-state-$hash.tar.gz; then
+  (cd src/barretenberg/nodejs_module && yarn --frozen-lockfile --prefer-offline)
+  if ! cache_download barretenberg-release-nodejs-module-$hash.tar.gz; then
     rm -f build-pic/CMakeCache.txt
     cmake --preset $pic_preset -DCMAKE_BUILD_TYPE=RelWithAssert
-    cmake --build --preset $pic_preset --target world_state_napi
-    cache_upload barretenberg-release-world-state-$hash.tar.gz build-pic/lib/world_state_napi.node
+    cmake --build --preset $pic_preset --target nodejs_module
+    cache_upload barretenberg-release-nodejs-module-$hash.tar.gz build-pic/lib/nodejs_module.node
   fi
 }
 
@@ -118,4 +118,4 @@ case "$cmd" in
   *)
     echo "Unknown command: $cmd"
     exit 1
-esac
+esac

barretenberg/cpp/cmake/lmdb.cmake

@@ -3,6 +3,7 @@ include(ExternalProject)
 set(LMDB_PREFIX "${CMAKE_BINARY_DIR}/_deps/lmdb")
 set(LMDB_INCLUDE "${LMDB_PREFIX}/src/lmdb_repo/libraries/liblmdb")
 set(LMDB_LIB "${LMDB_INCLUDE}/liblmdb.a")
+set(LMDB_HEADER "${LMDB_INCLUDE}/lmdb.h")
 set(LMDB_OBJECT "${LMDB_INCLUDE}/*.o")
 
 ExternalProject_Add(
@@ -15,7 +16,7 @@ ExternalProject_Add(
     BUILD_COMMAND make -C libraries/liblmdb -e XCFLAGS=-fPIC liblmdb.a
     INSTALL_COMMAND ""
     UPDATE_COMMAND "" # No update step
-    BUILD_BYPRODUCTS ${LMDB_LIB} ${LMDB_INCLUDE}
+    BUILD_BYPRODUCTS ${LMDB_LIB} ${LMDB_HEADER}
 )
 
 add_library(lmdb STATIC IMPORTED GLOBAL)

barretenberg/cpp/dockerfiles/Dockerfile.x86_64-linux-clang

@@ -28,7 +28,7 @@ RUN cmake --build --preset clang16 --target ultra_honk_rounds_bench --target bb
 
 RUN npm install --global yarn
 RUN cmake --preset clang16-pic
-RUN cmake --build --preset clang16-pic --target world_state_napi
+RUN cmake --build --preset clang16-pic --target nodejs_module
 
 FROM ubuntu:lunar
 WORKDIR /usr/src/barretenberg/cpp
@@ -40,4 +40,4 @@ COPY --from=builder /usr/src/barretenberg/cpp/build/bin/grumpkin_srs_gen /usr/sr
 # Copy libs for consuming projects.
 COPY --from=builder /usr/src/barretenberg/cpp/build/lib/libbarretenberg.a /usr/src/barretenberg/cpp/build/lib/libbarretenberg.a
 COPY --from=builder /usr/src/barretenberg/cpp/build/lib/libenv.a /usr/src/barretenberg/cpp/build/lib/libenv.a
-COPY --from=builder /usr/src/barretenberg/cpp/build-pic/lib/world_state_napi.node /usr/src/barretenberg/cpp/build-pic/lib/world_state_napi.node
+COPY --from=builder /usr/src/barretenberg/cpp/build-pic/lib/nodejs_module.node /usr/src/barretenberg/cpp/build-pic/lib/nodejs_module.node

barretenberg/cpp/scripts/lmdblib_tests.sh

@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+
+set -e
+
+# run commands relative to parent directory
+cd $(dirname $0)/..
+
+DEFAULT_TESTS=LMDBStoreTest.*:LMDBEnvironmentTest.*
+TEST=${1:-$DEFAULT_TESTS}
+PRESET=${PRESET:-clang16}
+
+cmake --build --preset $PRESET --target lmdblib_tests
+./build/bin/lmdblib_tests --gtest_filter=$TEST

barretenberg/cpp/src/CMakeLists.txt

@@ -58,7 +58,7 @@ if (ENABLE_PIC AND CMAKE_CXX_COMPILER_ID MATCHES "Clang")
   message("Building with Position Independent Code")
   set(CMAKE_CXX_FLAGS  "${CMAKE_CXX_FLAGS} -fPIC")
   set(CMAKE_C_FLAGS  "${CMAKE_C_FLAGS} -fPIC")
-  add_subdirectory(barretenberg/world_state_napi)
+  add_subdirectory(barretenberg/nodejs_module)
 endif()
 
 add_subdirectory(barretenberg/bb)
@@ -78,6 +78,7 @@ add_subdirectory(barretenberg/examples)
 add_subdirectory(barretenberg/flavor)
 add_subdirectory(barretenberg/goblin)
 add_subdirectory(barretenberg/grumpkin_srs_gen)
+add_subdirectory(barretenberg/lmdblib)
 add_subdirectory(barretenberg/numeric)
 add_subdirectory(barretenberg/plonk)
 add_subdirectory(barretenberg/plonk_honk_shared)
@@ -176,8 +177,9 @@ if(NOT DISABLE_AZTEC_VM)
 endif()
 
 if(NOT WASM)
-    # enable merkle trees
+    # enable merkle trees and lmdb
     list(APPEND BARRETENBERG_TARGET_OBJECTS $<TARGET_OBJECTS:crypto_merkle_tree_objects>)
+    list(APPEND BARRETENBERG_TARGET_OBJECTS $<TARGET_OBJECTS:lmdblib_objects>)
 endif()
 
 add_library(

barretenberg/cpp/src/barretenberg/benchmark/indexed_tree_bench/indexed_tree.bench.cpp

@@ -3,7 +3,6 @@
 #include "barretenberg/crypto/merkle_tree/hash.hpp"
 #include "barretenberg/crypto/merkle_tree/indexed_tree/content_addressed_indexed_tree.hpp"
 #include "barretenberg/crypto/merkle_tree/indexed_tree/indexed_leaf.hpp"
-#include "barretenberg/crypto/merkle_tree/lmdb_store/callbacks.hpp"
 #include "barretenberg/crypto/merkle_tree/lmdb_store/lmdb_tree_store.hpp"
 #include "barretenberg/crypto/merkle_tree/node_store/cached_content_addressed_tree_store.hpp"
 #include "barretenberg/crypto/merkle_tree/response.hpp"

barretenberg/cpp/src/barretenberg/crypto/merkle_tree/CMakeLists.txt

@@ -1,16 +1,11 @@
 # merkle tree is agnostic to hash function
 barretenberg_module(
     crypto_merkle_tree
-    lmdb
+    lmdblib
 )
 
 if (NOT FUZZING)
     # but the tests use pedersen and poseidon
     target_link_libraries(crypto_merkle_tree_tests PRIVATE stdlib_pedersen_hash stdlib_poseidon2)
-    add_dependencies(crypto_merkle_tree_tests lmdb_repo)
-    add_dependencies(crypto_merkle_tree_test_objects lmdb_repo)
 endif()
 
-add_dependencies(crypto_merkle_tree lmdb_repo)
-add_dependencies(crypto_merkle_tree_objects lmdb_repo)
-

barretenberg/cpp/src/barretenberg/crypto/merkle_tree/append_only_tree/content_addressed_append_only_tree.test.cpp

@@ -7,14 +7,14 @@
 #include "barretenberg/common/thread_pool.hpp"
 #include "barretenberg/crypto/merkle_tree/hash.hpp"
 #include "barretenberg/crypto/merkle_tree/hash_path.hpp"
-#include "barretenberg/crypto/merkle_tree/lmdb_store/lmdb_environment.hpp"
 #include "barretenberg/crypto/merkle_tree/lmdb_store/lmdb_tree_store.hpp"
 #include "barretenberg/crypto/merkle_tree/node_store/array_store.hpp"
 #include "barretenberg/crypto/merkle_tree/node_store/cached_content_addressed_tree_store.hpp"
 #include "barretenberg/crypto/merkle_tree/response.hpp"
 #include "barretenberg/crypto/merkle_tree/signal.hpp"
 #include "barretenberg/crypto/merkle_tree/types.hpp"
 #include "barretenberg/ecc/curves/bn254/fr.hpp"
+#include "barretenberg/lmdblib/lmdb_environment.hpp"
 #include "barretenberg/relations/relation_parameters.hpp"
 #include <algorithm>
 #include <array>
@@ -29,6 +29,7 @@
 
 using namespace bb;
 using namespace bb::crypto::merkle_tree;
+using namespace bb::lmdblib;
 
 using Store = ContentAddressedCachedTreeStore<bb::fr>;
 using TreeType = ContentAddressedAppendOnlyTree<Store, Poseidon2HashPolicy>;

barretenberg/cpp/src/barretenberg/crypto/merkle_tree/fixtures.hpp

@@ -29,7 +29,7 @@ static std::vector<fr> VALUES = create_values();
 inline std::string random_string()
 {
     std::stringstream ss;
-    ss << random_engine.get_random_uint256();
+    ss << random_engine.get_random_uint32();
     return ss.str();
 }