Merge branch 'master' into nv/storage-n

Dockerfile.aztec

@@ -3,7 +3,7 @@ ENV BB_WORKING_DIRECTORY=/usr/src/bb
 ENV BB_BINARY_PATH=/usr/src/barretenberg/cpp/build/bin/bb
 ENV ACVM_WORKING_DIRECTORY=/usr/src/acvm
 ENV ACVM_BINARY_PATH=/usr/src/noir/noir-repo/target/release/acvm
-RUN mkdir -p $BB_WORKING_DIRECTORY $ACVM_WORKING_DIRECTORY /usr/src/yarn-project/world-state/build
+RUN mkdir -p $BB_WORKING_DIRECTORY $ACVM_WORKING_DIRECTORY /usr/src/yarn-project/native/build
 
 COPY /usr/src /usr/src
 

Dockerfile.end-to-end

@@ -9,7 +9,7 @@ ENV BB_BINARY_PATH=/usr/src/barretenberg/cpp/build/bin/bb
 ENV ACVM_WORKING_DIRECTORY=/usr/src/acvm
 ENV ACVM_BINARY_PATH=/usr/src/noir/noir-repo/target/release/acvm
 ENV PROVER_AGENT_CONCURRENCY=8
-RUN mkdir -p $BB_WORKING_DIRECTORY $ACVM_WORKING_DIRECTORY /usr/src/yarn-project/world-state/build
+RUN mkdir -p $BB_WORKING_DIRECTORY $ACVM_WORKING_DIRECTORY /usr/src/yarn-project/native/build
 
 COPY /usr/src /usr/src
 COPY /anvil /opt/foundry/bin/anvil

aztec-nargo/Dockerfile

@@ -11,7 +11,7 @@ FROM --platform=linux/amd64 aztecprotocol/barretenberg-x86_64-linux-clang as bar
 FROM ubuntu:noble
 # Install Tini as nargo doesn't handle signals properly.
 # Install git as nargo needs it to clone.
-RUN apt-get update && apt-get install -y git tini jq curl && rm -rf /var/lib/apt/lists/* && apt-get clean
+RUN apt-get update && apt-get install -y git tini jq curl nodejs npm && rm -rf /var/lib/apt/lists/* && apt-get clean
 
 # Copy binaries to /usr/bin
 COPY --from=built-noir /usr/src/noir/noir-repo/target/release/nargo /usr/bin/nargo
@@ -21,4 +21,9 @@ COPY --from=barretenberg /usr/src/barretenberg/cpp/build/bin/bb /usr/bin/bb
 # Copy in script that calls both binaries
 COPY ./aztec-nargo/compile_then_postprocess.sh /usr/src/aztec-nargo/compile_then_postprocess.sh
 
+# Profiler
+COPY --from=built-noir /usr/src/noir/noir-repo/target/release/noir-profiler /usr/bin/noir-profiler
+COPY ./noir-projects/noir-contracts/scripts/flamegraph.sh /usr/bin/flamegraph.sh
+COPY ./noir-projects/noir-contracts/scripts/extractFunctionAsNoirArtifact.js /usr/bin/extractFunctionAsNoirArtifact.js
+
 ENTRYPOINT ["/usr/bin/tini", "--", "/usr/src/aztec-nargo/compile_then_postprocess.sh"]

aztec-nargo/Earthfile

@@ -4,7 +4,7 @@ run:
   FROM ubuntu:noble
   # Install Tini as nargo doesn't handle signals properly.
   # Install git as nargo needs it to clone.
-  RUN apt-get update && apt-get install -y git tini jq curl && rm -rf /var/lib/apt/lists/* && apt-get clean
+  RUN apt-get update && apt-get install -y git tini jq curl nodejs npm && rm -rf /var/lib/apt/lists/* && apt-get clean
 
   # Copy binaries to /usr/bin
   COPY ../+bootstrap/usr/src/noir/noir-repo/target/release/nargo /usr/bin/nargo
@@ -14,6 +14,11 @@ run:
   # Copy in script that calls both binaries
   COPY ./compile_then_postprocess.sh /usr/bin/compile_then_postprocess.sh
 
+  # Profiler
+  COPY ../+bootstrap/usr/src/noir/noir-repo/target/release/noir-profiler /usr/bin/noir-profiler
+  COPY ../+bootstrap/usr/src/noir-projects/noir-contracts/scripts/flamegraph.sh /usr/bin/flamegraph.sh
+  COPY ../+bootstrap/usr/src/noir-projects/noir-contracts/scripts/extractFunctionAsNoirArtifact.js /usr/bin/extractFunctionAsNoirArtifact.js
+
   ENV PATH "/usr/bin:${PATH}"
   ENTRYPOINT ["/usr/bin/tini", "--", "/usr/bin/compile_then_postprocess.sh"]
   SAVE IMAGE aztecprotocol/aztec-nargo

aztec-up/bin/aztec

@@ -72,6 +72,15 @@ elif [ "${1:-}" == "start" ]; then
   readarray -t ENV_VARS_TO_INJECT <"$LOCAL_ENV_VAR_FILE"
   export ENV_VARS_TO_INJECT="${ENV_VARS_TO_INJECT[*]}"
   ENV_VARS_TO_INJECT="${ENV_VARS_TO_INJECT[*]}" INHERIT_USER=0 $(dirname $0)/.aztec-run aztecprotocol/aztec "$@"
+elif [ "${1:-}" == "flamegraph" ]; then
+  docker run -it \
+    --entrypoint /usr/bin/flamegraph.sh \
+    --env PROFILER_PATH=/usr/bin/noir-profiler \
+    --env BACKEND_PATH=/usr/bin/bb \
+    --env SERVE=${SERVE:-0} \
+    $([ "${SERVE:-0}" == "1" ] && echo "-p 8000:8000" || echo "") \
+    -v $(realpath $(dirname $2))/:/tmp \
+    aztecprotocol/aztec-nargo:$VERSION /tmp/$(basename $2) $3
 else
   ENV_VARS_TO_INJECT="SECRET_KEY" SKIP_PORT_ASSIGNMENT=1 $(dirname $0)/.aztec-run aztecprotocol/aztec "$@"
 fi

barretenberg/.gitrepo

@@ -6,7 +6,7 @@
 [subrepo]
 remote = https://github.com/AztecProtocol/barretenberg
 	branch = master
-	commit = 2b718dec0624621c81bdc406bed8db0c30ed2a77
-	parent = 6ce23892c8762ffde6fc3198e1e392cd1a72969b
+	commit = 37c100901c570b61ca79b46f0fdb0bbc5ce22708
+	parent = 756b94101345d231b86233921398555512855273
 	method = merge
 	cmdver = 0.4.6

barretenberg/acir_tests/browser-test-app/yarn.lock

@@ -7,7 +7,7 @@ __metadata:
 
 "@aztec/bb.js@file:../../ts::locator=browser-test-app%40workspace%3A.":
   version: 0.72.1
-  resolution: "@aztec/bb.js@file:../../ts#../../ts::hash=7ab3ef&locator=browser-test-app%40workspace%3A."
+  resolution: "@aztec/bb.js@file:../../ts#../../ts::hash=90d55d&locator=browser-test-app%40workspace%3A."
   dependencies:
     comlink: "npm:^4.4.1"
     commander: "npm:^12.1.0"
@@ -17,7 +17,7 @@ __metadata:
     tslib: "npm:^2.4.0"
   bin:
     bb.js: ./dest/node/main.js
-  checksum: 10c0/463d29bb01102874431d5a83af71a18a92b6b7fd817311474744bfe4af2b82d71efa1796d0a00093bcc87b163922ff748cb6a8480d7c295c726a855d1bdf321d
+  checksum: 10c0/6ec39503ec545df13d297134ae28d8be01d02105a8518fa9cc15c54d1f93924956ce5c41913d1f2db31a8e05a5bad1fe7f5a0c1d5f1a6346f4703e307cc3f05a
   languageName: node
   linkType: hard
 

barretenberg/acir_tests/headless-test/package.json

@@ -10,7 +10,7 @@
   "dependencies": {
     "chalk": "^5.3.0",
     "commander": "^12.1.0",
-    "playwright": "^1.50.0",
+    "playwright": "1.49.0",
     "puppeteer": "^22.4.1"
   },
   "devDependencies": {

barretenberg/acir_tests/headless-test/yarn.lock

@@ -869,7 +869,7 @@ __metadata:
   dependencies:
     chalk: "npm:^5.3.0"
     commander: "npm:^12.1.0"
-    playwright: "npm:^1.50.0"
+    playwright: "npm:1.49.0"
     puppeteer: "npm:^22.4.1"
     ts-node: "npm:^10.9.2"
     typescript: "npm:^5.4.2"
@@ -1408,27 +1408,27 @@ __metadata:
   languageName: node
   linkType: hard
 
-"playwright-core@npm:1.50.0":
-  version: 1.50.0
-  resolution: "playwright-core@npm:1.50.0"
+"playwright-core@npm:1.49.0":
+  version: 1.49.0
+  resolution: "playwright-core@npm:1.49.0"
   bin:
     playwright-core: cli.js
-  checksum: 10c0/b0cc7fadcb2db68a7b8d730b26c7a7d17baad454a0697c781e08074a619e57779a90be9b57c4c741ff4895390bdfd093d8393a746e8bf68ae57ac452f4c1cdb2
+  checksum: 10c0/22c1a72fabdcc87bd1cd4d40a032d2c5b94cf94ba7484dc182048c3fa1c8ec26180b559d8cac4ca9870e8fd6bdf5ef9d9f54e7a31fd60d67d098fcffc5e4253b
   languageName: node
   linkType: hard
 
-"playwright@npm:^1.50.0":
-  version: 1.50.0
-  resolution: "playwright@npm:1.50.0"
+"playwright@npm:1.49.0":
+  version: 1.49.0
+  resolution: "playwright@npm:1.49.0"
   dependencies:
     fsevents: "npm:2.3.2"
-    playwright-core: "npm:1.50.0"
+    playwright-core: "npm:1.49.0"
   dependenciesMeta:
     fsevents:
       optional: true
   bin:
     playwright: cli.js
-  checksum: 10c0/0076a536433819b7122066a07c5fcfa56d40d09cbbec0a39061bbfa832c8a1f626df5e4fe206fbeba56b3a61f0e2b26d4ad3c2b402852d6f147a266fd18e4ddf
+  checksum: 10c0/e94d662747cd147d0573570fec90dadc013c1097595714036fc8934a075c5a82ab04a49111b03b1f762ea86429bdb7c94460901896901e20970b30ce817cc93f
   languageName: node
   linkType: hard
 

barretenberg/cpp/Earthfile

@@ -59,10 +59,10 @@ test-cache-read:
     --command="exit 1"
   SAVE ARTIFACT build/bin
 
-preset-release-world-state:
+preset-release-nodejs-module:
   FROM +source
-  DO +CACHE_BUILD_BIN --prefix=preset-release-world-state \
-    --command="cmake --preset clang16-pic -Bbuild && cmake --build build --target world_state_napi && mv ./build/lib/world_state_napi.node ./build/bin"
+  DO +CACHE_BUILD_BIN --prefix=preset-release-nodejs-module \
+    --command="cmake --preset clang16-pic -Bbuild && cmake --build build --target nodejs_module && mv ./build/lib/nodejs_module.node ./build/bin"
   SAVE ARTIFACT build/bin
 
 preset-release-assert:
@@ -317,4 +317,4 @@ build:
   BUILD +preset-wasm
   BUILD +preset-wasm-threads
   BUILD +preset-release
-  BUILD +preset-release-world-state
+  BUILD +preset-release-nodejs-module

barretenberg/cpp/bootstrap.sh

@@ -17,12 +17,12 @@ function build_native {
     cache_upload barretenberg-release-$hash.tar.gz build/bin
   fi
 
-  (cd src/barretenberg/world_state_napi && yarn --frozen-lockfile --prefer-offline)
-  if ! cache_download barretenberg-release-world-state-$hash.tar.gz; then
+  (cd src/barretenberg/nodejs_module && yarn --frozen-lockfile --prefer-offline)
+  if ! cache_download barretenberg-release-nodejs-module-$hash.tar.gz; then
     rm -f build-pic/CMakeCache.txt
     cmake --preset $pic_preset -DCMAKE_BUILD_TYPE=RelWithAssert
-    cmake --build --preset $pic_preset --target world_state_napi
-    cache_upload barretenberg-release-world-state-$hash.tar.gz build-pic/lib/world_state_napi.node
+    cmake --build --preset $pic_preset --target nodejs_module
+    cache_upload barretenberg-release-nodejs-module-$hash.tar.gz build-pic/lib/nodejs_module.node
   fi
 }
 
@@ -118,4 +118,4 @@ case "$cmd" in
   *)
     echo "Unknown command: $cmd"
     exit 1
-esac
+esac

barretenberg/cpp/cmake/lmdb.cmake

@@ -3,6 +3,7 @@ include(ExternalProject)
 set(LMDB_PREFIX "${CMAKE_BINARY_DIR}/_deps/lmdb")
 set(LMDB_INCLUDE "${LMDB_PREFIX}/src/lmdb_repo/libraries/liblmdb")
 set(LMDB_LIB "${LMDB_INCLUDE}/liblmdb.a")
+set(LMDB_HEADER "${LMDB_INCLUDE}/lmdb.h")
 set(LMDB_OBJECT "${LMDB_INCLUDE}/*.o")
 
 ExternalProject_Add(
@@ -15,7 +16,7 @@ ExternalProject_Add(
     BUILD_COMMAND make -C libraries/liblmdb -e XCFLAGS=-fPIC liblmdb.a
     INSTALL_COMMAND ""
     UPDATE_COMMAND "" # No update step
-    BUILD_BYPRODUCTS ${LMDB_LIB} ${LMDB_INCLUDE}
+    BUILD_BYPRODUCTS ${LMDB_LIB} ${LMDB_HEADER}
 )
 
 add_library(lmdb STATIC IMPORTED GLOBAL)

barretenberg/cpp/dockerfiles/Dockerfile.x86_64-linux-clang

@@ -28,7 +28,7 @@ RUN cmake --build --preset clang16 --target ultra_honk_rounds_bench --target bb
 
 RUN npm install --global yarn
 RUN cmake --preset clang16-pic
-RUN cmake --build --preset clang16-pic --target world_state_napi
+RUN cmake --build --preset clang16-pic --target nodejs_module
 
 FROM ubuntu:lunar
 WORKDIR /usr/src/barretenberg/cpp
@@ -40,4 +40,4 @@ COPY --from=builder /usr/src/barretenberg/cpp/build/bin/grumpkin_srs_gen /usr/sr
 # Copy libs for consuming projects.
 COPY --from=builder /usr/src/barretenberg/cpp/build/lib/libbarretenberg.a /usr/src/barretenberg/cpp/build/lib/libbarretenberg.a
 COPY --from=builder /usr/src/barretenberg/cpp/build/lib/libenv.a /usr/src/barretenberg/cpp/build/lib/libenv.a
-COPY --from=builder /usr/src/barretenberg/cpp/build-pic/lib/world_state_napi.node /usr/src/barretenberg/cpp/build-pic/lib/world_state_napi.node
+COPY --from=builder /usr/src/barretenberg/cpp/build-pic/lib/nodejs_module.node /usr/src/barretenberg/cpp/build-pic/lib/nodejs_module.node

barretenberg/cpp/scripts/lmdblib_tests.sh

@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+
+set -e
+
+# run commands relative to parent directory
+cd $(dirname $0)/..
+
+DEFAULT_TESTS=LMDBStoreTest.*:LMDBEnvironmentTest.*
+TEST=${1:-$DEFAULT_TESTS}
+PRESET=${PRESET:-clang16}
+
+cmake --build --preset $PRESET --target lmdblib_tests
+./build/bin/lmdblib_tests --gtest_filter=$TEST

barretenberg/cpp/src/CMakeLists.txt

@@ -58,7 +58,7 @@ if (ENABLE_PIC AND CMAKE_CXX_COMPILER_ID MATCHES "Clang")
   message("Building with Position Independent Code")
   set(CMAKE_CXX_FLAGS  "${CMAKE_CXX_FLAGS} -fPIC")
   set(CMAKE_C_FLAGS  "${CMAKE_C_FLAGS} -fPIC")
-  add_subdirectory(barretenberg/world_state_napi)
+  add_subdirectory(barretenberg/nodejs_module)
 endif()
 
 add_subdirectory(barretenberg/bb)
@@ -78,6 +78,7 @@ add_subdirectory(barretenberg/examples)
 add_subdirectory(barretenberg/flavor)
 add_subdirectory(barretenberg/goblin)
 add_subdirectory(barretenberg/grumpkin_srs_gen)
+add_subdirectory(barretenberg/lmdblib)
 add_subdirectory(barretenberg/numeric)
 add_subdirectory(barretenberg/plonk)
 add_subdirectory(barretenberg/plonk_honk_shared)
@@ -176,8 +177,9 @@ if(NOT DISABLE_AZTEC_VM)
 endif()
 
 if(NOT WASM)
-    # enable merkle trees
+    # enable merkle trees and lmdb
     list(APPEND BARRETENBERG_TARGET_OBJECTS $<TARGET_OBJECTS:crypto_merkle_tree_objects>)
+    list(APPEND BARRETENBERG_TARGET_OBJECTS $<TARGET_OBJECTS:lmdblib_objects>)
 endif()
 
 add_library(

barretenberg/cpp/src/barretenberg/benchmark/indexed_tree_bench/indexed_tree.bench.cpp

@@ -3,7 +3,6 @@
 #include "barretenberg/crypto/merkle_tree/hash.hpp"
 #include "barretenberg/crypto/merkle_tree/indexed_tree/content_addressed_indexed_tree.hpp"
 #include "barretenberg/crypto/merkle_tree/indexed_tree/indexed_leaf.hpp"
-#include "barretenberg/crypto/merkle_tree/lmdb_store/callbacks.hpp"
 #include "barretenberg/crypto/merkle_tree/lmdb_store/lmdb_tree_store.hpp"
 #include "barretenberg/crypto/merkle_tree/node_store/cached_content_addressed_tree_store.hpp"
 #include "barretenberg/crypto/merkle_tree/response.hpp"

barretenberg/cpp/src/barretenberg/crypto/merkle_tree/CMakeLists.txt

@@ -1,16 +1,11 @@
 # merkle tree is agnostic to hash function
 barretenberg_module(
     crypto_merkle_tree
-    lmdb
+    lmdblib
 )
 
 if (NOT FUZZING)
     # but the tests use pedersen and poseidon
     target_link_libraries(crypto_merkle_tree_tests PRIVATE stdlib_pedersen_hash stdlib_poseidon2)
-    add_dependencies(crypto_merkle_tree_tests lmdb_repo)
-    add_dependencies(crypto_merkle_tree_test_objects lmdb_repo)
 endif()
 
-add_dependencies(crypto_merkle_tree lmdb_repo)
-add_dependencies(crypto_merkle_tree_objects lmdb_repo)
-

barretenberg/cpp/src/barretenberg/crypto/merkle_tree/append_only_tree/content_addressed_append_only_tree.test.cpp

@@ -7,14 +7,14 @@
 #include "barretenberg/common/thread_pool.hpp"
 #include "barretenberg/crypto/merkle_tree/hash.hpp"
 #include "barretenberg/crypto/merkle_tree/hash_path.hpp"
-#include "barretenberg/crypto/merkle_tree/lmdb_store/lmdb_environment.hpp"
 #include "barretenberg/crypto/merkle_tree/lmdb_store/lmdb_tree_store.hpp"
 #include "barretenberg/crypto/merkle_tree/node_store/array_store.hpp"
 #include "barretenberg/crypto/merkle_tree/node_store/cached_content_addressed_tree_store.hpp"
 #include "barretenberg/crypto/merkle_tree/response.hpp"
 #include "barretenberg/crypto/merkle_tree/signal.hpp"
 #include "barretenberg/crypto/merkle_tree/types.hpp"
 #include "barretenberg/ecc/curves/bn254/fr.hpp"
+#include "barretenberg/lmdblib/lmdb_environment.hpp"
 #include "barretenberg/relations/relation_parameters.hpp"
 #include <algorithm>
 #include <array>
@@ -29,6 +29,7 @@
 
 using namespace bb;
 using namespace bb::crypto::merkle_tree;
+using namespace bb::lmdblib;
 
 using Store = ContentAddressedCachedTreeStore<bb::fr>;
 using TreeType = ContentAddressedAppendOnlyTree<Store, Poseidon2HashPolicy>;

barretenberg/cpp/src/barretenberg/crypto/merkle_tree/fixtures.hpp

@@ -29,7 +29,7 @@ static std::vector<fr> VALUES = create_values();
 inline std::string random_string()
 {
     std::stringstream ss;
-    ss << random_engine.get_random_uint256();
+    ss << random_engine.get_random_uint32();
     return ss.str();
 }