feat: Prototype native merkle trees (AztecProtocol#4457)

This PR contains work around understanding the performance of our hash
functions and merkle trees.

1. Parameterise our existing MerkleTree and NulliferTree to accept any
compatible hash function
2. Creates a new native AppendOnlyTree only allowing addition of leaves
at the end
3. Creates a new multi-threaded native IndexedMerkleTree.
4. Add tests to validate new trees against previous ones
5. Adds benchmarks for hash functions and new trees.
6. Exposes poseidon hash functions over the WASM and benchmarks it.

 ### Benchmarks

**Hash Functions**

**Pedersen - Native**

```
------------------------------------------------------------------------------------------
Benchmark                                                Time             CPU   Iterations
------------------------------------------------------------------------------------------
native_pedersen_hash_pair_bench/min_time:3.000       0.178 ms        0.178 ms        23450
```
**Pedersen - Single hash over WASM 1000 times**

Executed 1000 hashes at an average 593.7273874282837us / hash

**Pedersen - 1024 hashes over WASM 10 times**

Executed 10240 hashes at an average 581.7138370126486us / hash

**Poseidon - Native**
```
----------------------------------------------------------------------------------
Benchmark                                        Time             CPU   Iterations
----------------------------------------------------------------------------------
poseiden_hash_bench                          0.012 ms        0.012 ms        57022
```

**Poseidon - Single hash over WASM 1000 times**

Executed 1000 hashes at an average 48.4412202835083us / hash

**Poseidon - 1024 hashes over WASM 10 times**

Executed 10240 hashes at an average 37.44762847200036us / hash

### Merkle Trees

The following benchmarks follow the format of
{benchmark-type}/{batch-insertion-size}/{iteration-count}.

**Append Only Tree**

```
------------------------------------------------------------------------------------------------
Benchmark                                                      Time             CPU   Iterations
------------------------------------------------------------------------------------------------
append_only_tree_bench<Pedersen>/2/iterations:100           5.75 ms         5.75 ms          100
append_only_tree_bench<Pedersen>/4/iterations:100           6.02 ms         6.02 ms          100
append_only_tree_bench<Pedersen>/8/iterations:100           6.65 ms         6.65 ms          100
append_only_tree_bench<Pedersen>/16/iterations:100          7.89 ms         7.89 ms          100
append_only_tree_bench<Pedersen>/32/iterations:100          10.7 ms         10.7 ms          100
append_only_tree_bench<Pedersen>/64/iterations:100          16.3 ms         16.3 ms          100
append_only_tree_bench<Pedersen>/128/iterations:100         27.8 ms         27.8 ms          100

append_only_tree_bench<Poseidon2>/2/iterations:1000        0.395 ms        0.395 ms         1000
append_only_tree_bench<Poseidon2>/4/iterations:1000        0.407 ms        0.408 ms         1000
append_only_tree_bench<Poseidon2>/8/iterations:1000        0.454 ms        0.454 ms         1000
append_only_tree_bench<Poseidon2>/16/iterations:1000       0.531 ms        0.531 ms         1000
append_only_tree_bench<Poseidon2>/32/iterations:1000       0.726 ms        0.726 ms         1000
append_only_tree_bench<Poseidon2>/64/iterations:1000        1.11 ms         1.11 ms         1000
append_only_tree_bench<Poseidon2>/128/iterations:1000       1.89 ms         1.89 ms         1000
```

**Indexed Tree**
```
----------------------------------------------------------------------------------------------------------
Benchmark                                                                Time             CPU   Iterations
----------------------------------------------------------------------------------------------------------
single_thread_indexed_tree_bench<Pedersen>/2/iterations:100           18.2 ms         18.2 ms          100
single_thread_indexed_tree_bench<Pedersen>/4/iterations:100           31.3 ms         31.3 ms          100
single_thread_indexed_tree_bench<Pedersen>/8/iterations:100           57.2 ms         57.2 ms          100
single_thread_indexed_tree_bench<Pedersen>/16/iterations:100           108 ms          108 ms          100
single_thread_indexed_tree_bench<Pedersen>/32/iterations:100           210 ms          210 ms          100
single_thread_indexed_tree_bench<Pedersen>/64/iterations:100           415 ms          415 ms          100
single_thread_indexed_tree_bench<Pedersen>/128/iterations:100          824 ms          824 ms          100

multi_thread_indexed_tree_bench<Pedersen>/2/iterations:100            12.8 ms         6.65 ms          100
multi_thread_indexed_tree_bench<Pedersen>/4/iterations:100            14.8 ms         7.31 ms          100
multi_thread_indexed_tree_bench<Pedersen>/8/iterations:100            18.5 ms         8.80 ms          100
multi_thread_indexed_tree_bench<Pedersen>/16/iterations:100           23.9 ms         12.0 ms          100
multi_thread_indexed_tree_bench<Pedersen>/32/iterations:100           35.1 ms         18.5 ms          100
multi_thread_indexed_tree_bench<Pedersen>/64/iterations:100           59.3 ms         31.6 ms          100
multi_thread_indexed_tree_bench<Pedersen>/128/iterations:100           109 ms         66.5 ms          100

single_thread_indexed_tree_bench<Poseidon2>/2/iterations:1000         1.26 ms         1.26 ms         1000
single_thread_indexed_tree_bench<Poseidon2>/4/iterations:1000         2.15 ms         2.15 ms         1000
single_thread_indexed_tree_bench<Poseidon2>/8/iterations:1000         3.88 ms         3.88 ms         1000
single_thread_indexed_tree_bench<Poseidon2>/16/iterations:1000        7.41 ms         7.41 ms         1000
single_thread_indexed_tree_bench<Poseidon2>/32/iterations:1000        14.6 ms         14.6 ms         1000
single_thread_indexed_tree_bench<Poseidon2>/64/iterations:1000        28.7 ms         28.7 ms         1000
single_thread_indexed_tree_bench<Poseidon2>/128/iterations:1000       56.8 ms         56.8 ms         1000

multi_thread_indexed_tree_bench<Poseidon2>/2/iterations:1000          1.01 ms        0.812 ms         1000
multi_thread_indexed_tree_bench<Poseidon2>/4/iterations:1000          1.17 ms        0.865 ms         1000
multi_thread_indexed_tree_bench<Poseidon2>/8/iterations:1000          1.46 ms        0.979 ms         1000
multi_thread_indexed_tree_bench<Poseidon2>/16/iterations:1000         2.05 ms         1.23 ms         1000
multi_thread_indexed_tree_bench<Poseidon2>/32/iterations:1000         3.24 ms         1.70 ms         1000
multi_thread_indexed_tree_bench<Poseidon2>/64/iterations:1000         5.84 ms         2.54 ms         1000
multi_thread_indexed_tree_bench<Poseidon2>/128/iterations:1000        13.3 ms         6.36 ms         1000
```

---------

Co-authored-by: ludamad <adam@aztecprotocol.com>

barretenberg/cpp/scripts/stdlib-tests

@@ -3,7 +3,6 @@ stdlib_aes128_tests
 stdlib_blake2s_tests
 stdlib_blake3s_tests
 stdlib_ecdsa_tests
-stdlib_merkle_tree_tests
 stdlib_pedersen_commitment_tests
 stdlib_pedersen_hash_tests
 stdlib_poseidon2_tests

barretenberg/cpp/src/CMakeLists.txt

@@ -127,7 +127,7 @@ set(BARRETENBERG_TARGET_OBJECTS
     $<TARGET_OBJECTS:stdlib_blake2s_objects>
     $<TARGET_OBJECTS:stdlib_blake3s_objects>
     $<TARGET_OBJECTS:stdlib_keccak_objects>
-    $<TARGET_OBJECTS:stdlib_merkle_tree_objects>
+    $<TARGET_OBJECTS:crypto_merkle_tree_objects>
     $<TARGET_OBJECTS:stdlib_pedersen_commitment_objects>
     $<TARGET_OBJECTS:stdlib_pedersen_hash_objects>
     $<TARGET_OBJECTS:stdlib_poseidon2_objects>

barretenberg/cpp/src/barretenberg/benchmark/CMakeLists.txt

@@ -7,5 +7,9 @@ add_subdirectory(pippenger_bench)
 add_subdirectory(plonk_bench)
 add_subdirectory(protogalaxy_bench)
 add_subdirectory(relations_bench)
+add_subdirectory(widgets_bench)
+add_subdirectory(poseidon2_bench)
+add_subdirectory(merkle_tree_bench)
+add_subdirectory(indexed_tree_bench)
+add_subdirectory(append_only_tree_bench)
 add_subdirectory(ultra_bench)
-add_subdirectory(widgets_bench)

barretenberg/cpp/src/barretenberg/benchmark/append_only_tree_bench/CMakeLists.txt

@@ -0,0 +1 @@
+barretenberg_module(append_only_tree_bench crypto_poseidon2 crypto_merkle_tree)

barretenberg/cpp/src/barretenberg/benchmark/append_only_tree_bench/append_only_tree.bench.cpp

@@ -0,0 +1,54 @@
+#include "barretenberg/crypto/merkle_tree/append_only_tree/append_only_tree.hpp"
+#include "barretenberg/crypto/merkle_tree/array_store.hpp"
+#include "barretenberg/crypto/merkle_tree/hash.hpp"
+#include "barretenberg/numeric/random/engine.hpp"
+#include <benchmark/benchmark.h>
+
+using namespace benchmark;
+using namespace bb::crypto::merkle_tree;
+
+using Pedersen = AppendOnlyTree<ArrayStore, PedersenHashPolicy>;
+using Poseidon2 = AppendOnlyTree<ArrayStore, Poseidon2HashPolicy>;
+
+const size_t TREE_DEPTH = 32;
+const size_t MAX_BATCH_SIZE = 128;
+
+namespace {
+auto& random_engine = bb::numeric::get_randomness();
+} // namespace
+
+template <typename TreeType> void perform_batch_insert(TreeType& tree, const std::vector<fr>& values)
+{
+    tree.add_values(values);
+}
+
+template <typename TreeType> void append_only_tree_bench(State& state) noexcept
+{
+    const size_t batch_size = size_t(state.range(0));
+    const size_t depth = TREE_DEPTH;
+
+    ArrayStore store(depth, 1024 * 1024);
+    TreeType tree = TreeType(store, depth);
+
+    for (auto _ : state) {
+        state.PauseTiming();
+        std::vector<fr> values(batch_size);
+        for (size_t i = 0; i < batch_size; ++i) {
+            values[i] = fr(random_engine.get_random_uint256());
+        }
+        state.ResumeTiming();
+        perform_batch_insert(tree, values);
+    }
+}
+BENCHMARK(append_only_tree_bench<Pedersen>)
+    ->Unit(benchmark::kMillisecond)
+    ->RangeMultiplier(2)
+    ->Range(2, MAX_BATCH_SIZE)
+    ->Iterations(100);
+BENCHMARK(append_only_tree_bench<Poseidon2>)
+    ->Unit(benchmark::kMillisecond)
+    ->RangeMultiplier(2)
+    ->Range(2, MAX_BATCH_SIZE)
+    ->Iterations(1000);
+
+BENCHMARK_MAIN();

barretenberg/cpp/src/barretenberg/benchmark/goblin_bench/CMakeLists.txt

@@ -1 +1 @@
-barretenberg_module(goblin_bench ultra_honk eccvm stdlib_recursion stdlib_sha256 stdlib_merkle_tree stdlib_primitives)
+barretenberg_module(goblin_bench ultra_honk eccvm stdlib_recursion stdlib_sha256 crypto_merkle_tree stdlib_primitives)

barretenberg/cpp/src/barretenberg/benchmark/indexed_tree_bench/CMakeLists.txt

@@ -0,0 +1 @@
+barretenberg_module(indexed_tree_bench crypto_poseidon2 crypto_merkle_tree)

barretenberg/cpp/src/barretenberg/benchmark/indexed_tree_bench/indexed_tree.bench.cpp

@@ -0,0 +1,87 @@
+#include "barretenberg/crypto/merkle_tree/indexed_tree/indexed_tree.hpp"
+#include "barretenberg/crypto/merkle_tree/array_store.hpp"
+#include "barretenberg/crypto/merkle_tree/hash.hpp"
+#include "barretenberg/crypto/merkle_tree/indexed_tree/leaves_cache.hpp"
+#include "barretenberg/numeric/random/engine.hpp"
+#include <benchmark/benchmark.h>
+
+using namespace benchmark;
+using namespace bb::crypto::merkle_tree;
+
+using Poseidon2 = IndexedTree<ArrayStore, LeavesCache, Poseidon2HashPolicy>;
+using Pedersen = IndexedTree<ArrayStore, LeavesCache, PedersenHashPolicy>;
+
+const size_t TREE_DEPTH = 32;
+const size_t MAX_BATCH_SIZE = 128;
+
+namespace {
+auto& random_engine = bb::numeric::get_randomness();
+} // namespace
+
+template <typename TreeType>
+void perform_batch_insert(TreeType& tree, const std::vector<fr>& values, bool single_threaded)
+{
+    tree.add_or_update_values(values, single_threaded);
+}
+
+template <typename TreeType> void multi_thread_indexed_tree_bench(State& state) noexcept
+{
+    const size_t batch_size = size_t(state.range(0));
+    const size_t depth = TREE_DEPTH;
+
+    ArrayStore store(depth, 1024 * 1024);
+    TreeType tree = TreeType(store, depth, batch_size);
+
+    for (auto _ : state) {
+        state.PauseTiming();
+        std::vector<fr> values(batch_size);
+        for (size_t i = 0; i < batch_size; ++i) {
+            values[i] = fr(random_engine.get_random_uint256());
+        }
+        state.ResumeTiming();
+        perform_batch_insert(tree, values, false);
+    }
+}
+
+template <typename TreeType> void single_thread_indexed_tree_bench(State& state) noexcept
+{
+    const size_t batch_size = size_t(state.range(0));
+    const size_t depth = TREE_DEPTH;
+
+    ArrayStore store(depth, 1024 * 1024);
+    TreeType tree = TreeType(store, depth, batch_size);
+
+    for (auto _ : state) {
+        state.PauseTiming();
+        std::vector<fr> values(batch_size);
+        for (size_t i = 0; i < batch_size; ++i) {
+            values[i] = fr(random_engine.get_random_uint256());
+        }
+        state.ResumeTiming();
+        perform_batch_insert(tree, values, true);
+    }
+}
+BENCHMARK(single_thread_indexed_tree_bench<Pedersen>)
+    ->Unit(benchmark::kMillisecond)
+    ->RangeMultiplier(2)
+    ->Range(2, MAX_BATCH_SIZE)
+    ->Iterations(100);
+
+BENCHMARK(multi_thread_indexed_tree_bench<Pedersen>)
+    ->Unit(benchmark::kMillisecond)
+    ->RangeMultiplier(2)
+    ->Range(2, MAX_BATCH_SIZE)
+    ->Iterations(100);
+
+BENCHMARK(single_thread_indexed_tree_bench<Poseidon2>)
+    ->Unit(benchmark::kMillisecond)
+    ->RangeMultiplier(2)
+    ->Range(2, MAX_BATCH_SIZE)
+    ->Iterations(1000);
+BENCHMARK(multi_thread_indexed_tree_bench<Poseidon2>)
+    ->Unit(benchmark::kMillisecond)
+    ->RangeMultiplier(2)
+    ->Range(2, MAX_BATCH_SIZE)
+    ->Iterations(1000);
+
+BENCHMARK_MAIN();

barretenberg/cpp/src/barretenberg/benchmark/ivc_bench/CMakeLists.txt

@@ -1 +1 @@
-barretenberg_module(ivc_bench client_ivc stdlib_recursion stdlib_sha256 stdlib_merkle_tree stdlib_primitives)
+barretenberg_module(ivc_bench client_ivc stdlib_recursion stdlib_sha256 crypto_merkle_tree stdlib_primitives)

barretenberg/cpp/src/barretenberg/benchmark/merkle_tree_bench/CMakeLists.txt

@@ -0,0 +1 @@
+barretenberg_module(merkle_tree_bench crypto_poseidon2 crypto_merkle_tree)

barretenberg/cpp/src/barretenberg/stdlib/merkle_tree/merkle_tree.bench.cpp → barretenberg/cpp/src/barretenberg/benchmark/merkle_tree_bench/merkle_tree.bench.cpp

@@ -1,15 +1,17 @@
-#include "merkle_tree.hpp"
+#include "barretenberg/crypto/merkle_tree/merkle_tree.hpp"
+#include "barretenberg/crypto/merkle_tree/hash.hpp"
+#include "barretenberg/crypto/merkle_tree/memory_store.hpp"
 #include "barretenberg/numeric/random/engine.hpp"
-#include "hash.hpp"
-#include "memory_store.hpp"
 #include <benchmark/benchmark.h>
 
 using namespace benchmark;
-using namespace bb::stdlib::merkle_tree;
+using namespace bb::crypto::merkle_tree;
+
+using TreeType = MerkleTree<MemoryStore, PedersenHashPolicy>;
 
 namespace {
 auto& engine = bb::numeric::get_debug_randomness();
-}
+} // namespace
 
 constexpr size_t DEPTH = 256;
 constexpr size_t MAX = 4096;
@@ -33,7 +35,7 @@ BENCHMARK(hash)->MinTime(5);
 void update_first_element(State& state) noexcept
 {
     MemoryStore store;
-    MerkleTree<MemoryStore> db(store, DEPTH);
+    TreeType db(store, DEPTH);
 
     for (auto _ : state) {
         db.update_element(0, VALUES[1]);
@@ -46,7 +48,7 @@ void update_elements(State& state) noexcept
     for (auto _ : state) {
         state.PauseTiming();
         MemoryStore store;
-        MerkleTree<MemoryStore> db(store, DEPTH);
+        TreeType db(store, DEPTH);
         state.ResumeTiming();
         for (size_t i = 0; i < (size_t)state.range(0); ++i) {
             db.update_element(i, VALUES[i]);
@@ -60,10 +62,10 @@ void update_random_elements(State& state) noexcept
     for (auto _ : state) {
         state.PauseTiming();
         MemoryStore store;
-        MerkleTree db(store, DEPTH);
+        TreeType db(store, DEPTH);
         for (size_t i = 0; i < (size_t)state.range(0); i++) {
             state.PauseTiming();
-            auto index = MerkleTree<MemoryStore>::index_t(engine.get_random_uint256());
+            auto index = TreeType::index_t(engine.get_random_uint256());
             state.ResumeTiming();
             db.update_element(index, VALUES[i]);
         }

barretenberg/cpp/src/barretenberg/benchmark/poseidon2_bench/CMakeLists.txt

@@ -0,0 +1 @@
+barretenberg_module(poseidon2_bench crypto_poseidon2)

barretenberg/cpp/src/barretenberg/crypto/poseidon2/poseidon2.bench.cpp → barretenberg/cpp/src/barretenberg/benchmark/poseidon2_bench/poseidon2.bench.cpp

@@ -1,4 +1,4 @@
-#include "./poseidon2.hpp"
+#include "barretenberg/crypto/poseidon2/poseidon2.hpp"
 #include "barretenberg/ecc/curves/grumpkin/grumpkin.hpp"
 #include <benchmark/benchmark.h>
 
@@ -24,4 +24,20 @@ void native_poseidon2_commitment_bench(State& state) noexcept
 }
 BENCHMARK(native_poseidon2_commitment_bench)->Arg(10)->Arg(1000)->Arg(10000);
 
+grumpkin::fq poseiden_hash_impl(const grumpkin::fq& x, const grumpkin::fq& y)
+{
+    std::vector<grumpkin::fq> to_hash{ x, y };
+    return bb::crypto::Poseidon2<bb::crypto::Poseidon2Bn254ScalarFieldParams>::hash(to_hash);
+}
+
+void poseiden_hash_bench(State& state) noexcept
+{
+    grumpkin::fq x = grumpkin::fq::random_element();
+    grumpkin::fq y = grumpkin::fq::random_element();
+    for (auto _ : state) {
+        DoNotOptimize(poseiden_hash_impl(x, y));
+    }
+}
+BENCHMARK(poseiden_hash_bench)->Unit(benchmark::kMillisecond);
+
 BENCHMARK_MAIN();

barretenberg/cpp/src/barretenberg/benchmark/ultra_bench/CMakeLists.txt

@@ -2,5 +2,5 @@ barretenberg_module(ultra_bench
   ultra_honk
   stdlib_sha256
   stdlib_keccak
-  stdlib_merkle_tree
+  crypto_merkle_tree
   stdlib_recursion)

barretenberg/cpp/src/barretenberg/benchmark/ultra_bench/mock_proofs.hpp

@@ -2,17 +2,17 @@
 #include <benchmark/benchmark.h>
 #include <cstddef>
 
+#include "barretenberg/crypto/merkle_tree/membership.hpp"
+#include "barretenberg/crypto/merkle_tree/memory_store.hpp"
+#include "barretenberg/crypto/merkle_tree/memory_tree.hpp"
+#include "barretenberg/crypto/merkle_tree/merkle_tree.hpp"
 #include "barretenberg/goblin/mock_circuits.hpp"
 #include "barretenberg/plonk/composer/standard_composer.hpp"
 #include "barretenberg/plonk/composer/ultra_composer.hpp"
 #include "barretenberg/proof_system/types/circuit_type.hpp"
 #include "barretenberg/stdlib/encryption/ecdsa/ecdsa.hpp"
 #include "barretenberg/stdlib/hash/keccak/keccak.hpp"
 #include "barretenberg/stdlib/hash/sha256/sha256.hpp"
-#include "barretenberg/stdlib/merkle_tree/membership.hpp"
-#include "barretenberg/stdlib/merkle_tree/memory_store.hpp"
-#include "barretenberg/stdlib/merkle_tree/memory_tree.hpp"
-#include "barretenberg/stdlib/merkle_tree/merkle_tree.hpp"
 #include "barretenberg/stdlib/primitives/bool/bool.hpp"
 #include "barretenberg/stdlib/primitives/curves/secp256k1.hpp"
 #include "barretenberg/stdlib/primitives/field/field.hpp"

barretenberg/cpp/src/barretenberg/crypto/CMakeLists.txt

@@ -9,4 +9,5 @@ add_subdirectory(schnorr)
 add_subdirectory(sha256)
 add_subdirectory(ecdsa)
 add_subdirectory(aes128)
-add_subdirectory(poseidon2)
+add_subdirectory(poseidon2)
+add_subdirectory(merkle_tree)

barretenberg/cpp/src/barretenberg/crypto/merkle_tree/CMakeLists.txt

@@ -0,0 +1 @@
+barretenberg_module(crypto_merkle_tree stdlib_primitives stdlib_blake3s stdlib_pedersen_hash)