feat: Insert public data tree leaves one by one (#9989)

This PR:
 - Splits base rollup into public base and private base
 - Makes the private base only perform the fee write
 - The public base writes public data tree leaves one by one
 - The world state allows advancing the tree blocknumbers with no writes
 - We don't pad anymore the public data writes
- For now we get witnesses for "one by one" insertion in the public data
tree by calling world state one time per written item
- Sync still adds all the leaves in one go, since no individual
witnesses are necessary

barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_tree_store.cpp

@@ -188,7 +188,7 @@ void LMDBTreeStore::increment_node_reference_count(const fr& nodeHash, WriteTran
     NodePayload nodePayload;
     bool success = get_node_data(nodeHash, nodePayload, tx);
     if (!success) {
-        throw std::runtime_error("Failed to find node when attempting to increases reference count");
+        throw std::runtime_error("Failed to find node when attempting to increase reference count");
     }
     ++nodePayload.ref;
     // std::cout << "Incrementing siblng at " << nodeHash << ", to " << nodePayload.ref << std::endl;
@@ -212,7 +212,7 @@ void LMDBTreeStore::decrement_node_reference_count(const fr& nodeHash, NodePaylo
 {
     bool success = get_node_data(nodeHash, nodeData, tx);
     if (!success) {
-        throw std::runtime_error("Failed to find node when attempting to increases reference count");
+        throw std::runtime_error("Failed to find node when attempting to decrease reference count");
     }
     if (--nodeData.ref == 0) {
         // std::cout << "Deleting node at " << nodeHash << std::endl;

barretenberg/cpp/src/barretenberg/crypto/merkle_tree/node_store/cached_content_addressed_tree_store.hpp

@@ -12,6 +12,7 @@
 #include "msgpack/assert.hpp"
 #include <cstdint>
 #include <exception>
+#include <iostream>
 #include <memory>
 #include <mutex>
 #include <optional>
@@ -643,17 +644,13 @@ void ContentAddressedCachedTreeStore<LeafValueType>::commit(TreeMeta& finalMeta,
 
         // if the meta datas are different, we have uncommitted data
         bool metaToCommit = committedMeta != uncommittedMeta;
-        if (!metaToCommit) {
+        if (!metaToCommit && !asBlock) {
             return;
         }
+
         auto currentRootIter = nodes_.find(uncommittedMeta.root);
         dataPresent = currentRootIter != nodes_.end();
-        if (!dataPresent) {
-            // no uncommitted data present, if we were asked to commit as a block then we can't
-            if (asBlock) {
-                throw std::runtime_error("Can't commit as block if no data present");
-            }
-        } else {
+        if (dataPresent) {
             // data is present, hydrate persisted indices
             hydrate_indices_from_persisted_store(*tx);
         }
@@ -665,19 +662,27 @@ void ContentAddressedCachedTreeStore<LeafValueType>::commit(TreeMeta& finalMeta,
                 // std::cout << "Persisting data for block " << uncommittedMeta.unfinalisedBlockHeight + 1 << std::endl;
                 persist_leaf_indices(*tx);
                 persist_leaf_keys(uncommittedMeta.committedSize, *tx);
+            }
+            // If we are commiting a block, we need to persist the root, since the new block "references" this root
+            // However, if the root is the empty root we can't persist it, since it's not a real node
+            // We are abusing the trees in some tests, trying to add empty blocks to initial empty trees
+            // That is not expected behavior since the unwind operation will fail trying to decrease refcount
+            // for the empty root, which doesn't exist.
+            if (dataPresent || (asBlock && uncommittedMeta.size > 0)) {
                 persist_node(std::optional<fr>(uncommittedMeta.root), 0, *tx);
-                if (asBlock) {
-                    ++uncommittedMeta.unfinalisedBlockHeight;
-                    if (uncommittedMeta.oldestHistoricBlock == 0) {
-                        uncommittedMeta.oldestHistoricBlock = 1;
-                    }
-                    // std::cout << "New root " << uncommittedMeta.root << std::endl;
-                    BlockPayload block{ .size = uncommittedMeta.size,
-                                        .blockNumber = uncommittedMeta.unfinalisedBlockHeight,
-                                        .root = uncommittedMeta.root };
-                    dataStore_->write_block_data(uncommittedMeta.unfinalisedBlockHeight, block, *tx);
+            }
+            if (asBlock) {
+                ++uncommittedMeta.unfinalisedBlockHeight;
+                if (uncommittedMeta.oldestHistoricBlock == 0) {
+                    uncommittedMeta.oldestHistoricBlock = 1;
                 }
+                // std::cout << "New root " << uncommittedMeta.root << std::endl;
+                BlockPayload block{ .size = uncommittedMeta.size,
+                                    .blockNumber = uncommittedMeta.unfinalisedBlockHeight,
+                                    .root = uncommittedMeta.root };
+                dataStore_->write_block_data(uncommittedMeta.unfinalisedBlockHeight, block, *tx);
             }
+
             uncommittedMeta.committedSize = uncommittedMeta.size;
             persist_meta(uncommittedMeta, *tx);
             tx->commit();

barretenberg/cpp/src/barretenberg/world_state_napi/addon.cpp

@@ -560,7 +560,7 @@ bool WorldStateAddon::sync_block(msgpack::object& obj, msgpack::sbuffer& buf)
                                                   request.value.paddedNoteHashes,
                                                   request.value.paddedL1ToL2Messages,
                                                   request.value.paddedNullifiers,
-                                                  request.value.batchesOfPaddedPublicDataWrites);
+                                                  request.value.batchesOfPublicDataWrites);
 
     MsgHeader header(request.header.messageId);
     messaging::TypedMessage<WorldStateStatusFull> resp_msg(WorldStateMessageType::SYNC_BLOCK, header, { status });

barretenberg/cpp/src/barretenberg/world_state_napi/message.hpp

@@ -181,15 +181,15 @@ struct SyncBlockRequest {
     bb::fr blockHeaderHash;
     std::vector<bb::fr> paddedNoteHashes, paddedL1ToL2Messages;
     std::vector<crypto::merkle_tree::NullifierLeafValue> paddedNullifiers;
-    std::vector<std::vector<crypto::merkle_tree::PublicDataLeafValue>> batchesOfPaddedPublicDataWrites;
+    std::vector<std::vector<crypto::merkle_tree::PublicDataLeafValue>> batchesOfPublicDataWrites;
 
     MSGPACK_FIELDS(blockNumber,
                    blockStateRef,
                    blockHeaderHash,
                    paddedNoteHashes,
                    paddedL1ToL2Messages,
                    paddedNullifiers,
-                   batchesOfPaddedPublicDataWrites);
+                   batchesOfPublicDataWrites);
 };
 
 } // namespace bb::world_state

l1-contracts/src/core/libraries/ConstantsGen.sol

@@ -52,7 +52,6 @@ library Constants {
   uint256 internal constant L1_TO_L2_MSG_SUBTREE_HEIGHT = 4;
   uint256 internal constant NOTE_HASH_SUBTREE_SIBLING_PATH_LENGTH = 34;
   uint256 internal constant NULLIFIER_SUBTREE_SIBLING_PATH_LENGTH = 34;
-  uint256 internal constant PUBLIC_DATA_SUBTREE_SIBLING_PATH_LENGTH = 34;
   uint256 internal constant L1_TO_L2_MSG_SUBTREE_SIBLING_PATH_LENGTH = 35;
   uint256 internal constant MAX_NOTE_HASHES_PER_TX = 64;
   uint256 internal constant MAX_NULLIFIERS_PER_TX = 64;

noir-projects/noir-protocol-circuits/crates/rollup-lib/src/abis/base_or_merge_rollup_public_inputs.nr

@@ -6,8 +6,8 @@ use dep::types::{
     utils::reader::Reader,
 };
 
-global BASE_ROLLUP_TYPE = 0;
-global MERGE_ROLLUP_TYPE = 1;
+pub(crate) global BASE_ROLLUP_TYPE = 0;
+pub(crate) global MERGE_ROLLUP_TYPE = 1;
 
 pub struct BaseOrMergeRollupPublicInputs {
     // rollup_type is either 0 (base) or 1 (merge)

noir-projects/noir-protocol-circuits/crates/rollup-lib/src/abis/constant_rollup_data.nr

@@ -7,10 +7,10 @@ use dep::types::{
 
 pub struct ConstantRollupData {
     // Archive tree snapshot at the very beginning of the entire rollup.
-    last_archive: AppendOnlyTreeSnapshot,
-    vk_tree_root: Field,
-    protocol_contract_tree_root: Field,
-    global_variables: GlobalVariables,
+    pub last_archive: AppendOnlyTreeSnapshot,
+    pub vk_tree_root: Field,
+    pub protocol_contract_tree_root: Field,
+    pub global_variables: GlobalVariables,
 }
 
 impl Eq for ConstantRollupData {

noir-projects/noir-protocol-circuits/crates/rollup-lib/src/abis/mod.nr

@@ -1,5 +1,5 @@
-mod constant_rollup_data;
-mod base_or_merge_rollup_public_inputs;
-mod block_root_or_block_merge_public_inputs;
-mod previous_rollup_data;
-mod previous_rollup_block_data;
+pub(crate) mod constant_rollup_data;
+pub(crate) mod base_or_merge_rollup_public_inputs;
+pub(crate) mod block_root_or_block_merge_public_inputs;
+pub(crate) mod previous_rollup_data;
+pub(crate) mod previous_rollup_block_data;