chore(trie): move trie updates to reth-trie-common

crates/engine/invalid-block-hooks/Cargo.toml

@@ -20,7 +20,7 @@ reth-provider.workspace = true
 reth-revm = { workspace = true, features = ["serde"] }
 reth-rpc-api = { workspace = true, features = ["client"] }
 reth-tracing.workspace = true
-reth-trie = { workspace = true, features = ["serde"] }
+reth-trie.workspace = true
 
 # alloy
 alloy-primitives.workspace = true

crates/evm/execution-types/Cargo.toml

@@ -12,9 +12,10 @@ workspace = true
 
 [dependencies]
 reth-primitives.workspace = true
+reth-primitives-traits.workspace = true
 reth-execution-errors.workspace = true
+reth-trie-common.workspace = true
 reth-trie.workspace = true
-reth-primitives-traits.workspace = true
 
 revm.workspace = true
 
@@ -36,17 +37,17 @@ default = ["std"]
 optimism = ["reth-primitives/optimism", "revm/optimism"]
 serde = [
 	"dep:serde",
-	"reth-trie/serde",
+	"rand/serde",
 	"revm/serde",
 	"alloy-eips/serde",
 	"alloy-primitives/serde",
-	"rand/serde",
 	"reth-primitives-traits/serde",
+	"reth-trie-common/serde",
 ]
 serde-bincode-compat = [
 	"reth-primitives/serde-bincode-compat",
 	"reth-primitives-traits/serde-bincode-compat",
-	"reth-trie/serde-bincode-compat",
+	"reth-trie-common/serde-bincode-compat",
 	"serde_with",
 	"alloy-eips/serde-bincode-compat",
 ]

crates/evm/execution-types/src/chain.rs

@@ -11,7 +11,7 @@ use reth_primitives::{
     TransactionSignedEcRecovered,
 };
 use reth_primitives_traits::NodePrimitives;
-use reth_trie::updates::TrieUpdates;
+use reth_trie_common::updates::TrieUpdates;
 use revm::db::BundleState;
 
 /// A chain of blocks and their final state.
@@ -513,16 +513,14 @@ pub enum ChainSplit<N: NodePrimitives = reth_primitives::EthPrimitives> {
 /// Bincode-compatible [`Chain`] serde implementation.
 #[cfg(all(feature = "serde", feature = "serde-bincode-compat"))]
 pub(super) mod serde_bincode_compat {
-    use std::collections::BTreeMap;
-
+    use crate::ExecutionOutcome;
     use alloc::borrow::Cow;
     use alloy_primitives::BlockNumber;
     use reth_primitives::serde_bincode_compat::SealedBlockWithSenders;
-    use reth_trie::serde_bincode_compat::updates::TrieUpdates;
+    use reth_trie_common::serde_bincode_compat::updates::TrieUpdates;
     use serde::{ser::SerializeMap, Deserialize, Deserializer, Serialize, Serializer};
     use serde_with::{DeserializeAs, SerializeAs};
-
-    use crate::ExecutionOutcome;
+    use std::collections::BTreeMap;
 
     /// Bincode-compatible [`super::Chain`] serde implementation.
     ///

crates/revm/Cargo.toml

@@ -55,7 +55,6 @@ test-utils = [
 ]
 serde = [
     "revm/serde",
-    "reth-trie?/serde",
     "alloy-eips/serde",
     "alloy-primitives/serde",
     "alloy-consensus/serde",

crates/storage/provider/Cargo.toml

@@ -100,7 +100,6 @@ optimism = [
 serde = [
     "reth-execution-types/serde",
     "reth-trie-db/serde",
-    "reth-trie/serde",
     "alloy-consensus/serde",
     "alloy-eips/serde",
     "alloy-primitives/serde",

crates/trie/common/Cargo.toml

@@ -31,6 +31,9 @@ nybbles = { workspace = true, features = ["rlp"] }
 # `serde` feature
 serde = { workspace = true, optional = true }
 
+# `serde-bincode-compat` feature
+serde_with = { workspace = true, optional = true }
+
 # `test-utils` feature
 hash-db = { version = "=0.15.2", optional = true }
 plain_hasher = { version = "0.2", optional = true }
@@ -46,6 +49,7 @@ arbitrary = { workspace = true, features = ["derive"] }
 proptest.workspace = true
 proptest-arbitrary-interop.workspace = true
 criterion.workspace = true
+bincode.workspace = true
 
 [features]
 serde = [
@@ -59,6 +63,11 @@ serde = [
     "reth-primitives-traits/serde",
     "reth-codecs/serde"
 ]
+serde-bincode-compat = [
+	"serde_with",
+	"reth-primitives-traits/serde-bincode-compat",
+	"alloy-consensus/serde-bincode-compat"
+]
 test-utils = [
     "dep:plain_hasher",
     "dep:hash-db",

crates/trie/common/src/lib.rs

@@ -37,5 +37,19 @@ pub use proofs::*;
 
 pub mod root;
 
+/// Buffer for trie updates.
+pub mod updates;
+
+/// Bincode-compatible serde implementations for trie types.
+///
+/// `bincode` crate allows for more efficient serialization of trie types, because it allows
+/// non-string map keys.
+///
+/// Read more: <https://github.com/paradigmxyz/reth/issues/11370>
+#[cfg(all(feature = "serde", feature = "serde-bincode-compat"))]
+pub mod serde_bincode_compat {
+    pub use super::updates::serde_bincode_compat as updates;
+}
+
 /// Re-export
 pub use alloy_trie::{nodes::*, proof, BranchNodeCompact, HashBuilder, TrieMask, EMPTY_ROOT_HASH};

crates/trie/trie/src/updates.rs → crates/trie/common/src/updates.rs

@@ -1,4 +1,4 @@
-use crate::{walker::TrieWalker, BranchNodeCompact, HashBuilder, Nibbles};
+use crate::{BranchNodeCompact, HashBuilder, Nibbles};
 use alloy_primitives::B256;
 use std::collections::{HashMap, HashSet};
 
@@ -78,20 +78,19 @@ impl TrieUpdates {
     }
 
     /// Finalize state trie updates.
-    pub fn finalize<C>(
+    pub fn finalize(
         &mut self,
-        walker: TrieWalker<C>,
         hash_builder: HashBuilder,
+        removed_keys: HashSet<Nibbles>,
         destroyed_accounts: HashSet<B256>,
     ) {
-        // Retrieve deleted keys from trie walker.
-        let (_, removed_node_keys) = walker.split();
-        self.removed_nodes.extend(exclude_empty(removed_node_keys));
-
         // Retrieve updated nodes from hash builder.
         let (_, updated_nodes) = hash_builder.split();
         self.account_nodes.extend(exclude_empty_from_pair(updated_nodes));
 
+        // Add deleted node paths.
+        self.removed_nodes.extend(exclude_empty(removed_keys));
+
         // Add deleted storage tries for destroyed accounts.
         for destroyed in destroyed_accounts {
             self.storage_tries.entry(destroyed).or_default().set_deleted(true);
@@ -201,14 +200,13 @@ impl StorageTrieUpdates {
     }
 
     /// Finalize storage trie updates for by taking updates from walker and hash builder.
-    pub fn finalize<C>(&mut self, walker: TrieWalker<C>, hash_builder: HashBuilder) {
-        // Retrieve deleted keys from trie walker.
-        let (_, removed_keys) = walker.split();
-        self.removed_nodes.extend(exclude_empty(removed_keys));
-
+    pub fn finalize(&mut self, hash_builder: HashBuilder, removed_keys: HashSet<Nibbles>) {
         // Retrieve updated nodes from hash builder.
         let (_, updated_nodes) = hash_builder.split();
         self.storage_nodes.extend(exclude_empty_from_pair(updated_nodes));
+
+        // Add deleted node paths.
+        self.removed_nodes.extend(exclude_empty(removed_keys));
     }
 
     /// Convert storage trie updates into [`StorageTrieUpdatesSorted`].
@@ -229,10 +227,9 @@ impl StorageTrieUpdates {
 /// This also sorts the set before serializing.
 #[cfg(feature = "serde")]
 mod serde_nibbles_set {
-    use std::collections::HashSet;
-
-    use reth_trie_common::Nibbles;
+    use crate::Nibbles;
     use serde::{de::Error, Deserialize, Deserializer, Serialize, Serializer};
+    use std::collections::HashSet;
 
     pub(super) fn serialize<S>(map: &HashSet<Nibbles>, serializer: S) -> Result<S::Ok, S::Error>
     where
@@ -266,15 +263,14 @@ mod serde_nibbles_set {
 /// This also sorts the map's keys before encoding and serializing.
 #[cfg(feature = "serde")]
 mod serde_nibbles_map {
-    use std::{collections::HashMap, marker::PhantomData};
-
+    use crate::Nibbles;
     use alloy_primitives::hex;
-    use reth_trie_common::Nibbles;
     use serde::{
         de::{Error, MapAccess, Visitor},
         ser::SerializeMap,
         Deserialize, Deserializer, Serialize, Serializer,
     };
+    use std::{collections::HashMap, marker::PhantomData};
 
     pub(super) fn serialize<S, T>(
         map: &HashMap<Nibbles, T>,
@@ -340,9 +336,13 @@ mod serde_nibbles_map {
 /// Sorted trie updates used for lookups and insertions.
 #[derive(PartialEq, Eq, Clone, Default, Debug)]
 pub struct TrieUpdatesSorted {
-    pub(crate) account_nodes: Vec<(Nibbles, BranchNodeCompact)>,
-    pub(crate) removed_nodes: HashSet<Nibbles>,
-    pub(crate) storage_tries: HashMap<B256, StorageTrieUpdatesSorted>,
+    /// Sorted collection of updated state nodes with corresponding paths.
+    pub account_nodes: Vec<(Nibbles, BranchNodeCompact)>,
+    /// The set of removed state node keys.
+    pub removed_nodes: HashSet<Nibbles>,
+    /// Storage tries storage stored by hashed address of the account
+    /// the trie belongs to.
+    pub storage_tries: HashMap<B256, StorageTrieUpdatesSorted>,
 }
 
 impl TrieUpdatesSorted {
@@ -365,9 +365,12 @@ impl TrieUpdatesSorted {
 /// Sorted trie updates used for lookups and insertions.
 #[derive(PartialEq, Eq, Clone, Default, Debug)]
 pub struct StorageTrieUpdatesSorted {
-    pub(crate) is_deleted: bool,
-    pub(crate) storage_nodes: Vec<(Nibbles, BranchNodeCompact)>,
-    pub(crate) removed_nodes: HashSet<Nibbles>,
+    /// Flag indicating whether the trie has been deleted/wiped.
+    pub is_deleted: bool,
+    /// Sorted collection of updated storage nodes with corresponding paths.
+    pub storage_nodes: Vec<(Nibbles, BranchNodeCompact)>,
+    /// The set of removed storage node keys.
+    pub removed_nodes: HashSet<Nibbles>,
 }
 
 impl StorageTrieUpdatesSorted {
@@ -402,16 +405,15 @@ fn exclude_empty_from_pair<V>(
 /// Bincode-compatible trie updates type serde implementations.
 #[cfg(all(feature = "serde", feature = "serde-bincode-compat"))]
 pub mod serde_bincode_compat {
+    use crate::{BranchNodeCompact, Nibbles};
+    use alloy_primitives::B256;
+    use serde::{Deserialize, Deserializer, Serialize, Serializer};
+    use serde_with::{DeserializeAs, SerializeAs};
     use std::{
         borrow::Cow,
         collections::{HashMap, HashSet},
     };
 
-    use alloy_primitives::B256;
-    use reth_trie_common::{BranchNodeCompact, Nibbles};
-    use serde::{Deserialize, Deserializer, Serialize, Serializer};
-    use serde_with::{DeserializeAs, SerializeAs};
-
     /// Bincode-compatible [`super::TrieUpdates`] serde implementation.
     ///
     /// Intended to use with the [`serde_with::serde_as`] macro in the following way:

crates/trie/db/Cargo.toml

@@ -67,22 +67,22 @@ similar-asserts.workspace = true
 metrics = ["reth-metrics", "reth-trie/metrics", "dep:metrics"]
 serde = [
 	"dep:serde",
-	"reth-provider/serde",
-	"reth-trie/serde",
-	"reth-trie-common/serde",
+	"similar-asserts/serde",
+	"revm/serde",
 	"alloy-consensus/serde",
 	"alloy-primitives/serde",
-	"revm/serde",
-	"similar-asserts/serde"
+	"reth-trie/serde",
+	"reth-trie-common/serde",
+	"reth-provider/serde",
 ]
 test-utils = [
 	"triehash",
+	"revm/test-utils",
 	"reth-trie-common/test-utils",
 	"reth-chainspec/test-utils",
 	"reth-primitives/test-utils",
 	"reth-db/test-utils",
 	"reth-db-api/test-utils",
 	"reth-provider/test-utils",
 	"reth-trie/test-utils",
-	"revm/test-utils"
 ]

crates/trie/parallel/src/root.rs

@@ -193,11 +193,8 @@ where
 
         let root = hash_builder.root();
 
-        trie_updates.finalize(
-            account_node_iter.walker,
-            hash_builder,
-            prefix_sets.destroyed_accounts,
-        );
+        let removed_keys = account_node_iter.walker.take_removed_keys();
+        trie_updates.finalize(hash_builder, removed_keys, prefix_sets.destroyed_accounts);
 
         let stats = tracker.finish();
 

crates/trie/sparse/benches/root.rs

@@ -146,7 +146,7 @@ pub fn calculate_root_from_leaves_repeated(c: &mut Criterion) {
                                 hb.root();
 
                                 if storage_updates.peek().is_some() {
-                                    trie_updates.finalize(node_iter.walker, hb);
+                                    trie_updates.finalize(hb, node_iter.walker.take_removed_keys());
                                 }
                             }
                         },