From 76bea6d9b61437be8c89cef135ee646ae22e131d Mon Sep 17 00:00:00 2001 From: Javier Chatruc Date: Tue, 26 Nov 2024 11:28:10 -0300 Subject: [PATCH 01/11] Cache the sender address on mempool transactions --- cmd/ethrex/ethrex.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/ethrex/ethrex.rs b/cmd/ethrex/ethrex.rs index a1056f9e0..61e4c8968 100644 --- a/cmd/ethrex/ethrex.rs +++ b/cmd/ethrex/ethrex.rs @@ -223,7 +223,7 @@ async fn main() { }; let max_tries = 3; let url = format!("http://{authrpc_socket_addr}"); - let block_producer_engine = ethrex_dev::block_producer::start_block_producer(url, authrpc_jwtsecret.into(), head_block_hash, max_tries, 1000, ethrex_core::Address::default()); + let block_producer_engine = ethrex_dev::block_producer::start_block_producer(url, authrpc_jwtsecret.into(), head_block_hash, max_tries, 5000, ethrex_core::Address::default()); tracker.spawn(block_producer_engine); } else { let networking = ethrex_net::start_network( From 9dbc3756a6edb75f1f4e83a9878259469bc5cbb7 Mon Sep 17 00:00:00 2001 From: Javier Chatruc Date: Tue, 26 Nov 2024 12:10:19 -0300 Subject: [PATCH 02/11] Fix tests --- crates/blockchain/mempool.rs | 3 ++- crates/storage/store/storage.rs | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/crates/blockchain/mempool.rs b/crates/blockchain/mempool.rs index 9680f37b2..920b2cc20 100644 --- a/crates/blockchain/mempool.rs +++ b/crates/blockchain/mempool.rs @@ -313,7 +313,8 @@ mod tests { use super::{transaction_intrinsic_gas, validate_transaction}; use ethrex_core::types::{ - BlockHeader, ChainConfig, EIP1559Transaction, EIP4844Transaction, Transaction, TxKind, + BlockHeader, ChainConfig, EIP1559Transaction, EIP4844Transaction, Signable, Transaction, + TxKind, }; use ethrex_core::{Address, Bytes, H256, U256}; use ethrex_storage::EngineType; diff --git a/crates/storage/store/storage.rs b/crates/storage/store/storage.rs index 0a012b8d5..0de1eacca 100644 --- a/crates/storage/store/storage.rs +++ b/crates/storage/store/storage.rs @@ -928,6 +928,7 @@ mod tests { use bytes::Bytes; use ethereum_types::{H256, U256}; use ethrex_core::{ + serde_utils::blob, types::{Transaction, TxType, BYTES_PER_BLOB}, Bloom, }; From ee782038a7db0fd271f1cd7148daf63a8a1de24e Mon Sep 17 00:00:00 2001 From: Javier Chatruc Date: Tue, 26 Nov 2024 12:17:17 -0300 Subject: [PATCH 03/11] clippy --- crates/blockchain/mempool.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/blockchain/mempool.rs b/crates/blockchain/mempool.rs index 920b2cc20..a4118ebac 100644 --- a/crates/blockchain/mempool.rs +++ b/crates/blockchain/mempool.rs @@ -313,7 +313,7 @@ mod tests { use super::{transaction_intrinsic_gas, validate_transaction}; use ethrex_core::types::{ - BlockHeader, ChainConfig, EIP1559Transaction, EIP4844Transaction, Signable, Transaction, + BlockHeader, ChainConfig, EIP1559Transaction, EIP4844Transaction, Transaction, TxKind, }; use ethrex_core::{Address, Bytes, H256, U256}; From b4c52edcade204731e3f5ea07cb04fd0f9ec1d05 Mon Sep 17 00:00:00 2001 From: Javier Chatruc Date: Tue, 26 Nov 2024 14:56:23 -0300 Subject: [PATCH 04/11] clippy + fmt --- crates/blockchain/mempool.rs | 3 +-- crates/storage/store/storage.rs | 1 - 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/crates/blockchain/mempool.rs b/crates/blockchain/mempool.rs index a4118ebac..9680f37b2 100644 --- a/crates/blockchain/mempool.rs +++ b/crates/blockchain/mempool.rs @@ -313,8 +313,7 @@ mod tests { use super::{transaction_intrinsic_gas, validate_transaction}; use ethrex_core::types::{ - BlockHeader, ChainConfig, EIP1559Transaction, EIP4844Transaction, Transaction, - TxKind, + BlockHeader, ChainConfig, EIP1559Transaction, EIP4844Transaction, Transaction, TxKind, }; use ethrex_core::{Address, Bytes, H256, U256}; use ethrex_storage::EngineType; diff --git a/crates/storage/store/storage.rs b/crates/storage/store/storage.rs index 0de1eacca..0a012b8d5 100644 --- a/crates/storage/store/storage.rs +++ b/crates/storage/store/storage.rs @@ -928,7 +928,6 @@ mod tests { use bytes::Bytes; use ethereum_types::{H256, U256}; use ethrex_core::{ - serde_utils::blob, types::{Transaction, TxType, BYTES_PER_BLOB}, Bloom, }; From 00e9fd8e66fa4a604778f4823324cf7965d98dd8 Mon Sep 17 00:00:00 2001 From: Javier Chatruc Date: Wed, 27 Nov 2024 11:38:16 -0300 Subject: [PATCH 05/11] Try fix test --- cmd/ethrex/ethrex.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/ethrex/ethrex.rs b/cmd/ethrex/ethrex.rs index 61e4c8968..a1056f9e0 100644 --- a/cmd/ethrex/ethrex.rs +++ b/cmd/ethrex/ethrex.rs @@ -223,7 +223,7 @@ async fn main() { }; let max_tries = 3; let url = format!("http://{authrpc_socket_addr}"); - let block_producer_engine = ethrex_dev::block_producer::start_block_producer(url, authrpc_jwtsecret.into(), head_block_hash, max_tries, 5000, ethrex_core::Address::default()); + let block_producer_engine = ethrex_dev::block_producer::start_block_producer(url, authrpc_jwtsecret.into(), head_block_hash, max_tries, 1000, ethrex_core::Address::default()); tracker.spawn(block_producer_engine); } else { let networking = ethrex_net::start_network( From b0d0e0b83bd1c6b9917c0775b69e943848405d96 Mon Sep 17 00:00:00 2001 From: Javier Chatruc Date: Thu, 28 Nov 2024 19:30:13 -0300 Subject: [PATCH 06/11] Finish Redb migration --- crates/storage/store/engines/redb.rs | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/crates/storage/store/engines/redb.rs b/crates/storage/store/engines/redb.rs index 1a7ab7f53..65dd7c4eb 100644 --- a/crates/storage/store/engines/redb.rs +++ b/crates/storage/store/engines/redb.rs @@ -102,6 +102,27 @@ impl RedBStore { Ok(()) } + // Helper method to write into a redb table + fn write_to_multi<'k, 'v, 'a, K, V>( + &self, + table: MultimapTableDefinition<'a, K, V>, + key: impl Borrow>, + value: impl Borrow>, + ) -> Result<(), StoreError> + where + K: Key + 'static, + V: Key + 'static, + { + let write_txn = self.db.begin_write().unwrap(); + { + let mut table = write_txn.open_multimap_table(table).unwrap(); + table.insert(key, value).unwrap(); + } + write_txn.commit().unwrap(); + + Ok(()) + } + // Helper method to read from a redb table fn read<'k, 'a, K, V>( &self, From 4c6a120086d74a1e2987776468133d316d8cfe46 Mon Sep 17 00:00:00 2001 From: Javier Chatruc Date: Fri, 29 Nov 2024 11:18:35 -0300 Subject: [PATCH 07/11] Remove unwraps on engine --- crates/storage/store/engines/redb.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/crates/storage/store/engines/redb.rs b/crates/storage/store/engines/redb.rs index 65dd7c4eb..044f6a66d 100644 --- a/crates/storage/store/engines/redb.rs +++ b/crates/storage/store/engines/redb.rs @@ -113,12 +113,12 @@ impl RedBStore { K: Key + 'static, V: Key + 'static, { - let write_txn = self.db.begin_write().unwrap(); + let write_txn = self.db.begin_write()?; { - let mut table = write_txn.open_multimap_table(table).unwrap(); - table.insert(key, value).unwrap(); + let mut table = write_txn.open_multimap_table(table)?; + table.insert(key, value)?; } - write_txn.commit().unwrap(); + write_txn.commit()?; Ok(()) } From f8f3d4f8ecec2738948ca574559cc8b9ae758f43 Mon Sep 17 00:00:00 2001 From: Javier Chatruc Date: Fri, 29 Nov 2024 17:48:22 -0300 Subject: [PATCH 08/11] Batch heaviest write operations to speed up block production --- crates/blockchain/blockchain.rs | 4 +- crates/storage/store/engines/api.rs | 10 +++ crates/storage/store/engines/in_memory.rs | 27 +++++++ crates/storage/store/engines/libmdbx.rs | 51 +++++++++++++ crates/storage/store/engines/redb.rs | 92 ++++++++++++++++++++++- crates/storage/store/storage.rs | 48 +++++++----- crates/storage/trie/db.rs | 2 + crates/storage/trie/db/in_memory.rs | 10 +++ crates/storage/trie/db/libmdbx.rs | 9 +++ crates/storage/trie/db/libmdbx_dupsort.rs | 12 +++ crates/storage/trie/db/redb.rs | 13 ++++ crates/storage/trie/db/redb_multitable.rs | 13 ++++ crates/storage/trie/state.rs | 22 +++++- crates/storage/trie/trie.rs | 4 + test_data/genesis-l2.json | 2 +- 15 files changed, 293 insertions(+), 26 deletions(-) diff --git a/crates/blockchain/blockchain.rs b/crates/blockchain/blockchain.rs index 6b1aab8f6..27f0f968a 100644 --- a/crates/blockchain/blockchain.rs +++ b/crates/blockchain/blockchain.rs @@ -118,9 +118,7 @@ pub fn store_receipts( receipts: Vec, block_hash: BlockHash, ) -> Result<(), ChainError> { - for (index, receipt) in receipts.into_iter().enumerate() { - storage.add_receipt(block_hash, index as u64, receipt)?; - } + storage.add_receipts(block_hash, receipts)?; Ok(()) } diff --git a/crates/storage/store/engines/api.rs b/crates/storage/store/engines/api.rs index cca2e7333..f87f4c48c 100644 --- a/crates/storage/store/engines/api.rs +++ b/crates/storage/store/engines/api.rs @@ -78,6 +78,12 @@ pub trait StoreEngine: Debug + Send + Sync + RefUnwindSafe { index: Index, ) -> Result<(), StoreError>; + /// Store transaction locations in batch (one db transaction for all) + fn add_transaction_locations( + &self, + locations: Vec<(H256, BlockNumber, BlockHash, Index)>, + ) -> Result<(), StoreError>; + /// Obtain transaction location (block hash and index) fn get_transaction_location( &self, @@ -92,6 +98,10 @@ pub trait StoreEngine: Debug + Send + Sync + RefUnwindSafe { receipt: Receipt, ) -> Result<(), StoreError>; + /// Add receipt + fn add_receipts(&self, block_hash: BlockHash, receipts: Vec) + -> Result<(), StoreError>; + /// Obtain receipt for a canonical block represented by the block number. fn get_receipt( &self, diff --git a/crates/storage/store/engines/in_memory.rs b/crates/storage/store/engines/in_memory.rs index 4d4ed4dbb..b08318138 100644 --- a/crates/storage/store/engines/in_memory.rs +++ b/crates/storage/store/engines/in_memory.rs @@ -348,6 +348,33 @@ impl StoreEngine for Store { fn get_payload(&self, payload_id: u64) -> Result, StoreError> { Ok(self.inner().payloads.get(&payload_id).cloned()) } + + fn add_receipts( + &self, + block_hash: BlockHash, + receipts: Vec, + ) -> Result<(), StoreError> { + let mut store = self.inner(); + let entry = store.receipts.entry(block_hash).or_default(); + for (index, receipt) in receipts.into_iter().enumerate() { + entry.insert(index as u64, receipt); + } + Ok(()) + } + + fn add_transaction_locations( + &self, + locations: Vec<(H256, BlockNumber, BlockHash, Index)>, + ) -> Result<(), StoreError> { + for (transaction_hash, block_number, block_hash, index) in locations { + self.inner() + .transaction_locations + .entry(transaction_hash) + .or_default() + .push((block_number, block_hash, index)); + } + Ok(()) + } } impl Debug for Store { diff --git a/crates/storage/store/engines/libmdbx.rs b/crates/storage/store/engines/libmdbx.rs index 47c661d68..9b204a5e8 100644 --- a/crates/storage/store/engines/libmdbx.rs +++ b/crates/storage/store/engines/libmdbx.rs @@ -47,6 +47,21 @@ impl Store { txn.commit().map_err(StoreError::LibmdbxError) } + // Helper method to write into a libmdbx table in batch + fn write_batch(&self, key_values: Vec<(T::Key, T::Value)>) -> Result<(), StoreError> { + let txn = self + .db + .begin_readwrite() + .map_err(StoreError::LibmdbxError)?; + + for (key, value) in key_values { + txn.upsert::(key, value) + .map_err(StoreError::LibmdbxError)?; + } + + txn.commit().map_err(StoreError::LibmdbxError) + } + // Helper method to read from a libmdbx table fn read(&self, key: T::Key) -> Result, StoreError> { let txn = self.db.begin_read().map_err(StoreError::LibmdbxError)?; @@ -421,6 +436,42 @@ impl StoreEngine for Store { .read::(block_hash.into())? .map(|b| b.to())) } + + fn add_transaction_locations( + &self, + locations: Vec<(H256, BlockNumber, BlockHash, Index)>, + ) -> std::result::Result<(), StoreError> { + let key_values: Vec<(TransactionHashRLP, Rlp<(BlockNumber, BlockHash, Index)>)> = locations + .into_iter() + .map(|(tx_hash, block_number, block_hash, index)| { + (tx_hash.into(), (block_number, block_hash, index).into()) + }) + .collect(); + + self.write_batch::(key_values) + } + + fn add_receipts( + &self, + block_hash: BlockHash, + receipts: Vec, + ) -> std::result::Result<(), StoreError> { + let key_values = receipts + .into_iter() + .enumerate() + .map(|(index, receipt)| { + ( + <(H256, u64) as Into>>::into(( + block_hash, + index as u64, + )), + >::into(receipt), + ) + }) + .collect(); + + self.write_batch::(key_values) + } } impl Debug for Store { diff --git a/crates/storage/store/engines/redb.rs b/crates/storage/store/engines/redb.rs index 044f6a66d..93b6766e5 100644 --- a/crates/storage/store/engines/redb.rs +++ b/crates/storage/store/engines/redb.rs @@ -12,7 +12,9 @@ use ethrex_trie::{ db::{redb::RedBTrie, redb_multitable::RedBMultiTableTrieDB}, Trie, }; -use redb::{AccessGuard, Database, Key, MultimapTableDefinition, TableDefinition, TypeName, Value}; +use redb::{ + AccessGuard, Builder, Database, Key, MultimapTableDefinition, TableDefinition, TypeName, Value, +}; use crate::rlp::{BlockRLP, BlockTotalDifficultyRLP, Rlp, TransactionHashRLP}; use crate::{ @@ -102,6 +104,28 @@ impl RedBStore { Ok(()) } + // Helper method to write into a redb table + fn write_batch<'k, 'v, 'a, K, V>( + &self, + table: TableDefinition<'a, K, V>, + key_values: Vec<(impl Borrow>, impl Borrow>)>, + ) -> Result<(), StoreError> + where + K: Key + 'static, + V: Value + 'static, + { + let write_txn = self.db.begin_write()?; + { + let mut table = write_txn.open_table(table)?; + for (key, value) in key_values { + table.insert(key, value)?; + } + } + write_txn.commit()?; + + Ok(()) + } + // Helper method to write into a redb table fn write_to_multi<'k, 'v, 'a, K, V>( &self, @@ -123,6 +147,28 @@ impl RedBStore { Ok(()) } + // Helper method to write into a redb table + fn write_to_multi_batch<'k, 'v, 'a, K, V>( + &self, + table: MultimapTableDefinition<'a, K, V>, + key_values: Vec<(impl Borrow>, impl Borrow>)>, + ) -> Result<(), StoreError> + where + K: Key + 'static, + V: Key + 'static, + { + let write_txn = self.db.begin_write()?; + { + let mut table = write_txn.open_multimap_table(table)?; + for (key, value) in key_values { + table.insert(key, value)?; + } + } + write_txn.commit()?; + + Ok(()) + } + // Helper method to read from a redb table fn read<'k, 'a, K, V>( &self, @@ -562,6 +608,50 @@ impl StoreEngine for RedBStore { .read(PAYLOADS_TABLE, payload_id)? .map(|b| b.value().to())) } + + fn add_receipts( + &self, + block_hash: BlockHash, + receipts: Vec, + ) -> Result<(), StoreError> { + let key_values = receipts + .into_iter() + .enumerate() + .map(|(index, receipt)| { + ( + <(H256, u64) as Into>>::into(( + block_hash, + index as u64, + )), + >::into(receipt), + ) + }) + .collect(); + self.write_batch(RECEIPTS_TABLE, key_values) + } + + fn add_transaction_locations( + &self, + locations: Vec<(H256, BlockNumber, BlockHash, Index)>, + ) -> Result<(), StoreError> { + let key_values = locations + .into_iter() + .map(|(tx_hash, block_number, block_hash, index)| { + ( + >::into(tx_hash), + <(u64, H256, u64) as Into>>::into(( + block_number, + block_hash, + index, + )), + ) + }) + .collect(); + + self.write_to_multi_batch(TRANSACTION_LOCATIONS_TABLE, key_values)?; + + Ok(()) + } } impl redb::Value for ChainDataIndex { diff --git a/crates/storage/store/storage.rs b/crates/storage/store/storage.rs index 0a012b8d5..be85b6e2e 100644 --- a/crates/storage/store/storage.rs +++ b/crates/storage/store/storage.rs @@ -232,6 +232,26 @@ impl Store { .add_transaction_location(transaction_hash, block_number, block_hash, index) } + pub fn add_transaction_locations( + &self, + transactions: &Vec, + block_number: BlockNumber, + block_hash: BlockHash, + ) -> Result<(), StoreError> { + let mut locations = vec![]; + + for (index, transaction) in transactions.iter().enumerate() { + locations.push(( + transaction.compute_hash(), + block_number, + block_hash, + index as Index, + )); + } + + self.engine.add_transaction_locations(locations) + } + pub fn get_transaction_location( &self, transaction_hash: H256, @@ -466,6 +486,14 @@ impl Store { self.engine.add_receipt(block_hash, index, receipt) } + pub fn add_receipts( + &self, + block_hash: BlockHash, + receipts: Vec, + ) -> Result<(), StoreError> { + self.engine.add_receipts(block_hash, receipts) + } + pub fn get_receipt( &self, block_number: BlockNumber, @@ -487,24 +515,8 @@ impl Store { self.add_block_header(hash, header)?; self.add_block_number(hash, number)?; self.add_block_total_difficulty(hash, block_total_difficulty)?; - self.update_latest_total_difficulty(block_total_difficulty) - } - - fn add_transaction_locations( - &self, - transactions: &[Transaction], - block_number: BlockNumber, - block_hash: BlockHash, - ) -> Result<(), StoreError> { - for (index, transaction) in transactions.iter().enumerate() { - self.add_transaction_location( - transaction.compute_hash(), - block_number, - block_hash, - index as Index, - )?; - } - Ok(()) + let res = self.update_latest_total_difficulty(block_total_difficulty); + res } pub fn add_initial_state(&self, genesis: Genesis) -> Result<(), StoreError> { diff --git a/crates/storage/trie/db.rs b/crates/storage/trie/db.rs index d5c2c2b55..d69e337ed 100644 --- a/crates/storage/trie/db.rs +++ b/crates/storage/trie/db.rs @@ -14,4 +14,6 @@ use crate::error::TrieError; pub trait TrieDB { fn get(&self, key: Vec) -> Result>, TrieError>; fn put(&self, key: Vec, value: Vec) -> Result<(), TrieError>; + // fn put_batch(&self, key: Vec, value: Vec) -> Result<(), TrieError>; + fn put_batch(&self, key_values: Vec<(Vec, Vec)>) -> Result<(), TrieError>; } diff --git a/crates/storage/trie/db/in_memory.rs b/crates/storage/trie/db/in_memory.rs index 626039484..c42a7c5bd 100644 --- a/crates/storage/trie/db/in_memory.rs +++ b/crates/storage/trie/db/in_memory.rs @@ -25,4 +25,14 @@ impl TrieDB for InMemoryTrieDB { self.inner.lock().unwrap().insert(key, value); Ok(()) } + + fn put_batch(&self, key_values: Vec<(Vec, Vec)>) -> Result<(), TrieError> { + let mut db = self.inner.lock().unwrap(); + + for (key, value) in key_values { + db.insert(key, value); + } + + Ok(()) + } } diff --git a/crates/storage/trie/db/libmdbx.rs b/crates/storage/trie/db/libmdbx.rs index ebf2eec26..6f6d2d567 100644 --- a/crates/storage/trie/db/libmdbx.rs +++ b/crates/storage/trie/db/libmdbx.rs @@ -38,6 +38,15 @@ where .map_err(TrieError::LibmdbxError)?; txn.commit().map_err(TrieError::LibmdbxError) } + + fn put_batch(&self, key_values: Vec<(Vec, Vec)>) -> Result<(), TrieError> { + let txn = self.db.begin_readwrite().map_err(TrieError::LibmdbxError)?; + for (key, value) in key_values { + txn.upsert::(key, value) + .map_err(TrieError::LibmdbxError)?; + } + txn.commit().map_err(TrieError::LibmdbxError) + } } #[cfg(test)] diff --git a/crates/storage/trie/db/libmdbx_dupsort.rs b/crates/storage/trie/db/libmdbx_dupsort.rs index 03b23a19d..5a5c2420c 100644 --- a/crates/storage/trie/db/libmdbx_dupsort.rs +++ b/crates/storage/trie/db/libmdbx_dupsort.rs @@ -52,6 +52,18 @@ where .map_err(TrieError::LibmdbxError)?; txn.commit().map_err(TrieError::LibmdbxError) } + + fn put_batch(&self, key_values: Vec<(Vec, Vec)>) -> Result<(), TrieError> { + let txn = self.db.begin_readwrite().map_err(TrieError::LibmdbxError)?; + for (key, value) in key_values { + txn.upsert::( + (self.fixed_key.clone(), node_hash_to_fixed_size(key)), + value, + ) + .map_err(TrieError::LibmdbxError)?; + } + txn.commit().map_err(TrieError::LibmdbxError) + } } #[cfg(test)] diff --git a/crates/storage/trie/db/redb.rs b/crates/storage/trie/db/redb.rs index ba2e3489c..e7f38c50a 100644 --- a/crates/storage/trie/db/redb.rs +++ b/crates/storage/trie/db/redb.rs @@ -32,4 +32,17 @@ impl TrieDB for RedBTrie { Ok(()) } + + fn put_batch(&self, key_values: Vec<(Vec, Vec)>) -> Result<(), crate::TrieError> { + let write_txn = self.db.begin_write()?; + { + let mut table = write_txn.open_table(TABLE)?; + for (key, value) in key_values { + table.insert(&*key, &*value)?; + } + } + write_txn.commit()?; + + Ok(()) + } } diff --git a/crates/storage/trie/db/redb_multitable.rs b/crates/storage/trie/db/redb_multitable.rs index 988c50ead..0035ab5cf 100644 --- a/crates/storage/trie/db/redb_multitable.rs +++ b/crates/storage/trie/db/redb_multitable.rs @@ -54,4 +54,17 @@ impl TrieDB for RedBMultiTableTrieDB { Ok(()) } + + fn put_batch(&self, key_values: Vec<(Vec, Vec)>) -> Result<(), TrieError> { + let write_txn = self.db.begin_write()?; + { + let mut table = write_txn.open_multimap_table(STORAGE_TRIE_NODES_TABLE)?; + for (key, value) in key_values { + table.insert((self.fixed_key, node_hash_to_fixed_size(key)), &*value)?; + } + } + write_txn.commit()?; + + Ok(()) + } } diff --git a/crates/storage/trie/state.rs b/crates/storage/trie/state.rs index be42925e9..77fe4710e 100644 --- a/crates/storage/trie/state.rs +++ b/crates/storage/trie/state.rs @@ -56,6 +56,20 @@ impl TrieState { // Writes a node and its children into the DB fn commit_node(&mut self, node_hash: &NodeHash) -> Result<(), TrieError> { + let mut to_commit = vec![]; + self.commit_node_tail_recursive(node_hash, &mut to_commit)?; + + self.db.put_batch(to_commit)?; + + Ok(()) + } + + // Writes a node and its children into the DB + fn commit_node_tail_recursive( + &mut self, + node_hash: &NodeHash, + acc: &mut Vec<(Vec, Vec)>, + ) -> Result<(), TrieError> { let Some(node) = self.cache.remove(node_hash) else { // If the node is not in the cache then it means it is already stored in the DB return Ok(()); @@ -65,14 +79,16 @@ impl TrieState { Node::Branch(n) => { for child in n.choices.iter() { if child.is_valid() { - self.commit_node(child)?; + self.commit_node_tail_recursive(child, acc)?; } } } - Node::Extension(n) => self.commit_node(&n.child)?, + Node::Extension(n) => self.commit_node_tail_recursive(&n.child, acc)?, Node::Leaf(_) => {} } // Commit self - self.db.put(node_hash.into(), node.encode_to_vec()) + acc.push((node_hash.into(), node.encode_to_vec())); + + Ok(()) } } diff --git a/crates/storage/trie/trie.rs b/crates/storage/trie/trie.rs index 56f945093..067d75d01 100644 --- a/crates/storage/trie/trie.rs +++ b/crates/storage/trie/trie.rs @@ -247,6 +247,10 @@ impl Trie { fn put(&self, _key: Vec, _value: Vec) -> Result<(), TrieError> { Ok(()) } + + fn put_batch(&self, _key_values: Vec<(Vec, Vec)>) -> Result<(), TrieError> { + Ok(()) + } } Trie::new(Box::new(NullTrieDB)) diff --git a/test_data/genesis-l2.json b/test_data/genesis-l2.json index ac78f7606..44e04378a 100644 --- a/test_data/genesis-l2.json +++ b/test_data/genesis-l2.json @@ -546,7 +546,7 @@ "coinbase": "0x0000000000000000000000000000000000000000", "difficulty": "0x01", "extraData": "", - "gasLimit": "0x17d7840", + "gasLimit": "0x8f0d180", "nonce": "0x1234", "mixhash": "0x0000000000000000000000000000000000000000000000000000000000000000", "parentHash": "0x0000000000000000000000000000000000000000000000000000000000000000", From 2da7fb97b4f47d313935aeda1243f686db80484f Mon Sep 17 00:00:00 2001 From: Javier Chatruc Date: Fri, 29 Nov 2024 17:58:10 -0300 Subject: [PATCH 09/11] clippy --- crates/storage/store/engines/libmdbx.rs | 1 + crates/storage/store/engines/redb.rs | 4 +--- crates/storage/store/storage.rs | 5 ++--- 3 files changed, 4 insertions(+), 6 deletions(-) diff --git a/crates/storage/store/engines/libmdbx.rs b/crates/storage/store/engines/libmdbx.rs index 9b204a5e8..82d59b4ba 100644 --- a/crates/storage/store/engines/libmdbx.rs +++ b/crates/storage/store/engines/libmdbx.rs @@ -441,6 +441,7 @@ impl StoreEngine for Store { &self, locations: Vec<(H256, BlockNumber, BlockHash, Index)>, ) -> std::result::Result<(), StoreError> { + #[allow(clippy::type_complexity)] let key_values: Vec<(TransactionHashRLP, Rlp<(BlockNumber, BlockHash, Index)>)> = locations .into_iter() .map(|(tx_hash, block_number, block_hash, index)| { diff --git a/crates/storage/store/engines/redb.rs b/crates/storage/store/engines/redb.rs index 93b6766e5..7446cd8bc 100644 --- a/crates/storage/store/engines/redb.rs +++ b/crates/storage/store/engines/redb.rs @@ -12,9 +12,7 @@ use ethrex_trie::{ db::{redb::RedBTrie, redb_multitable::RedBMultiTableTrieDB}, Trie, }; -use redb::{ - AccessGuard, Builder, Database, Key, MultimapTableDefinition, TableDefinition, TypeName, Value, -}; +use redb::{AccessGuard, Database, Key, MultimapTableDefinition, TableDefinition, TypeName, Value}; use crate::rlp::{BlockRLP, BlockTotalDifficultyRLP, Rlp, TransactionHashRLP}; use crate::{ diff --git a/crates/storage/store/storage.rs b/crates/storage/store/storage.rs index be85b6e2e..945761a3f 100644 --- a/crates/storage/store/storage.rs +++ b/crates/storage/store/storage.rs @@ -234,7 +234,7 @@ impl Store { pub fn add_transaction_locations( &self, - transactions: &Vec, + transactions: &[Transaction], block_number: BlockNumber, block_hash: BlockHash, ) -> Result<(), StoreError> { @@ -515,8 +515,7 @@ impl Store { self.add_block_header(hash, header)?; self.add_block_number(hash, number)?; self.add_block_total_difficulty(hash, block_total_difficulty)?; - let res = self.update_latest_total_difficulty(block_total_difficulty); - res + self.update_latest_total_difficulty(block_total_difficulty) } pub fn add_initial_state(&self, genesis: Genesis) -> Result<(), StoreError> { From 4dec14276725a3f72841e6eee80a122b3f189e39 Mon Sep 17 00:00:00 2001 From: Javier Chatruc Date: Fri, 29 Nov 2024 20:00:50 -0300 Subject: [PATCH 10/11] Remove duplicate function --- crates/storage/store/engines/redb.rs | 21 --------------------- 1 file changed, 21 deletions(-) diff --git a/crates/storage/store/engines/redb.rs b/crates/storage/store/engines/redb.rs index 7446cd8bc..a3baa9015 100644 --- a/crates/storage/store/engines/redb.rs +++ b/crates/storage/store/engines/redb.rs @@ -124,27 +124,6 @@ impl RedBStore { Ok(()) } - // Helper method to write into a redb table - fn write_to_multi<'k, 'v, 'a, K, V>( - &self, - table: MultimapTableDefinition<'a, K, V>, - key: impl Borrow>, - value: impl Borrow>, - ) -> Result<(), StoreError> - where - K: Key + 'static, - V: Key + 'static, - { - let write_txn = self.db.begin_write()?; - { - let mut table = write_txn.open_multimap_table(table)?; - table.insert(key, value)?; - } - write_txn.commit()?; - - Ok(()) - } - // Helper method to write into a redb table fn write_to_multi_batch<'k, 'v, 'a, K, V>( &self, From 4845ce80da201a8ede0512b33fb76d6459e351a9 Mon Sep 17 00:00:00 2001 From: Javier Chatruc Date: Tue, 10 Dec 2024 11:13:32 -0300 Subject: [PATCH 11/11] Remove unwraps from in memory implementation --- crates/storage/trie/db/in_memory.rs | 14 +++++++++++--- crates/storage/trie/error.rs | 2 ++ 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/crates/storage/trie/db/in_memory.rs b/crates/storage/trie/db/in_memory.rs index c42a7c5bd..f9daa7080 100644 --- a/crates/storage/trie/db/in_memory.rs +++ b/crates/storage/trie/db/in_memory.rs @@ -18,16 +18,24 @@ impl InMemoryTrieDB { impl TrieDB for InMemoryTrieDB { fn get(&self, key: Vec) -> Result>, TrieError> { - Ok(self.inner.lock().unwrap().get(&key).cloned()) + Ok(self + .inner + .lock() + .map_err(|_| TrieError::LockError)? + .get(&key) + .cloned()) } fn put(&self, key: Vec, value: Vec) -> Result<(), TrieError> { - self.inner.lock().unwrap().insert(key, value); + self.inner + .lock() + .map_err(|_| TrieError::LockError)? + .insert(key, value); Ok(()) } fn put_batch(&self, key_values: Vec<(Vec, Vec)>) -> Result<(), TrieError> { - let mut db = self.inner.lock().unwrap(); + let mut db = self.inner.lock().map_err(|_| TrieError::LockError)?; for (key, value) in key_values { db.insert(key, value); diff --git a/crates/storage/trie/error.rs b/crates/storage/trie/error.rs index 34cf68c7f..7b7ffd49d 100644 --- a/crates/storage/trie/error.rs +++ b/crates/storage/trie/error.rs @@ -27,4 +27,6 @@ pub enum TrieError { Verify(String), #[error("Inconsistent internal tree structure")] InconsistentTree, + #[error("Lock Error: Panicked when trying to acquire a lock")] + LockError, }