From adb53bb2c99a1c278cb371e15a1cf0fb2f85aed9 Mon Sep 17 00:00:00 2001 From: joshieDo Date: Thu, 3 Aug 2023 12:20:41 +0000 Subject: [PATCH 01/15] prune receipts based on log emitters --- bin/reth/src/args/pruning_args.rs | 7 +++++ bin/reth/src/debug_cmd/execution.rs | 2 +- bin/reth/src/node/mod.rs | 7 +++-- crates/config/src/config.rs | 2 +- crates/primitives/src/prune/target.rs | 14 ++++++--- crates/stages/src/stages/execution.rs | 2 +- crates/stages/src/stages/mod.rs | 12 ++++---- crates/storage/provider/src/post_state/mod.rs | 30 +++++++++++++++++-- 8 files changed, 58 insertions(+), 18 deletions(-) diff --git a/bin/reth/src/args/pruning_args.rs b/bin/reth/src/args/pruning_args.rs index cb71dce15f0e..daec5b3e21db 100644 --- a/bin/reth/src/args/pruning_args.rs +++ b/bin/reth/src/args/pruning_args.rs @@ -31,6 +31,13 @@ impl PruningArgs { .map(|contract| PruneMode::Before(contract.block)), account_history: Some(PruneMode::Distance(128)), storage_history: Some(PruneMode::Distance(128)), + only_contract_logs: chain_spec + .deposit_contract + .as_ref() + .map(|contract| vec![(contract.block, vec![contract.address])]) + .into_iter() + .flatten() + .collect(), }, }) } else { diff --git a/bin/reth/src/debug_cmd/execution.rs b/bin/reth/src/debug_cmd/execution.rs index 4cbe6848c2fb..b3ce880104a2 100644 --- a/bin/reth/src/debug_cmd/execution.rs +++ b/bin/reth/src/debug_cmd/execution.rs @@ -147,7 +147,7 @@ impl Command { .clean_threshold .max(stage_conf.account_hashing.clean_threshold) .max(stage_conf.storage_hashing.clean_threshold), - config.prune.map(|prune| prune.parts).unwrap_or_default(), + config.prune.as_ref().map(|prune| prune.parts.clone()).unwrap_or_default(), )), ) .build(db, self.chain.clone()); diff --git a/bin/reth/src/node/mod.rs b/bin/reth/src/node/mod.rs index ed3beefba42d..c0934bb62f2d 100644 --- a/bin/reth/src/node/mod.rs +++ b/bin/reth/src/node/mod.rs @@ -301,7 +301,8 @@ impl Command { None }; - let prune_config = self.pruning.prune_config(Arc::clone(&self.chain)).or(config.prune); + let prune_config = + self.pruning.prune_config(Arc::clone(&self.chain)).or(config.prune.clone()); // Configure the pipeline let (mut pipeline, client) = if self.dev.dev { @@ -337,7 +338,7 @@ impl Command { db.clone(), &ctx.task_executor, metrics_tx, - prune_config, + prune_config.clone(), max_block, ) .await?; @@ -357,7 +358,7 @@ impl Command { db.clone(), &ctx.task_executor, metrics_tx, - prune_config, + prune_config.clone(), max_block, ) .await?; diff --git a/crates/config/src/config.rs b/crates/config/src/config.rs index 3fb414e8ee4e..cc54312553ef 100644 --- a/crates/config/src/config.rs +++ b/crates/config/src/config.rs @@ -279,7 +279,7 @@ impl Default for IndexHistoryConfig { } /// Pruning configuration. -#[derive(Debug, Clone, Copy, Deserialize, PartialEq, Serialize)] +#[derive(Debug, Clone, Deserialize, PartialEq, Serialize)] #[serde(default)] pub struct PruneConfig { /// Minimum pruning interval measured in blocks. diff --git a/crates/primitives/src/prune/target.rs b/crates/primitives/src/prune/target.rs index 939de0f3a227..9c29169a627b 100644 --- a/crates/primitives/src/prune/target.rs +++ b/crates/primitives/src/prune/target.rs @@ -1,12 +1,13 @@ use crate::{ - prune::PrunePartError, serde_helper::deserialize_opt_prune_mode_with_min_blocks, BlockNumber, - PruneMode, PrunePart, + prune::PrunePartError, serde_helper::deserialize_opt_prune_mode_with_min_blocks, Address, + BlockNumber, PruneMode, PrunePart, }; use paste::paste; use serde::{Deserialize, Serialize}; +use std::collections::BTreeMap; /// Pruning configuration for every part of the data that can be pruned. -#[derive(Debug, Clone, Default, Copy, Deserialize, Eq, PartialEq, Serialize)] +#[derive(Debug, Clone, Default, Deserialize, Eq, PartialEq, Serialize)] #[serde(default)] pub struct PruneModes { /// Sender Recovery pruning configuration. @@ -20,7 +21,8 @@ pub struct PruneModes { /// Transaction Lookup pruning configuration. #[serde(skip_serializing_if = "Option::is_none")] pub transaction_lookup: Option, - /// Receipts pruning configuration. + /// Receipts pruning configuration. Supersedes `PruneModes::only_contract_logs` and it's more + /// performant. #[serde( skip_serializing_if = "Option::is_none", deserialize_with = "deserialize_opt_prune_mode_with_min_blocks::<64, _>" @@ -38,6 +40,9 @@ pub struct PruneModes { deserialize_with = "deserialize_opt_prune_mode_with_min_blocks::<64, _>" )] pub storage_history: Option, + /// Includes only receipts with logs that were emitted by these addresses. Excludes every + /// other. It's superseded by `PruneModes::receipts`. + pub only_contract_logs: BTreeMap>, } macro_rules! impl_prune_parts { @@ -88,6 +93,7 @@ macro_rules! impl_prune_parts { $( $part: Some(PruneMode::Full), )+ + only_contract_logs: BTreeMap::new() } } diff --git a/crates/stages/src/stages/execution.rs b/crates/stages/src/stages/execution.rs index 2d0332a2ade0..6fe07013b90d 100644 --- a/crates/stages/src/stages/execution.rs +++ b/crates/stages/src/stages/execution.rs @@ -199,7 +199,7 @@ impl ExecutionStage { start_block: u64, max_block: u64, ) -> Result { - let mut prune_modes = self.prune_modes; + let mut prune_modes = self.prune_modes.clone(); // If we're not executing MerkleStage from scratch (by threshold or first-sync), then erase // changeset related pruning configurations diff --git a/crates/stages/src/stages/mod.rs b/crates/stages/src/stages/mod.rs index 777041fbcac1..1adc72bdf969 100644 --- a/crates/stages/src/stages/mod.rs +++ b/crates/stages/src/stages/mod.rs @@ -173,34 +173,34 @@ mod tests { // In an unpruned configuration there is 1 receipt, 3 changed accounts and 1 changed // storage. let mut prune = PruneModes::none(); - check_pruning(factory.clone(), prune, 1, 3, 1).await; + check_pruning(factory.clone(), prune.clone(), 1, 3, 1).await; prune.receipts = Some(PruneMode::Full); prune.account_history = Some(PruneMode::Full); prune.storage_history = Some(PruneMode::Full); // This will result in error for account_history and storage_history, which is caught. - check_pruning(factory.clone(), prune, 0, 0, 0).await; + check_pruning(factory.clone(), prune.clone(), 0, 0, 0).await; prune.receipts = Some(PruneMode::Before(1)); prune.account_history = Some(PruneMode::Before(1)); prune.storage_history = Some(PruneMode::Before(1)); - check_pruning(factory.clone(), prune, 1, 3, 1).await; + check_pruning(factory.clone(), prune.clone(), 1, 3, 1).await; prune.receipts = Some(PruneMode::Before(2)); prune.account_history = Some(PruneMode::Before(2)); prune.storage_history = Some(PruneMode::Before(2)); // The one account is the miner - check_pruning(factory.clone(), prune, 0, 1, 0).await; + check_pruning(factory.clone(), prune.clone(), 0, 1, 0).await; prune.receipts = Some(PruneMode::Distance(66)); prune.account_history = Some(PruneMode::Distance(66)); prune.storage_history = Some(PruneMode::Distance(66)); - check_pruning(factory.clone(), prune, 1, 3, 1).await; + check_pruning(factory.clone(), prune.clone(), 1, 3, 1).await; prune.receipts = Some(PruneMode::Distance(64)); prune.account_history = Some(PruneMode::Distance(64)); prune.storage_history = Some(PruneMode::Distance(64)); // The one account is the miner - check_pruning(factory.clone(), prune, 0, 1, 0).await; + check_pruning(factory.clone(), prune.clone(), 0, 1, 0).await; } } diff --git a/crates/storage/provider/src/post_state/mod.rs b/crates/storage/provider/src/post_state/mod.rs index 1aed0e666934..099af96ce403 100644 --- a/crates/storage/provider/src/post_state/mod.rs +++ b/crates/storage/provider/src/post_state/mod.rs @@ -657,21 +657,47 @@ impl PostState { let mut bodies_cursor = tx.cursor_read::()?; let mut receipts_cursor = tx.cursor_write::()?; + // Empty implies that there is going to be + // addresses to include in the filter in a future block. None means there isn't any kind + // of configuration. + let mut address_filter = None; + for (block, receipts) in self.receipts { - if self.prune_modes.should_prune_receipts(block, tip) { + if receipts.is_empty() || self.prune_modes.should_prune_receipts(block, tip) { continue } + if !self.prune_modes.only_contract_logs.is_empty() { + if address_filter.is_none() { + address_filter = Some((block, vec![])); + } + + // Get all addresses higher than prev_block up to this block + if let Some((prev_block, filter)) = &mut address_filter { + self.prune_modes + .only_contract_logs + .range(*prev_block..=block) + .for_each(|(_, addresses)| filter.extend(addresses)); + *prev_block = block; + } + } + let (_, body_indices) = bodies_cursor.seek_exact(block)?.expect("body indices exist"); let tx_range = body_indices.tx_num_range(); assert_eq!(receipts.len(), tx_range.clone().count(), "Receipt length mismatch"); for (tx_num, receipt) in tx_range.zip(receipts) { + // If there is an address_filter, and it does not contain any of the contract + // addresses, then skip writing this receipt. + if let Some((_, filter)) = &address_filter { + if !receipt.logs.iter().any(|log| filter.contains(&log.address)) { + continue + } + } receipts_cursor.append(tx_num, receipt)?; } } } - Ok(()) } } From 94eb76a4455e856ba276c339f1b00ffef65d4f2e Mon Sep 17 00:00:00 2001 From: joshieDo Date: Thu, 3 Aug 2023 12:26:17 +0000 Subject: [PATCH 02/15] start filter at block 0 --- crates/storage/provider/src/post_state/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/storage/provider/src/post_state/mod.rs b/crates/storage/provider/src/post_state/mod.rs index 099af96ce403..96330050b0c0 100644 --- a/crates/storage/provider/src/post_state/mod.rs +++ b/crates/storage/provider/src/post_state/mod.rs @@ -669,7 +669,7 @@ impl PostState { if !self.prune_modes.only_contract_logs.is_empty() { if address_filter.is_none() { - address_filter = Some((block, vec![])); + address_filter = Some((0, vec![])); } // Get all addresses higher than prev_block up to this block From a48206f270fafe260b7b59ce2c4a022a4aa713db Mon Sep 17 00:00:00 2001 From: joshieDo Date: Thu, 3 Aug 2023 12:37:39 +0000 Subject: [PATCH 03/15] improve doc --- crates/primitives/src/prune/target.rs | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/crates/primitives/src/prune/target.rs b/crates/primitives/src/prune/target.rs index 9c29169a627b..98a31e4a2323 100644 --- a/crates/primitives/src/prune/target.rs +++ b/crates/primitives/src/prune/target.rs @@ -21,8 +21,8 @@ pub struct PruneModes { /// Transaction Lookup pruning configuration. #[serde(skip_serializing_if = "Option::is_none")] pub transaction_lookup: Option, - /// Receipts pruning configuration. Supersedes `PruneModes::only_contract_logs` and it's more - /// performant. + /// Configuration for pruning of receipts. This setting overrides + /// `PruneModes::only_contract_logs` and offers improved performance. #[serde( skip_serializing_if = "Option::is_none", deserialize_with = "deserialize_opt_prune_mode_with_min_blocks::<64, _>" @@ -40,8 +40,11 @@ pub struct PruneModes { deserialize_with = "deserialize_opt_prune_mode_with_min_blocks::<64, _>" )] pub storage_history: Option, - /// Includes only receipts with logs that were emitted by these addresses. Excludes every - /// other. It's superseded by `PruneModes::receipts`. + /// Retains only those receipts that contain logs emitted by the specified addresses, + /// discarding all others. Note that this setting is overridden by `PruneModes::receipts`. + /// + /// The [`BlockNumber`] represents the starting block from which point onwards the receipts are + /// preserved. pub only_contract_logs: BTreeMap>, } From 28ace1635cc144043bb5b7aadbd229dc9b71371c Mon Sep 17 00:00:00 2001 From: joshieDo Date: Thu, 3 Aug 2023 14:18:38 +0000 Subject: [PATCH 04/15] smol impl PruneModes refactor --- crates/primitives/src/prune/mode.rs | 38 ++++++++++++++++++++++++++- crates/primitives/src/prune/target.rs | 28 +++----------------- 2 files changed, 40 insertions(+), 26 deletions(-) diff --git a/crates/primitives/src/prune/mode.rs b/crates/primitives/src/prune/mode.rs index b62a39041b8c..9a9ee5a0ca93 100644 --- a/crates/primitives/src/prune/mode.rs +++ b/crates/primitives/src/prune/mode.rs @@ -1,4 +1,4 @@ -use crate::BlockNumber; +use crate::{BlockNumber, PrunePart, PrunePartError}; use reth_codecs::{main_codec, Compact}; /// Prune mode. @@ -14,6 +14,42 @@ pub enum PruneMode { Before(BlockNumber), } +impl PruneMode { + /// Returns block up to which variant pruning needs to be done, inclusive, according to the + /// provided tip. + pub fn prune_target_block( + &self, + tip: BlockNumber, + min_blocks: u64, + prune_part: PrunePart, + ) -> Result, PrunePartError> { + Ok(match self { + PruneMode::Full if min_blocks == 0 => Some((tip, *self)), + PruneMode::Distance(distance) if *distance > tip => None, // Nothing to prune yet + PruneMode::Distance(distance) if *distance >= min_blocks => { + Some((tip - distance, *self)) + } + PruneMode::Before(n) if *n > tip => None, // Nothing to prune yet + PruneMode::Before(n) if tip - n >= min_blocks => Some((n - 1, *self)), + _ => return Err(PrunePartError::Configuration(prune_part)), + }) + } + + /// Check if target block should be pruned according to the provided prune mode and tip. + pub fn should_prune(&self, block: BlockNumber, tip: BlockNumber) -> bool { + match self { + PruneMode::Full => true, + PruneMode::Distance(distance) => { + if *distance > tip { + return false + } + block < tip - *distance + } + PruneMode::Before(n) => *n > block, + } + } +} + #[cfg(test)] impl Default for PruneMode { fn default() -> Self { diff --git a/crates/primitives/src/prune/target.rs b/crates/primitives/src/prune/target.rs index 98a31e4a2323..dca44764fdfa 100644 --- a/crates/primitives/src/prune/target.rs +++ b/crates/primitives/src/prune/target.rs @@ -59,7 +59,7 @@ macro_rules! impl_prune_parts { )] pub fn [](&self, block: BlockNumber, tip: BlockNumber) -> bool { if let Some(mode) = &self.$part { - return self.should_prune(mode, block, tip) + return mode.should_prune(block, tip) } false } @@ -74,16 +74,8 @@ macro_rules! impl_prune_parts { " pruning needs to be done, inclusive, according to the provided tip." )] pub fn [](&self, tip: BlockNumber) -> Result, PrunePartError> { - let min_blocks: u64 = $min_blocks.unwrap_or_default(); - match self.$part { - Some(mode) => Ok(match mode { - PruneMode::Full if min_blocks == 0 => Some((tip, mode)), - PruneMode::Distance(distance) if distance > tip => None, // Nothing to prune yet - PruneMode::Distance(distance) if distance >= min_blocks => Some((tip - distance, mode)), - PruneMode::Before(n) if n > tip => None, // Nothing to prune yet - PruneMode::Before(n) if tip - n >= min_blocks => Some((n - 1, mode)), - _ => return Err(PrunePartError::Configuration(PrunePart::$variant)), - }), + match self.$part { + Some(mode) => mode.prune_target_block(tip, $min_blocks.unwrap_or_default(), PrunePart::$variant), None => Ok(None) } } @@ -109,20 +101,6 @@ impl PruneModes { PruneModes::default() } - /// Check if target block should be pruned according to the provided prune mode and tip. - pub fn should_prune(&self, mode: &PruneMode, block: BlockNumber, tip: BlockNumber) -> bool { - match mode { - PruneMode::Full => true, - PruneMode::Distance(distance) => { - if *distance > tip { - return false - } - block < tip - *distance - } - PruneMode::Before(n) => *n > block, - } - } - impl_prune_parts!( (sender_recovery, SenderRecovery, Some(64)), (transaction_lookup, TransactionLookup, None), From f325c710ac56b8b44af5cdf88642739a0fe57acf Mon Sep 17 00:00:00 2001 From: joshieDo Date: Thu, 3 Aug 2023 15:17:19 +0000 Subject: [PATCH 05/15] only contract logs uses PruneMode --- bin/reth/src/args/pruning_args.rs | 17 ++++----- crates/primitives/src/lib.rs | 4 ++- crates/primitives/src/prune/mod.rs | 2 +- crates/primitives/src/prune/part.rs | 4 ++- crates/primitives/src/prune/target.rs | 35 +++++++++++++++++-- crates/storage/provider/src/post_state/mod.rs | 19 +++++----- 6 files changed, 60 insertions(+), 21 deletions(-) diff --git a/bin/reth/src/args/pruning_args.rs b/bin/reth/src/args/pruning_args.rs index daec5b3e21db..b5cd4a7c6052 100644 --- a/bin/reth/src/args/pruning_args.rs +++ b/bin/reth/src/args/pruning_args.rs @@ -2,7 +2,7 @@ use clap::Args; use reth_config::config::PruneConfig; -use reth_primitives::{ChainSpec, PruneMode, PruneModes}; +use reth_primitives::{ChainSpec, ContractLogPart, PruneMode, PruneModes}; use std::sync::Arc; /// Parameters for pruning and full node @@ -31,13 +31,14 @@ impl PruningArgs { .map(|contract| PruneMode::Before(contract.block)), account_history: Some(PruneMode::Distance(128)), storage_history: Some(PruneMode::Distance(128)), - only_contract_logs: chain_spec - .deposit_contract - .as_ref() - .map(|contract| vec![(contract.block, vec![contract.address])]) - .into_iter() - .flatten() - .collect(), + only_contract_logs: ContractLogPart( + chain_spec + .deposit_contract + .as_ref() + .map(|contract| (PruneMode::Before(contract.block), contract.address)) + .into_iter() + .collect(), + ), }, }) } else { diff --git a/crates/primitives/src/lib.rs b/crates/primitives/src/lib.rs index 249073804f1f..b668f5ca965b 100644 --- a/crates/primitives/src/lib.rs +++ b/crates/primitives/src/lib.rs @@ -79,7 +79,9 @@ pub use net::{ SEPOLIA_BOOTNODES, }; pub use peer::{PeerId, WithPeerId}; -pub use prune::{PruneCheckpoint, PruneMode, PruneModes, PrunePart, PrunePartError}; +pub use prune::{ + ContractLogPart, PruneCheckpoint, PruneMode, PruneModes, PrunePart, PrunePartError, +}; pub use receipt::{Receipt, ReceiptWithBloom, ReceiptWithBloomRef}; pub use revm_primitives::JumpMap; pub use serde_helper::JsonU256; diff --git a/crates/primitives/src/prune/mod.rs b/crates/primitives/src/prune/mod.rs index a3bcb959627e..e3900995ecbc 100644 --- a/crates/primitives/src/prune/mod.rs +++ b/crates/primitives/src/prune/mod.rs @@ -6,4 +6,4 @@ mod target; pub use checkpoint::PruneCheckpoint; pub use mode::PruneMode; pub use part::{PrunePart, PrunePartError}; -pub use target::PruneModes; +pub use target::{ContractLogPart, PruneModes}; diff --git a/crates/primitives/src/prune/part.rs b/crates/primitives/src/prune/part.rs index db49870735a7..d7a8fccd9023 100644 --- a/crates/primitives/src/prune/part.rs +++ b/crates/primitives/src/prune/part.rs @@ -10,8 +10,10 @@ pub enum PrunePart { SenderRecovery, /// Prune part responsible for the `TxHashNumber` table. TransactionLookup, - /// Prune part responsible for the `Receipts` table. + /// Prune part responsible for all `Receipts`. Receipts, + /// Prune part responsible for some `Receipts`. + ContractLogs, /// Prune part responsible for the `AccountChangeSet` and `AccountHistory` tables. AccountHistory, /// Prune part responsible for the `StorageChangeSet` and `StorageHistory` tables. diff --git a/crates/primitives/src/prune/target.rs b/crates/primitives/src/prune/target.rs index dca44764fdfa..28bc96797cf6 100644 --- a/crates/primitives/src/prune/target.rs +++ b/crates/primitives/src/prune/target.rs @@ -45,7 +45,7 @@ pub struct PruneModes { /// /// The [`BlockNumber`] represents the starting block from which point onwards the receipts are /// preserved. - pub only_contract_logs: BTreeMap>, + pub only_contract_logs: ContractLogPart, } macro_rules! impl_prune_parts { @@ -88,7 +88,7 @@ macro_rules! impl_prune_parts { $( $part: Some(PruneMode::Full), )+ - only_contract_logs: BTreeMap::new() + only_contract_logs: Default::default() } } @@ -109,3 +109,34 @@ impl PruneModes { (storage_history, StorageHistory, Some(64)) ); } + +/// Configuration for pruning receipts not associated with logs emitted by the specified contracts. +#[derive(Debug, Clone, PartialEq, Eq, Default, Serialize, Deserialize)] +pub struct ContractLogPart(pub Vec<(PruneMode, Address)>); + +impl ContractLogPart { + /// Checks if the configuration is empty + pub fn is_empty(&self) -> bool { + self.0.is_empty() + } + + /// Given the `tip` block number, flatten the struct so it can easily be queried for filtering + /// across a range of blocks. + /// + /// The [`BlockNumber`] key of the map should be viewed as `PruneMode::Before(block)`. + pub fn flatten( + &self, + tip: BlockNumber, + ) -> Result>, PrunePartError> { + let mut map = BTreeMap::new(); + for (mode, address) in self.0.iter() { + let block = mode + .prune_target_block(tip, 0, PrunePart::ContractLogs)? + .map(|(block, _)| block) + .unwrap_or_default(); + + map.entry(block).or_insert_with(Vec::new).push(address) + } + Ok(map) + } +} diff --git a/crates/storage/provider/src/post_state/mod.rs b/crates/storage/provider/src/post_state/mod.rs index 96330050b0c0..3d0acfc8e4f7 100644 --- a/crates/storage/provider/src/post_state/mod.rs +++ b/crates/storage/provider/src/post_state/mod.rs @@ -657,27 +657,30 @@ impl PostState { let mut bodies_cursor = tx.cursor_read::()?; let mut receipts_cursor = tx.cursor_write::()?; + let contract_log_pruner = + self.prune_modes.only_contract_logs.flatten(tip).expect("TODO"); // Empty implies that there is going to be // addresses to include in the filter in a future block. None means there isn't any kind // of configuration. - let mut address_filter = None; + let mut address_filter: Option<(u64, Vec<&Address>)> = None; for (block, receipts) in self.receipts { if receipts.is_empty() || self.prune_modes.should_prune_receipts(block, tip) { continue } - if !self.prune_modes.only_contract_logs.is_empty() { + if !contract_log_pruner.is_empty() { if address_filter.is_none() { address_filter = Some((0, vec![])); } - // Get all addresses higher than prev_block up to this block + // Get all addresses higher than the previous checked block up to the current + // one if let Some((prev_block, filter)) = &mut address_filter { - self.prune_modes - .only_contract_logs - .range(*prev_block..=block) - .for_each(|(_, addresses)| filter.extend(addresses)); + for (_, addresses) in contract_log_pruner.range(*prev_block..=block) { + filter.extend_from_slice(addresses.as_slice()) + } + *prev_block = block; } } @@ -690,7 +693,7 @@ impl PostState { // If there is an address_filter, and it does not contain any of the contract // addresses, then skip writing this receipt. if let Some((_, filter)) = &address_filter { - if !receipt.logs.iter().any(|log| filter.contains(&log.address)) { + if !receipt.logs.iter().any(|log| filter.contains(&&log.address)) { continue } } From 5345062eb7092062b37db218463c289d42caae62 Mon Sep 17 00:00:00 2001 From: joshieDo Date: Thu, 3 Aug 2023 15:32:19 +0000 Subject: [PATCH 06/15] smol refactor and renaming --- bin/reth/src/args/pruning_args.rs | 4 +- crates/primitives/src/lib.rs | 2 +- crates/primitives/src/prune/mod.rs | 37 +++++++++++++++- crates/primitives/src/prune/target.rs | 42 +++---------------- crates/storage/provider/src/post_state/mod.rs | 2 +- 5 files changed, 45 insertions(+), 42 deletions(-) diff --git a/bin/reth/src/args/pruning_args.rs b/bin/reth/src/args/pruning_args.rs index b5cd4a7c6052..7c6398194916 100644 --- a/bin/reth/src/args/pruning_args.rs +++ b/bin/reth/src/args/pruning_args.rs @@ -2,7 +2,7 @@ use clap::Args; use reth_config::config::PruneConfig; -use reth_primitives::{ChainSpec, ContractLogPart, PruneMode, PruneModes}; +use reth_primitives::{ChainSpec, ContractLogsPruneConfig, PruneMode, PruneModes}; use std::sync::Arc; /// Parameters for pruning and full node @@ -31,7 +31,7 @@ impl PruningArgs { .map(|contract| PruneMode::Before(contract.block)), account_history: Some(PruneMode::Distance(128)), storage_history: Some(PruneMode::Distance(128)), - only_contract_logs: ContractLogPart( + contract_logs_filter: ContractLogsPruneConfig( chain_spec .deposit_contract .as_ref() diff --git a/crates/primitives/src/lib.rs b/crates/primitives/src/lib.rs index b668f5ca965b..75c37010ceab 100644 --- a/crates/primitives/src/lib.rs +++ b/crates/primitives/src/lib.rs @@ -80,7 +80,7 @@ pub use net::{ }; pub use peer::{PeerId, WithPeerId}; pub use prune::{ - ContractLogPart, PruneCheckpoint, PruneMode, PruneModes, PrunePart, PrunePartError, + ContractLogsPruneConfig, PruneCheckpoint, PruneMode, PruneModes, PrunePart, PrunePartError, }; pub use receipt::{Receipt, ReceiptWithBloom, ReceiptWithBloomRef}; pub use revm_primitives::JumpMap; diff --git a/crates/primitives/src/prune/mod.rs b/crates/primitives/src/prune/mod.rs index e3900995ecbc..897d4839de2d 100644 --- a/crates/primitives/src/prune/mod.rs +++ b/crates/primitives/src/prune/mod.rs @@ -6,4 +6,39 @@ mod target; pub use checkpoint::PruneCheckpoint; pub use mode::PruneMode; pub use part::{PrunePart, PrunePartError}; -pub use target::{ContractLogPart, PruneModes}; +pub use target::PruneModes; + +use crate::{Address, BlockNumber}; +use serde::{Deserialize, Serialize}; +use std::collections::BTreeMap; + +/// Configuration for pruning receipts not associated with logs emitted by the specified contracts. +#[derive(Debug, Clone, PartialEq, Eq, Default, Serialize, Deserialize)] +pub struct ContractLogsPruneConfig(pub Vec<(PruneMode, Address)>); + +impl ContractLogsPruneConfig { + /// Checks if the configuration is empty + pub fn is_empty(&self) -> bool { + self.0.is_empty() + } + + /// Given the `tip` block number, flatten the struct so it can easily be queried for filtering + /// across a range of blocks. + /// + /// The [`BlockNumber`] key of the map should be viewed as `PruneMode::Before(block)`. + pub fn flatten( + &self, + tip: BlockNumber, + ) -> Result>, PrunePartError> { + let mut map = BTreeMap::new(); + for (mode, address) in self.0.iter() { + let block = mode + .prune_target_block(tip, 0, PrunePart::ContractLogs)? + .map(|(block, _)| block) + .unwrap_or_default(); + + map.entry(block).or_insert_with(Vec::new).push(address) + } + Ok(map) + } +} diff --git a/crates/primitives/src/prune/target.rs b/crates/primitives/src/prune/target.rs index 28bc96797cf6..267163dff807 100644 --- a/crates/primitives/src/prune/target.rs +++ b/crates/primitives/src/prune/target.rs @@ -1,10 +1,9 @@ use crate::{ - prune::PrunePartError, serde_helper::deserialize_opt_prune_mode_with_min_blocks, Address, - BlockNumber, PruneMode, PrunePart, + prune::PrunePartError, serde_helper::deserialize_opt_prune_mode_with_min_blocks, BlockNumber, + ContractLogsPruneConfig, PruneMode, PrunePart, }; use paste::paste; use serde::{Deserialize, Serialize}; -use std::collections::BTreeMap; /// Pruning configuration for every part of the data that can be pruned. #[derive(Debug, Clone, Default, Deserialize, Eq, PartialEq, Serialize)] @@ -22,7 +21,7 @@ pub struct PruneModes { #[serde(skip_serializing_if = "Option::is_none")] pub transaction_lookup: Option, /// Configuration for pruning of receipts. This setting overrides - /// `PruneModes::only_contract_logs` and offers improved performance. + /// `PruneModes::contract_logs_filter` and offers improved performance. #[serde( skip_serializing_if = "Option::is_none", deserialize_with = "deserialize_opt_prune_mode_with_min_blocks::<64, _>" @@ -45,7 +44,7 @@ pub struct PruneModes { /// /// The [`BlockNumber`] represents the starting block from which point onwards the receipts are /// preserved. - pub only_contract_logs: ContractLogPart, + pub contract_logs_filter: ContractLogsPruneConfig, } macro_rules! impl_prune_parts { @@ -88,7 +87,7 @@ macro_rules! impl_prune_parts { $( $part: Some(PruneMode::Full), )+ - only_contract_logs: Default::default() + contract_logs_filter: Default::default() } } @@ -109,34 +108,3 @@ impl PruneModes { (storage_history, StorageHistory, Some(64)) ); } - -/// Configuration for pruning receipts not associated with logs emitted by the specified contracts. -#[derive(Debug, Clone, PartialEq, Eq, Default, Serialize, Deserialize)] -pub struct ContractLogPart(pub Vec<(PruneMode, Address)>); - -impl ContractLogPart { - /// Checks if the configuration is empty - pub fn is_empty(&self) -> bool { - self.0.is_empty() - } - - /// Given the `tip` block number, flatten the struct so it can easily be queried for filtering - /// across a range of blocks. - /// - /// The [`BlockNumber`] key of the map should be viewed as `PruneMode::Before(block)`. - pub fn flatten( - &self, - tip: BlockNumber, - ) -> Result>, PrunePartError> { - let mut map = BTreeMap::new(); - for (mode, address) in self.0.iter() { - let block = mode - .prune_target_block(tip, 0, PrunePart::ContractLogs)? - .map(|(block, _)| block) - .unwrap_or_default(); - - map.entry(block).or_insert_with(Vec::new).push(address) - } - Ok(map) - } -} diff --git a/crates/storage/provider/src/post_state/mod.rs b/crates/storage/provider/src/post_state/mod.rs index 3d0acfc8e4f7..1aac5ba487eb 100644 --- a/crates/storage/provider/src/post_state/mod.rs +++ b/crates/storage/provider/src/post_state/mod.rs @@ -658,7 +658,7 @@ impl PostState { let mut receipts_cursor = tx.cursor_write::()?; let contract_log_pruner = - self.prune_modes.only_contract_logs.flatten(tip).expect("TODO"); + self.prune_modes.contract_logs_filter.flatten(tip).expect("TODO"); // Empty implies that there is going to be // addresses to include in the filter in a future block. None means there isn't any kind // of configuration. From cbd4d232632937494df508de04b8d91d27b541a6 Mon Sep 17 00:00:00 2001 From: joshieDo Date: Fri, 4 Aug 2023 09:55:32 +0000 Subject: [PATCH 07/15] min blocks is 128 even for contract logs --- crates/primitives/src/prune/mod.rs | 2 +- crates/primitives/src/prune/part.rs | 2 +- crates/storage/provider/src/post_state/mod.rs | 11 ++++++++--- 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/crates/primitives/src/prune/mod.rs b/crates/primitives/src/prune/mod.rs index 897d4839de2d..0b08620283c0 100644 --- a/crates/primitives/src/prune/mod.rs +++ b/crates/primitives/src/prune/mod.rs @@ -33,7 +33,7 @@ impl ContractLogsPruneConfig { let mut map = BTreeMap::new(); for (mode, address) in self.0.iter() { let block = mode - .prune_target_block(tip, 0, PrunePart::ContractLogs)? + .prune_target_block(tip, 128, PrunePart::ContractLogs)? .map(|(block, _)| block) .unwrap_or_default(); diff --git a/crates/primitives/src/prune/part.rs b/crates/primitives/src/prune/part.rs index d7a8fccd9023..be948dcd6296 100644 --- a/crates/primitives/src/prune/part.rs +++ b/crates/primitives/src/prune/part.rs @@ -12,7 +12,7 @@ pub enum PrunePart { TransactionLookup, /// Prune part responsible for all `Receipts`. Receipts, - /// Prune part responsible for some `Receipts`. + /// Prune part responsible for some `Receipts` filtered by logs. ContractLogs, /// Prune part responsible for the `AccountChangeSet` and `AccountHistory` tables. AccountHistory, diff --git a/crates/storage/provider/src/post_state/mod.rs b/crates/storage/provider/src/post_state/mod.rs index 1aac5ba487eb..d0b5a6caf672 100644 --- a/crates/storage/provider/src/post_state/mod.rs +++ b/crates/storage/provider/src/post_state/mod.rs @@ -6,6 +6,7 @@ use reth_db::{ transaction::{DbTx, DbTxMut}, DatabaseError as DbError, }; +use reth_interfaces::Error; use reth_primitives::{ bloom::logs_bloom, keccak256, proofs::calculate_receipt_root_ref, Account, Address, BlockNumber, Bloom, Bytecode, Log, PruneMode, PruneModes, Receipt, StorageEntry, H256, U256, @@ -600,7 +601,7 @@ impl PostState { mut self, tx: &TX, tip: BlockNumber, - ) -> Result<(), DbError> { + ) -> Result<(), Error> { self.write_history_to_db(tx, tip)?; // Write new storage state @@ -657,8 +658,12 @@ impl PostState { let mut bodies_cursor = tx.cursor_read::()?; let mut receipts_cursor = tx.cursor_write::()?; - let contract_log_pruner = - self.prune_modes.contract_logs_filter.flatten(tip).expect("TODO"); + let contract_log_pruner = self + .prune_modes + .contract_logs_filter + .flatten(tip) + .map_err(|e| Error::Custom(e.to_string()))?; + // Empty implies that there is going to be // addresses to include in the filter in a future block. None means there isn't any kind // of configuration. From dfa2d3fe3ed9208c1275d0d01dcd339ec103145d Mon Sep 17 00:00:00 2001 From: joshieDo Date: Fri, 4 Aug 2023 10:28:28 +0000 Subject: [PATCH 08/15] set MINIMUM_PRUNING_DISTANCE and apply it to contract log filters --- bin/reth/src/args/pruning_args.rs | 10 ++++---- crates/primitives/src/lib.rs | 1 + crates/primitives/src/prune/mod.rs | 2 +- crates/primitives/src/prune/target.rs | 3 +++ crates/storage/provider/src/post_state/mod.rs | 24 +++++++++++++------ 5 files changed, 28 insertions(+), 12 deletions(-) diff --git a/bin/reth/src/args/pruning_args.rs b/bin/reth/src/args/pruning_args.rs index 7c6398194916..f2008fd34399 100644 --- a/bin/reth/src/args/pruning_args.rs +++ b/bin/reth/src/args/pruning_args.rs @@ -2,7 +2,9 @@ use clap::Args; use reth_config::config::PruneConfig; -use reth_primitives::{ChainSpec, ContractLogsPruneConfig, PruneMode, PruneModes}; +use reth_primitives::{ + ChainSpec, ContractLogsPruneConfig, PruneMode, PruneModes, MINIMUM_PRUNING_DISTANCE, +}; use std::sync::Arc; /// Parameters for pruning and full node @@ -23,14 +25,14 @@ impl PruningArgs { Some(PruneConfig { block_interval: 5, parts: PruneModes { - sender_recovery: Some(PruneMode::Distance(128)), + sender_recovery: Some(PruneMode::Distance(MINIMUM_PRUNING_DISTANCE)), transaction_lookup: None, receipts: chain_spec .deposit_contract .as_ref() .map(|contract| PruneMode::Before(contract.block)), - account_history: Some(PruneMode::Distance(128)), - storage_history: Some(PruneMode::Distance(128)), + account_history: Some(PruneMode::Distance(MINIMUM_PRUNING_DISTANCE)), + storage_history: Some(PruneMode::Distance(MINIMUM_PRUNING_DISTANCE)), contract_logs_filter: ContractLogsPruneConfig( chain_spec .deposit_contract diff --git a/crates/primitives/src/lib.rs b/crates/primitives/src/lib.rs index 75c37010ceab..f1d8aa91b825 100644 --- a/crates/primitives/src/lib.rs +++ b/crates/primitives/src/lib.rs @@ -81,6 +81,7 @@ pub use net::{ pub use peer::{PeerId, WithPeerId}; pub use prune::{ ContractLogsPruneConfig, PruneCheckpoint, PruneMode, PruneModes, PrunePart, PrunePartError, + MINIMUM_PRUNING_DISTANCE, }; pub use receipt::{Receipt, ReceiptWithBloom, ReceiptWithBloomRef}; pub use revm_primitives::JumpMap; diff --git a/crates/primitives/src/prune/mod.rs b/crates/primitives/src/prune/mod.rs index 0b08620283c0..b46ab59647ad 100644 --- a/crates/primitives/src/prune/mod.rs +++ b/crates/primitives/src/prune/mod.rs @@ -6,7 +6,7 @@ mod target; pub use checkpoint::PruneCheckpoint; pub use mode::PruneMode; pub use part::{PrunePart, PrunePartError}; -pub use target::PruneModes; +pub use target::{PruneModes, MINIMUM_PRUNING_DISTANCE}; use crate::{Address, BlockNumber}; use serde::{Deserialize, Serialize}; diff --git a/crates/primitives/src/prune/target.rs b/crates/primitives/src/prune/target.rs index 267163dff807..9620569760e2 100644 --- a/crates/primitives/src/prune/target.rs +++ b/crates/primitives/src/prune/target.rs @@ -5,6 +5,9 @@ use crate::{ use paste::paste; use serde::{Deserialize, Serialize}; +/// Minimum distance necessary from the tip so blockchain tree can work correctly. +pub const MINIMUM_PRUNING_DISTANCE: u64 = 128; + /// Pruning configuration for every part of the data that can be pruned. #[derive(Debug, Clone, Default, Deserialize, Eq, PartialEq, Serialize)] #[serde(default)] diff --git a/crates/storage/provider/src/post_state/mod.rs b/crates/storage/provider/src/post_state/mod.rs index d0b5a6caf672..adceb15ece4a 100644 --- a/crates/storage/provider/src/post_state/mod.rs +++ b/crates/storage/provider/src/post_state/mod.rs @@ -9,7 +9,8 @@ use reth_db::{ use reth_interfaces::Error; use reth_primitives::{ bloom::logs_bloom, keccak256, proofs::calculate_receipt_root_ref, Account, Address, - BlockNumber, Bloom, Bytecode, Log, PruneMode, PruneModes, Receipt, StorageEntry, H256, U256, + BlockNumber, Bloom, Bytecode, Log, PruneMode, PruneModes, Receipt, StorageEntry, H256, + MINIMUM_PRUNING_DISTANCE, U256, }; use reth_trie::{ hashed_cursor::{HashedPostState, HashedPostStateCursorFactory, HashedStorage}, @@ -674,7 +675,12 @@ impl PostState { continue } - if !contract_log_pruner.is_empty() { + // All receipts from the last 128 blocks are required for blockchain tree, even with + // the contract log filters. + let prunable_receipts = + PruneMode::Distance(MINIMUM_PRUNING_DISTANCE).should_prune(block, tip); + + if prunable_receipts && !contract_log_pruner.is_empty() { if address_filter.is_none() { address_filter = Some((0, vec![])); } @@ -694,12 +700,16 @@ impl PostState { bodies_cursor.seek_exact(block)?.expect("body indices exist"); let tx_range = body_indices.tx_num_range(); assert_eq!(receipts.len(), tx_range.clone().count(), "Receipt length mismatch"); + for (tx_num, receipt) in tx_range.zip(receipts) { - // If there is an address_filter, and it does not contain any of the contract - // addresses, then skip writing this receipt. - if let Some((_, filter)) = &address_filter { - if !receipt.logs.iter().any(|log| filter.contains(&&log.address)) { - continue + if prunable_receipts { + // If there is an address_filter, and it does not contain any of the + // contract addresses, then skip writing this + // receipt. + if let Some((_, filter)) = &address_filter { + if !receipt.logs.iter().any(|log| filter.contains(&&log.address)) { + continue + } } } receipts_cursor.append(tx_num, receipt)?; From ad3fde0331dc5d843beaf5a70349ed9391075032 Mon Sep 17 00:00:00 2001 From: joshieDo Date: Fri, 4 Aug 2023 10:38:26 +0000 Subject: [PATCH 09/15] add MINIMUM_PRUNING_DISTANCE to flatten --- crates/primitives/src/prune/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/primitives/src/prune/mod.rs b/crates/primitives/src/prune/mod.rs index b46ab59647ad..e0315c3d14c1 100644 --- a/crates/primitives/src/prune/mod.rs +++ b/crates/primitives/src/prune/mod.rs @@ -33,7 +33,7 @@ impl ContractLogsPruneConfig { let mut map = BTreeMap::new(); for (mode, address) in self.0.iter() { let block = mode - .prune_target_block(tip, 128, PrunePart::ContractLogs)? + .prune_target_block(tip, MINIMUM_PRUNING_DISTANCE, PrunePart::ContractLogs)? .map(|(block, _)| block) .unwrap_or_default(); From 70c86e34a6846b999ec73b77938fb20672338bcc Mon Sep 17 00:00:00 2001 From: joshieDo Date: Fri, 4 Aug 2023 10:43:10 +0000 Subject: [PATCH 10/15] add docs to ContractLogsPruneConfig::flatten --- crates/primitives/src/prune/mod.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/crates/primitives/src/prune/mod.rs b/crates/primitives/src/prune/mod.rs index e0315c3d14c1..c6c542883782 100644 --- a/crates/primitives/src/prune/mod.rs +++ b/crates/primitives/src/prune/mod.rs @@ -32,6 +32,10 @@ impl ContractLogsPruneConfig { ) -> Result>, PrunePartError> { let mut map = BTreeMap::new(); for (mode, address) in self.0.iter() { + // Getting `None`, means that there is nothing to prune yet, so we need it to include in + // the BTreeMap (block = 0), otherwise it will be excluded. + // Reminder that this BTreeMap works as an inclusion list that excludes (prunes) all + // other receipts. let block = mode .prune_target_block(tip, MINIMUM_PRUNING_DISTANCE, PrunePart::ContractLogs)? .map(|(block, _)| block) From 901fa461c945d04b0e9573c871b6faaf5b7bd184 Mon Sep 17 00:00:00 2001 From: joshieDo Date: Fri, 4 Aug 2023 10:45:08 +0000 Subject: [PATCH 11/15] adds more docs --- crates/storage/provider/src/post_state/mod.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/crates/storage/provider/src/post_state/mod.rs b/crates/storage/provider/src/post_state/mod.rs index adceb15ece4a..9b20c789ff2a 100644 --- a/crates/storage/provider/src/post_state/mod.rs +++ b/crates/storage/provider/src/post_state/mod.rs @@ -671,12 +671,13 @@ impl PostState { let mut address_filter: Option<(u64, Vec<&Address>)> = None; for (block, receipts) in self.receipts { + // [`PrunePart::Receipts`] takes priority over [`PrunePart::ContractLogs`] if receipts.is_empty() || self.prune_modes.should_prune_receipts(block, tip) { continue } // All receipts from the last 128 blocks are required for blockchain tree, even with - // the contract log filters. + // [`PrunePart::ContractLogs`]. let prunable_receipts = PruneMode::Distance(MINIMUM_PRUNING_DISTANCE).should_prune(block, tip); From 9444eaede4fc8bccde0179d7ad7a1126ecb11ff4 Mon Sep 17 00:00:00 2001 From: joshieDo Date: Fri, 4 Aug 2023 12:16:38 +0000 Subject: [PATCH 12/15] rm newline --- crates/primitives/src/prune/mod.rs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/crates/primitives/src/prune/mod.rs b/crates/primitives/src/prune/mod.rs index c6c542883782..30d72bbfc3b5 100644 --- a/crates/primitives/src/prune/mod.rs +++ b/crates/primitives/src/prune/mod.rs @@ -3,14 +3,13 @@ mod mode; mod part; mod target; +use crate::{Address, BlockNumber}; pub use checkpoint::PruneCheckpoint; pub use mode::PruneMode; pub use part::{PrunePart, PrunePartError}; -pub use target::{PruneModes, MINIMUM_PRUNING_DISTANCE}; - -use crate::{Address, BlockNumber}; use serde::{Deserialize, Serialize}; use std::collections::BTreeMap; +pub use target::{PruneModes, MINIMUM_PRUNING_DISTANCE}; /// Configuration for pruning receipts not associated with logs emitted by the specified contracts. #[derive(Debug, Clone, PartialEq, Eq, Default, Serialize, Deserialize)] From 2da99069e61a84c1eac815c4d6a6bcb45e24688c Mon Sep 17 00:00:00 2001 From: joshieDo Date: Fri, 4 Aug 2023 12:56:00 +0000 Subject: [PATCH 13/15] add sanity tests to PruneMode --- crates/primitives/src/prune/mode.rs | 75 +++++++++++++++++++++++++++-- crates/primitives/src/prune/part.rs | 2 +- 2 files changed, 73 insertions(+), 4 deletions(-) diff --git a/crates/primitives/src/prune/mode.rs b/crates/primitives/src/prune/mode.rs index 9a9ee5a0ca93..3e39f876d7ce 100644 --- a/crates/primitives/src/prune/mode.rs +++ b/crates/primitives/src/prune/mode.rs @@ -23,7 +23,7 @@ impl PruneMode { min_blocks: u64, prune_part: PrunePart, ) -> Result, PrunePartError> { - Ok(match self { + let result = match self { PruneMode::Full if min_blocks == 0 => Some((tip, *self)), PruneMode::Distance(distance) if *distance > tip => None, // Nothing to prune yet PruneMode::Distance(distance) if *distance >= min_blocks => { @@ -32,7 +32,8 @@ impl PruneMode { PruneMode::Before(n) if *n > tip => None, // Nothing to prune yet PruneMode::Before(n) if tip - n >= min_blocks => Some((n - 1, *self)), _ => return Err(PrunePartError::Configuration(prune_part)), - }) + }; + Ok(result) } /// Check if target block should be pruned according to the provided prune mode and tip. @@ -59,10 +60,78 @@ impl Default for PruneMode { #[cfg(test)] mod tests { - use crate::prune::PruneMode; + use crate::{prune::PruneMode, PrunePart, PrunePartError, MINIMUM_PRUNING_DISTANCE}; use assert_matches::assert_matches; use serde::Deserialize; + #[test] + fn test_prune_target_block() { + let tip = 1000; + let min_blocks = MINIMUM_PRUNING_DISTANCE; + let prune_part = PrunePart::Receipts; + + let tests = vec![ + // MINIMUM_PRUNING_DISTANCE makes this impossible + (PruneMode::Full, Err(PrunePartError::Configuration(prune_part))), + // Nothing to prune + (PruneMode::Distance(tip + 1), Ok(None)), + (PruneMode::Distance(min_blocks + 1), Ok(Some(tip - (min_blocks + 1)))), + // Nothing to prune + (PruneMode::Before(tip + 1), Ok(None)), + ( + PruneMode::Before(tip - MINIMUM_PRUNING_DISTANCE), + Ok(Some(tip - MINIMUM_PRUNING_DISTANCE - 1)), + ), + ( + PruneMode::Before(tip - MINIMUM_PRUNING_DISTANCE - 1), + Ok(Some(tip - MINIMUM_PRUNING_DISTANCE - 2)), + ), + // MINIMUM_PRUNING_DISTANCE is 128 + (PruneMode::Before(tip - 1), Err(PrunePartError::Configuration(prune_part))), + ]; + + for (index, (mode, expected_result)) in tests.into_iter().enumerate() { + assert_eq!( + mode.prune_target_block(tip, min_blocks, prune_part), + expected_result.map(|r| r.map(|b| (b, mode))), + "Test {} failed", + index + 1, + ); + } + + // Test for a scenario where there are no minimum blocks and Full can be used + assert_eq!( + PruneMode::Full.prune_target_block(tip, 0, prune_part), + Ok(Some((tip, PruneMode::Full))), + ); + } + + #[test] + fn test_should_prune() { + let tip = 1000; + let should_prune = true; + + let tests = vec![ + (PruneMode::Distance(tip + 1), 1, !should_prune), + ( + PruneMode::Distance(MINIMUM_PRUNING_DISTANCE + 1), + tip - MINIMUM_PRUNING_DISTANCE - 1, + !should_prune, + ), + ( + PruneMode::Distance(MINIMUM_PRUNING_DISTANCE + 1), + tip - MINIMUM_PRUNING_DISTANCE - 2, + should_prune, + ), + (PruneMode::Before(tip + 1), 1, should_prune), + (PruneMode::Before(tip + 1), tip + 1, !should_prune), + ]; + + for (index, (mode, block, expected_result)) in tests.into_iter().enumerate() { + assert_eq!(mode.should_prune(block, tip), expected_result, "Test {} failed", index + 1,); + } + } + #[test] fn prune_mode_deserialize() { #[derive(Debug, Deserialize)] diff --git a/crates/primitives/src/prune/part.rs b/crates/primitives/src/prune/part.rs index be948dcd6296..03b64b916309 100644 --- a/crates/primitives/src/prune/part.rs +++ b/crates/primitives/src/prune/part.rs @@ -21,7 +21,7 @@ pub enum PrunePart { } /// PrunePart error type. -#[derive(Debug, Error)] +#[derive(Debug, Error, PartialEq, Eq)] pub enum PrunePartError { /// Invalid configuration of a prune part. #[error("The configuration provided for {0} is invalid.")] From a814441b5e56a91033070c76861e3594738aa3a3 Mon Sep 17 00:00:00 2001 From: joshieDo Date: Fri, 4 Aug 2023 14:46:36 +0000 Subject: [PATCH 14/15] turn ContractLogsPruneConfig to btreemap --- bin/reth/src/args/pruning_args.rs | 2 +- crates/primitives/src/prune/mod.rs | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/bin/reth/src/args/pruning_args.rs b/bin/reth/src/args/pruning_args.rs index f2008fd34399..fe9ace6f48e1 100644 --- a/bin/reth/src/args/pruning_args.rs +++ b/bin/reth/src/args/pruning_args.rs @@ -37,7 +37,7 @@ impl PruningArgs { chain_spec .deposit_contract .as_ref() - .map(|contract| (PruneMode::Before(contract.block), contract.address)) + .map(|contract| (contract.address, PruneMode::Before(contract.block))) .into_iter() .collect(), ), diff --git a/crates/primitives/src/prune/mod.rs b/crates/primitives/src/prune/mod.rs index 30d72bbfc3b5..57ef77c504f1 100644 --- a/crates/primitives/src/prune/mod.rs +++ b/crates/primitives/src/prune/mod.rs @@ -13,7 +13,7 @@ pub use target::{PruneModes, MINIMUM_PRUNING_DISTANCE}; /// Configuration for pruning receipts not associated with logs emitted by the specified contracts. #[derive(Debug, Clone, PartialEq, Eq, Default, Serialize, Deserialize)] -pub struct ContractLogsPruneConfig(pub Vec<(PruneMode, Address)>); +pub struct ContractLogsPruneConfig(pub BTreeMap); impl ContractLogsPruneConfig { /// Checks if the configuration is empty @@ -30,7 +30,7 @@ impl ContractLogsPruneConfig { tip: BlockNumber, ) -> Result>, PrunePartError> { let mut map = BTreeMap::new(); - for (mode, address) in self.0.iter() { + for (address, mode) in self.0.iter() { // Getting `None`, means that there is nothing to prune yet, so we need it to include in // the BTreeMap (block = 0), otherwise it will be excluded. // Reminder that this BTreeMap works as an inclusion list that excludes (prunes) all From 7ca2df972973bc4b14e5ee74dd3d3369d97edd0b Mon Sep 17 00:00:00 2001 From: joshieDo Date: Fri, 4 Aug 2023 14:57:02 +0000 Subject: [PATCH 15/15] rename flatten to group_by_block --- crates/primitives/src/prune/mod.rs | 6 +++--- crates/storage/provider/src/post_state/mod.rs | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/crates/primitives/src/prune/mod.rs b/crates/primitives/src/prune/mod.rs index 57ef77c504f1..bec2bfa26254 100644 --- a/crates/primitives/src/prune/mod.rs +++ b/crates/primitives/src/prune/mod.rs @@ -21,11 +21,11 @@ impl ContractLogsPruneConfig { self.0.is_empty() } - /// Given the `tip` block number, flatten the struct so it can easily be queried for filtering - /// across a range of blocks. + /// Given the `tip` block number, consolidates the structure so it can easily be queried for + /// filtering across a range of blocks. /// /// The [`BlockNumber`] key of the map should be viewed as `PruneMode::Before(block)`. - pub fn flatten( + pub fn group_by_block( &self, tip: BlockNumber, ) -> Result>, PrunePartError> { diff --git a/crates/storage/provider/src/post_state/mod.rs b/crates/storage/provider/src/post_state/mod.rs index 9b20c789ff2a..4032531fff38 100644 --- a/crates/storage/provider/src/post_state/mod.rs +++ b/crates/storage/provider/src/post_state/mod.rs @@ -662,7 +662,7 @@ impl PostState { let contract_log_pruner = self .prune_modes .contract_logs_filter - .flatten(tip) + .group_by_block(tip) .map_err(|e| Error::Custom(e.to_string()))?; // Empty implies that there is going to be