From ae6044dd4de022510a0cce6c1c1841a8967387e6 Mon Sep 17 00:00:00 2001 From: Alexey Shekhirin Date: Thu, 11 Jul 2024 14:24:09 +0100 Subject: [PATCH 1/2] chore(pruner): remove deletion limit per block, make it per run --- .../consensus/beacon/src/engine/test_utils.rs | 10 +++--- crates/node/builder/src/launch/mod.rs | 4 +-- crates/prune/prune/src/builder.rs | 16 ++------- crates/prune/prune/src/pruner.rs | 34 +++---------------- crates/prune/types/src/limiter.rs | 2 +- 5 files changed, 15 insertions(+), 51 deletions(-) diff --git a/crates/consensus/beacon/src/engine/test_utils.rs b/crates/consensus/beacon/src/engine/test_utils.rs index 7b5ee65ee941..356c5ac7976d 100644 --- a/crates/consensus/beacon/src/engine/test_utils.rs +++ b/crates/consensus/beacon/src/engine/test_utils.rs @@ -391,10 +391,13 @@ where // Setup blockchain tree let externals = TreeExternals::new(provider_factory.clone(), consensus, executor_factory); - let config = BlockchainTreeConfig::new(1, 2, 3, 2); let tree = Arc::new(ShareableBlockchainTree::new( - BlockchainTree::new(externals, config, PruneModes::default()) - .expect("failed to create tree"), + BlockchainTree::new( + externals, + BlockchainTreeConfig::new(1, 2, 3, 2), + PruneModes::default(), + ) + .expect("failed to create tree"), )); let latest = self.base_config.chain_spec.genesis_header().seal_slow(); let blockchain_provider = @@ -405,7 +408,6 @@ where vec![], 5, self.base_config.chain_spec.prune_delete_limit, - config.max_reorg_depth() as usize, None, watch::channel(FinishedExExHeight::NoExExs).1, ); diff --git a/crates/node/builder/src/launch/mod.rs b/crates/node/builder/src/launch/mod.rs index 3ca4e29fc88f..952b80401562 100644 --- a/crates/node/builder/src/launch/mod.rs +++ b/crates/node/builder/src/launch/mod.rs @@ -231,13 +231,11 @@ where let initial_target = ctx.node_config().debug.tip; - let mut pruner_builder = - ctx.pruner_builder().max_reorg_depth(ctx.tree_config().max_reorg_depth() as usize); + let mut pruner_builder = ctx.pruner_builder(); if let Some(exex_manager_handle) = &exex_manager_handle { pruner_builder = pruner_builder.finished_exex_height(exex_manager_handle.finished_height()); } - let pruner = pruner_builder.build(ctx.provider_factory().clone()); let pruner_events = pruner.events(); diff --git a/crates/prune/prune/src/builder.rs b/crates/prune/prune/src/builder.rs index 233e80a71a9a..3c706a66b29e 100644 --- a/crates/prune/prune/src/builder.rs +++ b/crates/prune/prune/src/builder.rs @@ -15,11 +15,7 @@ pub struct PrunerBuilder { block_interval: usize, /// Pruning configuration for every part of the data that can be pruned. segments: PruneModes, - /// The number of blocks that can be re-orged. - max_reorg_depth: usize, - /// The delete limit for pruner, per block. In the actual pruner run it will be multiplied by - /// the amount of blocks between pruner runs to account for the difference in amount of new - /// data coming in. + /// The delete limit for pruner, per run. prune_delete_limit: usize, /// Time a pruner job can run before timing out. timeout: Option, @@ -50,13 +46,7 @@ impl PrunerBuilder { self } - /// Sets the number of blocks that can be re-orged. - pub const fn max_reorg_depth(mut self, max_reorg_depth: usize) -> Self { - self.max_reorg_depth = max_reorg_depth; - self - } - - /// Sets the delete limit for pruner, per block. + /// Sets the delete limit for pruner, per run. pub const fn prune_delete_limit(mut self, prune_delete_limit: usize) -> Self { self.prune_delete_limit = prune_delete_limit; self @@ -89,7 +79,6 @@ impl PrunerBuilder { segments.into_vec(), self.block_interval, self.prune_delete_limit, - self.max_reorg_depth, self.timeout, self.finished_exex_height, ) @@ -101,7 +90,6 @@ impl Default for PrunerBuilder { Self { block_interval: 5, segments: PruneModes::none(), - max_reorg_depth: 64, prune_delete_limit: MAINNET.prune_delete_limit, timeout: None, finished_exex_height: watch::channel(FinishedExExHeight::NoExExs).1, diff --git a/crates/prune/prune/src/pruner.rs b/crates/prune/prune/src/pruner.rs index 60ff1ee80620..00cc10508d51 100644 --- a/crates/prune/prune/src/pruner.rs +++ b/crates/prune/prune/src/pruner.rs @@ -38,11 +38,8 @@ pub struct Pruner { /// number is updated with the tip block number the pruner was called with. It's used in /// conjunction with `min_block_interval` to determine when the pruning needs to be initiated. previous_tip_block_number: Option, - /// Maximum total entries to prune (delete from database) per block. - delete_limit_per_block: usize, - /// Maximum number of blocks to be pruned per run, as an additional restriction to - /// `previous_tip_block_number`. - prune_max_blocks_per_run: usize, + /// Maximum total entries to prune (delete from database) per run. + delete_limit: usize, /// Maximum time for a one pruner run. timeout: Option, /// The finished height of all `ExEx`'s. @@ -59,7 +56,6 @@ impl Pruner { segments: Vec>>, min_block_interval: usize, delete_limit: usize, - prune_max_blocks_per_run: usize, timeout: Option, finished_exex_height: watch::Receiver, ) -> Self { @@ -68,8 +64,7 @@ impl Pruner { segments, min_block_interval, previous_tip_block_number: None, - delete_limit_per_block: delete_limit, - prune_max_blocks_per_run, + delete_limit, timeout, finished_exex_height, metrics: Metrics::default(), @@ -105,25 +100,7 @@ impl Pruner { debug!(target: "pruner", %tip_block_number, "Pruner started"); let start = Instant::now(); - // Multiply `self.delete_limit` (number of rows to delete per block) by number of blocks - // since last pruner run. `self.previous_tip_block_number` is close to - // `tip_block_number`, usually within `self.block_interval` blocks, so - // `delete_limit` will not be too high. If it's too high, we additionally limit it by - // `self.prune_max_blocks_per_run`. - // - // Also see docs for `self.previous_tip_block_number`. - let blocks_since_last_run = self - .previous_tip_block_number - .map_or(1, |previous_tip_block_number| { - // Saturating subtraction is needed for the case when the chain was reverted, - // meaning current block number might be less than the previous tip - // block number. - tip_block_number.saturating_sub(previous_tip_block_number) as usize - }) - .min(self.prune_max_blocks_per_run); - - let mut limiter = PruneLimiter::default() - .set_deleted_entries_limit(self.delete_limit_per_block * blocks_since_last_run); + let mut limiter = PruneLimiter::default().set_deleted_entries_limit(self.delete_limit); if let Some(timeout) = self.timeout { limiter = limiter.set_time_limit(timeout); }; @@ -352,8 +329,7 @@ mod tests { let (finished_exex_height_tx, finished_exex_height_rx) = tokio::sync::watch::channel(FinishedExExHeight::NoExExs); - let mut pruner = - Pruner::new(provider_factory, vec![], 5, 0, 5, None, finished_exex_height_rx); + let mut pruner = Pruner::new(provider_factory, vec![], 5, 0, None, finished_exex_height_rx); // No last pruned block number was set before let first_block_number = 1; diff --git a/crates/prune/types/src/limiter.rs b/crates/prune/types/src/limiter.rs index 5927655ac0ef..09f0c4cd3f2d 100644 --- a/crates/prune/types/src/limiter.rs +++ b/crates/prune/types/src/limiter.rs @@ -7,7 +7,7 @@ use std::{ /// or the time it can run. #[derive(Debug, Clone, Default)] pub struct PruneLimiter { - /// Maximum entries (rows in the database) to delete from the database per block. + /// Maximum entries (rows in the database) to delete from the database per run. deleted_entries_limit: Option, /// Maximum duration of one prune run. time_limit: Option, From c66e2ab9cdcdda3cd763cf50ca359aa297d38f2d Mon Sep 17 00:00:00 2001 From: Alexey Shekhirin Date: Thu, 11 Jul 2024 16:23:27 +0100 Subject: [PATCH 2/2] bump delete limits 5x to adjust for removed per block accounting --- crates/chainspec/src/spec.rs | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/crates/chainspec/src/spec.rs b/crates/chainspec/src/spec.rs index 96eaeda537a6..f557a3d8de31 100644 --- a/crates/chainspec/src/spec.rs +++ b/crates/chainspec/src/spec.rs @@ -56,7 +56,7 @@ pub static MAINNET: Lazy> = Lazy::new(|| { b256!("649bbc62d0e31342afea4e5cd82d4049e7e1ee912fc0889aa790803be39038c5"), )), base_fee_params: BaseFeeParamsKind::Constant(BaseFeeParams::ethereum()), - prune_delete_limit: 3500, + prune_delete_limit: 20000, } .into() }); @@ -78,7 +78,7 @@ pub static SEPOLIA: Lazy> = Lazy::new(|| { b256!("649bbc62d0e31342afea4e5cd82d4049e7e1ee912fc0889aa790803be39038c5"), )), base_fee_params: BaseFeeParamsKind::Constant(BaseFeeParams::ethereum()), - prune_delete_limit: 1700, + prune_delete_limit: 10000, } .into() }); @@ -98,7 +98,7 @@ pub static HOLESKY: Lazy> = Lazy::new(|| { b256!("649bbc62d0e31342afea4e5cd82d4049e7e1ee912fc0889aa790803be39038c5"), )), base_fee_params: BaseFeeParamsKind::Constant(BaseFeeParams::ethereum()), - prune_delete_limit: 1700, + prune_delete_limit: 10000, } .into() }); @@ -143,7 +143,7 @@ pub static OP_MAINNET: Lazy> = Lazy::new(|| { ] .into(), ), - prune_delete_limit: 1700, + prune_delete_limit: 10000, ..Default::default() } .into() @@ -168,7 +168,7 @@ pub static OP_SEPOLIA: Lazy> = Lazy::new(|| { ] .into(), ), - prune_delete_limit: 1700, + prune_delete_limit: 10000, ..Default::default() } .into() @@ -193,7 +193,7 @@ pub static BASE_SEPOLIA: Lazy> = Lazy::new(|| { ] .into(), ), - prune_delete_limit: 1700, + prune_delete_limit: 10000, ..Default::default() } .into() @@ -218,7 +218,7 @@ pub static BASE_MAINNET: Lazy> = Lazy::new(|| { ] .into(), ), - prune_delete_limit: 1700, + prune_delete_limit: 10000, ..Default::default() } .into() @@ -300,9 +300,7 @@ pub struct ChainSpec { /// The parameters that configure how a block's base fee is computed pub base_fee_params: BaseFeeParamsKind, - /// The delete limit for pruner, per block. In the actual pruner run it will be multiplied by - /// the amount of blocks between pruner runs to account for the difference in amount of new - /// data coming in. + /// The delete limit for pruner, per run. pub prune_delete_limit: usize, }