From f445205728b02107ef961d135b28ea5ef70990a1 Mon Sep 17 00:00:00 2001 From: Alexey Shekhirin Date: Mon, 19 Feb 2024 16:40:12 +0000 Subject: [PATCH 1/4] feat(snapshot, stages): pass snapshot target per segment --- .../beacon/src/engine/hooks/snapshot.rs | 8 +++- crates/snapshot/src/snapshotter.rs | 43 ++++++++++++++----- crates/stages/src/error.rs | 1 + crates/stages/src/stages/snapshot.rs | 19 ++++++-- 4 files changed, 56 insertions(+), 15 deletions(-) diff --git a/crates/consensus/beacon/src/engine/hooks/snapshot.rs b/crates/consensus/beacon/src/engine/hooks/snapshot.rs index 58632359a5c4..11d926b381d9 100644 --- a/crates/consensus/beacon/src/engine/hooks/snapshot.rs +++ b/crates/consensus/beacon/src/engine/hooks/snapshot.rs @@ -7,7 +7,7 @@ use crate::{ use futures::FutureExt; use reth_db::database::Database; use reth_interfaces::RethResult; -use reth_primitives::BlockNumber; +use reth_primitives::{snapshot::HighestSnapshots, BlockNumber}; use reth_snapshot::{Snapshotter, SnapshotterWithResult}; use reth_tasks::TaskSpawner; use std::task::{ready, Context, Poll}; @@ -81,7 +81,11 @@ impl SnapshotHook { return Ok(None) }; - let targets = snapshotter.get_snapshot_targets(finalized_block_number)?; + let targets = snapshotter.get_snapshot_targets(HighestSnapshots { + headers: Some(finalized_block_number), + receipts: Some(finalized_block_number), + transactions: Some(finalized_block_number), + })?; // Check if the snapshotting of any data has been requested. if targets.any() { diff --git a/crates/snapshot/src/snapshotter.rs b/crates/snapshot/src/snapshotter.rs index 93be15e5dc20..e33d6c0e7410 100644 --- a/crates/snapshot/src/snapshotter.rs +++ b/crates/snapshot/src/snapshotter.rs @@ -137,32 +137,37 @@ impl Snapshotter { Ok(targets) } - /// Returns a snapshot targets at the provided finalized block number. + /// Returns a snapshot targets at the provided finalized block numbers per segment. /// The target is determined by the check against highest snapshots using /// [SnapshotProvider::get_highest_snapshots]. pub fn get_snapshot_targets( &self, - finalized_block_number: BlockNumber, + finalized_block_numbers: HighestSnapshots, ) -> RethResult { let highest_snapshots = self.snapshot_provider.get_highest_snapshots(); let targets = SnapshotTargets { - headers: self.get_snapshot_target(highest_snapshots.headers, finalized_block_number), + headers: finalized_block_numbers.headers.and_then(|finalized_block_number| { + self.get_snapshot_target(highest_snapshots.headers, finalized_block_number) + }), // Snapshot receipts only if they're not pruned according to the user configuration receipts: if self.prune_modes.receipts.is_none() && self.prune_modes.receipts_log_filter.is_empty() { - self.get_snapshot_target(highest_snapshots.receipts, finalized_block_number) + finalized_block_numbers.receipts.and_then(|finalized_block_number| { + self.get_snapshot_target(highest_snapshots.receipts, finalized_block_number) + }) } else { None }, - transactions: self - .get_snapshot_target(highest_snapshots.transactions, finalized_block_number), + transactions: finalized_block_numbers.transactions.and_then(|finalized_block_number| { + self.get_snapshot_target(highest_snapshots.transactions, finalized_block_number) + }), }; trace!( target: "snapshot", - %finalized_block_number, + ?finalized_block_numbers, ?highest_snapshots, ?targets, any = %targets.any(), @@ -237,7 +242,13 @@ mod tests { let mut snapshotter = Snapshotter::new(provider_factory, snapshot_provider.clone(), PruneModes::default()); - let targets = snapshotter.get_snapshot_targets(1).expect("get snapshot targets"); + let targets = snapshotter + .get_snapshot_targets(HighestSnapshots { + headers: Some(1), + receipts: Some(1), + transactions: Some(1), + }) + .expect("get snapshot targets"); assert_eq!( targets, SnapshotTargets { @@ -252,7 +263,13 @@ mod tests { HighestSnapshots { headers: Some(1), receipts: Some(1), transactions: Some(1) } ); - let targets = snapshotter.get_snapshot_targets(3).expect("get snapshot targets"); + let targets = snapshotter + .get_snapshot_targets(HighestSnapshots { + headers: Some(3), + receipts: Some(3), + transactions: Some(3), + }) + .expect("get snapshot targets"); assert_eq!( targets, SnapshotTargets { @@ -267,7 +284,13 @@ mod tests { HighestSnapshots { headers: Some(3), receipts: Some(3), transactions: Some(3) } ); - let targets = snapshotter.get_snapshot_targets(4).expect("get snapshot targets"); + let targets = snapshotter + .get_snapshot_targets(HighestSnapshots { + headers: Some(4), + receipts: Some(4), + transactions: Some(4), + }) + .expect("get snapshot targets"); assert_eq!( targets, SnapshotTargets { diff --git a/crates/stages/src/error.rs b/crates/stages/src/error.rs index 4758729e18b5..b06e03b9a418 100644 --- a/crates/stages/src/error.rs +++ b/crates/stages/src/error.rs @@ -136,6 +136,7 @@ impl StageError { StageError::ChannelClosed | StageError::InconsistentBlockNumber { .. } | StageError::InconsistentTxNumber { .. } | + StageError::Internal(_) | StageError::Fatal(_) ) } diff --git a/crates/stages/src/stages/snapshot.rs b/crates/stages/src/stages/snapshot.rs index bf0ed23abf83..3dbd37a232fa 100644 --- a/crates/stages/src/stages/snapshot.rs +++ b/crates/stages/src/stages/snapshot.rs @@ -1,7 +1,10 @@ use crate::{ExecInput, ExecOutput, Stage, StageError, UnwindInput, UnwindOutput}; use reth_db::database::Database; -use reth_primitives::stage::{StageCheckpoint, StageId}; -use reth_provider::{BlockNumReader, DatabaseProviderRW}; +use reth_primitives::{ + snapshot::HighestSnapshots, + stage::{StageCheckpoint, StageId}, +}; +use reth_provider::{DatabaseProviderRW, StageCheckpointReader}; use reth_snapshot::Snapshotter; /// The snapshot stage _copies_ all data from database to static files using [Snapshotter]. The @@ -30,7 +33,17 @@ impl Stage for SnapshotStage { provider: &DatabaseProviderRW, input: ExecInput, ) -> Result { - let targets = self.snapshotter.get_snapshot_targets(provider.best_block_number()?)?; + let targets = self.snapshotter.get_snapshot_targets(HighestSnapshots { + headers: provider + .get_stage_checkpoint(StageId::Headers)? + .map(|checkpoint| checkpoint.block_number), + receipts: provider + .get_stage_checkpoint(StageId::Execution)? + .map(|checkpoint| checkpoint.block_number), + transactions: provider + .get_stage_checkpoint(StageId::Bodies)? + .map(|checkpoint| checkpoint.block_number), + })?; self.snapshotter.run(targets)?; Ok(ExecOutput::done(input.checkpoint())) } From 78027b0fc4ce653c0ea2a3edc083e518784d92c5 Mon Sep 17 00:00:00 2001 From: Alexey Shekhirin Date: Mon, 19 Feb 2024 17:16:59 +0000 Subject: [PATCH 2/4] fix lint --- crates/consensus/beacon/src/engine/mod.rs | 2 +- crates/etl/src/lib.rs | 1 - crates/stages/src/stages/execution.rs | 2 +- 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/crates/consensus/beacon/src/engine/mod.rs b/crates/consensus/beacon/src/engine/mod.rs index 0cb3deca2e8c..c3a197b18f7f 100644 --- a/crates/consensus/beacon/src/engine/mod.rs +++ b/crates/consensus/beacon/src/engine/mod.rs @@ -2412,7 +2412,7 @@ mod tests { use reth_interfaces::test_utils::{generators, generators::random_block}; use reth_primitives::{ genesis::{Genesis, GenesisAllocator}, - Hardfork, + Hardfork, U256, }; use reth_provider::test_utils::blocks::BlockChainTestData; diff --git a/crates/etl/src/lib.rs b/crates/etl/src/lib.rs index 6ff6fc0c5a1e..d29dd54ad536 100644 --- a/crates/etl/src/lib.rs +++ b/crates/etl/src/lib.rs @@ -238,7 +238,6 @@ impl EtlFile { #[cfg(test)] mod tests { use reth_primitives::{TxHash, TxNumber}; - use tempfile::TempDir; use super::*; diff --git a/crates/stages/src/stages/execution.rs b/crates/stages/src/stages/execution.rs index 8d711ee1ef99..216963c6c377 100644 --- a/crates/stages/src/stages/execution.rs +++ b/crates/stages/src/stages/execution.rs @@ -598,7 +598,7 @@ mod tests { use reth_primitives::{ address, hex_literal::hex, keccak256, stage::StageUnitCheckpoint, Account, Address, Bytecode, ChainSpecBuilder, PruneMode, PruneModes, ReceiptsLogPruneConfig, SealedBlock, - StorageEntry, B256, MAINNET, U256, + StorageEntry, B256, U256, }; use reth_provider::{test_utils::create_test_provider_factory, AccountReader, ReceiptProvider}; use reth_revm::EvmProcessorFactory; From 0b611aa879e3da83ea410d23ae65734741d37910 Mon Sep 17 00:00:00 2001 From: Alexey Shekhirin Date: Mon, 19 Feb 2024 17:16:59 +0000 Subject: [PATCH 3/4] fix lint --- crates/consensus/beacon/src/engine/mod.rs | 2 +- crates/etl/src/lib.rs | 1 - crates/stages/src/stages/execution.rs | 2 +- 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/crates/consensus/beacon/src/engine/mod.rs b/crates/consensus/beacon/src/engine/mod.rs index 0cb3deca2e8c..c3a197b18f7f 100644 --- a/crates/consensus/beacon/src/engine/mod.rs +++ b/crates/consensus/beacon/src/engine/mod.rs @@ -2412,7 +2412,7 @@ mod tests { use reth_interfaces::test_utils::{generators, generators::random_block}; use reth_primitives::{ genesis::{Genesis, GenesisAllocator}, - Hardfork, + Hardfork, U256, }; use reth_provider::test_utils::blocks::BlockChainTestData; diff --git a/crates/etl/src/lib.rs b/crates/etl/src/lib.rs index 6ff6fc0c5a1e..d29dd54ad536 100644 --- a/crates/etl/src/lib.rs +++ b/crates/etl/src/lib.rs @@ -238,7 +238,6 @@ impl EtlFile { #[cfg(test)] mod tests { use reth_primitives::{TxHash, TxNumber}; - use tempfile::TempDir; use super::*; diff --git a/crates/stages/src/stages/execution.rs b/crates/stages/src/stages/execution.rs index 8d711ee1ef99..216963c6c377 100644 --- a/crates/stages/src/stages/execution.rs +++ b/crates/stages/src/stages/execution.rs @@ -598,7 +598,7 @@ mod tests { use reth_primitives::{ address, hex_literal::hex, keccak256, stage::StageUnitCheckpoint, Account, Address, Bytecode, ChainSpecBuilder, PruneMode, PruneModes, ReceiptsLogPruneConfig, SealedBlock, - StorageEntry, B256, MAINNET, U256, + StorageEntry, B256, U256, }; use reth_provider::{test_utils::create_test_provider_factory, AccountReader, ReceiptProvider}; use reth_revm::EvmProcessorFactory; From 070118374b746cdcc716a387d69d8768562e777c Mon Sep 17 00:00:00 2001 From: Alexey Shekhirin Date: Mon, 19 Feb 2024 22:08:09 +0000 Subject: [PATCH 4/4] fix comment --- crates/stages/src/stages/snapshot.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/crates/stages/src/stages/snapshot.rs b/crates/stages/src/stages/snapshot.rs index 3dbd37a232fa..f4951e2fbeb0 100644 --- a/crates/stages/src/stages/snapshot.rs +++ b/crates/stages/src/stages/snapshot.rs @@ -9,8 +9,7 @@ use reth_snapshot::Snapshotter; /// The snapshot stage _copies_ all data from database to static files using [Snapshotter]. The /// block range for copying is determined by the current highest blocks contained in static files -/// and [BlockNumReader::best_block_number], -/// i.e. the range is `highest_snapshotted_block_number..=best_block_number`. +/// and stage checkpoints for each segment individually. #[derive(Debug)] pub struct SnapshotStage { snapshotter: Snapshotter,