feat(snapshots): write Headers to snapshots during HeaderStage

bin/reth/src/commands/debug_cmd/execution.rs

@@ -214,7 +214,7 @@ impl Command {
             ProviderFactory::new(db.clone(), self.chain.clone(), data_dir.snapshots_path())?;
 
         debug!(target: "reth::cli", chain=%self.chain.chain, genesis=?self.chain.genesis_hash(), "Initializing genesis");
-        init_genesis(provider_factory.clone(), self.chain.clone())?;
+        init_genesis(provider_factory.clone())?;
 
         let consensus: Arc<dyn Consensus> = Arc::new(BeaconConsensus::new(Arc::clone(&self.chain)));
 

bin/reth/src/commands/debug_cmd/in_memory_merkle.rs

@@ -19,9 +19,8 @@ use reth_network::NetworkHandle;
 use reth_network_api::NetworkInfo;
 use reth_primitives::{fs, stage::StageId, BlockHashOrNumber, ChainSpec};
 use reth_provider::{
-    AccountExtReader, BlockWriter, ExecutorFactory, HashingWriter, HeaderProvider,
-    LatestStateProviderRef, OriginalValuesKnown, ProviderFactory, StageCheckpointReader,
-    StorageReader,
+    AccountExtReader, ExecutorFactory, HashingWriter, HeaderProvider, LatestStateProviderRef,
+    OriginalValuesKnown, ProviderFactory, StageCheckpointReader, StorageReader,
 };
 use reth_tasks::TaskExecutor;
 use reth_trie::{hashed_cursor::HashedPostStateCursorFactory, updates::TrieKey, StateRoot};
@@ -167,8 +166,10 @@ impl Command {
             .await?;
 
         let executor_factory = reth_revm::EvmProcessorFactory::new(self.chain.clone());
-        let mut executor =
-            executor_factory.with_state(LatestStateProviderRef::new(provider.tx_ref()));
+        let mut executor = executor_factory.with_state(LatestStateProviderRef::new(
+            provider.tx_ref(),
+            factory.snapshot_provider(),
+        ));
 
         let merkle_block_td =
             provider.header_td_by_number(merkle_block_number)?.unwrap_or_default();
@@ -201,7 +202,7 @@ impl Command {
         let provider_rw = factory.provider_rw()?;
 
         // Insert block, state and hashes
-        provider_rw.insert_block(
+        provider_rw.insert_historical_block(
             block
                 .clone()
                 .try_seal_with_senders()

bin/reth/src/commands/import.rs

@@ -98,7 +98,7 @@ impl ImportCommand {
 
         debug!(target: "reth::cli", chain=%self.chain.chain, genesis=?self.chain.genesis_hash(), "Initializing genesis");
 
-        init_genesis(provider_factory.clone(), self.chain.clone())?;
+        init_genesis(provider_factory.clone())?;
 
         let consensus = Arc::new(BeaconConsensus::new(self.chain.clone()));
         info!(target: "reth::cli", "Consensus engine initialized");

bin/reth/src/commands/init_cmd.rs

@@ -61,7 +61,8 @@ impl InitCommand {
             ProviderFactory::new(db.clone(), self.chain.clone(), data_dir.snapshots_path())?;
 
         info!(target: "reth::cli", "Writing genesis block");
-        let hash = init_genesis(provider_factory, self.chain)?;
+
+        let hash = init_genesis(provider_factory)?;
 
         info!(target: "reth::cli", hash = ?hash, "Genesis block written");
         Ok(())

bin/reth/src/commands/recover/storage_tries.rs

@@ -53,7 +53,7 @@ impl Command {
         let factory = ProviderFactory::new(&db, self.chain.clone(), data_dir.snapshots_path())?;
 
         debug!(target: "reth::cli", chain=%self.chain.chain, genesis=?self.chain.genesis_hash(), "Initializing genesis");
-        init_genesis(factory.clone(), self.chain.clone())?;
+        init_genesis(factory.clone())?;
 
         let mut provider = factory.provider_rw()?;
         let best_block = provider.best_block_number()?;

bin/reth/src/commands/stage/drop.rs

@@ -62,6 +62,7 @@ impl Command {
             open_db(db_path.as_ref(), DatabaseArguments::default().log_level(self.db.log_level))?;
         let provider_factory =
             ProviderFactory::new(db, self.chain.clone(), data_dir.snapshots_path())?;
+        let snapshot_provider = provider_factory.snapshot_provider();
 
         let tool = DbTool::new(provider_factory, self.chain.clone())?;
 
@@ -74,7 +75,7 @@ impl Command {
                     tx.clear::<tables::BlockOmmers>()?;
                     tx.clear::<tables::BlockWithdrawals>()?;
                     tx.put::<tables::SyncStage>(StageId::Bodies.to_string(), Default::default())?;
-                    insert_genesis_header::<DatabaseEnv>(tx, self.chain)?;
+                    insert_genesis_header::<DatabaseEnv>(tx, snapshot_provider, self.chain)?;
                 }
                 StageEnum::Senders => {
                     tx.clear::<tables::TxSenders>()?;
@@ -159,15 +160,15 @@ impl Command {
                         StageId::TotalDifficulty.to_string(),
                         Default::default(),
                     )?;
-                    insert_genesis_header::<DatabaseEnv>(tx, self.chain)?;
+                    insert_genesis_header::<DatabaseEnv>(tx, snapshot_provider, self.chain)?;
                 }
                 StageEnum::TxLookup => {
                     tx.clear::<tables::TxHashNumber>()?;
                     tx.put::<tables::SyncStage>(
                         StageId::TransactionLookup.to_string(),
                         Default::default(),
                     )?;
-                    insert_genesis_header::<DatabaseEnv>(tx, self.chain)?;
+                    insert_genesis_header::<DatabaseEnv>(tx, snapshot_provider, self.chain)?;
                 }
                 _ => {
                     info!("Nothing to do for stage {:?}", self.stage);

crates/blockchain-tree/src/blockchain_tree.rs

@@ -1288,7 +1288,7 @@ mod tests {
         let provider = factory.provider_rw().unwrap();
 
         provider
-            .insert_block(
+            .insert_historical_block(
                 genesis.try_seal_with_senders().expect("invalid tx signature in genesis"),
                 None,
             )

crates/blockchain-tree/src/externals.rs

@@ -1,9 +1,11 @@
 //! Blockchain tree externals.
 
-use reth_db::{cursor::DbCursorRO, database::Database, tables, transaction::DbTx};
+use reth_db::{
+    cursor::DbCursorRO, database::Database, snapshot::HeaderMask, tables, transaction::DbTx,
+};
 use reth_interfaces::{consensus::Consensus, RethResult};
-use reth_primitives::{BlockHash, BlockNumber};
-use reth_provider::ProviderFactory;
+use reth_primitives::{BlockHash, BlockNumber, SnapshotSegment};
+use reth_provider::{ProviderFactory, StatsReader};
 use std::{collections::BTreeMap, sync::Arc};
 
 /// A container for external components.
@@ -44,13 +46,33 @@ impl<DB: Database, EF> TreeExternals<DB, EF> {
         &self,
         num_hashes: usize,
     ) -> RethResult<BTreeMap<BlockNumber, BlockHash>> {
-        Ok(self
+        let mut hashes = self
             .provider_factory
             .provider()?
             .tx_ref()
             .cursor_read::<tables::CanonicalHeaders>()?
             .walk_back(None)?
             .take(num_hashes)
-            .collect::<Result<BTreeMap<BlockNumber, BlockHash>, _>>()?)
+            .collect::<Result<BTreeMap<BlockNumber, BlockHash>, _>>()?;
+
+        if hashes.len() < num_hashes {
+            let snapshot_provider = self.provider_factory.snapshot_provider();
+            let total_headers = snapshot_provider.count_entries::<tables::Headers>()? as u64;
+            if total_headers > 0 {
+                let range = total_headers
+                    .saturating_sub(1)
+                    .saturating_sub((num_hashes - hashes.len()) as u64)..
+                    total_headers;
+
+                hashes.extend(range.clone().zip(snapshot_provider.fetch_range_with_predicate(
+                    SnapshotSegment::Headers,
+                    range,
+                    |cursor, number| cursor.get_one::<HeaderMask<BlockHash>>(number.into()),
+                    |_| true,
+                )?));
+            }
+        }
+
+        Ok(hashes)
     }
 }

crates/consensus/beacon/Cargo.toml

@@ -58,6 +58,7 @@ reth-downloaders.workspace = true
 reth-node-builder.workspace = true
 
 assert_matches.workspace = true
+tempfile.workspace = true
 
 [features]
 optimism = [

crates/consensus/common/src/validation.rs

@@ -220,7 +220,7 @@ pub fn validate_block_standalone(
 
     // Check transaction root
     if let Err(error) = block.ensure_transaction_root_valid() {
-        return Err(ConsensusError::BodyTransactionRootDiff(error.into()));
+        return Err(ConsensusError::BodyTransactionRootDiff(error.into()))
     }
 
     // EIP-4895: Beacon chain push withdrawals as operations
@@ -280,19 +280,19 @@ fn check_gas_limit(
             return Err(ConsensusError::GasLimitInvalidIncrease {
                 parent_gas_limit,
                 child_gas_limit: child.gas_limit,
-            });
+            })
         }
     }
     // Check for a decrease in gas limit beyond the allowed threshold.
     else if parent_gas_limit - child.gas_limit >= parent_gas_limit / 1024 {
         return Err(ConsensusError::GasLimitInvalidDecrease {
             parent_gas_limit,
             child_gas_limit: child.gas_limit,
-        });
+        })
     }
     // Check if the child gas limit is below the minimum required limit.
     else if child.gas_limit < MINIMUM_GAS_LIMIT {
-        return Err(ConsensusError::GasLimitInvalidMinimum { child_gas_limit: child.gas_limit });
+        return Err(ConsensusError::GasLimitInvalidMinimum { child_gas_limit: child.gas_limit })
     }
 
     Ok(())

crates/ethereum-forks/src/hardfork.rs

@@ -55,7 +55,7 @@ impl Hardfork {
     /// Retrieves the activation block for the specified hardfork on the Ethereum mainnet.
     pub fn mainnet_activation_block(&self, chain: Chain) -> Option<u64> {
         if chain != Chain::mainnet() {
-            return None;
+            return None
         }
         match self {
             Hardfork::Frontier => Some(0),

crates/etl/src/lib.rs

@@ -212,7 +212,7 @@ impl EtlFile {
     /// Can return error if it reaches EOF before filling the internal buffers.
     pub(crate) fn read_next(&mut self) -> std::io::Result<Option<(Vec<u8>, Vec<u8>)>> {
         if self.len == 0 {
-            return Ok(None);
+            return Ok(None)
         }
 
         let mut buffer_key_length = [0; 8];

crates/net/network/src/transactions/fetcher.rs

@@ -288,7 +288,7 @@ impl TransactionFetcher {
                     );
 
                     max_retried_hashes.push(hash);
-                    continue;
+                    continue
                 }
                 *retries += 1;
             }