WIP: static files (#6444)

Co-authored-by: joshieDo <93316087+joshieDo@users.noreply.github.com>
Co-authored-by: Bjerg <onbjerg@users.noreply.github.com>
Co-authored-by: Matthias Seitz <matthias.seitz@outlook.de>
Co-authored-by: Oliver Nordbjerg <hi@notbjerg.me>
Co-authored-by: joshieDo <ranriver@protonmail.com>
Co-authored-by: Thomas Coratger <thomas.coratger@gmail.com>

.config/nextest.toml

@@ -4,4 +4,4 @@ slow-timeout = { period = "30s", terminate-after = 4 }
 
 [[profile.default.overrides]]
 filter = "test(general_state_tests)"
-slow-timeout = { period = "1m", terminate-after = 4 }
+slow-timeout = { period = "1m", terminate-after = 10 }

.github/workflows/hive.yml

@@ -37,8 +37,8 @@ jobs:
       - name: Checkout hive tests
         uses: actions/checkout@v4
         with:
-          repository: ethereum/hive
-          ref: master
+          repository: joshiedo/hive
+          ref: reth/disable-pruner
           path: hivetests
 
       - uses: actions/setup-go@v3
@@ -182,8 +182,8 @@ jobs:
       - name: Checkout hive tests
         uses: actions/checkout@v4
         with:
-          repository: ethereum/hive
-          ref: master
+          repository: joshiedo/hive
+          ref: reth/disable-pruner
           path: hivetests
 
       - name: Run ${{ matrix.sim }} simulator

CODEOWNERS

@@ -19,5 +19,5 @@ crates/metrics              @onbjerg
 crates/tracing              @onbjerg
 crates/tasks                @mattsse
 crates/prune                @shekhirin @joshieDo
-crates/snapshot             @joshieDo
+crates/static-file          @joshieDo @shekhirin
 .github/                    @onbjerg @gakonst @DaniPopes

Cargo.toml

@@ -8,6 +8,7 @@ members = [
     "crates/consensus/beacon-core/",
     "crates/consensus/common/",
     "crates/ethereum-forks/",
+    "crates/etl",
     "crates/interfaces/",
     "crates/metrics/",
     "crates/metrics/metrics-derive/",
@@ -41,8 +42,8 @@ members = [
     "crates/node-optimism/",
     "crates/node-core/",
     "crates/node-api/",
-    "crates/snapshot/",
     "crates/stages/",
+    "crates/static-file/",
     "crates/storage/codecs/",
     "crates/storage/codecs/derive/",
     "crates/storage/db/",
@@ -136,6 +137,7 @@ reth-ecies = { path = "crates/net/ecies" }
 reth-eth-wire = { path = "crates/net/eth-wire" }
 reth-ethereum-forks = { path = "crates/ethereum-forks" }
 reth-ethereum-payload-builder = { path = "crates/payload/ethereum" }
+reth-etl = { path = "crates/etl" }
 reth-optimism-payload-builder = { path = "crates/payload/optimism" }
 reth-interfaces = { path = "crates/interfaces" }
 reth-ipc = { path = "crates/rpc/ipc" }
@@ -161,8 +163,8 @@ reth-rpc-builder = { path = "crates/rpc/rpc-builder" }
 reth-rpc-engine-api = { path = "crates/rpc/rpc-engine-api" }
 reth-rpc-types = { path = "crates/rpc/rpc-types" }
 reth-rpc-types-compat = { path = "crates/rpc/rpc-types-compat" }
-reth-snapshot = { path = "crates/snapshot" }
 reth-stages = { path = "crates/stages" }
+reth-static-file = { path = "crates/static-file" }
 reth-tasks = { path = "crates/tasks" }
 reth-tokio-util = { path = "crates/tokio-util" }
 reth-tracing = { path = "crates/tracing" }

bin/reth/Cargo.toml

@@ -50,7 +50,7 @@ reth-payload-validator.workspace = true
 reth-basic-payload-builder.workspace = true
 reth-discv4.workspace = true
 reth-prune.workspace = true
-reth-snapshot = { workspace = true, features = ["clap"] }
+reth-static-file = { workspace = true, features = ["clap"] }
 reth-trie.workspace = true
 reth-nippy-jar.workspace = true
 reth-node-api.workspace = true

bin/reth/src/builder.rs

@@ -6,7 +6,7 @@ use fdlimit::raise_fd_limit;
 use futures::{future::Either, stream, stream_select, StreamExt};
 use reth_auto_seal_consensus::AutoSealBuilder;
 use reth_beacon_consensus::{
-    hooks::{EngineHooks, PruneHook},
+    hooks::{EngineHooks, PruneHook, StaticFileHook},
     BeaconConsensusEngine, MIN_BLOCKS_FOR_PIPELINE_RUN,
 };
 use reth_blockchain_tree::{config::BlockchainTreeConfig, ShareableBlockchainTree};
@@ -40,6 +40,7 @@ use reth_primitives::format_ether;
 use reth_provider::{providers::BlockchainProvider, ProviderFactory};
 use reth_prune::PrunerBuilder;
 use reth_rpc_engine_api::EngineApi;
+use reth_static_file::StaticFileProducer;
 use reth_tasks::{TaskExecutor, TaskManager};
 use reth_transaction_pool::TransactionPool;
 use std::{path::PathBuf, sync::Arc};
@@ -127,26 +128,18 @@ impl<DB: Database + DatabaseMetrics + DatabaseMetadata + 'static> NodeBuilderWit
 
         let prometheus_handle = self.config.install_prometheus_recorder()?;
 
-        let mut provider_factory =
-            ProviderFactory::new(Arc::clone(&self.db), Arc::clone(&self.config.chain));
-
-        // configure snapshotter
-        let snapshotter = reth_snapshot::Snapshotter::new(
-            provider_factory.clone(),
-            self.data_dir.snapshots_path(),
-            self.config.chain.snapshot_block_interval,
-        )?;
-
-        provider_factory = provider_factory.with_snapshots(
-            self.data_dir.snapshots_path(),
-            snapshotter.highest_snapshot_receiver(),
-        )?;
+        let provider_factory = ProviderFactory::new(
+            Arc::clone(&self.db),
+            Arc::clone(&self.config.chain),
+            self.data_dir.static_files_path(),
+        )?
+        .with_static_files_metrics();
 
         self.config.start_metrics_endpoint(prometheus_handle, Arc::clone(&self.db)).await?;
 
         debug!(target: "reth::cli", chain=%self.config.chain.chain, genesis=?self.config.chain.genesis_hash(), "Initializing genesis");
 
-        let genesis_hash = init_genesis(Arc::clone(&self.db), self.config.chain.clone())?;
+        let genesis_hash = init_genesis(provider_factory.clone())?;
 
         info!(target: "reth::cli", "{}", self.config.chain.display_hardforks());
 
@@ -270,6 +263,17 @@ impl<DB: Database + DatabaseMetrics + DatabaseMetadata + 'static> NodeBuilderWit
         };
         let max_block = self.config.max_block(&network_client, provider_factory.clone()).await?;
 
+        let mut hooks = EngineHooks::new();
+
+        let mut static_file_producer = StaticFileProducer::new(
+            provider_factory.clone(),
+            provider_factory.static_file_provider(),
+            prune_config.clone().unwrap_or_default().segments,
+        );
+        let static_file_producer_events = static_file_producer.events();
+        hooks.add(StaticFileHook::new(static_file_producer.clone(), Box::new(executor.clone())));
+        info!(target: "reth::cli", "StaticFileProducer initialized");
+
         // Configure the pipeline
         let (mut pipeline, client) = if self.config.dev.dev {
             info!(target: "reth::cli", "Starting Reth in dev mode");
@@ -301,6 +305,7 @@ impl<DB: Database + DatabaseMetrics + DatabaseMetadata + 'static> NodeBuilderWit
                     sync_metrics_tx,
                     prune_config.clone(),
                     max_block,
+                    static_file_producer,
                     evm_config,
                 )
                 .await?;
@@ -323,6 +328,7 @@ impl<DB: Database + DatabaseMetrics + DatabaseMetadata + 'static> NodeBuilderWit
                     sync_metrics_tx,
                     prune_config.clone(),
                     max_block,
+                    static_file_producer,
                     evm_config,
                 )
                 .await?;
@@ -333,22 +339,16 @@ impl<DB: Database + DatabaseMetrics + DatabaseMetadata + 'static> NodeBuilderWit
         let pipeline_events = pipeline.events();
 
         let initial_target = self.config.initial_pipeline_target(genesis_hash);
-        let mut hooks = EngineHooks::new();
 
-        let pruner_events = if let Some(prune_config) = prune_config {
-            let mut pruner = PrunerBuilder::new(prune_config.clone())
-                .max_reorg_depth(tree_config.max_reorg_depth() as usize)
-                .prune_delete_limit(self.config.chain.prune_delete_limit)
-                .build(provider_factory, snapshotter.highest_snapshot_receiver());
+        let prune_config = prune_config.unwrap_or_default();
+        let mut pruner = PrunerBuilder::new(prune_config.clone())
+            .max_reorg_depth(tree_config.max_reorg_depth() as usize)
+            .prune_delete_limit(self.config.chain.prune_delete_limit)
+            .build(provider_factory.clone());
 
-            let events = pruner.events();
-            hooks.add(PruneHook::new(pruner, Box::new(executor.clone())));
-
-            info!(target: "reth::cli", ?prune_config, "Pruner initialized");
-            Either::Left(events)
-        } else {
-            Either::Right(stream::empty())
-        };
+        let pruner_events = pruner.events();
+        hooks.add(PruneHook::new(pruner, Box::new(executor.clone())));
+        info!(target: "reth::cli", ?prune_config, "Pruner initialized");
 
         // Configure the consensus engine
         let (beacon_consensus_engine, beacon_engine_handle) = BeaconConsensusEngine::with_channel(
@@ -380,7 +380,8 @@ impl<DB: Database + DatabaseMetrics + DatabaseMetadata + 'static> NodeBuilderWit
             } else {
                 Either::Right(stream::empty())
             },
-            pruner_events.map(Into::into)
+            pruner_events.map(Into::into),
+            static_file_producer_events.map(Into::into),
         );
         executor.spawn_critical(
             "events task",