From 2e892bce1faa2cf85b6d8bec4aaa326e2c695e86 Mon Sep 17 00:00:00 2001 From: Dan Cline <6798349+Rjected@users.noreply.github.com> Date: Thu, 30 Nov 2023 23:41:04 -0500 Subject: [PATCH 01/33] feat: add NodeConfig to build and launch the node --- bin/reth/src/cli/mod.rs | 1 + bin/reth/src/cli/node_config.rs | 185 ++++++++++++++++++++++++++++++++ 2 files changed, 186 insertions(+) create mode 100644 bin/reth/src/cli/node_config.rs diff --git a/bin/reth/src/cli/mod.rs b/bin/reth/src/cli/mod.rs index 14f25a279293..d340c51958c2 100644 --- a/bin/reth/src/cli/mod.rs +++ b/bin/reth/src/cli/mod.rs @@ -22,6 +22,7 @@ use std::{fmt, fmt::Display, sync::Arc}; pub mod components; pub mod config; pub mod ext; +pub mod node_config; /// Default [directives](Directive) for [EnvFilter] which disables high-frequency debug logs from /// `hyper` and `trust-dns` diff --git a/bin/reth/src/cli/node_config.rs b/bin/reth/src/cli/node_config.rs new file mode 100644 index 000000000000..1c17a6a8c889 --- /dev/null +++ b/bin/reth/src/cli/node_config.rs @@ -0,0 +1,185 @@ +//! Support for customizing the node +use crate::{ + args::{ + DatabaseArgs, DebugArgs, DevArgs, NetworkArgs, PayloadBuilderArgs, PruningArgs, + RpcServerArgs, TxPoolArgs, + }, + cli::RethCliExt, +}; +use std::{net::SocketAddr, path::PathBuf, sync::Arc}; + +use reth_primitives::ChainSpec; + +/// Start the node +#[derive(Debug)] +pub struct NodeConfig { + /// The path to the data dir for all reth files and subdirectories. + /// + /// Defaults to the OS-specific data directory: + /// + /// - Linux: `$XDG_DATA_HOME/reth/` or `$HOME/.local/share/reth/` + /// - Windows: `{FOLDERID_RoamingAppData}/reth/` + /// - macOS: `$HOME/Library/Application Support/reth/` + pub datadir: PathBuf, + + /// The path to the configuration file to use. + pub config: Option, + + /// The chain this node is running. + /// + /// Possible values are either a built-in chain or the path to a chain specification file. + pub chain: Arc, + + /// Enable Prometheus metrics. + /// + /// The metrics will be served at the given interface and port. + pub metrics: Option, + + /// Add a new instance of a node. + /// + /// Configures the ports of the node to avoid conflicts with the defaults. + /// This is useful for running multiple nodes on the same machine. + /// + /// Max number of instances is 200. It is chosen in a way so that it's not possible to have + /// port numbers that conflict with each other. + /// + /// Changes to the following port numbers: + /// - DISCOVERY_PORT: default + `instance` - 1 + /// - AUTH_PORT: default + `instance` * 100 - 100 + /// - HTTP_RPC_PORT: default - `instance` + 1 + /// - WS_RPC_PORT: default + `instance` * 2 - 2 + pub instance: u16, + + /// Overrides the KZG trusted setup by reading from the supplied file. + pub trusted_setup_file: Option, + + /// All networking related arguments + pub network: NetworkArgs, + + /// All rpc related arguments + pub rpc: RpcServerArgs, + + /// All txpool related arguments with --txpool prefix + pub txpool: TxPoolArgs, + + /// All payload builder related arguments + pub builder: PayloadBuilderArgs, + + /// All debug related arguments with --debug prefix + pub debug: DebugArgs, + + /// All database related arguments + pub db: DatabaseArgs, + + /// All dev related arguments with --dev prefix + pub dev: DevArgs, + + /// All pruning related arguments + pub pruning: PruningArgs, + + /// Rollup related arguments + #[cfg(feature = "optimism")] + pub rollup: crate::args::RollupArgs, +} + +impl NodeConfig { + /// Launches the node, also adding any RPC extensions passed. + pub fn launch(self, ext: E::Node) -> NodeHandle { + todo!() + } + + /// Set the datadir for the node + pub fn datadir(mut self, datadir: impl Into) -> Self { + self.datadir = datadir.into(); + self + } + + /// Set the config file for the node + pub fn config(mut self, config: impl Into) -> Self { + self.config = Some(config.into()); + self + } + + /// Set the chain for the node + pub fn chain(mut self, chain: Arc) -> Self { + self.chain = chain.into(); + self + } + + /// Set the metrics address for the node + pub fn metrics(mut self, metrics: SocketAddr) -> Self { + self.metrics = Some(metrics); + self + } + + /// Set the instance for the node + pub fn instance(mut self, instance: u16) -> Self { + self.instance = instance; + self + } + + /// Set the trusted setup file for the node + pub fn trusted_setup_file(mut self, trusted_setup_file: impl Into) -> Self { + self.trusted_setup_file = Some(trusted_setup_file.into()); + self + } + + /// Set the network args for the node + pub fn network(mut self, network: NetworkArgs) -> Self { + self.network = network; + self + } + + /// Set the rpc args for the node + pub fn rpc(mut self, rpc: RpcServerArgs) -> Self { + self.rpc = rpc; + self + } + + /// Set the txpool args for the node + pub fn txpool(mut self, txpool: TxPoolArgs) -> Self { + self.txpool = txpool; + self + } + + /// Set the builder args for the node + pub fn builder(mut self, builder: PayloadBuilderArgs) -> Self { + self.builder = builder; + self + } + + /// Set the debug args for the node + pub fn debug(mut self, debug: DebugArgs) -> Self { + self.debug = debug; + self + } + + /// Set the database args for the node + pub fn db(mut self, db: DatabaseArgs) -> Self { + self.db = db; + self + } + + /// Set the dev args for the node + pub fn dev(mut self, dev: DevArgs) -> Self { + self.dev = dev; + self + } + + /// Set the pruning args for the node + pub fn pruning(mut self, pruning: PruningArgs) -> Self { + self.pruning = pruning; + self + } + + /// Set the rollup args for the node + #[cfg(feature = "optimism")] + pub fn rollup(mut self, rollup: crate::args::RollupArgs) -> Self { + self.rollup = rollup; + self + } +} + +/// The node handle +#[derive(Debug)] +pub struct NodeHandle; From 1c533c8d04a4e833c207d8f22210c8acee000e66 Mon Sep 17 00:00:00 2001 From: Dan Cline <6798349+Rjected@users.noreply.github.com> Date: Thu, 30 Nov 2023 23:45:25 -0500 Subject: [PATCH 02/33] add simple test --- bin/reth/src/cli/node_config.rs | 42 +++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/bin/reth/src/cli/node_config.rs b/bin/reth/src/cli/node_config.rs index 1c17a6a8c889..4c12c09ffecb 100644 --- a/bin/reth/src/cli/node_config.rs +++ b/bin/reth/src/cli/node_config.rs @@ -183,3 +183,45 @@ impl NodeConfig { /// The node handle #[derive(Debug)] pub struct NodeHandle; + +#[cfg(test)] +mod tests { + #[test] + fn test_node_config() { + use super::*; + use crate::args::NetworkArgs; + use reth_primitives::ChainSpec; + use std::net::Ipv4Addr; + + let config = NodeConfig::default() + .datadir("/tmp") + .config("/tmp/config.toml") + .chain(Arc::new(ChainSpec::default())) + .metrics(SocketAddr::new(Ipv4Addr::LOCALHOST.into(), 8080)) + .instance(1) + .trusted_setup_file("/tmp/trusted_setup") + .network(NetworkArgs::default()) + .rpc(RpcServerArgs::default()) + .txpool(TxPoolArgs::default()) + .builder(PayloadBuilderArgs::default()) + .debug(DebugArgs::default()) + .db(DatabaseArgs::default()) + .dev(DevArgs::default()) + .pruning(PruningArgs::default()); + + assert_eq!(config.datadir, PathBuf::from("/tmp")); + assert_eq!(config.config, Some(PathBuf::from("/tmp/config.toml"))); + assert_eq!(config.chain, Arc::new(ChainSpec::default())); + assert_eq!(config.metrics, Some(SocketAddr::new(Ipv4Addr::LOCALHOST.into(), 8080))); + assert_eq!(config.instance, 1); + assert_eq!(config.trusted_setup_file, Some(PathBuf::from("/tmp/trusted_setup"))); + assert_eq!(config.network, NetworkArgs::default()); + assert_eq!(config.rpc, RpcServerArgs::default()); + assert_eq!(config.txpool, TxPoolArgs::default()); + assert_eq!(config.builder, PayloadBuilderArgs::default()); + assert_eq!(config.debug, DebugArgs::default()); + assert_eq!(config.db, DatabaseArgs::default()); + assert_eq!(config.dev, DevArgs::default()); + assert_eq!(config.pruning, PruningArgs::default()); + } +} From 6e176e6d03271c1725184cc365d40a47a33ae9b6 Mon Sep 17 00:00:00 2001 From: Dan Cline <6798349+Rjected@users.noreply.github.com> Date: Fri, 1 Dec 2023 13:49:54 -0500 Subject: [PATCH 03/33] comment test --- bin/reth/src/cli/node_config.rs | 71 +++++++++++++++++---------------- 1 file changed, 36 insertions(+), 35 deletions(-) diff --git a/bin/reth/src/cli/node_config.rs b/bin/reth/src/cli/node_config.rs index 4c12c09ffecb..580fe44c0ab1 100644 --- a/bin/reth/src/cli/node_config.rs +++ b/bin/reth/src/cli/node_config.rs @@ -186,42 +186,43 @@ pub struct NodeHandle; #[cfg(test)] mod tests { + // use super::*; + // use crate::args::NetworkArgs; + // use reth_primitives::ChainSpec; + // use std::net::Ipv4Addr; + #[test] fn test_node_config() { - use super::*; - use crate::args::NetworkArgs; - use reth_primitives::ChainSpec; - use std::net::Ipv4Addr; - - let config = NodeConfig::default() - .datadir("/tmp") - .config("/tmp/config.toml") - .chain(Arc::new(ChainSpec::default())) - .metrics(SocketAddr::new(Ipv4Addr::LOCALHOST.into(), 8080)) - .instance(1) - .trusted_setup_file("/tmp/trusted_setup") - .network(NetworkArgs::default()) - .rpc(RpcServerArgs::default()) - .txpool(TxPoolArgs::default()) - .builder(PayloadBuilderArgs::default()) - .debug(DebugArgs::default()) - .db(DatabaseArgs::default()) - .dev(DevArgs::default()) - .pruning(PruningArgs::default()); - - assert_eq!(config.datadir, PathBuf::from("/tmp")); - assert_eq!(config.config, Some(PathBuf::from("/tmp/config.toml"))); - assert_eq!(config.chain, Arc::new(ChainSpec::default())); - assert_eq!(config.metrics, Some(SocketAddr::new(Ipv4Addr::LOCALHOST.into(), 8080))); - assert_eq!(config.instance, 1); - assert_eq!(config.trusted_setup_file, Some(PathBuf::from("/tmp/trusted_setup"))); - assert_eq!(config.network, NetworkArgs::default()); - assert_eq!(config.rpc, RpcServerArgs::default()); - assert_eq!(config.txpool, TxPoolArgs::default()); - assert_eq!(config.builder, PayloadBuilderArgs::default()); - assert_eq!(config.debug, DebugArgs::default()); - assert_eq!(config.db, DatabaseArgs::default()); - assert_eq!(config.dev, DevArgs::default()); - assert_eq!(config.pruning, PruningArgs::default()); + + // let config = NodeConfig::default() + // .datadir("/tmp") + // .config("/tmp/config.toml") + // .chain(Arc::new(ChainSpec::default())) + // .metrics(SocketAddr::new(Ipv4Addr::LOCALHOST.into(), 8080)) + // .instance(1) + // .trusted_setup_file("/tmp/trusted_setup") + // .network(NetworkArgs::default()) + // .rpc(RpcServerArgs::default()) + // .txpool(TxPoolArgs::default()) + // .builder(PayloadBuilderArgs::default()) + // .debug(DebugArgs::default()) + // .db(DatabaseArgs::default()) + // .dev(DevArgs::default()) + // .pruning(PruningArgs::default()); + + // assert_eq!(config.datadir, PathBuf::from("/tmp")); + // assert_eq!(config.config, Some(PathBuf::from("/tmp/config.toml"))); + // assert_eq!(config.chain, Arc::new(ChainSpec::default())); + // assert_eq!(config.metrics, Some(SocketAddr::new(Ipv4Addr::LOCALHOST.into(), 8080))); + // assert_eq!(config.instance, 1); + // assert_eq!(config.trusted_setup_file, Some(PathBuf::from("/tmp/trusted_setup"))); + // assert_eq!(config.network, NetworkArgs::default()); + // assert_eq!(config.rpc, RpcServerArgs::default()); + // assert_eq!(config.txpool, TxPoolArgs::default()); + // assert_eq!(config.builder, PayloadBuilderArgs::default()); + // assert_eq!(config.debug, DebugArgs::default()); + // assert_eq!(config.db, DatabaseArgs::default()); + // assert_eq!(config.dev, DevArgs::default()); + // assert_eq!(config.pruning, PruningArgs::default()); } } From 8f7f4d1e9952db56a82982f971d7b42c8455d864 Mon Sep 17 00:00:00 2001 From: Dan Cline <6798349+Rjected@users.noreply.github.com> Date: Fri, 1 Dec 2023 17:08:32 -0500 Subject: [PATCH 04/33] wip: copy paste launch method for NodeConfig --- bin/reth/src/cli/node_config.rs | 371 +++++++++++++++++++++++++++++++- 1 file changed, 367 insertions(+), 4 deletions(-) diff --git a/bin/reth/src/cli/node_config.rs b/bin/reth/src/cli/node_config.rs index 580fe44c0ab1..ce65fa36a087 100644 --- a/bin/reth/src/cli/node_config.rs +++ b/bin/reth/src/cli/node_config.rs @@ -2,13 +2,22 @@ use crate::{ args::{ DatabaseArgs, DebugArgs, DevArgs, NetworkArgs, PayloadBuilderArgs, PruningArgs, - RpcServerArgs, TxPoolArgs, + RpcServerArgs, TxPoolArgs, get_secret_key, }, - cli::RethCliExt, + cli::{RethCliExt, components::RethNodeComponentsImpl}, + version::SHORT_VERSION, init::init_genesis, }; use std::{net::SocketAddr, path::PathBuf, sync::Arc}; +use tokio::sync::{mpsc::unbounded_channel, oneshot}; +use fdlimit::raise_fd_limit; +use reth_db::init_db; +use reth_network::NetworkManager; +use reth_provider::ProviderFactory; +use tracing::{info, debug}; +use reth_primitives::{ + ChainSpec, DisplayHardforks, +}; -use reth_primitives::ChainSpec; /// Start the node #[derive(Debug)] @@ -84,7 +93,361 @@ pub struct NodeConfig { impl NodeConfig { /// Launches the node, also adding any RPC extensions passed. - pub fn launch(self, ext: E::Node) -> NodeHandle { + pub async fn launch(self, ext: E::Node) -> NodeHandle { + // TODO: add easy way to handle auto seal, etc, without introducing generics + // TODO: do we need ctx: CliContext here? + info!(target: "reth::cli", "reth {} starting", SHORT_VERSION); + + // Raise the fd limit of the process. + // Does not do anything on windows. + raise_fd_limit(); + + // get config + let config = self.load_config()?; + + let prometheus_handle = self.install_prometheus_recorder()?; + + let data_dir = self.data_dir(); + let db_path = data_dir.db_path(); + + info!(target: "reth::cli", path = ?db_path, "Opening database"); + // TODO: set up test database, see if we can configure either real or test db + let db = Arc::new(init_db(&db_path, self.db.log_level)?.with_metrics()); + info!(target: "reth::cli", "Database opened"); + + let mut provider_factory = ProviderFactory::new(Arc::clone(&db), Arc::clone(&self.chain)); + + // configure snapshotter + let snapshotter = reth_snapshot::Snapshotter::new( + provider_factory.clone(), + data_dir.snapshots_path(), + self.chain.snapshot_block_interval, + )?; + + provider_factory = provider_factory + .with_snapshots(data_dir.snapshots_path(), snapshotter.highest_snapshot_receiver()); + + self.start_metrics_endpoint(prometheus_handle, Arc::clone(&db)).await?; + + debug!(target: "reth::cli", chain=%self.chain.chain, genesis=?self.chain.genesis_hash(), "Initializing genesis"); + + let genesis_hash = init_genesis(Arc::clone(&db), self.chain.clone())?; + + info!(target: "reth::cli", "{}", DisplayHardforks::new(self.chain.hardforks())); + + let consensus = self.consensus(); + + debug!(target: "reth::cli", "Spawning stages metrics listener task"); + let (sync_metrics_tx, sync_metrics_rx) = unbounded_channel(); + let sync_metrics_listener = reth_stages::MetricsListener::new(sync_metrics_rx); + ctx.task_executor.spawn_critical("stages metrics listener task", sync_metrics_listener); + + let prune_config = + self.pruning.prune_config(Arc::clone(&self.chain))?.or(config.prune.clone()); + + // configure blockchain tree + let tree_externals = TreeExternals::new( + provider_factory.clone(), + Arc::clone(&consensus), + EvmProcessorFactory::new(self.chain.clone()), + ); + let tree_config = BlockchainTreeConfig::default(); + let tree = BlockchainTree::new( + tree_externals, + tree_config, + prune_config.clone().map(|config| config.segments), + )? + .with_sync_metrics_tx(sync_metrics_tx.clone()); + let canon_state_notification_sender = tree.canon_state_notification_sender(); + let blockchain_tree = ShareableBlockchainTree::new(tree); + debug!(target: "reth::cli", "configured blockchain tree"); + + // fetch the head block from the database + let head = self.lookup_head(Arc::clone(&db)).wrap_err("the head block is missing")?; + + // setup the blockchain provider + let blockchain_db = + BlockchainProvider::new(provider_factory.clone(), blockchain_tree.clone())?; + let blob_store = InMemoryBlobStore::default(); + let validator = TransactionValidationTaskExecutor::eth_builder(Arc::clone(&self.chain)) + .with_head_timestamp(head.timestamp) + .kzg_settings(self.kzg_settings()?) + .with_additional_tasks(1) + .build_with_tasks(blockchain_db.clone(), ctx.task_executor.clone(), blob_store.clone()); + + let transaction_pool = + reth_transaction_pool::Pool::eth_pool(validator, blob_store, self.txpool.pool_config()); + info!(target: "reth::cli", "Transaction pool initialized"); + + // spawn txpool maintenance task + { + let pool = transaction_pool.clone(); + let chain_events = blockchain_db.canonical_state_stream(); + let client = blockchain_db.clone(); + ctx.task_executor.spawn_critical( + "txpool maintenance task", + reth_transaction_pool::maintain::maintain_transaction_pool_future( + client, + pool, + chain_events, + ctx.task_executor.clone(), + Default::default(), + ), + ); + debug!(target: "reth::cli", "Spawned txpool maintenance task"); + } + + info!(target: "reth::cli", "Connecting to P2P network"); + let network_secret_path = + self.network.p2p_secret_key.clone().unwrap_or_else(|| data_dir.p2p_secret_path()); + debug!(target: "reth::cli", ?network_secret_path, "Loading p2p key file"); + let secret_key = get_secret_key(&network_secret_path)?; + let default_peers_path = data_dir.known_peers_path(); + let network_config = self.load_network_config( + &config, + Arc::clone(&db), + ctx.task_executor.clone(), + head, + secret_key, + default_peers_path.clone(), + ); + + let network_client = network_config.client.clone(); + let mut network_builder = NetworkManager::builder(network_config).await?; + + let components = RethNodeComponentsImpl { + provider: blockchain_db.clone(), + pool: transaction_pool.clone(), + network: network_builder.handle(), + task_executor: ctx.task_executor.clone(), + events: blockchain_db.clone(), + }; + + // allow network modifications + self.ext.configure_network(network_builder.network_mut(), &components)?; + + // launch network + let network = self.start_network( + network_builder, + &ctx.task_executor, + transaction_pool.clone(), + network_client, + default_peers_path, + ); + + info!(target: "reth::cli", peer_id = %network.peer_id(), local_addr = %network.local_addr(), enode = %network.local_node_record(), "Connected to P2P network"); + debug!(target: "reth::cli", peer_id = ?network.peer_id(), "Full peer ID"); + let network_client = network.fetch_client().await?; + + self.ext.on_components_initialized(&components)?; + + debug!(target: "reth::cli", "Spawning payload builder service"); + let payload_builder = self.ext.spawn_payload_builder_service(&self.builder, &components)?; + + let (consensus_engine_tx, consensus_engine_rx) = unbounded_channel(); + let max_block = if let Some(block) = self.debug.max_block { + Some(block) + } else if let Some(tip) = self.debug.tip { + Some(self.lookup_or_fetch_tip(&db, &network_client, tip).await?) + } else { + None + }; + + // Configure the pipeline + let (mut pipeline, client) = if self.dev.dev { + info!(target: "reth::cli", "Starting Reth in dev mode"); + + let mining_mode = if let Some(interval) = self.dev.block_time { + MiningMode::interval(interval) + } else if let Some(max_transactions) = self.dev.block_max_transactions { + MiningMode::instant( + max_transactions, + transaction_pool.pending_transactions_listener(), + ) + } else { + info!(target: "reth::cli", "No mining mode specified, defaulting to ReadyTransaction"); + MiningMode::instant(1, transaction_pool.pending_transactions_listener()) + }; + + let (_, client, mut task) = AutoSealBuilder::new( + Arc::clone(&self.chain), + blockchain_db.clone(), + transaction_pool.clone(), + consensus_engine_tx.clone(), + canon_state_notification_sender, + mining_mode, + ) + .build(); + + let mut pipeline = self + .build_networked_pipeline( + &config, + client.clone(), + Arc::clone(&consensus), + provider_factory, + &ctx.task_executor, + sync_metrics_tx, + prune_config.clone(), + max_block, + ) + .await?; + + let pipeline_events = pipeline.events(); + task.set_pipeline_events(pipeline_events); + debug!(target: "reth::cli", "Spawning auto mine task"); + ctx.task_executor.spawn(Box::pin(task)); + + (pipeline, EitherDownloader::Left(client)) + } else { + let pipeline = self + .build_networked_pipeline( + &config, + network_client.clone(), + Arc::clone(&consensus), + provider_factory, + &ctx.task_executor, + sync_metrics_tx, + prune_config.clone(), + max_block, + ) + .await?; + + (pipeline, EitherDownloader::Right(network_client)) + }; + + let pipeline_events = pipeline.events(); + + let initial_target = if let Some(tip) = self.debug.tip { + // Set the provided tip as the initial pipeline target. + debug!(target: "reth::cli", %tip, "Tip manually set"); + Some(tip) + } else if self.debug.continuous { + // Set genesis as the initial pipeline target. + // This will allow the downloader to start + debug!(target: "reth::cli", "Continuous sync mode enabled"); + Some(genesis_hash) + } else { + None + }; + + let mut hooks = EngineHooks::new(); + + let pruner_events = if let Some(prune_config) = prune_config { + let mut pruner = self.build_pruner( + &prune_config, + db.clone(), + tree_config, + snapshotter.highest_snapshot_receiver(), + ); + + let events = pruner.events(); + hooks.add(PruneHook::new(pruner, Box::new(ctx.task_executor.clone()))); + + info!(target: "reth::cli", ?prune_config, "Pruner initialized"); + Either::Left(events) + } else { + Either::Right(stream::empty()) + }; + + // Configure the consensus engine + let (beacon_consensus_engine, beacon_engine_handle) = BeaconConsensusEngine::with_channel( + client, + pipeline, + blockchain_db.clone(), + Box::new(ctx.task_executor.clone()), + Box::new(network.clone()), + max_block, + self.debug.continuous, + payload_builder.clone(), + initial_target, + MIN_BLOCKS_FOR_PIPELINE_RUN, + consensus_engine_tx, + consensus_engine_rx, + hooks, + )?; + info!(target: "reth::cli", "Consensus engine initialized"); + + let events = stream_select!( + network.event_listener().map(Into::into), + beacon_engine_handle.event_listener().map(Into::into), + pipeline_events.map(Into::into), + if self.debug.tip.is_none() { + Either::Left( + ConsensusLayerHealthEvents::new(Box::new(blockchain_db.clone())) + .map(Into::into), + ) + } else { + Either::Right(stream::empty()) + }, + pruner_events.map(Into::into) + ); + ctx.task_executor.spawn_critical( + "events task", + events::handle_events(Some(network.clone()), Some(head.number), events, db.clone()), + ); + + let engine_api = EngineApi::new( + blockchain_db.clone(), + self.chain.clone(), + beacon_engine_handle, + payload_builder.into(), + Box::new(ctx.task_executor.clone()), + ); + info!(target: "reth::cli", "Engine API handler initialized"); + + // extract the jwt secret from the args if possible + let default_jwt_path = data_dir.jwt_path(); + let jwt_secret = self.rpc.auth_jwt_secret(default_jwt_path)?; + + // adjust rpc port numbers based on instance number + self.adjust_instance_ports(); + + // Start RPC servers + let _rpc_server_handles = + self.rpc.start_servers(&components, engine_api, jwt_secret, &mut self.ext).await?; + + // Run consensus engine to completion + let (tx, rx) = oneshot::channel(); + info!(target: "reth::cli", "Starting consensus engine"); + ctx.task_executor.spawn_critical_blocking("consensus engine", async move { + let res = beacon_consensus_engine.await; + let _ = tx.send(res); + }); + + self.ext.on_node_started(&components)?; + + // If `enable_genesis_walkback` is set to true, the rollup client will need to + // perform the derivation pipeline from genesis, validating the data dir. + // When set to false, set the finalized, safe, and unsafe head block hashes + // on the rollup client using a fork choice update. This prevents the rollup + // client from performing the derivation pipeline from genesis, and instead + // starts syncing from the current tip in the DB. + #[cfg(feature = "optimism")] + if self.chain.is_optimism() && !self.rollup.enable_genesis_walkback { + let client = _rpc_server_handles.auth.http_client(); + reth_rpc_api::EngineApiClient::fork_choice_updated_v2( + &client, + reth_rpc_types::engine::ForkchoiceState { + head_block_hash: head.hash, + safe_block_hash: head.hash, + finalized_block_hash: head.hash, + }, + None, + ) + .await?; + } + + rx.await??; + + info!(target: "reth::cli", "Consensus engine has exited."); + + if self.debug.terminate { + Ok(()) + } else { + // The pipeline has finished downloading blocks up to `--debug.tip` or + // `--debug.max-block`. Keep other node components alive for further usage. + futures::future::pending().await + } todo!() } From fd85c8f33f1e7229fed62d7ba8a759081150df2c Mon Sep 17 00:00:00 2001 From: Dan Cline <6798349+Rjected@users.noreply.github.com> Date: Fri, 1 Dec 2023 18:04:44 -0500 Subject: [PATCH 05/33] add the other methods --- bin/reth/src/cli/node_config.rs | 539 +++++++++++++++++++++++++++++++- 1 file changed, 523 insertions(+), 16 deletions(-) diff --git a/bin/reth/src/cli/node_config.rs b/bin/reth/src/cli/node_config.rs index ce65fa36a087..ab0d5a856d2f 100644 --- a/bin/reth/src/cli/node_config.rs +++ b/bin/reth/src/cli/node_config.rs @@ -1,22 +1,87 @@ //! Support for customizing the node use crate::{ args::{ + get_secret_key, DatabaseArgs, DebugArgs, DevArgs, NetworkArgs, PayloadBuilderArgs, PruningArgs, - RpcServerArgs, TxPoolArgs, get_secret_key, + RpcServerArgs, TxPoolArgs, }, - cli::{RethCliExt, components::RethNodeComponentsImpl}, - version::SHORT_VERSION, init::init_genesis, + cli::{ + components::RethNodeComponentsImpl, + config::RethRpcConfig, + ext::{RethCliExt, RethNodeCommandConfig}, + }, + dirs::{ChainPath, DataDirPath}, + init::init_genesis, + node::cl_events::ConsensusLayerHealthEvents, + prometheus_exporter, + utils::get_single_header, + version::SHORT_VERSION, + node::events, }; -use std::{net::SocketAddr, path::PathBuf, sync::Arc}; -use tokio::sync::{mpsc::unbounded_channel, oneshot}; +use eyre::Context; use fdlimit::raise_fd_limit; -use reth_db::init_db; -use reth_network::NetworkManager; -use reth_provider::ProviderFactory; -use tracing::{info, debug}; +use futures::{future::Either, stream, stream_select, StreamExt}; +use metrics_exporter_prometheus::PrometheusHandle; +use reth_auto_seal_consensus::{AutoSealBuilder, AutoSealConsensus, MiningMode}; +use reth_beacon_consensus::{ + hooks::{EngineHooks, PruneHook}, + BeaconConsensus, BeaconConsensusEngine, MIN_BLOCKS_FOR_PIPELINE_RUN, +}; +use reth_blockchain_tree::{ + config::BlockchainTreeConfig, externals::TreeExternals, BlockchainTree, ShareableBlockchainTree, +}; +use reth_config::{config::PruneConfig, Config}; +use reth_db::{database::Database, init_db, DatabaseEnv}; +use reth_downloaders::{ + bodies::bodies::BodiesDownloaderBuilder, + headers::reverse_headers::ReverseHeadersDownloaderBuilder, +}; +use reth_interfaces::{ + consensus::Consensus, + p2p::{ + bodies::{client::BodiesClient, downloader::BodyDownloader}, + either::EitherDownloader, + headers::{client::HeadersClient, downloader::HeaderDownloader}, + }, + RethResult, +}; +use reth_network::{NetworkBuilder, NetworkConfig, NetworkEvents, NetworkHandle, NetworkManager}; +use reth_network_api::{NetworkInfo, PeersInfo}; use reth_primitives::{ - ChainSpec, DisplayHardforks, + constants::eip4844::{LoadKzgSettingsError, MAINNET_KZG_TRUSTED_SETUP}, + kzg::KzgSettings, + stage::StageId, + BlockHashOrNumber, BlockNumber, ChainSpec, DisplayHardforks, Head, SealedHeader, B256, }; +use reth_provider::{ + providers::BlockchainProvider, BlockHashReader, BlockReader, CanonStateSubscriptions, + HeaderProvider, HeaderSyncMode, ProviderFactory, StageCheckpointReader, +}; +use reth_prune::{segments::SegmentSet, Pruner}; +use reth_revm::EvmProcessorFactory; +use reth_revm_inspectors::stack::Hook; +use reth_rpc_engine_api::EngineApi; +use reth_snapshot::HighestSnapshotsTracker; +use reth_stages::{ + prelude::*, + stages::{ + AccountHashingStage, ExecutionStage, ExecutionStageThresholds, IndexAccountHistoryStage, + IndexStorageHistoryStage, MerkleStage, SenderRecoveryStage, StorageHashingStage, + TotalDifficultyStage, TransactionLookupStage, + }, +}; +use reth_tasks::TaskExecutor; +use reth_transaction_pool::{ + blobstore::InMemoryBlobStore, TransactionPool, TransactionValidationTaskExecutor, +}; +use secp256k1::SecretKey; +use std::{ + net::{SocketAddr, SocketAddrV4}, + path::PathBuf, + sync::Arc, +}; +use tokio::sync::{mpsc::unbounded_channel, oneshot, watch}; +use tracing::*; /// Start the node @@ -93,7 +158,7 @@ pub struct NodeConfig { impl NodeConfig { /// Launches the node, also adding any RPC extensions passed. - pub async fn launch(self, ext: E::Node) -> NodeHandle { + pub async fn launch(self, ext: E::Node) -> eyre::Result { // TODO: add easy way to handle auto seal, etc, without introducing generics // TODO: do we need ctx: CliContext here? info!(target: "reth::cli", "reth {} starting", SHORT_VERSION); @@ -224,7 +289,7 @@ impl NodeConfig { }; // allow network modifications - self.ext.configure_network(network_builder.network_mut(), &components)?; + ext.configure_network(network_builder.network_mut(), &components)?; // launch network let network = self.start_network( @@ -239,10 +304,10 @@ impl NodeConfig { debug!(target: "reth::cli", peer_id = ?network.peer_id(), "Full peer ID"); let network_client = network.fetch_client().await?; - self.ext.on_components_initialized(&components)?; + ext.on_components_initialized(&components)?; debug!(target: "reth::cli", "Spawning payload builder service"); - let payload_builder = self.ext.spawn_payload_builder_service(&self.builder, &components)?; + let payload_builder = ext.spawn_payload_builder_service(&self.builder, &components)?; let (consensus_engine_tx, consensus_engine_rx) = unbounded_channel(); let max_block = if let Some(block) = self.debug.max_block { @@ -404,7 +469,7 @@ impl NodeConfig { // Start RPC servers let _rpc_server_handles = - self.rpc.start_servers(&components, engine_api, jwt_secret, &mut self.ext).await?; + self.rpc.start_servers(&components, engine_api, jwt_secret, &mut ext).await?; // Run consensus engine to completion let (tx, rx) = oneshot::channel(); @@ -414,7 +479,7 @@ impl NodeConfig { let _ = tx.send(res); }); - self.ext.on_node_started(&components)?; + ext.on_node_started(&components)?; // If `enable_genesis_walkback` is set to true, the rollup client will need to // perform the derivation pipeline from genesis, validating the data dir. @@ -541,6 +606,448 @@ impl NodeConfig { self.rollup = rollup; self } + + /// Returns the [Consensus] instance to use. + /// + /// By default this will be a [BeaconConsensus] instance, but if the `--dev` flag is set, it + /// will be an [AutoSealConsensus] instance. + pub fn consensus(&self) -> Arc { + if self.dev.dev { + Arc::new(AutoSealConsensus::new(Arc::clone(&self.chain))) + } else { + Arc::new(BeaconConsensus::new(Arc::clone(&self.chain))) + } + } + + /// Constructs a [Pipeline] that's wired to the network + #[allow(clippy::too_many_arguments)] + async fn build_networked_pipeline( + &self, + config: &Config, + client: Client, + consensus: Arc, + provider_factory: ProviderFactory, + task_executor: &TaskExecutor, + metrics_tx: reth_stages::MetricEventsSender, + prune_config: Option, + max_block: Option, + ) -> eyre::Result> + where + DB: Database + Unpin + Clone + 'static, + Client: HeadersClient + BodiesClient + Clone + 'static, + { + // building network downloaders using the fetch client + let header_downloader = ReverseHeadersDownloaderBuilder::from(config.stages.headers) + .build(client.clone(), Arc::clone(&consensus)) + .into_task_with(task_executor); + + let body_downloader = BodiesDownloaderBuilder::from(config.stages.bodies) + .build(client, Arc::clone(&consensus), provider_factory.clone()) + .into_task_with(task_executor); + + let pipeline = self + .build_pipeline( + provider_factory, + config, + header_downloader, + body_downloader, + consensus, + max_block, + self.debug.continuous, + metrics_tx, + prune_config, + ) + .await?; + + Ok(pipeline) + } + + /// Returns the chain specific path to the data dir. + fn data_dir(&self) -> ChainPath { + self.datadir.unwrap_or_chain_default(self.chain.chain) + } + + /// Returns the path to the config file. + fn config_path(&self) -> PathBuf { + self.config.clone().unwrap_or_else(|| self.data_dir().config_path()) + } + + /// Loads the reth config with the given datadir root + fn load_config(&self) -> eyre::Result { + let config_path = self.config_path(); + let mut config = confy::load_path::(&config_path) + .wrap_err_with(|| format!("Could not load config file {:?}", config_path))?; + + info!(target: "reth::cli", path = ?config_path, "Configuration loaded"); + + // Update the config with the command line arguments + config.peers.connect_trusted_nodes_only = self.network.trusted_only; + + if !self.network.trusted_peers.is_empty() { + info!(target: "reth::cli", "Adding trusted nodes"); + self.network.trusted_peers.iter().for_each(|peer| { + config.peers.trusted_nodes.insert(*peer); + }); + } + + Ok(config) + } + + /// Loads the trusted setup params from a given file path or falls back to + /// `MAINNET_KZG_TRUSTED_SETUP`. + fn kzg_settings(&self) -> eyre::Result> { + if let Some(ref trusted_setup_file) = self.trusted_setup_file { + let trusted_setup = KzgSettings::load_trusted_setup_file(trusted_setup_file) + .map_err(LoadKzgSettingsError::KzgError)?; + Ok(Arc::new(trusted_setup)) + } else { + Ok(Arc::clone(&MAINNET_KZG_TRUSTED_SETUP)) + } + } + + fn install_prometheus_recorder(&self) -> eyre::Result { + prometheus_exporter::install_recorder() + } + + async fn start_metrics_endpoint( + &self, + prometheus_handle: PrometheusHandle, + db: Arc, + ) -> eyre::Result<()> { + if let Some(listen_addr) = self.metrics { + info!(target: "reth::cli", addr = %listen_addr, "Starting metrics endpoint"); + prometheus_exporter::serve( + listen_addr, + prometheus_handle, + db, + metrics_process::Collector::default(), + ) + .await?; + } + + Ok(()) + } + + /// Spawns the configured network and associated tasks and returns the [NetworkHandle] connected + /// to that network. + fn start_network( + &self, + builder: NetworkBuilder, + task_executor: &TaskExecutor, + pool: Pool, + client: C, + default_peers_path: PathBuf, + ) -> NetworkHandle + where + C: BlockReader + HeaderProvider + Clone + Unpin + 'static, + Pool: TransactionPool + Unpin + 'static, + { + let (handle, network, txpool, eth) = + builder.transactions(pool).request_handler(client).split_with_handle(); + + task_executor.spawn_critical("p2p txpool", txpool); + task_executor.spawn_critical("p2p eth request handler", eth); + + let known_peers_file = self.network.persistent_peers_file(default_peers_path); + task_executor + .spawn_critical_with_graceful_shutdown_signal("p2p network task", |shutdown| { + run_network_until_shutdown(shutdown, network, known_peers_file) + }); + + handle + } + + /// Fetches the head block from the database. + /// + /// If the database is empty, returns the genesis block. + fn lookup_head(&self, db: Arc) -> RethResult { + let factory = ProviderFactory::new(db, self.chain.clone()); + let provider = factory.provider()?; + + let head = provider.get_stage_checkpoint(StageId::Finish)?.unwrap_or_default().block_number; + + let header = provider + .header_by_number(head)? + .expect("the header for the latest block is missing, database is corrupt"); + + let total_difficulty = provider + .header_td_by_number(head)? + .expect("the total difficulty for the latest block is missing, database is corrupt"); + + let hash = provider + .block_hash(head)? + .expect("the hash for the latest block is missing, database is corrupt"); + + Ok(Head { + number: head, + hash, + difficulty: header.difficulty, + total_difficulty, + timestamp: header.timestamp, + }) + } + + /// Attempt to look up the block number for the tip hash in the database. + /// If it doesn't exist, download the header and return the block number. + /// + /// NOTE: The download is attempted with infinite retries. + async fn lookup_or_fetch_tip( + &self, + db: DB, + client: Client, + tip: B256, + ) -> RethResult + where + DB: Database, + Client: HeadersClient, + { + Ok(self.fetch_tip(db, client, BlockHashOrNumber::Hash(tip)).await?.number) + } + + /// Attempt to look up the block with the given number and return the header. + /// + /// NOTE: The download is attempted with infinite retries. + async fn fetch_tip( + &self, + db: DB, + client: Client, + tip: BlockHashOrNumber, + ) -> RethResult + where + DB: Database, + Client: HeadersClient, + { + let factory = ProviderFactory::new(db, self.chain.clone()); + let provider = factory.provider()?; + + let header = provider.header_by_hash_or_number(tip)?; + + // try to look up the header in the database + if let Some(header) = header { + info!(target: "reth::cli", ?tip, "Successfully looked up tip block in the database"); + return Ok(header.seal_slow()) + } + + info!(target: "reth::cli", ?tip, "Fetching tip block from the network."); + loop { + match get_single_header(&client, tip).await { + Ok(tip_header) => { + info!(target: "reth::cli", ?tip, "Successfully fetched tip"); + return Ok(tip_header) + } + Err(error) => { + error!(target: "reth::cli", %error, "Failed to fetch the tip. Retrying..."); + } + } + } + } + + fn load_network_config( + &self, + config: &Config, + db: Arc, + executor: TaskExecutor, + head: Head, + secret_key: SecretKey, + default_peers_path: PathBuf, + ) -> NetworkConfig>> { + let cfg_builder = self + .network + .network_config(config, self.chain.clone(), secret_key, default_peers_path) + .with_task_executor(Box::new(executor)) + .set_head(head) + .listener_addr(SocketAddr::V4(SocketAddrV4::new( + self.network.addr, + // set discovery port based on instance number + self.network.port + self.instance - 1, + ))) + .discovery_addr(SocketAddr::V4(SocketAddrV4::new( + self.network.addr, + // set discovery port based on instance number + self.network.port + self.instance - 1, + ))); + + // When `sequencer_endpoint` is configured, the node will forward all transactions to a + // Sequencer node for execution and inclusion on L1, and disable its own txpool + // gossip to prevent other parties in the network from learning about them. + #[cfg(feature = "optimism")] + let cfg_builder = cfg_builder + .sequencer_endpoint(self.rollup.sequencer_http.clone()) + .disable_tx_gossip(self.rollup.disable_txpool_gossip); + + cfg_builder.build(ProviderFactory::new(db, self.chain.clone())) + } + + #[allow(clippy::too_many_arguments)] + async fn build_pipeline( + &self, + provider_factory: ProviderFactory, + config: &Config, + header_downloader: H, + body_downloader: B, + consensus: Arc, + max_block: Option, + continuous: bool, + metrics_tx: reth_stages::MetricEventsSender, + prune_config: Option, + ) -> eyre::Result> + where + DB: Database + Clone + 'static, + H: HeaderDownloader + 'static, + B: BodyDownloader + 'static, + { + let stage_config = &config.stages; + + let mut builder = Pipeline::builder(); + + if let Some(max_block) = max_block { + debug!(target: "reth::cli", max_block, "Configuring builder to use max block"); + builder = builder.with_max_block(max_block) + } + + let (tip_tx, tip_rx) = watch::channel(B256::ZERO); + use reth_revm_inspectors::stack::InspectorStackConfig; + let factory = reth_revm::EvmProcessorFactory::new(self.chain.clone()); + + let stack_config = InspectorStackConfig { + use_printer_tracer: self.debug.print_inspector, + hook: if let Some(hook_block) = self.debug.hook_block { + Hook::Block(hook_block) + } else if let Some(tx) = self.debug.hook_transaction { + Hook::Transaction(tx) + } else if self.debug.hook_all { + Hook::All + } else { + Hook::None + }, + }; + + let factory = factory.with_stack_config(stack_config); + + let prune_modes = prune_config.map(|prune| prune.segments).unwrap_or_default(); + + let header_mode = + if continuous { HeaderSyncMode::Continuous } else { HeaderSyncMode::Tip(tip_rx) }; + let pipeline = builder + .with_tip_sender(tip_tx) + .with_metrics_tx(metrics_tx.clone()) + .add_stages( + DefaultStages::new( + provider_factory.clone(), + header_mode, + Arc::clone(&consensus), + header_downloader, + body_downloader, + factory.clone(), + ) + .set( + TotalDifficultyStage::new(consensus) + .with_commit_threshold(stage_config.total_difficulty.commit_threshold), + ) + .set(SenderRecoveryStage { + commit_threshold: stage_config.sender_recovery.commit_threshold, + }) + .set( + ExecutionStage::new( + factory, + ExecutionStageThresholds { + max_blocks: stage_config.execution.max_blocks, + max_changes: stage_config.execution.max_changes, + max_cumulative_gas: stage_config.execution.max_cumulative_gas, + }, + stage_config + .merkle + .clean_threshold + .max(stage_config.account_hashing.clean_threshold) + .max(stage_config.storage_hashing.clean_threshold), + prune_modes.clone(), + ) + .with_metrics_tx(metrics_tx), + ) + .set(AccountHashingStage::new( + stage_config.account_hashing.clean_threshold, + stage_config.account_hashing.commit_threshold, + )) + .set(StorageHashingStage::new( + stage_config.storage_hashing.clean_threshold, + stage_config.storage_hashing.commit_threshold, + )) + .set(MerkleStage::new_execution(stage_config.merkle.clean_threshold)) + .set(TransactionLookupStage::new( + stage_config.transaction_lookup.commit_threshold, + prune_modes.transaction_lookup, + )) + .set(IndexAccountHistoryStage::new( + stage_config.index_account_history.commit_threshold, + prune_modes.account_history, + )) + .set(IndexStorageHistoryStage::new( + stage_config.index_storage_history.commit_threshold, + prune_modes.storage_history, + )), + ) + .build(provider_factory); + + Ok(pipeline) + } + + /// Builds a [Pruner] with the given config. + fn build_pruner( + &self, + config: &PruneConfig, + db: DB, + tree_config: BlockchainTreeConfig, + highest_snapshots_rx: HighestSnapshotsTracker, + ) -> Pruner { + let segments = SegmentSet::default() + // Receipts + .segment_opt(config.segments.receipts.map(reth_prune::segments::Receipts::new)) + // Receipts by logs + .segment_opt((!config.segments.receipts_log_filter.is_empty()).then(|| { + reth_prune::segments::ReceiptsByLogs::new( + config.segments.receipts_log_filter.clone(), + ) + })) + // Transaction lookup + .segment_opt( + config + .segments + .transaction_lookup + .map(reth_prune::segments::TransactionLookup::new), + ) + // Sender recovery + .segment_opt( + config.segments.sender_recovery.map(reth_prune::segments::SenderRecovery::new), + ) + // Account history + .segment_opt( + config.segments.account_history.map(reth_prune::segments::AccountHistory::new), + ) + // Storage history + .segment_opt( + config.segments.storage_history.map(reth_prune::segments::StorageHistory::new), + ); + + Pruner::new( + db, + self.chain.clone(), + segments.into_vec(), + config.block_interval, + self.chain.prune_delete_limit, + tree_config.max_reorg_depth() as usize, + highest_snapshots_rx, + ) + } + + /// Change rpc port numbers based on the instance number. + fn adjust_instance_ports(&mut self) { + // auth port is scaled by a factor of instance * 100 + self.rpc.auth_port += self.instance * 100 - 100; + // http port is scaled by a factor of -instance + self.rpc.http_port -= self.instance - 1; + // ws port is scaled by a factor of instance * 2 + self.rpc.ws_port += self.instance * 2 - 2; + } } /// The node handle From 10ddd2ee8ca31e077969a897b8527971a1b9cece Mon Sep 17 00:00:00 2001 From: Dan Cline <6798349+Rjected@users.noreply.github.com> Date: Fri, 1 Dec 2023 20:22:56 -0500 Subject: [PATCH 06/33] make it compile now it really is a copy --- bin/reth/src/cli/node_config.rs | 33 ++++++++++++++++----------------- bin/reth/src/node/mod.rs | 2 +- 2 files changed, 17 insertions(+), 18 deletions(-) diff --git a/bin/reth/src/cli/node_config.rs b/bin/reth/src/cli/node_config.rs index ab0d5a856d2f..218a363cf9f7 100644 --- a/bin/reth/src/cli/node_config.rs +++ b/bin/reth/src/cli/node_config.rs @@ -1,9 +1,8 @@ //! Support for customizing the node use crate::{ args::{ - get_secret_key, - DatabaseArgs, DebugArgs, DevArgs, NetworkArgs, PayloadBuilderArgs, PruningArgs, - RpcServerArgs, TxPoolArgs, + get_secret_key, DatabaseArgs, DebugArgs, DevArgs, NetworkArgs, PayloadBuilderArgs, + PruningArgs, RpcServerArgs, TxPoolArgs, }, cli::{ components::RethNodeComponentsImpl, @@ -12,11 +11,11 @@ use crate::{ }, dirs::{ChainPath, DataDirPath}, init::init_genesis, - node::cl_events::ConsensusLayerHealthEvents, + node::{cl_events::ConsensusLayerHealthEvents, events, run_network_until_shutdown}, prometheus_exporter, + runner::CliContext, utils::get_single_header, version::SHORT_VERSION, - node::events, }; use eyre::Context; use fdlimit::raise_fd_limit; @@ -83,7 +82,6 @@ use std::{ use tokio::sync::{mpsc::unbounded_channel, oneshot, watch}; use tracing::*; - /// Start the node #[derive(Debug)] pub struct NodeConfig { @@ -94,7 +92,7 @@ pub struct NodeConfig { /// - Linux: `$XDG_DATA_HOME/reth/` or `$HOME/.local/share/reth/` /// - Windows: `{FOLDERID_RoamingAppData}/reth/` /// - macOS: `$HOME/Library/Application Support/reth/` - pub datadir: PathBuf, + pub datadir: ChainPath, /// The path to the configuration file to use. pub config: Option, @@ -158,9 +156,11 @@ pub struct NodeConfig { impl NodeConfig { /// Launches the node, also adding any RPC extensions passed. - pub async fn launch(self, ext: E::Node) -> eyre::Result { - // TODO: add easy way to handle auto seal, etc, without introducing generics - // TODO: do we need ctx: CliContext here? + pub async fn launch( + mut self, + mut ext: E::Node, + ctx: CliContext, + ) -> eyre::Result { info!(target: "reth::cli", "reth {} starting", SHORT_VERSION); // Raise the fd limit of the process. @@ -507,17 +507,16 @@ impl NodeConfig { info!(target: "reth::cli", "Consensus engine has exited."); if self.debug.terminate { - Ok(()) + todo!() } else { // The pipeline has finished downloading blocks up to `--debug.tip` or // `--debug.max-block`. Keep other node components alive for further usage. futures::future::pending().await } - todo!() } /// Set the datadir for the node - pub fn datadir(mut self, datadir: impl Into) -> Self { + pub fn datadir(mut self, datadir: ChainPath) -> Self { self.datadir = datadir.into(); self } @@ -663,8 +662,8 @@ impl NodeConfig { } /// Returns the chain specific path to the data dir. - fn data_dir(&self) -> ChainPath { - self.datadir.unwrap_or_chain_default(self.chain.chain) + fn data_dir(&self) -> &ChainPath { + &self.datadir } /// Returns the path to the config file. @@ -825,7 +824,7 @@ impl NodeConfig { // try to look up the header in the database if let Some(header) = header { info!(target: "reth::cli", ?tip, "Successfully looked up tip block in the database"); - return Ok(header.seal_slow()) + return Ok(header.seal_slow()); } info!(target: "reth::cli", ?tip, "Fetching tip block from the network."); @@ -833,7 +832,7 @@ impl NodeConfig { match get_single_header(&client, tip).await { Ok(tip_header) => { info!(target: "reth::cli", ?tip, "Successfully fetched tip"); - return Ok(tip_header) + return Ok(tip_header); } Err(error) => { error!(target: "reth::cli", %error, "Failed to fetch the tip. Retrying..."); diff --git a/bin/reth/src/node/mod.rs b/bin/reth/src/node/mod.rs index 7cecca9f2690..7bcc3a415257 100644 --- a/bin/reth/src/node/mod.rs +++ b/bin/reth/src/node/mod.rs @@ -985,7 +985,7 @@ impl NodeCommand { /// Drives the [NetworkManager] future until a [Shutdown](reth_tasks::shutdown::Shutdown) signal is /// received. If configured, this writes known peers to `persistent_peers_file` afterwards. -async fn run_network_until_shutdown( +pub(crate) async fn run_network_until_shutdown( shutdown: reth_tasks::shutdown::GracefulShutdown, network: NetworkManager, persistent_peers_file: Option, From f723a986677d4cbb5d1b3c328e848477ffb4de82 Mon Sep 17 00:00:00 2001 From: Dan Cline <6798349+Rjected@users.noreply.github.com> Date: Fri, 1 Dec 2023 20:32:45 -0500 Subject: [PATCH 07/33] add comments on what we need for the handle --- bin/reth/src/cli/node_config.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/bin/reth/src/cli/node_config.rs b/bin/reth/src/cli/node_config.rs index 218a363cf9f7..5e564e6edb69 100644 --- a/bin/reth/src/cli/node_config.rs +++ b/bin/reth/src/cli/node_config.rs @@ -1050,6 +1050,10 @@ impl NodeConfig { } /// The node handle +// We need from each component on init: +// * channels (for wiring into other components) +// * handles (for wiring into other components) +// * also for giving to the NodeHandle, for example everything rpc #[derive(Debug)] pub struct NodeHandle; From 9a7f7d7595a2156ee1292912235a86438b4f2cd1 Mon Sep 17 00:00:00 2001 From: Dan Cline <6798349+Rjected@users.noreply.github.com> Date: Fri, 1 Dec 2023 20:38:22 -0500 Subject: [PATCH 08/33] add rpc server handles to node handle --- bin/reth/src/cli/node_config.rs | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/bin/reth/src/cli/node_config.rs b/bin/reth/src/cli/node_config.rs index 5e564e6edb69..e39523ad6c6c 100644 --- a/bin/reth/src/cli/node_config.rs +++ b/bin/reth/src/cli/node_config.rs @@ -82,6 +82,8 @@ use std::{ use tokio::sync::{mpsc::unbounded_channel, oneshot, watch}; use tracing::*; +use super::components::RethRpcServerHandles; + /// Start the node #[derive(Debug)] pub struct NodeConfig { @@ -468,7 +470,7 @@ impl NodeConfig { self.adjust_instance_ports(); // Start RPC servers - let _rpc_server_handles = + let rpc_server_handles = self.rpc.start_servers(&components, engine_api, jwt_secret, &mut ext).await?; // Run consensus engine to completion @@ -502,6 +504,13 @@ impl NodeConfig { .await?; } + // we should return here + let _node_handle = NodeHandle { + rpc_server_handles + }; + + // TODO: we need a way to do this in the background because this method will just block + // until the consensus engine exits rx.await??; info!(target: "reth::cli", "Consensus engine has exited."); @@ -1055,7 +1064,10 @@ impl NodeConfig { // * handles (for wiring into other components) // * also for giving to the NodeHandle, for example everything rpc #[derive(Debug)] -pub struct NodeHandle; +pub struct NodeHandle { + /// The handles to the RPC servers + rpc_server_handles: RethRpcServerHandles, +} #[cfg(test)] mod tests { From 7841c270ef43255386314c2abbd577f857e02e1d Mon Sep 17 00:00:00 2001 From: Dan Cline <6798349+Rjected@users.noreply.github.com> Date: Fri, 1 Dec 2023 21:22:52 -0500 Subject: [PATCH 09/33] add node service --- bin/reth/src/cli/node_config.rs | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/bin/reth/src/cli/node_config.rs b/bin/reth/src/cli/node_config.rs index e39523ad6c6c..d562ae388c19 100644 --- a/bin/reth/src/cli/node_config.rs +++ b/bin/reth/src/cli/node_config.rs @@ -505,12 +505,14 @@ impl NodeConfig { } // we should return here - let _node_handle = NodeHandle { - rpc_server_handles - }; + let _node_handle = NodeHandle { rpc_server_handles }; // TODO: we need a way to do this in the background because this method will just block // until the consensus engine exits + // + // We should probably return the node handle AND `rx` so the `execute` method can do all of + // this stuff below, in the meantime we will spawn everything (analogous to foundry + // spawning the NodeService). rx.await??; info!(target: "reth::cli", "Consensus engine has exited."); @@ -1069,6 +1071,10 @@ pub struct NodeHandle { rpc_server_handles: RethRpcServerHandles, } +/// The node service +#[derive(Debug)] +pub struct NodeService; + #[cfg(test)] mod tests { // use super::*; From 0d5e7541fc60ec6f4a88403c3740c5acd96d8bd6 Mon Sep 17 00:00:00 2001 From: Dan Cline <6798349+Rjected@users.noreply.github.com> Date: Fri, 1 Dec 2023 21:30:11 -0500 Subject: [PATCH 10/33] use stage config for more things --- bin/reth/src/cli/node_config.rs | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/bin/reth/src/cli/node_config.rs b/bin/reth/src/cli/node_config.rs index d562ae388c19..d36bcaa97420 100644 --- a/bin/reth/src/cli/node_config.rs +++ b/bin/reth/src/cli/node_config.rs @@ -29,7 +29,7 @@ use reth_beacon_consensus::{ use reth_blockchain_tree::{ config::BlockchainTreeConfig, externals::TreeExternals, BlockchainTree, ShareableBlockchainTree, }; -use reth_config::{config::PruneConfig, Config}; +use reth_config::{config::{PruneConfig, StageConfig}, Config}; use reth_db::{database::Database, init_db, DatabaseEnv}; use reth_downloaders::{ bodies::bodies::BodiesDownloaderBuilder, @@ -348,7 +348,7 @@ impl NodeConfig { let mut pipeline = self .build_networked_pipeline( - &config, + &config.stages, client.clone(), Arc::clone(&consensus), provider_factory, @@ -368,7 +368,7 @@ impl NodeConfig { } else { let pipeline = self .build_networked_pipeline( - &config, + &config.stages, network_client.clone(), Arc::clone(&consensus), provider_factory, @@ -633,7 +633,7 @@ impl NodeConfig { #[allow(clippy::too_many_arguments)] async fn build_networked_pipeline( &self, - config: &Config, + config: &StageConfig, client: Client, consensus: Arc, provider_factory: ProviderFactory, @@ -647,18 +647,18 @@ impl NodeConfig { Client: HeadersClient + BodiesClient + Clone + 'static, { // building network downloaders using the fetch client - let header_downloader = ReverseHeadersDownloaderBuilder::from(config.stages.headers) + let header_downloader = ReverseHeadersDownloaderBuilder::from(config.headers) .build(client.clone(), Arc::clone(&consensus)) .into_task_with(task_executor); - let body_downloader = BodiesDownloaderBuilder::from(config.stages.bodies) + let body_downloader = BodiesDownloaderBuilder::from(config.bodies) .build(client, Arc::clone(&consensus), provider_factory.clone()) .into_task_with(task_executor); let pipeline = self .build_pipeline( provider_factory, - config, + &config, header_downloader, body_downloader, consensus, @@ -892,7 +892,7 @@ impl NodeConfig { async fn build_pipeline( &self, provider_factory: ProviderFactory, - config: &Config, + stage_config: &StageConfig, header_downloader: H, body_downloader: B, consensus: Arc, @@ -906,8 +906,6 @@ impl NodeConfig { H: HeaderDownloader + 'static, B: BodyDownloader + 'static, { - let stage_config = &config.stages; - let mut builder = Pipeline::builder(); if let Some(max_block) = max_block { From 781e578ab61d7535301ddfd9e4a25ad71c5dbd31 Mon Sep 17 00:00:00 2001 From: Dan Cline <6798349+Rjected@users.noreply.github.com> Date: Fri, 1 Dec 2023 21:58:05 -0500 Subject: [PATCH 11/33] impl Default for NodeConfig --- bin/reth/src/cli/node_config.rs | 48 +++++++++++++++++++++++++++------ bin/reth/src/dirs.rs | 5 ++++ 2 files changed, 45 insertions(+), 8 deletions(-) diff --git a/bin/reth/src/cli/node_config.rs b/bin/reth/src/cli/node_config.rs index d36bcaa97420..775169594d69 100644 --- a/bin/reth/src/cli/node_config.rs +++ b/bin/reth/src/cli/node_config.rs @@ -9,7 +9,7 @@ use crate::{ config::RethRpcConfig, ext::{RethCliExt, RethNodeCommandConfig}, }, - dirs::{ChainPath, DataDirPath}, + dirs::{ChainPath, DataDirPath, MaybePlatformPath}, init::init_genesis, node::{cl_events::ConsensusLayerHealthEvents, events, run_network_until_shutdown}, prometheus_exporter, @@ -29,7 +29,10 @@ use reth_beacon_consensus::{ use reth_blockchain_tree::{ config::BlockchainTreeConfig, externals::TreeExternals, BlockchainTree, ShareableBlockchainTree, }; -use reth_config::{config::{PruneConfig, StageConfig}, Config}; +use reth_config::{ + config::{PruneConfig, StageConfig}, + Config, +}; use reth_db::{database::Database, init_db, DatabaseEnv}; use reth_downloaders::{ bodies::bodies::BodiesDownloaderBuilder, @@ -50,7 +53,7 @@ use reth_primitives::{ constants::eip4844::{LoadKzgSettingsError, MAINNET_KZG_TRUSTED_SETUP}, kzg::KzgSettings, stage::StageId, - BlockHashOrNumber, BlockNumber, ChainSpec, DisplayHardforks, Head, SealedHeader, B256, + BlockHashOrNumber, BlockNumber, ChainSpec, DisplayHardforks, Head, SealedHeader, B256, MAINNET, }; use reth_provider::{ providers::BlockchainProvider, BlockHashReader, BlockReader, CanonStateSubscriptions, @@ -94,7 +97,7 @@ pub struct NodeConfig { /// - Linux: `$XDG_DATA_HOME/reth/` or `$HOME/.local/share/reth/` /// - Windows: `{FOLDERID_RoamingAppData}/reth/` /// - macOS: `$HOME/Library/Application Support/reth/` - pub datadir: ChainPath, + pub datadir: MaybePlatformPath, /// The path to the configuration file to use. pub config: Option, @@ -527,8 +530,8 @@ impl NodeConfig { } /// Set the datadir for the node - pub fn datadir(mut self, datadir: ChainPath) -> Self { - self.datadir = datadir.into(); + pub fn datadir(mut self, datadir: MaybePlatformPath) -> Self { + self.datadir = datadir; self } @@ -556,6 +559,12 @@ impl NodeConfig { self } + /// Set the [Chain] for the node + pub fn chain_spec(mut self, chain: Arc) -> Self { + self.chain = chain; + self + } + /// Set the trusted setup file for the node pub fn trusted_setup_file(mut self, trusted_setup_file: impl Into) -> Self { self.trusted_setup_file = Some(trusted_setup_file.into()); @@ -673,8 +682,8 @@ impl NodeConfig { } /// Returns the chain specific path to the data dir. - fn data_dir(&self) -> &ChainPath { - &self.datadir + fn data_dir(&self) -> ChainPath { + self.datadir.unwrap_or_chain_default(self.chain.chain) } /// Returns the path to the config file. @@ -1058,6 +1067,29 @@ impl NodeConfig { } } +impl Default for NodeConfig { + fn default() -> Self { + Self { + datadir: MaybePlatformPath::::default(), + config: None, + chain: MAINNET.clone(), + metrics: None, + instance: 1, + trusted_setup_file: None, + network: NetworkArgs::default(), + rpc: RpcServerArgs::default(), + txpool: TxPoolArgs::default(), + builder: PayloadBuilderArgs::default(), + debug: DebugArgs::default(), + db: DatabaseArgs::default(), + dev: DevArgs::default(), + pruning: PruningArgs::default(), + #[cfg(feature = "optimism")] + rollup: crate::args::RollupArgs::default(), + } + } +} + /// The node handle // We need from each component on init: // * channels (for wiring into other components) diff --git a/bin/reth/src/dirs.rs b/bin/reth/src/dirs.rs index 064b18e7f462..7bcbd2ce3105 100644 --- a/bin/reth/src/dirs.rs +++ b/bin/reth/src/dirs.rs @@ -190,6 +190,11 @@ impl MaybePlatformPath { ) } + /// Returns the default platform path for the specified [Chain]. + pub fn chain_default(chain: Chain) -> ChainPath { + PlatformPath::default().with_chain(chain) + } + /// Returns true if a custom path is set pub fn is_some(&self) -> bool { self.0.is_some() From bb262c719557715cceec8d1e071647dee99172cc Mon Sep 17 00:00:00 2001 From: Dan Cline <6798349+Rjected@users.noreply.github.com> Date: Fri, 1 Dec 2023 22:38:19 -0500 Subject: [PATCH 12/33] add simple test --- bin/reth/src/cli/node_config.rs | 107 +++++++++++++++----------------- 1 file changed, 50 insertions(+), 57 deletions(-) diff --git a/bin/reth/src/cli/node_config.rs b/bin/reth/src/cli/node_config.rs index 775169594d69..0577f0b668a2 100644 --- a/bin/reth/src/cli/node_config.rs +++ b/bin/reth/src/cli/node_config.rs @@ -13,7 +13,7 @@ use crate::{ init::init_genesis, node::{cl_events::ConsensusLayerHealthEvents, events, run_network_until_shutdown}, prometheus_exporter, - runner::CliContext, + runner::{CliContext, tokio_runtime}, utils::get_single_header, version::SHORT_VERSION, }; @@ -72,7 +72,7 @@ use reth_stages::{ TotalDifficultyStage, TransactionLookupStage, }, }; -use reth_tasks::TaskExecutor; +use reth_tasks::{TaskExecutor, TaskManager}; use reth_transaction_pool::{ blobstore::InMemoryBlobStore, TransactionPool, TransactionValidationTaskExecutor, }; @@ -85,7 +85,7 @@ use std::{ use tokio::sync::{mpsc::unbounded_channel, oneshot, watch}; use tracing::*; -use super::components::RethRpcServerHandles; +use super::{components::RethRpcServerHandles, ext::DefaultRethNodeCommandConfig}; /// Start the node #[derive(Debug)] @@ -161,10 +161,25 @@ pub struct NodeConfig { impl NodeConfig { /// Launches the node, also adding any RPC extensions passed. + /// + /// # Example + /// ```rust + /// + /// # use reth_tasks::TaskManager; + /// fn t() { + /// use reth_tasks::TaskSpawner; + /// let rt = tokio::runtime::Runtime::new().unwrap(); + /// let manager = TaskManager::new(rt.handle().clone()); + /// let executor = manager.executor(); + /// let config = NodeConfig::default(); + /// let ext = DefaultRethNodeCommandConfig; + /// let handle = config.launch(ext, executor); + /// } + /// ``` pub async fn launch( mut self, mut ext: E::Node, - ctx: CliContext, + executor: TaskExecutor, ) -> eyre::Result { info!(target: "reth::cli", "reth {} starting", SHORT_VERSION); @@ -210,7 +225,7 @@ impl NodeConfig { debug!(target: "reth::cli", "Spawning stages metrics listener task"); let (sync_metrics_tx, sync_metrics_rx) = unbounded_channel(); let sync_metrics_listener = reth_stages::MetricsListener::new(sync_metrics_rx); - ctx.task_executor.spawn_critical("stages metrics listener task", sync_metrics_listener); + executor.spawn_critical("stages metrics listener task", sync_metrics_listener); let prune_config = self.pruning.prune_config(Arc::clone(&self.chain))?.or(config.prune.clone()); @@ -243,7 +258,7 @@ impl NodeConfig { .with_head_timestamp(head.timestamp) .kzg_settings(self.kzg_settings()?) .with_additional_tasks(1) - .build_with_tasks(blockchain_db.clone(), ctx.task_executor.clone(), blob_store.clone()); + .build_with_tasks(blockchain_db.clone(), executor.clone(), blob_store.clone()); let transaction_pool = reth_transaction_pool::Pool::eth_pool(validator, blob_store, self.txpool.pool_config()); @@ -254,13 +269,13 @@ impl NodeConfig { let pool = transaction_pool.clone(); let chain_events = blockchain_db.canonical_state_stream(); let client = blockchain_db.clone(); - ctx.task_executor.spawn_critical( + executor.spawn_critical( "txpool maintenance task", reth_transaction_pool::maintain::maintain_transaction_pool_future( client, pool, chain_events, - ctx.task_executor.clone(), + executor.clone(), Default::default(), ), ); @@ -276,7 +291,7 @@ impl NodeConfig { let network_config = self.load_network_config( &config, Arc::clone(&db), - ctx.task_executor.clone(), + executor.clone(), head, secret_key, default_peers_path.clone(), @@ -289,7 +304,7 @@ impl NodeConfig { provider: blockchain_db.clone(), pool: transaction_pool.clone(), network: network_builder.handle(), - task_executor: ctx.task_executor.clone(), + task_executor: executor.clone(), events: blockchain_db.clone(), }; @@ -299,7 +314,7 @@ impl NodeConfig { // launch network let network = self.start_network( network_builder, - &ctx.task_executor, + &executor, transaction_pool.clone(), network_client, default_peers_path, @@ -355,7 +370,7 @@ impl NodeConfig { client.clone(), Arc::clone(&consensus), provider_factory, - &ctx.task_executor, + &executor, sync_metrics_tx, prune_config.clone(), max_block, @@ -365,7 +380,7 @@ impl NodeConfig { let pipeline_events = pipeline.events(); task.set_pipeline_events(pipeline_events); debug!(target: "reth::cli", "Spawning auto mine task"); - ctx.task_executor.spawn(Box::pin(task)); + executor.spawn(Box::pin(task)); (pipeline, EitherDownloader::Left(client)) } else { @@ -375,7 +390,7 @@ impl NodeConfig { network_client.clone(), Arc::clone(&consensus), provider_factory, - &ctx.task_executor, + &executor, sync_metrics_tx, prune_config.clone(), max_block, @@ -411,7 +426,7 @@ impl NodeConfig { ); let events = pruner.events(); - hooks.add(PruneHook::new(pruner, Box::new(ctx.task_executor.clone()))); + hooks.add(PruneHook::new(pruner, Box::new(executor.clone()))); info!(target: "reth::cli", ?prune_config, "Pruner initialized"); Either::Left(events) @@ -424,7 +439,7 @@ impl NodeConfig { client, pipeline, blockchain_db.clone(), - Box::new(ctx.task_executor.clone()), + Box::new(executor.clone()), Box::new(network.clone()), max_block, self.debug.continuous, @@ -451,7 +466,7 @@ impl NodeConfig { }, pruner_events.map(Into::into) ); - ctx.task_executor.spawn_critical( + executor.spawn_critical( "events task", events::handle_events(Some(network.clone()), Some(head.number), events, db.clone()), ); @@ -461,7 +476,7 @@ impl NodeConfig { self.chain.clone(), beacon_engine_handle, payload_builder.into(), - Box::new(ctx.task_executor.clone()), + Box::new(executor.clone()), ); info!(target: "reth::cli", "Engine API handler initialized"); @@ -479,7 +494,7 @@ impl NodeConfig { // Run consensus engine to completion let (tx, rx) = oneshot::channel(); info!(target: "reth::cli", "Starting consensus engine"); - ctx.task_executor.spawn_critical_blocking("consensus engine", async move { + executor.spawn_critical_blocking("consensus engine", async move { let res = beacon_consensus_engine.await; let _ = tx.send(res); }); @@ -1105,45 +1120,23 @@ pub struct NodeHandle { #[derive(Debug)] pub struct NodeService; + +/// A simple function to launch a node with the specified [NodeConfig], spawning tasks on the +/// [TaskExecutor] constructed from [tokio_runtime]. +pub async fn spawn_node(config: NodeConfig) -> eyre::Result { + let runtime = tokio_runtime()?; + let task_manager = TaskManager::new(runtime.handle().clone()); + let ext = DefaultRethNodeCommandConfig; + config.launch::<()>(ext, task_manager.executor()).await +} + + #[cfg(test)] mod tests { - // use super::*; - // use crate::args::NetworkArgs; - // use reth_primitives::ChainSpec; - // use std::net::Ipv4Addr; - - #[test] - fn test_node_config() { - - // let config = NodeConfig::default() - // .datadir("/tmp") - // .config("/tmp/config.toml") - // .chain(Arc::new(ChainSpec::default())) - // .metrics(SocketAddr::new(Ipv4Addr::LOCALHOST.into(), 8080)) - // .instance(1) - // .trusted_setup_file("/tmp/trusted_setup") - // .network(NetworkArgs::default()) - // .rpc(RpcServerArgs::default()) - // .txpool(TxPoolArgs::default()) - // .builder(PayloadBuilderArgs::default()) - // .debug(DebugArgs::default()) - // .db(DatabaseArgs::default()) - // .dev(DevArgs::default()) - // .pruning(PruningArgs::default()); - - // assert_eq!(config.datadir, PathBuf::from("/tmp")); - // assert_eq!(config.config, Some(PathBuf::from("/tmp/config.toml"))); - // assert_eq!(config.chain, Arc::new(ChainSpec::default())); - // assert_eq!(config.metrics, Some(SocketAddr::new(Ipv4Addr::LOCALHOST.into(), 8080))); - // assert_eq!(config.instance, 1); - // assert_eq!(config.trusted_setup_file, Some(PathBuf::from("/tmp/trusted_setup"))); - // assert_eq!(config.network, NetworkArgs::default()); - // assert_eq!(config.rpc, RpcServerArgs::default()); - // assert_eq!(config.txpool, TxPoolArgs::default()); - // assert_eq!(config.builder, PayloadBuilderArgs::default()); - // assert_eq!(config.debug, DebugArgs::default()); - // assert_eq!(config.db, DatabaseArgs::default()); - // assert_eq!(config.dev, DevArgs::default()); - // assert_eq!(config.pruning, PruningArgs::default()); + use super::*; + + #[tokio::test] + async fn test_node_config() { + let _handle = spawn_node(NodeConfig::default()).await; } } From c96b4e896e543c3b10a7d9b98bdc2f3de6fc5c20 Mon Sep 17 00:00:00 2001 From: Dan Cline <6798349+Rjected@users.noreply.github.com> Date: Sat, 2 Dec 2023 00:08:02 -0500 Subject: [PATCH 13/33] hmmmm temp / test db as well as real db, is challenging --- bin/reth/src/cli/db_type.rs | 77 ++++++++++++++++++++++++ bin/reth/src/cli/mod.rs | 1 + bin/reth/src/cli/node_config.rs | 101 ++++++++++++++++++++++---------- 3 files changed, 147 insertions(+), 32 deletions(-) create mode 100644 bin/reth/src/cli/db_type.rs diff --git a/bin/reth/src/cli/db_type.rs b/bin/reth/src/cli/db_type.rs new file mode 100644 index 000000000000..2baa247e4aff --- /dev/null +++ b/bin/reth/src/cli/db_type.rs @@ -0,0 +1,77 @@ +use std::sync::Arc; + +use crate::dirs::{DataDirPath, MaybePlatformPath}; +use reth_db::{ + database::Database, + init_db, + mdbx::{tx, RO, RW}, + test_utils::{create_test_rw_db, TempDatabase}, + DatabaseEnv, +}; +use reth_interfaces::db::LogLevel; +use reth_primitives::Chain; + +/// A type that represents either a _real_ (represented by a path), or _test_ database, which will +/// use a [TempDatabase]. +#[derive(Debug)] +pub enum DatabaseType { + /// The real database type + Real(MaybePlatformPath), + /// The test database type + Test, +} + +/// The [Default] implementation for [DatabaseType] uses the _real_ variant, using the default +/// value for the inner [MaybePlatformPath]. +impl Default for DatabaseType { + fn default() -> Self { + Self::Real(MaybePlatformPath::::default()) + } +} + +impl DatabaseType { + /// Creates a _test_ database + fn test() -> Self { + Self::Test + } +} + +/// Type that represents a [DatabaseType] and [LogLevel], used to build a database type +pub struct DatabaseBuilder { + /// The database type + db_type: DatabaseType, +} + +impl DatabaseBuilder { + /// Creates the [DatabaseBuilder] with the given [DatabaseType] + pub fn new(db_type: DatabaseType) -> Self { + Self { db_type } + } + + /// Initializes and returns the test database. If the [DatabaseType] is test, the [LogLevel] + /// and chain are not used. + pub fn build_db( + self, + log_level: Option, + chain: Chain, + ) -> eyre::Result { + match self.db_type { + DatabaseType::Test => Ok(DatabaseInstance::Test(create_test_rw_db())), + DatabaseType::Real(path) => { + let chain_dir = path.unwrap_or_chain_default(chain); + + tracing::info!(target: "reth::cli", path = ?chain_dir, "Opening database"); + Ok(DatabaseInstance::Real(Arc::new(init_db(chain_dir, log_level)?))) + } + } + } +} + +/// A constructed database type, without path information. +#[derive(Debug, Clone)] +pub enum DatabaseInstance { + /// The test database + Test(Arc>), + /// The right database + Real(Arc), +} diff --git a/bin/reth/src/cli/mod.rs b/bin/reth/src/cli/mod.rs index d340c51958c2..7957a1617c4f 100644 --- a/bin/reth/src/cli/mod.rs +++ b/bin/reth/src/cli/mod.rs @@ -21,6 +21,7 @@ use std::{fmt, fmt::Display, sync::Arc}; pub mod components; pub mod config; +pub mod db_type; pub mod ext; pub mod node_config; diff --git a/bin/reth/src/cli/node_config.rs b/bin/reth/src/cli/node_config.rs index 0577f0b668a2..9473f66fd2c7 100644 --- a/bin/reth/src/cli/node_config.rs +++ b/bin/reth/src/cli/node_config.rs @@ -13,7 +13,7 @@ use crate::{ init::init_genesis, node::{cl_events::ConsensusLayerHealthEvents, events, run_network_until_shutdown}, prometheus_exporter, - runner::{CliContext, tokio_runtime}, + runner::tokio_runtime, utils::get_single_header, version::SHORT_VERSION, }; @@ -85,19 +85,15 @@ use std::{ use tokio::sync::{mpsc::unbounded_channel, oneshot, watch}; use tracing::*; -use super::{components::RethRpcServerHandles, ext::DefaultRethNodeCommandConfig}; +use super::{ + components::RethRpcServerHandles, db_type::DatabaseType, ext::DefaultRethNodeCommandConfig, +}; /// Start the node #[derive(Debug)] pub struct NodeConfig { - /// The path to the data dir for all reth files and subdirectories. - /// - /// Defaults to the OS-specific data directory: - /// - /// - Linux: `$XDG_DATA_HOME/reth/` or `$HOME/.local/share/reth/` - /// - Windows: `{FOLDERID_RoamingAppData}/reth/` - /// - macOS: `$HOME/Library/Application Support/reth/` - pub datadir: MaybePlatformPath, + /// The test database + pub database: DatabaseType, /// The path to the configuration file to use. pub config: Option, @@ -160,20 +156,41 @@ pub struct NodeConfig { } impl NodeConfig { + /// Creates a testing [NodeConfig], causing the database to be launched ephemerally. + pub fn test() -> Self { + Self { + database: DatabaseType::default(), + config: None, + chain: MAINNET.clone(), + metrics: None, + instance: 1, + trusted_setup_file: None, + network: NetworkArgs::default(), + rpc: RpcServerArgs::default(), + txpool: TxPoolArgs::default(), + builder: PayloadBuilderArgs::default(), + debug: DebugArgs::default(), + db: DatabaseArgs::default(), + dev: DevArgs::default(), + pruning: PruningArgs::default(), + #[cfg(feature = "optimism")] + rollup: crate::args::RollupArgs::default(), + } + } + /// Launches the node, also adding any RPC extensions passed. /// /// # Example /// ```rust - /// /// # use reth_tasks::TaskManager; /// fn t() { - /// use reth_tasks::TaskSpawner; - /// let rt = tokio::runtime::Runtime::new().unwrap(); - /// let manager = TaskManager::new(rt.handle().clone()); - /// let executor = manager.executor(); - /// let config = NodeConfig::default(); - /// let ext = DefaultRethNodeCommandConfig; - /// let handle = config.launch(ext, executor); + /// use reth_tasks::TaskSpawner; + /// let rt = tokio::runtime::Runtime::new().unwrap(); + /// let manager = TaskManager::new(rt.handle().clone()); + /// let executor = manager.executor(); + /// let config = NodeConfig::default(); + /// let ext = DefaultRethNodeCommandConfig; + /// let handle = config.launch(ext, executor); /// } /// ``` pub async fn launch( @@ -192,11 +209,22 @@ impl NodeConfig { let prometheus_handle = self.install_prometheus_recorder()?; - let data_dir = self.data_dir(); + let data_dir = self.data_dir().expect("see below"); let db_path = data_dir.db_path(); - info!(target: "reth::cli", path = ?db_path, "Opening database"); // TODO: set up test database, see if we can configure either real or test db + // let db = + // DatabaseBuilder::new(self.database).build_db(self.db.log_level, self.chain.chain)?; + // TODO: ok, this doesn't work because: + // * Database is not object safe, and is sealed + // * DatabaseInstance is not sealed + // * ProviderFactory takes DB + // * But this would make it ProviderFactory OR + // ProviderFactory> + // * But we also have no Either impl for + // ProviderFactory> etc + // * Because Database is not object safe we can't return Box either + // * EitherDatabase is nontrivial due to associated types let db = Arc::new(init_db(&db_path, self.db.log_level)?.with_metrics()); info!(target: "reth::cli", "Database opened"); @@ -523,7 +551,7 @@ impl NodeConfig { } // we should return here - let _node_handle = NodeHandle { rpc_server_handles }; + let _node_handle = NodeHandle { _rpc_server_handles: rpc_server_handles }; // TODO: we need a way to do this in the background because this method will just block // until the consensus engine exits @@ -546,7 +574,7 @@ impl NodeConfig { /// Set the datadir for the node pub fn datadir(mut self, datadir: MaybePlatformPath) -> Self { - self.datadir = datadir; + self.database = DatabaseType::Real(datadir); self } @@ -696,19 +724,29 @@ impl NodeConfig { Ok(pipeline) } - /// Returns the chain specific path to the data dir. - fn data_dir(&self) -> ChainPath { - self.datadir.unwrap_or_chain_default(self.chain.chain) + /// Returns the chain specific path to the data dir. This returns `None` if the database is + /// configured for testing. + fn data_dir(&self) -> Option> { + match &self.database { + DatabaseType::Real(data_dir) => { + Some(data_dir.unwrap_or_chain_default(self.chain.chain)) + } + DatabaseType::Test => None, + } } /// Returns the path to the config file. - fn config_path(&self) -> PathBuf { - self.config.clone().unwrap_or_else(|| self.data_dir().config_path()) + fn config_path(&self) -> Option { + let chain_dir = self.data_dir()?; + + let config = self.config.clone().unwrap_or_else(|| chain_dir.config_path()); + Some(config) } /// Loads the reth config with the given datadir root fn load_config(&self) -> eyre::Result { - let config_path = self.config_path(); + let Some(config_path) = self.config_path() else { todo!() }; + let mut config = confy::load_path::(&config_path) .wrap_err_with(|| format!("Could not load config file {:?}", config_path))?; @@ -1085,7 +1123,7 @@ impl NodeConfig { impl Default for NodeConfig { fn default() -> Self { Self { - datadir: MaybePlatformPath::::default(), + database: DatabaseType::default(), config: None, chain: MAINNET.clone(), metrics: None, @@ -1113,14 +1151,13 @@ impl Default for NodeConfig { #[derive(Debug)] pub struct NodeHandle { /// The handles to the RPC servers - rpc_server_handles: RethRpcServerHandles, + _rpc_server_handles: RethRpcServerHandles, } /// The node service #[derive(Debug)] pub struct NodeService; - /// A simple function to launch a node with the specified [NodeConfig], spawning tasks on the /// [TaskExecutor] constructed from [tokio_runtime]. pub async fn spawn_node(config: NodeConfig) -> eyre::Result { @@ -1130,13 +1167,13 @@ pub async fn spawn_node(config: NodeConfig) -> eyre::Result { config.launch::<()>(ext, task_manager.executor()).await } - #[cfg(test)] mod tests { use super::*; #[tokio::test] async fn test_node_config() { + // we need to override the db let _handle = spawn_node(NodeConfig::default()).await; } } From 40ff8ec26e94763107ddb4d36dd610cf4506dc89 Mon Sep 17 00:00:00 2001 From: Dan Cline <6798349+Rjected@users.noreply.github.com> Date: Sat, 2 Dec 2023 14:35:04 -0500 Subject: [PATCH 14/33] rename for optimism --- bin/reth/src/cli/node_config.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/reth/src/cli/node_config.rs b/bin/reth/src/cli/node_config.rs index 9473f66fd2c7..e52b7277e0a9 100644 --- a/bin/reth/src/cli/node_config.rs +++ b/bin/reth/src/cli/node_config.rs @@ -537,7 +537,7 @@ impl NodeConfig { // starts syncing from the current tip in the DB. #[cfg(feature = "optimism")] if self.chain.is_optimism() && !self.rollup.enable_genesis_walkback { - let client = _rpc_server_handles.auth.http_client(); + let client = rpc_server_handles.auth.http_client(); reth_rpc_api::EngineApiClient::fork_choice_updated_v2( &client, reth_rpc_types::engine::ForkchoiceState { From 6b6c1b92cf2f61813bfcb84bdff8aec3979cef03 Mon Sep 17 00:00:00 2001 From: Dan Cline <6798349+Rjected@users.noreply.github.com> Date: Mon, 4 Dec 2023 19:07:10 -0500 Subject: [PATCH 15/33] move to 2-step build pattern --- bin/reth/src/cli/node_config.rs | 80 ++++++++++++++++++++++++++++++--- 1 file changed, 74 insertions(+), 6 deletions(-) diff --git a/bin/reth/src/cli/node_config.rs b/bin/reth/src/cli/node_config.rs index e52b7277e0a9..5a969eba3bf3 100644 --- a/bin/reth/src/cli/node_config.rs +++ b/bin/reth/src/cli/node_config.rs @@ -7,6 +7,7 @@ use crate::{ cli::{ components::RethNodeComponentsImpl, config::RethRpcConfig, + db_type::{DatabaseBuilder, DatabaseInstance}, ext::{RethCliExt, RethNodeCommandConfig}, }, dirs::{ChainPath, DataDirPath, MaybePlatformPath}, @@ -178,6 +179,18 @@ impl NodeConfig { } } + /// Install a database + pub fn install_db(self) -> eyre::Result> { + // let db_instance = + // DatabaseBuilder::new(self.database).build_db(self.db.log_level, self.chain.chain)?; + + // match db_instance { + // DatabaseInstance::Real(database) => Ok(NodeBuilderWithDatabase { config: self, database }), + // DatabaseInstance::Test(database) => Ok(NodeBuilderWithDatabase { config: self, database }), + // } + todo!() + } + /// Launches the node, also adding any RPC extensions passed. /// /// # Example @@ -198,6 +211,24 @@ impl NodeConfig { mut ext: E::Node, executor: TaskExecutor, ) -> eyre::Result { + let data_dir = self.data_dir().expect("see below"); + let db_path = data_dir.db_path(); + + // TODO: set up test database, see if we can configure either real or test db + // let db_instance = + // DatabaseBuilder::new(self.database).build_db(self.db.log_level, self.chain.chain)?; + + // match db_instance { + // DatabaseInstance::Real(database) => { + // // let builder = NodeBuilderWithDatabase { config: self, database, data_dir }; + // // todo!() + // }, + // DatabaseInstance::Test(database) => { + // // let builder = NodeBuilderWithDatabase { config: self, database, data_dir }; + // // todo!(); + // }, + // } + info!(target: "reth::cli", "reth {} starting", SHORT_VERSION); // Raise the fd limit of the process. @@ -209,12 +240,6 @@ impl NodeConfig { let prometheus_handle = self.install_prometheus_recorder()?; - let data_dir = self.data_dir().expect("see below"); - let db_path = data_dir.db_path(); - - // TODO: set up test database, see if we can configure either real or test db - // let db = - // DatabaseBuilder::new(self.database).build_db(self.db.log_level, self.chain.chain)?; // TODO: ok, this doesn't work because: // * Database is not object safe, and is sealed // * DatabaseInstance is not sealed @@ -1143,6 +1168,49 @@ impl Default for NodeConfig { } } +/// A version of the [NodeConfig] that has an installed database. This is used to construct the +/// [NodeHandle]. +/// +/// This also contains a path to a data dir that cannot be changed. +pub struct NodeBuilderWithDatabase { + /// The node config + pub config: NodeConfig, + /// The database + pub database: Arc, + /// The data dir + pub data_dir: ChainPath, +} + +impl NodeBuilderWithDatabase { + pub async fn launch( + mut self, + mut ext: E::Node, + executor: TaskExecutor, + ) -> eyre::Result { + let mut provider_factory = ProviderFactory::new(Arc::clone(&self.database), Arc::clone(&self.config.chain)); + + // configure snapshotter + let snapshotter = reth_snapshot::Snapshotter::new( + provider_factory.clone(), + self.data_dir.snapshots_path(), + self.config.chain.snapshot_block_interval, + )?; + + provider_factory = provider_factory + .with_snapshots(self.data_dir.snapshots_path(), snapshotter.highest_snapshot_receiver()); + + let prometheus_handle = self.config.install_prometheus_recorder()?; + self.config.start_metrics_endpoint(prometheus_handle, Arc::clone(&self.database)).await?; + + debug!(target: "reth::cli", chain=%self.config.chain.chain, genesis=?self.config.chain.genesis_hash(), "Initializing genesis"); + + let genesis_hash = init_genesis(Arc::clone(&self.database), self.config.chain.clone())?; + + info!(target: "reth::cli", "{}", DisplayHardforks::new(self.config.chain.hardforks())); + todo!() + } +} + /// The node handle // We need from each component on init: // * channels (for wiring into other components) From 9ce269519227192455f23441e37a8813c5ffef93 Mon Sep 17 00:00:00 2001 From: Dan Cline <6798349+Rjected@users.noreply.github.com> Date: Wed, 13 Dec 2023 18:41:00 +0200 Subject: [PATCH 16/33] start integrating with new generic changes --- bin/reth/src/cli/node_config.rs | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/bin/reth/src/cli/node_config.rs b/bin/reth/src/cli/node_config.rs index 5a969eba3bf3..6e36b3a092e2 100644 --- a/bin/reth/src/cli/node_config.rs +++ b/bin/reth/src/cli/node_config.rs @@ -6,7 +6,7 @@ use crate::{ }, cli::{ components::RethNodeComponentsImpl, - config::RethRpcConfig, + config::{RethRpcConfig, RethTransactionPoolConfig}, db_type::{DatabaseBuilder, DatabaseInstance}, ext::{RethCliExt, RethNodeCommandConfig}, }, @@ -34,7 +34,7 @@ use reth_config::{ config::{PruneConfig, StageConfig}, Config, }; -use reth_db::{database::Database, init_db, DatabaseEnv}; +use reth_db::{database::Database, init_db, DatabaseEnv, database_metrics::DatabaseMetrics}; use reth_downloaders::{ bodies::bodies::BodiesDownloaderBuilder, headers::reverse_headers::ReverseHeadersDownloaderBuilder, @@ -806,11 +806,14 @@ impl NodeConfig { prometheus_exporter::install_recorder() } - async fn start_metrics_endpoint( + async fn start_metrics_endpoint( &self, prometheus_handle: PrometheusHandle, - db: Arc, - ) -> eyre::Result<()> { + db: Metrics, + ) -> eyre::Result<()> + where + Metrics: DatabaseMetrics + 'static + Send + Sync, + { if let Some(listen_addr) = self.metrics { info!(target: "reth::cli", addr = %listen_addr, "Starting metrics endpoint"); prometheus_exporter::serve( @@ -1181,7 +1184,7 @@ pub struct NodeBuilderWithDatabase { pub data_dir: ChainPath, } -impl NodeBuilderWithDatabase { +impl NodeBuilderWithDatabase { pub async fn launch( mut self, mut ext: E::Node, From 3b7f84881e0f2c543cf1f8760da0be952b66e891 Mon Sep 17 00:00:00 2001 From: Dan Cline <6798349+Rjected@users.noreply.github.com> Date: Thu, 14 Dec 2023 15:39:23 +0200 Subject: [PATCH 17/33] add consensus engine rx --- bin/reth/src/cli/node_config.rs | 601 +++++++++++++++++--------------- 1 file changed, 321 insertions(+), 280 deletions(-) diff --git a/bin/reth/src/cli/node_config.rs b/bin/reth/src/cli/node_config.rs index 6e36b3a092e2..995d8fa4df81 100644 --- a/bin/reth/src/cli/node_config.rs +++ b/bin/reth/src/cli/node_config.rs @@ -25,7 +25,7 @@ use metrics_exporter_prometheus::PrometheusHandle; use reth_auto_seal_consensus::{AutoSealBuilder, AutoSealConsensus, MiningMode}; use reth_beacon_consensus::{ hooks::{EngineHooks, PruneHook}, - BeaconConsensus, BeaconConsensusEngine, MIN_BLOCKS_FOR_PIPELINE_RUN, + BeaconConsensus, BeaconConsensusEngine, MIN_BLOCKS_FOR_PIPELINE_RUN, BeaconConsensusEngineError, }; use reth_blockchain_tree::{ config::BlockchainTreeConfig, externals::TreeExternals, BlockchainTree, ShareableBlockchainTree, @@ -317,284 +317,6 @@ impl NodeConfig { reth_transaction_pool::Pool::eth_pool(validator, blob_store, self.txpool.pool_config()); info!(target: "reth::cli", "Transaction pool initialized"); - // spawn txpool maintenance task - { - let pool = transaction_pool.clone(); - let chain_events = blockchain_db.canonical_state_stream(); - let client = blockchain_db.clone(); - executor.spawn_critical( - "txpool maintenance task", - reth_transaction_pool::maintain::maintain_transaction_pool_future( - client, - pool, - chain_events, - executor.clone(), - Default::default(), - ), - ); - debug!(target: "reth::cli", "Spawned txpool maintenance task"); - } - - info!(target: "reth::cli", "Connecting to P2P network"); - let network_secret_path = - self.network.p2p_secret_key.clone().unwrap_or_else(|| data_dir.p2p_secret_path()); - debug!(target: "reth::cli", ?network_secret_path, "Loading p2p key file"); - let secret_key = get_secret_key(&network_secret_path)?; - let default_peers_path = data_dir.known_peers_path(); - let network_config = self.load_network_config( - &config, - Arc::clone(&db), - executor.clone(), - head, - secret_key, - default_peers_path.clone(), - ); - - let network_client = network_config.client.clone(); - let mut network_builder = NetworkManager::builder(network_config).await?; - - let components = RethNodeComponentsImpl { - provider: blockchain_db.clone(), - pool: transaction_pool.clone(), - network: network_builder.handle(), - task_executor: executor.clone(), - events: blockchain_db.clone(), - }; - - // allow network modifications - ext.configure_network(network_builder.network_mut(), &components)?; - - // launch network - let network = self.start_network( - network_builder, - &executor, - transaction_pool.clone(), - network_client, - default_peers_path, - ); - - info!(target: "reth::cli", peer_id = %network.peer_id(), local_addr = %network.local_addr(), enode = %network.local_node_record(), "Connected to P2P network"); - debug!(target: "reth::cli", peer_id = ?network.peer_id(), "Full peer ID"); - let network_client = network.fetch_client().await?; - - ext.on_components_initialized(&components)?; - - debug!(target: "reth::cli", "Spawning payload builder service"); - let payload_builder = ext.spawn_payload_builder_service(&self.builder, &components)?; - - let (consensus_engine_tx, consensus_engine_rx) = unbounded_channel(); - let max_block = if let Some(block) = self.debug.max_block { - Some(block) - } else if let Some(tip) = self.debug.tip { - Some(self.lookup_or_fetch_tip(&db, &network_client, tip).await?) - } else { - None - }; - - // Configure the pipeline - let (mut pipeline, client) = if self.dev.dev { - info!(target: "reth::cli", "Starting Reth in dev mode"); - - let mining_mode = if let Some(interval) = self.dev.block_time { - MiningMode::interval(interval) - } else if let Some(max_transactions) = self.dev.block_max_transactions { - MiningMode::instant( - max_transactions, - transaction_pool.pending_transactions_listener(), - ) - } else { - info!(target: "reth::cli", "No mining mode specified, defaulting to ReadyTransaction"); - MiningMode::instant(1, transaction_pool.pending_transactions_listener()) - }; - - let (_, client, mut task) = AutoSealBuilder::new( - Arc::clone(&self.chain), - blockchain_db.clone(), - transaction_pool.clone(), - consensus_engine_tx.clone(), - canon_state_notification_sender, - mining_mode, - ) - .build(); - - let mut pipeline = self - .build_networked_pipeline( - &config.stages, - client.clone(), - Arc::clone(&consensus), - provider_factory, - &executor, - sync_metrics_tx, - prune_config.clone(), - max_block, - ) - .await?; - - let pipeline_events = pipeline.events(); - task.set_pipeline_events(pipeline_events); - debug!(target: "reth::cli", "Spawning auto mine task"); - executor.spawn(Box::pin(task)); - - (pipeline, EitherDownloader::Left(client)) - } else { - let pipeline = self - .build_networked_pipeline( - &config.stages, - network_client.clone(), - Arc::clone(&consensus), - provider_factory, - &executor, - sync_metrics_tx, - prune_config.clone(), - max_block, - ) - .await?; - - (pipeline, EitherDownloader::Right(network_client)) - }; - - let pipeline_events = pipeline.events(); - - let initial_target = if let Some(tip) = self.debug.tip { - // Set the provided tip as the initial pipeline target. - debug!(target: "reth::cli", %tip, "Tip manually set"); - Some(tip) - } else if self.debug.continuous { - // Set genesis as the initial pipeline target. - // This will allow the downloader to start - debug!(target: "reth::cli", "Continuous sync mode enabled"); - Some(genesis_hash) - } else { - None - }; - - let mut hooks = EngineHooks::new(); - - let pruner_events = if let Some(prune_config) = prune_config { - let mut pruner = self.build_pruner( - &prune_config, - db.clone(), - tree_config, - snapshotter.highest_snapshot_receiver(), - ); - - let events = pruner.events(); - hooks.add(PruneHook::new(pruner, Box::new(executor.clone()))); - - info!(target: "reth::cli", ?prune_config, "Pruner initialized"); - Either::Left(events) - } else { - Either::Right(stream::empty()) - }; - - // Configure the consensus engine - let (beacon_consensus_engine, beacon_engine_handle) = BeaconConsensusEngine::with_channel( - client, - pipeline, - blockchain_db.clone(), - Box::new(executor.clone()), - Box::new(network.clone()), - max_block, - self.debug.continuous, - payload_builder.clone(), - initial_target, - MIN_BLOCKS_FOR_PIPELINE_RUN, - consensus_engine_tx, - consensus_engine_rx, - hooks, - )?; - info!(target: "reth::cli", "Consensus engine initialized"); - - let events = stream_select!( - network.event_listener().map(Into::into), - beacon_engine_handle.event_listener().map(Into::into), - pipeline_events.map(Into::into), - if self.debug.tip.is_none() { - Either::Left( - ConsensusLayerHealthEvents::new(Box::new(blockchain_db.clone())) - .map(Into::into), - ) - } else { - Either::Right(stream::empty()) - }, - pruner_events.map(Into::into) - ); - executor.spawn_critical( - "events task", - events::handle_events(Some(network.clone()), Some(head.number), events, db.clone()), - ); - - let engine_api = EngineApi::new( - blockchain_db.clone(), - self.chain.clone(), - beacon_engine_handle, - payload_builder.into(), - Box::new(executor.clone()), - ); - info!(target: "reth::cli", "Engine API handler initialized"); - - // extract the jwt secret from the args if possible - let default_jwt_path = data_dir.jwt_path(); - let jwt_secret = self.rpc.auth_jwt_secret(default_jwt_path)?; - - // adjust rpc port numbers based on instance number - self.adjust_instance_ports(); - - // Start RPC servers - let rpc_server_handles = - self.rpc.start_servers(&components, engine_api, jwt_secret, &mut ext).await?; - - // Run consensus engine to completion - let (tx, rx) = oneshot::channel(); - info!(target: "reth::cli", "Starting consensus engine"); - executor.spawn_critical_blocking("consensus engine", async move { - let res = beacon_consensus_engine.await; - let _ = tx.send(res); - }); - - ext.on_node_started(&components)?; - - // If `enable_genesis_walkback` is set to true, the rollup client will need to - // perform the derivation pipeline from genesis, validating the data dir. - // When set to false, set the finalized, safe, and unsafe head block hashes - // on the rollup client using a fork choice update. This prevents the rollup - // client from performing the derivation pipeline from genesis, and instead - // starts syncing from the current tip in the DB. - #[cfg(feature = "optimism")] - if self.chain.is_optimism() && !self.rollup.enable_genesis_walkback { - let client = rpc_server_handles.auth.http_client(); - reth_rpc_api::EngineApiClient::fork_choice_updated_v2( - &client, - reth_rpc_types::engine::ForkchoiceState { - head_block_hash: head.hash, - safe_block_hash: head.hash, - finalized_block_hash: head.hash, - }, - None, - ) - .await?; - } - - // we should return here - let _node_handle = NodeHandle { _rpc_server_handles: rpc_server_handles }; - - // TODO: we need a way to do this in the background because this method will just block - // until the consensus engine exits - // - // We should probably return the node handle AND `rx` so the `execute` method can do all of - // this stuff below, in the meantime we will spawn everything (analogous to foundry - // spawning the NodeService). - rx.await??; - - info!(target: "reth::cli", "Consensus engine has exited."); - - if self.debug.terminate { - todo!() - } else { - // The pipeline has finished downloading blocks up to `--debug.tip` or - // `--debug.max-block`. Keep other node components alive for further usage. - futures::future::pending().await - } } /// Set the datadir for the node @@ -1210,7 +932,324 @@ impl NodeBuilderWithDatabase { let genesis_hash = init_genesis(Arc::clone(&self.database), self.config.chain.clone())?; info!(target: "reth::cli", "{}", DisplayHardforks::new(self.config.chain.hardforks())); - todo!() + + let consensus = self.consensus(); + + debug!(target: "reth::cli", "Spawning stages metrics listener task"); + let (sync_metrics_tx, sync_metrics_rx) = unbounded_channel(); + let sync_metrics_listener = reth_stages::MetricsListener::new(sync_metrics_rx); + ctx.task_executor.spawn_critical("stages metrics listener task", sync_metrics_listener); + + let prune_config = + self.pruning.prune_config(Arc::clone(&self.chain))?.or(config.prune.clone()); + + // configure blockchain tree + let tree_externals = TreeExternals::new( + provider_factory.clone(), + Arc::clone(&consensus), + EvmProcessorFactory::new(self.chain.clone()), + ); + let tree_config = BlockchainTreeConfig::default(); + let tree = BlockchainTree::new( + tree_externals, + tree_config, + prune_config.clone().map(|config| config.segments), + )? + .with_sync_metrics_tx(sync_metrics_tx.clone()); + let canon_state_notification_sender = tree.canon_state_notification_sender(); + let blockchain_tree = ShareableBlockchainTree::new(tree); + debug!(target: "reth::cli", "configured blockchain tree"); + + // fetch the head block from the database + let head = self.lookup_head(Arc::clone(&db)).wrap_err("the head block is missing")?; + + // setup the blockchain provider + let blockchain_db = + BlockchainProvider::new(provider_factory.clone(), blockchain_tree.clone())?; + let blob_store = InMemoryBlobStore::default(); + let validator = TransactionValidationTaskExecutor::eth_builder(Arc::clone(&self.chain)) + .with_head_timestamp(head.timestamp) + .kzg_settings(self.kzg_settings()?) + .with_additional_tasks(1) + .build_with_tasks(blockchain_db.clone(), ctx.task_executor.clone(), blob_store.clone()); + + let transaction_pool = + reth_transaction_pool::Pool::eth_pool(validator, blob_store, self.txpool.pool_config()); + info!(target: "reth::cli", "Transaction pool initialized"); + + // spawn txpool maintenance task + { + let pool = transaction_pool.clone(); + let chain_events = blockchain_db.canonical_state_stream(); + let client = blockchain_db.clone(); + ctx.task_executor.spawn_critical( + "txpool maintenance task", + reth_transaction_pool::maintain::maintain_transaction_pool_future( + client, + pool, + chain_events, + ctx.task_executor.clone(), + Default::default(), + ), + ); + debug!(target: "reth::cli", "Spawned txpool maintenance task"); + } + + info!(target: "reth::cli", "Connecting to P2P network"); + let network_secret_path = + self.network.p2p_secret_key.clone().unwrap_or_else(|| data_dir.p2p_secret_path()); + debug!(target: "reth::cli", ?network_secret_path, "Loading p2p key file"); + let secret_key = get_secret_key(&network_secret_path)?; + let default_peers_path = data_dir.known_peers_path(); + let network_config = self.load_network_config( + &config, + Arc::clone(&db), + ctx.task_executor.clone(), + head, + secret_key, + default_peers_path.clone(), + ); + + let network_client = network_config.client.clone(); + let mut network_builder = NetworkManager::builder(network_config).await?; + + let components = RethNodeComponentsImpl { + provider: blockchain_db.clone(), + pool: transaction_pool.clone(), + network: network_builder.handle(), + task_executor: ctx.task_executor.clone(), + events: blockchain_db.clone(), + }; + + // allow network modifications + self.ext.configure_network(network_builder.network_mut(), &components)?; + + // launch network + let network = self.start_network( + network_builder, + &ctx.task_executor, + transaction_pool.clone(), + network_client, + default_peers_path, + ); + + info!(target: "reth::cli", peer_id = %network.peer_id(), local_addr = %network.local_addr(), enode = %network.local_node_record(), "Connected to P2P network"); + debug!(target: "reth::cli", peer_id = ?network.peer_id(), "Full peer ID"); + let network_client = network.fetch_client().await?; + + self.ext.on_components_initialized(&components)?; + + debug!(target: "reth::cli", "Spawning payload builder service"); + let payload_builder = self.ext.spawn_payload_builder_service(&self.builder, &components)?; + + let (consensus_engine_tx, consensus_engine_rx) = unbounded_channel(); + let max_block = if let Some(block) = self.debug.max_block { + Some(block) + } else if let Some(tip) = self.debug.tip { + Some(self.lookup_or_fetch_tip(&db, &network_client, tip).await?) + } else { + None + }; + + // Configure the pipeline + let (mut pipeline, client) = if self.dev.dev { + info!(target: "reth::cli", "Starting Reth in dev mode"); + + let mining_mode = if let Some(interval) = self.dev.block_time { + MiningMode::interval(interval) + } else if let Some(max_transactions) = self.dev.block_max_transactions { + MiningMode::instant( + max_transactions, + transaction_pool.pending_transactions_listener(), + ) + } else { + info!(target: "reth::cli", "No mining mode specified, defaulting to ReadyTransaction"); + MiningMode::instant(1, transaction_pool.pending_transactions_listener()) + }; + + let (_, client, mut task) = AutoSealBuilder::new( + Arc::clone(&self.chain), + blockchain_db.clone(), + transaction_pool.clone(), + consensus_engine_tx.clone(), + canon_state_notification_sender, + mining_mode, + ) + .build(); + + let mut pipeline = self + .build_networked_pipeline( + &config.stages, + client.clone(), + Arc::clone(&consensus), + provider_factory, + &ctx.task_executor, + sync_metrics_tx, + prune_config.clone(), + max_block, + ) + .await?; + + let pipeline_events = pipeline.events(); + task.set_pipeline_events(pipeline_events); + debug!(target: "reth::cli", "Spawning auto mine task"); + ctx.task_executor.spawn(Box::pin(task)); + + (pipeline, EitherDownloader::Left(client)) + } else { + let pipeline = self + .build_networked_pipeline( + &config.stages, + network_client.clone(), + Arc::clone(&consensus), + provider_factory, + &ctx.task_executor, + sync_metrics_tx, + prune_config.clone(), + max_block, + ) + .await?; + + (pipeline, EitherDownloader::Right(network_client)) + }; + + let pipeline_events = pipeline.events(); + + let initial_target = if let Some(tip) = self.debug.tip { + // Set the provided tip as the initial pipeline target. + debug!(target: "reth::cli", %tip, "Tip manually set"); + Some(tip) + } else if self.debug.continuous { + // Set genesis as the initial pipeline target. + // This will allow the downloader to start + debug!(target: "reth::cli", "Continuous sync mode enabled"); + Some(genesis_hash) + } else { + None + }; + + let mut hooks = EngineHooks::new(); + + let pruner_events = if let Some(prune_config) = prune_config { + let mut pruner = self.build_pruner( + &prune_config, + db.clone(), + tree_config, + snapshotter.highest_snapshot_receiver(), + ); + + let events = pruner.events(); + hooks.add(PruneHook::new(pruner, Box::new(ctx.task_executor.clone()))); + + info!(target: "reth::cli", ?prune_config, "Pruner initialized"); + Either::Left(events) + } else { + Either::Right(stream::empty()) + }; + + // Configure the consensus engine + let (beacon_consensus_engine, beacon_engine_handle) = BeaconConsensusEngine::with_channel( + client, + pipeline, + blockchain_db.clone(), + Box::new(ctx.task_executor.clone()), + Box::new(network.clone()), + max_block, + self.debug.continuous, + payload_builder.clone(), + initial_target, + MIN_BLOCKS_FOR_PIPELINE_RUN, + consensus_engine_tx, + consensus_engine_rx, + hooks, + )?; + info!(target: "reth::cli", "Consensus engine initialized"); + + let events = stream_select!( + network.event_listener().map(Into::into), + beacon_engine_handle.event_listener().map(Into::into), + pipeline_events.map(Into::into), + if self.debug.tip.is_none() { + Either::Left( + ConsensusLayerHealthEvents::new(Box::new(blockchain_db.clone())) + .map(Into::into), + ) + } else { + Either::Right(stream::empty()) + }, + pruner_events.map(Into::into) + ); + ctx.task_executor.spawn_critical( + "events task", + events::handle_events(Some(network.clone()), Some(head.number), events, db.clone()), + ); + + let engine_api = EngineApi::new( + blockchain_db.clone(), + self.chain.clone(), + beacon_engine_handle, + payload_builder.into(), + Box::new(ctx.task_executor.clone()), + ); + info!(target: "reth::cli", "Engine API handler initialized"); + + // extract the jwt secret from the args if possible + let default_jwt_path = data_dir.jwt_path(); + let jwt_secret = self.rpc.auth_jwt_secret(default_jwt_path)?; + + // adjust rpc port numbers based on instance number + self.adjust_instance_ports(); + + // Start RPC servers + let _rpc_server_handles = + self.rpc.start_servers(&components, engine_api, jwt_secret, &mut self.ext).await?; + + // Run consensus engine to completion + let (tx, rx) = oneshot::channel(); + info!(target: "reth::cli", "Starting consensus engine"); + ctx.task_executor.spawn_critical_blocking("consensus engine", async move { + let res = beacon_consensus_engine.await; + let _ = tx.send(res); + }); + + self.ext.on_node_started(&components)?; + + // If `enable_genesis_walkback` is set to true, the rollup client will need to + // perform the derivation pipeline from genesis, validating the data dir. + // When set to false, set the finalized, safe, and unsafe head block hashes + // on the rollup client using a fork choice update. This prevents the rollup + // client from performing the derivation pipeline from genesis, and instead + // starts syncing from the current tip in the DB. + #[cfg(feature = "optimism")] + if self.chain.is_optimism() && !self.rollup.enable_genesis_walkback { + let client = _rpc_server_handles.auth.http_client(); + reth_rpc_api::EngineApiClient::fork_choice_updated_v2( + &client, + reth_rpc_types::engine::ForkchoiceState { + head_block_hash: head.hash, + safe_block_hash: head.hash, + finalized_block_hash: head.hash, + }, + None, + ) + .await?; + } + + // construct node handle and return + let _node_handle = NodeHandle { _rpc_server_handles, _consensus_engine_rx: rx }; + return Ok(_node_handle) + + // rx.await??; + + // info!(target: "reth::cli", "Consensus engine has exited."); + + // if self.debug.terminate { + // Ok(()) + // } else { + // // The pipeline has finished downloading blocks up to `--debug.tip` or + // // `--debug.max-block`. Keep other node components alive for further usage. + // futures::future::pending().await + // } } } @@ -1223,6 +1262,8 @@ impl NodeBuilderWithDatabase { pub struct NodeHandle { /// The handles to the RPC servers _rpc_server_handles: RethRpcServerHandles, + /// The receiver half of the channel for the consensus engine + _consensus_engine_rx: oneshot::Receiver>, } /// The node service From c73520096cd7739239b593fce3c0e2445fc1d06f Mon Sep 17 00:00:00 2001 From: Dan Cline <6798349+Rjected@users.noreply.github.com> Date: Fri, 15 Dec 2023 10:48:45 +0200 Subject: [PATCH 18/33] fix more stuff --- bin/reth/src/cli/node_config.rs | 155 +++++++++++++++++++------------- 1 file changed, 95 insertions(+), 60 deletions(-) diff --git a/bin/reth/src/cli/node_config.rs b/bin/reth/src/cli/node_config.rs index 995d8fa4df81..071e5debe068 100644 --- a/bin/reth/src/cli/node_config.rs +++ b/bin/reth/src/cli/node_config.rs @@ -25,7 +25,8 @@ use metrics_exporter_prometheus::PrometheusHandle; use reth_auto_seal_consensus::{AutoSealBuilder, AutoSealConsensus, MiningMode}; use reth_beacon_consensus::{ hooks::{EngineHooks, PruneHook}, - BeaconConsensus, BeaconConsensusEngine, MIN_BLOCKS_FOR_PIPELINE_RUN, BeaconConsensusEngineError, + BeaconConsensus, BeaconConsensusEngine, BeaconConsensusEngineError, + MIN_BLOCKS_FOR_PIPELINE_RUN, }; use reth_blockchain_tree::{ config::BlockchainTreeConfig, externals::TreeExternals, BlockchainTree, ShareableBlockchainTree, @@ -34,7 +35,7 @@ use reth_config::{ config::{PruneConfig, StageConfig}, Config, }; -use reth_db::{database::Database, init_db, DatabaseEnv, database_metrics::DatabaseMetrics}; +use reth_db::{database::Database, database_metrics::DatabaseMetrics, init_db, DatabaseEnv}; use reth_downloaders::{ bodies::bodies::BodiesDownloaderBuilder, headers::reverse_headers::ReverseHeadersDownloaderBuilder, @@ -46,7 +47,7 @@ use reth_interfaces::{ either::EitherDownloader, headers::{client::HeadersClient, downloader::HeaderDownloader}, }, - RethResult, + RethResult, blockchain_tree::BlockchainTreeEngine, }; use reth_network::{NetworkBuilder, NetworkConfig, NetworkEvents, NetworkHandle, NetworkManager}; use reth_network_api::{NetworkInfo, PeersInfo}; @@ -58,7 +59,7 @@ use reth_primitives::{ }; use reth_provider::{ providers::BlockchainProvider, BlockHashReader, BlockReader, CanonStateSubscriptions, - HeaderProvider, HeaderSyncMode, ProviderFactory, StageCheckpointReader, + ExecutorFactory, HeaderProvider, HeaderSyncMode, ProviderFactory, StageCheckpointReader, BlockchainTreePendingStateProvider, }; use reth_prune::{segments::SegmentSet, Pruner}; use reth_revm::EvmProcessorFactory; @@ -75,7 +76,7 @@ use reth_stages::{ }; use reth_tasks::{TaskExecutor, TaskManager}; use reth_transaction_pool::{ - blobstore::InMemoryBlobStore, TransactionPool, TransactionValidationTaskExecutor, + blobstore::InMemoryBlobStore, TransactionPool, TransactionValidationTaskExecutor, EthTransactionValidator, EthPooledTransaction, CoinbaseTipOrdering, }; use secp256k1::SecretKey; use std::{ @@ -191,6 +192,50 @@ impl NodeConfig { todo!() } + /// Build a transaction pool and spawn the transaction pool maintenance task + pub fn build_and_spawn_txpool( + &self, + blockchain_db: BlockchainProvider, + head: Head, + executor: TaskExecutor, + ) -> eyre::Result, EthPooledTransaction>>, CoinbaseTipOrdering, InMemoryBlobStore>> + where + DB: Database + Unpin + Clone + 'static, + EF: ExecutorFactory + Clone + 'static, + Tree: BlockchainTreeEngine + BlockchainTreePendingStateProvider + CanonStateSubscriptions + Clone, + { + let blob_store = InMemoryBlobStore::default(); + let validator = TransactionValidationTaskExecutor::eth_builder(Arc::clone(&self.chain)) + .with_head_timestamp(head.timestamp) + .kzg_settings(self.kzg_settings()?) + .with_additional_tasks(1) + .build_with_tasks(blockchain_db.clone(), executor.clone(), blob_store.clone()); + + let transaction_pool = + reth_transaction_pool::Pool::eth_pool(validator, blob_store, self.txpool.pool_config()); + info!(target: "reth::cli", "Transaction pool initialized"); + + // spawn txpool maintenance task + { + let pool = transaction_pool.clone(); + let chain_events = blockchain_db.canonical_state_stream(); + let client = blockchain_db.clone(); + executor.spawn_critical( + "txpool maintenance task", + reth_transaction_pool::maintain::maintain_transaction_pool_future( + client, + pool, + chain_events, + executor.clone(), + Default::default(), + ), + ); + debug!(target: "reth::cli", "Spawned txpool maintenance task"); + } + + Ok(transaction_pool) + } + /// Launches the node, also adding any RPC extensions passed. /// /// # Example @@ -212,22 +257,22 @@ impl NodeConfig { executor: TaskExecutor, ) -> eyre::Result { let data_dir = self.data_dir().expect("see below"); - let db_path = data_dir.db_path(); // TODO: set up test database, see if we can configure either real or test db - // let db_instance = - // DatabaseBuilder::new(self.database).build_db(self.db.log_level, self.chain.chain)?; - - // match db_instance { - // DatabaseInstance::Real(database) => { - // // let builder = NodeBuilderWithDatabase { config: self, database, data_dir }; - // // todo!() - // }, - // DatabaseInstance::Test(database) => { - // // let builder = NodeBuilderWithDatabase { config: self, database, data_dir }; - // // todo!(); - // }, - // } + let database = std::mem::take(&mut self.database); + let db_instance = + DatabaseBuilder::new(database).build_db(self.db.log_level, self.chain.chain)?; + + return match db_instance { + DatabaseInstance::Real(database) => { + let builder = NodeBuilderWithDatabase { config: self, database, data_dir }; + builder.launch::(ext, executor).await + } + DatabaseInstance::Test(database) => { + let builder = NodeBuilderWithDatabase { config: self, database, data_dir }; + builder.launch::(ext, executor).await + } + }; info!(target: "reth::cli", "reth {} starting", SHORT_VERSION); @@ -316,7 +361,6 @@ impl NodeConfig { let transaction_pool = reth_transaction_pool::Pool::eth_pool(validator, blob_store, self.txpool.pool_config()); info!(target: "reth::cli", "Transaction pool initialized"); - } /// Set the datadir for the node @@ -912,7 +956,20 @@ impl NodeBuilderWithDatabase { mut ext: E::Node, executor: TaskExecutor, ) -> eyre::Result { - let mut provider_factory = ProviderFactory::new(Arc::clone(&self.database), Arc::clone(&self.config.chain)); + info!(target: "reth::cli", "reth {} starting", SHORT_VERSION); + + // Raise the fd limit of the process. + // Does not do anything on windows. + raise_fd_limit(); + + // get config + let config = self.config.load_config()?; + + let prometheus_handle = self.config.install_prometheus_recorder()?; + info!(target: "reth::cli", "Database opened"); + + let mut provider_factory = + ProviderFactory::new(Arc::clone(&self.database), Arc::clone(&self.config.chain)); // configure snapshotter let snapshotter = reth_snapshot::Snapshotter::new( @@ -921,10 +978,11 @@ impl NodeBuilderWithDatabase { self.config.chain.snapshot_block_interval, )?; - provider_factory = provider_factory - .with_snapshots(self.data_dir.snapshots_path(), snapshotter.highest_snapshot_receiver()); + provider_factory = provider_factory.with_snapshots( + self.data_dir.snapshots_path(), + snapshotter.highest_snapshot_receiver(), + ); - let prometheus_handle = self.config.install_prometheus_recorder()?; self.config.start_metrics_endpoint(prometheus_handle, Arc::clone(&self.database)).await?; debug!(target: "reth::cli", chain=%self.config.chain.chain, genesis=?self.config.chain.genesis_hash(), "Initializing genesis"); @@ -933,15 +991,18 @@ impl NodeBuilderWithDatabase { info!(target: "reth::cli", "{}", DisplayHardforks::new(self.config.chain.hardforks())); - let consensus = self.consensus(); + let consensus = self.config.consensus(); debug!(target: "reth::cli", "Spawning stages metrics listener task"); let (sync_metrics_tx, sync_metrics_rx) = unbounded_channel(); let sync_metrics_listener = reth_stages::MetricsListener::new(sync_metrics_rx); - ctx.task_executor.spawn_critical("stages metrics listener task", sync_metrics_listener); + executor.spawn_critical("stages metrics listener task", sync_metrics_listener); - let prune_config = - self.pruning.prune_config(Arc::clone(&self.chain))?.or(config.prune.clone()); + let prune_config = self + .config + .pruning + .prune_config(Arc::clone(&self.config.chain))? + .or(config.prune.clone()); // configure blockchain tree let tree_externals = TreeExternals::new( @@ -966,34 +1027,7 @@ impl NodeBuilderWithDatabase { // setup the blockchain provider let blockchain_db = BlockchainProvider::new(provider_factory.clone(), blockchain_tree.clone())?; - let blob_store = InMemoryBlobStore::default(); - let validator = TransactionValidationTaskExecutor::eth_builder(Arc::clone(&self.chain)) - .with_head_timestamp(head.timestamp) - .kzg_settings(self.kzg_settings()?) - .with_additional_tasks(1) - .build_with_tasks(blockchain_db.clone(), ctx.task_executor.clone(), blob_store.clone()); - - let transaction_pool = - reth_transaction_pool::Pool::eth_pool(validator, blob_store, self.txpool.pool_config()); - info!(target: "reth::cli", "Transaction pool initialized"); - - // spawn txpool maintenance task - { - let pool = transaction_pool.clone(); - let chain_events = blockchain_db.canonical_state_stream(); - let client = blockchain_db.clone(); - ctx.task_executor.spawn_critical( - "txpool maintenance task", - reth_transaction_pool::maintain::maintain_transaction_pool_future( - client, - pool, - chain_events, - ctx.task_executor.clone(), - Default::default(), - ), - ); - debug!(target: "reth::cli", "Spawned txpool maintenance task"); - } + let transaction_pool = self.config.build_and_spawn_txpool(blockchain_db, head, executor)?; info!(target: "reth::cli", "Connecting to P2P network"); let network_secret_path = @@ -1098,6 +1132,7 @@ impl NodeBuilderWithDatabase { (pipeline, EitherDownloader::Left(client)) } else { let pipeline = self + .config .build_networked_pipeline( &config.stages, network_client.clone(), @@ -1115,11 +1150,11 @@ impl NodeBuilderWithDatabase { let pipeline_events = pipeline.events(); - let initial_target = if let Some(tip) = self.debug.tip { + let initial_target = if let Some(tip) = self.config.debug.tip { // Set the provided tip as the initial pipeline target. debug!(target: "reth::cli", %tip, "Tip manually set"); Some(tip) - } else if self.debug.continuous { + } else if self.config.debug.continuous { // Set genesis as the initial pipeline target. // This will allow the downloader to start debug!(target: "reth::cli", "Continuous sync mode enabled"); @@ -1131,7 +1166,7 @@ impl NodeBuilderWithDatabase { let mut hooks = EngineHooks::new(); let pruner_events = if let Some(prune_config) = prune_config { - let mut pruner = self.build_pruner( + let mut pruner = self.config.build_pruner( &prune_config, db.clone(), tree_config, @@ -1237,7 +1272,7 @@ impl NodeBuilderWithDatabase { // construct node handle and return let _node_handle = NodeHandle { _rpc_server_handles, _consensus_engine_rx: rx }; - return Ok(_node_handle) + return Ok(_node_handle); // rx.await??; From ec3d0c22e9bce3943053d22eb7e3e01b6e2734f1 Mon Sep 17 00:00:00 2001 From: Dan Cline <6798349+Rjected@users.noreply.github.com> Date: Mon, 18 Dec 2023 19:15:03 +0200 Subject: [PATCH 19/33] fix node config structure and finish launch method --- bin/reth/src/cli/db_type.rs | 7 +- bin/reth/src/cli/node_config.rs | 517 +++++++++++++++++--------------- 2 files changed, 282 insertions(+), 242 deletions(-) diff --git a/bin/reth/src/cli/db_type.rs b/bin/reth/src/cli/db_type.rs index 2baa247e4aff..bc9d27460aa3 100644 --- a/bin/reth/src/cli/db_type.rs +++ b/bin/reth/src/cli/db_type.rs @@ -1,10 +1,9 @@ -use std::sync::Arc; +//! A real or test database type +use std::sync::Arc; use crate::dirs::{DataDirPath, MaybePlatformPath}; use reth_db::{ - database::Database, init_db, - mdbx::{tx, RO, RW}, test_utils::{create_test_rw_db, TempDatabase}, DatabaseEnv, }; @@ -31,7 +30,7 @@ impl Default for DatabaseType { impl DatabaseType { /// Creates a _test_ database - fn test() -> Self { + pub fn test() -> Self { Self::Test } } diff --git a/bin/reth/src/cli/node_config.rs b/bin/reth/src/cli/node_config.rs index 071e5debe068..466ca0ffdd86 100644 --- a/bin/reth/src/cli/node_config.rs +++ b/bin/reth/src/cli/node_config.rs @@ -1,4 +1,7 @@ //! Support for customizing the node +use super::{ + components::RethRpcServerHandles, db_type::DatabaseType, ext::DefaultRethNodeCommandConfig, +}; use crate::{ args::{ get_secret_key, DatabaseArgs, DebugArgs, DevArgs, NetworkArgs, PayloadBuilderArgs, @@ -35,19 +38,23 @@ use reth_config::{ config::{PruneConfig, StageConfig}, Config, }; -use reth_db::{database::Database, database_metrics::DatabaseMetrics, init_db, DatabaseEnv}; +use reth_db::{ + database::Database, + database_metrics::{DatabaseMetadata, DatabaseMetrics}, +}; use reth_downloaders::{ bodies::bodies::BodiesDownloaderBuilder, headers::reverse_headers::ReverseHeadersDownloaderBuilder, }; use reth_interfaces::{ + blockchain_tree::BlockchainTreeEngine, consensus::Consensus, p2p::{ bodies::{client::BodiesClient, downloader::BodyDownloader}, either::EitherDownloader, headers::{client::HeadersClient, downloader::HeaderDownloader}, }, - RethResult, blockchain_tree::BlockchainTreeEngine, + RethResult, }; use reth_network::{NetworkBuilder, NetworkConfig, NetworkEvents, NetworkHandle, NetworkManager}; use reth_network_api::{NetworkInfo, PeersInfo}; @@ -55,11 +62,13 @@ use reth_primitives::{ constants::eip4844::{LoadKzgSettingsError, MAINNET_KZG_TRUSTED_SETUP}, kzg::KzgSettings, stage::StageId, - BlockHashOrNumber, BlockNumber, ChainSpec, DisplayHardforks, Head, SealedHeader, B256, MAINNET, + BlockHashOrNumber, BlockNumber, ChainSpec, DisplayHardforks, Head, SealedHeader, TxHash, B256, + MAINNET, }; use reth_provider::{ - providers::BlockchainProvider, BlockHashReader, BlockReader, CanonStateSubscriptions, - ExecutorFactory, HeaderProvider, HeaderSyncMode, ProviderFactory, StageCheckpointReader, BlockchainTreePendingStateProvider, + providers::BlockchainProvider, BlockHashReader, BlockReader, + BlockchainTreePendingStateProvider, CanonStateSubscriptions, HeaderProvider, HeaderSyncMode, + ProviderFactory, StageCheckpointReader, }; use reth_prune::{segments::SegmentSet, Pruner}; use reth_revm::EvmProcessorFactory; @@ -73,10 +82,12 @@ use reth_stages::{ IndexStorageHistoryStage, MerkleStage, SenderRecoveryStage, StorageHashingStage, TotalDifficultyStage, TransactionLookupStage, }, + MetricEvent, }; use reth_tasks::{TaskExecutor, TaskManager}; use reth_transaction_pool::{ - blobstore::InMemoryBlobStore, TransactionPool, TransactionValidationTaskExecutor, EthTransactionValidator, EthPooledTransaction, CoinbaseTipOrdering, + blobstore::InMemoryBlobStore, CoinbaseTipOrdering, EthPooledTransaction, + EthTransactionValidator, TransactionPool, TransactionValidationTaskExecutor, }; use secp256k1::SecretKey; use std::{ @@ -84,12 +95,20 @@ use std::{ path::PathBuf, sync::Arc, }; -use tokio::sync::{mpsc::unbounded_channel, oneshot, watch}; +use tokio::sync::{ + mpsc::{unbounded_channel, Receiver, UnboundedSender}, + oneshot, watch, +}; use tracing::*; -use super::{ - components::RethRpcServerHandles, db_type::DatabaseType, ext::DefaultRethNodeCommandConfig, -}; +/// The default reth transaction pool type +type RethTransactionPool = reth_transaction_pool::Pool< + TransactionValidationTaskExecutor< + EthTransactionValidator, EthPooledTransaction>, + >, + CoinbaseTipOrdering, + InMemoryBlobStore, +>; /// Start the node #[derive(Debug)] @@ -161,7 +180,7 @@ impl NodeConfig { /// Creates a testing [NodeConfig], causing the database to be launched ephemerally. pub fn test() -> Self { Self { - database: DatabaseType::default(), + database: DatabaseType::test(), config: None, chain: MAINNET.clone(), metrics: None, @@ -180,29 +199,144 @@ impl NodeConfig { } } - /// Install a database - pub fn install_db(self) -> eyre::Result> { - // let db_instance = - // DatabaseBuilder::new(self.database).build_db(self.db.log_level, self.chain.chain)?; + /// Get the network secret from the given data dir + pub fn network_secret(&self, data_dir: &ChainPath) -> eyre::Result { + let network_secret_path = + self.network.p2p_secret_key.clone().unwrap_or_else(|| data_dir.p2p_secret_path()); + debug!(target: "reth::cli", ?network_secret_path, "Loading p2p key file"); + let secret_key = get_secret_key(&network_secret_path)?; + Ok(secret_key) + } + + /// Returns the initial pipeline target, based on whether or not the node is running in + /// `debug.tip` mode, `debug.continuous` mode, or neither. + /// + /// If running in `debug.tip` mode, the configured tip is returned. + /// Otherwise, if running in `debug.continuous` mode, the genesis hash is returned. + /// Otherwise, `None` is returned. This is what the node will do by default. + pub fn initial_pipeline_target(&self, genesis_hash: B256) -> Option { + if let Some(tip) = self.debug.tip { + // Set the provided tip as the initial pipeline target. + debug!(target: "reth::cli", %tip, "Tip manually set"); + Some(tip) + } else if self.debug.continuous { + // Set genesis as the initial pipeline target. + // This will allow the downloader to start + debug!(target: "reth::cli", "Continuous sync mode enabled"); + Some(genesis_hash) + } else { + None + } + } - // match db_instance { - // DatabaseInstance::Real(database) => Ok(NodeBuilderWithDatabase { config: self, database }), - // DatabaseInstance::Test(database) => Ok(NodeBuilderWithDatabase { config: self, database }), - // } - todo!() + /// Returns the max block that the node should run to, looking it up from the network if + /// necessary + pub async fn max_block( + &self, + network_client: &Client, + provider_factory: ProviderFactory, + ) -> eyre::Result> + where + DB: Database, + Client: HeadersClient, + { + let max_block = if let Some(block) = self.debug.max_block { + Some(block) + } else if let Some(tip) = self.debug.tip { + Some(self.lookup_or_fetch_tip(provider_factory, network_client, tip).await?) + } else { + None + }; + + Ok(max_block) + } + + /// Get the [MiningMode] from the given dev args + pub fn mining_mode(&self, pending_transactions_listener: Receiver) -> MiningMode { + if let Some(interval) = self.dev.block_time { + MiningMode::interval(interval) + } else if let Some(max_transactions) = self.dev.block_max_transactions { + MiningMode::instant(max_transactions, pending_transactions_listener) + } else { + info!(target: "reth::cli", "No mining mode specified, defaulting to ReadyTransaction"); + MiningMode::instant(1, pending_transactions_listener) + } + } + + /// Build a network and spawn it + pub async fn build_network( + &self, + config: &Config, + provider_factory: ProviderFactory, + executor: TaskExecutor, + head: Head, + data_dir: &ChainPath, + ) -> eyre::Result<(ProviderFactory, NetworkBuilder, (), ()>)> + where + DB: Database + Unpin + Clone + 'static, + { + info!(target: "reth::cli", "Connecting to P2P network"); + let network_secret_path = + self.network.p2p_secret_key.clone().unwrap_or_else(|| data_dir.p2p_secret_path()); + debug!(target: "reth::cli", ?network_secret_path, "Loading p2p key file"); + let secret_key = get_secret_key(&network_secret_path)?; + let default_peers_path = data_dir.known_peers_path(); + let network_config = self.load_network_config( + config, + provider_factory, + executor.clone(), + head, + secret_key, + default_peers_path.clone(), + ); + + let client = network_config.client.clone(); + let builder = NetworkManager::builder(network_config).await?; + Ok((client, builder)) + } + + /// Build the blockchain tree + pub fn build_blockchain_tree( + &self, + provider_factory: ProviderFactory, + consensus: Arc, + prune_config: Option, + sync_metrics_tx: UnboundedSender, + tree_config: BlockchainTreeConfig, + ) -> eyre::Result> + where + DB: Database + Unpin + Clone + 'static, + { + // configure blockchain tree + let tree_externals = TreeExternals::new( + provider_factory.clone(), + consensus.clone(), + EvmProcessorFactory::new(self.chain.clone()), + ); + let tree = BlockchainTree::new( + tree_externals, + tree_config, + prune_config.clone().map(|config| config.segments), + )? + .with_sync_metrics_tx(sync_metrics_tx.clone()); + + Ok(tree) } /// Build a transaction pool and spawn the transaction pool maintenance task - pub fn build_and_spawn_txpool( + pub fn build_and_spawn_txpool( &self, - blockchain_db: BlockchainProvider, + blockchain_db: &BlockchainProvider, head: Head, - executor: TaskExecutor, - ) -> eyre::Result, EthPooledTransaction>>, CoinbaseTipOrdering, InMemoryBlobStore>> + executor: &TaskExecutor, + ) -> eyre::Result> where DB: Database + Unpin + Clone + 'static, - EF: ExecutorFactory + Clone + 'static, - Tree: BlockchainTreeEngine + BlockchainTreePendingStateProvider + CanonStateSubscriptions + Clone, + Tree: BlockchainTreeEngine + + BlockchainTreePendingStateProvider + + CanonStateSubscriptions + + Clone + + 'static, { let blob_store = InMemoryBlobStore::default(); let validator = TransactionValidationTaskExecutor::eth_builder(Arc::clone(&self.chain)) @@ -253,17 +387,16 @@ impl NodeConfig { /// ``` pub async fn launch( mut self, - mut ext: E::Node, + ext: E::Node, executor: TaskExecutor, ) -> eyre::Result { let data_dir = self.data_dir().expect("see below"); - // TODO: set up test database, see if we can configure either real or test db let database = std::mem::take(&mut self.database); let db_instance = DatabaseBuilder::new(database).build_db(self.db.log_level, self.chain.chain)?; - return match db_instance { + match db_instance { DatabaseInstance::Real(database) => { let builder = NodeBuilderWithDatabase { config: self, database, data_dir }; builder.launch::(ext, executor).await @@ -272,95 +405,7 @@ impl NodeConfig { let builder = NodeBuilderWithDatabase { config: self, database, data_dir }; builder.launch::(ext, executor).await } - }; - - info!(target: "reth::cli", "reth {} starting", SHORT_VERSION); - - // Raise the fd limit of the process. - // Does not do anything on windows. - raise_fd_limit(); - - // get config - let config = self.load_config()?; - - let prometheus_handle = self.install_prometheus_recorder()?; - - // TODO: ok, this doesn't work because: - // * Database is not object safe, and is sealed - // * DatabaseInstance is not sealed - // * ProviderFactory takes DB - // * But this would make it ProviderFactory OR - // ProviderFactory> - // * But we also have no Either impl for - // ProviderFactory> etc - // * Because Database is not object safe we can't return Box either - // * EitherDatabase is nontrivial due to associated types - let db = Arc::new(init_db(&db_path, self.db.log_level)?.with_metrics()); - info!(target: "reth::cli", "Database opened"); - - let mut provider_factory = ProviderFactory::new(Arc::clone(&db), Arc::clone(&self.chain)); - - // configure snapshotter - let snapshotter = reth_snapshot::Snapshotter::new( - provider_factory.clone(), - data_dir.snapshots_path(), - self.chain.snapshot_block_interval, - )?; - - provider_factory = provider_factory - .with_snapshots(data_dir.snapshots_path(), snapshotter.highest_snapshot_receiver()); - - self.start_metrics_endpoint(prometheus_handle, Arc::clone(&db)).await?; - - debug!(target: "reth::cli", chain=%self.chain.chain, genesis=?self.chain.genesis_hash(), "Initializing genesis"); - - let genesis_hash = init_genesis(Arc::clone(&db), self.chain.clone())?; - - info!(target: "reth::cli", "{}", DisplayHardforks::new(self.chain.hardforks())); - - let consensus = self.consensus(); - - debug!(target: "reth::cli", "Spawning stages metrics listener task"); - let (sync_metrics_tx, sync_metrics_rx) = unbounded_channel(); - let sync_metrics_listener = reth_stages::MetricsListener::new(sync_metrics_rx); - executor.spawn_critical("stages metrics listener task", sync_metrics_listener); - - let prune_config = - self.pruning.prune_config(Arc::clone(&self.chain))?.or(config.prune.clone()); - - // configure blockchain tree - let tree_externals = TreeExternals::new( - provider_factory.clone(), - Arc::clone(&consensus), - EvmProcessorFactory::new(self.chain.clone()), - ); - let tree_config = BlockchainTreeConfig::default(); - let tree = BlockchainTree::new( - tree_externals, - tree_config, - prune_config.clone().map(|config| config.segments), - )? - .with_sync_metrics_tx(sync_metrics_tx.clone()); - let canon_state_notification_sender = tree.canon_state_notification_sender(); - let blockchain_tree = ShareableBlockchainTree::new(tree); - debug!(target: "reth::cli", "configured blockchain tree"); - - // fetch the head block from the database - let head = self.lookup_head(Arc::clone(&db)).wrap_err("the head block is missing")?; - - // setup the blockchain provider - let blockchain_db = - BlockchainProvider::new(provider_factory.clone(), blockchain_tree.clone())?; - let blob_store = InMemoryBlobStore::default(); - let validator = TransactionValidationTaskExecutor::eth_builder(Arc::clone(&self.chain)) - .with_head_timestamp(head.timestamp) - .kzg_settings(self.kzg_settings()?) - .with_additional_tasks(1) - .build_with_tasks(blockchain_db.clone(), executor.clone(), blob_store.clone()); - - let transaction_pool = - reth_transaction_pool::Pool::eth_pool(validator, blob_store, self.txpool.pool_config()); - info!(target: "reth::cli", "Transaction pool initialized"); + } } /// Set the datadir for the node @@ -377,7 +422,7 @@ impl NodeConfig { /// Set the chain for the node pub fn chain(mut self, chain: Arc) -> Self { - self.chain = chain.into(); + self.chain = chain; self } @@ -501,7 +546,7 @@ impl NodeConfig { let pipeline = self .build_pipeline( provider_factory, - &config, + config, header_downloader, body_downloader, consensus, @@ -602,7 +647,7 @@ impl NodeConfig { task_executor: &TaskExecutor, pool: Pool, client: C, - default_peers_path: PathBuf, + data_dir: &ChainPath, ) -> NetworkHandle where C: BlockReader + HeaderProvider + Clone + Unpin + 'static, @@ -614,6 +659,7 @@ impl NodeConfig { task_executor.spawn_critical("p2p txpool", txpool); task_executor.spawn_critical("p2p eth request handler", eth); + let default_peers_path = data_dir.known_peers_path(); let known_peers_file = self.network.persistent_peers_file(default_peers_path); task_executor .spawn_critical_with_graceful_shutdown_signal("p2p network task", |shutdown| { @@ -626,8 +672,7 @@ impl NodeConfig { /// Fetches the head block from the database. /// /// If the database is empty, returns the genesis block. - fn lookup_head(&self, db: Arc) -> RethResult { - let factory = ProviderFactory::new(db, self.chain.clone()); + fn lookup_head(&self, factory: ProviderFactory) -> RethResult { let provider = factory.provider()?; let head = provider.get_stage_checkpoint(StageId::Finish)?.unwrap_or_default().block_number; @@ -659,7 +704,7 @@ impl NodeConfig { /// NOTE: The download is attempted with infinite retries. async fn lookup_or_fetch_tip( &self, - db: DB, + provider_factory: ProviderFactory, client: Client, tip: B256, ) -> RethResult @@ -667,7 +712,7 @@ impl NodeConfig { DB: Database, Client: HeadersClient, { - Ok(self.fetch_tip(db, client, BlockHashOrNumber::Hash(tip)).await?.number) + Ok(self.fetch_tip(provider_factory, client, BlockHashOrNumber::Hash(tip)).await?.number) } /// Attempt to look up the block with the given number and return the header. @@ -675,7 +720,7 @@ impl NodeConfig { /// NOTE: The download is attempted with infinite retries. async fn fetch_tip( &self, - db: DB, + factory: ProviderFactory, client: Client, tip: BlockHashOrNumber, ) -> RethResult @@ -683,7 +728,6 @@ impl NodeConfig { DB: Database, Client: HeadersClient, { - let factory = ProviderFactory::new(db, self.chain.clone()); let provider = factory.provider()?; let header = provider.header_by_hash_or_number(tip)?; @@ -691,7 +735,7 @@ impl NodeConfig { // try to look up the header in the database if let Some(header) = header { info!(target: "reth::cli", ?tip, "Successfully looked up tip block in the database"); - return Ok(header.seal_slow()); + return Ok(header.seal_slow()) } info!(target: "reth::cli", ?tip, "Fetching tip block from the network."); @@ -699,7 +743,7 @@ impl NodeConfig { match get_single_header(&client, tip).await { Ok(tip_header) => { info!(target: "reth::cli", ?tip, "Successfully fetched tip"); - return Ok(tip_header); + return Ok(tip_header) } Err(error) => { error!(target: "reth::cli", %error, "Failed to fetch the tip. Retrying..."); @@ -708,15 +752,15 @@ impl NodeConfig { } } - fn load_network_config( + fn load_network_config( &self, config: &Config, - db: Arc, + provider_factory: ProviderFactory, executor: TaskExecutor, head: Head, secret_key: SecretKey, default_peers_path: PathBuf, - ) -> NetworkConfig>> { + ) -> NetworkConfig> { let cfg_builder = self .network .network_config(config, self.chain.clone(), secret_key, default_peers_path) @@ -741,7 +785,7 @@ impl NodeConfig { .sequencer_endpoint(self.rollup.sequencer_http.clone()) .disable_tx_gossip(self.rollup.disable_txpool_gossip); - cfg_builder.build(ProviderFactory::new(db, self.chain.clone())) + cfg_builder.build(provider_factory) } #[allow(clippy::too_many_arguments)] @@ -859,7 +903,7 @@ impl NodeConfig { fn build_pruner( &self, config: &PruneConfig, - db: DB, + provider_factory: ProviderFactory, tree_config: BlockchainTreeConfig, highest_snapshots_rx: HighestSnapshotsTracker, ) -> Pruner { @@ -893,8 +937,7 @@ impl NodeConfig { ); Pruner::new( - db, - self.chain.clone(), + provider_factory, segments.into_vec(), config.block_interval, self.chain.prune_delete_limit, @@ -950,7 +993,8 @@ pub struct NodeBuilderWithDatabase { pub data_dir: ChainPath, } -impl NodeBuilderWithDatabase { +impl NodeBuilderWithDatabase { + /// Launch the node with the given extensions and executor pub async fn launch( mut self, mut ext: E::Node, @@ -1005,104 +1049,86 @@ impl NodeBuilderWithDatabase { .or(config.prune.clone()); // configure blockchain tree - let tree_externals = TreeExternals::new( - provider_factory.clone(), - Arc::clone(&consensus), - EvmProcessorFactory::new(self.chain.clone()), - ); let tree_config = BlockchainTreeConfig::default(); - let tree = BlockchainTree::new( - tree_externals, + let tree = self.config.build_blockchain_tree( + provider_factory.clone(), + consensus.clone(), + prune_config.clone(), + sync_metrics_tx.clone(), tree_config, - prune_config.clone().map(|config| config.segments), - )? - .with_sync_metrics_tx(sync_metrics_tx.clone()); + )?; let canon_state_notification_sender = tree.canon_state_notification_sender(); let blockchain_tree = ShareableBlockchainTree::new(tree); debug!(target: "reth::cli", "configured blockchain tree"); // fetch the head block from the database - let head = self.lookup_head(Arc::clone(&db)).wrap_err("the head block is missing")?; + let head = self + .config + .lookup_head(provider_factory.clone()) + .wrap_err("the head block is missing")?; // setup the blockchain provider let blockchain_db = BlockchainProvider::new(provider_factory.clone(), blockchain_tree.clone())?; - let transaction_pool = self.config.build_and_spawn_txpool(blockchain_db, head, executor)?; - info!(target: "reth::cli", "Connecting to P2P network"); - let network_secret_path = - self.network.p2p_secret_key.clone().unwrap_or_else(|| data_dir.p2p_secret_path()); - debug!(target: "reth::cli", ?network_secret_path, "Loading p2p key file"); - let secret_key = get_secret_key(&network_secret_path)?; - let default_peers_path = data_dir.known_peers_path(); - let network_config = self.load_network_config( - &config, - Arc::clone(&db), - ctx.task_executor.clone(), - head, - secret_key, - default_peers_path.clone(), - ); + // build transaction pool + let transaction_pool = + self.config.build_and_spawn_txpool(&blockchain_db, head, &executor)?; - let network_client = network_config.client.clone(); - let mut network_builder = NetworkManager::builder(network_config).await?; + // build network + info!(target: "reth::cli", "Connecting to P2P network"); + let (network_client, mut network_builder) = self + .config + .build_network( + &config, + provider_factory.clone(), + executor.clone(), + head, + &self.data_dir, + ) + .await?; let components = RethNodeComponentsImpl { provider: blockchain_db.clone(), pool: transaction_pool.clone(), network: network_builder.handle(), - task_executor: ctx.task_executor.clone(), + task_executor: executor.clone(), events: blockchain_db.clone(), }; // allow network modifications - self.ext.configure_network(network_builder.network_mut(), &components)?; + ext.configure_network(network_builder.network_mut(), &components)?; // launch network - let network = self.start_network( + let network = self.config.start_network( network_builder, - &ctx.task_executor, + &executor, transaction_pool.clone(), network_client, - default_peers_path, + &self.data_dir, ); info!(target: "reth::cli", peer_id = %network.peer_id(), local_addr = %network.local_addr(), enode = %network.local_node_record(), "Connected to P2P network"); debug!(target: "reth::cli", peer_id = ?network.peer_id(), "Full peer ID"); let network_client = network.fetch_client().await?; - self.ext.on_components_initialized(&components)?; + ext.on_components_initialized(&components)?; debug!(target: "reth::cli", "Spawning payload builder service"); - let payload_builder = self.ext.spawn_payload_builder_service(&self.builder, &components)?; + let payload_builder = + ext.spawn_payload_builder_service(&self.config.builder, &components)?; let (consensus_engine_tx, consensus_engine_rx) = unbounded_channel(); - let max_block = if let Some(block) = self.debug.max_block { - Some(block) - } else if let Some(tip) = self.debug.tip { - Some(self.lookup_or_fetch_tip(&db, &network_client, tip).await?) - } else { - None - }; + let max_block = self.config.max_block(&network_client, provider_factory.clone()).await?; // Configure the pipeline - let (mut pipeline, client) = if self.dev.dev { + let (mut pipeline, client) = if self.config.dev.dev { info!(target: "reth::cli", "Starting Reth in dev mode"); - - let mining_mode = if let Some(interval) = self.dev.block_time { - MiningMode::interval(interval) - } else if let Some(max_transactions) = self.dev.block_max_transactions { - MiningMode::instant( - max_transactions, - transaction_pool.pending_transactions_listener(), - ) - } else { - info!(target: "reth::cli", "No mining mode specified, defaulting to ReadyTransaction"); - MiningMode::instant(1, transaction_pool.pending_transactions_listener()) - }; + let mining_mode = + self.config.mining_mode(transaction_pool.pending_transactions_listener()); let (_, client, mut task) = AutoSealBuilder::new( - Arc::clone(&self.chain), + Arc::clone(&self.config.chain), blockchain_db.clone(), transaction_pool.clone(), consensus_engine_tx.clone(), @@ -1112,12 +1138,13 @@ impl NodeBuilderWithDatabase { .build(); let mut pipeline = self + .config .build_networked_pipeline( &config.stages, client.clone(), Arc::clone(&consensus), - provider_factory, - &ctx.task_executor, + provider_factory.clone(), + &executor, sync_metrics_tx, prune_config.clone(), max_block, @@ -1127,7 +1154,7 @@ impl NodeBuilderWithDatabase { let pipeline_events = pipeline.events(); task.set_pipeline_events(pipeline_events); debug!(target: "reth::cli", "Spawning auto mine task"); - ctx.task_executor.spawn(Box::pin(task)); + executor.spawn(Box::pin(task)); (pipeline, EitherDownloader::Left(client)) } else { @@ -1137,8 +1164,8 @@ impl NodeBuilderWithDatabase { &config.stages, network_client.clone(), Arc::clone(&consensus), - provider_factory, - &ctx.task_executor, + provider_factory.clone(), + &executor.clone(), sync_metrics_tx, prune_config.clone(), max_block, @@ -1150,31 +1177,19 @@ impl NodeBuilderWithDatabase { let pipeline_events = pipeline.events(); - let initial_target = if let Some(tip) = self.config.debug.tip { - // Set the provided tip as the initial pipeline target. - debug!(target: "reth::cli", %tip, "Tip manually set"); - Some(tip) - } else if self.config.debug.continuous { - // Set genesis as the initial pipeline target. - // This will allow the downloader to start - debug!(target: "reth::cli", "Continuous sync mode enabled"); - Some(genesis_hash) - } else { - None - }; - + let initial_target = self.config.initial_pipeline_target(genesis_hash); let mut hooks = EngineHooks::new(); let pruner_events = if let Some(prune_config) = prune_config { let mut pruner = self.config.build_pruner( &prune_config, - db.clone(), + provider_factory.clone(), tree_config, snapshotter.highest_snapshot_receiver(), ); let events = pruner.events(); - hooks.add(PruneHook::new(pruner, Box::new(ctx.task_executor.clone()))); + hooks.add(PruneHook::new(pruner, Box::new(executor.clone()))); info!(target: "reth::cli", ?prune_config, "Pruner initialized"); Either::Left(events) @@ -1187,10 +1202,10 @@ impl NodeBuilderWithDatabase { client, pipeline, blockchain_db.clone(), - Box::new(ctx.task_executor.clone()), + Box::new(executor.clone()), Box::new(network.clone()), max_block, - self.debug.continuous, + self.config.debug.continuous, payload_builder.clone(), initial_target, MIN_BLOCKS_FOR_PIPELINE_RUN, @@ -1204,7 +1219,7 @@ impl NodeBuilderWithDatabase { network.event_listener().map(Into::into), beacon_engine_handle.event_listener().map(Into::into), pipeline_events.map(Into::into), - if self.debug.tip.is_none() { + if self.config.debug.tip.is_none() { Either::Left( ConsensusLayerHealthEvents::new(Box::new(blockchain_db.clone())) .map(Into::into), @@ -1214,40 +1229,45 @@ impl NodeBuilderWithDatabase { }, pruner_events.map(Into::into) ); - ctx.task_executor.spawn_critical( + executor.spawn_critical( "events task", - events::handle_events(Some(network.clone()), Some(head.number), events, db.clone()), + events::handle_events( + Some(network.clone()), + Some(head.number), + events, + self.database.clone(), + ), ); let engine_api = EngineApi::new( blockchain_db.clone(), - self.chain.clone(), + self.config.chain.clone(), beacon_engine_handle, payload_builder.into(), - Box::new(ctx.task_executor.clone()), + Box::new(executor.clone()), ); info!(target: "reth::cli", "Engine API handler initialized"); // extract the jwt secret from the args if possible - let default_jwt_path = data_dir.jwt_path(); - let jwt_secret = self.rpc.auth_jwt_secret(default_jwt_path)?; + let default_jwt_path = self.data_dir.jwt_path(); + let jwt_secret = self.config.rpc.auth_jwt_secret(default_jwt_path)?; // adjust rpc port numbers based on instance number - self.adjust_instance_ports(); + self.config.adjust_instance_ports(); // Start RPC servers - let _rpc_server_handles = - self.rpc.start_servers(&components, engine_api, jwt_secret, &mut self.ext).await?; + let rpc_server_handles = + self.config.rpc.start_servers(&components, engine_api, jwt_secret, &mut ext).await?; // Run consensus engine to completion let (tx, rx) = oneshot::channel(); info!(target: "reth::cli", "Starting consensus engine"); - ctx.task_executor.spawn_critical_blocking("consensus engine", async move { + executor.spawn_critical_blocking("consensus engine", async move { let res = beacon_consensus_engine.await; let _ = tx.send(res); }); - self.ext.on_node_started(&components)?; + ext.on_node_started(&components)?; // If `enable_genesis_walkback` is set to true, the rollup client will need to // perform the derivation pipeline from genesis, validating the data dir. @@ -1271,8 +1291,8 @@ impl NodeBuilderWithDatabase { } // construct node handle and return - let _node_handle = NodeHandle { _rpc_server_handles, _consensus_engine_rx: rx }; - return Ok(_node_handle); + let node_handle = NodeHandle { rpc_server_handles, consensus_engine_rx: rx }; + Ok(node_handle) // rx.await??; @@ -1296,14 +1316,25 @@ impl NodeBuilderWithDatabase { #[derive(Debug)] pub struct NodeHandle { /// The handles to the RPC servers - _rpc_server_handles: RethRpcServerHandles, - /// The receiver half of the channel for the consensus engine - _consensus_engine_rx: oneshot::Receiver>, + rpc_server_handles: RethRpcServerHandles, + + /// The receiver half of the channel for the consensus engine. + /// This can be used to wait for the consensus engine to exit. + consensus_engine_rx: oneshot::Receiver>, } -/// The node service -#[derive(Debug)] -pub struct NodeService; +impl NodeHandle { + /// Returns the [RethRpcServerHandles] for this node. + pub fn rpc_server_handles(&self) -> &RethRpcServerHandles { + &self.rpc_server_handles + } + + /// Waits for the consensus engine to exit. + pub async fn wait_for_consensus_exit(self) -> eyre::Result<()> { + self.consensus_engine_rx.await??; + Ok(()) + } +} /// A simple function to launch a node with the specified [NodeConfig], spawning tasks on the /// [TaskExecutor] constructed from [tokio_runtime]. @@ -1316,11 +1347,21 @@ pub async fn spawn_node(config: NodeConfig) -> eyre::Result { #[cfg(test)] mod tests { + use reth_primitives::U256; + use reth_rpc_api::EthApiClient; + use super::*; #[tokio::test] - async fn test_node_config() { - // we need to override the db - let _handle = spawn_node(NodeConfig::default()).await; + async fn block_number_node_config_test() { + // this launches a test node + let handle = spawn_node(NodeConfig::test()).await.unwrap(); + + // call a function on the node + let client = handle.rpc_server_handles().rpc.http_client().unwrap(); + let block_number = client.block_number().await.unwrap(); + + // it should be zero, since this is an ephemeral test node + assert_eq!(block_number, U256::ZERO); } } From 363ecb2102bba4b0339132f31913d142f06c47f8 Mon Sep 17 00:00:00 2001 From: Dan Cline <6798349+Rjected@users.noreply.github.com> Date: Mon, 18 Dec 2023 20:47:08 +0200 Subject: [PATCH 20/33] make node config work --- bin/reth/src/args/rpc_server_args.rs | 22 + bin/reth/src/cli/db_type.rs | 45 +- bin/reth/src/cli/node_config.rs | 370 ++++++------ bin/reth/src/node/mod.rs | 861 ++------------------------- 4 files changed, 305 insertions(+), 993 deletions(-) diff --git a/bin/reth/src/args/rpc_server_args.rs b/bin/reth/src/args/rpc_server_args.rs index 54f0f040fda0..26b4b7b9cce5 100644 --- a/bin/reth/src/args/rpc_server_args.rs +++ b/bin/reth/src/args/rpc_server_args.rs @@ -179,6 +179,28 @@ pub struct RpcServerArgs { } impl RpcServerArgs { + /// Enables the HTTP-RPC server. + pub fn with_http(mut self) -> Self { + self.http = true; + self + } + + /// Enables the WS-RPC server. + pub fn with_ws(mut self) -> Self { + self.ws = true; + self + } + + /// Change rpc port numbers based on the instance number. + pub fn adjust_instance_ports(&mut self, instance: u16) { + // auth port is scaled by a factor of instance * 100 + self.auth_port += instance * 100 - 100; + // http port is scaled by a factor of -instance + self.http_port -= instance - 1; + // ws port is scaled by a factor of instance * 2 + self.ws_port += instance * 2 - 2; + } + /// Configures and launches _all_ servers. /// /// Returns the handles for the launched regular RPC server(s) (if any) and the server handle diff --git a/bin/reth/src/cli/db_type.rs b/bin/reth/src/cli/db_type.rs index bc9d27460aa3..5f1229be58b4 100644 --- a/bin/reth/src/cli/db_type.rs +++ b/bin/reth/src/cli/db_type.rs @@ -1,7 +1,6 @@ //! A real or test database type -use std::sync::Arc; -use crate::dirs::{DataDirPath, MaybePlatformPath}; +use crate::dirs::{ChainPath, DataDirPath, MaybePlatformPath}; use reth_db::{ init_db, test_utils::{create_test_rw_db, TempDatabase}, @@ -9,6 +8,7 @@ use reth_db::{ }; use reth_interfaces::db::LogLevel; use reth_primitives::Chain; +use std::{str::FromStr, sync::Arc}; /// A type that represents either a _real_ (represented by a path), or _test_ database, which will /// use a [TempDatabase]. @@ -55,12 +55,21 @@ impl DatabaseBuilder { chain: Chain, ) -> eyre::Result { match self.db_type { - DatabaseType::Test => Ok(DatabaseInstance::Test(create_test_rw_db())), + DatabaseType::Test => { + let db = create_test_rw_db(); + let db_path_str = db.path().to_str().expect("Path is not valid unicode"); + let path = MaybePlatformPath::::from_str(db_path_str) + .expect("Path is not valid"); + let data_dir = path.unwrap_or_chain_default(chain); + + Ok(DatabaseInstance::Test { db, data_dir }) + } DatabaseType::Real(path) => { - let chain_dir = path.unwrap_or_chain_default(chain); + let data_dir = path.unwrap_or_chain_default(chain); - tracing::info!(target: "reth::cli", path = ?chain_dir, "Opening database"); - Ok(DatabaseInstance::Real(Arc::new(init_db(chain_dir, log_level)?))) + tracing::info!(target: "reth::cli", path = ?data_dir, "Opening database"); + let db = Arc::new(init_db(data_dir.clone(), log_level)?); + Ok(DatabaseInstance::Real { db, data_dir }) } } } @@ -70,7 +79,27 @@ impl DatabaseBuilder { #[derive(Debug, Clone)] pub enum DatabaseInstance { /// The test database - Test(Arc>), + Test { + /// The database + db: Arc>, + /// The data dir + data_dir: ChainPath, + }, /// The right database - Real(Arc), + Real { + /// The database + db: Arc, + /// The data dir + data_dir: ChainPath, + }, +} + +impl DatabaseInstance { + /// Returns the data dir for this database instance + pub fn data_dir(&self) -> &ChainPath { + match self { + Self::Test { data_dir, .. } => data_dir, + Self::Real { data_dir, .. } => data_dir, + } + } } diff --git a/bin/reth/src/cli/node_config.rs b/bin/reth/src/cli/node_config.rs index 466ca0ffdd86..a26d52ba661a 100644 --- a/bin/reth/src/cli/node_config.rs +++ b/bin/reth/src/cli/node_config.rs @@ -17,7 +17,6 @@ use crate::{ init::init_genesis, node::{cl_events::ConsensusLayerHealthEvents, events, run_network_until_shutdown}, prometheus_exporter, - runner::tokio_runtime, utils::get_single_header, version::SHORT_VERSION, }; @@ -95,9 +94,12 @@ use std::{ path::PathBuf, sync::Arc, }; -use tokio::sync::{ - mpsc::{unbounded_channel, Receiver, UnboundedSender}, - oneshot, watch, +use tokio::{ + runtime::Handle, + sync::{ + mpsc::{unbounded_channel, Receiver, UnboundedSender}, + oneshot, watch, + }, }; use tracing::*; @@ -199,6 +201,139 @@ impl NodeConfig { } } + /// Set the datadir for the node + pub fn with_datadir(mut self, datadir: MaybePlatformPath) -> Self { + self.database = DatabaseType::Real(datadir); + self + } + + /// Set the config file for the node + pub fn with_config(mut self, config: impl Into) -> Self { + self.config = Some(config.into()); + self + } + + /// Set the chain for the node + pub fn with_chain(mut self, chain: Arc) -> Self { + self.chain = chain; + self + } + + /// Set the metrics address for the node + pub fn with_metrics(mut self, metrics: SocketAddr) -> Self { + self.metrics = Some(metrics); + self + } + + /// Set the instance for the node + pub fn with_instance(mut self, instance: u16) -> Self { + self.instance = instance; + self + } + + /// Set the [Chain] for the node + pub fn with_chain_spec(mut self, chain: Arc) -> Self { + self.chain = chain; + self + } + + /// Set the trusted setup file for the node + pub fn with_trusted_setup_file(mut self, trusted_setup_file: impl Into) -> Self { + self.trusted_setup_file = Some(trusted_setup_file.into()); + self + } + + /// Set the network args for the node + pub fn with_network(mut self, network: NetworkArgs) -> Self { + self.network = network; + self + } + + /// Set the rpc args for the node + pub fn with_rpc(mut self, rpc: RpcServerArgs) -> Self { + self.rpc = rpc; + self + } + + /// Set the txpool args for the node + pub fn with_txpool(mut self, txpool: TxPoolArgs) -> Self { + self.txpool = txpool; + self + } + + /// Set the builder args for the node + pub fn with_builder(mut self, builder: PayloadBuilderArgs) -> Self { + self.builder = builder; + self + } + + /// Set the debug args for the node + pub fn with_debug(mut self, debug: DebugArgs) -> Self { + self.debug = debug; + self + } + + /// Set the database args for the node + pub fn with_db(mut self, db: DatabaseArgs) -> Self { + self.db = db; + self + } + + /// Set the dev args for the node + pub fn with_dev(mut self, dev: DevArgs) -> Self { + self.dev = dev; + self + } + + /// Set the pruning args for the node + pub fn with_pruning(mut self, pruning: PruningArgs) -> Self { + self.pruning = pruning; + self + } + + /// Set the rollup args for the node + #[cfg(feature = "optimism")] + pub fn with_rollup(mut self, rollup: crate::args::RollupArgs) -> Self { + self.rollup = rollup; + self + } + + /// Launches the node, also adding any RPC extensions passed. + /// + /// # Example + /// ```rust + /// # use reth_tasks::TaskManager; + /// fn t() { + /// use reth_tasks::TaskSpawner; + /// let rt = tokio::runtime::Runtime::new().unwrap(); + /// let manager = TaskManager::new(rt.handle().clone()); + /// let executor = manager.executor(); + /// let config = NodeConfig::default(); + /// let ext = DefaultRethNodeCommandConfig; + /// let handle = config.launch(ext, executor); + /// } + /// ``` + pub async fn launch( + mut self, + ext: E::Node, + executor: TaskExecutor, + ) -> eyre::Result { + let database = std::mem::take(&mut self.database); + let db_instance = + DatabaseBuilder::new(database).build_db(self.db.log_level, self.chain.chain)?; + + match db_instance { + DatabaseInstance::Real { db, data_dir } => { + let builder = NodeBuilderWithDatabase { config: self, db, data_dir }; + builder.launch::(ext, executor).await + } + DatabaseInstance::Test { db, data_dir } => { + let builder = NodeBuilderWithDatabase { config: self, db, data_dir }; + builder.launch::(ext, executor).await + } + } + } + /// Get the network secret from the given data dir pub fn network_secret(&self, data_dir: &ChainPath) -> eyre::Result { let network_secret_path = @@ -370,141 +505,6 @@ impl NodeConfig { Ok(transaction_pool) } - /// Launches the node, also adding any RPC extensions passed. - /// - /// # Example - /// ```rust - /// # use reth_tasks::TaskManager; - /// fn t() { - /// use reth_tasks::TaskSpawner; - /// let rt = tokio::runtime::Runtime::new().unwrap(); - /// let manager = TaskManager::new(rt.handle().clone()); - /// let executor = manager.executor(); - /// let config = NodeConfig::default(); - /// let ext = DefaultRethNodeCommandConfig; - /// let handle = config.launch(ext, executor); - /// } - /// ``` - pub async fn launch( - mut self, - ext: E::Node, - executor: TaskExecutor, - ) -> eyre::Result { - let data_dir = self.data_dir().expect("see below"); - - let database = std::mem::take(&mut self.database); - let db_instance = - DatabaseBuilder::new(database).build_db(self.db.log_level, self.chain.chain)?; - - match db_instance { - DatabaseInstance::Real(database) => { - let builder = NodeBuilderWithDatabase { config: self, database, data_dir }; - builder.launch::(ext, executor).await - } - DatabaseInstance::Test(database) => { - let builder = NodeBuilderWithDatabase { config: self, database, data_dir }; - builder.launch::(ext, executor).await - } - } - } - - /// Set the datadir for the node - pub fn datadir(mut self, datadir: MaybePlatformPath) -> Self { - self.database = DatabaseType::Real(datadir); - self - } - - /// Set the config file for the node - pub fn config(mut self, config: impl Into) -> Self { - self.config = Some(config.into()); - self - } - - /// Set the chain for the node - pub fn chain(mut self, chain: Arc) -> Self { - self.chain = chain; - self - } - - /// Set the metrics address for the node - pub fn metrics(mut self, metrics: SocketAddr) -> Self { - self.metrics = Some(metrics); - self - } - - /// Set the instance for the node - pub fn instance(mut self, instance: u16) -> Self { - self.instance = instance; - self - } - - /// Set the [Chain] for the node - pub fn chain_spec(mut self, chain: Arc) -> Self { - self.chain = chain; - self - } - - /// Set the trusted setup file for the node - pub fn trusted_setup_file(mut self, trusted_setup_file: impl Into) -> Self { - self.trusted_setup_file = Some(trusted_setup_file.into()); - self - } - - /// Set the network args for the node - pub fn network(mut self, network: NetworkArgs) -> Self { - self.network = network; - self - } - - /// Set the rpc args for the node - pub fn rpc(mut self, rpc: RpcServerArgs) -> Self { - self.rpc = rpc; - self - } - - /// Set the txpool args for the node - pub fn txpool(mut self, txpool: TxPoolArgs) -> Self { - self.txpool = txpool; - self - } - - /// Set the builder args for the node - pub fn builder(mut self, builder: PayloadBuilderArgs) -> Self { - self.builder = builder; - self - } - - /// Set the debug args for the node - pub fn debug(mut self, debug: DebugArgs) -> Self { - self.debug = debug; - self - } - - /// Set the database args for the node - pub fn db(mut self, db: DatabaseArgs) -> Self { - self.db = db; - self - } - - /// Set the dev args for the node - pub fn dev(mut self, dev: DevArgs) -> Self { - self.dev = dev; - self - } - - /// Set the pruning args for the node - pub fn pruning(mut self, pruning: PruningArgs) -> Self { - self.pruning = pruning; - self - } - - /// Set the rollup args for the node - #[cfg(feature = "optimism")] - pub fn rollup(mut self, rollup: crate::args::RollupArgs) -> Self { - self.rollup = rollup; - self - } - /// Returns the [Consensus] instance to use. /// /// By default this will be a [BeaconConsensus] instance, but if the `--dev` flag is set, it @@ -535,11 +535,11 @@ impl NodeConfig { Client: HeadersClient + BodiesClient + Clone + 'static, { // building network downloaders using the fetch client - let header_downloader = ReverseHeadersDownloaderBuilder::from(config.headers) + let header_downloader = ReverseHeadersDownloaderBuilder::new(config.headers) .build(client.clone(), Arc::clone(&consensus)) .into_task_with(task_executor); - let body_downloader = BodiesDownloaderBuilder::from(config.bodies) + let body_downloader = BodiesDownloaderBuilder::new(config.bodies) .build(client, Arc::clone(&consensus), provider_factory.clone()) .into_task_with(task_executor); @@ -946,14 +946,10 @@ impl NodeConfig { ) } - /// Change rpc port numbers based on the instance number. + /// Change rpc port numbers based on the instance number, using the inner + /// [RpcServerArgs::adjust_instance_ports] method. fn adjust_instance_ports(&mut self) { - // auth port is scaled by a factor of instance * 100 - self.rpc.auth_port += self.instance * 100 - 100; - // http port is scaled by a factor of -instance - self.rpc.http_port -= self.instance - 1; - // ws port is scaled by a factor of instance * 2 - self.rpc.ws_port += self.instance * 2 - 2; + self.rpc.adjust_instance_ports(self.instance); } } @@ -988,7 +984,7 @@ pub struct NodeBuilderWithDatabase { /// The node config pub config: NodeConfig, /// The database - pub database: Arc, + pub db: Arc, /// The data dir pub data_dir: ChainPath, } @@ -1013,7 +1009,7 @@ impl NodeBuilderWit info!(target: "reth::cli", "Database opened"); let mut provider_factory = - ProviderFactory::new(Arc::clone(&self.database), Arc::clone(&self.config.chain)); + ProviderFactory::new(Arc::clone(&self.db), Arc::clone(&self.config.chain)); // configure snapshotter let snapshotter = reth_snapshot::Snapshotter::new( @@ -1027,11 +1023,11 @@ impl NodeBuilderWit snapshotter.highest_snapshot_receiver(), ); - self.config.start_metrics_endpoint(prometheus_handle, Arc::clone(&self.database)).await?; + self.config.start_metrics_endpoint(prometheus_handle, Arc::clone(&self.db)).await?; debug!(target: "reth::cli", chain=%self.config.chain.chain, genesis=?self.config.chain.genesis_hash(), "Initializing genesis"); - let genesis_hash = init_genesis(Arc::clone(&self.database), self.config.chain.clone())?; + let genesis_hash = init_genesis(Arc::clone(&self.db), self.config.chain.clone())?; info!(target: "reth::cli", "{}", DisplayHardforks::new(self.config.chain.hardforks())); @@ -1235,7 +1231,7 @@ impl NodeBuilderWit Some(network.clone()), Some(head.number), events, - self.database.clone(), + self.db.clone(), ), ); @@ -1277,7 +1273,7 @@ impl NodeBuilderWit // starts syncing from the current tip in the DB. #[cfg(feature = "optimism")] if self.chain.is_optimism() && !self.rollup.enable_genesis_walkback { - let client = _rpc_server_handles.auth.http_client(); + let client = rpc_server_handles.auth.http_client(); reth_rpc_api::EngineApiClient::fork_choice_updated_v2( &client, reth_rpc_types::engine::ForkchoiceState { @@ -1291,28 +1287,17 @@ impl NodeBuilderWit } // construct node handle and return - let node_handle = NodeHandle { rpc_server_handles, consensus_engine_rx: rx }; + let node_handle = NodeHandle { + rpc_server_handles, + consensus_engine_rx: rx, + terminate: self.config.debug.terminate, + }; Ok(node_handle) - - // rx.await??; - - // info!(target: "reth::cli", "Consensus engine has exited."); - - // if self.debug.terminate { - // Ok(()) - // } else { - // // The pipeline has finished downloading blocks up to `--debug.tip` or - // // `--debug.max-block`. Keep other node components alive for further usage. - // futures::future::pending().await - // } } } -/// The node handle -// We need from each component on init: -// * channels (for wiring into other components) -// * handles (for wiring into other components) -// * also for giving to the NodeHandle, for example everything rpc +/// The [NodeHandle] contains the [RethRpcServerHandles] returned by the reth initialization +/// process, as well as a method for waiting for the node exit. #[derive(Debug)] pub struct NodeHandle { /// The handles to the RPC servers @@ -1321,6 +1306,9 @@ pub struct NodeHandle { /// The receiver half of the channel for the consensus engine. /// This can be used to wait for the consensus engine to exit. consensus_engine_rx: oneshot::Receiver>, + + /// Flag indicating whether the node should be terminated after the pipeline sync. + terminate: bool, } impl NodeHandle { @@ -1329,33 +1317,41 @@ impl NodeHandle { &self.rpc_server_handles } - /// Waits for the consensus engine to exit. - pub async fn wait_for_consensus_exit(self) -> eyre::Result<()> { + /// Waits for the node to exit, if it was configured to exit. + pub async fn wait_for_node_exit(self) -> eyre::Result<()> { self.consensus_engine_rx.await??; - Ok(()) + info!(target: "reth::cli", "Consensus engine has exited."); + + if self.terminate { + Ok(()) + } else { + // The pipeline has finished downloading blocks up to `--debug.tip` or + // `--debug.max-block`. Keep other node components alive for further usage. + futures::future::pending().await + } } } /// A simple function to launch a node with the specified [NodeConfig], spawning tasks on the -/// [TaskExecutor] constructed from [tokio_runtime]. +/// [TaskExecutor] constructed from [Handle::current]. pub async fn spawn_node(config: NodeConfig) -> eyre::Result { - let runtime = tokio_runtime()?; - let task_manager = TaskManager::new(runtime.handle().clone()); + let handle = Handle::current(); + let task_manager = TaskManager::new(handle); let ext = DefaultRethNodeCommandConfig; config.launch::<()>(ext, task_manager.executor()).await } #[cfg(test)] mod tests { + use super::*; use reth_primitives::U256; use reth_rpc_api::EthApiClient; - use super::*; - #[tokio::test] async fn block_number_node_config_test() { - // this launches a test node - let handle = spawn_node(NodeConfig::test()).await.unwrap(); + // this launches a test node with http + let rpc_args = RpcServerArgs::default().with_http(); + let handle = spawn_node(NodeConfig::test().with_rpc(rpc_args)).await.unwrap(); // call a function on the node let client = handle.rpc_server_handles().rpc.http_client().unwrap(); @@ -1364,4 +1360,14 @@ mod tests { // it should be zero, since this is an ephemeral test node assert_eq!(block_number, U256::ZERO); } + + #[tokio::test] + async fn rpc_handles_none_without_http() { + // this launches a test node _without_ http + let handle = spawn_node(NodeConfig::test()).await.unwrap(); + + // ensure that the `http_client` is none + let maybe_client = handle.rpc_server_handles().rpc.http_client(); + assert!(maybe_client.is_none()); + } } diff --git a/bin/reth/src/node/mod.rs b/bin/reth/src/node/mod.rs index 7bcc3a415257..1db8b8accb69 100644 --- a/bin/reth/src/node/mod.rs +++ b/bin/reth/src/node/mod.rs @@ -3,90 +3,24 @@ //! Starts the client use crate::{ args::{ - get_secret_key, utils::{chain_help, genesis_value_parser, parse_socket_address, SUPPORTED_CHAINS}, DatabaseArgs, DebugArgs, DevArgs, NetworkArgs, PayloadBuilderArgs, PruningArgs, RpcServerArgs, TxPoolArgs, }, - cli::{ - components::RethNodeComponentsImpl, - config::{RethRpcConfig, RethTransactionPoolConfig}, - ext::{RethCliExt, RethNodeCommandConfig}, - }, - dirs::{ChainPath, DataDirPath, MaybePlatformPath}, - init::init_genesis, - node::cl_events::ConsensusLayerHealthEvents, - prometheus_exporter, + cli::{db_type::DatabaseType, ext::RethCliExt, node_config::NodeConfig}, + dirs::{DataDirPath, MaybePlatformPath}, runner::CliContext, - utils::get_single_header, version::SHORT_VERSION, }; use clap::{value_parser, Parser}; -use eyre::Context; -use futures::{future::Either, pin_mut, stream, stream_select, StreamExt}; -use metrics_exporter_prometheus::PrometheusHandle; -use reth_auto_seal_consensus::{AutoSealBuilder, AutoSealConsensus, MiningMode}; -use reth_beacon_consensus::{ - hooks::{EngineHooks, PruneHook}, - BeaconConsensus, BeaconConsensusEngine, MIN_BLOCKS_FOR_PIPELINE_RUN, -}; -use reth_blockchain_tree::{ - config::BlockchainTreeConfig, externals::TreeExternals, BlockchainTree, ShareableBlockchainTree, -}; -use reth_config::{ - config::{PruneConfig, StageConfig}, - Config, -}; -use reth_db::{database::Database, database_metrics::DatabaseMetrics, init_db}; -use reth_downloaders::{ - bodies::bodies::BodiesDownloaderBuilder, - headers::reverse_headers::ReverseHeadersDownloaderBuilder, -}; -use reth_interfaces::{ - consensus::Consensus, - p2p::{ - bodies::{client::BodiesClient, downloader::BodyDownloader}, - either::EitherDownloader, - headers::{client::HeadersClient, downloader::HeaderDownloader}, - }, - RethResult, -}; -use reth_network::{NetworkBuilder, NetworkConfig, NetworkEvents, NetworkHandle, NetworkManager}; -use reth_network_api::{NetworkInfo, PeersInfo}; -use reth_primitives::{ - constants::eip4844::{LoadKzgSettingsError, MAINNET_KZG_TRUSTED_SETUP}, - fs, - kzg::KzgSettings, - stage::StageId, - BlockHashOrNumber, BlockNumber, ChainSpec, DisplayHardforks, Head, SealedHeader, B256, -}; -use reth_provider::{ - providers::BlockchainProvider, BlockHashReader, BlockReader, CanonStateSubscriptions, - HeaderProvider, HeaderSyncMode, ProviderFactory, StageCheckpointReader, -}; -use reth_prune::PrunerBuilder; -use reth_revm::EvmProcessorFactory; -use reth_revm_inspectors::stack::Hook; -use reth_rpc_engine_api::EngineApi; -use reth_stages::{ - prelude::*, - stages::{ - AccountHashingStage, ExecutionStage, ExecutionStageThresholds, IndexAccountHistoryStage, - IndexStorageHistoryStage, MerkleStage, SenderRecoveryStage, StorageHashingStage, - TotalDifficultyStage, TransactionLookupStage, - }, -}; -use reth_tasks::TaskExecutor; -use reth_transaction_pool::{ - blobstore::InMemoryBlobStore, TransactionPool, TransactionValidationTaskExecutor, -}; -use secp256k1::SecretKey; -use std::{ - net::{SocketAddr, SocketAddrV4}, - path::PathBuf, - sync::Arc, -}; -use tokio::sync::{mpsc::unbounded_channel, oneshot, watch}; +use futures::pin_mut; +use reth_auto_seal_consensus::AutoSealConsensus; +use reth_beacon_consensus::BeaconConsensus; +use reth_interfaces::consensus::Consensus; +use reth_network::NetworkManager; +use reth_primitives::ChainSpec; +use reth_provider::{BlockReader, HeaderProvider}; +use std::{net::SocketAddr, path::PathBuf, sync::Arc}; use tracing::*; pub mod cl_events; @@ -236,357 +170,59 @@ impl NodeCommand { } /// Execute `node` command - pub async fn execute(mut self, ctx: CliContext) -> eyre::Result<()> { + pub async fn execute(self, ctx: CliContext) -> eyre::Result<()> { info!(target: "reth::cli", "reth {} starting", SHORT_VERSION); - // Raise the fd limit of the process. - // Does not do anything on windows. - let _ = fdlimit::raise_fd_limit(); - - // get config - let config = self.load_config()?; - - let prometheus_handle = self.install_prometheus_recorder()?; - - let data_dir = self.data_dir(); - let db_path = data_dir.db_path(); - - info!(target: "reth::cli", path = ?db_path, "Opening database"); - let db = Arc::new(init_db(&db_path, self.db.log_level)?.with_metrics()); - info!(target: "reth::cli", "Database opened"); - - let mut provider_factory = ProviderFactory::new(Arc::clone(&db), Arc::clone(&self.chain)); - - // configure snapshotter - let snapshotter = reth_snapshot::Snapshotter::new( - provider_factory.clone(), - data_dir.snapshots_path(), - self.chain.snapshot_block_interval, - )?; - - provider_factory = provider_factory - .with_snapshots(data_dir.snapshots_path(), snapshotter.highest_snapshot_receiver())?; - - self.start_metrics_endpoint(prometheus_handle, Arc::clone(&db)).await?; - - debug!(target: "reth::cli", chain=%self.chain.chain, genesis=?self.chain.genesis_hash(), "Initializing genesis"); - - let genesis_hash = init_genesis(Arc::clone(&db), self.chain.clone())?; - - info!(target: "reth::cli", "{}", DisplayHardforks::new(self.chain.hardforks())); - - let consensus = self.consensus(); - - debug!(target: "reth::cli", "Spawning stages metrics listener task"); - let (sync_metrics_tx, sync_metrics_rx) = unbounded_channel(); - let sync_metrics_listener = reth_stages::MetricsListener::new(sync_metrics_rx); - ctx.task_executor.spawn_critical("stages metrics listener task", sync_metrics_listener); - - let prune_config = - self.pruning.prune_config(Arc::clone(&self.chain))?.or(config.prune.clone()); - - // configure blockchain tree - let tree_externals = TreeExternals::new( - provider_factory.clone(), - Arc::clone(&consensus), - EvmProcessorFactory::new(self.chain.clone()), - ); - let tree_config = BlockchainTreeConfig::default(); - let tree = BlockchainTree::new( - tree_externals, - tree_config, - prune_config.clone().map(|config| config.segments), - )? - .with_sync_metrics_tx(sync_metrics_tx.clone()); - let canon_state_notification_sender = tree.canon_state_notification_sender(); - let blockchain_tree = ShareableBlockchainTree::new(tree); - debug!(target: "reth::cli", "configured blockchain tree"); - - // fetch the head block from the database - let head = - self.lookup_head(provider_factory.clone()).wrap_err("the head block is missing")?; - - // setup the blockchain provider - let blockchain_db = - BlockchainProvider::new(provider_factory.clone(), blockchain_tree.clone())?; - let blob_store = InMemoryBlobStore::default(); - let validator = TransactionValidationTaskExecutor::eth_builder(Arc::clone(&self.chain)) - .with_head_timestamp(head.timestamp) - .kzg_settings(self.kzg_settings()?) - .with_additional_tasks(1) - .build_with_tasks(blockchain_db.clone(), ctx.task_executor.clone(), blob_store.clone()); - - let transaction_pool = - reth_transaction_pool::Pool::eth_pool(validator, blob_store, self.txpool.pool_config()); - info!(target: "reth::cli", "Transaction pool initialized"); - - // spawn txpool maintenance task - { - let pool = transaction_pool.clone(); - let chain_events = blockchain_db.canonical_state_stream(); - let client = blockchain_db.clone(); - ctx.task_executor.spawn_critical( - "txpool maintenance task", - reth_transaction_pool::maintain::maintain_transaction_pool_future( - client, - pool, - chain_events, - ctx.task_executor.clone(), - Default::default(), - ), - ); - debug!(target: "reth::cli", "Spawned txpool maintenance task"); - } - - info!(target: "reth::cli", "Connecting to P2P network"); - let network_secret_path = - self.network.p2p_secret_key.clone().unwrap_or_else(|| data_dir.p2p_secret_path()); - debug!(target: "reth::cli", ?network_secret_path, "Loading p2p key file"); - let secret_key = get_secret_key(&network_secret_path)?; - let default_peers_path = data_dir.known_peers_path(); - let network_config = self.load_network_config( - &config, - provider_factory.clone(), - ctx.task_executor.clone(), - head, - secret_key, - default_peers_path.clone(), - ); - - let network_client = network_config.client.clone(); - let mut network_builder = NetworkManager::builder(network_config).await?; - - let components = RethNodeComponentsImpl { - provider: blockchain_db.clone(), - pool: transaction_pool.clone(), - network: network_builder.handle(), - task_executor: ctx.task_executor.clone(), - events: blockchain_db.clone(), - }; - - // allow network modifications - self.ext.configure_network(network_builder.network_mut(), &components)?; - - // launch network - let network = self.start_network( - network_builder, - &ctx.task_executor, - transaction_pool.clone(), - network_client, - default_peers_path, - ); - - info!(target: "reth::cli", peer_id = %network.peer_id(), local_addr = %network.local_addr(), enode = %network.local_node_record(), "Connected to P2P network"); - debug!(target: "reth::cli", peer_id = ?network.peer_id(), "Full peer ID"); - let network_client = network.fetch_client().await?; - - self.ext.on_components_initialized(&components)?; - - debug!(target: "reth::cli", "Spawning payload builder service"); - let payload_builder = self.ext.spawn_payload_builder_service(&self.builder, &components)?; - - let (consensus_engine_tx, consensus_engine_rx) = unbounded_channel(); - let max_block = if let Some(block) = self.debug.max_block { - Some(block) - } else if let Some(tip) = self.debug.tip { - Some(self.lookup_or_fetch_tip(provider_factory.clone(), &network_client, tip).await?) - } else { - None - }; - - // Configure the pipeline - let (mut pipeline, client) = if self.dev.dev { - info!(target: "reth::cli", "Starting Reth in dev mode"); - - let mining_mode = if let Some(interval) = self.dev.block_time { - MiningMode::interval(interval) - } else if let Some(max_transactions) = self.dev.block_max_transactions { - MiningMode::instant( - max_transactions, - transaction_pool.pending_transactions_listener(), - ) - } else { - info!(target: "reth::cli", "No mining mode specified, defaulting to ReadyTransaction"); - MiningMode::instant(1, transaction_pool.pending_transactions_listener()) - }; - - let (_, client, mut task) = AutoSealBuilder::new( - Arc::clone(&self.chain), - blockchain_db.clone(), - transaction_pool.clone(), - consensus_engine_tx.clone(), - canon_state_notification_sender, - mining_mode, - ) - .build(); - - let mut pipeline = self - .build_networked_pipeline( - &config.stages, - client.clone(), - Arc::clone(&consensus), - provider_factory.clone(), - &ctx.task_executor, - sync_metrics_tx, - prune_config.clone(), - max_block, - ) - .await?; - - let pipeline_events = pipeline.events(); - task.set_pipeline_events(pipeline_events); - debug!(target: "reth::cli", "Spawning auto mine task"); - ctx.task_executor.spawn(Box::pin(task)); - - (pipeline, EitherDownloader::Left(client)) - } else { - let pipeline = self - .build_networked_pipeline( - &config.stages, - network_client.clone(), - Arc::clone(&consensus), - provider_factory.clone(), - &ctx.task_executor, - sync_metrics_tx, - prune_config.clone(), - max_block, - ) - .await?; - - (pipeline, EitherDownloader::Right(network_client)) - }; - - let pipeline_events = pipeline.events(); - - let initial_target = if let Some(tip) = self.debug.tip { - // Set the provided tip as the initial pipeline target. - debug!(target: "reth::cli", %tip, "Tip manually set"); - Some(tip) - } else if self.debug.continuous { - // Set genesis as the initial pipeline target. - // This will allow the downloader to start - debug!(target: "reth::cli", "Continuous sync mode enabled"); - Some(genesis_hash) - } else { - None - }; - - let mut hooks = EngineHooks::new(); - - let pruner_events = if let Some(prune_config) = prune_config { - let mut pruner = PrunerBuilder::new(prune_config.clone()) - .max_reorg_depth(tree_config.max_reorg_depth() as usize) - .prune_delete_limit(self.chain.prune_delete_limit) - .build(provider_factory, snapshotter.highest_snapshot_receiver()); + let Self { + datadir, + config, + chain, + metrics, + trusted_setup_file, + instance, + network, + rpc, + txpool, + builder, + debug, + db, + dev, + pruning, + #[cfg(feature = "optimism")] + rollup, + ext, + } = self; - let events = pruner.events(); - hooks.add(PruneHook::new(pruner, Box::new(ctx.task_executor.clone()))); + // set up real database + let database = DatabaseType::Real(datadir); - info!(target: "reth::cli", ?prune_config, "Pruner initialized"); - Either::Left(events) - } else { - Either::Right(stream::empty()) + // set up node config + let node_config = NodeConfig { + database, + config, + chain, + metrics, + instance, + trusted_setup_file, + network, + rpc, + txpool, + builder, + debug, + db, + dev, + pruning, + #[cfg(feature = "optimism")] + rollup, }; - // Configure the consensus engine - let (beacon_consensus_engine, beacon_engine_handle) = BeaconConsensusEngine::with_channel( - client, - pipeline, - blockchain_db.clone(), - Box::new(ctx.task_executor.clone()), - Box::new(network.clone()), - max_block, - self.debug.continuous, - payload_builder.clone(), - initial_target, - MIN_BLOCKS_FOR_PIPELINE_RUN, - consensus_engine_tx, - consensus_engine_rx, - hooks, - )?; - info!(target: "reth::cli", "Consensus engine initialized"); - - let events = stream_select!( - network.event_listener().map(Into::into), - beacon_engine_handle.event_listener().map(Into::into), - pipeline_events.map(Into::into), - if self.debug.tip.is_none() { - Either::Left( - ConsensusLayerHealthEvents::new(Box::new(blockchain_db.clone())) - .map(Into::into), - ) - } else { - Either::Right(stream::empty()) - }, - pruner_events.map(Into::into) - ); - ctx.task_executor.spawn_critical( - "events task", - events::handle_events(Some(network.clone()), Some(head.number), events, db.clone()), - ); - - let engine_api = EngineApi::new( - blockchain_db.clone(), - self.chain.clone(), - beacon_engine_handle, - payload_builder.into(), - Box::new(ctx.task_executor.clone()), - ); - info!(target: "reth::cli", "Engine API handler initialized"); + let executor = ctx.task_executor; - // extract the jwt secret from the args if possible - let default_jwt_path = data_dir.jwt_path(); - let jwt_secret = self.rpc.auth_jwt_secret(default_jwt_path)?; + // launch the node + let handle = node_config.launch::(ext, executor).await?; - // adjust rpc port numbers based on instance number - self.adjust_instance_ports(); - - // Start RPC servers - let _rpc_server_handles = - self.rpc.start_servers(&components, engine_api, jwt_secret, &mut self.ext).await?; - - // Run consensus engine to completion - let (tx, rx) = oneshot::channel(); - info!(target: "reth::cli", "Starting consensus engine"); - ctx.task_executor.spawn_critical_blocking("consensus engine", async move { - let res = beacon_consensus_engine.await; - let _ = tx.send(res); - }); - - self.ext.on_node_started(&components)?; - - // If `enable_genesis_walkback` is set to true, the rollup client will need to - // perform the derivation pipeline from genesis, validating the data dir. - // When set to false, set the finalized, safe, and unsafe head block hashes - // on the rollup client using a fork choice update. This prevents the rollup - // client from performing the derivation pipeline from genesis, and instead - // starts syncing from the current tip in the DB. - #[cfg(feature = "optimism")] - if self.chain.is_optimism() && !self.rollup.enable_genesis_walkback { - let client = _rpc_server_handles.auth.http_client(); - reth_rpc_api::EngineApiClient::fork_choice_updated_v2( - &client, - reth_rpc_types::engine::ForkchoiceState { - head_block_hash: head.hash, - safe_block_hash: head.hash, - finalized_block_hash: head.hash, - }, - None, - ) - .await?; - } - - rx.await??; - - info!(target: "reth::cli", "Consensus engine has exited."); - - if self.debug.terminate { - Ok(()) - } else { - // The pipeline has finished downloading blocks up to `--debug.tip` or - // `--debug.max-block`. Keep other node components alive for further usage. - futures::future::pending().await - } + // wait for node exit + handle.wait_for_node_exit().await } /// Returns the [Consensus] instance to use. @@ -600,387 +236,6 @@ impl NodeCommand { Arc::new(BeaconConsensus::new(Arc::clone(&self.chain))) } } - - /// Constructs a [Pipeline] that's wired to the network - #[allow(clippy::too_many_arguments)] - async fn build_networked_pipeline( - &self, - config: &StageConfig, - client: Client, - consensus: Arc, - provider_factory: ProviderFactory, - task_executor: &TaskExecutor, - metrics_tx: reth_stages::MetricEventsSender, - prune_config: Option, - max_block: Option, - ) -> eyre::Result> - where - DB: Database + Unpin + Clone + 'static, - Client: HeadersClient + BodiesClient + Clone + 'static, - { - // building network downloaders using the fetch client - let header_downloader = ReverseHeadersDownloaderBuilder::new(config.headers) - .build(client.clone(), Arc::clone(&consensus)) - .into_task_with(task_executor); - - let body_downloader = BodiesDownloaderBuilder::new(config.bodies) - .build(client, Arc::clone(&consensus), provider_factory.clone()) - .into_task_with(task_executor); - - let pipeline = self - .build_pipeline( - provider_factory, - config, - header_downloader, - body_downloader, - consensus, - max_block, - self.debug.continuous, - metrics_tx, - prune_config, - ) - .await?; - - Ok(pipeline) - } - - /// Returns the chain specific path to the data dir. - fn data_dir(&self) -> ChainPath { - self.datadir.unwrap_or_chain_default(self.chain.chain) - } - - /// Returns the path to the config file. - fn config_path(&self) -> PathBuf { - self.config.clone().unwrap_or_else(|| self.data_dir().config_path()) - } - - /// Loads the reth config with the given datadir root - fn load_config(&self) -> eyre::Result { - let config_path = self.config_path(); - let mut config = confy::load_path::(&config_path) - .wrap_err_with(|| format!("Could not load config file {:?}", config_path))?; - - info!(target: "reth::cli", path = ?config_path, "Configuration loaded"); - - // Update the config with the command line arguments - config.peers.connect_trusted_nodes_only = self.network.trusted_only; - - if !self.network.trusted_peers.is_empty() { - info!(target: "reth::cli", "Adding trusted nodes"); - self.network.trusted_peers.iter().for_each(|peer| { - config.peers.trusted_nodes.insert(*peer); - }); - } - - Ok(config) - } - - /// Loads the trusted setup params from a given file path or falls back to - /// `MAINNET_KZG_TRUSTED_SETUP`. - fn kzg_settings(&self) -> eyre::Result> { - if let Some(ref trusted_setup_file) = self.trusted_setup_file { - let trusted_setup = KzgSettings::load_trusted_setup_file(trusted_setup_file) - .map_err(LoadKzgSettingsError::KzgError)?; - Ok(Arc::new(trusted_setup)) - } else { - Ok(Arc::clone(&MAINNET_KZG_TRUSTED_SETUP)) - } - } - - fn install_prometheus_recorder(&self) -> eyre::Result { - prometheus_exporter::install_recorder() - } - - async fn start_metrics_endpoint( - &self, - prometheus_handle: PrometheusHandle, - db: Metrics, - ) -> eyre::Result<()> - where - Metrics: DatabaseMetrics + 'static + Send + Sync, - { - if let Some(listen_addr) = self.metrics { - info!(target: "reth::cli", addr = %listen_addr, "Starting metrics endpoint"); - prometheus_exporter::serve( - listen_addr, - prometheus_handle, - db, - metrics_process::Collector::default(), - ) - .await?; - } - - Ok(()) - } - - /// Spawns the configured network and associated tasks and returns the [NetworkHandle] connected - /// to that network. - fn start_network( - &self, - builder: NetworkBuilder, - task_executor: &TaskExecutor, - pool: Pool, - client: C, - default_peers_path: PathBuf, - ) -> NetworkHandle - where - C: BlockReader + HeaderProvider + Clone + Unpin + 'static, - Pool: TransactionPool + Unpin + 'static, - { - let (handle, network, txpool, eth) = - builder.transactions(pool).request_handler(client).split_with_handle(); - - task_executor.spawn_critical("p2p txpool", txpool); - task_executor.spawn_critical("p2p eth request handler", eth); - - let known_peers_file = self.network.persistent_peers_file(default_peers_path); - task_executor - .spawn_critical_with_graceful_shutdown_signal("p2p network task", |shutdown| { - run_network_until_shutdown(shutdown, network, known_peers_file) - }); - - handle - } - - /// Fetches the head block from the database. - /// - /// If the database is empty, returns the genesis block. - fn lookup_head(&self, factory: ProviderFactory) -> RethResult { - let provider = factory.provider()?; - - let head = provider.get_stage_checkpoint(StageId::Finish)?.unwrap_or_default().block_number; - - let header = provider - .header_by_number(head)? - .expect("the header for the latest block is missing, database is corrupt"); - - let total_difficulty = provider - .header_td_by_number(head)? - .expect("the total difficulty for the latest block is missing, database is corrupt"); - - let hash = provider - .block_hash(head)? - .expect("the hash for the latest block is missing, database is corrupt"); - - Ok(Head { - number: head, - hash, - difficulty: header.difficulty, - total_difficulty, - timestamp: header.timestamp, - }) - } - - /// Attempt to look up the block number for the tip hash in the database. - /// If it doesn't exist, download the header and return the block number. - /// - /// NOTE: The download is attempted with infinite retries. - async fn lookup_or_fetch_tip( - &self, - provider_factory: ProviderFactory, - client: Client, - tip: B256, - ) -> RethResult - where - DB: Database, - Client: HeadersClient, - { - Ok(self.fetch_tip(provider_factory, client, BlockHashOrNumber::Hash(tip)).await?.number) - } - - /// Attempt to look up the block with the given number and return the header. - /// - /// NOTE: The download is attempted with infinite retries. - async fn fetch_tip( - &self, - factory: ProviderFactory, - client: Client, - tip: BlockHashOrNumber, - ) -> RethResult - where - DB: Database, - Client: HeadersClient, - { - let provider = factory.provider()?; - - let header = provider.header_by_hash_or_number(tip)?; - - // try to look up the header in the database - if let Some(header) = header { - info!(target: "reth::cli", ?tip, "Successfully looked up tip block in the database"); - return Ok(header.seal_slow()) - } - - info!(target: "reth::cli", ?tip, "Fetching tip block from the network."); - loop { - match get_single_header(&client, tip).await { - Ok(tip_header) => { - info!(target: "reth::cli", ?tip, "Successfully fetched tip"); - return Ok(tip_header) - } - Err(error) => { - error!(target: "reth::cli", %error, "Failed to fetch the tip. Retrying..."); - } - } - } - } - - fn load_network_config( - &self, - config: &Config, - provider_factory: ProviderFactory, - executor: TaskExecutor, - head: Head, - secret_key: SecretKey, - default_peers_path: PathBuf, - ) -> NetworkConfig> { - let cfg_builder = self - .network - .network_config(config, self.chain.clone(), secret_key, default_peers_path) - .with_task_executor(Box::new(executor)) - .set_head(head) - .listener_addr(SocketAddr::V4(SocketAddrV4::new( - self.network.addr, - // set discovery port based on instance number - self.network.port + self.instance - 1, - ))) - .discovery_addr(SocketAddr::V4(SocketAddrV4::new( - self.network.addr, - // set discovery port based on instance number - self.network.port + self.instance - 1, - ))); - - // When `sequencer_endpoint` is configured, the node will forward all transactions to a - // Sequencer node for execution and inclusion on L1, and disable its own txpool - // gossip to prevent other parties in the network from learning about them. - #[cfg(feature = "optimism")] - let cfg_builder = cfg_builder - .sequencer_endpoint(self.rollup.sequencer_http.clone()) - .disable_tx_gossip(self.rollup.disable_txpool_gossip); - - cfg_builder.build(provider_factory) - } - - #[allow(clippy::too_many_arguments)] - async fn build_pipeline( - &self, - provider_factory: ProviderFactory, - config: &StageConfig, - header_downloader: H, - body_downloader: B, - consensus: Arc, - max_block: Option, - continuous: bool, - metrics_tx: reth_stages::MetricEventsSender, - prune_config: Option, - ) -> eyre::Result> - where - DB: Database + Clone + 'static, - H: HeaderDownloader + 'static, - B: BodyDownloader + 'static, - { - let mut builder = Pipeline::builder(); - - if let Some(max_block) = max_block { - debug!(target: "reth::cli", max_block, "Configuring builder to use max block"); - builder = builder.with_max_block(max_block) - } - - let (tip_tx, tip_rx) = watch::channel(B256::ZERO); - use reth_revm_inspectors::stack::InspectorStackConfig; - let factory = reth_revm::EvmProcessorFactory::new(self.chain.clone()); - - let stack_config = InspectorStackConfig { - use_printer_tracer: self.debug.print_inspector, - hook: if let Some(hook_block) = self.debug.hook_block { - Hook::Block(hook_block) - } else if let Some(tx) = self.debug.hook_transaction { - Hook::Transaction(tx) - } else if self.debug.hook_all { - Hook::All - } else { - Hook::None - }, - }; - - let factory = factory.with_stack_config(stack_config); - - let prune_modes = prune_config.map(|prune| prune.segments).unwrap_or_default(); - - let header_mode = - if continuous { HeaderSyncMode::Continuous } else { HeaderSyncMode::Tip(tip_rx) }; - let pipeline = builder - .with_tip_sender(tip_tx) - .with_metrics_tx(metrics_tx.clone()) - .add_stages( - DefaultStages::new( - provider_factory.clone(), - header_mode, - Arc::clone(&consensus), - header_downloader, - body_downloader, - factory.clone(), - ) - .set( - TotalDifficultyStage::new(consensus) - .with_commit_threshold(config.total_difficulty.commit_threshold), - ) - .set(SenderRecoveryStage { - commit_threshold: config.sender_recovery.commit_threshold, - }) - .set( - ExecutionStage::new( - factory, - ExecutionStageThresholds { - max_blocks: config.execution.max_blocks, - max_changes: config.execution.max_changes, - max_cumulative_gas: config.execution.max_cumulative_gas, - }, - config - .merkle - .clean_threshold - .max(config.account_hashing.clean_threshold) - .max(config.storage_hashing.clean_threshold), - prune_modes.clone(), - ) - .with_metrics_tx(metrics_tx), - ) - .set(AccountHashingStage::new( - config.account_hashing.clean_threshold, - config.account_hashing.commit_threshold, - )) - .set(StorageHashingStage::new( - config.storage_hashing.clean_threshold, - config.storage_hashing.commit_threshold, - )) - .set(MerkleStage::new_execution(config.merkle.clean_threshold)) - .set(TransactionLookupStage::new( - config.transaction_lookup.commit_threshold, - prune_modes.transaction_lookup, - )) - .set(IndexAccountHistoryStage::new( - config.index_account_history.commit_threshold, - prune_modes.account_history, - )) - .set(IndexStorageHistoryStage::new( - config.index_storage_history.commit_threshold, - prune_modes.storage_history, - )), - ) - .build(provider_factory); - - Ok(pipeline) - } - - /// Change rpc port numbers based on the instance number. - fn adjust_instance_ports(&mut self) { - // auth port is scaled by a factor of instance * 100 - self.rpc.auth_port += self.instance * 100 - 100; - // http port is scaled by a factor of -instance - self.rpc.http_port -= self.instance - 1; - // ws port is scaled by a factor of instance * 2 - self.rpc.ws_port += self.instance * 2 - 2; - } } /// Drives the [NetworkManager] future until a [Shutdown](reth_tasks::shutdown::Shutdown) signal is @@ -1149,7 +404,7 @@ mod tests { #[test] fn parse_instance() { let mut cmd = NodeCommand::<()>::parse_from(["reth"]); - cmd.adjust_instance_ports(); + cmd.rpc.adjust_instance_ports(cmd.instance); cmd.network.port = DEFAULT_DISCOVERY_PORT + cmd.instance - 1; // check rpc port numbers assert_eq!(cmd.rpc.auth_port, 8551); @@ -1159,7 +414,7 @@ mod tests { assert_eq!(cmd.network.port, 30303); let mut cmd = NodeCommand::<()>::parse_from(["reth", "--instance", "2"]); - cmd.adjust_instance_ports(); + cmd.rpc.adjust_instance_ports(cmd.instance); cmd.network.port = DEFAULT_DISCOVERY_PORT + cmd.instance - 1; // check rpc port numbers assert_eq!(cmd.rpc.auth_port, 8651); @@ -1169,7 +424,7 @@ mod tests { assert_eq!(cmd.network.port, 30304); let mut cmd = NodeCommand::<()>::parse_from(["reth", "--instance", "3"]); - cmd.adjust_instance_ports(); + cmd.rpc.adjust_instance_ports(cmd.instance); cmd.network.port = DEFAULT_DISCOVERY_PORT + cmd.instance - 1; // check rpc port numbers assert_eq!(cmd.rpc.auth_port, 8751); From 7b413a93a6caf1dbbad1a1a7f5746153c1c4a67d Mon Sep 17 00:00:00 2001 From: Dan Cline <6798349+Rjected@users.noreply.github.com> Date: Mon, 18 Dec 2023 20:50:01 +0200 Subject: [PATCH 21/33] rename to NodeBuilder --- bin/reth/src/cli/node_config.rs | 16 ++++++++-------- bin/reth/src/node/mod.rs | 4 ++-- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/bin/reth/src/cli/node_config.rs b/bin/reth/src/cli/node_config.rs index a26d52ba661a..e921a9ffbad4 100644 --- a/bin/reth/src/cli/node_config.rs +++ b/bin/reth/src/cli/node_config.rs @@ -114,7 +114,7 @@ type RethTransactionPool = reth_transaction_pool::Pool< /// Start the node #[derive(Debug)] -pub struct NodeConfig { +pub struct NodeBuilder { /// The test database pub database: DatabaseType, @@ -178,7 +178,7 @@ pub struct NodeConfig { pub rollup: crate::args::RollupArgs, } -impl NodeConfig { +impl NodeBuilder { /// Creates a testing [NodeConfig], causing the database to be launched ephemerally. pub fn test() -> Self { Self { @@ -953,7 +953,7 @@ impl NodeConfig { } } -impl Default for NodeConfig { +impl Default for NodeBuilder { fn default() -> Self { Self { database: DatabaseType::default(), @@ -982,7 +982,7 @@ impl Default for NodeConfig { /// This also contains a path to a data dir that cannot be changed. pub struct NodeBuilderWithDatabase { /// The node config - pub config: NodeConfig, + pub config: NodeBuilder, /// The database pub db: Arc, /// The data dir @@ -1332,9 +1332,9 @@ impl NodeHandle { } } -/// A simple function to launch a node with the specified [NodeConfig], spawning tasks on the +/// A simple function to launch a node with the specified [NodeBuilder], spawning tasks on the /// [TaskExecutor] constructed from [Handle::current]. -pub async fn spawn_node(config: NodeConfig) -> eyre::Result { +pub async fn spawn_node(config: NodeBuilder) -> eyre::Result { let handle = Handle::current(); let task_manager = TaskManager::new(handle); let ext = DefaultRethNodeCommandConfig; @@ -1351,7 +1351,7 @@ mod tests { async fn block_number_node_config_test() { // this launches a test node with http let rpc_args = RpcServerArgs::default().with_http(); - let handle = spawn_node(NodeConfig::test().with_rpc(rpc_args)).await.unwrap(); + let handle = spawn_node(NodeBuilder::test().with_rpc(rpc_args)).await.unwrap(); // call a function on the node let client = handle.rpc_server_handles().rpc.http_client().unwrap(); @@ -1364,7 +1364,7 @@ mod tests { #[tokio::test] async fn rpc_handles_none_without_http() { // this launches a test node _without_ http - let handle = spawn_node(NodeConfig::test()).await.unwrap(); + let handle = spawn_node(NodeBuilder::test()).await.unwrap(); // ensure that the `http_client` is none let maybe_client = handle.rpc_server_handles().rpc.http_client(); diff --git a/bin/reth/src/node/mod.rs b/bin/reth/src/node/mod.rs index 1db8b8accb69..0b70db8e3c9b 100644 --- a/bin/reth/src/node/mod.rs +++ b/bin/reth/src/node/mod.rs @@ -7,7 +7,7 @@ use crate::{ DatabaseArgs, DebugArgs, DevArgs, NetworkArgs, PayloadBuilderArgs, PruningArgs, RpcServerArgs, TxPoolArgs, }, - cli::{db_type::DatabaseType, ext::RethCliExt, node_config::NodeConfig}, + cli::{db_type::DatabaseType, ext::RethCliExt, node_config::NodeBuilder}, dirs::{DataDirPath, MaybePlatformPath}, runner::CliContext, version::SHORT_VERSION, @@ -197,7 +197,7 @@ impl NodeCommand { let database = DatabaseType::Real(datadir); // set up node config - let node_config = NodeConfig { + let node_config = NodeBuilder { database, config, chain, From f3f9f51bc5d77a4a75d98d859c50fedb92810cac Mon Sep 17 00:00:00 2001 From: Dan Cline <6798349+Rjected@users.noreply.github.com> Date: Mon, 18 Dec 2023 20:51:28 +0200 Subject: [PATCH 22/33] move file to node_builder --- bin/reth/src/cli/mod.rs | 2 +- bin/reth/src/cli/{node_config.rs => node_builder.rs} | 0 bin/reth/src/node/mod.rs | 2 +- 3 files changed, 2 insertions(+), 2 deletions(-) rename bin/reth/src/cli/{node_config.rs => node_builder.rs} (100%) diff --git a/bin/reth/src/cli/mod.rs b/bin/reth/src/cli/mod.rs index 7957a1617c4f..0cd519433020 100644 --- a/bin/reth/src/cli/mod.rs +++ b/bin/reth/src/cli/mod.rs @@ -23,7 +23,7 @@ pub mod components; pub mod config; pub mod db_type; pub mod ext; -pub mod node_config; +pub mod node_builder; /// Default [directives](Directive) for [EnvFilter] which disables high-frequency debug logs from /// `hyper` and `trust-dns` diff --git a/bin/reth/src/cli/node_config.rs b/bin/reth/src/cli/node_builder.rs similarity index 100% rename from bin/reth/src/cli/node_config.rs rename to bin/reth/src/cli/node_builder.rs diff --git a/bin/reth/src/node/mod.rs b/bin/reth/src/node/mod.rs index 0b70db8e3c9b..bb3d2eae95fc 100644 --- a/bin/reth/src/node/mod.rs +++ b/bin/reth/src/node/mod.rs @@ -7,7 +7,7 @@ use crate::{ DatabaseArgs, DebugArgs, DevArgs, NetworkArgs, PayloadBuilderArgs, PruningArgs, RpcServerArgs, TxPoolArgs, }, - cli::{db_type::DatabaseType, ext::RethCliExt, node_config::NodeBuilder}, + cli::{db_type::DatabaseType, ext::RethCliExt, node_builder::NodeBuilder}, dirs::{DataDirPath, MaybePlatformPath}, runner::CliContext, version::SHORT_VERSION, From 354e86af49432cc0629cb72d3f2abc0b8928e6a4 Mon Sep 17 00:00:00 2001 From: Dan Cline <6798349+Rjected@users.noreply.github.com> Date: Mon, 18 Dec 2023 20:59:25 +0200 Subject: [PATCH 23/33] add with_instance_number --- bin/reth/src/cli/node_builder.rs | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/bin/reth/src/cli/node_builder.rs b/bin/reth/src/cli/node_builder.rs index e921a9ffbad4..9f47c447200c 100644 --- a/bin/reth/src/cli/node_builder.rs +++ b/bin/reth/src/cli/node_builder.rs @@ -291,6 +291,12 @@ impl NodeBuilder { self } + /// Set the node instance number + pub fn with_instance_number(mut self, instance: u16) -> Self { + self.instance = instance; + self + } + /// Set the rollup args for the node #[cfg(feature = "optimism")] pub fn with_rollup(mut self, rollup: crate::args::RollupArgs) -> Self { @@ -1370,4 +1376,12 @@ mod tests { let maybe_client = handle.rpc_server_handles().rpc.http_client(); assert!(maybe_client.is_none()); } + + #[tokio::test] + async fn launch_multiple_nodes() { + // need to fix subscriber thing + todo!(); + // let _first_handle = spawn_node(NodeBuilder::test()).await.unwrap(); + // let _second_handle = spawn_node(NodeBuilder::test().with_instance(1)).await.unwrap(); + } } From 17e975a7a52bf19ecbb59e0907b69d1706250812 Mon Sep 17 00:00:00 2001 From: Dan Cline <6798349+Rjected@users.noreply.github.com> Date: Wed, 20 Dec 2023 14:04:41 -0500 Subject: [PATCH 24/33] integrate PrunerBuilder --- bin/reth/src/cli/node_builder.rs | 64 ++++---------------------------- bin/reth/src/node/mod.rs | 2 +- 2 files changed, 8 insertions(+), 58 deletions(-) diff --git a/bin/reth/src/cli/node_builder.rs b/bin/reth/src/cli/node_builder.rs index 9f47c447200c..4bc31145d2f5 100644 --- a/bin/reth/src/cli/node_builder.rs +++ b/bin/reth/src/cli/node_builder.rs @@ -69,11 +69,10 @@ use reth_provider::{ BlockchainTreePendingStateProvider, CanonStateSubscriptions, HeaderProvider, HeaderSyncMode, ProviderFactory, StageCheckpointReader, }; -use reth_prune::{segments::SegmentSet, Pruner}; +use reth_prune::PrunerBuilder; use reth_revm::EvmProcessorFactory; use reth_revm_inspectors::stack::Hook; use reth_rpc_engine_api::EngineApi; -use reth_snapshot::HighestSnapshotsTracker; use reth_stages::{ prelude::*, stages::{ @@ -905,53 +904,6 @@ impl NodeBuilder { Ok(pipeline) } - /// Builds a [Pruner] with the given config. - fn build_pruner( - &self, - config: &PruneConfig, - provider_factory: ProviderFactory, - tree_config: BlockchainTreeConfig, - highest_snapshots_rx: HighestSnapshotsTracker, - ) -> Pruner { - let segments = SegmentSet::default() - // Receipts - .segment_opt(config.segments.receipts.map(reth_prune::segments::Receipts::new)) - // Receipts by logs - .segment_opt((!config.segments.receipts_log_filter.is_empty()).then(|| { - reth_prune::segments::ReceiptsByLogs::new( - config.segments.receipts_log_filter.clone(), - ) - })) - // Transaction lookup - .segment_opt( - config - .segments - .transaction_lookup - .map(reth_prune::segments::TransactionLookup::new), - ) - // Sender recovery - .segment_opt( - config.segments.sender_recovery.map(reth_prune::segments::SenderRecovery::new), - ) - // Account history - .segment_opt( - config.segments.account_history.map(reth_prune::segments::AccountHistory::new), - ) - // Storage history - .segment_opt( - config.segments.storage_history.map(reth_prune::segments::StorageHistory::new), - ); - - Pruner::new( - provider_factory, - segments.into_vec(), - config.block_interval, - self.chain.prune_delete_limit, - tree_config.max_reorg_depth() as usize, - highest_snapshots_rx, - ) - } - /// Change rpc port numbers based on the instance number, using the inner /// [RpcServerArgs::adjust_instance_ports] method. fn adjust_instance_ports(&mut self) { @@ -1006,7 +958,7 @@ impl NodeBuilderWit // Raise the fd limit of the process. // Does not do anything on windows. - raise_fd_limit(); + raise_fd_limit()?; // get config let config = self.config.load_config()?; @@ -1027,7 +979,7 @@ impl NodeBuilderWit provider_factory = provider_factory.with_snapshots( self.data_dir.snapshots_path(), snapshotter.highest_snapshot_receiver(), - ); + )?; self.config.start_metrics_endpoint(prometheus_handle, Arc::clone(&self.db)).await?; @@ -1183,12 +1135,10 @@ impl NodeBuilderWit let mut hooks = EngineHooks::new(); let pruner_events = if let Some(prune_config) = prune_config { - let mut pruner = self.config.build_pruner( - &prune_config, - provider_factory.clone(), - tree_config, - snapshotter.highest_snapshot_receiver(), - ); + let mut pruner = PrunerBuilder::new(prune_config.clone()) + .max_reorg_depth(tree_config.max_reorg_depth() as usize) + .prune_delete_limit(self.config.chain.prune_delete_limit) + .build(provider_factory, snapshotter.highest_snapshot_receiver()); let events = pruner.events(); hooks.add(PruneHook::new(pruner, Box::new(executor.clone()))); diff --git a/bin/reth/src/node/mod.rs b/bin/reth/src/node/mod.rs index bb3d2eae95fc..ec1a67857d84 100644 --- a/bin/reth/src/node/mod.rs +++ b/bin/reth/src/node/mod.rs @@ -20,7 +20,7 @@ use reth_interfaces::consensus::Consensus; use reth_network::NetworkManager; use reth_primitives::ChainSpec; use reth_provider::{BlockReader, HeaderProvider}; -use std::{net::SocketAddr, path::PathBuf, sync::Arc}; +use std::{fs, net::SocketAddr, path::PathBuf, sync::Arc}; use tracing::*; pub mod cl_events; From 0dac9bdb5761e5f5f71ab8f72efd2a1c7551d969 Mon Sep 17 00:00:00 2001 From: Dan Cline <6798349+Rjected@users.noreply.github.com> Date: Wed, 20 Dec 2023 14:29:04 -0500 Subject: [PATCH 25/33] fix docs --- bin/reth/src/cli/node_builder.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/bin/reth/src/cli/node_builder.rs b/bin/reth/src/cli/node_builder.rs index 4bc31145d2f5..1547f14efd29 100644 --- a/bin/reth/src/cli/node_builder.rs +++ b/bin/reth/src/cli/node_builder.rs @@ -178,7 +178,7 @@ pub struct NodeBuilder { } impl NodeBuilder { - /// Creates a testing [NodeConfig], causing the database to be launched ephemerally. + /// Creates a testing [NodeBuilder], causing the database to be launched ephemerally. pub fn test() -> Self { Self { database: DatabaseType::test(), @@ -230,7 +230,7 @@ impl NodeBuilder { self } - /// Set the [Chain] for the node + /// Set the [ChainSpec] for the node pub fn with_chain_spec(mut self, chain: Arc) -> Self { self.chain = chain; self @@ -313,7 +313,7 @@ impl NodeBuilder { /// let rt = tokio::runtime::Runtime::new().unwrap(); /// let manager = TaskManager::new(rt.handle().clone()); /// let executor = manager.executor(); - /// let config = NodeConfig::default(); + /// let config = NodeBuilder::default(); /// let ext = DefaultRethNodeCommandConfig; /// let handle = config.launch(ext, executor); /// } @@ -934,7 +934,7 @@ impl Default for NodeBuilder { } } -/// A version of the [NodeConfig] that has an installed database. This is used to construct the +/// A version of the [NodeBuilder] that has an installed database. This is used to construct the /// [NodeHandle]. /// /// This also contains a path to a data dir that cannot be changed. @@ -1228,7 +1228,7 @@ impl NodeBuilderWit // client from performing the derivation pipeline from genesis, and instead // starts syncing from the current tip in the DB. #[cfg(feature = "optimism")] - if self.chain.is_optimism() && !self.rollup.enable_genesis_walkback { + if self.config.chain.is_optimism() && !self.config.rollup.enable_genesis_walkback { let client = rpc_server_handles.auth.http_client(); reth_rpc_api::EngineApiClient::fork_choice_updated_v2( &client, From 89dc77a73ec973dbc2b8cb169d84c7bce2adebc7 Mon Sep 17 00:00:00 2001 From: Dan Cline <6798349+Rjected@users.noreply.github.com> Date: Thu, 21 Dec 2023 11:25:58 -0500 Subject: [PATCH 26/33] fix example --- bin/reth/src/cli/node_builder.rs | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/bin/reth/src/cli/node_builder.rs b/bin/reth/src/cli/node_builder.rs index 1547f14efd29..a5a139f4eee3 100644 --- a/bin/reth/src/cli/node_builder.rs +++ b/bin/reth/src/cli/node_builder.rs @@ -307,15 +307,20 @@ impl NodeBuilder { /// /// # Example /// ```rust - /// # use reth_tasks::TaskManager; - /// fn t() { - /// use reth_tasks::TaskSpawner; - /// let rt = tokio::runtime::Runtime::new().unwrap(); - /// let manager = TaskManager::new(rt.handle().clone()); + /// # use reth_tasks::{TaskManager, TaskSpawner}; + /// # use reth::cli::{ + /// # node_builder::NodeBuilder, + /// # ext::DefaultRethNodeCommandConfig, + /// # }; + /// # use tokio::runtime::Handle; + /// + /// async fn t() { + /// let handle = Handle::current(); + /// let manager = TaskManager::new(handle); /// let executor = manager.executor(); - /// let config = NodeBuilder::default(); - /// let ext = DefaultRethNodeCommandConfig; - /// let handle = config.launch(ext, executor); + /// let builder = NodeBuilder::default(); + /// let ext = DefaultRethNodeCommandConfig::default(); + /// let handle = builder.launch::<()>(ext, executor).await.unwrap(); /// } /// ``` pub async fn launch( From e2ca3905ccc4533f53e61e828e9173db3f2abb9e Mon Sep 17 00:00:00 2001 From: Dan Cline <6798349+Rjected@users.noreply.github.com> Date: Thu, 21 Dec 2023 13:03:43 -0500 Subject: [PATCH 27/33] add statics for instance num and prometheus recorder --- Cargo.lock | 1 + bin/reth/Cargo.toml | 1 + bin/reth/src/args/rpc_server_args.rs | 3 ++ bin/reth/src/cli/node_builder.rs | 56 +++++++++++++++++++--------- crates/storage/db/src/lib.rs | 10 ++++- 5 files changed, 51 insertions(+), 20 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index cf1100a8a4cf..aa91ab5243b0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5589,6 +5589,7 @@ dependencies = [ "metrics-exporter-prometheus", "metrics-process", "metrics-util", + "once_cell", "pin-project", "pretty_assertions", "procfs", diff --git a/bin/reth/Cargo.toml b/bin/reth/Cargo.toml index 9dfd694db948..1e9295b88ef0 100644 --- a/bin/reth/Cargo.toml +++ b/bin/reth/Cargo.toml @@ -73,6 +73,7 @@ metrics-util = "0.15.0" metrics-process = "1.0.9" reth-metrics.workspace = true metrics.workspace = true +once_cell.workspace = true # test vectors generation proptest.workspace = true diff --git a/bin/reth/src/args/rpc_server_args.rs b/bin/reth/src/args/rpc_server_args.rs index 26b4b7b9cce5..4a6e47c419ed 100644 --- a/bin/reth/src/args/rpc_server_args.rs +++ b/bin/reth/src/args/rpc_server_args.rs @@ -192,7 +192,10 @@ impl RpcServerArgs { } /// Change rpc port numbers based on the instance number. + /// + /// Warning: if `instance` is zero, this will panic. pub fn adjust_instance_ports(&mut self, instance: u16) { + debug_assert_ne!(instance, 0, "instance must be non-zero"); // auth port is scaled by a factor of instance * 100 self.auth_port += instance * 100 - 100; // http port is scaled by a factor of -instance diff --git a/bin/reth/src/cli/node_builder.rs b/bin/reth/src/cli/node_builder.rs index a5a139f4eee3..6120cf9f5874 100644 --- a/bin/reth/src/cli/node_builder.rs +++ b/bin/reth/src/cli/node_builder.rs @@ -24,6 +24,7 @@ use eyre::Context; use fdlimit::raise_fd_limit; use futures::{future::Either, stream, stream_select, StreamExt}; use metrics_exporter_prometheus::PrometheusHandle; +use once_cell::sync::Lazy; use reth_auto_seal_consensus::{AutoSealBuilder, AutoSealConsensus, MiningMode}; use reth_beacon_consensus::{ hooks::{EngineHooks, PruneHook}, @@ -84,8 +85,8 @@ use reth_stages::{ }; use reth_tasks::{TaskExecutor, TaskManager}; use reth_transaction_pool::{ - blobstore::InMemoryBlobStore, CoinbaseTipOrdering, EthPooledTransaction, - EthTransactionValidator, TransactionPool, TransactionValidationTaskExecutor, + blobstore::InMemoryBlobStore, EthTransactionPool, TransactionPool, + TransactionValidationTaskExecutor, }; use secp256k1::SecretKey; use std::{ @@ -102,14 +103,10 @@ use tokio::{ }; use tracing::*; -/// The default reth transaction pool type -type RethTransactionPool = reth_transaction_pool::Pool< - TransactionValidationTaskExecutor< - EthTransactionValidator, EthPooledTransaction>, - >, - CoinbaseTipOrdering, - InMemoryBlobStore, ->; +/// The default prometheus recorder handle. We use a global static to ensure that it is only +/// installed once. +pub static PROMETHEUS_RECORDER_HANDLE: Lazy = + Lazy::new(|| prometheus_exporter::install_recorder().unwrap()); /// Start the node #[derive(Debug)] @@ -474,7 +471,7 @@ impl NodeBuilder { blockchain_db: &BlockchainProvider, head: Head, executor: &TaskExecutor, - ) -> eyre::Result> + ) -> eyre::Result, InMemoryBlobStore>> where DB: Database + Unpin + Clone + 'static, Tree: BlockchainTreeEngine @@ -624,7 +621,7 @@ impl NodeBuilder { } fn install_prometheus_recorder(&self) -> eyre::Result { - prometheus_exporter::install_recorder() + Ok(PROMETHEUS_RECORDER_HANDLE.clone()) } async fn start_metrics_endpoint( @@ -1307,12 +1304,31 @@ mod tests { use super::*; use reth_primitives::U256; use reth_rpc_api::EthApiClient; + use std::sync::atomic::{AtomicU16, Ordering}; + + /// A simple static atomic used to assign instance numbers to test nodes. + static TEST_NODE_INSTANCE: AtomicU16 = AtomicU16::new(1); + + /// A method that ensures the node is spawned with a unique instance number. This calls + /// [spawn_node] under the hood, and just modifies the [NodeBuilder] to have a unique instance + /// number. + /// + /// IPC is also disabled by default. It is up to the test / caller to enable it and ensure that + /// the IPC path is unique. + async fn spawn_test_node(config: NodeBuilder) -> eyre::Result { + let instance = TEST_NODE_INSTANCE.fetch_add(1, Ordering::SeqCst); + let mut config = config.with_instance(instance); + + // disable ipc by default + config.rpc.ipcdisable = true; + spawn_node(config).await + } #[tokio::test] async fn block_number_node_config_test() { // this launches a test node with http let rpc_args = RpcServerArgs::default().with_http(); - let handle = spawn_node(NodeBuilder::test().with_rpc(rpc_args)).await.unwrap(); + let handle = spawn_test_node(NodeBuilder::test().with_rpc(rpc_args)).await.unwrap(); // call a function on the node let client = handle.rpc_server_handles().rpc.http_client().unwrap(); @@ -1325,7 +1341,7 @@ mod tests { #[tokio::test] async fn rpc_handles_none_without_http() { // this launches a test node _without_ http - let handle = spawn_node(NodeBuilder::test()).await.unwrap(); + let handle = spawn_test_node(NodeBuilder::test()).await.unwrap(); // ensure that the `http_client` is none let maybe_client = handle.rpc_server_handles().rpc.http_client(); @@ -1334,9 +1350,13 @@ mod tests { #[tokio::test] async fn launch_multiple_nodes() { - // need to fix subscriber thing - todo!(); - // let _first_handle = spawn_node(NodeBuilder::test()).await.unwrap(); - // let _second_handle = spawn_node(NodeBuilder::test().with_instance(1)).await.unwrap(); + // spawn_test_node takes roughly 1 second per node, so this test takes ~4 seconds + let num_nodes = 4; + + let mut handles = Vec::new(); + for _ in 0..num_nodes { + let handle = spawn_test_node(NodeBuilder::test()).await.unwrap(); + handles.push(handle); + } } } diff --git a/crates/storage/db/src/lib.rs b/crates/storage/db/src/lib.rs index 16ef51b4b24e..a57779d4f043 100644 --- a/crates/storage/db/src/lib.rs +++ b/crates/storage/db/src/lib.rs @@ -226,9 +226,15 @@ pub mod test_utils { } } + /// Get a temporary directory path to use for the database + pub fn tempdir_path() -> PathBuf { + let builder = tempfile::Builder::new().prefix("reth-test-").rand_bytes(8).tempdir(); + builder.expect(ERROR_TEMPDIR).into_path() + } + /// Create read/write database for testing pub fn create_test_rw_db() -> Arc> { - let path = tempfile::TempDir::new().expect(ERROR_TEMPDIR).into_path(); + let path = tempdir_path(); let emsg = format!("{}: {:?}", ERROR_DB_CREATION, path); let db = init_db(&path, None).expect(&emsg); @@ -245,7 +251,7 @@ pub mod test_utils { /// Create read only database for testing pub fn create_test_ro_db() -> Arc> { - let path = tempfile::TempDir::new().expect(ERROR_TEMPDIR).into_path(); + let path = tempdir_path(); { init_db(path.as_path(), None).expect(ERROR_DB_CREATION); } From 461b8a407cba0e0ddaa9a6485798f821cfc3df74 Mon Sep 17 00:00:00 2001 From: Dan Cline <6798349+Rjected@users.noreply.github.com> Date: Thu, 21 Dec 2023 13:29:53 -0500 Subject: [PATCH 28/33] set default subscriber and return guard instead of setting global default --- bin/reth/src/cli/mod.rs | 12 ++++++++---- crates/tracing/src/lib.rs | 8 +++++--- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/bin/reth/src/cli/mod.rs b/bin/reth/src/cli/mod.rs index 0cd519433020..e8d0fa7f2aa4 100644 --- a/bin/reth/src/cli/mod.rs +++ b/bin/reth/src/cli/mod.rs @@ -18,6 +18,7 @@ use reth_tracing::{ BoxedLayer, FileWorkerGuard, }; use std::{fmt, fmt::Display, sync::Arc}; +use tracing::subscriber::DefaultGuard; pub mod components; pub mod config; @@ -83,7 +84,7 @@ impl Cli { self.logs.log_file_directory = self.logs.log_file_directory.join(self.chain.chain.to_string()); - let _guard = self.init_tracing()?; + let _guards = self.init_tracing()?; let runner = CliRunner; match self.command { @@ -104,15 +105,18 @@ impl Cli { /// /// If file logging is enabled, this function returns a guard that must be kept alive to ensure /// that all logs are flushed to disk. - pub fn init_tracing(&self) -> eyre::Result> { + /// + /// This also returns a guard for the default tracing subscriber, which must be kept alive for + /// the subscriber to remain set. + pub fn init_tracing(&self) -> eyre::Result<(Option, DefaultGuard)> { let mut layers = vec![reth_tracing::stdout(self.verbosity.directive(), &self.logs.color.to_string())]; let (additional_layers, guard) = self.logs.layers()?; layers.extend(additional_layers); - reth_tracing::init(layers); - Ok(guard) + let default_guard = reth_tracing::init(layers); + Ok((guard, default_guard)) } /// Configures the given node extension. diff --git a/crates/tracing/src/lib.rs b/crates/tracing/src/lib.rs index 61481013ff44..1c2a9814de54 100644 --- a/crates/tracing/src/lib.rs +++ b/crates/tracing/src/lib.rs @@ -19,7 +19,7 @@ use rolling_file::{RollingConditionBasic, RollingFileAppender}; use std::path::Path; -use tracing::Subscriber; +use tracing::{subscriber::DefaultGuard, Subscriber}; use tracing_subscriber::{ filter::Directive, prelude::*, registry::LookupSpan, EnvFilter, Layer, Registry, }; @@ -32,8 +32,10 @@ pub use tracing_subscriber; pub type BoxedLayer = Box + Send + Sync>; /// Initializes a new [Subscriber] based on the given layers. -pub fn init(layers: Vec>) { - tracing_subscriber::registry().with(layers).init(); +pub fn init(layers: Vec>) -> DefaultGuard { + // Just set the default subscriber, relying on the caller to keep the default guard alive for + // the lifetime of the program. + tracing_subscriber::registry().with(layers).set_default() } /// Builds a new tracing layer that writes to stdout. From cc636231f3c1f41bc2642f94eed518dc490b66bf Mon Sep 17 00:00:00 2001 From: Dan Cline <6798349+Rjected@users.noreply.github.com> Date: Thu, 21 Dec 2023 14:33:26 -0500 Subject: [PATCH 29/33] use try_init instead of always using set_default --- bin/reth/src/cli/mod.rs | 12 ++++-------- crates/tracing/src/lib.rs | 9 ++++----- 2 files changed, 8 insertions(+), 13 deletions(-) diff --git a/bin/reth/src/cli/mod.rs b/bin/reth/src/cli/mod.rs index e8d0fa7f2aa4..0cd519433020 100644 --- a/bin/reth/src/cli/mod.rs +++ b/bin/reth/src/cli/mod.rs @@ -18,7 +18,6 @@ use reth_tracing::{ BoxedLayer, FileWorkerGuard, }; use std::{fmt, fmt::Display, sync::Arc}; -use tracing::subscriber::DefaultGuard; pub mod components; pub mod config; @@ -84,7 +83,7 @@ impl Cli { self.logs.log_file_directory = self.logs.log_file_directory.join(self.chain.chain.to_string()); - let _guards = self.init_tracing()?; + let _guard = self.init_tracing()?; let runner = CliRunner; match self.command { @@ -105,18 +104,15 @@ impl Cli { /// /// If file logging is enabled, this function returns a guard that must be kept alive to ensure /// that all logs are flushed to disk. - /// - /// This also returns a guard for the default tracing subscriber, which must be kept alive for - /// the subscriber to remain set. - pub fn init_tracing(&self) -> eyre::Result<(Option, DefaultGuard)> { + pub fn init_tracing(&self) -> eyre::Result> { let mut layers = vec![reth_tracing::stdout(self.verbosity.directive(), &self.logs.color.to_string())]; let (additional_layers, guard) = self.logs.layers()?; layers.extend(additional_layers); - let default_guard = reth_tracing::init(layers); - Ok((guard, default_guard)) + reth_tracing::init(layers); + Ok(guard) } /// Configures the given node extension. diff --git a/crates/tracing/src/lib.rs b/crates/tracing/src/lib.rs index 1c2a9814de54..2fc551a5f65e 100644 --- a/crates/tracing/src/lib.rs +++ b/crates/tracing/src/lib.rs @@ -19,7 +19,7 @@ use rolling_file::{RollingConditionBasic, RollingFileAppender}; use std::path::Path; -use tracing::{subscriber::DefaultGuard, Subscriber}; +use tracing::Subscriber; use tracing_subscriber::{ filter::Directive, prelude::*, registry::LookupSpan, EnvFilter, Layer, Registry, }; @@ -32,10 +32,9 @@ pub use tracing_subscriber; pub type BoxedLayer = Box + Send + Sync>; /// Initializes a new [Subscriber] based on the given layers. -pub fn init(layers: Vec>) -> DefaultGuard { - // Just set the default subscriber, relying on the caller to keep the default guard alive for - // the lifetime of the program. - tracing_subscriber::registry().with(layers).set_default() +pub fn init(layers: Vec>) { + // To avoid panicking in tests, we silently fail if we cannot initialize the subscriber. + let _ = tracing_subscriber::registry().with(layers).try_init(); } /// Builds a new tracing layer that writes to stdout. From 0779b10b4e2162f5c7f3f4651ef4a7e03ab68e32 Mon Sep 17 00:00:00 2001 From: Dan Cline <6798349+Rjected@users.noreply.github.com> Date: Thu, 21 Dec 2023 14:53:27 -0500 Subject: [PATCH 30/33] use manual instance nums --- bin/reth/src/cli/node_builder.rs | 34 +++++++++++--------------------- 1 file changed, 11 insertions(+), 23 deletions(-) diff --git a/bin/reth/src/cli/node_builder.rs b/bin/reth/src/cli/node_builder.rs index 6120cf9f5874..08c96f1445bc 100644 --- a/bin/reth/src/cli/node_builder.rs +++ b/bin/reth/src/cli/node_builder.rs @@ -1304,31 +1304,17 @@ mod tests { use super::*; use reth_primitives::U256; use reth_rpc_api::EthApiClient; - use std::sync::atomic::{AtomicU16, Ordering}; - - /// A simple static atomic used to assign instance numbers to test nodes. - static TEST_NODE_INSTANCE: AtomicU16 = AtomicU16::new(1); - - /// A method that ensures the node is spawned with a unique instance number. This calls - /// [spawn_node] under the hood, and just modifies the [NodeBuilder] to have a unique instance - /// number. - /// - /// IPC is also disabled by default. It is up to the test / caller to enable it and ensure that - /// the IPC path is unique. - async fn spawn_test_node(config: NodeBuilder) -> eyre::Result { - let instance = TEST_NODE_INSTANCE.fetch_add(1, Ordering::SeqCst); - let mut config = config.with_instance(instance); - - // disable ipc by default - config.rpc.ipcdisable = true; - spawn_node(config).await - } #[tokio::test] async fn block_number_node_config_test() { // this launches a test node with http let rpc_args = RpcServerArgs::default().with_http(); - let handle = spawn_test_node(NodeBuilder::test().with_rpc(rpc_args)).await.unwrap(); + + // NOTE: tests here manually set an instance number. The alternative would be to use an + // atomic counter. This works for `cargo test` but if tests would be run in `nextest` then + // they would become flaky. So new tests should manually set a unique instance number. + let handle = + spawn_node(NodeBuilder::test().with_rpc(rpc_args).with_instance(1)).await.unwrap(); // call a function on the node let client = handle.rpc_server_handles().rpc.http_client().unwrap(); @@ -1341,7 +1327,7 @@ mod tests { #[tokio::test] async fn rpc_handles_none_without_http() { // this launches a test node _without_ http - let handle = spawn_test_node(NodeBuilder::test()).await.unwrap(); + let handle = spawn_node(NodeBuilder::test().with_instance(2)).await.unwrap(); // ensure that the `http_client` is none let maybe_client = handle.rpc_server_handles().rpc.http_client(); @@ -1353,9 +1339,11 @@ mod tests { // spawn_test_node takes roughly 1 second per node, so this test takes ~4 seconds let num_nodes = 4; + let starting_instance = 3; let mut handles = Vec::new(); - for _ in 0..num_nodes { - let handle = spawn_test_node(NodeBuilder::test()).await.unwrap(); + for i in 0..num_nodes { + let handle = + spawn_node(NodeBuilder::test().with_instance(starting_instance + i)).await.unwrap(); handles.push(handle); } } From 484b0a4b51971f41c45dba27f43a848d5896e71c Mon Sep 17 00:00:00 2001 From: Dan Cline <6798349+Rjected@users.noreply.github.com> Date: Tue, 26 Dec 2023 13:50:24 -0500 Subject: [PATCH 31/33] add run_until_graceful_shutdown method with shutdown hook --- bin/reth/src/cli/node_builder.rs | 16 +++++++----- bin/reth/src/node/mod.rs | 43 +------------------------------ bin/reth/src/utils.rs | 29 +++++++++++++++++++-- crates/net/network/src/manager.rs | 32 ++++++++++++++++++++++- 4 files changed, 69 insertions(+), 51 deletions(-) diff --git a/bin/reth/src/cli/node_builder.rs b/bin/reth/src/cli/node_builder.rs index 08c96f1445bc..f72a863b1d9c 100644 --- a/bin/reth/src/cli/node_builder.rs +++ b/bin/reth/src/cli/node_builder.rs @@ -15,9 +15,9 @@ use crate::{ }, dirs::{ChainPath, DataDirPath, MaybePlatformPath}, init::init_genesis, - node::{cl_events::ConsensusLayerHealthEvents, events, run_network_until_shutdown}, + node::{cl_events::ConsensusLayerHealthEvents, events}, prometheus_exporter, - utils::get_single_header, + utils::{get_single_header, write_peers_to_file}, version::SHORT_VERSION, }; use eyre::Context; @@ -668,10 +668,14 @@ impl NodeBuilder { let default_peers_path = data_dir.known_peers_path(); let known_peers_file = self.network.persistent_peers_file(default_peers_path); - task_executor - .spawn_critical_with_graceful_shutdown_signal("p2p network task", |shutdown| { - run_network_until_shutdown(shutdown, network, known_peers_file) - }); + task_executor.spawn_critical_with_graceful_shutdown_signal( + "p2p network task", + |shutdown| { + network.run_until_graceful_shutdown(shutdown, |network| { + write_peers_to_file(network, known_peers_file) + }) + }, + ); handle } diff --git a/bin/reth/src/node/mod.rs b/bin/reth/src/node/mod.rs index ec1a67857d84..664da40e14fa 100644 --- a/bin/reth/src/node/mod.rs +++ b/bin/reth/src/node/mod.rs @@ -13,14 +13,11 @@ use crate::{ version::SHORT_VERSION, }; use clap::{value_parser, Parser}; -use futures::pin_mut; use reth_auto_seal_consensus::AutoSealConsensus; use reth_beacon_consensus::BeaconConsensus; use reth_interfaces::consensus::Consensus; -use reth_network::NetworkManager; use reth_primitives::ChainSpec; -use reth_provider::{BlockReader, HeaderProvider}; -use std::{fs, net::SocketAddr, path::PathBuf, sync::Arc}; +use std::{net::SocketAddr, path::PathBuf, sync::Arc}; use tracing::*; pub mod cl_events; @@ -238,44 +235,6 @@ impl NodeCommand { } } -/// Drives the [NetworkManager] future until a [Shutdown](reth_tasks::shutdown::Shutdown) signal is -/// received. If configured, this writes known peers to `persistent_peers_file` afterwards. -pub(crate) async fn run_network_until_shutdown( - shutdown: reth_tasks::shutdown::GracefulShutdown, - network: NetworkManager, - persistent_peers_file: Option, -) where - C: BlockReader + HeaderProvider + Clone + Unpin + 'static, -{ - pin_mut!(network, shutdown); - - let mut graceful_guard = None; - tokio::select! { - _ = &mut network => {}, - guard = shutdown => { - graceful_guard = Some(guard); - }, - } - - if let Some(file_path) = persistent_peers_file { - let known_peers = network.all_peers().collect::>(); - if let Ok(known_peers) = serde_json::to_string_pretty(&known_peers) { - trace!(target: "reth::cli", peers_file =?file_path, num_peers=%known_peers.len(), "Saving current peers"); - let parent_dir = file_path.parent().map(fs::create_dir_all).transpose(); - match parent_dir.and_then(|_| fs::write(&file_path, known_peers)) { - Ok(_) => { - info!(target: "reth::cli", peers_file=?file_path, "Wrote network peers to file"); - } - Err(err) => { - warn!(target: "reth::cli", ?err, peers_file=?file_path, "Failed to write network peers to file"); - } - } - } - } - - drop(graceful_guard) -} - #[cfg(test)] mod tests { use super::*; diff --git a/bin/reth/src/utils.rs b/bin/reth/src/utils.rs index 74af5288cebb..38a824fb9a99 100644 --- a/bin/reth/src/utils.rs +++ b/bin/reth/src/utils.rs @@ -15,9 +15,11 @@ use reth_interfaces::p2p::{ headers::client::{HeadersClient, HeadersRequest}, priority::Priority, }; +use reth_network::NetworkManager; use reth_primitives::{ fs, BlockHashOrNumber, ChainSpec, HeadersDirection, SealedBlock, SealedHeader, }; +use reth_provider::BlockReader; use reth_rpc::{JwtError, JwtSecret}; use std::{ env::VarError, @@ -25,7 +27,7 @@ use std::{ rc::Rc, sync::Arc, }; -use tracing::{debug, info}; +use tracing::{debug, info, trace, warn}; /// Exposing `open_db_read_only` function pub mod db { @@ -255,8 +257,8 @@ impl ListFilter { self.len = len; } } -/// Attempts to retrieve or create a JWT secret from the specified path. +/// Attempts to retrieve or create a JWT secret from the specified path. pub fn get_or_create_jwt_secret_from_path(path: &Path) -> Result { if path.exists() { debug!(target: "reth::cli", ?path, "Reading JWT auth secret file"); @@ -266,3 +268,26 @@ pub fn get_or_create_jwt_secret_from_path(path: &Path) -> Result(network: &NetworkManager, persistent_peers_file: Option) +where + C: BlockReader + Unpin, +{ + if let Some(file_path) = persistent_peers_file { + let known_peers = network.all_peers().collect::>(); + if let Ok(known_peers) = serde_json::to_string_pretty(&known_peers) { + trace!(target: "reth::cli", peers_file =?file_path, num_peers=%known_peers.len(), "Saving current peers"); + let parent_dir = file_path.parent().map(fs::create_dir_all).transpose(); + match parent_dir.and_then(|_| fs::write(&file_path, known_peers)) { + Ok(_) => { + info!(target: "reth::cli", peers_file=?file_path, "Wrote network peers to file"); + } + Err(err) => { + warn!(target: "reth::cli", ?err, peers_file=?file_path, "Failed to write network peers to file"); + } + } + } + } +} diff --git a/crates/net/network/src/manager.rs b/crates/net/network/src/manager.rs index 88cb8bdcbcdf..b725d68171d1 100644 --- a/crates/net/network/src/manager.rs +++ b/crates/net/network/src/manager.rs @@ -33,7 +33,7 @@ use crate::{ transactions::NetworkTransactionEvent, FetchClient, NetworkBuilder, }; -use futures::{Future, StreamExt}; +use futures::{pin_mut, Future, StreamExt}; use parking_lot::Mutex; use reth_eth_wire::{ capability::{Capabilities, CapabilityMessage}, @@ -45,6 +45,7 @@ use reth_network_api::ReputationChangeKind; use reth_primitives::{ForkId, NodeRecord, PeerId, B256}; use reth_provider::{BlockNumReader, BlockReader}; use reth_rpc_types::{EthProtocolInfo, NetworkStatus}; +use reth_tasks::shutdown::GracefulShutdown; use reth_tokio_util::EventListeners; use secp256k1::SecretKey; use std::{ @@ -596,6 +597,35 @@ where } } +impl NetworkManager +where + C: BlockReader + Unpin, +{ + /// Drives the [NetworkManager] future until a [GracefulShutdown](GracefulShutdown) signal is + /// received. + /// + /// This also run the given function `shutdown_hook` afterwards. + pub async fn run_until_graceful_shutdown( + self, + shutdown: GracefulShutdown, + shutdown_hook: impl FnOnce(&mut Self), + ) { + let network = self; + pin_mut!(network, shutdown); + + let mut graceful_guard = None; + tokio::select! { + _ = &mut network => {}, + guard = shutdown => { + graceful_guard = Some(guard); + }, + } + + shutdown_hook(&mut network); + drop(graceful_guard); + } +} + impl Future for NetworkManager where C: BlockReader + Unpin, From 5ffc12dd78ef54e0b99434bd0f8c77b08373c723 Mon Sep 17 00:00:00 2001 From: Dan Cline <6798349+Rjected@users.noreply.github.com> Date: Tue, 26 Dec 2023 13:55:35 -0500 Subject: [PATCH 32/33] fix docs --- crates/net/network/src/manager.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/crates/net/network/src/manager.rs b/crates/net/network/src/manager.rs index b725d68171d1..456b542e33a5 100644 --- a/crates/net/network/src/manager.rs +++ b/crates/net/network/src/manager.rs @@ -601,8 +601,7 @@ impl NetworkManager where C: BlockReader + Unpin, { - /// Drives the [NetworkManager] future until a [GracefulShutdown](GracefulShutdown) signal is - /// received. + /// Drives the [NetworkManager] future until a [GracefulShutdown] signal is received. /// /// This also run the given function `shutdown_hook` afterwards. pub async fn run_until_graceful_shutdown( From 5bca9c8ae65053f60efd2cba5760c51e8e74357a Mon Sep 17 00:00:00 2001 From: Dan Cline <6798349+Rjected@users.noreply.github.com> Date: Tue, 26 Dec 2023 14:01:26 -0500 Subject: [PATCH 33/33] improve DatabaseBuilder docs --- bin/reth/src/cli/db_type.rs | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/bin/reth/src/cli/db_type.rs b/bin/reth/src/cli/db_type.rs index 5f1229be58b4..a852f88c2a85 100644 --- a/bin/reth/src/cli/db_type.rs +++ b/bin/reth/src/cli/db_type.rs @@ -47,8 +47,11 @@ impl DatabaseBuilder { Self { db_type } } - /// Initializes and returns the test database. If the [DatabaseType] is test, the [LogLevel] - /// and chain are not used. + /// Initializes and returns the [DatabaseInstance] depending on the current database type. If + /// the [DatabaseType] is test, the [LogLevel] is not used. + /// + /// If the [DatabaseType] is test, then the [ChainPath] constructed will be derived from the db + /// path of the [TempDatabase] and the given chain. pub fn build_db( self, log_level: Option, @@ -75,7 +78,7 @@ impl DatabaseBuilder { } } -/// A constructed database type, without path information. +/// A constructed database type, with a [ChainPath]. #[derive(Debug, Clone)] pub enum DatabaseInstance { /// The test database @@ -85,7 +88,7 @@ pub enum DatabaseInstance { /// The data dir data_dir: ChainPath, }, - /// The right database + /// The real database Real { /// The database db: Arc,