From 3d9f107e1cfb5a615b886d1bb1090f87d607d797 Mon Sep 17 00:00:00 2001 From: Gregory Hill Date: Mon, 9 May 2022 14:58:41 +0100 Subject: [PATCH] feat: add instant-seal for dev chain Signed-off-by: Gregory Hill --- Cargo.lock | 41 +++++++++- parachain/Cargo.toml | 2 + parachain/src/cli.rs | 6 ++ parachain/src/command.rs | 96 ++++++++++++---------- parachain/src/service.rs | 166 ++++++++++++++++++++++++++++++++++++-- rpc/Cargo.toml | 1 + rpc/src/lib.rs | 14 +++- standalone/src/service.rs | 1 + 8 files changed, 279 insertions(+), 48 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ca8f0bd1bb..aba574233f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3556,6 +3556,7 @@ dependencies = [ "frame-benchmarking", "frame-benchmarking-cli", "frame-support", + "futures 0.3.21", "hex-literal 0.2.2", "interbtc-primitives", "interbtc-rpc", @@ -3582,6 +3583,7 @@ dependencies = [ "sc-cli", "sc-client-api", "sc-consensus", + "sc-consensus-manual-seal", "sc-executor", "sc-network", "sc-rpc", @@ -3638,6 +3640,7 @@ dependencies = [ "module-replace-rpc", "module-vault-registry-rpc", "pallet-transaction-payment-rpc", + "sc-consensus-manual-seal", "sc-rpc", "sc-rpc-api", "sc-transaction-pool-api", @@ -10191,6 +10194,42 @@ dependencies = [ "sp-runtime", ] +[[package]] +name = "sc-consensus-manual-seal" +version = "0.10.0-dev" +source = "git+https://github.com/paritytech/substrate?branch=polkadot-v0.9.18#fc3fd073d3a0acf9933c3994b660ebd7b5833f65" +dependencies = [ + "assert_matches", + "async-trait", + "futures 0.3.21", + "jsonrpc-core", + "jsonrpc-core-client", + "jsonrpc-derive", + "log", + "parity-scale-codec", + "sc-client-api", + "sc-consensus", + "sc-consensus-aura", + "sc-consensus-babe", + "sc-consensus-epochs", + "sc-transaction-pool", + "sc-transaction-pool-api", + "serde", + "sp-api", + "sp-blockchain", + "sp-consensus", + "sp-consensus-aura", + "sp-consensus-babe", + "sp-consensus-slots", + "sp-core", + "sp-inherents", + "sp-keystore", + "sp-runtime", + "sp-timestamp", + "substrate-prometheus-endpoint", + "thiserror", +] + [[package]] name = "sc-consensus-slots" version = "0.10.0-dev" @@ -12829,7 +12868,7 @@ checksum = "4ee73e6e4924fe940354b8d4d98cad5231175d615cd855b758adc658c0aac6a0" dependencies = [ "cfg-if 1.0.0", "digest 0.10.3", - "rand 0.8.5", + "rand 0.6.5", "static_assertions", ] diff --git a/parachain/Cargo.toml b/parachain/Cargo.toml index a931591683..6185d55651 100644 --- a/parachain/Cargo.toml +++ b/parachain/Cargo.toml @@ -20,6 +20,7 @@ log = "0.4.8" codec = { package = "parity-scale-codec", version = "3.0.0" } serde = { version = "1.0.130", features = ["derive"] } hex-literal = "0.2.1" +futures = "0.3.15" # Parachain dependencies interlay-runtime = { package = "interlay-runtime-parachain", path = "./runtime/interlay" } @@ -42,6 +43,7 @@ module-refund-rpc-runtime-api = { path = "../crates/refund/rpc/runtime-api" } sc-cli = { git = "https://github.com/paritytech/substrate", branch = "polkadot-v0.9.18", features = ["wasmtime"] } sc-client-api = { git = "https://github.com/paritytech/substrate", branch = "polkadot-v0.9.18" } sc-consensus = { git = "https://github.com/paritytech/substrate", branch = "polkadot-v0.9.18" } +sc-consensus-manual-seal = { git = "https://github.com/paritytech/substrate", branch = "polkadot-v0.9.18" } sc-executor = { git = "https://github.com/paritytech/substrate", branch = "polkadot-v0.9.18", features = ["wasmtime"] } sc-rpc = { git = "https://github.com/paritytech/substrate", branch = "polkadot-v0.9.18" } sc-service = { git = "https://github.com/paritytech/substrate", branch = "polkadot-v0.9.18" } diff --git a/parachain/src/cli.rs b/parachain/src/cli.rs index 2f40ef2fe9..30b9e7a511 100644 --- a/parachain/src/cli.rs +++ b/parachain/src/cli.rs @@ -115,6 +115,12 @@ pub struct Cli { /// Relaychain arguments #[clap(raw = true)] pub relaychain_args: Vec, + + /// Instant block sealing + /// + /// Can only be used with `--dev` + #[clap(long = "instant-seal", requires = "dev")] + pub instant_seal: bool, } #[derive(Debug)] diff --git a/parachain/src/command.rs b/parachain/src/command.rs index 42c7c39f95..6432462979 100644 --- a/parachain/src/command.rs +++ b/parachain/src/command.rs @@ -100,6 +100,27 @@ fn load_spec(id: &str) -> std::result::Result, St }) } +macro_rules! with_runtime_or_err { + ($chain_spec:expr, { $( $code:tt )* }) => { + if $chain_spec.is_interlay() { + #[allow(unused_imports)] + use { interlay_runtime::RuntimeApi, crate::service::InterlayRuntimeExecutor as Executor }; + $( $code )* + + } else if $chain_spec.is_kintsugi() { + #[allow(unused_imports)] + use { kintsugi_runtime::RuntimeApi, crate::service::KintsugiRuntimeExecutor as Executor }; + $( $code )* + + } else { + #[allow(unused_imports)] + use { testnet_runtime::RuntimeApi, crate::service::TestnetRuntimeExecutor as Executor }; + $( $code )* + + } + } +} + impl SubstrateCli for Cli { fn impl_name() -> String { "interBTC Parachain".into() @@ -212,6 +233,7 @@ macro_rules! construct_async_run { runner.async_run(|$config| { let $components = new_partial::( &$config, + true, )?; let task_manager = $components.task_manager; { $( $code )* }.map(|v| (v, task_manager)) @@ -223,6 +245,7 @@ macro_rules! construct_async_run { KintsugiRuntimeExecutor, >( &$config, + true, )?; let task_manager = $components.task_manager; { $( $code )* }.map(|v| (v, task_manager)) @@ -234,6 +257,7 @@ macro_rules! construct_async_run { TestnetRuntimeExecutor, >( &$config, + true, )?; let task_manager = $components.task_manager; { $( $code )* }.map(|v| (v, task_manager)) @@ -291,15 +315,11 @@ pub fn run() -> Result<()> { Some(Subcommand::Benchmark(cmd)) => { if cfg!(feature = "runtime-benchmarks") { let runner = cli.create_runner(cmd)?; - if runner.config().chain_spec.is_interlay() { - runner.sync_run(|config| cmd.run::(config)) - } else if runner.config().chain_spec.is_kintsugi() { - runner.sync_run(|config| cmd.run::(config)) - } else if runner.config().chain_spec.is_testnet() { - runner.sync_run(|config| cmd.run::(config)) - } else { - Err("Chain doesn't support benchmarking".into()) - } + let chain_spec = &runner.config().chain_spec; + + with_runtime_or_err!(chain_spec, { + return runner.sync_run(|config| cmd.run::(config)); + }) } else { Err("Benchmarking wasn't enabled when building the node. \ You can enable it with `--features runtime-benchmarks`." @@ -349,12 +369,29 @@ pub fn run() -> Result<()> { let runner = cli.create_runner(&cli.run.normalize())?; runner - .run_node_until_exit(|config| async move { start_node(cli, config).await }) + .run_node_until_exit(|config| async move { + if cli.instant_seal { + start_instant(config).await + } else { + start_node(cli, config).await + } + }) .map_err(Into::into) } } } +async fn start_instant(config: Configuration) -> sc_service::error::Result { + with_runtime_or_err!(config.chain_spec, { + { + crate::service::start_instant::(config) + .await + .map(|r| r.0) + .map_err(Into::into) + } + }) +} + async fn start_node(cli: Cli, config: Configuration) -> sc_service::error::Result { let para_id = chain_spec::Extensions::try_get(&*config.chain_spec).map(|e| e.para_id); @@ -387,37 +424,14 @@ async fn start_node(cli: Cli, config: Configuration) -> sc_service::error::Resul if config.role.is_authority() { "yes" } else { "no" } ); - if config.chain_spec.is_interlay() { - crate::service::start_node::( - config, - polkadot_config, - collator_options, - id, - ) - .await - .map(|r| r.0) - .map_err(Into::into) - } else if config.chain_spec.is_kintsugi() { - crate::service::start_node::( - config, - polkadot_config, - collator_options, - id, - ) - .await - .map(|r| r.0) - .map_err(Into::into) - } else { - crate::service::start_node::( - config, - polkadot_config, - collator_options, - id, - ) - .await - .map(|r| r.0) - .map_err(Into::into) - } + with_runtime_or_err!(config.chain_spec, { + { + crate::service::start_node::(config, polkadot_config, collator_options, id) + .await + .map(|r| r.0) + .map_err(Into::into) + } + }) } impl DefaultConfigurationValues for RelayChainCli { diff --git a/parachain/src/service.rs b/parachain/src/service.rs index 06bb55299b..adb6c412d5 100644 --- a/parachain/src/service.rs +++ b/parachain/src/service.rs @@ -6,13 +6,16 @@ use cumulus_client_service::{ prepare_node_config, start_collator, start_full_node, StartCollatorParams, StartFullNodeParams, }; use cumulus_primitives_core::ParaId; +use cumulus_primitives_parachain_inherent::{MockValidationDataInherentDataProvider, MockXcmConfig}; use cumulus_relay_chain_inprocess_interface::build_inprocess_relay_chain; use cumulus_relay_chain_interface::{RelayChainError, RelayChainInterface, RelayChainResult}; use cumulus_relay_chain_rpc_interface::RelayChainRPCInterface; use polkadot_service::CollatorPair; +use futures::StreamExt; use primitives::*; -use sc_client_api::ExecutorProvider; +use sc_client_api::{ExecutorProvider, HeaderBackend}; +use sc_consensus::LongestChain; use sc_executor::NativeElseWasmExecutor; use sc_network::NetworkService; use sc_service::{Configuration, PartialComponents, Role, TFullBackend, TFullClient, TaskManager}; @@ -148,17 +151,20 @@ type FullBackend = TFullBackend; type FullClient = sc_service::TFullClient>; +type MaybeFullSelectChain = Option>; + /// Starts a `ServiceBuilder` for a full service. /// /// Use this macro if you don't actually need the full service, but just the builder in order to /// be able to perform chain operations. pub fn new_partial( config: &Configuration, + instant_seal: bool, ) -> Result< PartialComponents< TFullClient>, TFullBackend, - (), + MaybeFullSelectChain, sc_consensus::DefaultImportQueue>>, sc_transaction_pool::FullPool>>, (Option, Option), @@ -213,8 +219,21 @@ where client.clone(), ); + let select_chain = if instant_seal { + Some(LongestChain::new(backend.clone())) + } else { + None + }; + let client_clone = client.clone(); - let import_queue = { + let import_queue = if instant_seal { + // instance sealing + sc_consensus_manual_seal::import_queue( + Box::new(client.clone()), + &task_manager.spawn_essential_handle(), + registry, + ) + } else { cumulus_client_consensus_aura::import_queue::( cumulus_client_consensus_aura::ImportQueueParams { block_import: client.clone(), @@ -253,7 +272,7 @@ where keystore_container, task_manager, transaction_pool, - select_chain: (), + select_chain, other: (telemetry, telemetry_worker_handle), }; @@ -316,7 +335,7 @@ where let parachain_config = prepare_node_config(parachain_config); - let params = new_partial(¶chain_config)?; + let params = new_partial(¶chain_config, false)?; let (mut telemetry, telemetry_worker_handle) = params.other; let client = params.client.clone(); @@ -362,6 +381,7 @@ where client: client.clone(), pool: transaction_pool.clone(), deny_unsafe, + command_sink: None, }; Ok(interbtc_rpc::create_full(deps)) @@ -542,3 +562,139 @@ where ) .await } + +pub async fn start_instant( + config: Configuration, +) -> sc_service::error::Result<(TaskManager, Arc>)> +where + RuntimeApi: ConstructRuntimeApi> + Send + Sync + 'static, + RuntimeApi::RuntimeApi: RuntimeApiCollection>, + RuntimeApi::RuntimeApi: sp_consensus_aura::AuraApi, + Executor: sc_executor::NativeExecutionDispatch + 'static, +{ + let sc_service::PartialComponents { + client, + backend, + mut task_manager, + import_queue, + keystore_container, + select_chain: maybe_select_chain, + transaction_pool, + other: (mut telemetry, _), + } = new_partial::(&config, true)?; + + let (network, system_rpc_tx, network_starter) = sc_service::build_network(sc_service::BuildNetworkParams { + config: &config, + client: client.clone(), + transaction_pool: transaction_pool.clone(), + spawn_handle: task_manager.spawn_handle(), + import_queue, + block_announce_validator_builder: None, + warp_sync: None, + })?; + + if config.offchain_worker.enabled { + sc_service::build_offchain_workers(&config, task_manager.spawn_handle(), client.clone(), network.clone()); + }; + + let prometheus_registry = config.prometheus_registry().cloned(); + + let role = config.role.clone(); + + let select_chain = maybe_select_chain.expect("`new_partial` will return some `select_chain`; qed"); + + let command_sink = if role.is_authority() { + let proposer_factory = sc_basic_authorship::ProposerFactory::new( + task_manager.spawn_handle(), + client.clone(), + transaction_pool.clone(), + prometheus_registry.as_ref(), + telemetry.as_ref().map(|x| x.handle()), + ); + + // Channel for the rpc handler to communicate with the authorship task. + let (command_sink, commands_stream) = futures::channel::mpsc::channel(1024); + + let pool = transaction_pool.pool().clone(); + let import_stream = pool.validated_pool().import_notification_stream().map(|_| { + sc_consensus_manual_seal::rpc::EngineCommand::SealNewBlock { + create_empty: true, + finalize: true, + parent_hash: None, + sender: None, + } + }); + + let client_for_cidp = client.clone(); + + let authorship_future = sc_consensus_manual_seal::run_manual_seal(sc_consensus_manual_seal::ManualSealParams { + block_import: client.clone(), + env: proposer_factory, + client: client.clone(), + pool: transaction_pool.clone(), + commands_stream: futures::stream_select!(commands_stream, import_stream), + select_chain, + consensus_data_provider: None, + create_inherent_data_providers: move |block: Hash, _| { + let current_para_block = client_for_cidp + .number(block) + .expect("Header lookup should succeed") + .expect("Header passed in as parent should be present in backend."); + let client_for_xcm = client_for_cidp.clone(); + async move { + let mocked_parachain = MockValidationDataInherentDataProvider { + current_para_block, + relay_offset: 1000, + relay_blocks_per_para_block: 2, + xcm_config: MockXcmConfig::new(&*client_for_xcm, block, Default::default(), Default::default()), + raw_downward_messages: vec![], + raw_horizontal_messages: vec![], + }; + Ok((sp_timestamp::InherentDataProvider::from_system_time(), mocked_parachain)) + } + }, + }); + // we spawn the future on a background thread managed by service. + task_manager.spawn_essential_handle().spawn_blocking( + "instant-seal", + Some("block-authoring"), + authorship_future, + ); + Some(command_sink) + } else { + None + }; + + let rpc_extensions_builder = { + let client = client.clone(); + let transaction_pool = transaction_pool.clone(); + + Box::new(move |deny_unsafe, _| { + let deps = interbtc_rpc::FullDeps { + client: client.clone(), + pool: transaction_pool.clone(), + deny_unsafe, + command_sink: command_sink.clone(), + }; + + Ok(interbtc_rpc::create_full(deps)) + }) + }; + + sc_service::spawn_tasks(sc_service::SpawnTasksParams { + rpc_extensions_builder, + client: client.clone(), + transaction_pool, + task_manager: &mut task_manager, + config, + keystore: keystore_container.sync_keystore(), + backend, + network, + system_rpc_tx, + telemetry: telemetry.as_mut(), + })?; + + network_starter.start_network(); + + Ok((task_manager, client)) +} diff --git a/rpc/Cargo.toml b/rpc/Cargo.toml index 4118a7076a..b07f24f7ea 100644 --- a/rpc/Cargo.toml +++ b/rpc/Cargo.toml @@ -24,6 +24,7 @@ primitives = { package = "interbtc-primitives", path = "../primitives" } sc-rpc = { git = "https://github.com/paritytech/substrate", branch = "polkadot-v0.9.18" } sc-rpc-api = { git = "https://github.com/paritytech/substrate", branch = "polkadot-v0.9.18" } sc-transaction-pool-api = { git = "https://github.com/paritytech/substrate", branch = "polkadot-v0.9.18" } +sc-consensus-manual-seal = { git = "https://github.com/paritytech/substrate", branch = "polkadot-v0.9.18" } sp-api = { git = "https://github.com/paritytech/substrate", branch = "polkadot-v0.9.18" } sp-core = { git = "https://github.com/paritytech/substrate", branch = "polkadot-v0.9.18" } sp-arithmetic = { git = "https://github.com/paritytech/substrate", branch = "polkadot-v0.9.18" } diff --git a/rpc/src/lib.rs b/rpc/src/lib.rs index b1b1bb5744..66e02ce664 100644 --- a/rpc/src/lib.rs +++ b/rpc/src/lib.rs @@ -7,8 +7,9 @@ use primitives::{ issue::IssueRequest, redeem::RedeemRequest, refund::RefundRequest, replace::ReplaceRequest, AccountId, Balance, - Block, BlockNumber, CurrencyId, H256Le, Nonce, VaultId, + Block, BlockNumber, CurrencyId, H256Le, Hash, Nonce, VaultId, }; +use sc_consensus_manual_seal::rpc::{EngineCommand, ManualSeal, ManualSealApi}; pub use sc_rpc_api::DenyUnsafe; use sc_transaction_pool_api::TransactionPool; use sp_api::ProvideRuntimeApi; @@ -28,6 +29,8 @@ pub struct FullDeps { pub pool: Arc

, /// Whether to deny unsafe calls pub deny_unsafe: DenyUnsafe, + /// Manual seal command sink + pub command_sink: Option>>, } /// Instantiate all full RPC extensions. @@ -87,8 +90,17 @@ where client, pool, deny_unsafe, + command_sink, } = deps; + if let Some(command_sink) = command_sink { + io.extend_with( + // We provide the rpc handler with the sending end of the channel to allow the rpc + // send EngineCommands to the background block authorship task. + ManualSealApi::to_delegate(ManualSeal::new(command_sink)), + ); + } + io.extend_with(SystemApi::to_delegate(FullSystem::new( client.clone(), pool, diff --git a/standalone/src/service.rs b/standalone/src/service.rs index a7b2e02e59..36a374f025 100644 --- a/standalone/src/service.rs +++ b/standalone/src/service.rs @@ -206,6 +206,7 @@ pub fn new_full(mut config: Configuration) -> Result<(TaskManager, RpcHandlers), client: client.clone(), pool: pool.clone(), deny_unsafe, + command_sink: None, }; Ok(interbtc_rpc::create_full(deps))