diff --git a/Cargo.lock b/Cargo.lock index ea34146f16f9b..b840cadce3e5c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -269,9 +269,9 @@ checksum = "e91831deabf0d6d7ec49552e489aed63b7456a7a3c46cff62adad428110b0af0" [[package]] name = "async-trait" -version = "0.1.58" +version = "0.1.59" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e805d94e6b5001b651426cf4cd446b1ab5f319d27bab5c644f61de0a804360c" +checksum = "31e6e93155431f3931513b243d371981bb2770112b370c82745a1d19d2f99364" dependencies = [ "proc-macro2", "quote", @@ -2031,6 +2031,7 @@ version = "0.10.0-dev" dependencies = [ "env_logger", "frame-support", + "futures", "log", "pallet-elections-phragmen", "parity-scale-codec", @@ -2042,6 +2043,7 @@ dependencies = [ "sp-version", "substrate-rpc-client", "tokio", + "tracing-subscriber 0.3.16", ] [[package]] @@ -2874,9 +2876,9 @@ checksum = "879d54834c8c76457ef4293a689b2a8c59b076067ad77b15efafbb05f92a592b" [[package]] name = "itertools" -version = "0.10.3" +version = "0.10.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9a9d19fa1e79b6215ff29b9d6880b706147f16e9b1dbb1e4e5947b5b02bc5e3" +checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473" dependencies = [ "either", ] @@ -3788,6 +3790,15 @@ dependencies = [ "regex-automata", ] +[[package]] +name = "matchers" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8263075bb86c5a1b1427b5ae862e8889656f126e9f77c484496e8b47cf5c5558" +dependencies = [ + "regex-automata", +] + [[package]] name = "matches" version = "0.1.8" @@ -4271,6 +4282,7 @@ dependencies = [ "sp-core", "sp-finality-grandpa", "sp-inherents", + "sp-io", "sp-keyring", "sp-keystore", "sp-runtime", @@ -4427,6 +4439,7 @@ dependencies = [ "sp-core", "sp-finality-grandpa", "sp-inherents", + "sp-io", "sp-keyring", "sp-runtime", "sp-timestamp", @@ -4529,6 +4542,16 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "61807f77802ff30975e01f4f071c8ba10c022052f98b3294119f3e615d13e5be" +[[package]] +name = "nu-ansi-term" +version = "0.46.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77a8165726e8236064dbb45459242600304b42a5ea24ee2948e18e023bf7ba84" +dependencies = [ + "overload", + "winapi", +] + [[package]] name = "num-bigint" version = "0.4.3" @@ -4667,6 +4690,12 @@ dependencies = [ "winapi", ] +[[package]] +name = "overload" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39" + [[package]] name = "pallet-alliance" version = "4.0.0-dev" @@ -7585,7 +7614,7 @@ dependencies = [ "substrate-test-runtime", "tempfile", "tracing", - "tracing-subscriber", + "tracing-subscriber 0.2.25", "wasmi 0.13.0", "wat", ] @@ -8337,7 +8366,7 @@ dependencies = [ "thiserror", "tracing", "tracing-log", - "tracing-subscriber", + "tracing-subscriber 0.2.25", ] [[package]] @@ -8705,9 +8734,9 @@ dependencies = [ [[package]] name = "sharded-slab" -version = "0.1.1" +version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "79c719719ee05df97490f80a45acfc99e5a30ce98a1e4fb67aee422745ae14e3" +checksum = "900fba806f70c630b0a382d0d825e17a0f19fcd059a2ade1ff237bcddf446b31" dependencies = [ "lazy_static", ] @@ -8763,9 +8792,9 @@ checksum = "03b634d87b960ab1a38c4fe143b508576f075e7c978bfad18217645ebfdfa2ec" [[package]] name = "smallvec" -version = "1.8.0" +version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2dd574626839106c320a323308629dcb1acfc96e32a8cba364ddc61ac23ee83" +checksum = "a507befe795404456341dfab10cef66ead4c041f62b8b11bbb92bffe5d0953e0" [[package]] name = "snap" @@ -9555,7 +9584,7 @@ dependencies = [ "sp-std", "tracing", "tracing-core", - "tracing-subscriber", + "tracing-subscriber 0.2.25", ] [[package]] @@ -10431,9 +10460,9 @@ dependencies = [ [[package]] name = "tracing-core" -version = "0.1.28" +version = "0.1.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b7358be39f2f274f322d2aaed611acc57f382e8eb1e5b48cb9ae30933495ce7" +checksum = "24eb03ba0eab1fd845050058ce5e616558e8f8d8fca633e6b163fe25c797213a" dependencies = [ "once_cell", "valuable", @@ -10479,7 +10508,7 @@ dependencies = [ "ansi_term", "chrono", "lazy_static", - "matchers", + "matchers 0.0.1", "parking_lot 0.11.2", "regex", "serde", @@ -10493,6 +10522,24 @@ dependencies = [ "tracing-serde", ] +[[package]] +name = "tracing-subscriber" +version = "0.3.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6176eae26dd70d0c919749377897b54a9276bd7061339665dd68777926b5a70" +dependencies = [ + "matchers 0.1.0", + "nu-ansi-term", + "once_cell", + "regex", + "sharded-slab", + "smallvec", + "thread_local", + "tracing", + "tracing-core", + "tracing-log", +] + [[package]] name = "treeline" version = "0.1.0" @@ -10605,6 +10652,7 @@ dependencies = [ "clap 4.0.11", "frame-remote-externalities", "frame-try-runtime", + "hex", "log", "parity-scale-codec", "sc-chain-spec", @@ -10612,10 +10660,13 @@ dependencies = [ "sc-executor", "sc-service", "serde", + "sp-api", "sp-core", + "sp-debug-derive", "sp-externalities", "sp-io", "sp-keystore", + "sp-rpc", "sp-runtime", "sp-state-machine", "sp-version", diff --git a/bin/node-template/node/Cargo.toml b/bin/node-template/node/Cargo.toml index 2ea841093d0e2..364cfa25d3c6b 100644 --- a/bin/node-template/node/Cargo.toml +++ b/bin/node-template/node/Cargo.toml @@ -36,6 +36,7 @@ sc-finality-grandpa = { version = "0.10.0-dev", path = "../../../client/finality sp-finality-grandpa = { version = "4.0.0-dev", path = "../../../primitives/finality-grandpa" } sc-client-api = { version = "4.0.0-dev", path = "../../../client/api" } sp-runtime = { version = "7.0.0", path = "../../../primitives/runtime" } +sp-io = { version = "7.0.0", path = "../../../primitives/io" } sp-timestamp = { version = "4.0.0-dev", path = "../../../primitives/timestamp" } sp-inherents = { version = "4.0.0-dev", path = "../../../primitives/inherents" } sp-keyring = { version = "7.0.0", path = "../../../primitives/keyring" } diff --git a/bin/node-template/node/src/command.rs b/bin/node-template/node/src/command.rs index 6d293b7b85fcc..15cd69b34b5b2 100644 --- a/bin/node-template/node/src/command.rs +++ b/bin/node-template/node/src/command.rs @@ -174,6 +174,8 @@ pub fn run() -> sc_cli::Result<()> { }, #[cfg(feature = "try-runtime")] Some(Subcommand::TryRuntime(cmd)) => { + use crate::service::ExecutorDispatch; + use sc_executor::{sp_wasm_interface::ExtendedHostFunctions, NativeExecutionDispatch}; let runner = cli.create_runner(cmd)?; runner.async_run(|config| { // we don't need any of the components of new_partial, just a runtime, or a task @@ -182,7 +184,13 @@ pub fn run() -> sc_cli::Result<()> { let task_manager = sc_service::TaskManager::new(config.tokio_handle.clone(), registry) .map_err(|e| sc_cli::Error::Service(sc_service::Error::Prometheus(e)))?; - Ok((cmd.run::(config), task_manager)) + Ok(( + cmd.run::::ExtendHostFunctions, + >>(), + task_manager, + )) }) }, #[cfg(not(feature = "try-runtime"))] diff --git a/bin/node-template/runtime/src/lib.rs b/bin/node-template/runtime/src/lib.rs index 938282c662b5c..f76b2c449ee4a 100644 --- a/bin/node-template/runtime/src/lib.rs +++ b/bin/node-template/runtime/src/lib.rs @@ -539,22 +539,23 @@ impl_runtime_apis! { #[cfg(feature = "try-runtime")] impl frame_try_runtime::TryRuntime for Runtime { - fn on_runtime_upgrade() -> (Weight, Weight) { + fn on_runtime_upgrade(checks: bool) -> (Weight, Weight) { // NOTE: intentional unwrap: we don't want to propagate the error backwards, and want to // have a backtrace here. If any of the pre/post migration checks fail, we shall stop // right here and right now. - let weight = Executive::try_runtime_upgrade().unwrap(); + let weight = Executive::try_runtime_upgrade(checks).unwrap(); (weight, BlockWeights::get().max_block) } fn execute_block( block: Block, state_root_check: bool, + signature_check: bool, select: frame_try_runtime::TryStateSelect ) -> Weight { // NOTE: intentional unwrap: we don't want to propagate the error backwards, and want to // have a backtrace here. - Executive::try_execute_block(block, state_root_check, select).expect("execute-block failed") + Executive::try_execute_block(block, state_root_check, signature_check, select).expect("execute-block failed") } } } diff --git a/bin/node/cli/Cargo.toml b/bin/node/cli/Cargo.toml index 4ee4bcd033921..6b50115fd9a00 100644 --- a/bin/node/cli/Cargo.toml +++ b/bin/node/cli/Cargo.toml @@ -59,6 +59,7 @@ sp-keystore = { version = "0.13.0", path = "../../../primitives/keystore" } sp-consensus = { version = "0.10.0-dev", path = "../../../primitives/consensus/common" } sp-transaction-pool = { version = "4.0.0-dev", path = "../../../primitives/transaction-pool" } sp-transaction-storage-proof = { version = "4.0.0-dev", path = "../../../primitives/transaction-storage-proof" } +sp-io = { path = "../../../primitives/io" } # client dependencies sc-client-api = { version = "4.0.0-dev", path = "../../../client/api" } diff --git a/bin/node/cli/src/command.rs b/bin/node/cli/src/command.rs index 108d7743843b6..fd464bbc914a5 100644 --- a/bin/node/cli/src/command.rs +++ b/bin/node/cli/src/command.rs @@ -227,6 +227,7 @@ pub fn run() -> Result<()> { }, #[cfg(feature = "try-runtime")] Some(Subcommand::TryRuntime(cmd)) => { + use sc_executor::{sp_wasm_interface::ExtendedHostFunctions, NativeExecutionDispatch}; let runner = cli.create_runner(cmd)?; runner.async_run(|config| { // we don't need any of the components of new_partial, just a runtime, or a task @@ -236,7 +237,13 @@ pub fn run() -> Result<()> { sc_service::TaskManager::new(config.tokio_handle.clone(), registry) .map_err(|e| sc_cli::Error::Service(sc_service::Error::Prometheus(e)))?; - Ok((cmd.run::(config), task_manager)) + Ok(( + cmd.run::::ExtendHostFunctions, + >>(), + task_manager, + )) }) }, #[cfg(not(feature = "try-runtime"))] diff --git a/bin/node/runtime/src/lib.rs b/bin/node/runtime/src/lib.rs index 00d2a54d1e774..0e3bee8821fc2 100644 --- a/bin/node/runtime/src/lib.rs +++ b/bin/node/runtime/src/lib.rs @@ -2158,29 +2158,23 @@ impl_runtime_apis! { #[cfg(feature = "try-runtime")] impl frame_try_runtime::TryRuntime for Runtime { - fn on_runtime_upgrade() -> (Weight, Weight) { + fn on_runtime_upgrade(checks: bool) -> (Weight, Weight) { // NOTE: intentional unwrap: we don't want to propagate the error backwards, and want to // have a backtrace here. If any of the pre/post migration checks fail, we shall stop // right here and right now. - let weight = Executive::try_runtime_upgrade().unwrap(); + let weight = Executive::try_runtime_upgrade(checks).unwrap(); (weight, RuntimeBlockWeights::get().max_block) } fn execute_block( block: Block, state_root_check: bool, + signature_check: bool, select: frame_try_runtime::TryStateSelect ) -> Weight { - log::info!( - target: "node-runtime", - "try-runtime: executing block {:?} / root checks: {:?} / try-state-select: {:?}", - block.header.hash(), - state_root_check, - select, - ); // NOTE: intentional unwrap: we don't want to propagate the error backwards, and want to // have a backtrace here. - Executive::try_execute_block(block, state_root_check, select).unwrap() + Executive::try_execute_block(block, state_root_check, signature_check, select).unwrap() } } diff --git a/frame/bags-list/remote-tests/src/migration.rs b/frame/bags-list/remote-tests/src/migration.rs index b013472b4c90e..759906a4ef479 100644 --- a/frame/bags-list/remote-tests/src/migration.rs +++ b/frame/bags-list/remote-tests/src/migration.rs @@ -30,7 +30,7 @@ pub async fn execute( ws_url: String, ) where Runtime: RuntimeT, - Block: BlockT, + Block: BlockT + DeserializeOwned, Block::Header: DeserializeOwned, { let mut ext = Builder::::new() diff --git a/frame/bags-list/remote-tests/src/snapshot.rs b/frame/bags-list/remote-tests/src/snapshot.rs index cfe065924bd92..0163ca200a15d 100644 --- a/frame/bags-list/remote-tests/src/snapshot.rs +++ b/frame/bags-list/remote-tests/src/snapshot.rs @@ -25,7 +25,7 @@ use sp_runtime::{traits::Block as BlockT, DeserializeOwned}; pub async fn execute(voter_limit: Option, currency_unit: u64, ws_url: String) where Runtime: crate::RuntimeT, - Block: BlockT, + Block: BlockT + DeserializeOwned, Block::Header: DeserializeOwned, { use frame_support::storage::generator::StorageMap; @@ -38,14 +38,18 @@ where pallets: vec![pallet_bags_list::Pallet::::name() .to_string()], at: None, + hashed_prefixes: vec![ + >::prefix_hash(), + >::prefix_hash(), + >::map_storage_final_prefix(), + >::map_storage_final_prefix(), + ], + hashed_keys: vec![ + >::counter_storage_final_key().to_vec(), + >::counter_storage_final_key().to_vec(), + ], ..Default::default() })) - .inject_hashed_prefix(&>::prefix_hash()) - .inject_hashed_prefix(&>::prefix_hash()) - .inject_hashed_prefix(&>::map_storage_final_prefix()) - .inject_hashed_prefix(&>::map_storage_final_prefix()) - .inject_hashed_key(&>::counter_storage_final_key()) - .inject_hashed_key(&>::counter_storage_final_key()) .build() .await .unwrap(); diff --git a/frame/bags-list/remote-tests/src/try_state.rs b/frame/bags-list/remote-tests/src/try_state.rs index d3fb63f045a64..514c80d72ab67 100644 --- a/frame/bags-list/remote-tests/src/try_state.rs +++ b/frame/bags-list/remote-tests/src/try_state.rs @@ -31,7 +31,7 @@ pub async fn execute( ws_url: String, ) where Runtime: crate::RuntimeT, - Block: BlockT, + Block: BlockT + DeserializeOwned, Block::Header: DeserializeOwned, { let mut ext = Builder::::new() @@ -39,10 +39,12 @@ pub async fn execute( transport: ws_url.to_string().into(), pallets: vec![pallet_bags_list::Pallet::::name() .to_string()], + hashed_prefixes: vec![ + >::prefix_hash(), + >::prefix_hash(), + ], ..Default::default() })) - .inject_hashed_prefix(&>::prefix_hash()) - .inject_hashed_prefix(&>::prefix_hash()) .build() .await .unwrap(); diff --git a/frame/executive/Cargo.toml b/frame/executive/Cargo.toml index b3e4247445710..a6d16e9b0793f 100644 --- a/frame/executive/Cargo.toml +++ b/frame/executive/Cargo.toml @@ -49,4 +49,4 @@ std = [ "sp-std/std", "sp-tracing/std", ] -try-runtime = ["frame-support/try-runtime", "frame-try-runtime/try-runtime" ] +try-runtime = ["frame-support/try-runtime", "frame-try-runtime/try-runtime", "sp-runtime/try-runtime"] diff --git a/frame/executive/src/lib.rs b/frame/executive/src/lib.rs index 6f59ac72eb2fd..1f20e93f43c30 100644 --- a/frame/executive/src/lib.rs +++ b/frame/executive/src/lib.rs @@ -227,27 +227,71 @@ where { /// Execute given block, but don't as strict is the normal block execution. /// - /// Some consensus related checks such as the state root check can be switched off via - /// `try_state_root`. Some additional non-consensus checks can be additionally enabled via - /// `try_state`. + /// Some checks can be disabled via: + /// + /// - `state_root_check` + /// - `signature_check` /// /// Should only be used for testing ONLY. pub fn try_execute_block( block: Block, - try_state_root: bool, + state_root_check: bool, + signature_check: bool, select: frame_try_runtime::TryStateSelect, - ) -> Result { - use frame_support::traits::TryState; + ) -> Result { + frame_support::log::info!( + target: "frame::executive", + "try-runtime: executing block #{:?} / state root check: {:?} / signature check: {:?} / try-state-select: {:?}", + block.header().number(), + state_root_check, + signature_check, + select, + ); Self::initialize_block(block.header()); Self::initial_checks(&block); let (header, extrinsics) = block.deconstruct(); - Self::execute_extrinsics_with_book_keeping(extrinsics, *header.number()); + let try_apply_extrinsic = |uxt: Block::Extrinsic| -> ApplyExtrinsicResult { + sp_io::init_tracing(); + let encoded = uxt.encode(); + let encoded_len = encoded.len(); + + // skip signature verification. + let xt = if signature_check { + uxt.check(&Default::default()) + } else { + uxt.unchecked_into_checked_i_know_what_i_am_doing(&Default::default()) + }?; + >::note_extrinsic(encoded); + + let dispatch_info = xt.get_dispatch_info(); + let r = Applyable::apply::(xt, &dispatch_info, encoded_len)?; - // run the try-state checks of all pallets. - >::try_state( + >::note_applied_extrinsic(&r, dispatch_info); + + Ok(r.map(|_| ()).map_err(|e| e.error)) + }; + + for e in extrinsics { + if let Err(err) = try_apply_extrinsic(e.clone()) { + frame_support::log::error!( + target: "runtime::executive", "executing transaction {:?} failed due to {:?}. Aborting the rest of the block execution.", + e, + err, + ); + break + } + } + + // post-extrinsics book-keeping + >::note_finished_extrinsics(); + Self::idle_and_finalize_hook(*header.number()); + + // run the try-state checks of all pallets, ensuring they don't alter any state. + let _guard = frame_support::StorageNoopGuard::default(); + >::try_state( *header.number(), select, ) @@ -255,6 +299,7 @@ where frame_support::log::error!(target: "runtime::executive", "failure: {:?}", e); e })?; + drop(_guard); // do some of the checks that would normally happen in `final_checks`, but perhaps skip // the state root check. @@ -266,7 +311,7 @@ where assert!(header_item == computed_item, "Digest item must match that calculated."); } - if try_state_root { + if state_root_check { let storage_root = new_header.state_root(); header.state_root().check_equal(storage_root); assert!( @@ -286,9 +331,30 @@ where /// Execute all `OnRuntimeUpgrade` of this runtime, including the pre and post migration checks. /// - /// This should only be used for testing. - pub fn try_runtime_upgrade() -> Result { - let weight = Self::execute_on_runtime_upgrade(); + /// Runs the try-state code both before and after the migration function if `checks` is set to + /// `true`. Also, if set to `true`, it runs the `pre_upgrade` and `post_upgrade` hooks. + pub fn try_runtime_upgrade(checks: bool) -> Result { + if checks { + let _guard = frame_support::StorageNoopGuard::default(); + >::try_state( + frame_system::Pallet::::block_number(), + frame_try_runtime::TryStateSelect::All, + )?; + } + + let weight = + <(COnRuntimeUpgrade, AllPalletsWithSystem) as OnRuntimeUpgrade>::try_on_runtime_upgrade( + checks, + )?; + + if checks { + let _guard = frame_support::StorageNoopGuard::default(); + >::try_state( + frame_system::Pallet::::block_number(), + frame_try_runtime::TryStateSelect::All, + )?; + } + Ok(weight) } } @@ -314,7 +380,7 @@ where UnsignedValidator: ValidateUnsigned>, { /// Execute all `OnRuntimeUpgrade` of this runtime, and return the aggregate weight. - pub fn execute_on_runtime_upgrade() -> frame_support::weights::Weight { + pub fn execute_on_runtime_upgrade() -> Weight { <(COnRuntimeUpgrade, AllPalletsWithSystem) as OnRuntimeUpgrade>::on_runtime_upgrade() } diff --git a/frame/staking/Cargo.toml b/frame/staking/Cargo.toml index a7fca045cc4ba..f6b3b95d0beb9 100644 --- a/frame/staking/Cargo.toml +++ b/frame/staking/Cargo.toml @@ -62,7 +62,6 @@ std = [ "sp-runtime/std", "sp-staking/std", "pallet-session/std", - "pallet-bags-list/std", "frame-system/std", "pallet-authorship/std", "sp-application-crypto/std", diff --git a/frame/staking/src/pallet/impls.rs b/frame/staking/src/pallet/impls.rs index 830b33ceb69a2..d3b9b6a2b1e83 100644 --- a/frame/staking/src/pallet/impls.rs +++ b/frame/staking/src/pallet/impls.rs @@ -1636,9 +1636,9 @@ impl Pallet { ensure!( T::VoterList::iter() .all(|x| >::contains_key(&x) || >::contains_key(&x)), - "VoterList contains non-nominators" + "VoterList contains non-staker" ); - T::VoterList::try_state()?; + Self::check_nominators()?; Self::check_exposures()?; Self::check_ledgers()?; @@ -1651,7 +1651,10 @@ impl Pallet { Nominators::::count() + Validators::::count(), "wrong external count" ); - + ensure!( + ::TargetList::count() == Validators::::count(), + "wrong external count" + ); ensure!( ValidatorCount::::get() <= ::MaxWinners::get(), @@ -1692,7 +1695,7 @@ impl Pallet { >::iter() .filter_map( |(nominator, nomination)| { - if nomination.submitted_in > era { + if nomination.submitted_in < era { Some(nominator) } else { None diff --git a/frame/state-trie-migration/src/lib.rs b/frame/state-trie-migration/src/lib.rs index 823ea08a0b573..23f73bb56b173 100644 --- a/frame/state-trie-migration/src/lib.rs +++ b/frame/state-trie-migration/src/lib.rs @@ -1619,7 +1619,10 @@ pub(crate) mod remote_tests { use frame_system::Pallet as System; use remote_externalities::Mode; use sp_core::H256; - use sp_runtime::traits::{Block as BlockT, HashFor, Header as _, One, Zero}; + use sp_runtime::{ + traits::{Block as BlockT, HashFor, Header as _, One, Zero}, + DeserializeOwned, + }; use thousands::Separable; #[allow(dead_code)] @@ -1648,12 +1651,12 @@ pub(crate) mod remote_tests { pub(crate) async fn run_with_limits(limits: MigrationLimits, mode: Mode) where Runtime: crate::Config, - Block: BlockT, + Block: BlockT + DeserializeOwned, Block::Header: serde::de::DeserializeOwned, { let mut ext = remote_externalities::Builder::::new() .mode(mode) - .state_version(sp_core::storage::StateVersion::V0) + .overwrite_state_version(sp_core::storage::StateVersion::V0) .build() .await .unwrap(); diff --git a/frame/support/procedural/src/pallet/expand/hooks.rs b/frame/support/procedural/src/pallet/expand/hooks.rs index d8d009cf3c940..0aa7c1e7aaf06 100644 --- a/frame/support/procedural/src/pallet/expand/hooks.rs +++ b/frame/support/procedural/src/pallet/expand/hooks.rs @@ -50,7 +50,7 @@ pub fn expand_hooks(def: &mut Def) -> proc_macro2::TokenStream { } else { // default. quote::quote! { - #frame_support::log::info!( + #frame_support::log::debug!( target: #frame_support::LOG_TARGET, "✅ no migration for {}", pallet_name, diff --git a/frame/support/src/dispatch.rs b/frame/support/src/dispatch.rs index 1696e9a63915e..93cf08c131641 100644 --- a/frame/support/src/dispatch.rs +++ b/frame/support/src/dispatch.rs @@ -2188,7 +2188,7 @@ macro_rules! decl_module { $system::Config >::PalletInfo as $crate::traits::PalletInfo>::name::().unwrap_or(""); - $crate::log::info!( + $crate::log::debug!( target: $crate::LOG_TARGET, "✅ no migration for {}", pallet_name, diff --git a/frame/support/src/traits/hooks.rs b/frame/support/src/traits/hooks.rs index 3415682c0b382..3f7db1fa046bd 100644 --- a/frame/support/src/traits/hooks.rs +++ b/frame/support/src/traits/hooks.rs @@ -22,9 +22,6 @@ use impl_trait_for_tuples::impl_for_tuples; use sp_runtime::traits::AtLeast32BitUnsigned; use sp_std::prelude::*; -#[cfg(all(feature = "try-runtime", test))] -use codec::{Decode, Encode}; - /// The block initialization trait. /// /// Implementing this lets you express what should happen for your pallet when the block is @@ -136,6 +133,29 @@ pub trait OnRuntimeUpgrade { Weight::zero() } + /// Same as `on_runtime_upgrade`, but perform the optional `pre_upgrade` and `post_upgrade` as + /// well. + #[cfg(feature = "try-runtime")] + fn try_on_runtime_upgrade(checks: bool) -> Result { + let maybe_state = if checks { + let _guard = frame_support::StorageNoopGuard::default(); + let state = Self::pre_upgrade()?; + Some(state) + } else { + None + }; + + let weight = Self::on_runtime_upgrade(); + + if let Some(state) = maybe_state { + let _guard = frame_support::StorageNoopGuard::default(); + // we want to panic if any checks fail right here right now. + Self::post_upgrade(state)? + } + + Ok(weight) + } + /// Execute some pre-checks prior to a runtime upgrade. /// /// Return a `Vec` that can contain arbitrary encoded data (usually some pre-upgrade state), @@ -143,6 +163,9 @@ pub trait OnRuntimeUpgrade { /// should be returned if there is no such need. /// /// This hook is never meant to be executed on-chain but is meant to be used by testing tools. + /// + /// This hook must not write to any state, as it would make the main `on_runtime_upgrade` path + /// inaccurate. #[cfg(feature = "try-runtime")] fn pre_upgrade() -> Result, &'static str> { Ok(Vec::new()) @@ -155,6 +178,9 @@ pub trait OnRuntimeUpgrade { /// be passed in, in such case `post_upgrade` should ignore it. /// /// This hook is never meant to be executed on-chain but is meant to be used by testing tools. + /// + /// This hook must not write to any state, as it would make the main `on_runtime_upgrade` path + /// inaccurate. #[cfg(feature = "try-runtime")] fn post_upgrade(_state: Vec) -> Result<(), &'static str> { Ok(()) @@ -165,7 +191,6 @@ pub trait OnRuntimeUpgrade { #[cfg_attr(all(feature = "tuples-96", not(feature = "tuples-128")), impl_for_tuples(96))] #[cfg_attr(feature = "tuples-128", impl_for_tuples(128))] impl OnRuntimeUpgrade for Tuple { - #[cfg(not(feature = "try-runtime"))] fn on_runtime_upgrade() -> Weight { let mut weight = Weight::zero(); for_tuples!( #( weight = weight.saturating_add(Tuple::on_runtime_upgrade()); )* ); @@ -176,40 +201,10 @@ impl OnRuntimeUpgrade for Tuple { /// We are executing pre- and post-checks sequentially in order to be able to test several /// consecutive migrations for the same pallet without errors. Therefore pre and post upgrade /// hooks for tuples are a noop. - fn on_runtime_upgrade() -> Weight { - use scale_info::prelude::format; - + fn try_on_runtime_upgrade(checks: bool) -> Result { let mut weight = Weight::zero(); - // migration index in the tuple, start with 1 for better readability - let mut i = 1; - for_tuples!( #( - let _guard = frame_support::StorageNoopGuard::default(); - // we want to panic if any checks fail right here right now. - let state = Tuple::pre_upgrade().expect(&format!("PreUpgrade failed for migration #{}", i)); - drop(_guard); - - weight = weight.saturating_add(Tuple::on_runtime_upgrade()); - - let _guard = frame_support::StorageNoopGuard::default(); - // we want to panic if any checks fail right here right now. - Tuple::post_upgrade(state).expect(&format!("PostUpgrade failed for migration #{}", i)); - drop(_guard); - - i += 1; - )* ); - weight - } - - #[cfg(feature = "try-runtime")] - /// noop - fn pre_upgrade() -> Result, &'static str> { - Ok(Vec::new()) - } - - #[cfg(feature = "try-runtime")] - /// noop - fn post_upgrade(_state: Vec) -> Result<(), &'static str> { - Ok(()) + for_tuples!( #( weight = weight.saturating_add(Tuple::try_on_runtime_upgrade(checks)?); )* ); + Ok(weight) } } @@ -274,6 +269,8 @@ pub trait Hooks { /// /// It should focus on certain checks to ensure that the state is sensible. This is never /// executed in a consensus code-path, therefore it can consume as much weight as it needs. + /// + /// This hook should not alter any storage. #[cfg(feature = "try-runtime")] fn try_state(_n: BlockNumber) -> Result<(), &'static str> { Ok(()) @@ -440,110 +437,4 @@ mod tests { ON_IDLE_INVOCATION_ORDER.clear(); } } - - #[cfg(feature = "try-runtime")] - #[test] - #[allow(dead_code)] - fn on_runtime_upgrade_tuple() { - use frame_support::parameter_types; - use sp_io::TestExternalities; - - struct Test1; - struct Test2; - struct Test3; - - parameter_types! { - static Test1Assertions: u8 = 0; - static Test2Assertions: u8 = 0; - static Test3Assertions: u8 = 0; - static EnableSequentialTest: bool = false; - static SequentialAssertions: u8 = 0; - } - - impl OnRuntimeUpgrade for Test1 { - fn pre_upgrade() -> Result, &'static str> { - Ok("Test1".encode()) - } - fn post_upgrade(state: Vec) -> Result<(), &'static str> { - let s: String = Decode::decode(&mut state.as_slice()).unwrap(); - Test1Assertions::mutate(|val| *val += 1); - if EnableSequentialTest::get() { - SequentialAssertions::mutate(|val| *val += 1); - } - assert_eq!(s, "Test1"); - Ok(()) - } - } - - impl OnRuntimeUpgrade for Test2 { - fn pre_upgrade() -> Result, &'static str> { - Ok(100u32.encode()) - } - fn post_upgrade(state: Vec) -> Result<(), &'static str> { - let s: u32 = Decode::decode(&mut state.as_slice()).unwrap(); - Test2Assertions::mutate(|val| *val += 1); - if EnableSequentialTest::get() { - assert_eq!(SequentialAssertions::get(), 1); - SequentialAssertions::mutate(|val| *val += 1); - } - assert_eq!(s, 100); - Ok(()) - } - } - - impl OnRuntimeUpgrade for Test3 { - fn pre_upgrade() -> Result, &'static str> { - Ok(true.encode()) - } - fn post_upgrade(state: Vec) -> Result<(), &'static str> { - let s: bool = Decode::decode(&mut state.as_slice()).unwrap(); - Test3Assertions::mutate(|val| *val += 1); - if EnableSequentialTest::get() { - assert_eq!(SequentialAssertions::get(), 2); - SequentialAssertions::mutate(|val| *val += 1); - } - assert_eq!(s, true); - Ok(()) - } - } - - TestExternalities::default().execute_with(|| { - type TestEmpty = (); - let origin_state = ::pre_upgrade().unwrap(); - assert!(origin_state.is_empty()); - ::post_upgrade(origin_state).unwrap(); - - type Test1Tuple = (Test1,); - let origin_state = ::pre_upgrade().unwrap(); - assert!(origin_state.is_empty()); - ::post_upgrade(origin_state).unwrap(); - assert_eq!(Test1Assertions::get(), 0); - ::on_runtime_upgrade(); - assert_eq!(Test1Assertions::take(), 1); - - type Test321 = (Test3, Test2, Test1); - ::on_runtime_upgrade(); - assert_eq!(Test1Assertions::take(), 1); - assert_eq!(Test2Assertions::take(), 1); - assert_eq!(Test3Assertions::take(), 1); - - // enable sequential tests - EnableSequentialTest::mutate(|val| *val = true); - - type Test123 = (Test1, Test2, Test3); - ::on_runtime_upgrade(); - assert_eq!(Test1Assertions::take(), 1); - assert_eq!(Test2Assertions::take(), 1); - assert_eq!(Test3Assertions::take(), 1); - - // reset assertions - SequentialAssertions::take(); - - type TestNested123 = (Test1, (Test2, Test3)); - ::on_runtime_upgrade(); - assert_eq!(Test1Assertions::take(), 1); - assert_eq!(Test2Assertions::take(), 1); - assert_eq!(Test3Assertions::take(), 1); - }); - } } diff --git a/frame/support/src/traits/try_runtime.rs b/frame/support/src/traits/try_runtime.rs index 640bb566a65af..f741ca56a56fc 100644 --- a/frame/support/src/traits/try_runtime.rs +++ b/frame/support/src/traits/try_runtime.rs @@ -85,6 +85,8 @@ impl sp_std::str::FromStr for Select { /// /// Usually, these checks should check all of the invariants that are expected to be held on all of /// the storage items of your pallet. +/// +/// This hook should not alter any storage. pub trait TryState { /// Execute the state checks. fn try_state(_: BlockNumber, _: Select) -> Result<(), &'static str>; diff --git a/frame/support/test/tests/pallet_ui/storage_ensure_span_are_ok_on_wrong_gen.stderr b/frame/support/test/tests/pallet_ui/storage_ensure_span_are_ok_on_wrong_gen.stderr index 999d8585c221a..a3af9897be5c7 100644 --- a/frame/support/test/tests/pallet_ui/storage_ensure_span_are_ok_on_wrong_gen.stderr +++ b/frame/support/test/tests/pallet_ui/storage_ensure_span_are_ok_on_wrong_gen.stderr @@ -28,7 +28,7 @@ error[E0277]: the trait bound `Bar: EncodeLike` is not satisfied <&[(T,)] as EncodeLike>> <&[(T,)] as EncodeLike>> <&[T] as EncodeLike>> - and 279 others + and 280 others = note: required for `Bar` to implement `FullEncode` = note: required for `Bar` to implement `FullCodec` = note: required for `frame_support::pallet_prelude::StorageValue<_GeneratedPrefixForStorageFoo, Bar>` to implement `PartialStorageInfoTrait` @@ -103,7 +103,7 @@ error[E0277]: the trait bound `Bar: EncodeLike` is not satisfied <&[(T,)] as EncodeLike>> <&[(T,)] as EncodeLike>> <&[T] as EncodeLike>> - and 279 others + and 280 others = note: required for `Bar` to implement `FullEncode` = note: required for `Bar` to implement `FullCodec` = note: required for `frame_support::pallet_prelude::StorageValue<_GeneratedPrefixForStorageFoo, Bar>` to implement `StorageEntryMetadataBuilder` diff --git a/frame/support/test/tests/pallet_ui/storage_ensure_span_are_ok_on_wrong_gen_unnamed.stderr b/frame/support/test/tests/pallet_ui/storage_ensure_span_are_ok_on_wrong_gen_unnamed.stderr index e2870ffb9e86f..9e87f87825b2a 100644 --- a/frame/support/test/tests/pallet_ui/storage_ensure_span_are_ok_on_wrong_gen_unnamed.stderr +++ b/frame/support/test/tests/pallet_ui/storage_ensure_span_are_ok_on_wrong_gen_unnamed.stderr @@ -28,7 +28,7 @@ error[E0277]: the trait bound `Bar: EncodeLike` is not satisfied <&[(T,)] as EncodeLike>> <&[(T,)] as EncodeLike>> <&[T] as EncodeLike>> - and 279 others + and 280 others = note: required for `Bar` to implement `FullEncode` = note: required for `Bar` to implement `FullCodec` = note: required for `frame_support::pallet_prelude::StorageValue<_GeneratedPrefixForStorageFoo, Bar>` to implement `PartialStorageInfoTrait` @@ -103,7 +103,7 @@ error[E0277]: the trait bound `Bar: EncodeLike` is not satisfied <&[(T,)] as EncodeLike>> <&[(T,)] as EncodeLike>> <&[T] as EncodeLike>> - and 279 others + and 280 others = note: required for `Bar` to implement `FullEncode` = note: required for `Bar` to implement `FullCodec` = note: required for `frame_support::pallet_prelude::StorageValue<_GeneratedPrefixForStorageFoo, Bar>` to implement `StorageEntryMetadataBuilder` diff --git a/frame/try-runtime/src/lib.rs b/frame/try-runtime/src/lib.rs index ed1247bd8e6f2..99c68d4dc65b8 100644 --- a/frame/try-runtime/src/lib.rs +++ b/frame/try-runtime/src/lib.rs @@ -33,12 +33,21 @@ sp_api::decl_runtime_apis! { /// /// Returns the consumed weight of the migration in case of a successful one, combined with /// the total allowed block weight of the runtime. - fn on_runtime_upgrade() -> (Weight, Weight); + /// + /// If `checks` is `true`, `pre_migrate` and `post_migrate` of each migration and + /// `try_state` of all pallets will be executed. Else, no. If checks are executed, the PoV + /// tracking is likely inaccurate. + fn on_runtime_upgrade(checks: bool) -> (Weight, Weight); - /// Execute the given block, but don't check that its state root matches that of yours. + /// Execute the given block, but optionally disable state-root and signature checks. /// - /// This is only sensible where the incoming block is from a different network, yet it has - /// the same block format as the runtime implementing this API. - fn execute_block(block: Block, state_root_check: bool, try_state: TryStateSelect) -> Weight; + /// Optionally, a number of `try_state` hooks can also be executed after the block + /// execution. + fn execute_block( + block: Block, + state_root_check: bool, + signature_check: bool, + try_state: TryStateSelect, + ) -> Weight; } } diff --git a/primitives/runtime/Cargo.toml b/primitives/runtime/Cargo.toml index 8ea6ed3eb3b19..4202110cd60c2 100644 --- a/primitives/runtime/Cargo.toml +++ b/primitives/runtime/Cargo.toml @@ -41,6 +41,7 @@ substrate-test-runtime-client = { version = "2.0.0", path = "../../test-utils/ru [features] runtime-benchmarks = [] +try-runtime = [] default = ["std"] std = [ "codec/std", diff --git a/primitives/runtime/src/generic/unchecked_extrinsic.rs b/primitives/runtime/src/generic/unchecked_extrinsic.rs index 5d378410e4756..bab4411167a7f 100644 --- a/primitives/runtime/src/generic/unchecked_extrinsic.rs +++ b/primitives/runtime/src/generic/unchecked_extrinsic.rs @@ -149,6 +149,22 @@ where None => CheckedExtrinsic { signed: None, function: self.function }, }) } + + #[cfg(feature = "try-runtime")] + fn unchecked_into_checked_i_know_what_i_am_doing( + self, + lookup: &Lookup, + ) -> Result { + Ok(match self.signature { + Some((signed, _, extra)) => { + let signed = lookup.lookup(signed)?; + let raw_payload = SignedPayload::new(self.function, extra)?; + let (function, extra, _) = raw_payload.deconstruct(); + CheckedExtrinsic { signed: Some((signed, extra)), function } + }, + None => CheckedExtrinsic { signed: None, function: self.function }, + }) + } } impl ExtrinsicMetadata diff --git a/primitives/runtime/src/testing.rs b/primitives/runtime/src/testing.rs index 0cd78ba6267dd..81762b3fc4f9f 100644 --- a/primitives/runtime/src/testing.rs +++ b/primitives/runtime/src/testing.rs @@ -326,6 +326,14 @@ impl Checkable for TestXt Result { Ok(self) } + + #[cfg(feature = "try-runtime")] + fn unchecked_into_checked_i_know_what_i_am_doing( + self, + _: &Context, + ) -> Result { + unreachable!() + } } impl traits::Extrinsic for TestXt { diff --git a/primitives/runtime/src/traits.rs b/primitives/runtime/src/traits.rs index c69f8616b4be5..375475141b818 100644 --- a/primitives/runtime/src/traits.rs +++ b/primitives/runtime/src/traits.rs @@ -990,6 +990,20 @@ pub trait Checkable: Sized { /// Check self, given an instance of Context. fn check(self, c: &Context) -> Result; + + /// Blindly check self. + /// + /// ## WARNING + /// + /// DO NOT USE IN PRODUCTION. This is only meant to be used in testing environments. A runtime + /// compiled with `try-runtime` should never be in production. Moreover, the name of this + /// function is deliberately chosen to prevent developers from ever calling it in consensus + /// code-paths. + #[cfg(feature = "try-runtime")] + fn unchecked_into_checked_i_know_what_i_am_doing( + self, + c: &Context, + ) -> Result; } /// A "checkable" piece of information, used by the standard Substrate Executive in order to @@ -1011,6 +1025,14 @@ impl Checkable for T { fn check(self, _c: &Context) -> Result { BlindCheckable::check(self) } + + #[cfg(feature = "try-runtime")] + fn unchecked_into_checked_i_know_what_i_am_doing( + self, + _: &Context, + ) -> Result { + unreachable!(); + } } /// A lazy call (module function and argument values) that can be executed via its `dispatch` diff --git a/primitives/storage/src/lib.rs b/primitives/storage/src/lib.rs index 79c1012196bde..237787710a7e7 100644 --- a/primitives/storage/src/lib.rs +++ b/primitives/storage/src/lib.rs @@ -407,6 +407,7 @@ impl ChildTrieParentKeyId { /// V0 and V1 uses a same trie implementation, but V1 will write external value node in the trie for /// value with size at least `TRIE_VALUE_NODE_THRESHOLD`. #[derive(Debug, Clone, Copy, Eq, PartialEq)] +#[cfg_attr(feature = "std", derive(Encode, Decode))] pub enum StateVersion { /// Old state version, no value nodes. V0 = 0, diff --git a/utils/frame/remote-externalities/Cargo.toml b/utils/frame/remote-externalities/Cargo.toml index 4cb847867f374..ad8230fe29dcf 100644 --- a/utils/frame/remote-externalities/Cargo.toml +++ b/utils/frame/remote-externalities/Cargo.toml @@ -22,12 +22,14 @@ sp-core = { version = "7.0.0", path = "../../../primitives/core" } sp-io = { version = "7.0.0", path = "../../../primitives/io" } sp-runtime = { version = "7.0.0", path = "../../../primitives/runtime" } sp-version = { version = "5.0.0", path = "../../../primitives/version" } +tokio = { version = "1.22.0", features = ["macros", "rt-multi-thread"] } substrate-rpc-client = { path = "../rpc/client" } +futures = "0.3" [dev-dependencies] -tokio = { version = "1.22.0", features = ["macros", "rt-multi-thread"] } frame-support = { version = "4.0.0-dev", path = "../../../frame/support" } pallet-elections-phragmen = { version = "5.0.0-dev", path = "../../../frame/elections-phragmen" } +tracing-subscriber = { version = "0.3.16", features = ["env-filter"] } [features] remote-test = ["frame-support"] diff --git a/utils/frame/remote-externalities/src/lib.rs b/utils/frame/remote-externalities/src/lib.rs index db062e246ceef..fb63b4275172d 100644 --- a/utils/frame/remote-externalities/src/lib.rs +++ b/utils/frame/remote-externalities/src/lib.rs @@ -21,7 +21,7 @@ //! based chain, or a local state snapshot file. use codec::{Decode, Encode}; - +use futures::{channel::mpsc, stream::StreamExt}; use log::*; use serde::de::DeserializeOwned; use sp_core::{ @@ -36,8 +36,11 @@ pub use sp_io::TestExternalities; use sp_runtime::{traits::Block as BlockT, StateVersion}; use std::{ fs, + num::NonZeroUsize, + ops::{Deref, DerefMut}, path::{Path, PathBuf}, sync::Arc, + thread, }; use substrate_rpc_client::{ rpc_params, ws_client, BatchRequestBuilder, ChainApi, ClientT, StateApi, WsClient, @@ -48,18 +51,49 @@ type TopKeyValues = Vec; type ChildKeyValues = Vec<(ChildInfo, Vec)>; const LOG_TARGET: &str = "remote-ext"; -const DEFAULT_TARGET: &str = "wss://rpc.polkadot.io:443"; -const BATCH_SIZE: usize = 1000; -const PAGE: u32 = 1000; +const DEFAULT_WS_ENDPOINT: &str = "wss://rpc.polkadot.io:443"; +const DEFAULT_VALUE_DOWNLOAD_BATCH: usize = 4096; +// NOTE: increasing this value does not seem to impact speed all that much. +const DEFAULT_KEY_DOWNLOAD_PAGE: u32 = 1000; +/// The snapshot that we store on disk. +#[derive(Decode, Encode)] +struct Snapshot { + state_version: StateVersion, + block_hash: B::Hash, + top: TopKeyValues, + child: ChildKeyValues, +} + +/// An externalities that acts exactly the same as [`sp_io::TestExternalities`] but has a few extra +/// bits and pieces to it, and can be loaded remotely. +pub struct RemoteExternalities { + /// The inner externalities. + pub inner_ext: TestExternalities, + /// The block hash it which we created this externality env. + pub block_hash: B::Hash, +} + +impl Deref for RemoteExternalities { + type Target = TestExternalities; + fn deref(&self) -> &Self::Target { + &self.inner_ext + } +} + +impl DerefMut for RemoteExternalities { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.inner_ext + } +} /// The execution mode. #[derive(Clone)] pub enum Mode { - /// Online. Potentially writes to a cache file. + /// Online. Potentially writes to a snapshot file. Online(OnlineConfig), /// Offline. Uses a state snapshot file and needs not any client config. Offline(OfflineConfig), - /// Prefer using a cache file if it exists, else use a remote server. + /// Prefer using a snapshot file if it exists, else use a remote server. OfflineOrElseOnline(OfflineConfig, OnlineConfig), } @@ -95,6 +129,13 @@ impl Transport { } } + fn as_client_cloned(&self) -> Option> { + match self { + Self::RemoteClient(client) => Some(client.clone()), + _ => None, + } + } + // Open a new WebSocket connection if it's not connected. async fn map_uri(&mut self) -> Result<(), &'static str> { if let Self::Uri(uri) = self { @@ -134,38 +175,56 @@ pub struct OnlineConfig { pub at: Option, /// An optional state snapshot file to WRITE to, not for reading. Not written if set to `None`. pub state_snapshot: Option, - /// The pallets to scrape. If empty, entire chain state will be scraped. + /// The pallets to scrape. These values are hashed and added to `hashed_prefix`. pub pallets: Vec, /// Transport config. pub transport: Transport, /// Lookout for child-keys, and scrape them as well if set to true. - pub scrape_children: bool, + pub child_trie: bool, + /// Storage entry key prefixes to be injected into the externalities. The *hashed* prefix must + /// be given. + pub hashed_prefixes: Vec>, + /// Storage entry keys to be injected into the externalities. The *hashed* key must be given. + pub hashed_keys: Vec>, } impl OnlineConfig { - /// Return rpc (ws) client. + /// Return rpc (ws) client reference. fn rpc_client(&self) -> &WsClient { self.transport .as_client() .expect("ws client must have been initialized by now; qed.") } + + /// Return a cloned rpc (ws) client, suitable for being moved to threads. + fn rpc_client_cloned(&self) -> Arc { + self.transport + .as_client_cloned() + .expect("ws client must have been initialized by now; qed.") + } + + fn at_expected(&self) -> B::Hash { + self.at.expect("block at must be initialized; qed") + } } impl Default for OnlineConfig { fn default() -> Self { Self { - transport: Transport::Uri(DEFAULT_TARGET.to_owned()), + transport: Transport::from(DEFAULT_WS_ENDPOINT.to_owned()), + child_trie: true, at: None, state_snapshot: None, - pallets: vec![], - scrape_children: true, + pallets: Default::default(), + hashed_keys: Default::default(), + hashed_prefixes: Default::default(), } } } impl From for OnlineConfig { - fn from(s: String) -> Self { - Self { transport: s.into(), ..Default::default() } + fn from(t: String) -> Self { + Self { transport: t.into(), ..Default::default() } } } @@ -196,20 +255,18 @@ impl Default for SnapshotConfig { /// Builder for remote-externalities. pub struct Builder { - /// Custom key-pairs to be injected into the externalities. The *hashed* keys and values must - /// be given. + /// Custom key-pairs to be injected into the final externalities. The *hashed* keys and values + /// must be given. hashed_key_values: Vec, - /// Storage entry key prefixes to be injected into the externalities. The *hashed* prefix must - /// be given. - hashed_prefixes: Vec>, - /// Storage entry keys to be injected into the externalities. The *hashed* key must be given. - hashed_keys: Vec>, /// The keys that will be excluded from the final externality. The *hashed* key must be given. hashed_blacklist: Vec>, - /// connectivity mode, online or offline. + /// Connectivity mode, online or offline. mode: Mode, - /// The state version being used. - state_version: StateVersion, + /// If provided, overwrite the state version with this. Otherwise, the state_version of the + /// remote node is used. All cache files also store their state version. + /// + /// Overwrite only with care. + overwrite_state_version: Option, } // NOTE: ideally we would use `DefaultNoBound` here, but not worth bringing in frame-support for @@ -219,10 +276,8 @@ impl Default for Builder { Self { mode: Default::default(), hashed_key_values: Default::default(), - hashed_prefixes: Default::default(), - hashed_keys: Default::default(), hashed_blacklist: Default::default(), - state_version: StateVersion::V1, + overwrite_state_version: None, } } } @@ -252,20 +307,22 @@ where B::Hash: DeserializeOwned, B::Header: DeserializeOwned, { + /// Get the number of threads to use. + fn threads() -> NonZeroUsize { + thread::available_parallelism() + .unwrap_or(NonZeroUsize::new(4usize).expect("4 is non-zero; qed")) + } + async fn rpc_get_storage( &self, key: StorageKey, maybe_at: Option, - ) -> Result { + ) -> Result, &'static str> { trace!(target: LOG_TARGET, "rpc: get_storage"); - match self.as_online().rpc_client().storage(key, maybe_at).await { - Ok(Some(res)) => Ok(res), - Ok(None) => Err("get_storage not found"), - Err(e) => { - error!(target: LOG_TARGET, "Error = {:?}", e); - Err("rpc get_storage failed.") - }, - } + self.as_online().rpc_client().storage(key, maybe_at).await.map_err(|e| { + error!(target: LOG_TARGET, "Error = {:?}", e); + "rpc get_storage failed." + }) } /// Get the latest finalized head. @@ -293,7 +350,12 @@ where let page = self .as_online() .rpc_client() - .storage_keys_paged(Some(prefix.clone()), PAGE, last_key.clone(), Some(at)) + .storage_keys_paged( + Some(prefix.clone()), + DEFAULT_KEY_DOWNLOAD_PAGE, + last_key.clone(), + Some(at), + ) .await .map_err(|e| { error!(target: LOG_TARGET, "Error = {:?}", e); @@ -303,7 +365,7 @@ where all_keys.extend(page); - if page_len < PAGE as usize { + if page_len < DEFAULT_KEY_DOWNLOAD_PAGE as usize { log::debug!(target: LOG_TARGET, "last page received: {}", page_len); break all_keys } else { @@ -322,7 +384,7 @@ where Ok(keys) } - /// Synonym of `rpc_get_pairs_unsafe` that uses paged queries to first get the keys, and then + /// Synonym of `getPairs` that uses paged queries to first get the keys, and then /// map them to values one by one. /// /// This can work with public nodes. But, expect it to be darn slow. @@ -330,79 +392,169 @@ where &self, prefix: StorageKey, at: B::Hash, + pending_ext: &mut TestExternalities, ) -> Result, &'static str> { - let keys = self.rpc_get_keys_paged(prefix, at).await?; - let keys_count = keys.len(); - log::debug!(target: LOG_TARGET, "Querying a total of {} keys", keys.len()); + let keys = self.rpc_get_keys_paged(prefix.clone(), at).await?; + if keys.is_empty() { + return Ok(Default::default()) + } - let mut key_values: Vec = vec![]; - let mut batch_success = true; + let client = self.as_online().rpc_client_cloned(); + let threads = Self::threads().get(); + let thread_chunk_size = (keys.len() + threads - 1) / threads; - let client = self.as_online().rpc_client(); - for chunk_keys in keys.chunks(BATCH_SIZE) { - let mut batch = BatchRequestBuilder::new(); + log::info!( + target: LOG_TARGET, + "Querying a total of {} keys from prefix {:?}, splitting among {} threads, {} keys per thread", + keys.len(), + HexDisplay::from(&prefix), + threads, + thread_chunk_size, + ); - for key in chunk_keys.iter() { - batch - .insert("state_getStorage", rpc_params![key, at]) - .map_err(|_| "Invalid batch params")?; - } + let mut handles = Vec::new(); + let keys_chunked: Vec> = + keys.chunks(thread_chunk_size).map(|s| s.into()).collect::>(); + + enum Message { + /// This thread completed the assigned work. + Terminated, + /// The thread produced the following batch response. + Batch(Vec<(Vec, Vec)>), + /// A request from the batch failed. + BatchFailed(String), + } - let batch_response = - client.batch_request::>(batch).await.map_err(|e| { - log::error!( - target: LOG_TARGET, - "failed to execute batch: {:?}. Error: {:?}", - chunk_keys.iter().map(HexDisplay::from).collect::>(), - e - ); - "batch failed." - })?; + let (tx, mut rx) = mpsc::unbounded::(); + + for thread_keys in keys_chunked { + let thread_client = client.clone(); + let thread_sender = tx.clone(); + let handle = std::thread::spawn(move || { + let rt = tokio::runtime::Runtime::new().unwrap(); + let mut thread_key_values = Vec::with_capacity(thread_keys.len()); + + for chunk_keys in thread_keys.chunks(DEFAULT_VALUE_DOWNLOAD_BATCH) { + let mut batch = BatchRequestBuilder::new(); + + for key in chunk_keys.iter() { + batch + .insert("state_getStorage", rpc_params![key, at]) + .map_err(|_| "Invalid batch params") + .unwrap(); + } + + let batch_response = rt + .block_on(thread_client.batch_request::>(batch)) + .map_err(|e| { + log::error!( + target: LOG_TARGET, + "failed to execute batch: {:?}. Error: {:?}", + chunk_keys.iter().map(HexDisplay::from).collect::>(), + e + ); + "batch failed." + }) + .unwrap(); + + // Check if we got responses for all submitted requests. + assert_eq!(chunk_keys.len(), batch_response.len()); + + let mut batch_kv = Vec::with_capacity(chunk_keys.len()); + for (key, maybe_value) in chunk_keys.into_iter().zip(batch_response) { + match maybe_value { + Ok(Some(data)) => { + thread_key_values.push((key.clone(), data.clone())); + batch_kv.push((key.clone().0, data.0)); + }, + Ok(None) => { + log::warn!( + target: LOG_TARGET, + "key {:?} had none corresponding value.", + &key + ); + let data = StorageData(vec![]); + thread_key_values.push((key.clone(), data.clone())); + batch_kv.push((key.clone().0, data.0)); + }, + Err(e) => { + let reason = format!("key {:?} failed: {:?}", &key, e); + log::error!(target: LOG_TARGET, "Reason: {}", reason); + // Signal failures to the main thread, stop aggregating (key, value) + // pairs and return immediately an error. + thread_sender.unbounded_send(Message::BatchFailed(reason)).unwrap(); + return Default::default() + }, + }; + + if thread_key_values.len() % (thread_keys.len() / 10).max(1) == 0 { + let ratio: f64 = + thread_key_values.len() as f64 / thread_keys.len() as f64; + log::debug!( + target: LOG_TARGET, + "[thread = {:?}] progress = {:.2} [{} / {}]", + std::thread::current().id(), + ratio, + thread_key_values.len(), + thread_keys.len(), + ); + } + } + + // Send this batch to the main thread to start inserting. + thread_sender.unbounded_send(Message::Batch(batch_kv)).unwrap(); + } - assert_eq!(chunk_keys.len(), batch_response.len()); + thread_sender.unbounded_send(Message::Terminated).unwrap(); + thread_key_values + }); - for (key, maybe_value) in chunk_keys.into_iter().zip(batch_response) { - match maybe_value { - Ok(Some(v)) => { - key_values.push((key.clone(), v)); - }, - Ok(None) => { - log::warn!( - target: LOG_TARGET, - "key {:?} had none corresponding value.", - &key - ); - key_values.push((key.clone(), StorageData(vec![]))); - }, - Err(e) => { - log::error!(target: LOG_TARGET, "key {:?} failed: {:?}", &key, e); - batch_success = false; - }, - }; + handles.push(handle); + } - if key_values.len() % (10 * BATCH_SIZE) == 0 { - let ratio: f64 = key_values.len() as f64 / keys_count as f64; - log::debug!( - target: LOG_TARGET, - "progress = {:.2} [{} / {}]", - ratio, - key_values.len(), - keys_count, - ); - } + // first, wait until all threads send a `Terminated` message, in the meantime populate + // `pending_ext`. + let mut terminated = 0usize; + let mut batch_failed = false; + loop { + match rx.next().await.unwrap() { + Message::Batch(kv) => { + for (k, v) in kv { + // skip writing the child root data. + if is_default_child_storage_key(k.as_ref()) { + continue + } + pending_ext.insert(k, v); + } + }, + Message::BatchFailed(error) => { + log::error!(target: LOG_TARGET, "Batch processing failed: {:?}", error); + batch_failed = true; + break + }, + Message::Terminated => { + terminated += 1; + if terminated == handles.len() { + break + } + }, } } - if batch_success { - Ok(key_values) - } else { - Err("batch failed.") + // Ensure all threads finished execution before returning. + let keys_and_values = + handles.into_iter().flat_map(|h| h.join().unwrap()).collect::>(); + + if batch_failed { + return Err("Batch failed.") } + + Ok(keys_and_values) } /// Get the values corresponding to `child_keys` at the given `prefixed_top_key`. pub(crate) async fn rpc_child_get_storage_paged( - &self, + client: &WsClient, prefixed_top_key: &StorageKey, child_keys: Vec, at: B::Hash, @@ -410,7 +562,7 @@ where let mut child_kv_inner = vec![]; let mut batch_success = true; - for batch_child_key in child_keys.chunks(BATCH_SIZE) { + for batch_child_key in child_keys.chunks(DEFAULT_VALUE_DOWNLOAD_BATCH) { let mut batch_request = BatchRequestBuilder::new(); for key in batch_child_key { @@ -426,12 +578,8 @@ where .map_err(|_| "Invalid batch params")?; } - let batch_response = self - .as_online() - .rpc_client() - .batch_request::>(batch_request) - .await - .map_err(|e| { + let batch_response = + client.batch_request::>(batch_request).await.map_err(|e| { log::error!( target: LOG_TARGET, "failed to execute batch: {:?}. Error: {:?}", @@ -472,7 +620,7 @@ where } pub(crate) async fn rpc_child_get_keys( - &self, + client: &WsClient, prefixed_top_key: &StorageKey, child_prefix: StorageKey, at: B::Hash, @@ -480,7 +628,7 @@ where // This is deprecated and will generate a warning which causes the CI to fail. #[allow(warnings)] let child_keys = substrate_rpc_client::ChildStateApi::storage_keys( - self.as_online().rpc_client(), + client, PrefixedStorageKey::new(prefixed_top_key.as_ref().to_vec()), child_prefix, Some(at), @@ -493,7 +641,8 @@ where debug!( target: LOG_TARGET, - "scraped {} child-keys of the child-bearing top key: {}", + "[thread = {:?}] scraped {} child-keys of the child-bearing top key: {}", + std::thread::current().id(), child_keys.len(), HexDisplay::from(prefixed_top_key) ); @@ -502,214 +651,341 @@ where } } -// Internal methods -impl Builder +impl Builder where B::Hash: DeserializeOwned, B::Header: DeserializeOwned, { - /// Save the given data to the top keys snapshot. - fn save_top_snapshot(&self, data: &[KeyValue], path: &PathBuf) -> Result<(), &'static str> { - let mut path = path.clone(); - let encoded = data.encode(); - path.set_extension("top"); - debug!( - target: LOG_TARGET, - "writing {} bytes to state snapshot file {:?}", - encoded.len(), - path - ); - fs::write(path, encoded).map_err(|_| "fs::write failed.")?; - Ok(()) - } - - /// Save the given data to the child keys snapshot. - fn save_child_snapshot( - &self, - data: &ChildKeyValues, - path: &PathBuf, - ) -> Result<(), &'static str> { - let mut path = path.clone(); - path.set_extension("child"); - let encoded = data.encode(); - debug!( - target: LOG_TARGET, - "writing {} bytes to state snapshot file {:?}", - encoded.len(), - path - ); - fs::write(path, encoded).map_err(|_| "fs::write failed.")?; - Ok(()) - } - - fn load_top_snapshot(&self, path: &PathBuf) -> Result { - let mut path = path.clone(); - path.set_extension("top"); - info!(target: LOG_TARGET, "loading top key-pairs from snapshot {:?}", path); - let bytes = fs::read(path).map_err(|_| "fs::read failed.")?; - Decode::decode(&mut &*bytes).map_err(|e| { - log::error!(target: LOG_TARGET, "{:?}", e); - "decode failed" - }) - } - - fn load_child_snapshot(&self, path: &PathBuf) -> Result { - let mut path = path.clone(); - path.set_extension("child"); - info!(target: LOG_TARGET, "loading child key-pairs from snapshot {:?}", path); - let bytes = fs::read(path).map_err(|_| "fs::read failed.")?; - Decode::decode(&mut &*bytes).map_err(|e| { - log::error!(target: LOG_TARGET, "{:?}", e); - "decode failed" - }) - } - - /// Load all the `top` keys from the remote config, and maybe write then to cache. - async fn load_top_remote_and_maybe_save(&self) -> Result { - let top_kv = self.load_top_remote().await?; - if let Some(c) = &self.as_online().state_snapshot { - self.save_top_snapshot(&top_kv, &c.path)?; - } - Ok(top_kv) - } - /// Load all of the child keys from the remote config, given the already scraped list of top key /// pairs. /// - /// Stores all values to cache as well, if provided. - async fn load_child_remote_and_maybe_save( + /// `top_kv` need not be only child-bearing top keys. It should be all of the top keys that are + /// included thus far. + /// + /// This function concurrently populates `pending_ext`. the return value is only for writing to + /// cache, we can also optimize further. + async fn load_child_remote( &self, top_kv: &[KeyValue], + pending_ext: &mut TestExternalities, ) -> Result { - let child_kv = self.load_child_remote(top_kv).await?; - if let Some(c) = &self.as_online().state_snapshot { - self.save_child_snapshot(&child_kv, &c.path)?; - } - Ok(child_kv) - } - - /// Load all of the child keys from the remote config, given the already scraped list of top key - /// pairs. - /// - /// `top_kv` need not be only child-bearing top keys. It should be all of the top keys that are - /// included thus far. - async fn load_child_remote(&self, top_kv: &[KeyValue]) -> Result { let child_roots = top_kv - .iter() - .filter_map(|(k, _)| is_default_child_storage_key(k.as_ref()).then(|| k)) + .into_iter() + .filter_map(|(k, _)| is_default_child_storage_key(k.as_ref()).then(|| k.clone())) .collect::>(); + if child_roots.is_empty() { + return Ok(Default::default()) + } + + // div-ceil simulation. + let threads = Self::threads().get(); + let child_roots_per_thread = (child_roots.len() + threads - 1) / threads; + info!( target: LOG_TARGET, - "👩‍👦 scraping child-tree data from {} top keys", - child_roots.len() + "👩‍👦 scraping child-tree data from {} top keys, split among {} threads, {} top keys per thread", + child_roots.len(), + threads, + child_roots_per_thread, ); - let mut child_kv = vec![]; - for prefixed_top_key in child_roots { - let at = self.as_online().at.expect("at must be initialized in online mode."); - let child_keys = - self.rpc_child_get_keys(prefixed_top_key, StorageKey(vec![]), at).await?; - let child_kv_inner = - self.rpc_child_get_storage_paged(prefixed_top_key, child_keys, at).await?; - - let prefixed_top_key = PrefixedStorageKey::new(prefixed_top_key.clone().0); - let un_prefixed = match ChildType::from_prefixed_key(&prefixed_top_key) { - Some((ChildType::ParentKeyId, storage_key)) => storage_key, - None => { - log::error!(target: LOG_TARGET, "invalid key: {:?}", prefixed_top_key); - return Err("Invalid child key") - }, - }; + // NOTE: the threading done here is the simpler, yet slightly un-elegant because we are + // splitting child root among threads, and it is very common for these root to have vastly + // different child tries underneath them, causing some threads to finish way faster than + // others. Certainly still better than single thread though. + let mut handles = vec![]; + let client = self.as_online().rpc_client_cloned(); + let at = self.as_online().at_expected(); + + enum Message { + Terminated, + Batch((ChildInfo, Vec<(Vec, Vec)>)), + } + let (tx, mut rx) = mpsc::unbounded::(); + + for thread_child_roots in child_roots + .chunks(child_roots_per_thread) + .map(|x| x.into()) + .collect::>>() + { + let thread_client = client.clone(); + let thread_sender = tx.clone(); + let handle = thread::spawn(move || { + let rt = tokio::runtime::Runtime::new().unwrap(); + let mut thread_child_kv = vec![]; + for prefixed_top_key in thread_child_roots { + let child_keys = rt.block_on(Self::rpc_child_get_keys( + &thread_client, + &prefixed_top_key, + StorageKey(vec![]), + at, + ))?; + let child_kv_inner = rt.block_on(Self::rpc_child_get_storage_paged( + &thread_client, + &prefixed_top_key, + child_keys, + at, + ))?; + + let prefixed_top_key = PrefixedStorageKey::new(prefixed_top_key.clone().0); + let un_prefixed = match ChildType::from_prefixed_key(&prefixed_top_key) { + Some((ChildType::ParentKeyId, storage_key)) => storage_key, + None => { + log::error!(target: LOG_TARGET, "invalid key: {:?}", prefixed_top_key); + return Err("Invalid child key") + }, + }; + + thread_sender + .unbounded_send(Message::Batch(( + ChildInfo::new_default(un_prefixed), + child_kv_inner + .iter() + .cloned() + .map(|(k, v)| (k.0, v.0)) + .collect::>(), + ))) + .unwrap(); + thread_child_kv.push((ChildInfo::new_default(un_prefixed), child_kv_inner)); + } + + thread_sender.unbounded_send(Message::Terminated).unwrap(); + Ok(thread_child_kv) + }); + handles.push(handle); + } - child_kv.push((ChildInfo::new_default(un_prefixed), child_kv_inner)); + // first, wait until all threads send a `Terminated` message, in the meantime populate + // `pending_ext`. + let mut terminated = 0usize; + loop { + match rx.next().await.unwrap() { + Message::Batch((info, kvs)) => + for (k, v) in kvs { + pending_ext.insert_child(info.clone(), k, v); + }, + Message::Terminated => { + terminated += 1; + if terminated == handles.len() { + break + } + }, + } } + let child_kv = handles + .into_iter() + .flat_map(|h| h.join().unwrap()) + .flatten() + .collect::>(); Ok(child_kv) } /// Build `Self` from a network node denoted by `uri`. - async fn load_top_remote(&self) -> Result { + /// + /// This function concurrently populates `pending_ext`. the return value is only for writing to + /// cache, we can also optimize further. + async fn load_top_remote( + &self, + pending_ext: &mut TestExternalities, + ) -> Result { let config = self.as_online(); let at = self .as_online() .at .expect("online config must be initialized by this point; qed."); - log::info!(target: LOG_TARGET, "scraping key-pairs from remote @ {:?}", at); - - let mut keys_and_values = if config.pallets.len() > 0 { - let mut filtered_kv = vec![]; - for p in config.pallets.iter() { - let hashed_prefix = StorageKey(twox_128(p.as_bytes()).to_vec()); - let pallet_kv = self.rpc_get_pairs_paged(hashed_prefix.clone(), at).await?; - log::info!( - target: LOG_TARGET, - "downloaded data for module {} (count: {} / prefix: {}).", - p, - pallet_kv.len(), - HexDisplay::from(&hashed_prefix), - ); - filtered_kv.extend(pallet_kv); - } - filtered_kv - } else { - log::info!(target: LOG_TARGET, "downloading data for all pallets."); - self.rpc_get_pairs_paged(StorageKey(vec![]), at).await? - }; + log::info!(target: LOG_TARGET, "scraping key-pairs from remote at block height {:?}", at); - for prefix in &self.hashed_prefixes { + let mut keys_and_values = Vec::new(); + for prefix in &config.hashed_prefixes { + let now = std::time::Instant::now(); + let additional_key_values = + self.rpc_get_pairs_paged(StorageKey(prefix.to_vec()), at, pending_ext).await?; + let elapsed = now.elapsed(); log::info!( target: LOG_TARGET, - "adding data for hashed prefix: {:?}", - HexDisplay::from(prefix) + "adding data for hashed prefix: {:?}, took {:?}s", + HexDisplay::from(prefix), + elapsed.as_secs() ); - let additional_key_values = - self.rpc_get_pairs_paged(StorageKey(prefix.to_vec()), at).await?; keys_and_values.extend(additional_key_values); } - for key in &self.hashed_keys { + for key in &config.hashed_keys { let key = StorageKey(key.to_vec()); log::info!( target: LOG_TARGET, "adding data for hashed key: {:?}", HexDisplay::from(&key) ); - let value = self.rpc_get_storage(key.clone(), Some(at)).await?; - keys_and_values.push((key, value)); + match self.rpc_get_storage(key.clone(), Some(at)).await? { + Some(value) => { + pending_ext.insert(key.clone().0, value.clone().0); + keys_and_values.push((key, value)); + }, + None => { + log::warn!( + target: LOG_TARGET, + "no data found for hashed key: {:?}", + HexDisplay::from(&key) + ); + }, + } } Ok(keys_and_values) } - pub(crate) async fn init_remote_client(&mut self) -> Result<(), &'static str> { + /// The entry point of execution, if `mode` is online. + /// + /// initializes the remote client in `transport`, and sets the `at` field, if not specified. + async fn init_remote_client(&mut self) -> Result<(), &'static str> { // First, initialize the ws client. self.as_online_mut().transport.map_uri().await?; // Then, if `at` is not set, set it. if self.as_online().at.is_none() { let at = self.rpc_get_head().await?; + log::info!( + target: LOG_TARGET, + "since no at is provided, setting it to latest finalized head, {:?}", + at + ); self.as_online_mut().at = Some(at); } + // Then, a few transformation that we want to perform in the online config: + let online_config = self.as_online_mut(); + online_config + .pallets + .iter() + .for_each(|p| online_config.hashed_prefixes.push(twox_128(p.as_bytes()).to_vec())); + + if online_config.child_trie { + online_config.hashed_prefixes.push(DEFAULT_CHILD_STORAGE_KEY_PREFIX.to_vec()); + } + + // Finally, if by now, we have put any limitations on prefixes that we are interested in, we + // download everything. + if online_config + .hashed_prefixes + .iter() + .filter(|p| *p != DEFAULT_CHILD_STORAGE_KEY_PREFIX) + .count() == 0 + { + log::info!( + target: LOG_TARGET, + "since no prefix is filtered, the data for all pallets will be downloaded" + ); + online_config.hashed_prefixes.push(vec![]); + } + Ok(()) } - pub(crate) async fn pre_build( - mut self, - ) -> Result<(TopKeyValues, ChildKeyValues), &'static str> { - let mut top_kv = match self.mode.clone() { - Mode::Offline(config) => self.load_top_snapshot(&config.state_snapshot.path)?, - Mode::Online(_) => { - self.init_remote_client().await?; - self.load_top_remote_and_maybe_save().await? - }, + /// Load the data from a remote server. The main code path is calling into `load_top_remote` and + /// `load_child_remote`. + /// + /// Must be called after `init_remote_client`. + async fn load_remote_and_maybe_save(&mut self) -> Result { + let state_version = + StateApi::::runtime_version(self.as_online().rpc_client(), None) + .await + .map_err(|e| { + error!(target: LOG_TARGET, "Error = {:?}", e); + "rpc runtime_version failed." + }) + .map(|v| v.state_version())?; + let mut pending_ext = TestExternalities::new_with_code_and_state( + Default::default(), + Default::default(), + self.overwrite_state_version.unwrap_or(state_version), + ); + let top_kv = self.load_top_remote(&mut pending_ext).await?; + let child_kv = self.load_child_remote(&top_kv, &mut pending_ext).await?; + + if let Some(path) = self.as_online().state_snapshot.clone().map(|c| c.path) { + let snapshot = Snapshot:: { + state_version, + top: top_kv, + child: child_kv, + block_hash: self + .as_online() + .at + .expect("set to `Some` in `init_remote_client`; must be called before; qed"), + }; + let encoded = snapshot.encode(); + log::info!( + target: LOG_TARGET, + "writing snapshot of {} bytes to {:?}", + encoded.len(), + path + ); + std::fs::write(path, encoded).map_err(|_| "fs::write failed")?; + } + + Ok(pending_ext) + } + + fn load_snapshot(&mut self, path: PathBuf) -> Result, &'static str> { + info!(target: LOG_TARGET, "loading data from snapshot {:?}", path); + let bytes = fs::read(path).map_err(|_| "fs::read failed.")?; + Decode::decode(&mut &*bytes).map_err(|_| "decode failed") + } + + async fn do_load_remote(&mut self) -> Result, &'static str> { + self.init_remote_client().await?; + let block_hash = self.as_online().at_expected(); + let inner_ext = self.load_remote_and_maybe_save().await?; + Ok(RemoteExternalities { block_hash, inner_ext }) + } + + fn do_load_offline( + &mut self, + config: OfflineConfig, + ) -> Result, &'static str> { + let Snapshot { block_hash, top, child, state_version } = + self.load_snapshot(config.state_snapshot.path.clone())?; + + let mut inner_ext = TestExternalities::new_with_code_and_state( + Default::default(), + Default::default(), + self.overwrite_state_version.unwrap_or(state_version), + ); + + info!(target: LOG_TARGET, "injecting a total of {} top keys", top.len()); + for (k, v) in top { + // skip writing the child root data. + if is_default_child_storage_key(k.as_ref()) { + continue + } + inner_ext.insert(k.0, v.0); + } + + info!( + target: LOG_TARGET, + "injecting a total of {} child keys", + child.iter().flat_map(|(_, kv)| kv).count() + ); + + for (info, key_values) in child { + for (k, v) in key_values { + inner_ext.insert_child(info.clone(), k.0, v.0); + } + } + + Ok(RemoteExternalities { inner_ext, block_hash }) + } + + pub(crate) async fn pre_build(mut self) -> Result, &'static str> { + let mut ext = match self.mode.clone() { + Mode::Offline(config) => self.do_load_offline(config)?, + Mode::Online(_) => self.do_load_remote().await?, Mode::OfflineOrElseOnline(offline_config, _) => { - if let Ok(kv) = self.load_top_snapshot(&offline_config.state_snapshot.path) { - kv - } else { - self.init_remote_client().await?; - self.load_top_remote_and_maybe_save().await? + match self.do_load_offline(offline_config) { + Ok(x) => x, + Err(_) => self.do_load_remote().await?, } }, }; @@ -721,7 +997,9 @@ where "extending externalities with {} manually injected key-values", self.hashed_key_values.len() ); - top_kv.extend(self.hashed_key_values.clone()); + for (k, v) in self.hashed_key_values { + ext.insert(k.0, v.0); + } } // exclude manual key values. @@ -731,87 +1009,34 @@ where "excluding externalities from {} keys", self.hashed_blacklist.len() ); - top_kv.retain(|(k, _)| !self.hashed_blacklist.contains(&k.0)) + for k in self.hashed_blacklist { + ext.execute_with(|| sp_io::storage::clear(&k)); + } } - let child_kv = match self.mode.clone() { - Mode::Online(_) => self.load_child_remote_and_maybe_save(&top_kv).await?, - Mode::OfflineOrElseOnline(offline_config, _) => { - if let Ok(kv) = self.load_child_snapshot(&offline_config.state_snapshot.path) { - kv - } else { - self.load_child_remote_and_maybe_save(&top_kv).await? - } - }, - Mode::Offline(ref config) => self - .load_child_snapshot(&config.state_snapshot.path) - .map_err(|why| { - log::warn!( - target: LOG_TARGET, - "failed to load child-key file due to {:?}.", - why - ) - }) - .unwrap_or_default(), - }; - - Ok((top_kv, child_kv)) + Ok(ext) } } // Public methods -impl Builder { +impl Builder +where + B::Hash: DeserializeOwned, + B::Header: DeserializeOwned, +{ /// Create a new builder. pub fn new() -> Self { Default::default() } /// Inject a manual list of key and values to the storage. - pub fn inject_hashed_key_value(mut self, injections: &[KeyValue]) -> Self { + pub fn inject_hashed_key_value(mut self, injections: Vec) -> Self { for i in injections { self.hashed_key_values.push(i.clone()); } self } - /// Inject a hashed prefix. This is treated as-is, and should be pre-hashed. - /// - /// Only relevant is `Mode::Online` is being used. Noop otherwise. - /// - /// This should be used to inject a "PREFIX", like a storage (double) map. - pub fn inject_hashed_prefix(mut self, hashed: &[u8]) -> Self { - self.hashed_prefixes.push(hashed.to_vec()); - self - } - - /// Just a utility wrapper of [`Self::inject_hashed_prefix`] that injects - /// [`DEFAULT_CHILD_STORAGE_KEY_PREFIX`] as a prefix. - /// - /// Only relevant is `Mode::Online` is being used. Noop otherwise. - /// - /// If set, this will guarantee that the child-tree data of ALL pallets will be downloaded. - /// - /// This is not needed if the entire state is being downloaded. - /// - /// Otherwise, the only other way to make sure a child-tree is manually included is to inject - /// its root (`DEFAULT_CHILD_STORAGE_KEY_PREFIX`, plus some other postfix) into - /// [`Self::inject_hashed_key`]. Unfortunately, there's no federated way of managing child tree - /// roots as of now and each pallet does its own thing. Therefore, it is not possible for this - /// library to automatically include child trees of pallet X, when its top keys are included. - pub fn inject_default_child_tree_prefix(self) -> Self { - self.inject_hashed_prefix(DEFAULT_CHILD_STORAGE_KEY_PREFIX) - } - - /// Inject a hashed key to scrape. This is treated as-is, and should be pre-hashed. - /// - /// Only relevant is `Mode::Online` is being used. Noop otherwise. - /// - /// This should be used to inject a "KEY", like a storage value. - pub fn inject_hashed_key(mut self, hashed: &[u8]) -> Self { - self.hashed_keys.push(hashed.to_vec()); - self - } - /// Blacklist this hashed key from the final externalities. This is treated as-is, and should be /// pre-hashed. pub fn blacklist_hashed_key(mut self, hashed: &[u8]) -> Self { @@ -826,64 +1051,20 @@ impl Builder { } /// The state version to use. - pub fn state_version(mut self, version: StateVersion) -> Self { - self.state_version = version; - self - } - - /// overwrite the `at` value, if `mode` is set to [`Mode::Online`]. - /// - /// noop if `mode` is [`Mode::Offline`] - pub fn overwrite_online_at(mut self, at: B::Hash) -> Self { - if let Mode::Online(mut online) = self.mode.clone() { - online.at = Some(at); - self.mode = Mode::Online(online); - } + pub fn overwrite_state_version(mut self, version: StateVersion) -> Self { + self.overwrite_state_version = Some(version); self } -} - -// Public methods -impl Builder -where - B::Header: DeserializeOwned, -{ - /// Build the test externalities. - pub async fn build(self) -> Result { - let state_version = self.state_version; - let (top_kv, child_kv) = self.pre_build().await?; - let mut ext = TestExternalities::new_with_code_and_state( - Default::default(), - Default::default(), - state_version, - ); - - info!(target: LOG_TARGET, "injecting a total of {} top keys", top_kv.len()); - for (k, v) in top_kv { - // skip writing the child root data. - if is_default_child_storage_key(k.as_ref()) { - continue - } - ext.insert(k.0, v.0); - } - - info!( - target: LOG_TARGET, - "injecting a total of {} child keys", - child_kv.iter().flat_map(|(_, kv)| kv).count() - ); - - for (info, key_values) in child_kv { - for (k, v) in key_values { - ext.insert_child(info.clone(), k.0, v.0); - } - } + pub async fn build(self) -> Result, &'static str> { + let mut ext = self.pre_build().await?; ext.commit_all().unwrap(); + info!( target: LOG_TARGET, - "initialized state externalities with storage root {:?}", - ext.as_backend().root() + "initialized state externalities with storage root {:?} and state_version {:?}", + ext.as_backend().root(), + ext.state_version ); Ok(ext) @@ -892,16 +1073,16 @@ where #[cfg(test)] mod test_prelude { + use tracing_subscriber::EnvFilter; + pub(crate) use super::*; pub(crate) use sp_runtime::testing::{Block as RawBlock, ExtrinsicWrapper, H256 as Hash}; - pub(crate) type Block = RawBlock>; pub(crate) fn init_logger() { - let _ = env_logger::Builder::from_default_env() - .format_module_path(true) - .format_level(true) - .filter_module(LOG_TARGET, log::LevelFilter::Debug) + let _ = tracing_subscriber::fmt() + .with_env_filter(EnvFilter::from_default_env()) + .with_level(true) .try_init(); } } @@ -910,7 +1091,7 @@ mod test_prelude { mod tests { use super::test_prelude::*; - #[tokio::test] + #[tokio::test(flavor = "multi_thread")] async fn can_load_state_snapshot() { init_logger(); Builder::::new() @@ -919,15 +1100,15 @@ mod tests { })) .build() .await - .expect("Can't read state snapshot file") + .unwrap() .execute_with(|| {}); } - #[tokio::test] - async fn can_exclude_from_cache() { + #[tokio::test(flavor = "multi_thread")] + async fn can_exclude_from_snapshot() { init_logger(); - // get the first key from the cache file. + // get the first key from the snapshot file. let some_key = Builder::::new() .mode(Mode::Offline(OfflineConfig { state_snapshot: SnapshotConfig::new("test_data/proxy_test"), @@ -957,19 +1138,88 @@ mod tests { #[cfg(all(test, feature = "remote-test"))] mod remote_tests { use super::test_prelude::*; + use std::os::unix::fs::MetadataExt; + + #[tokio::test(flavor = "multi_thread")] + async fn state_version_is_kept_and_can_be_altered() { + const CACHE: &'static str = "state_version_is_kept_and_can_be_altered"; + init_logger(); + + // first, build a snapshot. + let ext = Builder::::new() + .mode(Mode::Online(OnlineConfig { + pallets: vec!["Proxy".to_owned()], + child_trie: false, + state_snapshot: Some(SnapshotConfig::new(CACHE)), + ..Default::default() + })) + .build() + .await + .unwrap(); + + // now re-create the same snapshot. + let cached_ext = Builder::::new() + .mode(Mode::Offline(OfflineConfig { state_snapshot: SnapshotConfig::new(CACHE) })) + .build() + .await + .unwrap(); + + assert_eq!(ext.state_version, cached_ext.state_version); + + // now overwrite it + let other = match ext.state_version { + StateVersion::V0 => StateVersion::V1, + StateVersion::V1 => StateVersion::V0, + }; + let cached_ext = Builder::::new() + .mode(Mode::Offline(OfflineConfig { state_snapshot: SnapshotConfig::new(CACHE) })) + .overwrite_state_version(other) + .build() + .await + .unwrap(); + + assert_eq!(cached_ext.state_version, other); + } + + #[tokio::test(flavor = "multi_thread")] + async fn snapshot_block_hash_works() { + const CACHE: &'static str = "snapshot_block_hash_works"; + init_logger(); + + // first, build a snapshot. + let ext = Builder::::new() + .mode(Mode::Online(OnlineConfig { + pallets: vec!["Proxy".to_owned()], + child_trie: false, + state_snapshot: Some(SnapshotConfig::new(CACHE)), + ..Default::default() + })) + .build() + .await + .unwrap(); - #[tokio::test] + // now re-create the same snapshot. + let cached_ext = Builder::::new() + .mode(Mode::Offline(OfflineConfig { state_snapshot: SnapshotConfig::new(CACHE) })) + .build() + .await + .unwrap(); + + assert_eq!(ext.block_hash, cached_ext.block_hash); + } + + #[tokio::test(flavor = "multi_thread")] async fn offline_else_online_works() { + const CACHE: &'static str = "offline_else_online_works_data"; init_logger(); - // this shows that in the second run, we use the remote and create a cache. + // this shows that in the second run, we use the remote and create a snapshot. Builder::::new() .mode(Mode::OfflineOrElseOnline( - OfflineConfig { - state_snapshot: SnapshotConfig::new("offline_else_online_works_data"), - }, + OfflineConfig { state_snapshot: SnapshotConfig::new(CACHE) }, OnlineConfig { pallets: vec!["Proxy".to_owned()], - state_snapshot: Some(SnapshotConfig::new("offline_else_online_works_data")), + child_trie: false, + state_snapshot: Some(SnapshotConfig::new(CACHE)), ..Default::default() }, )) @@ -981,12 +1231,8 @@ mod remote_tests { // this shows that in the second run, we are not using the remote Builder::::new() .mode(Mode::OfflineOrElseOnline( - OfflineConfig { - state_snapshot: SnapshotConfig::new("offline_else_online_works_data"), - }, + OfflineConfig { state_snapshot: SnapshotConfig::new(CACHE) }, OnlineConfig { - pallets: vec!["Proxy".to_owned()], - state_snapshot: Some(SnapshotConfig::new("offline_else_online_works_data")), transport: "ws://non-existent:666".to_owned().into(), ..Default::default() }, @@ -1000,51 +1246,20 @@ mod remote_tests { .unwrap() .into_iter() .map(|d| d.unwrap()) - .filter(|p| { - p.path().file_name().unwrap_or_default() == "offline_else_online_works_data" || - p.path().extension().unwrap_or_default() == "top" || - p.path().extension().unwrap_or_default() == "child" - }) + .filter(|p| p.path().file_name().unwrap_or_default() == CACHE) .collect::>(); - assert!(to_delete.len() > 0); - for d in to_delete { - std::fs::remove_file(d.path()).unwrap(); - } - } - #[tokio::test] - #[ignore = "too slow"] - async fn can_build_one_big_pallet() { - init_logger(); - Builder::::new() - .mode(Mode::Online(OnlineConfig { - pallets: vec!["System".to_owned()], - ..Default::default() - })) - .build() - .await - .unwrap() - .execute_with(|| {}); + assert!(to_delete.len() == 1); + std::fs::remove_file(to_delete[0].path()).unwrap(); } - #[tokio::test] + #[tokio::test(flavor = "multi_thread")] async fn can_build_one_small_pallet() { init_logger(); Builder::::new() .mode(Mode::Online(OnlineConfig { - transport: "wss://kusama-rpc.polkadot.io:443".to_owned().into(), - pallets: vec!["Council".to_owned()], - ..Default::default() - })) - .build() - .await - .unwrap() - .execute_with(|| {}); - - Builder::::new() - .mode(Mode::Online(OnlineConfig { - transport: "wss://rpc.polkadot.io:443".to_owned().into(), - pallets: vec!["Council".to_owned()], + pallets: vec!["Proxy".to_owned()], + child_trie: false, ..Default::default() })) .build() @@ -1053,24 +1268,13 @@ mod remote_tests { .execute_with(|| {}); } - #[tokio::test] + #[tokio::test(flavor = "multi_thread")] async fn can_build_few_pallet() { init_logger(); Builder::::new() .mode(Mode::Online(OnlineConfig { - transport: "wss://kusama-rpc.polkadot.io:443".to_owned().into(), - pallets: vec!["Proxy".to_owned(), "Multisig".to_owned()], - ..Default::default() - })) - .build() - .await - .unwrap() - .execute_with(|| {}); - - Builder::::new() - .mode(Mode::Online(OnlineConfig { - transport: "wss://rpc.polkadot.io:443".to_owned().into(), pallets: vec!["Proxy".to_owned(), "Multisig".to_owned()], + child_trie: false, ..Default::default() })) .build() @@ -1079,13 +1283,16 @@ mod remote_tests { .execute_with(|| {}); } - #[tokio::test] - async fn can_create_top_snapshot() { + #[tokio::test(flavor = "multi_thread")] + async fn can_create_snapshot() { + const CACHE: &'static str = "can_create_snapshot"; init_logger(); + Builder::::new() .mode(Mode::Online(OnlineConfig { - state_snapshot: Some(SnapshotConfig::new("can_create_top_snapshot_data")), + state_snapshot: Some(SnapshotConfig::new(CACHE)), pallets: vec!["Proxy".to_owned()], + child_trie: false, ..Default::default() })) .build() @@ -1097,38 +1304,29 @@ mod remote_tests { .unwrap() .into_iter() .map(|d| d.unwrap()) - .filter(|p| { - p.path().file_name().unwrap_or_default() == "can_create_top_snapshot_data" || - p.path().extension().unwrap_or_default() == "top" || - p.path().extension().unwrap_or_default() == "child" - }) + .filter(|p| p.path().file_name().unwrap_or_default() == CACHE) .collect::>(); - assert!(to_delete.len() > 0); + let snap: Snapshot = Builder::::new().load_snapshot(CACHE.into()).unwrap(); + assert!(matches!(snap, Snapshot { top, child, .. } if top.len() > 0 && child.len() == 0)); - for d in to_delete { - use std::os::unix::fs::MetadataExt; - if d.path().extension().unwrap_or_default() == "top" { - // if this is the top snapshot it must not be empty. - assert!(std::fs::metadata(d.path()).unwrap().size() > 1); - } else { - // the child is empty for this pallet. - assert!(std::fs::metadata(d.path()).unwrap().size() == 1); - } - std::fs::remove_file(d.path()).unwrap(); - } + assert!(to_delete.len() == 1); + let to_delete = to_delete.first().unwrap(); + assert!(std::fs::metadata(to_delete.path()).unwrap().size() > 1); + std::fs::remove_file(to_delete.path()).unwrap(); } - #[tokio::test] + #[tokio::test(flavor = "multi_thread")] async fn can_create_child_snapshot() { + const CACHE: &'static str = "can_create_child_snapshot"; init_logger(); Builder::::new() .mode(Mode::Online(OnlineConfig { - state_snapshot: Some(SnapshotConfig::new("can_create_child_snapshot_data")), + state_snapshot: Some(SnapshotConfig::new(CACHE)), pallets: vec!["Crowdloan".to_owned()], + child_trie: true, ..Default::default() })) - .inject_default_child_tree_prefix() .build() .await .unwrap() @@ -1138,72 +1336,46 @@ mod remote_tests { .unwrap() .into_iter() .map(|d| d.unwrap()) - .filter(|p| { - p.path().file_name().unwrap_or_default() == "can_create_child_snapshot_data" || - p.path().extension().unwrap_or_default() == "top" || - p.path().extension().unwrap_or_default() == "child" - }) + .filter(|p| p.path().file_name().unwrap_or_default() == CACHE) .collect::>(); - assert!(to_delete.len() > 0); + let snap: Snapshot = Builder::::new().load_snapshot(CACHE.into()).unwrap(); + assert!(matches!(snap, Snapshot { top, child, .. } if top.len() > 0 && child.len() > 0)); - for d in to_delete { - use std::os::unix::fs::MetadataExt; - // if this is the top snapshot it must not be empty - if d.path().extension().unwrap_or_default() == "child" { - assert!(std::fs::metadata(d.path()).unwrap().size() > 1); - } else { - assert!(std::fs::metadata(d.path()).unwrap().size() > 1); - } - std::fs::remove_file(d.path()).unwrap(); - } + assert!(to_delete.len() == 1); + let to_delete = to_delete.first().unwrap(); + assert!(std::fs::metadata(to_delete.path()).unwrap().size() > 1); + std::fs::remove_file(to_delete.path()).unwrap(); } - #[tokio::test] - async fn can_fetch_all() { + #[tokio::test(flavor = "multi_thread")] + async fn can_build_big_pallet() { + if std::option_env!("TEST_WS").is_none() { + return + } init_logger(); Builder::::new() .mode(Mode::Online(OnlineConfig { - state_snapshot: Some(SnapshotConfig::new("can_fetch_all_data")), + transport: std::option_env!("TEST_WS").unwrap().to_owned().into(), + pallets: vec!["Staking".to_owned()], + child_trie: false, ..Default::default() })) .build() .await .unwrap() .execute_with(|| {}); - - let to_delete = std::fs::read_dir(Path::new(".")) - .unwrap() - .into_iter() - .map(|d| d.unwrap()) - .filter(|p| { - p.path().file_name().unwrap_or_default() == "can_fetch_all_data" || - p.path().extension().unwrap_or_default() == "top" || - p.path().extension().unwrap_or_default() == "child" - }) - .collect::>(); - - assert!(to_delete.len() > 0); - - for d in to_delete { - use std::os::unix::fs::MetadataExt; - // if we download everything, child tree must also be filled. - if d.path().extension().unwrap_or_default() == "child" { - assert!(std::fs::metadata(d.path()).unwrap().size() > 1); - } else { - assert!(std::fs::metadata(d.path()).unwrap().size() > 1); - } - std::fs::remove_file(d.path()).unwrap(); - } } - #[tokio::test] - async fn can_build_child_tree() { + #[tokio::test(flavor = "multi_thread")] + async fn can_fetch_all() { + if std::option_env!("TEST_WS").is_none() { + return + } init_logger(); Builder::::new() .mode(Mode::Online(OnlineConfig { - transport: "wss://rpc.polkadot.io:443".to_owned().into(), - pallets: vec!["Crowdloan".to_owned()], + transport: std::option_env!("TEST_WS").unwrap().to_owned().into(), ..Default::default() })) .build() diff --git a/utils/frame/remote-externalities/test_data/proxy_test b/utils/frame/remote-externalities/test_data/proxy_test new file mode 100644 index 0000000000000..6673bd6765ad8 Binary files /dev/null and b/utils/frame/remote-externalities/test_data/proxy_test differ diff --git a/utils/frame/remote-externalities/test_data/proxy_test.top b/utils/frame/remote-externalities/test_data/proxy_test.top deleted file mode 100644 index 548ce9cdba4f1..0000000000000 Binary files a/utils/frame/remote-externalities/test_data/proxy_test.top and /dev/null differ diff --git a/utils/frame/rpc/client/src/lib.rs b/utils/frame/rpc/client/src/lib.rs index a211fc6c6983e..a6f73ba6784b2 100644 --- a/utils/frame/rpc/client/src/lib.rs +++ b/utils/frame/rpc/client/src/lib.rs @@ -46,6 +46,7 @@ pub use jsonrpsee::{ core::{ client::{ClientT, Subscription, SubscriptionClientT}, params::BatchRequestBuilder, + Error, RpcResult, }, rpc_params, ws_client::{WsClient, WsClientBuilder}, diff --git a/utils/frame/try-runtime/cli/Cargo.toml b/utils/frame/try-runtime/cli/Cargo.toml index 2b095fc9419b9..d6f211392c6cf 100644 --- a/utils/frame/try-runtime/cli/Cargo.toml +++ b/utils/frame/try-runtime/cli/Cargo.toml @@ -12,11 +12,6 @@ description = "Cli command runtime testing and dry-running" targets = ["x86_64-unknown-linux-gnu"] [dependencies] -clap = { version = "4.0.9", features = ["derive"] } -log = "0.4.17" -parity-scale-codec = "3.0.0" -serde = "1.0.136" -zstd = { version = "0.11.2", default-features = false } remote-externalities = { version = "0.10.0-dev", path = "../../remote-externalities", package = "frame-remote-externalities" } sc-chain-spec = { version = "4.0.0-dev", path = "../../../../client/chain-spec" } sc-cli = { version = "0.10.0-dev", path = "../../../../client/cli" } @@ -27,16 +22,27 @@ sp-externalities = { version = "0.13.0", path = "../../../../primitives/external sp-io = { version = "7.0.0", path = "../../../../primitives/io" } sp-keystore = { version = "0.13.0", path = "../../../../primitives/keystore" } sp-runtime = { version = "7.0.0", path = "../../../../primitives/runtime" } +sp-rpc = { version = "6.0.0", path = "../../../../primitives/rpc" } sp-state-machine = { version = "0.13.0", path = "../../../../primitives/state-machine" } sp-version = { version = "5.0.0", path = "../../../../primitives/version" } +sp-debug-derive = { path = "../../../../primitives/debug-derive" } +sp-api = { path = "../../../../primitives/api" } sp-weights = { version = "4.0.0", path = "../../../../primitives/weights" } frame-try-runtime = { optional = true, path = "../../../../frame/try-runtime" } substrate-rpc-client = { path = "../../rpc/client" } +parity-scale-codec = "3.0.0" +hex = "0.4.3" +clap = { version = "4.0.9", features = ["derive"] } +log = "0.4.17" +serde = "1.0.136" +zstd = { version = "0.11.2", default-features = false } + [dev-dependencies] tokio = "1.22.0" [features] try-runtime = [ + "sp-debug-derive/force-debug", "frame-try-runtime/try-runtime", ] diff --git a/utils/frame/try-runtime/cli/src/commands/create_snapshot.rs b/utils/frame/try-runtime/cli/src/commands/create_snapshot.rs new file mode 100644 index 0000000000000..ef39c3d9846ce --- /dev/null +++ b/utils/frame/try-runtime/cli/src/commands/create_snapshot.rs @@ -0,0 +1,78 @@ +// This file is part of Substrate. + +// Copyright (C) 2022 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use crate::{build_executor, LiveState, SharedParams, State, LOG_TARGET}; +use sc_executor::sp_wasm_interface::HostFunctions; +use sp_runtime::traits::{Block as BlockT, NumberFor}; +use std::{fmt::Debug, str::FromStr}; +use substrate_rpc_client::{ws_client, StateApi}; + +/// Configurations of the [`crate::Command::CreateSnapshot`]. +#[derive(Debug, Clone, clap::Parser)] +pub struct CreateSnapshotCmd { + /// The source of the snapshot. Must be a remote node. + #[clap(flatten)] + pub from: LiveState, + + /// The snapshot path to write to. + /// + /// If not provided `-@.snap` will be used. + pub snapshot_path: Option, +} + +/// inner command for `Command::CreateSnapshot`. +pub(crate) async fn create_snapshot( + shared: SharedParams, + command: CreateSnapshotCmd, +) -> sc_cli::Result<()> +where + Block: BlockT + serde::de::DeserializeOwned, + Block::Hash: FromStr + serde::de::DeserializeOwned, + Block::Header: serde::de::DeserializeOwned, + ::Err: Debug, + NumberFor: FromStr, + as FromStr>::Err: Debug, + HostFns: HostFunctions, +{ + let snapshot_path = command.snapshot_path; + if !matches!(shared.runtime, crate::Runtime::Existing) { + return Err("creating a snapshot is only possible with --runtime existing.".into()) + } + + let path = match snapshot_path { + Some(path) => path, + None => { + let rpc = ws_client(&command.from.uri).await.unwrap(); + let remote_spec = StateApi::::runtime_version(&rpc, None).await.unwrap(); + let path_str = format!( + "{}-{}@{}.snap", + remote_spec.spec_name.to_lowercase(), + remote_spec.spec_version, + command.from.at.clone().unwrap_or("latest".to_owned()) + ); + log::info!(target: LOG_TARGET, "snapshot path not provided (-s), using '{}'", path_str); + path_str.into() + }, + }; + + let executor = build_executor::(&shared); + let _ = State::Live(command.from) + .into_ext::(&shared, &executor, Some(path.into())) + .await?; + + Ok(()) +} diff --git a/utils/frame/try-runtime/cli/src/commands/execute_block.rs b/utils/frame/try-runtime/cli/src/commands/execute_block.rs index 56d88b9cb8919..80d34002fa771 100644 --- a/utils/frame/try-runtime/cli/src/commands/execute_block.rs +++ b/utils/frame/try-runtime/cli/src/commands/execute_block.rs @@ -16,30 +16,25 @@ // limitations under the License. use crate::{ - build_executor, ensure_matching_spec, extract_code, full_extensions, hash_of, local_spec, - state_machine_call_with_proof, SharedParams, State, LOG_TARGET, + build_executor, full_extensions, rpc_err_handler, state_machine_call_with_proof, LiveState, + SharedParams, State, LOG_TARGET, }; use parity_scale_codec::Encode; -use sc_service::{Configuration, NativeExecutionDispatch}; -use sp_core::storage::well_known_keys; -use sp_runtime::traits::{Block as BlockT, Header as HeaderT, NumberFor}; +use sc_executor::sp_wasm_interface::HostFunctions; +use sp_rpc::{list::ListOrValue, number::NumberOrHex}; +use sp_runtime::{ + generic::SignedBlock, + traits::{Block as BlockT, Header as HeaderT, NumberFor}, +}; use std::{fmt::Debug, str::FromStr}; use substrate_rpc_client::{ws_client, ChainApi}; -/// Configurations of the [`Command::ExecuteBlock`]. +/// Configurations of the [`crate::Command::ExecuteBlock`]. /// /// This will always call into `TryRuntime_execute_block`, which can optionally skip the state-root /// check (useful for trying a unreleased runtime), and can execute runtime sanity checks as well. #[derive(Debug, Clone, clap::Parser)] pub struct ExecuteBlockCmd { - /// Overwrite the wasm code in state or not. - #[arg(long)] - overwrite_wasm_code: bool, - - /// If set the state root check is disabled. - #[arg(long)] - no_state_root_check: bool, - /// Which try-state targets to execute when running this command. /// /// Expected values: @@ -49,69 +44,28 @@ pub struct ExecuteBlockCmd { /// `Staking, System`). /// - `rr-[x]` where `[x]` is a number. Then, the given number of pallets are checked in a /// round-robin fashion. - #[arg(long, default_value = "none")] - try_state: frame_try_runtime::TryStateSelect, - - /// The block hash at which to fetch the block. - /// - /// If the `live` state type is being used, then this can be omitted, and is equal to whatever - /// the `state::at` is. Only use this (with care) when combined with a snapshot. - #[arg( - long, - value_parser = crate::parse::hash - )] - block_at: Option, + #[arg(long, default_value = "all")] + pub try_state: frame_try_runtime::TryStateSelect, /// The ws uri from which to fetch the block. /// - /// If the `live` state type is being used, then this can be omitted, and is equal to whatever - /// the `state::uri` is. Only use this (with care) when combined with a snapshot. + /// This will always fetch the next block of whatever `state` is referring to, because this is + /// the only sensible combination. In other words, if you have the state of block `n`, you + /// should execute block `n+1` on top of it. + /// + /// If `state` is `Live`, this can be ignored and the same uri is used for both. #[arg( long, value_parser = crate::parse::url )] - block_ws_uri: Option, + pub block_ws_uri: Option, /// The state type to use. - /// - /// For this command only, if the `live` is used, then state of the parent block is fetched. - /// - /// If `block_at` is provided, then the [`State::Live::at`] is being ignored. #[command(subcommand)] - state: State, + pub state: State, } impl ExecuteBlockCmd { - async fn block_at(&self, ws_uri: String) -> sc_cli::Result - where - Block::Hash: FromStr + serde::de::DeserializeOwned, - ::Err: Debug, - Block::Header: serde::de::DeserializeOwned, - { - let rpc = ws_client(&ws_uri).await?; - - match (&self.block_at, &self.state) { - (Some(block_at), State::Snap { .. }) => hash_of::(block_at), - (Some(block_at), State::Live { .. }) => { - log::warn!(target: LOG_TARGET, "--block-at is provided while state type is live. the `Live::at` will be ignored"); - hash_of::(block_at) - }, - (None, State::Live { at: None, .. }) => { - log::warn!( - target: LOG_TARGET, - "No --block-at or --at provided, using the latest finalized block instead" - ); - ChainApi::<(), Block::Hash, Block::Header, ()>::finalized_head(&rpc) - .await - .map_err(|e| e.to_string().into()) - }, - (None, State::Live { at: Some(at), .. }) => hash_of::(at), - _ => { - panic!("either `--block-at` must be provided, or state must be `live with a proper `--at``"); - }, - } - } - fn block_ws_uri(&self) -> String where Block::Hash: FromStr, @@ -123,7 +77,7 @@ impl ExecuteBlockCmd { log::error!(target: LOG_TARGET, "--block-uri is provided while state type is live, Are you sure you know what you are doing?"); block_ws_uri.to_owned() }, - (None, State::Live { uri, .. }) => uri.clone(), + (None, State::Live(LiveState { uri, .. })) => uri.clone(), (None, State::Snap { .. }) => { panic!("either `--block-uri` must be provided, or state must be `live`"); }, @@ -131,10 +85,9 @@ impl ExecuteBlockCmd { } } -pub(crate) async fn execute_block( +pub(crate) async fn execute_block( shared: SharedParams, command: ExecuteBlockCmd, - config: Configuration, ) -> sc_cli::Result<()> where Block: BlockT + serde::de::DeserializeOwned, @@ -142,79 +95,77 @@ where ::Err: Debug, Block::Hash: serde::de::DeserializeOwned, Block::Header: serde::de::DeserializeOwned, - NumberFor: FromStr, - as FromStr>::Err: Debug, - ExecDispatch: NativeExecutionDispatch + 'static, + as TryInto>::Error: Debug, + HostFns: HostFunctions, { - let executor = build_executor::(&shared, &config); - let execution = shared.execution; + let executor = build_executor::(&shared); + let ext = command.state.into_ext::(&shared, &executor, None).await?; + // get the block number associated with this block. let block_ws_uri = command.block_ws_uri::(); - let block_at = command.block_at::(block_ws_uri.clone()).await?; let rpc = ws_client(&block_ws_uri).await?; - let block: Block = ChainApi::<(), Block::Hash, Block::Header, _>::block(&rpc, Some(block_at)) - .await - .unwrap() - .unwrap(); - let parent_hash = block.header().parent_hash(); - log::info!( - target: LOG_TARGET, - "fetched block #{:?} from {:?}, parent_hash to fetch the state {:?}", - block.header().number(), - block_ws_uri, - parent_hash - ); - - let ext = { - let builder = command - .state - .builder::()? - // make sure the state is being build with the parent hash, if it is online. - .overwrite_online_at(parent_hash.to_owned()) - .state_version(shared.state_version); - - let builder = if command.overwrite_wasm_code { - log::info!( - target: LOG_TARGET, - "replacing the in-storage :code: with the local code from {}'s chain_spec (your local repo)", - config.chain_spec.name(), - ); - let (code_key, code) = extract_code(&config.chain_spec)?; - builder.inject_hashed_key_value(&[(code_key, code)]) - } else { - builder.inject_hashed_key(well_known_keys::CODE) - }; - - builder.build().await? - }; + let next_hash = next_hash_of::(&rpc, ext.block_hash).await?; + + log::info!(target: LOG_TARGET, "fetching next block: {:?} ", next_hash); + + let block = ChainApi::<(), Block::Hash, Block::Header, SignedBlock>::block( + &rpc, + Some(next_hash), + ) + .await + .map_err(rpc_err_handler)? + .expect("header exists, block should also exist; qed") + .block; // A digest item gets added when the runtime is processing the block, so we need to pop // the last one to be consistent with what a gossiped block would contain. let (mut header, extrinsics) = block.deconstruct(); header.digest_mut().pop(); let block = Block::new(header, extrinsics); - let payload = (block.clone(), !command.no_state_root_check, command.try_state).encode(); - - let (expected_spec_name, expected_spec_version, _) = - local_spec::(&ext, &executor); - ensure_matching_spec::( - block_ws_uri.clone(), - expected_spec_name, - expected_spec_version, - shared.no_spec_check_panic, - ) - .await; - let _ = state_machine_call_with_proof::( + // for now, hardcoded for the sake of simplicity. We might customize them one day. + let state_root_check = false; + let signature_check = false; + let payload = (block.clone(), state_root_check, signature_check, command.try_state).encode(); + + let _ = state_machine_call_with_proof::( &ext, &executor, - execution, "TryRuntime_execute_block", &payload, full_extensions(), )?; - log::info!(target: LOG_TARGET, "Core_execute_block executed without errors."); - Ok(()) } + +pub(crate) async fn next_hash_of( + rpc: &substrate_rpc_client::WsClient, + hash: Block::Hash, +) -> sc_cli::Result +where + Block: BlockT + serde::de::DeserializeOwned, + Block::Header: serde::de::DeserializeOwned, +{ + let number = ChainApi::<(), Block::Hash, Block::Header, ()>::header(rpc, Some(hash)) + .await + .map_err(rpc_err_handler) + .and_then(|maybe_header| maybe_header.ok_or("header_not_found").map(|h| *h.number()))?; + + let next = number + sp_runtime::traits::One::one(); + + let next_hash = match ChainApi::<(), Block::Hash, Block::Header, ()>::block_hash( + rpc, + Some(ListOrValue::Value(NumberOrHex::Number( + next.try_into().map_err(|_| "failed to convert number to block number")?, + ))), + ) + .await + .map_err(rpc_err_handler)? + { + ListOrValue::Value(t) => t.expect("value passed in; value comes out; qed"), + _ => unreachable!(), + }; + + Ok(next_hash) +} diff --git a/utils/frame/try-runtime/cli/src/commands/follow_chain.rs b/utils/frame/try-runtime/cli/src/commands/follow_chain.rs index 1cc371c8f22fd..4eb3b3a8f35a9 100644 --- a/utils/frame/try-runtime/cli/src/commands/follow_chain.rs +++ b/utils/frame/try-runtime/cli/src/commands/follow_chain.rs @@ -16,32 +16,33 @@ // limitations under the License. use crate::{ - build_executor, ensure_matching_spec, extract_code, full_extensions, local_spec, parse, - state_machine_call_with_proof, SharedParams, LOG_TARGET, + build_executor, full_extensions, parse, rpc_err_handler, state_machine_call_with_proof, + LiveState, SharedParams, State, LOG_TARGET, }; use parity_scale_codec::{Decode, Encode}; -use remote_externalities::{Builder, Mode, OnlineConfig}; -use sc_executor::NativeExecutionDispatch; -use sc_service::Configuration; +use sc_executor::sp_wasm_interface::HostFunctions; use serde::{de::DeserializeOwned, Serialize}; use sp_core::H256; -use sp_runtime::traits::{Block as BlockT, Header as HeaderT, NumberFor}; +use sp_runtime::{ + generic::SignedBlock, + traits::{Block as BlockT, Header as HeaderT, NumberFor}, +}; use std::{fmt::Debug, str::FromStr}; use substrate_rpc_client::{ws_client, ChainApi, FinalizedHeaders, Subscription, WsClient}; const SUB: &str = "chain_subscribeFinalizedHeads"; const UN_SUB: &str = "chain_unsubscribeFinalizedHeads"; -/// Configurations of the [`Command::FollowChain`]. +/// Configurations of the [`crate::Command::FollowChain`]. #[derive(Debug, Clone, clap::Parser)] pub struct FollowChainCmd { /// The url to connect to. #[arg(short, long, value_parser = parse::url)] - uri: String, + pub uri: String, /// If set, then the state root check is enabled. #[arg(long)] - state_root_check: bool, + pub state_root_check: bool, /// Which try-state targets to execute when running this command. /// @@ -52,12 +53,12 @@ pub struct FollowChainCmd { /// `Staking, System`). /// - `rr-[x]` where `[x]` is a number. Then, the given number of pallets are checked in a /// round-robin fashion. - #[arg(long, default_value = "none")] - try_state: frame_try_runtime::TryStateSelect, + #[arg(long, default_value = "all")] + pub try_state: frame_try_runtime::TryStateSelect, /// If present, a single connection to a node will be kept and reused for fetching blocks. #[arg(long)] - keep_connection: bool, + pub keep_connection: bool, } /// Start listening for with `SUB` at `url`. @@ -77,10 +78,9 @@ async fn start_subscribing( +pub(crate) async fn follow_chain( shared: SharedParams, command: FollowChainCmd, - config: Configuration, ) -> sc_cli::Result<()> where Block: BlockT + DeserializeOwned, @@ -89,26 +89,35 @@ where ::Err: Debug, NumberFor: FromStr, as FromStr>::Err: Debug, - ExecDispatch: NativeExecutionDispatch + 'static, + HostFns: HostFunctions, { - let mut maybe_state_ext = None; let (rpc, subscription) = start_subscribing::(&command.uri).await?; - - let (code_key, code) = extract_code(&config.chain_spec)?; - let executor = build_executor::(&shared, &config); - let execution = shared.execution; - let mut finalized_headers: FinalizedHeaders = FinalizedHeaders::new(&rpc, subscription); + let mut maybe_state_ext = None; + let executor = build_executor::(&shared); + while let Some(header) = finalized_headers.next().await { let hash = header.hash(); let number = header.number(); - let block: Block = ChainApi::<(), Block::Hash, Block::Header, _>::block(&rpc, Some(hash)) - .await - .unwrap() - .unwrap(); + let block = + ChainApi::<(), Block::Hash, Block::Header, SignedBlock>::block(&rpc, Some(hash)) + .await + .or_else(|e| { + if matches!(e, substrate_rpc_client::Error::ParseError(_)) { + log::error!( + "failed to parse the block format of remote against the local \ + codebase. The block format has changed, and follow-chain cannot run in \ + this case. Try running this command in a branch of your codebase that has \ + the same block format as the remote chain. For now, we replace the block with an empty one" + ); + } + Err(rpc_err_handler(e)) + })? + .expect("if header exists, block should also exist.") + .block; log::debug!( target: LOG_TARGET, @@ -120,49 +129,40 @@ where // create an ext at the state of this block, whatever is the first subscription event. if maybe_state_ext.is_none() { - let builder = Builder::::new() - .mode(Mode::Online(OnlineConfig { - transport: command.uri.clone().into(), - at: Some(*header.parent_hash()), - ..Default::default() - })) - .state_version(shared.state_version); - - let new_ext = builder - .inject_hashed_key_value(&[(code_key.clone(), code.clone())]) - .build() - .await?; - log::info!( - target: LOG_TARGET, - "initialized state externalities at {:?}, storage root {:?}", - number, - new_ext.as_backend().root() - ); - - let (expected_spec_name, expected_spec_version, spec_state_version) = - local_spec::(&new_ext, &executor); - ensure_matching_spec::( - command.uri.clone(), - expected_spec_name, - expected_spec_version, - shared.no_spec_check_panic, - ) - .await; - - maybe_state_ext = Some((new_ext, spec_state_version)); + let state = State::Live(LiveState { + uri: command.uri.clone(), + // a bit dodgy, we have to un-parse the has to a string again and re-parse it + // inside. + at: Some(hex::encode(header.parent_hash().encode())), + pallet: vec![], + child_tree: true, + }); + let ext = state.into_ext::(&shared, &executor, None).await?; + maybe_state_ext = Some(ext); } - let (state_ext, spec_state_version) = + let state_ext = maybe_state_ext.as_mut().expect("state_ext either existed or was just created"); - let (mut changes, encoded_result) = state_machine_call_with_proof::( + let result = state_machine_call_with_proof::( state_ext, &executor, - execution, "TryRuntime_execute_block", (block, command.state_root_check, command.try_state.clone()).encode().as_ref(), full_extensions(), - )?; + ); + + if let Err(why) = result { + log::error!( + target: LOG_TARGET, + "failed to execute block {:?} due to {:?}", + number, + why + ); + continue + } + + let (mut changes, encoded_result) = result.expect("checked to be Ok; qed"); let consumed_weight = ::decode(&mut &*encoded_result) .map_err(|e| format!("failed to decode weight: {:?}", e))?; @@ -171,13 +171,13 @@ where .drain_storage_changes( &state_ext.backend, &mut Default::default(), - // Note that in case a block contains a runtime upgrade, - // state version could potentially be incorrect here, - // this is very niche and would only result in unaligned - // roots, so this use case is ignored for now. - *spec_state_version, + // Note that in case a block contains a runtime upgrade, state version could + // potentially be incorrect here, this is very niche and would only result in + // unaligned roots, so this use case is ignored for now. + state_ext.state_version, ) .unwrap(); + state_ext.backend.apply_transaction( storage_changes.transaction_storage_root, storage_changes.transaction, diff --git a/utils/frame/try-runtime/cli/src/commands/mod.rs b/utils/frame/try-runtime/cli/src/commands/mod.rs index 4861d94f077ce..ab0a066585f6a 100644 --- a/utils/frame/try-runtime/cli/src/commands/mod.rs +++ b/utils/frame/try-runtime/cli/src/commands/mod.rs @@ -15,7 +15,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -pub(crate) mod execute_block; -pub(crate) mod follow_chain; -pub(crate) mod offchain_worker; -pub(crate) mod on_runtime_upgrade; +pub mod create_snapshot; +pub mod execute_block; +pub mod follow_chain; +pub mod offchain_worker; +pub mod on_runtime_upgrade; diff --git a/utils/frame/try-runtime/cli/src/commands/offchain_worker.rs b/utils/frame/try-runtime/cli/src/commands/offchain_worker.rs index 8d2585372b4a8..c55de7da64817 100644 --- a/utils/frame/try-runtime/cli/src/commands/offchain_worker.rs +++ b/utils/frame/try-runtime/cli/src/commands/offchain_worker.rs @@ -16,34 +16,18 @@ // limitations under the License. use crate::{ - build_executor, ensure_matching_spec, extract_code, full_extensions, hash_of, local_spec, - parse, state_machine_call, SharedParams, State, LOG_TARGET, + build_executor, commands::execute_block::next_hash_of, full_extensions, parse, rpc_err_handler, + state_machine_call, LiveState, SharedParams, State, LOG_TARGET, }; use parity_scale_codec::Encode; -use sc_executor::NativeExecutionDispatch; -use sc_service::Configuration; -use sp_core::storage::well_known_keys; -use sp_runtime::traits::{Block as BlockT, Header, NumberFor}; +use sc_executor::sp_wasm_interface::HostFunctions; +use sp_runtime::traits::{Block as BlockT, NumberFor}; use std::{fmt::Debug, str::FromStr}; use substrate_rpc_client::{ws_client, ChainApi}; -/// Configurations of the [`Command::OffchainWorker`]. +/// Configurations of the [`crate::Command::OffchainWorker`]. #[derive(Debug, Clone, clap::Parser)] pub struct OffchainWorkerCmd { - /// Overwrite the wasm code in state or not. - #[arg(long)] - overwrite_wasm_code: bool, - - /// The block hash at which to fetch the header. - /// - /// If the `live` state type is being used, then this can be omitted, and is equal to whatever - /// the `state::at` is. Only use this (with care) when combined with a snapshot. - #[arg( - long, - value_parser = parse::hash - )] - header_at: Option, - /// The ws uri from which to fetch the header. /// /// If the `live` state type is being used, then this can be omitted, and is equal to whatever @@ -52,7 +36,7 @@ pub struct OffchainWorkerCmd { long, value_parser = parse::url )] - header_ws_uri: Option, + pub header_ws_uri: Option, /// The state type to use. #[command(subcommand)] @@ -60,24 +44,6 @@ pub struct OffchainWorkerCmd { } impl OffchainWorkerCmd { - fn header_at(&self) -> sc_cli::Result - where - Block::Hash: FromStr, - ::Err: Debug, - { - match (&self.header_at, &self.state) { - (Some(header_at), State::Snap { .. }) => hash_of::(header_at), - (Some(header_at), State::Live { .. }) => { - log::error!(target: LOG_TARGET, "--header-at is provided while state type is live, this will most likely lead to a nonsensical result."); - hash_of::(header_at) - }, - (None, State::Live { at: Some(at), .. }) => hash_of::(at), - _ => { - panic!("either `--header-at` must be provided, or state must be `live` with a proper `--at`"); - }, - } - } - fn header_ws_uri(&self) -> String where Block::Hash: FromStr, @@ -89,7 +55,7 @@ impl OffchainWorkerCmd { log::error!(target: LOG_TARGET, "--header-uri is provided while state type is live, this will most likely lead to a nonsensical result."); header_ws_uri.to_owned() }, - (None, State::Live { uri, .. }) => uri.clone(), + (None, State::Live(LiveState { uri, .. })) => uri.clone(), (None, State::Snap { .. }) => { panic!("either `--header-uri` must be provided, or state must be `live`"); }, @@ -97,76 +63,42 @@ impl OffchainWorkerCmd { } } -pub(crate) async fn offchain_worker( +pub(crate) async fn offchain_worker( shared: SharedParams, command: OffchainWorkerCmd, - config: Configuration, ) -> sc_cli::Result<()> where Block: BlockT + serde::de::DeserializeOwned, - Block::Hash: FromStr, Block::Header: serde::de::DeserializeOwned, + Block::Hash: FromStr, ::Err: Debug, NumberFor: FromStr, as FromStr>::Err: Debug, - ExecDispatch: NativeExecutionDispatch + 'static, + HostFns: HostFunctions, { - let executor = build_executor(&shared, &config); - let execution = shared.execution; + let executor = build_executor(&shared); + // we first build the externalities with the remote code. + let ext = command.state.into_ext::(&shared, &executor, None).await?; - let header_at = command.header_at::()?; let header_ws_uri = command.header_ws_uri::(); let rpc = ws_client(&header_ws_uri).await?; - let header = ChainApi::<(), Block::Hash, Block::Header, ()>::header(&rpc, Some(header_at)) - .await - .unwrap() - .unwrap(); - log::info!( - target: LOG_TARGET, - "fetched header from {:?}, block number: {:?}", - header_ws_uri, - header.number() - ); - - let ext = { - let builder = command.state.builder::()?.state_version(shared.state_version); - - let builder = if command.overwrite_wasm_code { - log::info!( - target: LOG_TARGET, - "replacing the in-storage :code: with the local code from {}'s chain_spec (your local repo)", - config.chain_spec.name(), - ); - let (code_key, code) = extract_code(&config.chain_spec)?; - builder.inject_hashed_key_value(&[(code_key, code)]) - } else { - builder.inject_hashed_key(well_known_keys::CODE) - }; + let next_hash = next_hash_of::(&rpc, ext.block_hash).await?; + log::info!(target: LOG_TARGET, "fetching next header: {:?} ", next_hash); - builder.build().await? - }; - - let (expected_spec_name, expected_spec_version, _) = - local_spec::(&ext, &executor); - ensure_matching_spec::( - header_ws_uri, - expected_spec_name, - expected_spec_version, - shared.no_spec_check_panic, - ) - .await; + let header = ChainApi::<(), Block::Hash, Block::Header, ()>::header(&rpc, Some(next_hash)) + .await + .map_err(rpc_err_handler) + .map(|maybe_header| maybe_header.ok_or("Header does not exist"))??; + let payload = header.encode(); - let _ = state_machine_call::( + let _ = state_machine_call::( &ext, &executor, - execution, "OffchainWorkerApi_offchain_worker", - header.encode().as_ref(), + &payload, full_extensions(), )?; - log::info!(target: LOG_TARGET, "OffchainWorkerApi_offchain_worker executed without errors."); - Ok(()) } diff --git a/utils/frame/try-runtime/cli/src/commands/on_runtime_upgrade.rs b/utils/frame/try-runtime/cli/src/commands/on_runtime_upgrade.rs index fba34ddfb5060..80fb5d31f71a9 100644 --- a/utils/frame/try-runtime/cli/src/commands/on_runtime_upgrade.rs +++ b/utils/frame/try-runtime/cli/src/commands/on_runtime_upgrade.rs @@ -15,31 +15,31 @@ // See the License for the specific language governing permissions and // limitations under the License. -use std::{fmt::Debug, str::FromStr}; - -use parity_scale_codec::Decode; -use sc_executor::NativeExecutionDispatch; -use sc_service::Configuration; +use crate::{build_executor, state_machine_call_with_proof, SharedParams, State, LOG_TARGET}; +use parity_scale_codec::{Decode, Encode}; +use sc_executor::sp_wasm_interface::HostFunctions; use sp_runtime::traits::{Block as BlockT, NumberFor}; use sp_weights::Weight; +use std::{fmt::Debug, str::FromStr}; -use crate::{ - build_executor, ensure_matching_spec, extract_code, local_spec, state_machine_call_with_proof, - SharedParams, State, LOG_TARGET, -}; - -/// Configurations of the [`Command::OnRuntimeUpgrade`]. +/// Configurations of the [`crate::Command::OnRuntimeUpgrade`]. #[derive(Debug, Clone, clap::Parser)] pub struct OnRuntimeUpgradeCmd { /// The state type to use. #[command(subcommand)] pub state: State, + + /// Execute `try_state`, `pre_upgrade` and `post_upgrade` checks as well. + /// + /// This will perform more checks, but it will also makes the reported PoV/Weight be + /// inaccurate. + #[clap(long)] + pub checks: bool, } -pub(crate) async fn on_runtime_upgrade( +pub(crate) async fn on_runtime_upgrade( shared: SharedParams, command: OnRuntimeUpgradeCmd, - config: Configuration, ) -> sc_cli::Result<()> where Block: BlockT + serde::de::DeserializeOwned, @@ -48,40 +48,22 @@ where Block::Header: serde::de::DeserializeOwned, NumberFor: FromStr, as FromStr>::Err: Debug, - ExecDispatch: NativeExecutionDispatch + 'static, + HostFns: HostFunctions, { - let executor = build_executor(&shared, &config); - let execution = shared.execution; - - let ext = { - let builder = command.state.builder::()?.state_version(shared.state_version); - let (code_key, code) = extract_code(&config.chain_spec)?; - builder.inject_hashed_key_value(&[(code_key, code)]).build().await? - }; + let executor = build_executor(&shared); + let ext = command.state.into_ext::(&shared, &executor, None).await?; - if let Some(uri) = command.state.live_uri() { - let (expected_spec_name, expected_spec_version, _) = - local_spec::(&ext, &executor); - ensure_matching_spec::( - uri, - expected_spec_name, - expected_spec_version, - shared.no_spec_check_panic, - ) - .await; - } - - let (_, encoded_result) = state_machine_call_with_proof::( + let (_, encoded_result) = state_machine_call_with_proof::( &ext, &executor, - execution, "TryRuntime_on_runtime_upgrade", - &[], + command.checks.encode().as_ref(), Default::default(), // we don't really need any extensions here. )?; let (weight, total_weight) = <(Weight, Weight) as Decode>::decode(&mut &*encoded_result) .map_err(|e| format!("failed to decode weight: {:?}", e))?; + log::info!( target: LOG_TARGET, "TryRuntime_on_runtime_upgrade executed without errors. Consumed weight = ({} ps, {} byte), total weight = ({} ps, {} byte) ({:.2} %, {:.2} %).", diff --git a/utils/frame/try-runtime/cli/src/lib.rs b/utils/frame/try-runtime/cli/src/lib.rs index f54354342bf28..47a9dfa3f6544 100644 --- a/utils/frame/try-runtime/cli/src/lib.rs +++ b/utils/frame/try-runtime/cli/src/lib.rs @@ -22,7 +22,7 @@ //! > As the name suggests, `try-runtime` is a detailed testing framework that gives you a lot of //! control over what is being executed in which environment. It is recommended that user's first //! familiarize themselves with substrate in depth, particularly the execution model. It is critical -//! to deeply understand how the wasm/native interactions, and the runtime apis work in the +//! to deeply understand how the wasm/client/runtime interactions, and the runtime apis work in the //! substrate runtime, before commencing to working with `try-runtime`. //! //! #### Resources @@ -35,101 +35,102 @@ //! //! --- //! -//! ## Overview +//! ## Background Knowledge //! //! The basis of all try-runtime commands is the same: connect to a live node, scrape its *state* //! and put it inside a `TestExternalities`, then call into a *specific runtime-api* using the given //! state and some *runtime*. //! +//! Alternatively, the state could come from a snapshot file. +//! //! All of the variables in the above statement are made *italic*. Let's look at each of them: //! //! 1. **State** is the key-value pairs of data that comprise the canonical information that any //! blockchain is keeping. A state can be full (all key-value pairs), or be partial (only pairs -//! related to some pallets). Moreover, some keys are special and are not related to specific -//! pallets, known as [`well_known_keys`] in substrate. The most important of these is the -//! `:CODE:` key, which contains the code used for execution, when wasm execution is chosen. +//! related to some pallets/prefixes). Moreover, some keys are especial and are not related to +//! specific pallets, known as [`well_known_keys`] in substrate. The most important of these is +//! the `:CODE:` key, which contains the code used for execution, when wasm execution is chosen. //! //! 2. *A runtime-api* call is a call into a function defined in the runtime, *on top of a given //! state*. Each subcommand of `try-runtime` utilizes a specific *runtime-api*. //! //! 3. Finally, the **runtime** is the actual code that is used to execute the aforementioned -//! runtime-api. All substrate based chains always have two runtimes: native and wasm. The -//! decision of which one is chosen is non-trivial. First, let's look at the options: -//! -//! 1. Native: this means that the runtime that is **in your codebase**, aka whatever you see in -//! your editor, is being used. This runtime is easier for diagnostics. We refer to this as -//! the "local runtime". -//! -//! 2. Wasm: this means that whatever is stored in the `:CODE:` key of the state that your -//! scrape is being used. In plain sight, since the entire state (including `:CODE:`) is -//! scraped from a remote chain, you could conclude that the wasm runtime, if used, is always -//! equal to the canonical runtime of the live chain (i.e. NOT the "local runtime"). That's -//! factually true, but then the testing would be quite lame. Typically, with try-runtime, -//! you don't want to execute whatever code is already on the live chain. Instead, you want -//! your local runtime (which typically includes a non-released feature) to be used. This is -//! why try-runtime overwrites the wasm runtime (at `:CODE:`) with the local runtime as well. -//! That being said, this behavior can be controlled in certain subcommands with a special -//! flag (`--overwrite-wasm-code`). -//! -//! The decision of which runtime is eventually used is based on two facts: -//! -//! 1. `--execution` flag. If you specify `wasm`, then it is *always* wasm. If it is `native`, then -//! if and ONLY IF the spec versions match, then the native runtime is used. Else, wasm runtime -//! is used again. -//! 2. `--chain` flag (if present in your cli), which determines *which local runtime*, is selected. -//! This will specify: -//! 1. which native runtime is used, if you select `--execution Native` -//! 2. which wasm runtime is used to replace the `:CODE:`, if try-runtime is instructed to do -//! so. -//! -//! All in all, if the term "local runtime" is used in the rest of this crate's documentation, it -//! means either the native runtime, or the wasm runtime when overwritten inside `:CODE:`. In other -//! words, it means your... well, "local runtime", regardless of wasm or native. -//! -//! //! See [`Command`] for more information about each command's specific customization flags, and -//! assumptions regarding the runtime being used. +//! runtime-api. Everything in this crate assumes wasm execution, which means the runtime that +//! you use is the one stored onchain, namely under the `:CODE:` key. +//! +//! To recap, a typical try-runtime command does the following: +//! +//! 1. Download the state of a live chain, and write to an `externalities`. +//! 2. Overwrite the `:CODE:` with a given wasm blob +//! 3. Test some functionality via calling a runtime-api. +//! +//! ## Usage +//! +//! To use any of the provided commands, [`SharedParams`] must be provided. The most important of +//! which being [`SharedParams::runtime`], which specifies which runtime to use. Furthermore, +//! [`SharedParams::overwrite_state_version`] can be used to alter the state-version (see +//! for more info). +//! +//! Then, the specific command has to be specified. See [`Command`] for more information about each +//! command's specific customization flags, and assumptions regarding the runtime being used. +//! +//! Said briefly, this CLI is capable of executing: +//! +//! * [`Command::OnRuntimeUpgrade`]: execute all the `on_runtime_upgrade` hooks. +//! * [`Command::ExecuteBlock`]: re-execute the given block. +//! * [`Command::OffchainWorker`]: re-execute the given block's offchain worker code path. +//! * [`Command::FollowChain`]: continuously execute the blocks of a remote chain on top of a given +//! runtime. +//! * [`Command::CreateSnapshot`]: Create a snapshot file from a remote node. //! //! Finally, To make sure there are no errors regarding this, always run any `try-runtime` command //! with `executor=trace` logging targets, which will specify which runtime is being used per api -//! call. -//! -//! Furthermore, other relevant log targets are: `try-runtime::cli`, `remote-ext`, and `runtime`. +//! call. Moreover, `remote-ext`, `try-runtime` and `runtime` logs targets will also be useful. //! //! ## Spec name check //! //! A common pitfall is that you might be running some test on top of the state of chain `x`, with //! the runtime of chain `y`. To avoid this all commands do a spec-name check before executing -//! anything by default. This will check the spec name of the remote node your are connected to, -//! with the spec name of your local runtime and ensure that they match. +//! anything by default. This will check the, if any alterations are being made to the `:CODE:`, +//! then the spec names match. The spec versions are warned, but are not mandated to match. //! -//! Should you need to disable this on certain occasions, a top level flag of `--no-spec-name-check` -//! can be used. +//! > If anything, in most cases, we expect spec-versions to NOT match, because try-runtime is all +//! > about testing unreleased runtimes. //! -//! The spec version is also always inspected, but if it is a mismatch, it will only emit a warning. -//! -//! ## Note nodes that operate with `try-runtime` +//! ## Note on nodes that respond to `try-runtime` requests. //! //! There are a number of flags that need to be preferably set on a running node in order to work //! well with try-runtime's expensive RPC queries: //! -//! - set `--rpc-max-payload 1000` to ensure large RPC queries can work. -//! - set `--ws-max-out-buffer-capacity 1000` to ensure the websocket connection can handle large -//! RPC queries. +//! - set `--rpc-max-response-size 1000` and +//! - `--rpc-max-request-size 1000` to ensure connections are not dropped in case the state is +//! large. //! - set `--rpc-cors all` to ensure ws connections can come through. //! //! Note that *none* of the try-runtime operations need unsafe RPCs. //! -//! ## Migration Best Practices +//! ## Note on signature and state-root checks +//! +//! All of the commands calling into `TryRuntime_execute_block` ([`Command::ExecuteBlock`] and +//! [`Command::FollowChain`]) disable both state root and signature checks. This is because in 99% +//! of the cases, the runtime that is being tested is different from the one that is stored in the +//! canonical chain state. This implies: +//! +//! 1. the state root will NEVER match, because `:CODE:` is different between the two. +//! 2. replaying all transactions will fail, because the spec-version is part of the transaction +//! signature. +//! +//! ## Best Practices //! -//! One of the main use-cases of try-runtime is using it for testing storage migrations. The -//! following points makes sure you can *effectively* test your migrations with try-runtime. +//! Try-runtime is all about battle-testing unreleased runtime. The following list of suggestions +//! help developers maximize the testing coverage and make base use of `try-runtime`. //! //! #### Adding pre/post hooks //! //! One of the gems that come only in the `try-runtime` feature flag is the `pre_upgrade` and -//! `post_upgrade` hooks for `OnRuntimeUpgrade`. This trait is implemented either inside the -//! pallet, or manually in a runtime, to define a migration. In both cases, these functions can be -//! added, given the right flag: +//! `post_upgrade` hooks for `OnRuntimeUpgrade`. This trait is implemented either inside the pallet, +//! or manually in a runtime, to define a migration. In both cases, these functions can be added, +//! given the right flag: //! //! ```ignore //! @@ -147,6 +148,19 @@ //! encoded data (usually some pre-upgrade state) which will be passed to `post_upgrade` after //! upgrading and used for post checking. //! +//! ## State Consistency +//! +//! Similarly, each pallet can expose a function in `#[pallet::hooks]` section as follows: +//! +//! ``` +//! #[cfg(feature = try-runtime)] +//! fn try_state(_) -> Result<(), &'static str> {} +//! ``` +//! +//! which is called on numerous code paths in the try-runtime tool. These checks should ensure that +//! the state of the pallet is consistent and correct. See `frame_support::try_runtime::TryState` +//! for more info. +//! //! #### Logging //! //! It is super helpful to make sure your migration code uses logging (always with a `runtime` log @@ -161,216 +175,260 @@ //! //! ## Examples //! -//! Run the migrations of the local runtime on the state of polkadot, from the polkadot repo where -//! we have `--chain polkadot-dev`, on the latest finalized block's state +//! For the following examples, we assume the existence of the following: //! -//! ```sh -//! RUST_LOG=runtime=trace,try-runtime::cli=trace,executor=trace \ -//! cargo run try-runtime \ -//! --execution Native \ -//! --chain polkadot-dev \ -//! on-runtime-upgrade \ -//! live \ -//! --uri wss://rpc.polkadot.io -//! # note that we don't pass any --at, nothing means latest block. +//! 1. a substrate node compiled without `--feature try-runtime`, called `substrate`. This will be +//! the running node that you connect to. then, after some changes to this node, you compile it with +//! `--features try-runtime`. This gives you: +//! 2. a substrate binary that has the try-runtime sub-command enabled. +//! 3. a wasm blob that has try-runtime functionality. +//! +//! ```bash +//! # this is like your running deployed node. +//! cargo build --release && cp target/release/substrate . +//! +//! # this is like your WIP branch. +//! cargo build --release --features try-runtime +//! cp target/release/substrate substrate-try-runtime +//! cp ./target/release/wbuild/kitchensink-runtime/kitchensink_runtime.wasm runtime-try-runtime.wasm //! ``` //! -//! Same as previous one, but let's say we want to run this command from the substrate repo, where -//! we don't have a matching spec name/version. +//! > The above example is with `substrate`'s `kitchensink-runtime`, but is applicable to any +//! > substrate-based chain that has implemented `try-runtime-cli`. +//! +//! * If you run `try-runtime` subcommand against `substrate` binary listed above, you get the +//! following error. +//! +//! ```bash +//! [substrate] ./substrate try-runtime +//! Error: Input("TryRuntime wasn't enabled when building the node. You can enable it with `--features try-runtime`.") +//! ``` //! -//! ```sh -//! RUST_LOG=runtime=trace,try-runtime::cli=trace,executor=trace \ -//! cargo run try-runtime \ -//! --execution Native \ -//! --chain dev \ -//! --no-spec-name-check \ # mind this one! +//! * If you run the same against `substrate-try-runtime`, it will work. +//! +//! ```bash +//! [substrate] ./substrate-try-runtime try-runtime +//! Try some command against runtime state +//! +//! Usage: substrate-try-runtime try-runtime [OPTIONS] --runtime +//! +//! Commands: +//! on-runtime-upgrade Execute the migrations of the "local runtime" +//! execute-block Executes the given block against some state +//! offchain-worker Executes *the offchain worker hooks* of a given block against some state +//! follow-chain Follow the given chain's finalized blocks and apply all of its extrinsics +//! create-snapshot Create a new snapshot file +//! help Print this message or the help of the given subcommand(s) +//! +//! Options: +//! --chain +//! Specify the chain specification +//! --dev +//! Specify the development chain +//! -d, --base-path +//! Specify custom base path +//! -l, --log ... +//! Sets a custom logging filter. Syntax is `=`, e.g. -lsync=debug +//! --detailed-log-output +//! Enable detailed log output +//! --disable-log-color +//! Disable log color output +//! --enable-log-reloading +//! Enable feature to dynamically update and reload the log filter +//! --tracing-targets +//! Sets a custom profiling filter. Syntax is the same as for logging: `=` +//! --tracing-receiver +//! Receiver to process tracing messages [default: log] [possible values: log] +//! --runtime +//! The runtime to use +//! --wasm-execution +//! Type of wasm execution used [default: compiled] [possible values: interpreted-i-know-what-i-do, compiled] +//! --wasm-instantiation-strategy +//! The WASM instantiation method to use [default: pooling-copy-on-write] [possible values: pooling-copy-on-write, recreate-instance-copy-on-write, pooling, recreate-instance, legacy-instance-reuse] +//! --heap-pages +//! The number of 64KB pages to allocate for Wasm execution. Defaults to [`sc_service::Configuration.default_heap_pages`] +//! --overwrite-state-version +//! Overwrite the `state_version` +//! -h, --help +//! Print help information (use `--help` for more detail) +//! -V, --version +//! Print version information +//! ``` +//! +//! * Run the migrations of a given runtime on top of a live state. +//! +//! ```bash +//! # assuming there's `./substrate --dev --tmp --ws-port 9999` or similar running. +//! ./substrate-try-runtime \ +//! try-runtime \ +//! --runtime kitchensink_runtime.wasm \ +//! -lruntime=debug \ //! on-runtime-upgrade \ -//! live \ -//! --uri wss://rpc.polkadot.io +//! live --uri ws://localhost:9999 //! ``` //! -//! Same as the previous one, but run it at specific block number's state. This means that this +//! * Same as the previous one, but run it at specific block number's state. This means that this //! block hash's state shall not yet have been pruned in `rpc.polkadot.io`. //! -//! ```sh -//! RUST_LOG=runtime=trace,try-runtime::cli=trace,executor=trace \ -//! cargo run try-runtime \ -//! --execution Native \ -//! --chain dev \ -//! --no-spec-name-check \ # mind this one! on-runtime-upgrade \ +//! ```bash +//! ./substrate-try-runtime \ +//! try-runtime \ +//! --runtime kitchensink_runtime.wasm \ +//! -lruntime=debug \ //! on-runtime-upgrade \ -//! live \ -//! --uri wss://rpc.polkadot.io \ -//! --at +//! live --uri ws://localhost:9999 \ +//! # replace with your desired block hash! +//! --at 0xa1b16c1efd889a9f17375ec4dd5c1b4351a2be17fa069564fced10d23b9b3836 //! ``` //! -//! Moving to `execute-block` and `offchain-workers`. For these commands, you always needs to -//! specify a block hash. For the rest of these examples, we assume we're in the polkadot repo. -//! -//! First, let's assume you are in a branch that has the same spec name/version as the live polkadot -//! network. -//! -//! ```sh -//! RUST_LOG=runtime=trace,try-runtime::cli=trace,executor=trace \ -//! cargo run try-runtime \ -//! --execution Wasm \ -//! --chain polkadot-dev \ -//! --uri wss://rpc.polkadot.io \ -//! execute-block \ -//! live \ -//! --at +//! * Executing the same command with the [`Runtime::Existing`] will fail because the existing +//! runtime, stored onchain in `substrate` binary that we compiled earlier does not have +//! `try-runtime` feature! +//! +//! ```bash +//! ./substrate-try-runtime try-runtime --runtime existing -lruntime=debug on-runtime-upgrade live --uri ws://localhost:9999 +//! ... +//! Error: Input("given runtime is NOT compiled with try-runtime feature!") //! ``` //! -//! This is wasm, so it will technically execute the code that lives on the live network. Let's say -//! you want to execute your local runtime. Since you have a matching spec versions, you can simply -//! change `--execution Wasm` to `--execution Native` to achieve this. Your logs of `executor=trace` -//! should show something among the lines of: +//! * Now, let's use a snapshot file. First, we create the snapshot: //! -//! ```text -//! Request for native execution succeeded (native: polkadot-9900 (parity-polkadot-0.tx7.au0), chain: polkadot-9900 (parity-polkadot-0.tx7.au0)) +//! ```bash +//! ./substrate-try-runtime try-runtime --runtime existing -lruntime=debug create-snapshot --uri ws://localhost:9999 +//! 2022-12-13 10:28:17.516 INFO main try-runtime::cli: snapshot path not provided (-s), using 'node-268@latest.snap' +//! 2022-12-13 10:28:17.516 INFO main remote-ext: since no at is provided, setting it to latest finalized head, 0xe7d0b614dfe89af65b33577aae46a6f958c974bf52f8a5e865a0f4faeb578d22 +//! 2022-12-13 10:28:17.516 INFO main remote-ext: since no prefix is filtered, the data for all pallets will be downloaded +//! 2022-12-13 10:28:17.550 INFO main remote-ext: writing snapshot of 1611464 bytes to "node-268@latest.snap" +//! 2022-12-13 10:28:17.551 INFO main remote-ext: initialized state externalities with storage root 0x925e4e95de4c08474fb7f976c4472fa9b8a1091619cd7820a793bf796ee6d932 and state_version V1 //! ``` //! -//! If you don't have matching spec versions, then are doomed to execute wasm. In this case, you can -//! manually overwrite the wasm code with your local runtime: -//! -//! ```sh -//! RUST_LOG=runtime=trace,try-runtime::cli=trace,executor=trace \ -//! cargo run try-runtime \ -//! --execution Wasm \ -//! --chain polkadot-dev \ -//! execute-block \ -//! live \ -//! --uri wss://rpc.polkadot.io \ -//! --at \ -//! --overwrite-wasm-code +//! > Note that the snapshot contains the `existing` runtime, which does not have the correct +//! > `try-runtime` feature. In the following commands, we still need to overwrite the runtime. +//! +//! Then, we can use it to have the same command as before, `on-runtime-upgrade` +//! +//! ```bash +//! try-runtime \ +//! --runtime runtime-try-runtime.wasm \ +//! -lruntime=debug \ +//! on-runtime-upgrade \ +//! snap -s node-268@latest.snap //! ``` //! -//! For all of these blocks, the block with hash `` is being used, and the initial state -//! is the state of the parent hash. This is because by omitting `ExecuteBlockCmd::block_at`, the -//! `--at` is used for both. This should be good enough for 99% of the cases. The only case where -//! you need to specify `block-at` and `block-ws-uri` is with snapshots. Let's say you have a file -//! `snap` and you know it corresponds to the state of the parent block of `X`. Then you'd do: -//! -//! ```sh -//! RUST_LOG=runtime=trace,try-runtime::cli=trace,executor=trace \ -//! cargo run try-runtime \ -//! --execution Wasm \ -//! --chain polkadot-dev \ -//! --uri wss://rpc.polkadot.io \ -//! execute-block \ -//! --block-at \ -//! --block-ws-uri wss://rpc.polkadot.io \ -//! --overwrite-wasm-code \ -//! snap \ -//! -s snap \ +//! * Execute the latest finalized block with the given runtime. +//! +//! ```bash +//! ./substrate-try-runtime try-runtime \ +//! --runtime runtime-try-runtime.wasm \ +//! -lruntime=debug \ +//! execute-block live \ +//! --uri ws://localhost:999 +//! ``` +//! +//! This can still be customized at a given block with `--at`. If you want to use a snapshot, you +//! can still use `--block-ws-uri` to provide a node form which the block data can be fetched. +//! +//! Moreover, this runs the `frame_support::try_runtime::TryState` hooks as well. The hooks to run +//! can be customized with the `--try-state`. For example: +//! +//! ```bash +//! ./substrate-try-runtime try-runtime \ +//! --runtime runtime-try-runtime.wasm \ +//! -lruntime=debug \ +//! execute-block live \ +//! --try-state System,Staking \ +//! --uri ws://localhost:999 +//! ``` +//! +//! Will only run the `try-state` of the two given pallets. See +//! [`frame_try_runtime::TryStateSelect`] for more information. +//! +//! * Follow our live chain's blocks using `follow-chain`, whilst running the try-state of 3 pallets +//! in a round robin fashion +//! +//! ```bash +//! ./substrate-try-runtime \ +//! try-runtime \ +//! --runtime runtime-try-runtime.wasm \ +//! -lruntime=debug \ +//! follow-chain \ +//! --uri ws://localhost:9999 \ +//! --try-state rr-3 //! ``` #![cfg(feature = "try-runtime")] use parity_scale_codec::Decode; use remote_externalities::{ - Builder, Mode, OfflineConfig, OnlineConfig, SnapshotConfig, TestExternalities, + Builder, Mode, OfflineConfig, OnlineConfig, RemoteExternalities, SnapshotConfig, + TestExternalities, }; -use sc_chain_spec::ChainSpec; use sc_cli::{ - execution_method_from_cli, CliConfiguration, ExecutionStrategy, WasmExecutionMethod, - WasmtimeInstantiationStrategy, DEFAULT_WASMTIME_INSTANTIATION_STRATEGY, - DEFAULT_WASM_EXECUTION_METHOD, + CliConfiguration, RuntimeVersion, WasmExecutionMethod, WasmtimeInstantiationStrategy, + DEFAULT_WASMTIME_INSTANTIATION_STRATEGY, DEFAULT_WASM_EXECUTION_METHOD, }; -use sc_executor::NativeElseWasmExecutor; -use sc_service::{Configuration, NativeExecutionDispatch}; +use sc_executor::{sp_wasm_interface::HostFunctions, WasmExecutor}; +use sp_api::HashT; use sp_core::{ + hexdisplay::HexDisplay, offchain::{ testing::{TestOffchainExt, TestTransactionPoolExt}, OffchainDbExt, OffchainWorkerExt, TransactionPoolExt, }, - storage::{well_known_keys, StorageData, StorageKey}, + storage::well_known_keys, testing::TaskExecutor, - traits::TaskExecutorExt, + traits::{ReadRuntimeVersion, TaskExecutorExt}, twox_128, H256, }; use sp_externalities::Extensions; use sp_keystore::{testing::KeyStore, KeystoreExt}; use sp_runtime::{ - traits::{Block as BlockT, NumberFor}, + traits::{BlakeTwo256, Block as BlockT, NumberFor}, DeserializeOwned, }; -use sp_state_machine::{OverlayedChanges, StateMachine, TrieBackendBuilder}; +use sp_state_machine::{CompactProof, OverlayedChanges, StateMachine, TrieBackendBuilder}; use sp_version::StateVersion; use std::{fmt::Debug, path::PathBuf, str::FromStr}; -use substrate_rpc_client::{ws_client, StateApi}; -mod commands; +pub mod commands; pub(crate) mod parse; pub(crate) const LOG_TARGET: &str = "try-runtime::cli"; /// Possible commands of `try-runtime`. #[derive(Debug, Clone, clap::Subcommand)] pub enum Command { - /// Execute the migrations of the "local runtime". + /// Execute the migrations of the given runtime /// - /// This uses a custom runtime api call, namely "TryRuntime_on_runtime_upgrade". + /// This uses a custom runtime api call, namely "TryRuntime_on_runtime_upgrade". The code path + /// only triggers all of the `on_runtime_upgrade` hooks in the runtime, and optionally + /// `try_state`. /// - /// This always overwrites the wasm code with the local runtime (specified by `--chain`), to - /// ensure the new migrations are being executed. Re-executing already existing migrations is - /// evidently not very exciting. + /// See [`frame_try_runtime::TryRuntime`] and + /// [`commands::on_runtime_upgrade::OnRuntimeUpgradeCmd`] for more information. OnRuntimeUpgrade(commands::on_runtime_upgrade::OnRuntimeUpgradeCmd), /// Executes the given block against some state. /// - /// Unlike [`Command::OnRuntimeUpgrade`], this command needs two inputs: the state, and the - /// block data. Since the state could be cached (see [`State::Snap`]), different flags are - /// provided for both. `--block-at` and `--block-uri`, if provided, are only used for fetching - /// the block. For convenience, these flags can be both emitted, if the [`State::Live`] is - /// being used. - /// - /// Note that by default, this command does not overwrite the code, so in wasm execution, the - /// live chain's code is used. This can be disabled if desired, see - /// `ExecuteBlockCmd::overwrite_wasm_code`. + /// This uses a custom runtime api call, namely "TryRuntime_execute_block". Some checks, such + /// as state-root and signature checks are always disabled, and additional checks like + /// `try-state` can be enabled. /// - /// Note that if you do overwrite the wasm code, or generally use the local runtime for this, - /// you might - /// - not be able to decode the block, if the block format has changed. - /// - quite possibly will get a signature verification failure, since the spec and - /// transaction version are part of the signature's payload, and if they differ between - /// your local runtime and the remote counterparts, the signatures cannot be verified. - /// - almost certainly will get a state root mismatch, since, well, you are executing a - /// different state transition function. - /// - /// To make testing slightly more dynamic, you can disable the state root check by enabling - /// `ExecuteBlockCmd::no_check`. If you get signature verification errors, you should manually - /// tweak your local runtime's spec version to fix this. - /// - /// A subtle detail of execute block is that if you want to execute block 100 of a live chain - /// again, you need to scrape the state of block 99. This is already done automatically if you - /// use [`State::Live`], and the parent hash of the target block is used to scrape the state. - /// If [`State::Snap`] is being used, then this needs to be manually taken into consideration. - /// - /// This does not execute the same runtime api as normal block import do, namely - /// `Core_execute_block`. Instead, it uses `TryRuntime_execute_block`, which can optionally - /// skip state-root check (useful for trying a unreleased runtime), and can execute runtime - /// sanity checks as well. + /// See [`frame_try_runtime::TryRuntime`] and [`commands::execute_block::ExecuteBlockCmd`] for + /// more information. ExecuteBlock(commands::execute_block::ExecuteBlockCmd), /// Executes *the offchain worker hooks* of a given block against some state. /// - /// Similar to [`Command::ExecuteBlock`], this command needs two inputs: the state, and the - /// header data. Likewise, `--header-at` and `--header-uri` can be filled, or omitted if - /// [`State::Live`] is used. - /// - /// Similar to [`Command::ExecuteBlock`], this command does not overwrite the code, so in wasm - /// execution, the live chain's code is used. This can be disabled if desired, see - /// `OffchainWorkerCmd::overwrite_wasm_code`. - /// /// This executes the same runtime api as normal block import, namely /// `OffchainWorkerApi_offchain_worker`. + /// + /// See [`frame_try_runtime::TryRuntime`] and [`commands::offchain_worker::OffchainWorkerCmd`] + /// for more information. OffchainWorker(commands::offchain_worker::OffchainWorkerCmd), /// Follow the given chain's finalized blocks and apply all of its extrinsics. /// - /// This is essentially repeated calls to [`Command::ExecuteBlock`], whilst the local runtime - /// is always at use, the state root check is disabled, and the state is persisted between - /// executions. + /// This is essentially repeated calls to [`Command::ExecuteBlock`]. /// /// This allows the behavior of a new runtime to be inspected over a long period of time, with /// realistic transactions coming as input. @@ -382,7 +440,38 @@ pub enum Command { /// connections, starts listening for finalized block events. Upon first block notification, it /// initializes the state from the remote node, and starts applying that block, plus all the /// blocks that follow, to the same growing state. + /// + /// This can only work if the block format between the remote chain and the new runtime being + /// tested has remained the same, otherwise block decoding might fail. FollowChain(commands::follow_chain::FollowChainCmd), + + /// Create a new snapshot file. + CreateSnapshot(commands::create_snapshot::CreateSnapshotCmd), +} + +#[derive(Debug, Clone)] +pub enum Runtime { + /// Use the given path to the wasm binary file. + /// + /// It must have been compiled with `try-runtime`. + Path(PathBuf), + + /// Use the code of the remote node, or the snapshot. + /// + /// In almost all cases, this is not what you want, because the code in the remote node does + /// not have any of the try-runtime custom runtime APIs. + Existing, +} + +impl FromStr for Runtime { + type Err = String; + + fn from_str(s: &str) -> Result { + Ok(match s.to_lowercase().as_ref() { + "existing" => Runtime::Existing, + x @ _ => Runtime::Path(x.into()), + }) + } } /// Shared parameters of the `try-runtime` commands @@ -390,13 +479,23 @@ pub enum Command { #[group(skip)] pub struct SharedParams { /// Shared parameters of substrate cli. + /// + /// TODO: this is only needed because try-runtime is embedded in the substrate CLI. It should + /// go away. #[allow(missing_docs)] #[clap(flatten)] pub shared_params: sc_cli::SharedParams, - /// The execution strategy that should be used. - #[arg(long, value_name = "STRATEGY", value_enum, ignore_case = true, default_value_t = ExecutionStrategy::Wasm)] - pub execution: ExecutionStrategy, + /// The runtime to use. + /// + /// Must be a path to a wasm blob, compiled with `try-runtime` feature flag. + /// + /// Or, `existing`, indicating that you don't want to overwrite the runtime. This will use + /// whatever comes from the remote node, or the snapshot file. This will most likely not work + /// against a remote node, as no (sane) blockchain should compile its onchain wasm with + /// `try-runtime` feature. + #[arg(long)] + pub runtime: Runtime, /// Type of wasm execution used. #[arg( @@ -424,13 +523,11 @@ pub struct SharedParams { #[arg(long)] pub heap_pages: Option, - /// When enabled, the spec check will not panic, and instead only show a warning. - #[arg(long)] - pub no_spec_check_panic: bool, - - /// State version that is used by the chain. - #[arg(long, default_value_t = StateVersion::V1, value_parser = parse::state_version)] - pub state_version: StateVersion, + /// Overwrite the `state_version`. + /// + /// Otherwise `remote-externalities` will automatically set the correct state version. + #[arg(long, value_parser = parse::state_version)] + pub overwrite_state_version: Option, } /// Our `try-runtime` command. @@ -445,6 +542,41 @@ pub struct TryRuntimeCmd { pub command: Command, } +/// A `Live` variant [`State`] +#[derive(Debug, Clone, clap::Args)] +pub struct LiveState { + /// The url to connect to. + #[arg( + short, + long, + value_parser = parse::url, + )] + uri: String, + + /// The block hash at which to fetch the state. + /// + /// If non provided, then the latest finalized head is used. + #[arg( + short, + long, + value_parser = parse::hash, + )] + at: Option, + + /// A pallet to scrape. Can be provided multiple times. If empty, entire chain state will + /// be scraped. + #[arg(short, long, num_args = 1..)] + pallet: Vec, + + /// Fetch the child-keys as well. + /// + /// Default is `false`, if specific `--pallets` are specified, `true` otherwise. In other + /// words, if you scrape the whole state the child tree data is included out of the box. + /// Otherwise, it must be enabled explicitly using this flag. + #[arg(long)] + child_tree: bool, +} + /// The source of runtime *state* to use. #[derive(Debug, Clone, clap::Subcommand)] pub enum State { @@ -457,128 +589,167 @@ pub enum State { }, /// Use a live chain as the source of runtime state. - Live { - /// The url to connect to. - #[arg( - short, - long, - value_parser = parse::url, - )] - uri: String, - - /// The block hash at which to fetch the state. - /// - /// If non provided, then the latest finalized head is used. This is particularly useful - /// for [`Command::OnRuntimeUpgrade`]. - #[arg( - short, - long, - value_parser = parse::hash, - )] - at: Option, - - /// An optional state snapshot file to WRITE to. Not written if set to `None`. - #[arg(short, long)] - snapshot_path: Option, - - /// A pallet to scrape. Can be provided multiple times. If empty, entire chain state will - /// be scraped. - #[arg(short, long, num_args = 1..)] - pallet: Vec, - - /// Fetch the child-keys as well. - /// - /// Default is `false`, if specific `--pallets` are specified, `true` otherwise. In other - /// words, if you scrape the whole state the child tree data is included out of the box. - /// Otherwise, it must be enabled explicitly using this flag. - #[arg(long)] - child_tree: bool, - }, + Live(LiveState), } impl State { - /// Create the [`remote_externalities::Builder`] from self. - pub(crate) fn builder(&self) -> sc_cli::Result> + /// Create the [`remote_externalities::RemoteExternalities`] using [`remote-externalities`] from + /// self. + /// + /// This will override the code as it sees fit based on [`SharedParams::Runtime`]. It will also + /// check the spec-version and name. + pub(crate) async fn into_ext( + &self, + shared: &SharedParams, + executor: &WasmExecutor, + state_snapshot: Option, + ) -> sc_cli::Result> where Block::Hash: FromStr, + Block::Header: DeserializeOwned, + Block::Hash: DeserializeOwned, ::Err: Debug, { - Ok(match self { + let builder = match self { State::Snap { snapshot_path } => Builder::::new().mode(Mode::Offline(OfflineConfig { state_snapshot: SnapshotConfig::new(snapshot_path), })), - State::Live { snapshot_path, pallet, uri, at, child_tree } => { + State::Live(LiveState { pallet, uri, at, child_tree }) => { let at = match at { Some(at_str) => Some(hash_of::(at_str)?), None => None, }; - let mut builder = Builder::::new() - .mode(Mode::Online(OnlineConfig { - transport: uri.to_owned().into(), - state_snapshot: snapshot_path.as_ref().map(SnapshotConfig::new), - pallets: pallet.clone(), - scrape_children: true, - at, - })) - .inject_hashed_key( - &[twox_128(b"System"), twox_128(b"LastRuntimeUpgrade")].concat(), - ); - if *child_tree { - builder = builder.inject_default_child_tree_prefix(); - } - builder + Builder::::new().mode(Mode::Online(OnlineConfig { + at, + transport: uri.to_owned().into(), + state_snapshot, + pallets: pallet.clone(), + child_trie: *child_tree, + hashed_keys: vec![ + // we always download the code, but we almost always won't use it, based on + // `Runtime`. + well_known_keys::CODE.to_vec(), + // we will always download this key, since it helps detect if we should do + // runtime migration or not. + [twox_128(b"System"), twox_128(b"LastRuntimeUpgrade")].concat(), + [twox_128(b"System"), twox_128(b"Number")].concat(), + ], + hashed_prefixes: vec![], + })) }, - }) - } + }; + + // possibly overwrite the state version, should hardly be needed. + let builder = if let Some(state_version) = shared.overwrite_state_version { + log::warn!( + target: LOG_TARGET, + "overwriting state version to {:?}, you better know what you are doing.", + state_version + ); + builder.overwrite_state_version(state_version) + } else { + builder + }; + + // then, we prepare to replace the code based on what the CLI wishes. + let maybe_code_to_overwrite = match shared.runtime { + Runtime::Path(ref path) => Some(std::fs::read(path).map_err(|e| { + format!("error while reading runtime file from {:?}: {:?}", path, e) + })?), + Runtime::Existing => None, + }; + + // build the main ext. + let mut ext = builder.build().await?; + + // actually replace the code if needed. + if let Some(new_code) = maybe_code_to_overwrite { + let original_code = ext + .execute_with(|| sp_io::storage::get(well_known_keys::CODE)) + .expect("':CODE:' is always downloaded in try-runtime-cli; qed"); + + // NOTE: see the impl notes of `read_runtime_version`, the ext is almost not used here, + // only as a backup. + ext.insert(well_known_keys::CODE.to_vec(), new_code.clone()); + let old_version = ::decode( + &mut &*executor.read_runtime_version(&original_code, &mut ext.ext()).unwrap(), + ) + .unwrap(); + log::info!( + target: LOG_TARGET, + "original spec: {:?}-{:?}, code hash: {:?}", + old_version.spec_name, + old_version.spec_version, + HexDisplay::from(BlakeTwo256::hash(&original_code).as_fixed_bytes()), + ); + let new_version = ::decode( + &mut &*executor.read_runtime_version(&new_code, &mut ext.ext()).unwrap(), + ) + .unwrap(); + log::info!( + target: LOG_TARGET, + "new spec: {:?}-{:?}, code hash: {:?}", + new_version.spec_name, + new_version.spec_version, + HexDisplay::from(BlakeTwo256::hash(&new_code).as_fixed_bytes()) + ); - /// Get the uri, if self is `Live`. - pub(crate) fn live_uri(&self) -> Option { - match self { - State::Live { uri, .. } => Some(uri.clone()), - _ => None, + if new_version.spec_name != old_version.spec_name { + return Err("Spec names must match.".into()) + } + } + + // whatever runtime we have in store now must have been compiled with try-runtime feature. + if !ensure_try_runtime::(&executor, &mut ext) { + return Err("given runtime is NOT compiled with try-runtime feature!".into()) } + + Ok(ext) } } impl TryRuntimeCmd { - pub async fn run(&self, config: Configuration) -> sc_cli::Result<()> + pub async fn run(&self) -> sc_cli::Result<()> where Block: BlockT + DeserializeOwned, Block::Header: DeserializeOwned, Block::Hash: FromStr, ::Err: Debug, - NumberFor: FromStr, as FromStr>::Err: Debug, - ExecDispatch: NativeExecutionDispatch + 'static, + as TryInto>::Error: Debug, + NumberFor: FromStr, + HostFns: HostFunctions, { match &self.command { Command::OnRuntimeUpgrade(ref cmd) => - commands::on_runtime_upgrade::on_runtime_upgrade::( + commands::on_runtime_upgrade::on_runtime_upgrade::( self.shared.clone(), cmd.clone(), - config, ) .await, Command::OffchainWorker(cmd) => - commands::offchain_worker::offchain_worker::( + commands::offchain_worker::offchain_worker::( self.shared.clone(), cmd.clone(), - config, ) .await, Command::ExecuteBlock(cmd) => - commands::execute_block::execute_block::( + commands::execute_block::execute_block::( self.shared.clone(), cmd.clone(), - config, ) .await, Command::FollowChain(cmd) => - commands::follow_chain::follow_chain::( + commands::follow_chain::follow_chain::( + self.shared.clone(), + cmd.clone(), + ) + .await, + Command::CreateSnapshot(cmd) => + commands::create_snapshot::create_snapshot::( self.shared.clone(), cmd.clone(), - config, ) .await, } @@ -598,22 +769,6 @@ impl CliConfiguration for TryRuntimeCmd { } } -/// Extract `:code` from the given chain spec and return as `StorageData` along with the -/// corresponding `StorageKey`. -pub(crate) fn extract_code(spec: &Box) -> sc_cli::Result<(StorageKey, StorageData)> { - let genesis_storage = spec.build_storage()?; - let code = StorageData( - genesis_storage - .top - .get(well_known_keys::CODE) - .expect("code key must exist in genesis storage; qed") - .to_vec(), - ); - let code_key = StorageKey(well_known_keys::CODE.to_vec()); - - Ok((code_key, code)) -} - /// Get the hash type of the generic `Block` from a `hash_str`. pub(crate) fn hash_of(hash_str: &str) -> sc_cli::Result where @@ -625,67 +780,6 @@ where .map_err(|e| format!("Could not parse block hash: {:?}", e).into()) } -/// Check the spec_name of an `ext` -/// -/// If the spec names don't match, if `relaxed`, then it emits a warning, else it panics. -/// If the spec versions don't match, it only ever emits a warning. -pub(crate) async fn ensure_matching_spec( - uri: String, - expected_spec_name: String, - expected_spec_version: u32, - relaxed: bool, -) { - let rpc = ws_client(&uri).await.unwrap(); - match StateApi::::runtime_version(&rpc, None) - .await - .map(|version| (String::from(version.spec_name.clone()), version.spec_version)) - .map(|(spec_name, spec_version)| (spec_name.to_lowercase(), spec_version)) - { - Ok((name, version)) => { - // first, deal with spec name - if expected_spec_name.to_lowercase() == name { - log::info!(target: LOG_TARGET, "found matching spec name: {:?}", name); - } else { - let msg = format!( - "version mismatch: remote spec name: '{}', expected (local chain spec, aka. `--chain`): '{}'", - name, - expected_spec_name - ); - if relaxed { - log::warn!(target: LOG_TARGET, "{}", msg); - } else { - panic!("{}", msg); - } - } - - if expected_spec_version == version { - log::info!(target: LOG_TARGET, "found matching spec version: {:?}", version); - } else { - let msg = format!( - "spec version mismatch (local {} != remote {}). This could cause some issues.", - expected_spec_version, version - ); - if relaxed { - log::warn!(target: LOG_TARGET, "{}", msg); - } else { - panic!("{}", msg); - } - } - }, - Err(why) => { - let msg = format!( - "failed to fetch runtime version from {}: {:?}. Skipping the check", - uri, why - ); - if relaxed { - log::error!(target: LOG_TARGET, "{}", msg); - } else { - panic!("{}", msg); - } - }, - } -} - /// Build all extensions that we typically use. pub(crate) fn full_extensions() -> Extensions { let mut extensions = Extensions::default(); @@ -700,29 +794,43 @@ pub(crate) fn full_extensions() -> Extensions { extensions } -/// Build a default execution that we typically use. -pub(crate) fn build_executor( - shared: &SharedParams, - config: &sc_service::Configuration, -) -> NativeElseWasmExecutor { - let heap_pages = shared.heap_pages.or(config.default_heap_pages); - let max_runtime_instances = config.max_runtime_instances; - let runtime_cache_size = config.runtime_cache_size; - - NativeElseWasmExecutor::::new( - execution_method_from_cli(shared.wasm_method, shared.wasmtime_instantiation_strategy), +pub(crate) fn build_executor(shared: &SharedParams) -> WasmExecutor { + let heap_pages = shared.heap_pages.or(Some(2048)); + let max_runtime_instances = 8; + let runtime_cache_size = 2; + + WasmExecutor::new( + sc_executor::WasmExecutionMethod::Interpreted, heap_pages, max_runtime_instances, + None, runtime_cache_size, ) } +/// Ensure that the given `ext` is compiled with `try-runtime` +fn ensure_try_runtime( + executor: &WasmExecutor, + ext: &mut TestExternalities, +) -> bool { + use sp_api::RuntimeApiInfo; + let final_code = ext + .execute_with(|| sp_io::storage::get(well_known_keys::CODE)) + .expect("':CODE:' is always downloaded in try-runtime-cli; qed"); + let final_version = ::decode( + &mut &*executor.read_runtime_version(&final_code, &mut ext.ext()).unwrap(), + ) + .unwrap(); + final_version + .api_version(&>::ID) + .is_some() +} + /// Execute the given `method` and `data` on top of `ext`, returning the results (encoded) and the /// state `changes`. -pub(crate) fn state_machine_call( +pub(crate) fn state_machine_call( ext: &TestExternalities, - executor: &NativeElseWasmExecutor, - execution: sc_cli::ExecutionStrategy, + executor: &WasmExecutor, method: &'static str, data: &[u8], extensions: Extensions, @@ -738,7 +846,7 @@ pub(crate) fn state_machine_call(Into::into)?; @@ -749,28 +857,23 @@ pub(crate) fn state_machine_call( +pub(crate) fn state_machine_call_with_proof( ext: &TestExternalities, - executor: &NativeElseWasmExecutor, - execution: sc_cli::ExecutionStrategy, + executor: &WasmExecutor, method: &'static str, data: &[u8], extensions: Extensions, ) -> sc_cli::Result<(OverlayedChanges, Vec)> { use parity_scale_codec::Encode; - use sp_core::hexdisplay::HexDisplay; let mut changes = Default::default(); let backend = ext.backend.clone(); let runtime_code_backend = sp_state_machine::backend::BackendRuntimeCode::new(&backend); - let proving_backend = TrieBackendBuilder::wrap(&backend).with_recorder(Default::default()).build(); - let runtime_code = runtime_code_backend.runtime_code()?; let pre_root = *backend.root(); - let encoded_results = StateMachine::new( &proving_backend, &mut changes, @@ -781,7 +884,7 @@ pub(crate) fn state_machine_call_with_proof(Into::into)?; @@ -792,11 +895,24 @@ pub(crate) fn state_machine_call_with_proof(pre_root) - .map_err(|e| format!("failed to generate compact proof {}: {:?}", method, e))?; + .map_err(|e| { + log::error!(target: LOG_TARGET, "failed to generate compact proof {}: {:?}", method, e); + e + }) + .unwrap_or(CompactProof { encoded_nodes: Default::default() }); let compact_proof_size = compact_proof.encoded_size(); let compressed_proof = zstd::stream::encode_all(&compact_proof.encode()[..], 0) - .map_err(|e| format!("failed to generate compact proof {}: {:?}", method, e))?; + .map_err(|e| { + log::error!( + target: LOG_TARGET, + "failed to generate compressed proof {}: {:?}", + method, + e + ); + e + }) + .unwrap_or_default(); let proof_nodes = proof.into_nodes(); @@ -814,8 +930,8 @@ pub(crate) fn state_machine_call_with_proof>()), + "proof: 0x{}... / {} nodes", + HexDisplay::from(&proof_nodes.iter().flatten().cloned().take(10).collect::>()), proof_nodes.len() ); log::debug!(target: LOG_TARGET, "proof size: {}", humanize(proof_size)); @@ -825,28 +941,13 @@ pub(crate) fn state_machine_call_with_proof( - ext: &TestExternalities, - executor: &NativeElseWasmExecutor, -) -> (String, u32, sp_core::storage::StateVersion) { - let (_, encoded) = state_machine_call::( - ext, - executor, - sc_cli::ExecutionStrategy::NativeElseWasm, - "Core_version", - &[], - Default::default(), - ) - .expect("all runtimes should have version; qed"); - ::decode(&mut &*encoded) - .map_err(|e| format!("failed to decode output: {:?}", e)) - .map(|v| { - let state_version = v.state_version(); - (v.spec_name.into(), v.spec_version, state_version) - }) - .expect("all runtimes should have version; qed") +pub(crate) fn rpc_err_handler(error: impl Debug) -> &'static str { + log::error!(target: LOG_TARGET, "rpc error: {:?}", error); + "rpc error." }