From 9ce7f32cae12ce637bd3b85d255dd55ba7198f9d Mon Sep 17 00:00:00 2001 From: Oliver Tale-Yazdi Date: Wed, 2 Mar 2022 11:17:19 +0000 Subject: [PATCH] Add benchmark-block Signed-off-by: Oliver Tale-Yazdi --- Cargo.lock | 14 + bin/node-template/node/Cargo.toml | 10 + bin/node-template/node/src/cli.rs | 4 + bin/node-template/node/src/command.rs | 75 ++++- bin/node-template/node/src/service.rs | 2 +- bin/node/cli/Cargo.toml | 6 + bin/node/cli/src/cli.rs | 4 + bin/node/cli/src/command.rs | 70 +++- frame/aura/src/lib.rs | 5 +- frame/babe/src/lib.rs | 11 +- utils/frame/benchmarking-cli/Cargo.toml | 6 + utils/frame/benchmarking-cli/src/block/cmd.rs | 303 ++++++++++++++++++ utils/frame/benchmarking-cli/src/block/mod.rs | 20 ++ .../benchmarking-cli/src/block/weights.hbs | 87 +++++ utils/frame/benchmarking-cli/src/lib.rs | 2 + .../benchmarking-cli/src/storage/record.rs | 2 +- .../benchmarking-cli/src/storage/write.rs | 1 + 17 files changed, 610 insertions(+), 12 deletions(-) create mode 100644 utils/frame/benchmarking-cli/src/block/cmd.rs create mode 100644 utils/frame/benchmarking-cli/src/block/mod.rs create mode 100644 utils/frame/benchmarking-cli/src/block/weights.hbs diff --git a/Cargo.lock b/Cargo.lock index 5c1b6ed88fb4a..5739ced35f011 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2126,6 +2126,7 @@ dependencies = [ "clap 3.0.7", "frame-benchmarking", "frame-support", + "futures 0.3.19", "handlebars", "hash-db", "hex", @@ -2136,9 +2137,11 @@ dependencies = [ "memory-db", "parity-scale-codec", "rand 0.8.4", + "sc-block-builder", "sc-cli", "sc-client-api", "sc-client-db", + "sc-consensus", "sc-executor", "sc-service", "serde", @@ -2146,9 +2149,11 @@ dependencies = [ "serde_nanos", "sp-api", "sp-blockchain", + "sp-consensus", "sp-core", "sp-database", "sp-externalities", + "sp-inherents", "sp-keystore", "sp-runtime", "sp-state-machine", @@ -4787,6 +4792,7 @@ version = "3.0.0-dev" dependencies = [ "assert_cmd", "async-std", + "chrono", "clap 3.0.7", "clap_complete", "criterion", @@ -4845,6 +4851,7 @@ dependencies = [ "sp-blockchain", "sp-consensus", "sp-consensus-babe", + "sp-consensus-slots", "sp-core", "sp-finality-grandpa", "sp-inherents", @@ -5059,6 +5066,9 @@ dependencies = [ "frame-benchmarking", "frame-benchmarking-cli", "jsonrpc-core", + "log 0.4.14", + "node-cli", + "node-runtime", "node-template-runtime", "pallet-transaction-payment-rpc", "sc-basic-authorship", @@ -5080,8 +5090,12 @@ dependencies = [ "sp-blockchain", "sp-consensus", "sp-consensus-aura", + "sp-consensus-babe", + "sp-consensus-slots", "sp-core", "sp-finality-grandpa", + "sp-inherents", + "sp-keyring", "sp-runtime", "sp-timestamp", "substrate-build-script-utils", diff --git a/bin/node-template/node/Cargo.toml b/bin/node-template/node/Cargo.toml index 98e8af96d3f8d..6e0d8bbc46d34 100644 --- a/bin/node-template/node/Cargo.toml +++ b/bin/node-template/node/Cargo.toml @@ -37,6 +37,12 @@ sc-client-api = { version = "4.0.0-dev", path = "../../../client/api" } sp-runtime = { version = "6.0.0", path = "../../../primitives/runtime" } sp-timestamp = { version = "4.0.0-dev", path = "../../../primitives/timestamp" } +# OTY added +sp-consensus-slots = { version = "0.10.0-dev", path = "../../../primitives/consensus/slots" } +sp-consensus-babe = { version = "0.10.0-dev", path = "../../../primitives/consensus/babe" } +sp-inherents = { version = "4.0.0-dev", path = "../../../primitives/inherents" } +log = "0.4.8" + # These dependencies are used for the node template's RPCs jsonrpc-core = "18.0.0" sc-rpc = { version = "4.0.0-dev", path = "../../../client/rpc" } @@ -57,6 +63,10 @@ node-template-runtime = { version = "4.0.0-dev", path = "../runtime" } # CLI-specific dependencies try-runtime-cli = { version = "0.10.0-dev", optional = true, path = "../../../utils/frame/try-runtime/cli" } +# OTY added +sp-keyring = { version = "6.0.0", path = "../../../primitives/keyring" } +node-runtime = { version = "3.0.0-dev", path = "../../node/runtime" } +node-cli = { version = "3.0.0-dev", path = "../../node/cli" } [build-dependencies] substrate-build-script-utils = { version = "3.0.0", path = "../../../utils/build-script-utils" } diff --git a/bin/node-template/node/src/cli.rs b/bin/node-template/node/src/cli.rs index c4d27b71e4994..9db00ccdb5dd9 100644 --- a/bin/node-template/node/src/cli.rs +++ b/bin/node-template/node/src/cli.rs @@ -40,6 +40,10 @@ pub enum Subcommand { #[clap(name = "benchmark", about = "Benchmark runtime pallets.")] Benchmark(frame_benchmarking_cli::BenchmarkCmd), + /// The custom benchmark subcommand benchmarking runtime pallets. + #[clap(name = "benchmark-block", about = "Benchmark runtime pallets.")] + BenchmarkBlock(frame_benchmarking_cli::BlockCmd), + /// Try some command against runtime state. #[cfg(feature = "try-runtime")] TryRuntime(try_runtime_cli::TryRuntimeCmd), diff --git a/bin/node-template/node/src/command.rs b/bin/node-template/node/src/command.rs index 72c7a75b387bb..346c139ebb1ab 100644 --- a/bin/node-template/node/src/command.rs +++ b/bin/node-template/node/src/command.rs @@ -3,10 +3,13 @@ use crate::{ cli::{Cli, Subcommand}, service, }; -use node_template_runtime::Block; +use node_cli::service::create_extrinsic; +use node_runtime::SystemCall; use sc_cli::{ChainSpec, RuntimeVersion, SubstrateCli}; use sc_service::PartialComponents; +use std::sync::Arc; + impl SubstrateCli for Cli { fn impl_name() -> String { "Substrate Node".into() @@ -46,6 +49,47 @@ impl SubstrateCli for Cli { } } +use super::service::FullClient; +struct ExtrinsicGen { + client: Arc, +} +use node_runtime::UncheckedExtrinsic; +use node_template_runtime::Block; +use sp_keyring::Sr25519Keyring; +use sp_runtime::{traits::Block as BlockT, OpaqueExtrinsic}; +/* +impl frame_benchmarking_cli::block::cmd::ExtrinsicGenerator for ExtrinsicGen { + fn noop(&self, nonce: u32) -> Option { + let src = Sr25519Keyring::Alice.pair(); + + let extrinsic: OpaqueExtrinsic = create_extrinsic( + self.client.as_ref(), + src.clone(), + SystemCall::remark { remark: vec![] }, + Some(nonce), + ) + .into(); + None + } +}*/ + +#[derive(Default)] +struct InherentProv {} + +impl frame_benchmarking_cli::block::cmd::BlockInherentDataProvider for InherentProv { + fn providers( + &self, + block: u64, + ) -> std::result::Result>, sp_inherents::Error> + { + log::info!("Creating inherents for block #{}", block); + let d = std::time::Duration::from_millis(0); + let timestamp = sp_timestamp::InherentDataProvider::new(d.into()); + + Ok(vec![Arc::new(timestamp)]) + } +} + /// Parse and run command line arguments pub fn run() -> sc_cli::Result<()> { let cli = Cli::from_args(); @@ -98,6 +142,35 @@ pub fn run() -> sc_cli::Result<()> { Ok((cmd.run(client, backend), task_manager)) }) }, + Some(Subcommand::BenchmarkBlock(cmd)) => { + unimplemented!(); + }, + /*let runner = cli.create_runner(cmd)?; + runner.async_run(|mut config| { + config.role = sc_service::Role::Full; + + let PartialComponents { client, task_manager, backend, import_queue, .. } = + service::new_partial(&config)?; + let db = backend.expose_db(); + let storage = backend.expose_storage(); + let ext_gen = ExtrinsicGen { client: client.clone() }; + let provs: InherentProv = Default::default(); + + Ok(( + cmd.run( + config, + client.clone(), + import_queue, + db, + storage, + client.clone(), + Arc::new(provs), + Arc::new(ext_gen), + ), + task_manager, + )) + }) + },*/ Some(Subcommand::Benchmark(cmd)) => if cfg!(feature = "runtime-benchmarks") { let runner = cli.create_runner(cmd)?; diff --git a/bin/node-template/node/src/service.rs b/bin/node-template/node/src/service.rs index fc7dc9b978df3..e2a8cb4ed834b 100644 --- a/bin/node-template/node/src/service.rs +++ b/bin/node-template/node/src/service.rs @@ -31,7 +31,7 @@ impl sc_executor::NativeExecutionDispatch for ExecutorDispatch { } } -type FullClient = +pub(crate) type FullClient = sc_service::TFullClient>; type FullBackend = sc_service::TFullBackend; type FullSelectChain = sc_consensus::LongestChain; diff --git a/bin/node/cli/Cargo.toml b/bin/node/cli/Cargo.toml index b4a91712fd16c..a5c5c0530b854 100644 --- a/bin/node/cli/Cargo.toml +++ b/bin/node/cli/Cargo.toml @@ -96,6 +96,12 @@ frame-benchmarking-cli = { version = "4.0.0-dev", optional = true, path = "../.. node-inspect = { version = "0.9.0-dev", optional = true, path = "../inspect" } try-runtime-cli = { version = "0.10.0-dev", optional = true, path = "../../../utils/frame/try-runtime/cli" } +# OTY added +sp-consensus-slots = { version = "0.10.0-dev", path = "../../../primitives/consensus/slots" } +# OTY +sp-blockchain = { version = "4.0.0-dev", path = "../../../primitives/blockchain" } +chrono = "0.4.19" + [target.'cfg(any(target_arch="x86_64", target_arch="aarch64"))'.dependencies] node-executor = { version = "3.0.0-dev", path = "../executor", features = ["wasmtime"] } sc-cli = { version = "0.10.0-dev", optional = true, path = "../../../client/cli", features = ["wasmtime"] } diff --git a/bin/node/cli/src/cli.rs b/bin/node/cli/src/cli.rs index 386215854b963..b6c6da310822d 100644 --- a/bin/node/cli/src/cli.rs +++ b/bin/node/cli/src/cli.rs @@ -42,6 +42,10 @@ pub enum Subcommand { #[clap(name = "benchmark", about = "Benchmark runtime pallets.")] Benchmark(frame_benchmarking_cli::BenchmarkCmd), + /// Sub command for benchmarking the storage speed. + #[clap(name = "benchmark-block", about = "Benchmark TODO.")] + BenchmarkBlock(frame_benchmarking_cli::BlockCmd), + /// Sub command for benchmarking the storage speed. #[clap(name = "benchmark-storage", about = "Benchmark storage speed.")] BenchmarkStorage(frame_benchmarking_cli::StorageCmd), diff --git a/bin/node/cli/src/command.rs b/bin/node/cli/src/command.rs index cc6480bb90d55..0a181f3b9b44f 100644 --- a/bin/node/cli/src/command.rs +++ b/bin/node/cli/src/command.rs @@ -18,10 +18,12 @@ use crate::{chain_spec, service, service::new_partial, Cli, Subcommand}; use node_executor::ExecutorDispatch; -use node_runtime::{Block, RuntimeApi}; +use node_runtime::RuntimeApi; use sc_cli::{ChainSpec, Result, RuntimeVersion, SubstrateCli}; use sc_service::PartialComponents; +use std::sync::Arc; + impl SubstrateCli for Cli { fn impl_name() -> String { "Substrate Node".into() @@ -68,6 +70,46 @@ impl SubstrateCli for Cli { &node_runtime::VERSION } } +struct ExtrinsicGen { + client: Arc, +} +use crate::service::{create_extrinsic, FullClient}; +use node_primitives::Block; +use node_runtime::{BalancesCall, SystemCall}; +use sp_keyring::Sr25519Keyring; +use sp_runtime::AccountId32; +use sp_runtime::{traits::Block as BlockT, MultiAddress, OpaqueExtrinsic}; +impl frame_benchmarking_cli::block::cmd::ExtrinsicGenerator for ExtrinsicGen { + fn noop(&self, nonce: u32) -> Option { + let src = Sr25519Keyring::Alice.pair(); + + let extrinsic: OpaqueExtrinsic = create_extrinsic( + self.client.as_ref(), + src.clone(), + SystemCall::remark { remark: vec![] }, + Some(nonce), + ) + .into(); + Some(extrinsic) + } +} + +#[derive(Default)] +struct InherentProv {} + +impl frame_benchmarking_cli::block::cmd::BlockInherentDataProvider for InherentProv { + fn providers( + &self, + block: u64, + ) -> std::result::Result>, sp_inherents::Error> + { + log::info!("Creating inherents for block #{}", block); + let d = std::time::Duration::from_millis(0); + let timestamp = sp_timestamp::InherentDataProvider::new(d.into()); + + Ok(vec![Arc::new(timestamp)]) + } +} /// Parse command line arguments into service configuration. pub fn run() -> Result<()> { @@ -95,6 +137,32 @@ pub fn run() -> Result<()> { You can enable it with `--features runtime-benchmarks`." .into()) }, + Some(Subcommand::BenchmarkBlock(cmd)) => { + let runner = cli.create_runner(cmd)?; + runner.async_run(|mut config| { + config.role = sc_service::Role::Full; + + let PartialComponents { client, task_manager, backend, .. } = new_partial(&config)?; + let db = backend.expose_db(); + let storage = backend.expose_storage(); + let ext_gen = ExtrinsicGen { client: client.clone() }; + + let provs: InherentProv = Default::default(); + Ok(( + cmd.run( + config, + client.clone(), + client.clone(), + db, + storage, + client.clone(), + Arc::new(provs), + Arc::new(ext_gen), + ), + task_manager, + )) + }) + }, Some(Subcommand::BenchmarkStorage(cmd)) => { if !cfg!(feature = "runtime-benchmarks") { return Err("Benchmarking wasn't enabled when building the node. \ diff --git a/frame/aura/src/lib.rs b/frame/aura/src/lib.rs index 657965c60a3f1..83fb6cb392a27 100644 --- a/frame/aura/src/lib.rs +++ b/frame/aura/src/lib.rs @@ -282,9 +282,6 @@ impl OnTimestampSet for Pallet { let timestamp_slot = moment / slot_duration; let timestamp_slot = Slot::from(timestamp_slot.saturated_into::()); - assert!( - CurrentSlot::::get() == timestamp_slot, - "Timestamp slot must match `CurrentSlot`" - ); + assert!(CurrentSlot::::get() == timestamp_slot, "sdaf`"); } } diff --git a/frame/babe/src/lib.rs b/frame/babe/src/lib.rs index f673c8b43bee0..22d0fd62891a6 100644 --- a/frame/babe/src/lib.rs +++ b/frame/babe/src/lib.rs @@ -842,10 +842,13 @@ impl OnTimestampSet for Pallet { let timestamp_slot = moment / slot_duration; let timestamp_slot = Slot::from(timestamp_slot.saturated_into::()); - assert!( - CurrentSlot::::get() == timestamp_slot, - "Timestamp slot must match `CurrentSlot`" - ); + let c = CurrentSlot::::get(); + if c != timestamp_slot { + panic!( + "d: {:?}, moment: {:?}, want: {:?}, got: {:?}", + slot_duration, moment, c, timestamp_slot + ); + } } } diff --git a/utils/frame/benchmarking-cli/Cargo.toml b/utils/frame/benchmarking-cli/Cargo.toml index 20122c4279366..46c7b985c3dbd 100644 --- a/utils/frame/benchmarking-cli/Cargo.toml +++ b/utils/frame/benchmarking-cli/Cargo.toml @@ -49,6 +49,12 @@ hex = "0.4.3" memory-db = "0.29.0" rand = { version = "0.8.4", features = ["small_rng"] } +sc-block-builder = { version = "0.10.0-dev", path = "../../../client/block-builder" } +sp-inherents = { version = "4.0.0-dev", path = "../../../primitives/inherents" } +futures = "0.3.19" +sp-consensus = { version = "0.10.0-dev", path = "../../../primitives/consensus/common" } +sc-consensus = { version = "0.10.0-dev", path = "../../../client/consensus/common" } + [features] default = ["db", "sc-client-db/runtime-benchmarks"] db = ["sc-client-db/with-kvdb-rocksdb", "sc-client-db/with-parity-db"] diff --git a/utils/frame/benchmarking-cli/src/block/cmd.rs b/utils/frame/benchmarking-cli/src/block/cmd.rs new file mode 100644 index 0000000000000..4a6a9dfb33f37 --- /dev/null +++ b/utils/frame/benchmarking-cli/src/block/cmd.rs @@ -0,0 +1,303 @@ +// This file is part of Substrate. + +// Copyright (C) 2022 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use sc_cli::{CliConfiguration, DatabaseParams, PruningParams, Result, SharedParams}; +use sc_client_api::{Backend as ClientBackend, StorageProvider, UsageProvider}; +use sc_client_db::DbHash; +use sc_service::Configuration; +use sp_blockchain::HeaderBackend; +use sp_database::{ColumnId, Database}; +use sp_runtime::{ + OpaqueExtrinsic, + traits::{Block as BlockT, HashFor}, + transaction_validity::{InvalidTransaction, TransactionValidityError}, +}; +use sp_state_machine::Storage; +use sp_storage::StateVersion; +use sp_inherents::InherentData; +use sc_block_builder::{BlockBuilderProvider, BlockBuilder}; +use sc_block_builder::BlockBuilderApi; +use sp_api::{ApiExt, ProvideRuntimeApi}; +use sc_consensus::{ + block_import::{BlockImportParams, ForkChoiceStrategy}, + BlockImport, StateAction, +}; +use sp_blockchain::{ApplyExtrinsicFailed::Validity, Error::ApplyExtrinsicFailed}; +use sp_consensus::BlockOrigin; +use sp_api::BlockId; + +use clap::{Args, Parser}; +use log::info; +use rand::prelude::*; +use serde::Serialize; +use std::{boxed::Box, fmt::Debug, sync::Arc, time::Instant}; + +use crate::storage::{record::{Stats, StatSelect}, template::TemplateData}; + +#[derive(Debug, Parser)] +pub struct BlockCmd { + #[allow(missing_docs)] + #[clap(flatten)] + pub shared_params: SharedParams, + + #[allow(missing_docs)] + #[clap(flatten)] + pub database_params: DatabaseParams, + + #[allow(missing_docs)] + #[clap(flatten)] + pub pruning_params: PruningParams, + + #[allow(missing_docs)] + #[clap(flatten)] + pub params: BlockParams, +} + +#[derive(Debug, Default, Serialize, Clone, PartialEq, Args)] +pub struct BlockParams { + /// Path to write the *weight* file to. Can be a file or directory. + /// For substrate this should be `frame/support/src/weights`. + #[clap(long, default_value = ".")] + pub weight_path: String, + + /// Select a specific metric to calculate the final weight output. + #[clap(long = "metric", default_value = "average")] + pub weight_metric: StatSelect, + + /// Multiply the resulting weight with the given factor. Must be positive. + /// Is calculated before `weight_add`. + #[clap(long = "mul", default_value = "1")] + pub weight_mul: f64, + + /// Add the given offset to the resulting weight. + /// Is calculated after `weight_mul`. + #[clap(long = "add", default_value = "0")] + pub weight_add: u64, + + /// Skip the `read` benchmark. + #[clap(long)] + pub skip_read: bool, + + /// Skip the `write` benchmark. + #[clap(long)] + pub skip_write: bool, + + /// Rounds of warmups before measuring. + /// Only supported for `read` benchmarks. + #[clap(long, default_value = "1")] + pub warmups: u32, + + /// State cache size. + #[clap(long, default_value = "0")] + pub state_cache_size: usize, + + /// Limit them number of extrinsics to put in a block. + #[clap(long)] + pub max_ext_per_block: Option, + + /// Repeats for measuring the time of a an empty block execution. + #[clap(long, default_value = "1000")] + pub repeat_empty_block: u32, + + /// Repeats for measuring the time of a a full block execution. + #[clap(long, default_value = "100")] + pub repeat_full_block: u32, +} + +/// The results of multiple runs in ns. +type BenchRecord = Vec; + +pub trait ExtrinsicGenerator { + fn noop(&self, nonce: u32) -> Option; +} + +impl BlockCmd { + pub async fn run( + &self, + cfg: Configuration, + client: Arc, + mut iq: IQ, + db: (Arc>, ColumnId), + storage: Arc>>, + ra: Arc, + idps: Arc, + ext_gen: Arc, + ) -> Result<()> + where + // TODO remove the Extrinsic = OpaqueExtrinsic + Block: BlockT, + BA: ClientBackend, + C: BlockBuilderProvider + + UsageProvider + + StorageProvider + + HeaderBackend, + RA: ProvideRuntimeApi, + IQ: sc_consensus::BlockImport, + API: ApiExt + BlockBuilderApi, + { + let genesis = BlockId::Number(client.info().best_number); + //let mut template = TemplateData::new(&cfg, &self.params); + info!("Creating empty block"); + // Block builders are not cloneable, so we need two. + let mut build_empty = client.new_block(Default::default()).unwrap(); + let mut build_full = client.new_block(Default::default()).unwrap(); + let mut data = sp_inherents::InherentData::new(); + idps.provide_inherent_data(&mut data).unwrap(); + info!("Creating inherents from {} inherent datas", data.len()); + let inherents = build_empty.create_inherents(data).unwrap(); + info!("Inserting {} inherent extrinsic", inherents.len()); + for ext in inherents.clone() { + build_empty.push(ext.clone())?; + build_full.push(ext.clone())?; + } + // Build a block with only inherents aka empty. + let empty_block = build_empty.build()?.block; + // Interesting part here: + // Benchmark 1; Execution time of a an empty block. + let mut rec_empty = BenchRecord::new(); + info!("Executing an empty block {} times", self.params.repeat_empty_block); + for i in 0..self.params.repeat_empty_block { + let block = empty_block.clone(); + let start = Instant::now(); + ra.runtime_api().execute_block(&genesis, block).unwrap(); + rec_empty.push(start.elapsed().as_nanos() as u64); + } + + info!("Estimating max NO-OPs per block, capped at {}", self.max_ext_per_block()); + let mut noops = Vec::new(); + for nonce in 0..self.max_ext_per_block() { + let ext = ext_gen.noop(nonce).expect("Need noop"); + match build_full.push(ext.clone()) { + Ok(_) => {}, + Err(ApplyExtrinsicFailed(Validity(TransactionValidityError::Invalid( + InvalidTransaction::ExhaustsResources, + )))) => break, + Err(error) => panic!("{}", error), + } + noops.push(ext); + } + info!("Max NO-OPs per block {}, continuing with benchmark.", noops.len()); + let full_block = build_full.build()?.block; + + // Interesting part here: + // Benchmark 2; Execution time of a block filled with NO-OPs. + let mut rec_full = BenchRecord::new(); + info!("Executing a full block {} times", self.params.repeat_full_block); + for i in 0..self.params.repeat_full_block { + let block = full_block.clone(); + let start = Instant::now(); + ra.runtime_api().execute_block(&genesis, block).unwrap(); + let took = start.elapsed().as_nanos(); + // Divide by the number of extrinsics in a block. + rec_full.push(took as u64 / noops.len() as u64); + } + + let start = Instant::now(); + Self::must_import(full_block, &mut iq); + let speed = noops.len() as f32 / start.elapsed().as_secs_f32(); + info!("Imported block {:.2} NO-OP/s", speed); + + let stats = Stats::new(&rec_empty).unwrap(); + info!("Executing an empty block [ns]:\n{:?}", stats); + + let stats = Stats::new(&rec_full).unwrap(); + info!("Executing NO-OP extrinsic [ns]:\n{:?}", stats); + + Ok(()) + } + + /*fn new_block<'a, Block, BA, RA, C>(client: Arc, idps: Arc,) -> Result> + where Block: BlockT, + C: BlockBuilderProvider + 'a, + RA: sp_api::ProvideRuntimeApi, + BA: sc_client_api::Backend { + let builder = client.new_block(Default::default()).unwrap(); + Ok(builder) + }*/ + + fn must_import(block: B, iq: &mut IQ) + where + B: BlockT, + IQ: sc_consensus::BlockImport, + { + let mut params = BlockImportParams::new(BlockOrigin::File, block.header().clone()); + params.body = Some(block.extrinsics().to_vec()); + params.fork_choice = Some(ForkChoiceStrategy::LongestChain); + let block_hash = block.hash(); + let import = + futures::executor::block_on(iq.import_block(params, Default::default())).unwrap(); + if !matches!(import, sc_consensus::ImportResult::Imported(_)) { + panic!("Could not import block: {:?}", import) + } + } + + /// Creates an rng from a random seed. + pub(crate) fn setup_rng() -> impl rand::Rng { + let seed = rand::thread_rng().gen::(); + info!("Using seed {}", seed); + StdRng::seed_from_u64(seed) + } + + fn max_ext_per_block(&self) -> u32 { + self.params.max_ext_per_block.unwrap_or(u32::MAX) + } +} + +// Boilerplate +impl CliConfiguration for BlockCmd { + fn shared_params(&self) -> &SharedParams { + &self.shared_params + } + + fn database_params(&self) -> Option<&DatabaseParams> { + Some(&self.database_params) + } + + fn pruning_params(&self) -> Option<&PruningParams> { + Some(&self.pruning_params) + } + + fn state_cache_size(&self) -> Result { + Ok(self.params.state_cache_size) + } +} + + + + /*info!("Importing empty block as warmup"); + let block = block_builder.build()?.block; + info!("Executing block"); + ra.runtime_api().execute_block(&BlockId::Number(genesis), block.clone()).expect("Must execute"); + info!("Importing block"); + Self::must_import(block, &mut iq); + + info!("Creating empty block"); + let mut block_builder = client.new_block(Default::default()).unwrap(); + + let idps = block_inherents.providers(1).unwrap(); + info!("Creating inherent data from {} providers", idps.len()); + + let mut data = sp_inherents::InherentData::new(); + for idp in idps.clone() { + idp.provide_inherent_data(&mut data).unwrap(); + } + info!("Creating inherents from {} inherent datas", data.len()); + let inherents = block_builder.create_inherents(data).unwrap(); + info!("Inserting {} extrinsic", inherents.len()); + for ext in inherents.clone() { + block_builder.push(ext.clone())?; + }*/ diff --git a/utils/frame/benchmarking-cli/src/block/mod.rs b/utils/frame/benchmarking-cli/src/block/mod.rs new file mode 100644 index 0000000000000..4e33e24202b1c --- /dev/null +++ b/utils/frame/benchmarking-cli/src/block/mod.rs @@ -0,0 +1,20 @@ +// This file is part of Substrate. + +// Copyright (C) 2022 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +pub mod cmd; + +pub use cmd::BlockCmd; diff --git a/utils/frame/benchmarking-cli/src/block/weights.hbs b/utils/frame/benchmarking-cli/src/block/weights.hbs new file mode 100644 index 0000000000000..eafdd1114e766 --- /dev/null +++ b/utils/frame/benchmarking-cli/src/block/weights.hbs @@ -0,0 +1,87 @@ +// This file is part of Substrate. + +// Copyright (C) 2022 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! THIS FILE WAS AUTO-GENERATED USING THE SUBSTRATE BENCHMARK CLI VERSION {{version}} +//! DATE: {{date}} +//! +//! DATABASE: `{{db_name}}`, RUNTIME: `{{runtime_name}}` +//! SKIP-WRITE: `{{params.skip_write}}`, SKIP-READ: `{{params.skip_read}}`, WARMUPS: `{{params.warmups}}` +//! STATE-VERSION: `V{{params.state_version}}`, STATE-CACHE-SIZE: `{{params.state_cache_size}}` +//! WEIGHT-PATH: `{{params.weight_path}}` +//! METRIC: `{{params.weight_metric}}`, WEIGHT-MUL: `{{params.weight_mul}}`, WEIGHT-ADD: `{{params.weight_add}}` + +// Executed Command: +{{#each args as |arg|}} +// {{arg}} +{{/each}} + +pub mod constants { + use frame_support::{ + parameter_types, + weights::{Weight, constants, RuntimeDbWeight}, + }; + + parameter_types! { + /// Importing a block with 0 txs. + pub const BlockExecutionWeight: Weight = {{underscore block}} * constants::WEIGHT_PER_NANOS; + /// Executing a System remarks (no-op) txs. + pub const ExtrinsicBaseWeight: Weight = {{underscore extrinsic}} * constants::WEIGHT_PER_NANOS; + } + + #[cfg(test)] + mod test_base_weights { + use frame_support::weights::constants; + + /// Checks that the block weight exist and is sane. + // NOTE: If this test fails but you are sure that the generated values are fine, + // you can delete it. + #[test] + fn sane_block() { + use super::constants::BlockExecutionWeight as W; + + // At least 100 µs. + assert!( + W::get() >= 100 * constants::WEIGHT_PER_MICROS, + "Weight should be at least 100 µs." + ); + // At most 50 ms. + assert!( + W::get() <= 10 * constants::WEIGHT_PER_MILLIS, + "Weight should be at most 10 ms." + ); + } + + /// Checks that the extrinsic weight exist and is sane. + // NOTE: If this test fails but you are sure that the generated values are fine, + // you can delete it. + #[test] + fn sane_extrinsic() { + use super::constants::ExtrinsicBaseWeight as W; + + // At least 10 µs. + assert!( + W::get() >= 10 * constants::WEIGHT_PER_MICROS, + "Weight should be at least 100 µs." + ); + // At most 1 ms. + assert!( + W::get() <= constants::WEIGHT_PER_MILLIS, + "Weight should be at most 10 ms." + ); + } + } +} diff --git a/utils/frame/benchmarking-cli/src/lib.rs b/utils/frame/benchmarking-cli/src/lib.rs index 56aab0321ccd0..7de6929b2eafb 100644 --- a/utils/frame/benchmarking-cli/src/lib.rs +++ b/utils/frame/benchmarking-cli/src/lib.rs @@ -15,6 +15,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +pub mod block; mod command; mod storage; mod writer; @@ -22,6 +23,7 @@ mod writer; use sc_cli::{ExecutionStrategy, WasmExecutionMethod}; use std::{fmt::Debug, path::PathBuf}; +pub use block::BlockCmd; pub use storage::StorageCmd; // Add a more relaxed parsing for pallet names by allowing pallet directory names with `-` to be diff --git a/utils/frame/benchmarking-cli/src/storage/record.rs b/utils/frame/benchmarking-cli/src/storage/record.rs index 00a613c713007..3bb1ec21965fb 100644 --- a/utils/frame/benchmarking-cli/src/storage/record.rs +++ b/utils/frame/benchmarking-cli/src/storage/record.rs @@ -154,7 +154,7 @@ impl Stats { fn percentile(mut xs: Vec, p: f64) -> u64 { xs.sort(); let index = (xs.len() as f64 * p).ceil() as usize; - xs[index] + xs[index-1] } } diff --git a/utils/frame/benchmarking-cli/src/storage/write.rs b/utils/frame/benchmarking-cli/src/storage/write.rs index eb9ba11f30696..49ebbcc2349a9 100644 --- a/utils/frame/benchmarking-cli/src/storage/write.rs +++ b/utils/frame/benchmarking-cli/src/storage/write.rs @@ -51,6 +51,7 @@ impl StorageCmd { let mut record = BenchRecord::default(); let supports_rc = db.supports_ref_counting(); + // TODO use HeaderBackend.info() let block = BlockId::Number(client.usage_info().chain.best_number); let header = client.header(block)?.ok_or("Header not found")?; let original_root = *header.state_root();