diff --git a/Cargo.lock b/Cargo.lock index 82dea83813d0b..806b683f81743 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2139,6 +2139,7 @@ dependencies = [ "memory-db", "parity-scale-codec", "rand 0.8.4", + "sc-block-builder", "sc-cli", "sc-client-api", "sc-client-db", @@ -2152,6 +2153,7 @@ dependencies = [ "sp-core", "sp-database", "sp-externalities", + "sp-inherents", "sp-keystore", "sp-runtime", "sp-state-machine", diff --git a/bin/node/cli/src/cli.rs b/bin/node/cli/src/cli.rs index 386215854b963..a911cc26ef87c 100644 --- a/bin/node/cli/src/cli.rs +++ b/bin/node/cli/src/cli.rs @@ -42,6 +42,13 @@ pub enum Subcommand { #[clap(name = "benchmark", about = "Benchmark runtime pallets.")] Benchmark(frame_benchmarking_cli::BenchmarkCmd), + /// Sub command for benchmarking the per-block and per-extrinsic execution overhead. + #[clap( + name = "benchmark-overhead", + about = "Benchmark the per-block and per-extrinsic execution overhead." + )] + BenchmarkOverhead(frame_benchmarking_cli::OverheadCmd), + /// Sub command for benchmarking the storage speed. #[clap(name = "benchmark-storage", about = "Benchmark storage speed.")] BenchmarkStorage(frame_benchmarking_cli::StorageCmd), diff --git a/bin/node/cli/src/command.rs b/bin/node/cli/src/command.rs index cc6480bb90d55..e208e324ee2aa 100644 --- a/bin/node/cli/src/command.rs +++ b/bin/node/cli/src/command.rs @@ -18,10 +18,13 @@ use crate::{chain_spec, service, service::new_partial, Cli, Subcommand}; use node_executor::ExecutorDispatch; -use node_runtime::{Block, RuntimeApi}; +use node_primitives::Block; +use node_runtime::RuntimeApi; use sc_cli::{ChainSpec, Result, RuntimeVersion, SubstrateCli}; use sc_service::PartialComponents; +use std::sync::Arc; + impl SubstrateCli for Cli { fn impl_name() -> String { "Substrate Node".into() @@ -95,13 +98,21 @@ pub fn run() -> Result<()> { You can enable it with `--features runtime-benchmarks`." .into()) }, - Some(Subcommand::BenchmarkStorage(cmd)) => { - if !cfg!(feature = "runtime-benchmarks") { - return Err("Benchmarking wasn't enabled when building the node. \ - You can enable it with `--features runtime-benchmarks`." - .into()) - } + Some(Subcommand::BenchmarkOverhead(cmd)) => { + let runner = cli.create_runner(cmd)?; + runner.async_run(|mut config| { + use super::command_helper::{inherent_data, ExtrinsicBuilder}; + // We don't use the authority role since that would start producing blocks + // in the background which would mess with our benchmark. + config.role = sc_service::Role::Full; + let PartialComponents { client, task_manager, .. } = new_partial(&config)?; + let ext_builder = ExtrinsicBuilder::new(client.clone()); + + Ok((cmd.run(config, client, inherent_data()?, Arc::new(ext_builder)), task_manager)) + }) + }, + Some(Subcommand::BenchmarkStorage(cmd)) => { let runner = cli.create_runner(cmd)?; runner.async_run(|config| { let PartialComponents { client, task_manager, backend, .. } = new_partial(&config)?; diff --git a/bin/node/cli/src/command_helper.rs b/bin/node/cli/src/command_helper.rs new file mode 100644 index 0000000000000..51fe7a5c5a7bf --- /dev/null +++ b/bin/node/cli/src/command_helper.rs @@ -0,0 +1,69 @@ +// This file is part of Substrate. + +// Copyright (C) 2022 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +//! Contains code to setup the command invocations in [`super::command`] which would +//! otherwise bloat that module. + +use crate::service::{create_extrinsic, FullClient}; + +use node_runtime::SystemCall; +use sc_cli::Result; +use sp_inherents::{InherentData, InherentDataProvider}; +use sp_keyring::Sr25519Keyring; +use sp_runtime::OpaqueExtrinsic; + +use std::{sync::Arc, time::Duration}; + +/// Generates extrinsics for the `benchmark-overhead` command. +pub struct ExtrinsicBuilder { + client: Arc, +} + +impl ExtrinsicBuilder { + /// Creates a new [`Self`] from the given client. + pub fn new(client: Arc) -> Self { + Self { client } + } +} + +impl frame_benchmarking_cli::ExtrinsicBuilder for ExtrinsicBuilder { + fn remark(&self, nonce: u32) -> std::result::Result { + let acc = Sr25519Keyring::Bob.pair(); + let extrinsic: OpaqueExtrinsic = create_extrinsic( + self.client.as_ref(), + acc, + SystemCall::remark { remark: vec![] }, + Some(nonce), + ) + .into(); + + Ok(extrinsic) + } +} + +/// Generates inherent data for the `benchmark-overhead` command. +pub fn inherent_data() -> Result { + let mut inherent_data = InherentData::new(); + let d = Duration::from_millis(0); + let timestamp = sp_timestamp::InherentDataProvider::new(d.into()); + + timestamp + .provide_inherent_data(&mut inherent_data) + .map_err(|e| format!("creating inherent data: {:?}", e))?; + Ok(inherent_data) +} diff --git a/bin/node/cli/src/lib.rs b/bin/node/cli/src/lib.rs index 791140a25484d..06c0bcccbc296 100644 --- a/bin/node/cli/src/lib.rs +++ b/bin/node/cli/src/lib.rs @@ -38,6 +38,8 @@ pub mod service; mod cli; #[cfg(feature = "cli")] mod command; +#[cfg(feature = "cli")] +mod command_helper; #[cfg(feature = "cli")] pub use cli::*; diff --git a/bin/node/cli/tests/benchmark_overhead_works.rs b/bin/node/cli/tests/benchmark_overhead_works.rs new file mode 100644 index 0000000000000..550221ee2f70f --- /dev/null +++ b/bin/node/cli/tests/benchmark_overhead_works.rs @@ -0,0 +1,46 @@ +// This file is part of Substrate. + +// Copyright (C) 2022 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +use assert_cmd::cargo::cargo_bin; +use std::process::Command; +use tempfile::tempdir; + +/// Tests that the `benchmark-overhead` command works for the substrate dev runtime. +#[test] +fn benchmark_overhead_works() { + let tmp_dir = tempdir().expect("could not create a temp dir"); + let base_path = tmp_dir.path(); + + // Only put 10 extrinsics into the block otherwise it takes forever to build it + // especially for a non-release build. + let status = Command::new(cargo_bin("substrate")) + .args(&["benchmark-overhead", "--dev", "-d"]) + .arg(base_path) + .arg("--weight-path") + .arg(base_path) + .args(["--warmup", "10", "--repeat", "10"]) + .args(["--add", "100", "--mul", "1.2", "--metric", "p75"]) + .args(["--max-ext-per-block", "10"]) + .status() + .unwrap(); + assert!(status.success()); + + // Weight files have been created. + assert!(base_path.join("block_weights.rs").exists()); + assert!(base_path.join("extrinsic_weights.rs").exists()); +} diff --git a/utils/frame/benchmarking-cli/Cargo.toml b/utils/frame/benchmarking-cli/Cargo.toml index 81e7396db3e68..5575bb833ca77 100644 --- a/utils/frame/benchmarking-cli/Cargo.toml +++ b/utils/frame/benchmarking-cli/Cargo.toml @@ -16,6 +16,7 @@ targets = ["x86_64-unknown-linux-gnu"] frame-benchmarking = { version = "4.0.0-dev", path = "../../../frame/benchmarking" } frame-support = { version = "4.0.0-dev", path = "../../../frame/support" } sp-core = { version = "6.0.0", path = "../../../primitives/core" } +sc-block-builder = { version = "0.10.0-dev", path = "../../../client/block-builder" } sc-service = { version = "0.10.0-dev", default-features = false, path = "../../../client/service" } sc-client-api = { version = "4.0.0-dev", path = "../../../client/api" } sc-cli = { version = "0.10.0-dev", path = "../../../client/cli" } @@ -26,6 +27,7 @@ sp-api = { version = "4.0.0-dev", path = "../../../primitives/api" } sp-externalities = { version = "0.12.0", path = "../../../primitives/externalities" } sp-database = { version = "4.0.0-dev", path = "../../../primitives/database" } sp-blockchain = { version = "4.0.0-dev", path = "../../../primitives/blockchain" } +sp-inherents = { version = "4.0.0-dev", path = "../../../primitives/inherents" } sp-keystore = { version = "0.12.0", path = "../../../primitives/keystore" } sp-storage = { version = "6.0.0", path = "../../../primitives/storage" } sp-runtime = { version = "6.0.0", path = "../../../primitives/runtime" } diff --git a/utils/frame/benchmarking-cli/src/lib.rs b/utils/frame/benchmarking-cli/src/lib.rs index 9815fe88a7f02..e06d57963dad3 100644 --- a/utils/frame/benchmarking-cli/src/lib.rs +++ b/utils/frame/benchmarking-cli/src/lib.rs @@ -16,12 +16,15 @@ // limitations under the License. mod command; +pub mod overhead; +mod post_processing; mod storage; mod writer; use sc_cli::{ExecutionStrategy, WasmExecutionMethod, DEFAULT_WASM_EXECUTION_METHOD}; use std::{fmt::Debug, path::PathBuf}; +pub use overhead::{ExtrinsicBuilder, OverheadCmd}; pub use storage::StorageCmd; // Add a more relaxed parsing for pallet names by allowing pallet directory names with `-` to be diff --git a/utils/frame/benchmarking-cli/src/overhead/bench.rs b/utils/frame/benchmarking-cli/src/overhead/bench.rs new file mode 100644 index 0000000000000..3e18c6a86db24 --- /dev/null +++ b/utils/frame/benchmarking-cli/src/overhead/bench.rs @@ -0,0 +1,210 @@ +// This file is part of Substrate. + +// Copyright (C) 2022 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Contains the core benchmarking logic. + +use sc_block_builder::{BlockBuilderApi, BlockBuilderProvider}; +use sc_cli::{Error, Result}; +use sc_client_api::Backend as ClientBackend; +use sp_api::{ApiExt, BlockId, Core, ProvideRuntimeApi}; +use sp_blockchain::{ + ApplyExtrinsicFailed::Validity, + Error::{ApplyExtrinsicFailed, RuntimeApiError}, +}; +use sp_runtime::{ + traits::{Block as BlockT, Zero}, + transaction_validity::{InvalidTransaction, TransactionValidityError}, + OpaqueExtrinsic, +}; + +use clap::Args; +use log::info; +use serde::Serialize; +use std::{marker::PhantomData, sync::Arc, time::Instant}; + +use crate::{overhead::cmd::ExtrinsicBuilder, storage::record::Stats}; + +/// Parameters to configure an *overhead* benchmark. +#[derive(Debug, Default, Serialize, Clone, PartialEq, Args)] +pub struct BenchmarkParams { + /// Rounds of warmups before measuring. + #[clap(long, default_value = "100")] + pub warmup: u32, + + /// How many times the benchmark should be repeated. + #[clap(long, default_value = "1000")] + pub repeat: u32, + + /// Maximal number of extrinsics that should be put into a block. + /// + /// Only useful for debugging. + #[clap(long)] + pub max_ext_per_block: Option, +} + +/// The results of multiple runs in nano seconds. +pub(crate) type BenchRecord = Vec; + +/// Type of a benchmark. +#[derive(Serialize, Clone, PartialEq, Copy)] +pub(crate) enum BenchmarkType { + /// Measure the per-extrinsic execution overhead. + Extrinsic, + /// Measure the per-block execution overhead. + Block, +} + +/// Holds all objects needed to run the *overhead* benchmarks. +pub(crate) struct Benchmark { + client: Arc, + params: BenchmarkParams, + inherent_data: sp_inherents::InherentData, + ext_builder: Arc, + _p: PhantomData<(Block, BA)>, +} + +impl Benchmark +where + Block: BlockT, + BA: ClientBackend, + C: BlockBuilderProvider + ProvideRuntimeApi, + C::Api: ApiExt + BlockBuilderApi, +{ + /// Create a new [`Self`] from the arguments. + pub fn new( + client: Arc, + params: BenchmarkParams, + inherent_data: sp_inherents::InherentData, + ext_builder: Arc, + ) -> Self { + Self { client, params, inherent_data, ext_builder, _p: PhantomData } + } + + /// Run the specified benchmark. + pub fn bench(&self, bench_type: BenchmarkType) -> Result { + let (block, num_ext) = self.build_block(bench_type)?; + let record = self.measure_block(&block, num_ext, bench_type)?; + Stats::new(&record) + } + + /// Builds a block for the given benchmark type. + /// + /// Returns the block and the number of extrinsics in the block + /// that are not inherents. + fn build_block(&self, bench_type: BenchmarkType) -> Result<(Block, u64)> { + let mut builder = self.client.new_block(Default::default())?; + // Create and insert the inherents. + let inherents = builder.create_inherents(self.inherent_data.clone())?; + for inherent in inherents { + builder.push(inherent)?; + } + + // Return early if we just want a block with inherents and no additional extrinsics. + if bench_type == BenchmarkType::Block { + return Ok((builder.build()?.block, 0)) + } + + // Put as many extrinsics into the block as possible and count them. + info!("Building block, this takes some time..."); + let mut num_ext = 0; + for nonce in 0..self.max_ext_per_block() { + let ext = self.ext_builder.remark(nonce)?; + match builder.push(ext.clone()) { + Ok(()) => {}, + Err(ApplyExtrinsicFailed(Validity(TransactionValidityError::Invalid( + InvalidTransaction::ExhaustsResources, + )))) => break, // Block is full + Err(e) => return Err(Error::Client(e)), + } + num_ext += 1; + } + if num_ext == 0 { + return Err("A Block must hold at least one extrinsic".into()) + } + info!("Extrinsics per block: {}", num_ext); + let block = builder.build()?.block; + + Ok((block, num_ext)) + } + + /// Measures the time that it take to execute a block or an extrinsic. + fn measure_block( + &self, + block: &Block, + num_ext: u64, + bench_type: BenchmarkType, + ) -> Result { + let mut record = BenchRecord::new(); + if bench_type == BenchmarkType::Extrinsic && num_ext == 0 { + return Err("Cannot measure the extrinsic time of an empty block".into()) + } + let genesis = BlockId::Number(Zero::zero()); + + info!("Running {} warmups...", self.params.warmup); + for _ in 0..self.params.warmup { + self.client + .runtime_api() + .execute_block(&genesis, block.clone()) + .map_err(|e| Error::Client(RuntimeApiError(e)))?; + } + + info!("Executing block {} times", self.params.repeat); + // Interesting part here: + // Execute a block multiple times and record each execution time. + for _ in 0..self.params.repeat { + let block = block.clone(); + let runtime_api = self.client.runtime_api(); + let start = Instant::now(); + + runtime_api + .execute_block(&genesis, block) + .map_err(|e| Error::Client(RuntimeApiError(e)))?; + + let elapsed = start.elapsed().as_nanos(); + if bench_type == BenchmarkType::Extrinsic { + // Checked for non-zero div above. + record.push((elapsed as f64 / num_ext as f64).ceil() as u64); + } else { + record.push(elapsed as u64); + } + } + + Ok(record) + } + + fn max_ext_per_block(&self) -> u32 { + self.params.max_ext_per_block.unwrap_or(u32::MAX) + } +} + +impl BenchmarkType { + /// Short name of the benchmark type. + pub(crate) fn short_name(&self) -> &'static str { + match self { + Self::Extrinsic => "extrinsic", + Self::Block => "block", + } + } + + /// Long name of the benchmark type. + pub(crate) fn long_name(&self) -> &'static str { + match self { + Self::Extrinsic => "ExtrinsicBase", + Self::Block => "BlockExecution", + } + } +} diff --git a/utils/frame/benchmarking-cli/src/overhead/cmd.rs b/utils/frame/benchmarking-cli/src/overhead/cmd.rs new file mode 100644 index 0000000000000..8c75627fe2462 --- /dev/null +++ b/utils/frame/benchmarking-cli/src/overhead/cmd.rs @@ -0,0 +1,118 @@ +// This file is part of Substrate. + +// Copyright (C) 2022 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Contains the [`OverheadCmd`] as entry point for the CLI to execute +//! the *overhead* benchmarks. + +use sc_block_builder::{BlockBuilderApi, BlockBuilderProvider}; +use sc_cli::{CliConfiguration, Result, SharedParams}; +use sc_client_api::Backend as ClientBackend; +use sc_service::Configuration; +use sp_api::{ApiExt, ProvideRuntimeApi}; +use sp_runtime::{traits::Block as BlockT, OpaqueExtrinsic}; + +use clap::{Args, Parser}; +use log::info; +use serde::Serialize; +use std::{fmt::Debug, sync::Arc}; + +use crate::{ + overhead::{ + bench::{Benchmark, BenchmarkParams, BenchmarkType}, + template::TemplateData, + }, + post_processing::WeightParams, +}; + +/// Benchmarks the per-block and per-extrinsic execution overhead. +#[derive(Debug, Parser)] +pub struct OverheadCmd { + #[allow(missing_docs)] + #[clap(flatten)] + pub shared_params: SharedParams, + + #[allow(missing_docs)] + #[clap(flatten)] + pub params: OverheadParams, +} + +/// Configures the benchmark, the post-processing and weight generation. +#[derive(Debug, Default, Serialize, Clone, PartialEq, Args)] +pub struct OverheadParams { + #[allow(missing_docs)] + #[clap(flatten)] + pub weight: WeightParams, + + #[allow(missing_docs)] + #[clap(flatten)] + pub bench: BenchmarkParams, +} + +/// Used by the benchmark to build signed extrinsics. +/// +/// The built extrinsics only need to be valid in the first block +/// who's parent block is the genesis block. +pub trait ExtrinsicBuilder { + /// Build a `System::remark` extrinsic. + fn remark(&self, nonce: u32) -> std::result::Result; +} + +impl OverheadCmd { + /// Measures the per-block and per-extrinsic execution overhead. + /// + /// Writes the results to console and into two instances of the + /// `weights.hbs` template, one for each benchmark. + pub async fn run( + &self, + cfg: Configuration, + client: Arc, + inherent_data: sp_inherents::InherentData, + ext_builder: Arc, + ) -> Result<()> + where + Block: BlockT, + BA: ClientBackend, + C: BlockBuilderProvider + ProvideRuntimeApi, + C::Api: ApiExt + BlockBuilderApi, + { + let bench = Benchmark::new(client, self.params.bench.clone(), inherent_data, ext_builder); + + // per-block execution overhead + { + let stats = bench.bench(BenchmarkType::Block)?; + info!("Per-block execution overhead [ns]:\n{:?}", stats); + let template = TemplateData::new(BenchmarkType::Block, &cfg, &self.params, &stats)?; + template.write(&self.params.weight.weight_path)?; + } + // per-extrinsic execution overhead + { + let stats = bench.bench(BenchmarkType::Extrinsic)?; + info!("Per-extrinsic execution overhead [ns]:\n{:?}", stats); + let template = TemplateData::new(BenchmarkType::Extrinsic, &cfg, &self.params, &stats)?; + template.write(&self.params.weight.weight_path)?; + } + + Ok(()) + } +} + +// Boilerplate +impl CliConfiguration for OverheadCmd { + fn shared_params(&self) -> &SharedParams { + &self.shared_params + } +} diff --git a/utils/frame/benchmarking-cli/src/overhead/mod.rs b/utils/frame/benchmarking-cli/src/overhead/mod.rs new file mode 100644 index 0000000000000..abdeac22b7898 --- /dev/null +++ b/utils/frame/benchmarking-cli/src/overhead/mod.rs @@ -0,0 +1,22 @@ +// This file is part of Substrate. + +// Copyright (C) 2022 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +mod bench; +pub mod cmd; +mod template; + +pub use cmd::{ExtrinsicBuilder, OverheadCmd}; diff --git a/utils/frame/benchmarking-cli/src/overhead/template.rs b/utils/frame/benchmarking-cli/src/overhead/template.rs new file mode 100644 index 0000000000000..f6fb8ed9d929e --- /dev/null +++ b/utils/frame/benchmarking-cli/src/overhead/template.rs @@ -0,0 +1,110 @@ +// This file is part of Substrate. + +// Copyright (C) 2022 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Converts a benchmark result into [`TemplateData`] and writes +//! it into the `weights.hbs` template. + +use sc_cli::Result; +use sc_service::Configuration; + +use handlebars::Handlebars; +use log::info; +use serde::Serialize; +use std::{env, fs, path::PathBuf}; + +use crate::{ + overhead::{bench::BenchmarkType, cmd::OverheadParams}, + storage::record::Stats, +}; + +static VERSION: &'static str = env!("CARGO_PKG_VERSION"); +static TEMPLATE: &str = include_str!("./weights.hbs"); + +/// Data consumed by Handlebar to fill out the `weights.hbs` template. +#[derive(Serialize, Debug, Clone)] +pub(crate) struct TemplateData { + /// Short name of the benchmark. Can be "block" or "extrinsic". + long_name: String, + /// Long name of the benchmark. Can be "BlockExecution" or "ExtrinsicBase". + short_name: String, + /// Name of the runtime. Taken from the chain spec. + runtime_name: String, + /// Version of the benchmarking CLI used. + version: String, + /// Date that the template was filled out. + date: String, + /// Command line arguments that were passed to the CLI. + args: Vec, + /// Params of the executed command. + params: OverheadParams, + /// Stats about the benchmark result. + stats: Stats, + /// The resulting weight in ns. + weight: u64, +} + +impl TemplateData { + /// Returns a new [`Self`] from the given params. + pub(crate) fn new( + t: BenchmarkType, + cfg: &Configuration, + params: &OverheadParams, + stats: &Stats, + ) -> Result { + let weight = params.weight.calc_weight(stats)?; + + Ok(TemplateData { + short_name: t.short_name().into(), + long_name: t.long_name().into(), + runtime_name: cfg.chain_spec.name().into(), + version: VERSION.into(), + date: chrono::Utc::now().format("%Y-%m-%d (Y/M/D)").to_string(), + args: env::args().collect::>(), + params: params.clone(), + stats: stats.clone(), + weight, + }) + } + + /// Fill out the `weights.hbs` HBS template with its own data. + /// Writes the result to `path` which can be a directory or a file. + pub fn write(&self, path: &Option) -> Result<()> { + let mut handlebars = Handlebars::new(); + // Format large integers with underscores. + handlebars.register_helper("underscore", Box::new(crate::writer::UnderscoreHelper)); + // Don't HTML escape any characters. + handlebars.register_escape_fn(|s| -> String { s.to_string() }); + + let out_path = self.build_path(path)?; + let mut fd = fs::File::create(&out_path)?; + info!("Writing weights to {:?}", fs::canonicalize(&out_path)?); + handlebars + .render_template_to_write(&TEMPLATE, &self, &mut fd) + .map_err(|e| format!("HBS template write: {:?}", e).into()) + } + + /// Build a path for the weight file. + fn build_path(&self, weight_out: &Option) -> Result { + let mut path = weight_out.clone().unwrap_or(PathBuf::from(".")); + + if !path.is_dir() { + return Err("Need directory as --weight-path".into()) + } + path.push(format!("{}_weights.rs", self.short_name)); + Ok(path) + } +} diff --git a/utils/frame/benchmarking-cli/src/overhead/weights.hbs b/utils/frame/benchmarking-cli/src/overhead/weights.hbs new file mode 100644 index 0000000000000..0f6b7f3e9119f --- /dev/null +++ b/utils/frame/benchmarking-cli/src/overhead/weights.hbs @@ -0,0 +1,92 @@ +// This file is part of Substrate. + +// Copyright (C) 2022 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! THIS FILE WAS AUTO-GENERATED USING THE SUBSTRATE BENCHMARK CLI VERSION {{version}} +//! DATE: {{date}} +//! +//! SHORT-NAME: `{{short_name}}`, LONG-NAME: `{{long_name}}`, RUNTIME: `{{runtime_name}}` +//! WARMUPS: `{{params.bench.warmup}}`, REPEAT: `{{params.bench.repeat}}` +//! WEIGHT-PATH: `{{params.weight.weight_path}}` +//! WEIGHT-METRIC: `{{params.weight.weight_metric}}`, WEIGHT-MUL: `{{params.weight.weight_mul}}`, WEIGHT-ADD: `{{params.weight.weight_add}}` + +// Executed Command: +{{#each args as |arg|}} +// {{arg}} +{{/each}} + +use frame_support::{ + parameter_types, + weights::{constants::WEIGHT_PER_NANOS, Weight}, +}; + +parameter_types! { + {{#if (eq short_name "block")}} + /// Time to execute an empty block. + {{else}} + /// Time to execute a NO-OP extrinsic eg. `System::remark`. + {{/if}} + /// Calculated by multiplying the *{{params.weight.weight_metric}}* with `{{params.weight.weight_mul}}` and adding `{{params.weight.weight_add}}`. + /// + /// Stats [ns]: + /// Min, Max: {{underscore stats.min}}, {{underscore stats.max}} + /// Average: {{underscore stats.avg}} + /// Median: {{underscore stats.median}} + /// StdDev: {{stats.stddev}} + /// + /// Percentiles [ns]: + /// 99th: {{underscore stats.p99}} + /// 95th: {{underscore stats.p95}} + /// 75th: {{underscore stats.p75}} + pub const {{long_name}}Weight: Weight = {{underscore weight}} * WEIGHT_PER_NANOS; +} + +#[cfg(test)] +mod test_weights { + use frame_support::weights::constants; + + /// Checks that the weight exists and is sane. + // NOTE: If this test fails but you are sure that the generated values are fine, + // you can delete it. + #[test] + fn sane() { + let w = super::{{long_name}}Weight::get(); + + {{#if (eq short_name "block")}} + // At least 100 µs. + assert!( + w >= 100 * constants::WEIGHT_PER_MICROS, + "Weight should be at least 100 µs." + ); + // At most 50 ms. + assert!( + w <= 50 * constants::WEIGHT_PER_MILLIS, + "Weight should be at most 50 ms." + ); + {{else}} + // At least 10 µs. + assert!( + w >= 10 * constants::WEIGHT_PER_MICROS, + "Weight should be at least 10 µs." + ); + // At most 1 ms. + assert!( + w <= constants::WEIGHT_PER_MILLIS, + "Weight should be at most 1 ms." + ); + {{/if}} + } +} diff --git a/utils/frame/benchmarking-cli/src/post_processing/mod.rs b/utils/frame/benchmarking-cli/src/post_processing/mod.rs new file mode 100644 index 0000000000000..fb20d9bd0c488 --- /dev/null +++ b/utils/frame/benchmarking-cli/src/post_processing/mod.rs @@ -0,0 +1,95 @@ +// This file is part of Substrate. + +// Copyright (C) 2022 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Calculates a weight from the statistics of a benchmark result. + +use sc_cli::Result; + +use clap::Args; +use serde::Serialize; +use std::path::PathBuf; + +use crate::storage::record::{StatSelect, Stats}; + +/// Configures the weight generation. +#[derive(Debug, Default, Serialize, Clone, PartialEq, Args)] +pub struct WeightParams { + /// File or directory to write the *weight* files to. + /// + /// For Substrate this should be `frame/support/src/weights`. + #[clap(long)] + pub weight_path: Option, + + /// Select a specific metric to calculate the final weight output. + #[clap(long = "metric", default_value = "average")] + pub weight_metric: StatSelect, + + /// Multiply the resulting weight with the given factor. Must be positive. + /// + /// Is applied before `weight_add`. + #[clap(long = "mul", default_value = "1")] + pub weight_mul: f64, + + /// Add the given offset to the resulting weight. + /// + /// Is applied after `weight_mul`. + #[clap(long = "add", default_value = "0")] + pub weight_add: u64, +} + +/// Calculates the final weight by multiplying the selected metric with +/// `weight_mul` and adding `weight_add`. +/// Does not use safe casts and can overflow. +impl WeightParams { + pub(crate) fn calc_weight(&self, stat: &Stats) -> Result { + if self.weight_mul.is_sign_negative() || !self.weight_mul.is_normal() { + return Err("invalid floating number for `weight_mul`".into()) + } + let s = stat.select(self.weight_metric) as f64; + let w = s.mul_add(self.weight_mul, self.weight_add as f64).ceil(); + Ok(w as u64) // No safe cast here since there is no `From` for `u64`. + } +} + +#[cfg(test)] +mod test_weight_params { + use super::WeightParams; + use crate::storage::record::{StatSelect, Stats}; + + #[test] + fn calc_weight_works() { + let stats = Stats { avg: 113, ..Default::default() }; + let params = WeightParams { + weight_metric: StatSelect::Average, + weight_mul: 0.75, + weight_add: 3, + ..Default::default() + }; + + let want = (113.0f64 * 0.75 + 3.0).ceil() as u64; // Ceil for overestimation. + let got = params.calc_weight(&stats).unwrap(); + assert_eq!(want, got); + } + + #[test] + fn calc_weight_detects_negative_mul() { + let stats = Stats::default(); + let params = WeightParams { weight_mul: -0.75, ..Default::default() }; + + assert!(params.calc_weight(&stats).is_err()); + } +} diff --git a/utils/frame/benchmarking-cli/src/storage/cmd.rs b/utils/frame/benchmarking-cli/src/storage/cmd.rs index ad7d13a2022e4..c38e6636e5a3e 100644 --- a/utils/frame/benchmarking-cli/src/storage/cmd.rs +++ b/utils/frame/benchmarking-cli/src/storage/cmd.rs @@ -33,8 +33,8 @@ use serde::Serialize; use sp_runtime::generic::BlockId; use std::{fmt::Debug, path::PathBuf, sync::Arc}; -use super::{record::StatSelect, template::TemplateData}; - +use super::template::TemplateData; +use crate::post_processing::WeightParams; /// Benchmark the storage of a Substrate node with a live chain snapshot. #[derive(Debug, Parser)] pub struct StorageCmd { @@ -58,24 +58,9 @@ pub struct StorageCmd { /// Parameters for modifying the benchmark behaviour and the post processing of the results. #[derive(Debug, Default, Serialize, Clone, PartialEq, Args)] pub struct StorageParams { - /// Path to write the *weight* file to. Can be a file or directory. - /// For substrate this should be `frame/support/src/weights`. - #[clap(long)] - pub weight_path: Option, - - /// Select a specific metric to calculate the final weight output. - #[clap(long = "metric", default_value = "average")] - pub weight_metric: StatSelect, - - /// Multiply the resulting weight with the given factor. Must be positive. - /// Is calculated before `weight_add`. - #[clap(long = "mul", default_value = "1")] - pub weight_mul: f64, - - /// Add the given offset to the resulting weight. - /// Is calculated after `weight_mul`. - #[clap(long = "add", default_value = "0")] - pub weight_add: u64, + #[allow(missing_docs)] + #[clap(flatten)] + pub weight_params: WeightParams, /// Skip the `read` benchmark. #[clap(long)] @@ -153,7 +138,7 @@ impl StorageCmd { template.set_stats(None, Some(stats))?; } - template.write(&self.params.weight_path, &self.params.template_path) + template.write(&self.params.weight_params.weight_path, &self.params.template_path) } /// Returns the specified state version. diff --git a/utils/frame/benchmarking-cli/src/storage/record.rs b/utils/frame/benchmarking-cli/src/storage/record.rs index 667274bef0dd5..530fa4cdfe965 100644 --- a/utils/frame/benchmarking-cli/src/storage/record.rs +++ b/utils/frame/benchmarking-cli/src/storage/record.rs @@ -36,25 +36,25 @@ pub(crate) struct BenchRecord { #[derive(Serialize, Default, Clone)] pub(crate) struct Stats { /// Sum of all values. - sum: u64, + pub(crate) sum: u64, /// Minimal observed value. - min: u64, + pub(crate) min: u64, /// Maximal observed value. - max: u64, + pub(crate) max: u64, /// Average of all values. - avg: u64, + pub(crate) avg: u64, /// Median of all values. - median: u64, + pub(crate) median: u64, /// Standard derivation of all values. - stddev: f64, + pub(crate) stddev: f64, /// 99th percentile. At least 99% of all values are below this threshold. - p99: u64, + pub(crate) p99: u64, /// 95th percentile. At least 95% of all values are below this threshold. - p95: u64, + pub(crate) p95: u64, /// 75th percentile. At least 75% of all values are below this threshold. - p75: u64, + pub(crate) p75: u64, } /// Selects a specific field from a [`Stats`] object. @@ -159,8 +159,8 @@ impl Stats { /// This is best effort since it ignores the interpolation case. fn percentile(mut xs: Vec, p: f64) -> u64 { xs.sort(); - let index = (xs.len() as f64 * p).ceil() as usize; - xs[index] + let index = (xs.len() as f64 * p).ceil() as usize - 1; + xs[index.clamp(0, xs.len() - 1)] } } @@ -195,3 +195,40 @@ impl FromStr for StatSelect { } } } + +#[cfg(test)] +mod test_stats { + use super::Stats; + use rand::{seq::SliceRandom, thread_rng}; + + #[test] + fn stats_correct() { + let mut data: Vec = (1..=100).collect(); + data.shuffle(&mut thread_rng()); + let stats = Stats::new(&data).unwrap(); + + assert_eq!(stats.sum, 5050); + assert_eq!(stats.min, 1); + assert_eq!(stats.max, 100); + + assert_eq!(stats.avg, 50); + assert_eq!(stats.median, 50); // 50.5 to be exact. + assert_eq!(stats.stddev, 28.87); // Rounded with 1/100 precision. + + assert_eq!(stats.p99, 99); + assert_eq!(stats.p95, 95); + assert_eq!(stats.p75, 75); + } + + #[test] + fn no_panic_short_lengths() { + // Empty input does error. + assert!(Stats::new(&vec![]).is_err()); + + // Different small input lengths are fine. + for l in 1..10 { + let data = (0..=l).collect(); + assert!(Stats::new(&data).is_ok()); + } + } +} diff --git a/utils/frame/benchmarking-cli/src/storage/template.rs b/utils/frame/benchmarking-cli/src/storage/template.rs index 13b825d891a51..10e6902b934bc 100644 --- a/utils/frame/benchmarking-cli/src/storage/template.rs +++ b/utils/frame/benchmarking-cli/src/storage/template.rs @@ -77,11 +77,11 @@ impl TemplateData { write: Option<(Stats, Stats)>, ) -> Result<()> { if let Some(read) = read { - self.read_weight = calc_weight(&read.0, &self.params)?; + self.read_weight = self.params.weight_params.calc_weight(&read.0)?; self.read = Some(read); } if let Some(write) = write { - self.write_weight = calc_weight(&write.0, &self.params)?; + self.write_weight = self.params.weight_params.calc_weight(&write.0)?; self.write = Some(write); } Ok(()) @@ -130,15 +130,3 @@ impl TemplateData { path } } - -/// Calculates the final weight by multiplying the selected metric with -/// `mul` and adding `add`. -/// Does not use safe casts and can overflow. -fn calc_weight(stat: &Stats, params: &StorageParams) -> Result { - if params.weight_mul.is_sign_negative() || !params.weight_mul.is_normal() { - return Err("invalid floating number for `weight_mul`".into()) - } - let s = stat.select(params.weight_metric) as f64; - let w = s.mul_add(params.weight_mul, params.weight_add as f64).ceil(); - Ok(w as u64) // No safe cast here since there is no `From` for `u64`. -}