diff --git a/Cargo.lock b/Cargo.lock index 58297393c834d..826ed90d4553b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2127,20 +2127,34 @@ dependencies = [ "frame-benchmarking", "frame-support", "handlebars", + "hash-db", + "hex", + "itertools", + "kvdb", "linked-hash-map", "log 0.4.14", + "memory-db", "parity-scale-codec", + "rand 0.8.4", "sc-cli", + "sc-client-api", "sc-client-db", "sc-executor", "sc-service", "serde", "serde_json", + "serde_nanos", + "sp-api", + "sp-blockchain", "sp-core", + "sp-database", "sp-externalities", "sp-keystore", "sp-runtime", "sp-state-machine", + "sp-std", + "sp-storage", + "sp-trie", ] [[package]] @@ -3151,9 +3165,9 @@ checksum = "47be2f14c678be2fdcab04ab1171db51b2762ce6f0a8ee87c8dd4a04ed216135" [[package]] name = "itertools" -version = "0.10.0" +version = "0.10.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37d572918e350e82412fe766d24b15e6682fb2ed2bbe018280caa810397cb319" +checksum = "a9a9d19fa1e79b6215ff29b9d6880b706147f16e9b1dbb1e4e5947b5b02bc5e3" dependencies = [ "either", ] @@ -7414,9 +7428,9 @@ dependencies = [ [[package]] name = "rand_distr" -version = "0.4.1" +version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "051b398806e42b9cd04ad9ec8f81e355d0a382c543ac6672c62f5a5b452ef142" +checksum = "32cb0b9bc82b0a0876c2dd994a7e7a2683d3e7390ca40e6886785ef0c7e3ee31" dependencies = [ "num-traits", "rand 0.8.4", @@ -9289,6 +9303,15 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_nanos" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e44969a61f5d316be20a42ff97816efb3b407a924d06824c3d8a49fa8450de0e" +dependencies = [ + "serde", +] + [[package]] name = "sha-1" version = "0.8.2" diff --git a/bin/node/cli/src/cli.rs b/bin/node/cli/src/cli.rs index 00393c52f8b68..386215854b963 100644 --- a/bin/node/cli/src/cli.rs +++ b/bin/node/cli/src/cli.rs @@ -42,6 +42,10 @@ pub enum Subcommand { #[clap(name = "benchmark", about = "Benchmark runtime pallets.")] Benchmark(frame_benchmarking_cli::BenchmarkCmd), + /// Sub command for benchmarking the storage speed. + #[clap(name = "benchmark-storage", about = "Benchmark storage speed.")] + BenchmarkStorage(frame_benchmarking_cli::StorageCmd), + /// Try some command against runtime state. #[cfg(feature = "try-runtime")] TryRuntime(try_runtime_cli::TryRuntimeCmd), diff --git a/bin/node/cli/src/command.rs b/bin/node/cli/src/command.rs index d9ba53785ba0c..cc6480bb90d55 100644 --- a/bin/node/cli/src/command.rs +++ b/bin/node/cli/src/command.rs @@ -95,6 +95,22 @@ pub fn run() -> Result<()> { You can enable it with `--features runtime-benchmarks`." .into()) }, + Some(Subcommand::BenchmarkStorage(cmd)) => { + if !cfg!(feature = "runtime-benchmarks") { + return Err("Benchmarking wasn't enabled when building the node. \ + You can enable it with `--features runtime-benchmarks`." + .into()) + } + + let runner = cli.create_runner(cmd)?; + runner.async_run(|config| { + let PartialComponents { client, task_manager, backend, .. } = new_partial(&config)?; + let db = backend.expose_db(); + let storage = backend.expose_storage(); + + Ok((cmd.run(config, client, db, storage), task_manager)) + }) + }, Some(Subcommand::Key(cmd)) => cmd.run(&cli), Some(Subcommand::Sign(cmd)) => cmd.run(), Some(Subcommand::Verify(cmd)) => cmd.run(), diff --git a/client/cli/src/arg_enums.rs b/client/cli/src/arg_enums.rs index 249e3c639e4ef..b3bcddd418467 100644 --- a/client/cli/src/arg_enums.rs +++ b/client/cli/src/arg_enums.rs @@ -185,7 +185,7 @@ impl Into for RpcMethods { } /// Database backend -#[derive(Debug, Clone, Copy)] +#[derive(Debug, Clone, PartialEq, Copy)] pub enum Database { /// Facebooks RocksDB RocksDb, diff --git a/client/cli/src/params/database_params.rs b/client/cli/src/params/database_params.rs index dd11c21f432b1..70c929ecf202f 100644 --- a/client/cli/src/params/database_params.rs +++ b/client/cli/src/params/database_params.rs @@ -21,7 +21,7 @@ use clap::Args; use sc_service::TransactionStorageMode; /// Parameters for block import. -#[derive(Debug, Clone, Args)] +#[derive(Debug, Clone, PartialEq, Args)] pub struct DatabaseParams { /// Select database backend to use. #[clap( diff --git a/client/cli/src/params/pruning_params.rs b/client/cli/src/params/pruning_params.rs index a8516ee1453ac..de9628ecf7ad9 100644 --- a/client/cli/src/params/pruning_params.rs +++ b/client/cli/src/params/pruning_params.rs @@ -21,7 +21,7 @@ use clap::Args; use sc_service::{KeepBlocks, PruningMode, Role}; /// Parameters to define the pruning mode -#[derive(Debug, Clone, Args)] +#[derive(Debug, Clone, PartialEq, Args)] pub struct PruningParams { /// Specify the state pruning mode, a number of blocks to keep or 'archive'. /// diff --git a/client/cli/src/params/shared_params.rs b/client/cli/src/params/shared_params.rs index a4f2271e12c48..3fd0fc78a7910 100644 --- a/client/cli/src/params/shared_params.rs +++ b/client/cli/src/params/shared_params.rs @@ -22,7 +22,7 @@ use sc_service::config::BasePath; use std::path::PathBuf; /// Shared parameters used by all `CoreParams`. -#[derive(Debug, Clone, Args)] +#[derive(Debug, Clone, PartialEq, Args)] pub struct SharedParams { /// Specify the chain specification. /// diff --git a/client/db/Cargo.toml b/client/db/Cargo.toml index 82607f56f7d1f..a9fa370a8ead2 100644 --- a/client/db/Cargo.toml +++ b/client/db/Cargo.toml @@ -45,5 +45,6 @@ tempfile = "3" [features] default = [] test-helpers = [] +runtime-benchmarks = [] with-kvdb-rocksdb = ["kvdb-rocksdb"] with-parity-db = ["parity-db"] diff --git a/client/db/src/lib.rs b/client/db/src/lib.rs index a2d0cad72845f..3fd3bf8d09042 100644 --- a/client/db/src/lib.rs +++ b/client/db/src/lib.rs @@ -106,7 +106,8 @@ const DEFAULT_CHILD_RATIO: (usize, usize) = (1, 10); pub type DbState = sp_state_machine::TrieBackend>>, HashFor>; -const DB_HASH_LEN: usize = 32; +/// Length of a [`DbHash`]. +pub const DB_HASH_LEN: usize = 32; /// Hash type that this backend uses for the database. pub type DbHash = sp_core::H256; @@ -1050,6 +1051,23 @@ impl Backend { Self::new(db_setting, canonicalization_delay).expect("failed to create test-db") } + /// Expose the Database that is used by this backend. + /// The second argument is the Column that stores the State. + /// + /// Should only be needed for benchmarking. + #[cfg(any(feature = "runtime-benchmarks"))] + pub fn expose_db(&self) -> (Arc>, sp_database::ColumnId) { + (self.storage.db.clone(), columns::STATE) + } + + /// Expose the Storage that is used by this backend. + /// + /// Should only be needed for benchmarking. + #[cfg(any(feature = "runtime-benchmarks"))] + pub fn expose_storage(&self) -> Arc>> { + self.storage.clone() + } + fn from_database( db: Arc>, canonicalization_delay: u64, diff --git a/utils/frame/benchmarking-cli/Cargo.toml b/utils/frame/benchmarking-cli/Cargo.toml index bbfcbb7e8b818..dc6c5eb221652 100644 --- a/utils/frame/benchmarking-cli/Cargo.toml +++ b/utils/frame/benchmarking-cli/Cargo.toml @@ -17,13 +17,21 @@ frame-benchmarking = { version = "4.0.0-dev", path = "../../../frame/benchmarkin frame-support = { version = "4.0.0-dev", path = "../../../frame/support" } sp-core = { version = "5.0.0", path = "../../../primitives/core" } sc-service = { version = "0.10.0-dev", default-features = false, path = "../../../client/service" } +sc-client-api = { version = "4.0.0-dev", path = "../../../client/api" } sc-cli = { version = "0.10.0-dev", path = "../../../client/cli" } sc-client-db = { version = "0.10.0-dev", path = "../../../client/db" } sc-executor = { version = "0.10.0-dev", path = "../../../client/executor" } + +sp-api = { version = "4.0.0-dev", path = "../../../primitives/api" } sp-externalities = { version = "0.11.0", path = "../../../primitives/externalities" } +sp-database = { version = "4.0.0-dev", path = "../../../primitives/database" } +sp-blockchain = { version = "4.0.0-dev", path = "../../../primitives/blockchain" } sp-keystore = { version = "0.11.0", path = "../../../primitives/keystore" } +sp-storage = { version = "5.0.0", path = "../../../primitives/storage" } sp-runtime = { version = "5.0.0", path = "../../../primitives/runtime" } +sp-std = { version = "4.0.0", default-features = false, path = "../../../primitives/std" } sp-state-machine = { version = "0.11.0", path = "../../../primitives/state-machine" } +sp-trie = { version = "5.0.0", path = "../../../primitives/trie" } codec = { version = "3.0.0", package = "parity-scale-codec" } clap = { version = "3.0", features = ["derive"] } chrono = "0.4" @@ -33,7 +41,14 @@ handlebars = "4.1.6" Inflector = "0.11.4" linked-hash-map = "0.5.4" log = "0.4.8" +itertools = "0.10.3" +serde_nanos = "0.1.2" +kvdb = "0.11.0" +hash-db = "0.15.2" +hex = "0.4.3" +memory-db = "0.29.0" +rand = { version = "0.8.4", features = ["small_rng"] } [features] -default = ["db"] +default = ["db", "sc-client-db/runtime-benchmarks"] db = ["sc-client-db/with-kvdb-rocksdb", "sc-client-db/with-parity-db"] diff --git a/utils/frame/benchmarking-cli/src/lib.rs b/utils/frame/benchmarking-cli/src/lib.rs index 3b0f4843d2065..56aab0321ccd0 100644 --- a/utils/frame/benchmarking-cli/src/lib.rs +++ b/utils/frame/benchmarking-cli/src/lib.rs @@ -16,11 +16,14 @@ // limitations under the License. mod command; +mod storage; mod writer; use sc_cli::{ExecutionStrategy, WasmExecutionMethod}; use std::{fmt::Debug, path::PathBuf}; +pub use storage::StorageCmd; + // Add a more relaxed parsing for pallet names by allowing pallet directory names with `-` to be // used like crate names with `_` fn parse_pallet_name(pallet: &str) -> String { diff --git a/utils/frame/benchmarking-cli/src/storage/cmd.rs b/utils/frame/benchmarking-cli/src/storage/cmd.rs new file mode 100644 index 0000000000000..4376b616286a4 --- /dev/null +++ b/utils/frame/benchmarking-cli/src/storage/cmd.rs @@ -0,0 +1,171 @@ +// This file is part of Substrate. + +// Copyright (C) 2022 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use sc_cli::{CliConfiguration, DatabaseParams, PruningParams, Result, SharedParams}; +use sc_client_api::{Backend as ClientBackend, StorageProvider, UsageProvider}; +use sc_client_db::DbHash; +use sc_service::Configuration; +use sp_blockchain::HeaderBackend; +use sp_database::{ColumnId, Database}; +use sp_runtime::traits::{Block as BlockT, HashFor}; +use sp_state_machine::Storage; +use sp_storage::StateVersion; + +use clap::{Args, Parser}; +use log::info; +use rand::prelude::*; +use serde::Serialize; +use std::{fmt::Debug, sync::Arc}; + +use super::{record::StatSelect, template::TemplateData}; + +/// Benchmark the storage of a Substrate node with a live chain snapshot. +#[derive(Debug, Parser)] +pub struct StorageCmd { + #[allow(missing_docs)] + #[clap(flatten)] + pub shared_params: SharedParams, + + #[allow(missing_docs)] + #[clap(flatten)] + pub database_params: DatabaseParams, + + #[allow(missing_docs)] + #[clap(flatten)] + pub pruning_params: PruningParams, + + #[allow(missing_docs)] + #[clap(flatten)] + pub params: StorageParams, +} + +/// Parameters for modifying the benchmark behaviour and the post processing of the results. +#[derive(Debug, Default, Serialize, Clone, PartialEq, Args)] +pub struct StorageParams { + /// Path to write the *weight* file to. Can be a file or directory. + /// For substrate this should be `frame/support/src/weights`. + #[clap(long, default_value = ".")] + pub weight_path: String, + + /// Select a specific metric to calculate the final weight output. + #[clap(long = "metric", default_value = "average")] + pub weight_metric: StatSelect, + + /// Multiply the resulting weight with the given factor. Must be positive. + /// Is calculated before `weight_add`. + #[clap(long = "mul", default_value = "1")] + pub weight_mul: f64, + + /// Add the given offset to the resulting weight. + /// Is calculated after `weight_mul`. + #[clap(long = "add", default_value = "0")] + pub weight_add: u64, + + /// Skip the `read` benchmark. + #[clap(long)] + pub skip_read: bool, + + /// Skip the `write` benchmark. + #[clap(long)] + pub skip_write: bool, + + /// Rounds of warmups before measuring. + /// Only supported for `read` benchmarks. + #[clap(long, default_value = "1")] + pub warmups: u32, + + /// The `StateVersion` to use. Substrate `--dev` should use `V1` and Polkadot `V0`. + /// Selecting the wrong version can corrupt the DB. + #[clap(long, possible_values = ["0", "1"])] + pub state_version: u8, + + /// State cache size. + #[clap(long, default_value = "0")] + pub state_cache_size: usize, +} + +impl StorageCmd { + /// Calls into the Read and Write benchmarking functions. + /// Processes the output and writes it into files and stdout. + pub async fn run( + &self, + cfg: Configuration, + client: Arc, + db: (Arc>, ColumnId), + storage: Arc>>, + ) -> Result<()> + where + BA: ClientBackend, + Block: BlockT, + C: UsageProvider + StorageProvider + HeaderBackend, + { + let mut template = TemplateData::new(&cfg, &self.params); + + if !self.params.skip_read { + let record = self.bench_read(client.clone())?; + record.save_json(&cfg, "read")?; + let stats = record.calculate_stats()?; + info!("Time summary [ns]:\n{:?}\nValue size summary:\n{:?}", stats.0, stats.1); + template.set_stats(Some(stats), None)?; + } + + if !self.params.skip_write { + let record = self.bench_write(client, db, storage)?; + record.save_json(&cfg, "write")?; + let stats = record.calculate_stats()?; + info!("Time summary [ns]:\n{:?}\nValue size summary:\n{:?}", stats.0, stats.1); + template.set_stats(None, Some(stats))?; + } + + template.write(&self.params.weight_path) + } + + /// Returns the specified state version. + pub(crate) fn state_version(&self) -> StateVersion { + match self.params.state_version { + 0 => StateVersion::V0, + 1 => StateVersion::V1, + _ => unreachable!("Clap set to only allow 0 and 1"), + } + } + + /// Creates an rng from a random seed. + pub(crate) fn setup_rng() -> impl rand::Rng { + let seed = rand::thread_rng().gen::(); + info!("Using seed {}", seed); + StdRng::seed_from_u64(seed) + } +} + +// Boilerplate +impl CliConfiguration for StorageCmd { + fn shared_params(&self) -> &SharedParams { + &self.shared_params + } + + fn database_params(&self) -> Option<&DatabaseParams> { + Some(&self.database_params) + } + + fn pruning_params(&self) -> Option<&PruningParams> { + Some(&self.pruning_params) + } + + fn state_cache_size(&self) -> Result { + Ok(self.params.state_cache_size) + } +} diff --git a/utils/frame/benchmarking-cli/src/storage/mod.rs b/utils/frame/benchmarking-cli/src/storage/mod.rs new file mode 100644 index 0000000000000..9849cbcb6097b --- /dev/null +++ b/utils/frame/benchmarking-cli/src/storage/mod.rs @@ -0,0 +1,24 @@ +// This file is part of Substrate. + +// Copyright (C) 2022 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +pub mod cmd; +pub mod read; +pub mod record; +pub mod template; +pub mod write; + +pub use cmd::StorageCmd; diff --git a/utils/frame/benchmarking-cli/src/storage/read.rs b/utils/frame/benchmarking-cli/src/storage/read.rs new file mode 100644 index 0000000000000..3974c4010f632 --- /dev/null +++ b/utils/frame/benchmarking-cli/src/storage/read.rs @@ -0,0 +1,76 @@ +// This file is part of Substrate. + +// Copyright (C) 2022 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use sc_cli::Result; +use sc_client_api::{Backend as ClientBackend, StorageProvider, UsageProvider}; +use sp_core::storage::StorageKey; +use sp_runtime::{ + generic::BlockId, + traits::{Block as BlockT, Header as HeaderT}, +}; + +use log::info; +use rand::prelude::*; +use std::{fmt::Debug, sync::Arc, time::Instant}; + +use super::{cmd::StorageCmd, record::BenchRecord}; + +impl StorageCmd { + /// Benchmarks the time it takes to read a single Storage item. + /// Uses the latest state that is available for the given client. + pub(crate) fn bench_read(&self, client: Arc) -> Result + where + C: UsageProvider + StorageProvider, + B: BlockT + Debug, + BA: ClientBackend, + <::Header as HeaderT>::Number: From, + { + let mut record = BenchRecord::default(); + let block = BlockId::Number(client.usage_info().chain.best_number); + + info!("Preparing keys from block {}", block); + // Load all keys and randomly shuffle them. + let empty_prefix = StorageKey(Vec::new()); + let mut keys = client.storage_keys(&block, &empty_prefix)?; + let mut rng = Self::setup_rng(); + keys.shuffle(&mut rng); + + // Run some rounds of the benchmark as warmup. + for i in 0..self.params.warmups { + info!("Warmup round {}/{}", i + 1, self.params.warmups); + for key in keys.clone() { + let _ = client + .storage(&block, &key) + .expect("Checked above to exist") + .ok_or("Value unexpectedly empty")?; + } + } + + // Interesting part here: + // Read all the keys in the database and measure the time it takes to access each. + info!("Reading {} keys", keys.len()); + for key in keys.clone() { + let start = Instant::now(); + let v = client + .storage(&block, &key) + .expect("Checked above to exist") + .ok_or("Value unexpectedly empty")?; + record.append(v.0.len(), start.elapsed())?; + } + Ok(record) + } +} diff --git a/utils/frame/benchmarking-cli/src/storage/record.rs b/utils/frame/benchmarking-cli/src/storage/record.rs new file mode 100644 index 0000000000000..00a613c713007 --- /dev/null +++ b/utils/frame/benchmarking-cli/src/storage/record.rs @@ -0,0 +1,191 @@ +// This file is part of Substrate. + +// Copyright (C) 2022 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Calculates statistics and fills out the `weight.hbs` template. + +use sc_cli::Result; +use sc_service::Configuration; + +use log::info; +use serde::Serialize; +use std::{fmt, fs, result, str::FromStr, time::Duration}; + +/// Raw output of a Storage benchmark. +#[derive(Debug, Default, Clone, Serialize)] +pub(crate) struct BenchRecord { + /// Multi-Map of value sizes and the time that it took to access them. + ns_per_size: Vec<(u64, u64)>, +} + +/// Various statistics that help to gauge the quality of the produced weights. +/// Will be written to the weight file and printed to console. +#[derive(Serialize, Default, Clone)] +pub(crate) struct Stats { + /// Sum of all values. + sum: u64, + /// Minimal observed value. + min: u64, + /// Maximal observed value. + max: u64, + + /// Average of all values. + avg: u64, + /// Median of all values. + median: u64, + /// Standard derivation of all values. + stddev: f64, + + /// 99th percentile. At least 99% of all values are below this threshold. + p99: u64, + /// 95th percentile. At least 95% of all values are below this threshold. + p95: u64, + /// 75th percentile. At least 75% of all values are below this threshold. + p75: u64, +} + +/// Selects a specific field from a [`Stats`] object. +/// Not all fields are available. +#[derive(Debug, Clone, Copy, Serialize, PartialEq)] +pub enum StatSelect { + /// Select the maximum. + Maximum, + /// Select the average. + Average, + /// Select the median. + Median, + /// Select the 99th percentile. + P99Percentile, + /// Select the 95th percentile. + P95Percentile, + /// Select the 75th percentile. + P75Percentile, +} + +impl BenchRecord { + /// Appends a new record. Uses safe casts. + pub fn append(&mut self, size: usize, d: Duration) -> Result<()> { + let size: u64 = size.try_into().map_err(|e| format!("Size overflow u64: {}", e))?; + let ns: u64 = d + .as_nanos() + .try_into() + .map_err(|e| format!("Nanoseconds overflow u64: {}", e))?; + self.ns_per_size.push((size, ns)); + Ok(()) + } + + /// Returns the statistics for *time* and *value size*. + pub(crate) fn calculate_stats(self) -> Result<(Stats, Stats)> { + let (size, time): (Vec<_>, Vec<_>) = self.ns_per_size.into_iter().unzip(); + let size = Stats::new(&size)?; + let time = Stats::new(&time)?; + Ok((time, size)) // The swap of time/size here is intentional. + } + + /// Saves the raw results in a json file in the current directory. + /// Prefixes it with the DB name and suffixed with `path_suffix`. + pub fn save_json(&self, cfg: &Configuration, path_suffix: &str) -> Result<()> { + let path = format!("{}_{}.json", cfg.database, path_suffix).to_lowercase(); + let json = serde_json::to_string_pretty(&self) + .map_err(|e| format!("Serializing as JSON: {:?}", e))?; + fs::write(&path, json)?; + info!("Raw data written to {:?}", fs::canonicalize(&path)?); + Ok(()) + } +} + +impl Stats { + /// Calculates statistics and returns them. + pub fn new(xs: &Vec) -> Result { + if xs.is_empty() { + return Err("Empty input is invalid".into()) + } + let (avg, stddev) = Self::avg_and_stddev(&xs); + + Ok(Self { + sum: xs.iter().sum(), + min: *xs.iter().min().expect("Checked for non-empty above"), + max: *xs.iter().max().expect("Checked for non-empty above"), + + avg: avg as u64, + median: Self::percentile(xs.clone(), 0.50), + stddev: (stddev * 100.0).round() / 100.0, // round to 1/100 + + p99: Self::percentile(xs.clone(), 0.99), + p95: Self::percentile(xs.clone(), 0.95), + p75: Self::percentile(xs.clone(), 0.75), + }) + } + + /// Returns the selected stat. + pub(crate) fn select(&self, s: StatSelect) -> u64 { + match s { + StatSelect::Maximum => self.max, + StatSelect::Average => self.avg, + StatSelect::Median => self.median, + StatSelect::P99Percentile => self.p99, + StatSelect::P95Percentile => self.p95, + StatSelect::P75Percentile => self.p75, + } + } + + /// Returns the *average* and the *standard derivation*. + fn avg_and_stddev(xs: &Vec) -> (f64, f64) { + let avg = xs.iter().map(|x| *x as f64).sum::() / xs.len() as f64; + let variance = xs.iter().map(|x| (*x as f64 - avg).powi(2)).sum::() / xs.len() as f64; + (avg, variance.sqrt()) + } + + /// Returns the specified percentile for the given data. + /// This is best effort since it ignores the interpolation case. + fn percentile(mut xs: Vec, p: f64) -> u64 { + xs.sort(); + let index = (xs.len() as f64 * p).ceil() as usize; + xs[index] + } +} + +impl fmt::Debug for Stats { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "Total: {}\n", self.sum)?; + write!(f, "Min: {}, Max: {}\n", self.min, self.max)?; + write!(f, "Average: {}, Median: {}, Stddev: {}\n", self.avg, self.median, self.stddev)?; + write!(f, "Percentiles 99th, 95th, 75th: {}, {}, {}", self.p99, self.p95, self.p75) + } +} + +impl Default for StatSelect { + /// Returns the `Average` selector. + fn default() -> Self { + Self::Average + } +} + +impl FromStr for StatSelect { + type Err = &'static str; + + fn from_str(day: &str) -> result::Result { + match day.to_lowercase().as_str() { + "max" => Ok(Self::Maximum), + "average" => Ok(Self::Average), + "median" => Ok(Self::Median), + "p99" => Ok(Self::P99Percentile), + "p95" => Ok(Self::P95Percentile), + "p75" => Ok(Self::P75Percentile), + _ => Err("String was not a StatSelect"), + } + } +} diff --git a/utils/frame/benchmarking-cli/src/storage/template.rs b/utils/frame/benchmarking-cli/src/storage/template.rs new file mode 100644 index 0000000000000..56e0869a914a1 --- /dev/null +++ b/utils/frame/benchmarking-cli/src/storage/template.rs @@ -0,0 +1,126 @@ +// This file is part of Substrate. + +// Copyright (C) 2022 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use sc_cli::Result; +use sc_service::Configuration; + +use log::info; +use serde::Serialize; +use std::{env, fs, path::PathBuf}; + +use super::{cmd::StorageParams, record::Stats}; + +static VERSION: &'static str = env!("CARGO_PKG_VERSION"); +static TEMPLATE: &str = include_str!("./weights.hbs"); + +/// Data consumed by Handlebar to fill out the `weights.hbs` template. +#[derive(Serialize, Default, Debug, Clone)] +pub(crate) struct TemplateData { + /// Name of the database used. + db_name: String, + /// Name of the runtime. Taken from the chain spec. + runtime_name: String, + /// Version of the benchmarking CLI used. + version: String, + /// Date that the template was filled out. + date: String, + /// Command line arguments that were passed to the CLI. + args: Vec, + /// Storage params of the executed command. + params: StorageParams, + /// The weight for one `read`. + read_weight: u64, + /// The weight for one `write`. + write_weight: u64, + /// Stats about a `read` benchmark. Contains *time* and *value size* stats. + /// The *value size* stats are currently not used in the template. + read: Option<(Stats, Stats)>, + /// Stats about a `write` benchmark. Contains *time* and *value size* stats. + /// The *value size* stats are currently not used in the template. + write: Option<(Stats, Stats)>, +} + +impl TemplateData { + /// Returns a new [`Self`] from the given configuration. + pub fn new(cfg: &Configuration, params: &StorageParams) -> Self { + TemplateData { + db_name: format!("{}", cfg.database), + runtime_name: cfg.chain_spec.name().into(), + version: VERSION.into(), + date: chrono::Utc::now().format("%Y-%m-%d (Y/M/D)").to_string(), + args: env::args().collect::>(), + params: params.clone(), + ..Default::default() + } + } + + /// Sets the stats and calculates the final weights. + pub fn set_stats( + &mut self, + read: Option<(Stats, Stats)>, + write: Option<(Stats, Stats)>, + ) -> Result<()> { + if let Some(read) = read { + self.read_weight = calc_weight(&read.0, &self.params)?; + self.read = Some(read); + } + if let Some(write) = write { + self.write_weight = calc_weight(&write.0, &self.params)?; + self.write = Some(write); + } + Ok(()) + } + + /// Filles out the `weights.hbs` HBS template with its own data. + /// Writes the result to `path` which can be a directory or file. + pub fn write(&self, path: &str) -> Result<()> { + let mut handlebars = handlebars::Handlebars::new(); + // Format large integers with underscore. + handlebars.register_helper("underscore", Box::new(crate::writer::UnderscoreHelper)); + // Don't HTML escape any characters. + handlebars.register_escape_fn(|s| -> String { s.to_string() }); + + let out_path = self.build_path(path); + let mut fd = fs::File::create(&out_path)?; + info!("Writing weights to {:?}", fs::canonicalize(&out_path)?); + handlebars + .render_template_to_write(&TEMPLATE, &self, &mut fd) + .map_err(|e| format!("HBS template write: {:?}", e).into()) + } + + /// Builds a path for the weight file. + fn build_path(&self, weight_out: &str) -> PathBuf { + let mut path = PathBuf::from(weight_out); + if path.is_dir() { + path.push(format!("{}_weights.rs", self.db_name.to_lowercase())); + path.set_extension("rs"); + } + path + } +} + +/// Calculates the final weight by multiplying the selected metric with +/// `mul` and adding `add`. +/// Does not use safe casts and can overflow. +fn calc_weight(stat: &Stats, params: &StorageParams) -> Result { + if params.weight_mul.is_sign_negative() || !params.weight_mul.is_normal() { + return Err("invalid floating number for `weight_mul`".into()) + } + let s = stat.select(params.weight_metric) as f64; + let w = s.mul_add(params.weight_mul, params.weight_add as f64).ceil(); + Ok(w as u64) // No safe cast here since there is no `From` for `u64`. +} diff --git a/utils/frame/benchmarking-cli/src/storage/weights.hbs b/utils/frame/benchmarking-cli/src/storage/weights.hbs new file mode 100644 index 0000000000000..ffeb1fe04d81c --- /dev/null +++ b/utils/frame/benchmarking-cli/src/storage/weights.hbs @@ -0,0 +1,107 @@ +// This file is part of Substrate. + +// Copyright (C) 2022 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! THIS FILE WAS AUTO-GENERATED USING THE SUBSTRATE BENCHMARK CLI VERSION {{version}} +//! DATE: {{date}} +//! +//! DATABASE: `{{db_name}}`, RUNTIME: `{{runtime_name}}` +//! SKIP-WRITE: `{{params.skip_write}}`, SKIP-READ: `{{params.skip_read}}`, WARMUPS: `{{params.warmups}}` +//! STATE-VERSION: `V{{params.state_version}}`, STATE-CACHE-SIZE: `{{params.state_cache_size}}` +//! WEIGHT-PATH: `{{params.weight_path}}` +//! METRIC: `{{params.weight_metric}}`, WEIGHT-MUL: `{{params.weight_mul}}`, WEIGHT-ADD: `{{params.weight_add}}` + +// Executed Command: +{{#each args as |arg|}} +// {{arg}} +{{/each}} + +/// Storage DB weights for the {{runtime_name}} runtime and {{db_name}}. +pub mod constants { + use frame_support::{parameter_types, weights::{RuntimeDbWeight, constants}}; + + parameter_types! { + {{#if (eq db_name "ParityDb")}} + /// ParityDB can be enabled with a feature flag, but is still experimental. These weights + /// are available for brave runtime engineers who may want to try this out as default. + {{else}} + /// By default, Substrate uses RocksDB, so this will be the weight used throughout + /// the runtime. + {{/if}} + pub const {{db_name}}Weight: RuntimeDbWeight = RuntimeDbWeight { + /// Time to read one storage item. + /// Calculated by multiplying the *{{params.weight_metric}}* of all values with `{{params.weight_mul}}` and adding `{{params.weight_add}}`. + /// + /// Stats [ns]: + /// Min, Max: {{underscore read.0.min}}, {{underscore read.0.max}} + /// Average: {{underscore read.0.avg}} + /// Median: {{underscore read.0.median}} + /// StdDev: {{read.0.stddev}} + /// + /// Percentiles [ns]: + /// 99th: {{underscore read.0.p99}} + /// 95th: {{underscore read.0.p95}} + /// 75th: {{underscore read.0.p75}} + read: {{underscore read_weight}} * constants::WEIGHT_PER_NANOS, + + /// Time to write one storage item. + /// Calculated by multiplying the *{{params.weight_metric}}* of all values with `{{params.weight_mul}}` and adding `{{params.weight_add}}`. + /// + /// Stats [ns]: + /// Min, Max: {{underscore write.0.min}}, {{underscore write.0.max}} + /// Average: {{underscore write.0.avg}} + /// Median: {{underscore write.0.median}} + /// StdDev: {{write.0.stddev}} + /// + /// Percentiles [ns]: + /// 99th: {{underscore write.0.p99}} + /// 95th: {{underscore write.0.p95}} + /// 75th: {{underscore write.0.p75}} + write: {{underscore write_weight}} * constants::WEIGHT_PER_NANOS, + }; + } + + #[cfg(test)] + mod test_db_weights { + use super::constants::{{db_name}}Weight as W; + use frame_support::weights::constants; + + /// Checks that all weights exist and have sane values. + // NOTE: If this test fails but you are sure that the generated values are fine, + // you can delete it. + #[test] + fn bound() { + // At least 1 µs. + assert!( + W::get().reads(1) >= constants::WEIGHT_PER_MICROS, + "Read weight should be at least 1 µs." + ); + assert!( + W::get().writes(1) >= constants::WEIGHT_PER_MICROS, + "Write weight should be at least 1 µs." + ); + // At most 1 ms. + assert!( + W::get().reads(1) <= constants::WEIGHT_PER_MILLIS, + "Read weight should be at most 1 ms." + ); + assert!( + W::get().writes(1) <= constants::WEIGHT_PER_MILLIS, + "Write weight should be at most 1 ms." + ); + } + } +} diff --git a/utils/frame/benchmarking-cli/src/storage/write.rs b/utils/frame/benchmarking-cli/src/storage/write.rs new file mode 100644 index 0000000000000..eb9ba11f30696 --- /dev/null +++ b/utils/frame/benchmarking-cli/src/storage/write.rs @@ -0,0 +1,131 @@ +// This file is part of Substrate. + +// Copyright (C) 2022 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use sc_cli::Result; +use sc_client_api::UsageProvider; +use sc_client_db::{DbHash, DbState, DB_HASH_LEN}; +use sp_api::StateBackend; +use sp_blockchain::HeaderBackend; +use sp_database::{ColumnId, Transaction}; +use sp_runtime::{ + generic::BlockId, + traits::{Block as BlockT, HashFor, Header as HeaderT}, +}; +use sp_trie::PrefixedMemoryDB; + +use log::info; +use rand::prelude::*; +use std::{fmt::Debug, sync::Arc, time::Instant}; + +use super::{cmd::StorageCmd, record::BenchRecord}; + +impl StorageCmd { + /// Benchmarks the time it takes to write a single Storage item. + /// Uses the latest state that is available for the given client. + pub(crate) fn bench_write( + &self, + client: Arc, + (db, state_col): (Arc>, ColumnId), + storage: Arc>>, + ) -> Result + where + Block: BlockT
+ Debug, + H: HeaderT, + C: UsageProvider + HeaderBackend, + { + // Store the time that it took to write each value. + let mut record = BenchRecord::default(); + + let supports_rc = db.supports_ref_counting(); + let block = BlockId::Number(client.usage_info().chain.best_number); + let header = client.header(block)?.ok_or("Header not found")?; + let original_root = *header.state_root(); + let trie = DbState::::new(storage.clone(), original_root); + + info!("Preparing keys from block {}", block); + // Load all KV pairs and randomly shuffle them. + let mut kvs = trie.pairs(); + let mut rng = Self::setup_rng(); + kvs.shuffle(&mut rng); + + info!("Writing {} keys", kvs.len()); + // Write each value in one commit. + for (k, original_v) in kvs.iter() { + // Create a random value to overwrite with. + // NOTE: We use a possibly higher entropy than the original value, + // could be improved but acts as an over-estimation which is fine for now. + let mut new_v = vec![0; original_v.len()]; + rng.fill_bytes(&mut new_v[..]); + + // Interesting part here: + let start = Instant::now(); + // Create a TX that will modify the Trie in the DB and + // calculate the root hash of the Trie after the modification. + let replace = vec![(k.as_ref(), Some(new_v.as_ref()))]; + let (_, stx) = trie.storage_root(replace.iter().cloned(), self.state_version()); + // Only the keep the insertions, since we do not want to benchmark pruning. + let tx = convert_tx::(stx.clone(), true, state_col, supports_rc); + db.commit(tx).map_err(|e| format!("Writing to the Database: {}", e))?; + record.append(new_v.len(), start.elapsed())?; + + // Now undo the changes by removing what was added. + let tx = convert_tx::(stx.clone(), false, state_col, supports_rc); + db.commit(tx).map_err(|e| format!("Writing to the Database: {}", e))?; + } + Ok(record) + } +} + +/// Converts a Trie transaction into a DB transaction. +/// Removals are ignored and will not be included in the final tx. +/// `invert_inserts` replaces all inserts with removals. +/// +/// The keys of Trie transactions are prefixed, this is treated differently by each DB. +/// ParityDB can use an optimization where only the last `DB_HASH_LEN` byte are needed. +/// The last `DB_HASH_LEN` byte are the hash of the actual stored data, everything +/// before that is the route in the Patricia Trie. +/// RocksDB cannot do this and needs the whole route, hence no key truncating for RocksDB. +/// +/// TODO: +/// This copies logic from [`sp_client_db::Backend::try_commit_operation`] and should be +/// refactored to use a canonical `sanitize_key` function from `sp_client_db` which +/// does not yet exist. +fn convert_tx( + mut tx: PrefixedMemoryDB>, + invert_inserts: bool, + col: ColumnId, + supports_rc: bool, +) -> Transaction { + let mut ret = Transaction::::default(); + + for (mut k, (v, rc)) in tx.drain().into_iter() { + if supports_rc { + let _prefix = k.drain(0..k.len() - DB_HASH_LEN); + } + + if rc > 0 { + if invert_inserts { + ret.set(col, k.as_ref(), &v); + } else { + ret.remove(col, &k); + } + } + // < 0 means removal - ignored. + // 0 means no modification. + } + ret +} diff --git a/utils/frame/benchmarking-cli/src/writer.rs b/utils/frame/benchmarking-cli/src/writer.rs index 1e31c4e98e56d..17f1221e46d8b 100644 --- a/utils/frame/benchmarking-cli/src/writer.rs +++ b/utils/frame/benchmarking-cli/src/writer.rs @@ -436,7 +436,7 @@ where // A Handlebars helper to add an underscore after every 3rd character, // i.e. a separator for large numbers. #[derive(Clone, Copy)] -struct UnderscoreHelper; +pub(crate) struct UnderscoreHelper; impl handlebars::HelperDef for UnderscoreHelper { fn call<'reg: 'rc, 'rc>( &self,