From 746846415d34088f99fab9260a6badbba9f33323 Mon Sep 17 00:00:00 2001 From: Shawn Tabrizi Date: Wed, 17 Jun 2020 13:41:29 +0200 Subject: [PATCH 01/36] initial mockup --- client/db/src/bench.rs | 64 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 64 insertions(+) diff --git a/client/db/src/bench.rs b/client/db/src/bench.rs index 99ce1edae00c5..07b6736b7a11b 100644 --- a/client/db/src/bench.rs +++ b/client/db/src/bench.rs @@ -50,6 +50,38 @@ impl sp_state_machine::Storage> for StorageDb { root: Cell, @@ -59,6 +91,8 @@ pub struct BenchmarkingState { genesis: HashMap, (Vec, i32)>, record: Cell>>, shared_cache: SharedCache, // shared cache is always empty + key_tracker: RefCell, KeyTracker>>, + read_write_tracker: RefCell, } impl BenchmarkingState { @@ -76,6 +110,8 @@ impl BenchmarkingState { genesis_root: Default::default(), record: Default::default(), shared_cache: new_shared_cache(0, (1, 10)), + key_tracker: Default::default(), + read_write_tracker: Default::default(), }; state.reopen()?; @@ -109,6 +145,33 @@ impl BenchmarkingState { )); Ok(()) } + + fn add_read_key(&self, key: &[u8]) { + let mut key_tracker = self.key_tracker.borrow_mut(); + let mut read_write_tracker = self.read_write_tracker.borrow_mut(); + + let maybe_tracker = key_tracker.get(&key.to_vec()); + + let has_been_read = KeyTracker { + has_been_read: true, + has_been_written: false, + }; + + match maybe_tracker { + None => { + key_tracker.insert(key.to_vec(), has_been_read); + read_write_tracker.add_read(); + }, + Some(tracker) => { + if !tracker.has_been_read { + key_tracker.insert(key.to_vec(), has_been_read); + read_write_tracker.add_read(); + } else { + read_write_tracker.add_repeat_read(); + } + } + } + } } fn state_err() -> String { @@ -137,6 +200,7 @@ impl StateBackend> for BenchmarkingState { } fn exists_storage(&self, key: &[u8]) -> Result { + self.add_read_key(key); self.state.borrow().as_ref().ok_or_else(state_err)?.exists_storage(key) } From 947d68ce7a324e543c7e1208b33a659ab0ccb53b Mon Sep 17 00:00:00 2001 From: Shawn Tabrizi Date: Wed, 17 Jun 2020 15:56:52 +0200 Subject: [PATCH 02/36] add and wipe --- client/db/src/bench.rs | 40 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/client/db/src/bench.rs b/client/db/src/bench.rs index 07b6736b7a11b..eb01a542ca547 100644 --- a/client/db/src/bench.rs +++ b/client/db/src/bench.rs @@ -146,6 +146,11 @@ impl BenchmarkingState { Ok(()) } + fn wipe_tracker(&self) { + *self.key_tracker.borrow_mut() = HashMap::new(); + *self.read_write_tracker.borrow_mut() = Default::default(); + } + fn add_read_key(&self, key: &[u8]) { let mut key_tracker = self.key_tracker.borrow_mut(); let mut read_write_tracker = self.read_write_tracker.borrow_mut(); @@ -172,6 +177,34 @@ impl BenchmarkingState { } } } + + fn add_write_key(&self, key: &[u8]) { + let mut key_tracker = self.key_tracker.borrow_mut(); + let mut read_write_tracker = self.read_write_tracker.borrow_mut(); + + let maybe_tracker = key_tracker.get(&key.to_vec()); + + // If we have written to the key, we also consider that we have read from it. + let has_been_written = KeyTracker { + has_been_read: true, + has_been_written: true, + }; + + match maybe_tracker { + None => { + key_tracker.insert(key.to_vec(), has_been_written); + read_write_tracker.add_write(); + }, + Some(tracker) => { + if !tracker.has_been_written { + key_tracker.insert(key.to_vec(), has_been_written); + read_write_tracker.add_write(); + } else { + read_write_tracker.add_repeat_write(); + } + } + } + } } fn state_err() -> String { @@ -184,10 +217,12 @@ impl StateBackend> for BenchmarkingState { type TrieBackendStorage = as StateBackend>>::TrieBackendStorage; fn storage(&self, key: &[u8]) -> Result>, Self::Error> { + self.add_read_key(key); self.state.borrow().as_ref().ok_or_else(state_err)?.storage(key) } fn storage_hash(&self, key: &[u8]) -> Result, Self::Error> { + self.add_read_key(key); self.state.borrow().as_ref().ok_or_else(state_err)?.storage_hash(key) } @@ -196,6 +231,7 @@ impl StateBackend> for BenchmarkingState { child_info: &ChildInfo, key: &[u8], ) -> Result>, Self::Error> { + self.add_read_key(key); self.state.borrow().as_ref().ok_or_else(state_err)?.child_storage(child_info, key) } @@ -209,10 +245,12 @@ impl StateBackend> for BenchmarkingState { child_info: &ChildInfo, key: &[u8], ) -> Result { + self.add_read_key(key); self.state.borrow().as_ref().ok_or_else(state_err)?.exists_child_storage(child_info, key) } fn next_storage_key(&self, key: &[u8]) -> Result>, Self::Error> { + self.add_read_key(key); self.state.borrow().as_ref().ok_or_else(state_err)?.next_storage_key(key) } @@ -221,6 +259,7 @@ impl StateBackend> for BenchmarkingState { child_info: &ChildInfo, key: &[u8], ) -> Result>, Self::Error> { + self.add_read_key(key); self.state.borrow().as_ref().ok_or_else(state_err)?.next_child_storage_key(child_info, key) } @@ -336,6 +375,7 @@ impl StateBackend> for BenchmarkingState { self.root.set(self.genesis_root.clone()); self.reopen()?; + self.wipe_tracker(); Ok(()) } From 9c77f5081048d06efbd822f0a25b16007814752e Mon Sep 17 00:00:00 2001 From: Shawn Tabrizi Date: Wed, 17 Jun 2020 16:21:51 +0200 Subject: [PATCH 03/36] track writes --- client/db/src/bench.rs | 13 +++++++++---- primitives/state-machine/src/backend.rs | 2 +- primitives/state-machine/src/ext.rs | 1 + 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/client/db/src/bench.rs b/client/db/src/bench.rs index eb01a542ca547..5e03a2ae1a625 100644 --- a/client/db/src/bench.rs +++ b/client/db/src/bench.rs @@ -27,7 +27,7 @@ use sp_trie::{MemoryDB, prefixed_key}; use sp_core::storage::ChildInfo; use sp_runtime::traits::{Block as BlockT, HashFor}; use sp_runtime::Storage; -use sp_state_machine::{DBValue, backend::Backend as StateBackend}; +use sp_state_machine::{DBValue, backend::Backend as StateBackend, StorageCollection}; use kvdb::{KeyValueDB, DBTransaction}; use crate::storage_cache::{CachingState, SharedCache, new_shared_cache}; @@ -125,7 +125,7 @@ impl BenchmarkingState { ); state.genesis = transaction.clone().drain(); state.genesis_root = root.clone(); - state.commit(root, transaction)?; + state.commit(root, transaction, Vec::new())?; state.record.take(); Ok(state) } @@ -333,7 +333,7 @@ impl StateBackend> for BenchmarkingState { None } - fn commit(&self, storage_root: as Hasher>::Out, mut transaction: Self::Transaction) + fn commit(&self, storage_root: as Hasher>::Out, mut transaction: Self::Transaction, storage_changes: StorageCollection) -> Result<(), Self::Error> { if let Some(db) = self.db.take() { @@ -351,7 +351,12 @@ impl StateBackend> for BenchmarkingState { self.record.set(keys); db.write(db_transaction).map_err(|_| String::from("Error committing transaction"))?; self.root.set(storage_root); - self.db.set(Some(db)) + self.db.set(Some(db)); + + // Track DB Writes + storage_changes.iter().for_each(|(key, _)| { + self.add_write_key(key); + }); } else { return Err("Trying to commit to a closed db".into()) } diff --git a/primitives/state-machine/src/backend.rs b/primitives/state-machine/src/backend.rs index 20a3ab7500a98..9b5a2659bdf15 100644 --- a/primitives/state-machine/src/backend.rs +++ b/primitives/state-machine/src/backend.rs @@ -212,7 +212,7 @@ pub trait Backend: std::fmt::Debug { } /// Commit given transaction to storage. - fn commit(&self, _: H::Out, _: Self::Transaction) -> Result<(), Self::Error> { + fn commit(&self, _: H::Out, _: Self::Transaction, _: StorageCollection) -> Result<(), Self::Error> { unimplemented!() } } diff --git a/primitives/state-machine/src/ext.rs b/primitives/state-machine/src/ext.rs index 7e805250e726a..bb94573740877 100644 --- a/primitives/state-machine/src/ext.rs +++ b/primitives/state-machine/src/ext.rs @@ -570,6 +570,7 @@ where self.backend.commit( changes.transaction_storage_root, changes.transaction, + changes.main_storage_changes, ).expect(EXT_NOT_ALLOWED_TO_FAIL); self.storage_transaction_cache.reset(); } From 2f198b9819d5079425b4ac296848fa24feddde81 Mon Sep 17 00:00:00 2001 From: Shawn Tabrizi Date: Thu, 18 Jun 2020 05:18:12 +0200 Subject: [PATCH 04/36] start to add to pipeline --- client/db/src/bench.rs | 10 +++++++++- frame/benchmarking/src/lib.rs | 8 ++++++++ frame/benchmarking/src/utils.rs | 5 +++++ primitives/externalities/src/lib.rs | 7 +++++++ primitives/state-machine/src/backend.rs | 5 +++++ primitives/state-machine/src/basic.rs | 4 ++++ primitives/state-machine/src/ext.rs | 4 ++++ primitives/state-machine/src/read_only.rs | 4 ++++ 8 files changed, 46 insertions(+), 1 deletion(-) diff --git a/client/db/src/bench.rs b/client/db/src/bench.rs index 5e03a2ae1a625..c715feb3678c8 100644 --- a/client/db/src/bench.rs +++ b/client/db/src/bench.rs @@ -50,13 +50,15 @@ impl sp_state_machine::Storage> for StorageDb StateBackend> for BenchmarkingState { Ok(()) } + /// Get the key tracking information for the state db. + fn read_write_count(&self) -> (u32, u32, u32, u32) { + let count = *self.read_write_tracker.borrow_mut(); + (count.reads, count.repeat_reads, count.writes, count.repeat_writes) + } + fn register_overlay_stats(&mut self, stats: &sp_state_machine::StateMachineStats) { self.state.borrow_mut().as_mut().map(|s| s.register_overlay_stats(stats)); } diff --git a/frame/benchmarking/src/lib.rs b/frame/benchmarking/src/lib.rs index 47e83cffbcd3e..452037cc67349 100644 --- a/frame/benchmarking/src/lib.rs +++ b/frame/benchmarking/src/lib.rs @@ -791,6 +791,9 @@ macro_rules! impl_benchmark { results.push((c.clone(), elapsed_extrinsic, elapsed_storage_root)); + frame_support::debug::trace!(target: "benchmark", "Read Count {}", $crate::benchmarking::read_write_count()); + + // Wipe the DB back to the genesis state. $crate::benchmarking::wipe_db(); } @@ -891,6 +894,10 @@ macro_rules! impl_benchmark { let elapsed_extrinsic = finish_extrinsic - start_extrinsic; frame_support::debug::trace!(target: "benchmark", "End Benchmark: {} ns", elapsed_extrinsic); + frame_support::debug::trace!(target: "benchmark", "TEST"); + + frame_support::debug::trace!(target: "benchmark", "Read Count {}", $crate::benchmarking::read_write_count()); + // Time the storage root recalculation. let start_storage_root = $crate::benchmarking::current_time(); $crate::storage_root(); @@ -899,6 +906,7 @@ macro_rules! impl_benchmark { results.push((c.clone(), elapsed_extrinsic, elapsed_storage_root)); + frame_support::debug::trace!(target: "benchmark", "Read Count {}", $crate::benchmarking::read_write_count()); // Wipe the DB back to the genesis state. $crate::benchmarking::wipe_db(); } diff --git a/frame/benchmarking/src/utils.rs b/frame/benchmarking/src/utils.rs index 31ec3783cc061..e8999346e8e48 100644 --- a/frame/benchmarking/src/utils.rs +++ b/frame/benchmarking/src/utils.rs @@ -83,6 +83,11 @@ pub trait Benchmarking { fn commit_db(&mut self) { self.commit() } + + /// Get the read/write count + fn read_write_count(&self) -> u32 { + self.read_write_count().0 + } } /// The pallet benchmarking trait. diff --git a/primitives/externalities/src/lib.rs b/primitives/externalities/src/lib.rs index cfb1d0878a491..037ce748059b2 100644 --- a/primitives/externalities/src/lib.rs +++ b/primitives/externalities/src/lib.rs @@ -210,6 +210,13 @@ pub trait Externalities: ExtensionStore { /// /// Commits all changes to the database and clears all caches. fn commit(&mut self); + + /// !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + /// Benchmarking related functionality and shouldn't be used anywhere else! + /// !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + /// + /// Gets the current read/write count for the benchmarking process. + fn read_write_count(&self) -> (u32, u32, u32, u32); } /// Extension for the [`Externalities`] trait. diff --git a/primitives/state-machine/src/backend.rs b/primitives/state-machine/src/backend.rs index 9b5a2659bdf15..d78f89c612517 100644 --- a/primitives/state-machine/src/backend.rs +++ b/primitives/state-machine/src/backend.rs @@ -215,6 +215,11 @@ pub trait Backend: std::fmt::Debug { fn commit(&self, _: H::Out, _: Self::Transaction, _: StorageCollection) -> Result<(), Self::Error> { unimplemented!() } + + /// Get the read/write count of the db + fn read_write_count(&self) -> (u32, u32, u32, u32) { + unimplemented!() + } } impl<'a, T: Backend, H: Hasher> Backend for &'a T { diff --git a/primitives/state-machine/src/basic.rs b/primitives/state-machine/src/basic.rs index 917e41f33d78b..4228261dd43d7 100644 --- a/primitives/state-machine/src/basic.rs +++ b/primitives/state-machine/src/basic.rs @@ -310,6 +310,10 @@ impl Externalities for BasicExternalities { fn wipe(&mut self) {} fn commit(&mut self) {} + + fn read_write_count(&self) -> (u32, u32, u32, u32) { + unimplemented!("read_write_count is not supported in Basic") + } } impl sp_externalities::ExtensionStore for BasicExternalities { diff --git a/primitives/state-machine/src/ext.rs b/primitives/state-machine/src/ext.rs index bb94573740877..32c71023f83a3 100644 --- a/primitives/state-machine/src/ext.rs +++ b/primitives/state-machine/src/ext.rs @@ -574,6 +574,10 @@ where ).expect(EXT_NOT_ALLOWED_TO_FAIL); self.storage_transaction_cache.reset(); } + + fn read_write_count(&self) -> (u32, u32, u32, u32) { + self.backend.read_write_count() + } } diff --git a/primitives/state-machine/src/read_only.rs b/primitives/state-machine/src/read_only.rs index 817282f8e71a3..37209145092b1 100644 --- a/primitives/state-machine/src/read_only.rs +++ b/primitives/state-machine/src/read_only.rs @@ -173,6 +173,10 @@ impl<'a, H: Hasher, B: 'a + Backend> Externalities for ReadOnlyExternalities< fn wipe(&mut self) {} fn commit(&mut self) {} + + fn read_write_count(&self) -> (u32, u32, u32, u32) { + unimplemented!("read_write_count is not supported in ReadOnlyExternalities") + } } impl<'a, H: Hasher, B: 'a + Backend> sp_externalities::ExtensionStore for ReadOnlyExternalities<'a, H, B> { From 12395aefc1f5d309f0d4667ea570519c159aa714 Mon Sep 17 00:00:00 2001 From: Shawn Tabrizi Date: Thu, 18 Jun 2020 09:21:49 +0200 Subject: [PATCH 05/36] return all reads/writes --- frame/benchmarking/src/lib.rs | 10 +++------- frame/benchmarking/src/utils.rs | 4 ++-- primitives/runtime-interface/src/impls.rs | 4 ++++ 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/frame/benchmarking/src/lib.rs b/frame/benchmarking/src/lib.rs index 452037cc67349..9382e6c6b6df5 100644 --- a/frame/benchmarking/src/lib.rs +++ b/frame/benchmarking/src/lib.rs @@ -783,6 +783,8 @@ macro_rules! impl_benchmark { let elapsed_extrinsic = finish_extrinsic - start_extrinsic; frame_support::debug::trace!(target: "benchmark", "End Benchmark: {} ns", elapsed_extrinsic); + frame_support::debug::trace!(target: "benchmark", "Read/Write Count {:?}", $crate::benchmarking::read_write_count()); + // Time the storage root recalculation. let start_storage_root = $crate::benchmarking::current_time(); $crate::storage_root(); @@ -791,9 +793,6 @@ macro_rules! impl_benchmark { results.push((c.clone(), elapsed_extrinsic, elapsed_storage_root)); - frame_support::debug::trace!(target: "benchmark", "Read Count {}", $crate::benchmarking::read_write_count()); - - // Wipe the DB back to the genesis state. $crate::benchmarking::wipe_db(); } @@ -894,9 +893,7 @@ macro_rules! impl_benchmark { let elapsed_extrinsic = finish_extrinsic - start_extrinsic; frame_support::debug::trace!(target: "benchmark", "End Benchmark: {} ns", elapsed_extrinsic); - frame_support::debug::trace!(target: "benchmark", "TEST"); - - frame_support::debug::trace!(target: "benchmark", "Read Count {}", $crate::benchmarking::read_write_count()); + frame_support::debug::trace!(target: "benchmark", "Read/Write Count {:?}", $crate::benchmarking::read_write_count()); // Time the storage root recalculation. let start_storage_root = $crate::benchmarking::current_time(); @@ -906,7 +903,6 @@ macro_rules! impl_benchmark { results.push((c.clone(), elapsed_extrinsic, elapsed_storage_root)); - frame_support::debug::trace!(target: "benchmark", "Read Count {}", $crate::benchmarking::read_write_count()); // Wipe the DB back to the genesis state. $crate::benchmarking::wipe_db(); } diff --git a/frame/benchmarking/src/utils.rs b/frame/benchmarking/src/utils.rs index e8999346e8e48..ff9e80ed4fe60 100644 --- a/frame/benchmarking/src/utils.rs +++ b/frame/benchmarking/src/utils.rs @@ -85,8 +85,8 @@ pub trait Benchmarking { } /// Get the read/write count - fn read_write_count(&self) -> u32 { - self.read_write_count().0 + fn read_write_count(&self) -> (u32, u32, u32, u32) { + self.read_write_count() } } diff --git a/primitives/runtime-interface/src/impls.rs b/primitives/runtime-interface/src/impls.rs index 217316c3dd79c..259d3517f001d 100644 --- a/primitives/runtime-interface/src/impls.rs +++ b/primitives/runtime-interface/src/impls.rs @@ -365,6 +365,10 @@ impl PassBy for Option { type PassBy = Codec; } +impl PassBy for (u32, u32, u32, u32) { + type PassBy = Codec; +} + /// Implement `PassBy` with `Inner` for the given fixed sized hash types. macro_rules! for_primitive_types { { $( $hash:ident $n:expr ),* $(,)? } => { From 69f137df57b16abe36feb0397a8d54358dc07837 Mon Sep 17 00:00:00 2001 From: Shawn Tabrizi Date: Thu, 18 Jun 2020 09:29:07 +0200 Subject: [PATCH 06/36] Log reads and writes from bench db --- client/db/src/bench.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/client/db/src/bench.rs b/client/db/src/bench.rs index c715feb3678c8..6984f1330ff6d 100644 --- a/client/db/src/bench.rs +++ b/client/db/src/bench.rs @@ -24,7 +24,7 @@ use std::collections::HashMap; use hash_db::{Prefix, Hasher}; use sp_trie::{MemoryDB, prefixed_key}; -use sp_core::storage::ChildInfo; +use sp_core::{storage::ChildInfo, hexdisplay::HexDisplay}; use sp_runtime::traits::{Block as BlockT, HashFor}; use sp_runtime::Storage; use sp_state_machine::{DBValue, backend::Backend as StateBackend, StorageCollection}; @@ -154,6 +154,8 @@ impl BenchmarkingState { } fn add_read_key(&self, key: &[u8]) { + log::trace!(target: "benchmark", "Read: {}", HexDisplay::from(&key)); + let mut key_tracker = self.key_tracker.borrow_mut(); let mut read_write_tracker = self.read_write_tracker.borrow_mut(); @@ -181,6 +183,8 @@ impl BenchmarkingState { } fn add_write_key(&self, key: &[u8]) { + log::trace!(target: "benchmark", "Write: {}", HexDisplay::from(&key)); + let mut key_tracker = self.key_tracker.borrow_mut(); let mut read_write_tracker = self.read_write_tracker.borrow_mut(); From 60f22c33a630b6409a2a72731f588f953f0f1f38 Mon Sep 17 00:00:00 2001 From: Shawn Tabrizi Date: Thu, 18 Jun 2020 09:51:58 +0200 Subject: [PATCH 07/36] causes panic --- frame/benchmarking/src/lib.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/frame/benchmarking/src/lib.rs b/frame/benchmarking/src/lib.rs index 9382e6c6b6df5..3dc9a53b3b14e 100644 --- a/frame/benchmarking/src/lib.rs +++ b/frame/benchmarking/src/lib.rs @@ -783,6 +783,8 @@ macro_rules! impl_benchmark { let elapsed_extrinsic = finish_extrinsic - start_extrinsic; frame_support::debug::trace!(target: "benchmark", "End Benchmark: {} ns", elapsed_extrinsic); + $crate::benchmarking::commit_db(); + frame_support::debug::trace!(target: "benchmark", "Read/Write Count {:?}", $crate::benchmarking::read_write_count()); // Time the storage root recalculation. From e08ffef16a5a57825e983e5837b305469ebc71bc Mon Sep 17 00:00:00 2001 From: arkpar Date: Thu, 18 Jun 2020 10:11:03 +0200 Subject: [PATCH 08/36] Allow multiple commits --- client/db/src/bench.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/client/db/src/bench.rs b/client/db/src/bench.rs index 6984f1330ff6d..a1d6184e15ef5 100644 --- a/client/db/src/bench.rs +++ b/client/db/src/bench.rs @@ -354,7 +354,9 @@ impl StateBackend> for BenchmarkingState { } keys.push(key); } - self.record.set(keys); + let mut record = self.record.take(); + record.extend(keys); + self.record.set(record); db.write(db_transaction).map_err(|_| String::from("Error committing transaction"))?; self.root.set(storage_root); self.db.set(Some(db)); From 86ecbeb7c5e56287c6f814a417c875a708311ad3 Mon Sep 17 00:00:00 2001 From: Shawn Tabrizi Date: Thu, 18 Jun 2020 11:33:56 +0200 Subject: [PATCH 09/36] commit before ending benchmark --- frame/benchmarking/src/lib.rs | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/frame/benchmarking/src/lib.rs b/frame/benchmarking/src/lib.rs index 3dc9a53b3b14e..4a54ab7d5f9c7 100644 --- a/frame/benchmarking/src/lib.rs +++ b/frame/benchmarking/src/lib.rs @@ -775,16 +775,17 @@ macro_rules! impl_benchmark { // This will enable worst case scenario for reading from the database. $crate::benchmarking::commit_db(); + frame_support::debug::trace!(target: "benchmark", "BEFORE {:?}", $crate::benchmarking::read_write_count()); + // Time the extrinsic logic. frame_support::debug::trace!(target: "benchmark", "Start Benchmark: {:?} {:?}", name, component_value); let start_extrinsic = $crate::benchmarking::current_time(); closure_to_benchmark()?; let finish_extrinsic = $crate::benchmarking::current_time(); let elapsed_extrinsic = finish_extrinsic - start_extrinsic; - frame_support::debug::trace!(target: "benchmark", "End Benchmark: {} ns", elapsed_extrinsic); - + // Commit the changes to get proper write count $crate::benchmarking::commit_db(); - + frame_support::debug::trace!(target: "benchmark", "End Benchmark: {} ns", elapsed_extrinsic); frame_support::debug::trace!(target: "benchmark", "Read/Write Count {:?}", $crate::benchmarking::read_write_count()); // Time the storage root recalculation. @@ -893,8 +894,9 @@ macro_rules! impl_benchmark { closure_to_benchmark()?; let finish_extrinsic = $crate::benchmarking::current_time(); let elapsed_extrinsic = finish_extrinsic - start_extrinsic; + // Commit the changes to get proper write count + $crate::benchmarking::commit_db(); frame_support::debug::trace!(target: "benchmark", "End Benchmark: {} ns", elapsed_extrinsic); - frame_support::debug::trace!(target: "benchmark", "Read/Write Count {:?}", $crate::benchmarking::read_write_count()); // Time the storage root recalculation. From 004bea17a339f47f12df3511a1a0638afb82c558 Mon Sep 17 00:00:00 2001 From: Shawn Tabrizi Date: Thu, 18 Jun 2020 12:03:43 +0200 Subject: [PATCH 10/36] doesn't work??? --- client/db/src/bench.rs | 5 +++++ frame/benchmarking/src/lib.rs | 2 ++ frame/benchmarking/src/utils.rs | 5 +++++ primitives/externalities/src/lib.rs | 7 +++++++ primitives/state-machine/src/backend.rs | 5 +++++ primitives/state-machine/src/basic.rs | 4 ++++ primitives/state-machine/src/ext.rs | 4 ++++ primitives/state-machine/src/read_only.rs | 4 ++++ 8 files changed, 36 insertions(+) diff --git a/client/db/src/bench.rs b/client/db/src/bench.rs index a1d6184e15ef5..29ffbd9fd128e 100644 --- a/client/db/src/bench.rs +++ b/client/db/src/bench.rs @@ -398,6 +398,11 @@ impl StateBackend> for BenchmarkingState { (count.reads, count.repeat_reads, count.writes, count.repeat_writes) } + /// Reset the key tracking information for the state db. + fn reset_read_write_count(&mut self) { + self.wipe_tracker() + } + fn register_overlay_stats(&mut self, stats: &sp_state_machine::StateMachineStats) { self.state.borrow_mut().as_mut().map(|s| s.register_overlay_stats(stats)); } diff --git a/frame/benchmarking/src/lib.rs b/frame/benchmarking/src/lib.rs index 4a54ab7d5f9c7..a61fbc5465f07 100644 --- a/frame/benchmarking/src/lib.rs +++ b/frame/benchmarking/src/lib.rs @@ -774,6 +774,7 @@ macro_rules! impl_benchmark { // Commit the externalities to the database, flushing the DB cache. // This will enable worst case scenario for reading from the database. $crate::benchmarking::commit_db(); + $crate::benchmarking::reset_read_write_count(); frame_support::debug::trace!(target: "benchmark", "BEFORE {:?}", $crate::benchmarking::read_write_count()); @@ -887,6 +888,7 @@ macro_rules! impl_benchmark { // Commit the externalities to the database, flushing the DB cache. // This will enable worst case scenario for reading from the database. $crate::benchmarking::commit_db(); + $crate::benchmarking::reset_read_write_count(); // Time the extrinsic logic. frame_support::debug::trace!(target: "benchmark", "Start Benchmark: {:?} {:?}", name, component_value); diff --git a/frame/benchmarking/src/utils.rs b/frame/benchmarking/src/utils.rs index ff9e80ed4fe60..b94b5d8b0b415 100644 --- a/frame/benchmarking/src/utils.rs +++ b/frame/benchmarking/src/utils.rs @@ -88,6 +88,11 @@ pub trait Benchmarking { fn read_write_count(&self) -> (u32, u32, u32, u32) { self.read_write_count() } + + /// Reset the read/write count + fn reset_read_write_count(&mut self) { + self.reset_read_write_count() + } } /// The pallet benchmarking trait. diff --git a/primitives/externalities/src/lib.rs b/primitives/externalities/src/lib.rs index 037ce748059b2..9365b37d32e2c 100644 --- a/primitives/externalities/src/lib.rs +++ b/primitives/externalities/src/lib.rs @@ -217,6 +217,13 @@ pub trait Externalities: ExtensionStore { /// /// Gets the current read/write count for the benchmarking process. fn read_write_count(&self) -> (u32, u32, u32, u32); + + /// !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + /// Benchmarking related functionality and shouldn't be used anywhere else! + /// !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + /// + /// Resets read/write count for the benchmarking process. + fn reset_read_write_count(&mut self); } /// Extension for the [`Externalities`] trait. diff --git a/primitives/state-machine/src/backend.rs b/primitives/state-machine/src/backend.rs index d78f89c612517..48983ee8d0cef 100644 --- a/primitives/state-machine/src/backend.rs +++ b/primitives/state-machine/src/backend.rs @@ -220,6 +220,11 @@ pub trait Backend: std::fmt::Debug { fn read_write_count(&self) -> (u32, u32, u32, u32) { unimplemented!() } + + /// Get the read/write count of the db + fn reset_read_write_count(&mut self) { + unimplemented!() + } } impl<'a, T: Backend, H: Hasher> Backend for &'a T { diff --git a/primitives/state-machine/src/basic.rs b/primitives/state-machine/src/basic.rs index 4228261dd43d7..826da75a28d35 100644 --- a/primitives/state-machine/src/basic.rs +++ b/primitives/state-machine/src/basic.rs @@ -314,6 +314,10 @@ impl Externalities for BasicExternalities { fn read_write_count(&self) -> (u32, u32, u32, u32) { unimplemented!("read_write_count is not supported in Basic") } + + fn reset_read_write_count(&mut self) { + unimplemented!("reset_read_write_count is not supported in Basic") + } } impl sp_externalities::ExtensionStore for BasicExternalities { diff --git a/primitives/state-machine/src/ext.rs b/primitives/state-machine/src/ext.rs index 32c71023f83a3..b8f09658fc3cb 100644 --- a/primitives/state-machine/src/ext.rs +++ b/primitives/state-machine/src/ext.rs @@ -578,6 +578,10 @@ where fn read_write_count(&self) -> (u32, u32, u32, u32) { self.backend.read_write_count() } + + fn reset_read_write_count(&mut self) { + self.backend.reset_read_write_count() + } } diff --git a/primitives/state-machine/src/read_only.rs b/primitives/state-machine/src/read_only.rs index 37209145092b1..b660807701b29 100644 --- a/primitives/state-machine/src/read_only.rs +++ b/primitives/state-machine/src/read_only.rs @@ -177,6 +177,10 @@ impl<'a, H: Hasher, B: 'a + Backend> Externalities for ReadOnlyExternalities< fn read_write_count(&self) -> (u32, u32, u32, u32) { unimplemented!("read_write_count is not supported in ReadOnlyExternalities") } + + fn reset_read_write_count(&mut self) { + unimplemented!("reset_read_write_count is not supported in ReadOnlyExternalities") + } } impl<'a, H: Hasher, B: 'a + Backend> sp_externalities::ExtensionStore for ReadOnlyExternalities<'a, H, B> { From 1b25a8bd98a555854f553d2d4dc67fdc8a490808 Mon Sep 17 00:00:00 2001 From: Shawn Tabrizi Date: Thu, 18 Jun 2020 12:11:46 +0200 Subject: [PATCH 11/36] fix --- client/db/src/bench.rs | 2 +- primitives/state-machine/src/backend.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/client/db/src/bench.rs b/client/db/src/bench.rs index 29ffbd9fd128e..1ebf7340d8726 100644 --- a/client/db/src/bench.rs +++ b/client/db/src/bench.rs @@ -399,7 +399,7 @@ impl StateBackend> for BenchmarkingState { } /// Reset the key tracking information for the state db. - fn reset_read_write_count(&mut self) { + fn reset_read_write_count(&self) { self.wipe_tracker() } diff --git a/primitives/state-machine/src/backend.rs b/primitives/state-machine/src/backend.rs index 48983ee8d0cef..4a7bdc02d0aea 100644 --- a/primitives/state-machine/src/backend.rs +++ b/primitives/state-machine/src/backend.rs @@ -222,7 +222,7 @@ pub trait Backend: std::fmt::Debug { } /// Get the read/write count of the db - fn reset_read_write_count(&mut self) { + fn reset_read_write_count(&self) { unimplemented!() } } From c9952159c5ee2b2bb5da72b12d6bd0637efd7f25 Mon Sep 17 00:00:00 2001 From: Shawn Tabrizi Date: Thu, 18 Jun 2020 12:14:59 +0200 Subject: [PATCH 12/36] Update lib.rs --- frame/benchmarking/src/lib.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/frame/benchmarking/src/lib.rs b/frame/benchmarking/src/lib.rs index a61fbc5465f07..fbbf5501faa37 100644 --- a/frame/benchmarking/src/lib.rs +++ b/frame/benchmarking/src/lib.rs @@ -774,9 +774,9 @@ macro_rules! impl_benchmark { // Commit the externalities to the database, flushing the DB cache. // This will enable worst case scenario for reading from the database. $crate::benchmarking::commit_db(); - $crate::benchmarking::reset_read_write_count(); - frame_support::debug::trace!(target: "benchmark", "BEFORE {:?}", $crate::benchmarking::read_write_count()); + // Reset the read/write counter so we don't count operations in the setup process. + $crate::benchmarking::reset_read_write_count(); // Time the extrinsic logic. frame_support::debug::trace!(target: "benchmark", "Start Benchmark: {:?} {:?}", name, component_value); @@ -888,6 +888,8 @@ macro_rules! impl_benchmark { // Commit the externalities to the database, flushing the DB cache. // This will enable worst case scenario for reading from the database. $crate::benchmarking::commit_db(); + + // Reset the read/write counter so we don't count operations in the setup process. $crate::benchmarking::reset_read_write_count(); // Time the extrinsic logic. From 2eef7faab7ad9d94afeaaed30976b34c393508b9 Mon Sep 17 00:00:00 2001 From: Shawn Tabrizi Date: Thu, 18 Jun 2020 12:37:17 +0200 Subject: [PATCH 13/36] switch to struct for `BenchmarkResults` --- frame/benchmarking/src/analysis.rs | 18 +++++++++--------- frame/benchmarking/src/lib.rs | 12 ++++++++++-- frame/benchmarking/src/utils.rs | 7 ++++++- utils/frame/benchmarking-cli/src/command.rs | 6 +++--- 4 files changed, 28 insertions(+), 15 deletions(-) diff --git a/frame/benchmarking/src/analysis.rs b/frame/benchmarking/src/analysis.rs index 0446430975550..7488ea428d443 100644 --- a/frame/benchmarking/src/analysis.rs +++ b/frame/benchmarking/src/analysis.rs @@ -31,22 +31,22 @@ pub struct Analysis { impl Analysis { pub fn median_slopes(r: &Vec) -> Option { - let results = r[0].0.iter().enumerate().map(|(i, &(param, _))| { + let results = r[0].components.iter().enumerate().map(|(i, &(param, _))| { let mut counted = BTreeMap::, usize>::new(); - for (params, _, _) in r.iter() { - let mut p = params.iter().map(|x| x.1).collect::>(); + for result in r.iter() { + let mut p = result.components.iter().map(|x| x.1).collect::>(); p[i] = 0; *counted.entry(p).or_default() += 1; } let others: Vec = counted.iter().max_by_key(|i| i.1).expect("r is not empty; qed").0.clone(); let values = r.iter() .filter(|v| - v.0.iter() + v.components.iter() .map(|x| x.1) .zip(others.iter()) .enumerate() .all(|(j, (v1, v2))| j == i || v1 == *v2) - ).map(|(ps, v, _)| (ps[i].1, *v)) + ).map(|result| (result.components[i].1, result.extrinsic_time)) .collect::>(); (format!("{:?}", param), i, others, values) }).collect::>(); @@ -99,9 +99,9 @@ impl Analysis { pub fn min_squares_iqr(r: &Vec) -> Option { let mut results = BTreeMap::, Vec>::new(); - for &(ref params, t, _) in r.iter() { - let p = params.iter().map(|x| x.1).collect::>(); - results.entry(p).or_default().push(t); + for result in r.iter() { + let p = result.components.iter().map(|x| x.1).collect::>(); + results.entry(p).or_default().push(result.extrinsic_time); } for (_, rs) in results.iter_mut() { rs.sort(); @@ -111,7 +111,7 @@ impl Analysis { let mut data = vec![("Y", results.iter().flat_map(|x| x.1.iter().map(|v| *v as f64)).collect())]; - let names = r[0].0.iter().map(|x| format!("{:?}", x.0)).collect::>(); + let names = r[0].components.iter().map(|x| format!("{:?}", x.0)).collect::>(); data.extend(names.iter() .enumerate() .map(|(i, p)| ( diff --git a/frame/benchmarking/src/lib.rs b/frame/benchmarking/src/lib.rs index fbbf5501faa37..eb9a4695793d8 100644 --- a/frame/benchmarking/src/lib.rs +++ b/frame/benchmarking/src/lib.rs @@ -795,7 +795,11 @@ macro_rules! impl_benchmark { let finish_storage_root = $crate::benchmarking::current_time(); let elapsed_storage_root = finish_storage_root - start_storage_root; - results.push((c.clone(), elapsed_extrinsic, elapsed_storage_root)); + results.push($crate::BenchmarkResults { + components: c.clone(), + extrinsic_time: elapsed_extrinsic, + storage_root_time: elapsed_storage_root + }); // Wipe the DB back to the genesis state. $crate::benchmarking::wipe_db(); @@ -909,7 +913,11 @@ macro_rules! impl_benchmark { let finish_storage_root = $crate::benchmarking::current_time(); let elapsed_storage_root = finish_storage_root - start_storage_root; - results.push((c.clone(), elapsed_extrinsic, elapsed_storage_root)); + results.push($crate::BenchmarkResults { + components: c.clone(), + extrinsic_time: elapsed_extrinsic, + storage_root_time: elapsed_storage_root + }); // Wipe the DB back to the genesis state. $crate::benchmarking::wipe_db(); diff --git a/frame/benchmarking/src/utils.rs b/frame/benchmarking/src/utils.rs index b94b5d8b0b415..76f153b4f33ea 100644 --- a/frame/benchmarking/src/utils.rs +++ b/frame/benchmarking/src/utils.rs @@ -44,7 +44,12 @@ pub struct BenchmarkBatch { /// Results from running benchmarks on a FRAME pallet. /// Contains duration of the function call in nanoseconds along with the benchmark parameters /// used for that benchmark result. -pub type BenchmarkResults = (Vec<(BenchmarkParameter, u32)>, u128, u128); +#[derive(Encode, Decode, Clone, PartialEq, Debug)] +pub struct BenchmarkResults { + pub components: Vec<(BenchmarkParameter, u32)>, + pub extrinsic_time: u128, + pub storage_root_time: u128, +} sp_api::decl_runtime_apis! { /// Runtime api for benchmarking a FRAME runtime. diff --git a/utils/frame/benchmarking-cli/src/command.rs b/utils/frame/benchmarking-cli/src/command.rs index f867d75d2ab17..4fc2b69926547 100644 --- a/utils/frame/benchmarking-cli/src/command.rs +++ b/utils/frame/benchmarking-cli/src/command.rs @@ -107,15 +107,15 @@ impl BenchmarkCmd { if self.raw_data { // Print the table header - batch.results[0].0.iter().for_each(|param| print!("{:?},", param.0)); + batch.results[0].components.iter().for_each(|param| print!("{:?},", param.0)); print!("extrinsic_time,storage_root_time\n"); // Print the values batch.results.iter().for_each(|result| { - let parameters = &result.0; + let parameters = &result.components; parameters.iter().for_each(|param| print!("{:?},", param.1)); // Print extrinsic time and storage root time - print!("{:?},{:?}\n", result.1, result.2); + print!("{:?},{:?}\n", result.extrinsic_time, result.storage_root_time); }); println!(); From 473f492e5d92ad65247f5d9e24a317d77a7b4971 Mon Sep 17 00:00:00 2001 From: Shawn Tabrizi Date: Thu, 18 Jun 2020 12:48:52 +0200 Subject: [PATCH 14/36] add to output --- frame/benchmarking/src/lib.rs | 18 ++++++++++++++---- frame/benchmarking/src/utils.rs | 6 +++++- utils/frame/benchmarking-cli/src/command.rs | 11 +++++++++-- 3 files changed, 28 insertions(+), 7 deletions(-) diff --git a/frame/benchmarking/src/lib.rs b/frame/benchmarking/src/lib.rs index eb9a4695793d8..c0cdb5ae1fe83 100644 --- a/frame/benchmarking/src/lib.rs +++ b/frame/benchmarking/src/lib.rs @@ -787,7 +787,8 @@ macro_rules! impl_benchmark { // Commit the changes to get proper write count $crate::benchmarking::commit_db(); frame_support::debug::trace!(target: "benchmark", "End Benchmark: {} ns", elapsed_extrinsic); - frame_support::debug::trace!(target: "benchmark", "Read/Write Count {:?}", $crate::benchmarking::read_write_count()); + let read_write_count = $crate::benchmarking::read_write_count(); + frame_support::debug::trace!(target: "benchmark", "Read/Write Count {:?}", read_write_count); // Time the storage root recalculation. let start_storage_root = $crate::benchmarking::current_time(); @@ -798,7 +799,11 @@ macro_rules! impl_benchmark { results.push($crate::BenchmarkResults { components: c.clone(), extrinsic_time: elapsed_extrinsic, - storage_root_time: elapsed_storage_root + storage_root_time: elapsed_storage_root, + reads: read_write_count.0, + repeat_reads: read_write_count.1, + writes: read_write_count.2, + repeat_writes: read_write_count.3, }); // Wipe the DB back to the genesis state. @@ -905,7 +910,8 @@ macro_rules! impl_benchmark { // Commit the changes to get proper write count $crate::benchmarking::commit_db(); frame_support::debug::trace!(target: "benchmark", "End Benchmark: {} ns", elapsed_extrinsic); - frame_support::debug::trace!(target: "benchmark", "Read/Write Count {:?}", $crate::benchmarking::read_write_count()); + let read_write_count = $crate::benchmarking::read_write_count(); + frame_support::debug::trace!(target: "benchmark", "Read/Write Count {:?}", read_write_count); // Time the storage root recalculation. let start_storage_root = $crate::benchmarking::current_time(); @@ -916,7 +922,11 @@ macro_rules! impl_benchmark { results.push($crate::BenchmarkResults { components: c.clone(), extrinsic_time: elapsed_extrinsic, - storage_root_time: elapsed_storage_root + storage_root_time: elapsed_storage_root, + reads: read_write_count.0, + repeat_reads: read_write_count.1, + writes: read_write_count.2, + repeat_writes: read_write_count.3, }); // Wipe the DB back to the genesis state. diff --git a/frame/benchmarking/src/utils.rs b/frame/benchmarking/src/utils.rs index 76f153b4f33ea..0f896506adb11 100644 --- a/frame/benchmarking/src/utils.rs +++ b/frame/benchmarking/src/utils.rs @@ -44,11 +44,15 @@ pub struct BenchmarkBatch { /// Results from running benchmarks on a FRAME pallet. /// Contains duration of the function call in nanoseconds along with the benchmark parameters /// used for that benchmark result. -#[derive(Encode, Decode, Clone, PartialEq, Debug)] +#[derive(Encode, Decode, Default, Clone, PartialEq, Debug)] pub struct BenchmarkResults { pub components: Vec<(BenchmarkParameter, u32)>, pub extrinsic_time: u128, pub storage_root_time: u128, + pub reads: u32, + pub repeat_reads: u32, + pub writes: u32, + pub repeat_writes: u32, } sp_api::decl_runtime_apis! { diff --git a/utils/frame/benchmarking-cli/src/command.rs b/utils/frame/benchmarking-cli/src/command.rs index 4fc2b69926547..c5719bcf6ae86 100644 --- a/utils/frame/benchmarking-cli/src/command.rs +++ b/utils/frame/benchmarking-cli/src/command.rs @@ -109,13 +109,20 @@ impl BenchmarkCmd { // Print the table header batch.results[0].components.iter().for_each(|param| print!("{:?},", param.0)); - print!("extrinsic_time,storage_root_time\n"); + print!("extrinsic_time,storage_root_time,reads,repeat_reads,writes,repeat_writes\n"); // Print the values batch.results.iter().for_each(|result| { let parameters = &result.components; parameters.iter().for_each(|param| print!("{:?},", param.1)); // Print extrinsic time and storage root time - print!("{:?},{:?}\n", result.extrinsic_time, result.storage_root_time); + print!("{:?},{:?},{:?},{:?},{:?},{:?}\n", + result.extrinsic_time, + result.storage_root_time, + result.reads, + result.repeat_reads, + result.writes, + result.repeat_writes, + ); }); println!(); From 0ae5300c8814e81ed2eaf4995def6ebde929193c Mon Sep 17 00:00:00 2001 From: Shawn Tabrizi Date: Thu, 18 Jun 2020 13:14:53 +0200 Subject: [PATCH 15/36] fix test --- frame/benchmarking/src/analysis.rs | 48 ++++++++++++++++++++---------- 1 file changed, 32 insertions(+), 16 deletions(-) diff --git a/frame/benchmarking/src/analysis.rs b/frame/benchmarking/src/analysis.rs index 7488ea428d443..900744f7e38d2 100644 --- a/frame/benchmarking/src/analysis.rs +++ b/frame/benchmarking/src/analysis.rs @@ -222,17 +222,33 @@ mod tests { use super::*; use crate::BenchmarkParameter; + fn benchmark_result( + components: Vec<(BenchmarkParameter, u32)>, + extrinsic_time: u128, + storage_root_time: u128, + ) -> BenchmarkResults { + BenchmarkResults { + components, + extrinsic_time, + storage_root_time, + reads: 0, + repeat_reads: 0, + writes: 0, + repeat_writes: 0, + } + } + #[test] fn analysis_median_slopes_should_work() { let a = Analysis::median_slopes(&vec![ - (vec![(BenchmarkParameter::n, 1), (BenchmarkParameter::m, 5)], 11_500_000, 0), - (vec![(BenchmarkParameter::n, 2), (BenchmarkParameter::m, 5)], 12_500_000, 0), - (vec![(BenchmarkParameter::n, 3), (BenchmarkParameter::m, 5)], 13_500_000, 0), - (vec![(BenchmarkParameter::n, 4), (BenchmarkParameter::m, 5)], 14_500_000, 0), - (vec![(BenchmarkParameter::n, 3), (BenchmarkParameter::m, 1)], 13_100_000, 0), - (vec![(BenchmarkParameter::n, 3), (BenchmarkParameter::m, 3)], 13_300_000, 0), - (vec![(BenchmarkParameter::n, 3), (BenchmarkParameter::m, 7)], 13_700_000, 0), - (vec![(BenchmarkParameter::n, 3), (BenchmarkParameter::m, 10)], 14_000_000, 0), + benchmark_result(vec![(BenchmarkParameter::n, 1), (BenchmarkParameter::m, 5)], 11_500_000, 0), + benchmark_result(vec![(BenchmarkParameter::n, 2), (BenchmarkParameter::m, 5)], 12_500_000, 0), + benchmark_result(vec![(BenchmarkParameter::n, 3), (BenchmarkParameter::m, 5)], 13_500_000, 0), + benchmark_result(vec![(BenchmarkParameter::n, 4), (BenchmarkParameter::m, 5)], 14_500_000, 0), + benchmark_result(vec![(BenchmarkParameter::n, 3), (BenchmarkParameter::m, 1)], 13_100_000, 0), + benchmark_result(vec![(BenchmarkParameter::n, 3), (BenchmarkParameter::m, 3)], 13_300_000, 0), + benchmark_result(vec![(BenchmarkParameter::n, 3), (BenchmarkParameter::m, 7)], 13_700_000, 0), + benchmark_result(vec![(BenchmarkParameter::n, 3), (BenchmarkParameter::m, 10)], 14_000_000, 0), ]).unwrap(); assert_eq!(a.base, 10_000_000); assert_eq!(a.slopes, vec![1_000_000, 100_000]); @@ -241,14 +257,14 @@ mod tests { #[test] fn analysis_median_min_squares_should_work() { let a = Analysis::min_squares_iqr(&vec![ - (vec![(BenchmarkParameter::n, 1), (BenchmarkParameter::m, 5)], 11_500_000, 0), - (vec![(BenchmarkParameter::n, 2), (BenchmarkParameter::m, 5)], 12_500_000, 0), - (vec![(BenchmarkParameter::n, 3), (BenchmarkParameter::m, 5)], 13_500_000, 0), - (vec![(BenchmarkParameter::n, 4), (BenchmarkParameter::m, 5)], 14_500_000, 0), - (vec![(BenchmarkParameter::n, 3), (BenchmarkParameter::m, 1)], 13_100_000, 0), - (vec![(BenchmarkParameter::n, 3), (BenchmarkParameter::m, 3)], 13_300_000, 0), - (vec![(BenchmarkParameter::n, 3), (BenchmarkParameter::m, 7)], 13_700_000, 0), - (vec![(BenchmarkParameter::n, 3), (BenchmarkParameter::m, 10)], 14_000_000, 0), + benchmark_result(vec![(BenchmarkParameter::n, 1), (BenchmarkParameter::m, 5)], 11_500_000, 0), + benchmark_result(vec![(BenchmarkParameter::n, 2), (BenchmarkParameter::m, 5)], 12_500_000, 0), + benchmark_result(vec![(BenchmarkParameter::n, 3), (BenchmarkParameter::m, 5)], 13_500_000, 0), + benchmark_result(vec![(BenchmarkParameter::n, 4), (BenchmarkParameter::m, 5)], 14_500_000, 0), + benchmark_result(vec![(BenchmarkParameter::n, 3), (BenchmarkParameter::m, 1)], 13_100_000, 0), + benchmark_result(vec![(BenchmarkParameter::n, 3), (BenchmarkParameter::m, 3)], 13_300_000, 0), + benchmark_result(vec![(BenchmarkParameter::n, 3), (BenchmarkParameter::m, 7)], 13_700_000, 0), + benchmark_result(vec![(BenchmarkParameter::n, 3), (BenchmarkParameter::m, 10)], 14_000_000, 0), ]).unwrap(); assert_eq!(a.base, 10_000_000); assert_eq!(a.slopes, vec![1_000_000, 100_000]); From fd99bdac1979eb4a1c432a9d3e2182a94dc2fa32 Mon Sep 17 00:00:00 2001 From: Shawn Tabrizi Date: Thu, 18 Jun 2020 13:16:35 +0200 Subject: [PATCH 16/36] line width --- client/db/src/bench.rs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/client/db/src/bench.rs b/client/db/src/bench.rs index 1ebf7340d8726..1d6a421dcbb74 100644 --- a/client/db/src/bench.rs +++ b/client/db/src/bench.rs @@ -339,8 +339,11 @@ impl StateBackend> for BenchmarkingState { None } - fn commit(&self, storage_root: as Hasher>::Out, mut transaction: Self::Transaction, storage_changes: StorageCollection) - -> Result<(), Self::Error> + fn commit(&self, + storage_root: as Hasher>::Out, + mut transaction: Self::Transaction, + storage_changes: StorageCollection, + ) -> Result<(), Self::Error> { if let Some(db) = self.db.take() { let mut db_transaction = DBTransaction::new(); From 79cd015aef687111035eb600728b27cd0bdb046b Mon Sep 17 00:00:00 2001 From: Shawn Tabrizi Date: Thu, 18 Jun 2020 14:35:45 +0200 Subject: [PATCH 17/36] @kianenigma review --- client/db/src/bench.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/client/db/src/bench.rs b/client/db/src/bench.rs index 1d6a421dcbb74..7ee5c06fe204d 100644 --- a/client/db/src/bench.rs +++ b/client/db/src/bench.rs @@ -159,7 +159,7 @@ impl BenchmarkingState { let mut key_tracker = self.key_tracker.borrow_mut(); let mut read_write_tracker = self.read_write_tracker.borrow_mut(); - let maybe_tracker = key_tracker.get(&key.to_vec()); + let maybe_tracker = key_tracker.get(key); let has_been_read = KeyTracker { has_been_read: true, @@ -188,7 +188,7 @@ impl BenchmarkingState { let mut key_tracker = self.key_tracker.borrow_mut(); let mut read_write_tracker = self.read_write_tracker.borrow_mut(); - let maybe_tracker = key_tracker.get(&key.to_vec()); + let maybe_tracker = key_tracker.get(key); // If we have written to the key, we also consider that we have read from it. let has_been_written = KeyTracker { From 9464475c4d2e4cae213a8b7af673840ec520ccf1 Mon Sep 17 00:00:00 2001 From: Shawn Tabrizi Date: Sat, 20 Jun 2020 08:14:26 +0200 Subject: [PATCH 18/36] Add Whitelist to DB Tracking in Benchmarks Pipeline (#6405) * hardcoded whitelist * Add whitelist to pipeline * Remove whitelist pipeline from CLI, add to runtime * clean-up unused db initialized whitelist --- Cargo.lock | 1 + bin/node/runtime/Cargo.toml | 1 + bin/node/runtime/src/lib.rs | 20 ++++++++++++++++++- client/db/src/bench.rs | 24 +++++++++++++++++++++++ frame/benchmarking/src/lib.rs | 12 +++++++++++- frame/benchmarking/src/utils.rs | 5 +++++ primitives/externalities/src/lib.rs | 7 +++++++ primitives/state-machine/src/backend.rs | 5 +++++ primitives/state-machine/src/basic.rs | 4 ++++ primitives/state-machine/src/ext.rs | 4 ++++ primitives/state-machine/src/read_only.rs | 4 ++++ 11 files changed, 85 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 2c1d3e2c4b1e3..c8df9639ebb47 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3509,6 +3509,7 @@ dependencies = [ "frame-system", "frame-system-benchmarking", "frame-system-rpc-runtime-api", + "hex-literal", "integer-sqrt", "node-primitives", "pallet-authority-discovery", diff --git a/bin/node/runtime/Cargo.toml b/bin/node/runtime/Cargo.toml index 7cc4018fb6c4c..c0b640ab27c99 100644 --- a/bin/node/runtime/Cargo.toml +++ b/bin/node/runtime/Cargo.toml @@ -18,6 +18,7 @@ codec = { package = "parity-scale-codec", version = "1.3.0", default-features = integer-sqrt = { version = "0.1.2" } serde = { version = "1.0.102", optional = true } static_assertions = "1.1.0" +hex-literal = "0.2.1" # primitives sp-authority-discovery = { version = "2.0.0-rc3", default-features = false, path = "../../../primitives/authority-discovery" } diff --git a/bin/node/runtime/src/lib.rs b/bin/node/runtime/src/lib.rs index cf3d2622988a0..d668eb62c638c 100644 --- a/bin/node/runtime/src/lib.rs +++ b/bin/node/runtime/src/lib.rs @@ -1074,8 +1074,26 @@ impl_runtime_apis! { impl pallet_offences_benchmarking::Trait for Runtime {} impl frame_system_benchmarking::Trait for Runtime {} + let whitelist: Vec> = vec![ + // Block Number + // frame_system::Number::::hashed_key().to_vec(), + hex_literal::hex!("26aa394eea5630e07c48ae0c9558cef702a5c1b19ab7a04f536c519aca4983ac").to_vec(), + // Total Issuance + hex_literal::hex!("c2261276cc9d1f8598ea4b6a74b15c2f57c875e4cff74148e4628f264b974c80").to_vec(), + // Execution Phase + hex_literal::hex!("26aa394eea5630e07c48ae0c9558cef7ff553b5a9862a516939d82b3d3d8661a").to_vec(), + // Event Count + hex_literal::hex!("26aa394eea5630e07c48ae0c9558cef70a98fdbe9ce6c55837576c60c7af3850").to_vec(), + // System Events + hex_literal::hex!("26aa394eea5630e07c48ae0c9558cef780d41e5e16056765bc8461851072c9d7").to_vec(), + // Caller 0 Account + hex_literal::hex!("26aa394eea5630e07c48ae0c9558cef7b99d880ec681799c0cf30e8886371da946c154ffd9992e395af90b5b13cc6f295c77033fce8a9045824a6690bbf99c6db269502f0a8d1d2a008542d5690a0749").to_vec(), + // Treasury Account + hex_literal::hex!("26aa394eea5630e07c48ae0c9558cef7b99d880ec681799c0cf30e8886371da95ecffd7b6c0f78751baa9d281e0bfa3a6d6f646c70792f74727372790000000000000000000000000000000000000000").to_vec(), + ]; + let mut batches = Vec::::new(); - let params = (&pallet, &benchmark, &lowest_range_values, &highest_range_values, &steps, repeat); + let params = (&pallet, &benchmark, &lowest_range_values, &highest_range_values, &steps, repeat, &whitelist); add_benchmark!(params, batches, b"balances", Balances); add_benchmark!(params, batches, b"collective", Council); diff --git a/client/db/src/bench.rs b/client/db/src/bench.rs index 7ee5c06fe204d..384c2813cafa6 100644 --- a/client/db/src/bench.rs +++ b/client/db/src/bench.rs @@ -95,6 +95,7 @@ pub struct BenchmarkingState { shared_cache: SharedCache, // shared cache is always empty key_tracker: RefCell, KeyTracker>>, read_write_tracker: RefCell, + whitelist: RefCell>>, } impl BenchmarkingState { @@ -114,8 +115,11 @@ impl BenchmarkingState { shared_cache: new_shared_cache(0, (1, 10)), key_tracker: Default::default(), read_write_tracker: Default::default(), + whitelist: Default::default(), }; + state.add_whitelist_to_tracker(); + state.reopen()?; let child_delta = genesis.children_default.iter().map(|(_storage_key, child_content)| ( &child_content.child_info, @@ -148,8 +152,24 @@ impl BenchmarkingState { Ok(()) } + fn add_whitelist_to_tracker(&self) { + let mut key_tracker = self.key_tracker.borrow_mut(); + + let whitelisted = KeyTracker { + has_been_read: true, + has_been_written: true, + }; + + let whitelist = self.whitelist.borrow_mut(); + + whitelist.iter().for_each(|key| { + key_tracker.insert(key.to_vec(), whitelisted); + }); + } + fn wipe_tracker(&self) { *self.key_tracker.borrow_mut() = HashMap::new(); + self.add_whitelist_to_tracker(); *self.read_write_tracker.borrow_mut() = Default::default(); } @@ -406,6 +426,10 @@ impl StateBackend> for BenchmarkingState { self.wipe_tracker() } + fn set_whitelist(&self, new: Vec>) { + *self.whitelist.borrow_mut() = new; + } + fn register_overlay_stats(&mut self, stats: &sp_state_machine::StateMachineStats) { self.state.borrow_mut().as_mut().map(|s| s.register_overlay_stats(stats)); } diff --git a/frame/benchmarking/src/lib.rs b/frame/benchmarking/src/lib.rs index c0cdb5ae1fe83..d8fa1c98185c4 100644 --- a/frame/benchmarking/src/lib.rs +++ b/frame/benchmarking/src/lib.rs @@ -708,6 +708,7 @@ macro_rules! impl_benchmark { highest_range_values: &[u32], steps: &[u32], repeat: u32, + whitelist: &[Vec] ) -> Result, &'static str> { // Map the input to the selected benchmark. let extrinsic = sp_std::str::from_utf8(extrinsic) @@ -717,6 +718,9 @@ macro_rules! impl_benchmark { _ => return Err("Could not find extrinsic."), }; + // Add whitelist to DB + $crate::benchmarking::set_whitelist(whitelist.to_vec()); + // Warm up the DB $crate::benchmarking::commit_db(); $crate::benchmarking::wipe_db(); @@ -831,6 +835,7 @@ macro_rules! impl_benchmark { highest_range_values: &[u32], steps: &[u32], repeat: u32, + whitelist: &[Vec] ) -> Result, &'static str> { // Map the input to the selected benchmark. let extrinsic = sp_std::str::from_utf8(extrinsic) @@ -840,6 +845,9 @@ macro_rules! impl_benchmark { _ => return Err("Could not find extrinsic."), }; + // Add whitelist to DB + $crate::benchmarking::set_whitelist(whitelist.to_vec()); + // Warm up the DB $crate::benchmarking::commit_db(); $crate::benchmarking::wipe_db(); @@ -1074,7 +1082,7 @@ macro_rules! impl_benchmark_tests { #[macro_export] macro_rules! add_benchmark { ( $params:ident, $batches:ident, $name:literal, $( $location:tt )* ) => ( - let (pallet, benchmark, lowest_range_values, highest_range_values, steps, repeat) = $params; + let (pallet, benchmark, lowest_range_values, highest_range_values, steps, repeat, whitelist) = $params; if &pallet[..] == &$name[..] || &pallet[..] == &b"*"[..] { if &pallet[..] == &b"*"[..] || &benchmark[..] == &b"*"[..] { for benchmark in $( $location )*::benchmarks().into_iter() { @@ -1085,6 +1093,7 @@ macro_rules! add_benchmark { &highest_range_values[..], &steps[..], repeat, + whitelist, )?, pallet: $name.to_vec(), benchmark: benchmark.to_vec(), @@ -1098,6 +1107,7 @@ macro_rules! add_benchmark { &highest_range_values[..], &steps[..], repeat, + whitelist, )?, pallet: $name.to_vec(), benchmark: benchmark.clone(), diff --git a/frame/benchmarking/src/utils.rs b/frame/benchmarking/src/utils.rs index 0f896506adb11..7f9d9121100e3 100644 --- a/frame/benchmarking/src/utils.rs +++ b/frame/benchmarking/src/utils.rs @@ -102,6 +102,10 @@ pub trait Benchmarking { fn reset_read_write_count(&mut self) { self.reset_read_write_count() } + + fn set_whitelist(&mut self, new: Vec>) { + self.set_whitelist(new) + } } /// The pallet benchmarking trait. @@ -125,6 +129,7 @@ pub trait Benchmarking { highest_range_values: &[u32], steps: &[u32], repeat: u32, + whitelist: &[Vec] ) -> Result, &'static str>; } diff --git a/primitives/externalities/src/lib.rs b/primitives/externalities/src/lib.rs index 9365b37d32e2c..d4ded98004336 100644 --- a/primitives/externalities/src/lib.rs +++ b/primitives/externalities/src/lib.rs @@ -224,6 +224,13 @@ pub trait Externalities: ExtensionStore { /// /// Resets read/write count for the benchmarking process. fn reset_read_write_count(&mut self); + + /// !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + /// Benchmarking related functionality and shouldn't be used anywhere else! + /// !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + /// + /// Adds new storage keys to the DB tracking whitelist. + fn set_whitelist(&mut self, new: Vec>); } /// Extension for the [`Externalities`] trait. diff --git a/primitives/state-machine/src/backend.rs b/primitives/state-machine/src/backend.rs index 4a7bdc02d0aea..9ec03c4d1e249 100644 --- a/primitives/state-machine/src/backend.rs +++ b/primitives/state-machine/src/backend.rs @@ -225,6 +225,11 @@ pub trait Backend: std::fmt::Debug { fn reset_read_write_count(&self) { unimplemented!() } + + /// Update the whitelist for tracking db reads/writes + fn set_whitelist(&self, _: Vec>) { + unimplemented!() + } } impl<'a, T: Backend, H: Hasher> Backend for &'a T { diff --git a/primitives/state-machine/src/basic.rs b/primitives/state-machine/src/basic.rs index 826da75a28d35..9db9b206915ea 100644 --- a/primitives/state-machine/src/basic.rs +++ b/primitives/state-machine/src/basic.rs @@ -318,6 +318,10 @@ impl Externalities for BasicExternalities { fn reset_read_write_count(&mut self) { unimplemented!("reset_read_write_count is not supported in Basic") } + + fn set_whitelist(&mut self, _: Vec>) { + unimplemented!("set_whitelist is not supported in Basic") + } } impl sp_externalities::ExtensionStore for BasicExternalities { diff --git a/primitives/state-machine/src/ext.rs b/primitives/state-machine/src/ext.rs index b8f09658fc3cb..345524c1ad6e9 100644 --- a/primitives/state-machine/src/ext.rs +++ b/primitives/state-machine/src/ext.rs @@ -582,6 +582,10 @@ where fn reset_read_write_count(&mut self) { self.backend.reset_read_write_count() } + + fn set_whitelist(&mut self, new: Vec>) { + self.backend.set_whitelist(new) + } } diff --git a/primitives/state-machine/src/read_only.rs b/primitives/state-machine/src/read_only.rs index b660807701b29..3c9b4d2569729 100644 --- a/primitives/state-machine/src/read_only.rs +++ b/primitives/state-machine/src/read_only.rs @@ -181,6 +181,10 @@ impl<'a, H: Hasher, B: 'a + Backend> Externalities for ReadOnlyExternalities< fn reset_read_write_count(&mut self) { unimplemented!("reset_read_write_count is not supported in ReadOnlyExternalities") } + + fn set_whitelist(&mut self, _: Vec>) { + unimplemented!("set_whitelist is not supported in ReadOnlyExternalities") + } } impl<'a, H: Hasher, B: 'a + Backend> sp_externalities::ExtensionStore for ReadOnlyExternalities<'a, H, B> { From 956cda1f0c58b883c6293026e1637e574fd57690 Mon Sep 17 00:00:00 2001 From: Shawn Tabrizi Date: Mon, 22 Jun 2020 20:34:55 +0200 Subject: [PATCH 19/36] Add regression analysis to DB Tracking (#6475) * Add selector * add tests * debug formatter for easy formula --- frame/benchmarking/src/analysis.rs | 114 ++++++++++++++------ frame/benchmarking/src/lib.rs | 2 +- utils/frame/benchmarking-cli/src/command.rs | 24 ++++- 3 files changed, 104 insertions(+), 36 deletions(-) diff --git a/frame/benchmarking/src/analysis.rs b/frame/benchmarking/src/analysis.rs index 900744f7e38d2..621f3a2941ff5 100644 --- a/frame/benchmarking/src/analysis.rs +++ b/frame/benchmarking/src/analysis.rs @@ -29,8 +29,15 @@ pub struct Analysis { model: Option, } +pub enum BenchmarkSelector { + ExtrinsicTime, + StorageRootTime, + Reads, + Writes, +} + impl Analysis { - pub fn median_slopes(r: &Vec) -> Option { + pub fn median_slopes(r: &Vec, selector: BenchmarkSelector) -> Option { let results = r[0].components.iter().enumerate().map(|(i, &(param, _))| { let mut counted = BTreeMap::, usize>::new(); for result in r.iter() { @@ -46,7 +53,16 @@ impl Analysis { .zip(others.iter()) .enumerate() .all(|(j, (v1, v2))| j == i || v1 == *v2) - ).map(|result| (result.components[i].1, result.extrinsic_time)) + ).map(|result| { + // Extract the data we are interested in analyzing + let data = match selector { + BenchmarkSelector::ExtrinsicTime => result.extrinsic_time, + BenchmarkSelector::StorageRootTime => result.storage_root_time, + BenchmarkSelector::Reads => result.reads.into(), + BenchmarkSelector::Writes => result.writes.into(), + }; + (result.components[i].1, data) + }) .collect::>(); (format!("{:?}", param), i, others, values) }).collect::>(); @@ -97,12 +113,18 @@ impl Analysis { }) } - pub fn min_squares_iqr(r: &Vec) -> Option { + pub fn min_squares_iqr(r: &Vec, selector: BenchmarkSelector) -> Option { let mut results = BTreeMap::, Vec>::new(); for result in r.iter() { let p = result.components.iter().map(|x| x.1).collect::>(); - results.entry(p).or_default().push(result.extrinsic_time); + results.entry(p).or_default().push(match selector { + BenchmarkSelector::ExtrinsicTime => result.extrinsic_time, + BenchmarkSelector::StorageRootTime => result.storage_root_time, + BenchmarkSelector::Reads => result.reads.into(), + BenchmarkSelector::Writes => result.writes.into(), + }) } + for (_, rs) in results.iter_mut() { rs.sort(); let ql = rs.len() / 4; @@ -217,6 +239,16 @@ impl std::fmt::Display for Analysis { } } +impl std::fmt::Debug for Analysis { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + write!(f, "{}", self.base)?; + for (&m, n) in self.slopes.iter().zip(self.names.iter()) { + write!(f, " + ({} * {})", m, n)?; + } + write!(f,"") + } +} + #[cfg(test)] mod tests { use super::*; @@ -226,47 +258,69 @@ mod tests { components: Vec<(BenchmarkParameter, u32)>, extrinsic_time: u128, storage_root_time: u128, + reads: u32, + writes: u32, ) -> BenchmarkResults { BenchmarkResults { components, extrinsic_time, storage_root_time, - reads: 0, + reads, repeat_reads: 0, - writes: 0, + writes, repeat_writes: 0, } } #[test] fn analysis_median_slopes_should_work() { - let a = Analysis::median_slopes(&vec![ - benchmark_result(vec![(BenchmarkParameter::n, 1), (BenchmarkParameter::m, 5)], 11_500_000, 0), - benchmark_result(vec![(BenchmarkParameter::n, 2), (BenchmarkParameter::m, 5)], 12_500_000, 0), - benchmark_result(vec![(BenchmarkParameter::n, 3), (BenchmarkParameter::m, 5)], 13_500_000, 0), - benchmark_result(vec![(BenchmarkParameter::n, 4), (BenchmarkParameter::m, 5)], 14_500_000, 0), - benchmark_result(vec![(BenchmarkParameter::n, 3), (BenchmarkParameter::m, 1)], 13_100_000, 0), - benchmark_result(vec![(BenchmarkParameter::n, 3), (BenchmarkParameter::m, 3)], 13_300_000, 0), - benchmark_result(vec![(BenchmarkParameter::n, 3), (BenchmarkParameter::m, 7)], 13_700_000, 0), - benchmark_result(vec![(BenchmarkParameter::n, 3), (BenchmarkParameter::m, 10)], 14_000_000, 0), - ]).unwrap(); - assert_eq!(a.base, 10_000_000); - assert_eq!(a.slopes, vec![1_000_000, 100_000]); + let data = vec![ + benchmark_result(vec![(BenchmarkParameter::n, 1), (BenchmarkParameter::m, 5)], 11_500_000, 0, 3, 10), + benchmark_result(vec![(BenchmarkParameter::n, 2), (BenchmarkParameter::m, 5)], 12_500_000, 0, 4, 10), + benchmark_result(vec![(BenchmarkParameter::n, 3), (BenchmarkParameter::m, 5)], 13_500_000, 0, 5, 10), + benchmark_result(vec![(BenchmarkParameter::n, 4), (BenchmarkParameter::m, 5)], 14_500_000, 0, 6, 10), + benchmark_result(vec![(BenchmarkParameter::n, 3), (BenchmarkParameter::m, 1)], 13_100_000, 0, 5, 2), + benchmark_result(vec![(BenchmarkParameter::n, 3), (BenchmarkParameter::m, 3)], 13_300_000, 0, 5, 6), + benchmark_result(vec![(BenchmarkParameter::n, 3), (BenchmarkParameter::m, 7)], 13_700_000, 0, 5, 14), + benchmark_result(vec![(BenchmarkParameter::n, 3), (BenchmarkParameter::m, 10)], 14_000_000, 0, 5, 20), + ]; + + let extrinsic_time = Analysis::median_slopes(&data, BenchmarkSelector::ExtrinsicTime).unwrap(); + assert_eq!(extrinsic_time.base, 10_000_000); + assert_eq!(extrinsic_time.slopes, vec![1_000_000, 100_000]); + + let reads = Analysis::median_slopes(&data, BenchmarkSelector::Reads).unwrap(); + assert_eq!(reads.base, 2); + assert_eq!(reads.slopes, vec![1, 0]); + + let writes = Analysis::median_slopes(&data, BenchmarkSelector::Writes).unwrap(); + assert_eq!(writes.base, 0); + assert_eq!(writes.slopes, vec![0, 2]); } #[test] fn analysis_median_min_squares_should_work() { - let a = Analysis::min_squares_iqr(&vec![ - benchmark_result(vec![(BenchmarkParameter::n, 1), (BenchmarkParameter::m, 5)], 11_500_000, 0), - benchmark_result(vec![(BenchmarkParameter::n, 2), (BenchmarkParameter::m, 5)], 12_500_000, 0), - benchmark_result(vec![(BenchmarkParameter::n, 3), (BenchmarkParameter::m, 5)], 13_500_000, 0), - benchmark_result(vec![(BenchmarkParameter::n, 4), (BenchmarkParameter::m, 5)], 14_500_000, 0), - benchmark_result(vec![(BenchmarkParameter::n, 3), (BenchmarkParameter::m, 1)], 13_100_000, 0), - benchmark_result(vec![(BenchmarkParameter::n, 3), (BenchmarkParameter::m, 3)], 13_300_000, 0), - benchmark_result(vec![(BenchmarkParameter::n, 3), (BenchmarkParameter::m, 7)], 13_700_000, 0), - benchmark_result(vec![(BenchmarkParameter::n, 3), (BenchmarkParameter::m, 10)], 14_000_000, 0), - ]).unwrap(); - assert_eq!(a.base, 10_000_000); - assert_eq!(a.slopes, vec![1_000_000, 100_000]); + let data = vec![ + benchmark_result(vec![(BenchmarkParameter::n, 1), (BenchmarkParameter::m, 5)], 11_500_000, 0, 3, 10), + benchmark_result(vec![(BenchmarkParameter::n, 2), (BenchmarkParameter::m, 5)], 12_500_000, 0, 4, 10), + benchmark_result(vec![(BenchmarkParameter::n, 3), (BenchmarkParameter::m, 5)], 13_500_000, 0, 5, 10), + benchmark_result(vec![(BenchmarkParameter::n, 4), (BenchmarkParameter::m, 5)], 14_500_000, 0, 6, 10), + benchmark_result(vec![(BenchmarkParameter::n, 3), (BenchmarkParameter::m, 1)], 13_100_000, 0, 5, 2), + benchmark_result(vec![(BenchmarkParameter::n, 3), (BenchmarkParameter::m, 3)], 13_300_000, 0, 5, 6), + benchmark_result(vec![(BenchmarkParameter::n, 3), (BenchmarkParameter::m, 7)], 13_700_000, 0, 5, 14), + benchmark_result(vec![(BenchmarkParameter::n, 3), (BenchmarkParameter::m, 10)], 14_000_000, 0, 5, 20), + ]; + + let extrinsic_time = Analysis::min_squares_iqr(&data, BenchmarkSelector::ExtrinsicTime).unwrap(); + assert_eq!(extrinsic_time.base, 10_000_000); + assert_eq!(extrinsic_time.slopes, vec![1_000_000, 100_000]); + + let reads = Analysis::min_squares_iqr(&data, BenchmarkSelector::Reads).unwrap(); + assert_eq!(reads.base, 2); + assert_eq!(reads.slopes, vec![1, 0]); + + let writes = Analysis::min_squares_iqr(&data, BenchmarkSelector::Writes).unwrap(); + assert_eq!(writes.base, 0); + assert_eq!(writes.slopes, vec![0, 2]); } } diff --git a/frame/benchmarking/src/lib.rs b/frame/benchmarking/src/lib.rs index d8fa1c98185c4..f0b31b4fbe65c 100644 --- a/frame/benchmarking/src/lib.rs +++ b/frame/benchmarking/src/lib.rs @@ -26,7 +26,7 @@ mod analysis; pub use utils::*; #[cfg(feature = "std")] -pub use analysis::Analysis; +pub use analysis::{Analysis, BenchmarkSelector}; #[doc(hidden)] pub use sp_io::storage::root as storage_root; pub use sp_runtime::traits::Zero; diff --git a/utils/frame/benchmarking-cli/src/command.rs b/utils/frame/benchmarking-cli/src/command.rs index c5719bcf6ae86..7f55672885d46 100644 --- a/utils/frame/benchmarking-cli/src/command.rs +++ b/utils/frame/benchmarking-cli/src/command.rs @@ -17,7 +17,7 @@ use crate::BenchmarkCmd; use codec::{Decode, Encode}; -use frame_benchmarking::{Analysis, BenchmarkBatch}; +use frame_benchmarking::{Analysis, BenchmarkBatch, BenchmarkSelector}; use sc_cli::{SharedParams, CliConfiguration, ExecutionStrategy, Result}; use sc_client_db::BenchmarkingState; use sc_executor::NativeExecutor; @@ -130,13 +130,27 @@ impl BenchmarkCmd { // Conduct analysis. if !self.no_median_slopes { - if let Some(analysis) = Analysis::median_slopes(&batch.results) { - println!("Median Slopes Analysis\n========\n{}", analysis); + println!("Median Slopes Analysis\n========"); + if let Some(analysis) = Analysis::median_slopes(&batch.results, BenchmarkSelector::ExtrinsicTime) { + println!("-- Extrinsic Time --\n{}", analysis); + } + if let Some(analysis) = Analysis::median_slopes(&batch.results, BenchmarkSelector::Reads) { + println!("Reads = {:?}", analysis); + } + if let Some(analysis) = Analysis::median_slopes(&batch.results, BenchmarkSelector::Writes) { + println!("Writes = {:?}", analysis); } } if !self.no_min_squares { - if let Some(analysis) = Analysis::min_squares_iqr(&batch.results) { - println!("Min Squares Analysis\n========\n{}", analysis); + println!("Min Squares Analysis\n========"); + if let Some(analysis) = Analysis::min_squares_iqr(&batch.results, BenchmarkSelector::ExtrinsicTime) { + println!("-- Extrinsic Time --\n{}", analysis); + } + if let Some(analysis) = Analysis::min_squares_iqr(&batch.results, BenchmarkSelector::Reads) { + println!("Reads = {:?}", analysis); + } + if let Some(analysis) = Analysis::min_squares_iqr(&batch.results, BenchmarkSelector::Writes) { + println!("Writes = {:?}", analysis); } } }, From 962a3f53b798f1460687be0a5029b04999280a84 Mon Sep 17 00:00:00 2001 From: Shawn Tabrizi Date: Tue, 23 Jun 2020 10:50:58 +0200 Subject: [PATCH 20/36] initial idea --- frame/benchmarking/src/analysis.rs | 10 ++--- utils/frame/benchmarking-cli/src/command.rs | 5 ++- utils/frame/benchmarking-cli/src/lib.rs | 1 + utils/frame/benchmarking-cli/src/writer.rs | 48 +++++++++++++++++++++ 4 files changed, 58 insertions(+), 6 deletions(-) create mode 100644 utils/frame/benchmarking-cli/src/writer.rs diff --git a/frame/benchmarking/src/analysis.rs b/frame/benchmarking/src/analysis.rs index 621f3a2941ff5..c17e206c34c6c 100644 --- a/frame/benchmarking/src/analysis.rs +++ b/frame/benchmarking/src/analysis.rs @@ -22,11 +22,11 @@ use linregress::{FormulaRegressionBuilder, RegressionDataBuilder, RegressionMode use crate::BenchmarkResults; pub struct Analysis { - base: u128, - slopes: Vec, - names: Vec, - value_dists: Option, u128, u128)>>, - model: Option, + pub base: u128, + pub slopes: Vec, + pub names: Vec, + pub value_dists: Option, u128, u128)>>, + pub model: Option, } pub enum BenchmarkSelector { diff --git a/utils/frame/benchmarking-cli/src/command.rs b/utils/frame/benchmarking-cli/src/command.rs index 7f55672885d46..51ee33ec13a67 100644 --- a/utils/frame/benchmarking-cli/src/command.rs +++ b/utils/frame/benchmarking-cli/src/command.rs @@ -89,13 +89,16 @@ impl BenchmarkCmd { let results = , String> as Decode>::decode(&mut &result[..]) .map_err(|e| format!("Failed to decode benchmark results: {:?}", e))?; + let mut file = crate::writer::open_file()?; + crate::writer::write_results(&mut file, results.clone())?; + match results { Ok(batches) => for batch in batches.into_iter() { // Print benchmark metadata println!( "Pallet: {:?}, Extrinsic: {:?}, Lowest values: {:?}, Highest values: {:?}, Steps: {:?}, Repeat: {:?}", String::from_utf8(batch.pallet).expect("Encoded from String; qed"), - String::from_utf8(batch.benchmark).expect("Encoded from String; qed"), + String::from_utf8(batch.benchmark.clone()).expect("Encoded from String; qed"), self.lowest_range_values, self.highest_range_values, self.steps, diff --git a/utils/frame/benchmarking-cli/src/lib.rs b/utils/frame/benchmarking-cli/src/lib.rs index 149b971577fab..2772490b37e2c 100644 --- a/utils/frame/benchmarking-cli/src/lib.rs +++ b/utils/frame/benchmarking-cli/src/lib.rs @@ -16,6 +16,7 @@ // limitations under the License. mod command; +mod writer; use sc_cli::{ExecutionStrategy, WasmExecutionMethod}; use std::fmt::Debug; diff --git a/utils/frame/benchmarking-cli/src/writer.rs b/utils/frame/benchmarking-cli/src/writer.rs new file mode 100644 index 0000000000000..9ccf2b2463524 --- /dev/null +++ b/utils/frame/benchmarking-cli/src/writer.rs @@ -0,0 +1,48 @@ +use std::fs::{File, OpenOptions}; +use std::io::prelude::*; +use frame_benchmarking::{BenchmarkBatch, BenchmarkSelector, Analysis}; + +pub fn open_file() -> Result { + File::create("benchmarks.rs") +} + +pub fn write_results(file: &mut File, batches: Result, String>) -> Result<(), std::io::Error> { + let batches = batches.unwrap(); + //let f = &mut std::fmt::Formatter::new(); + batches.iter().for_each(|batch| { + // function name + write!(file, "pub fn {}_{}(", + String::from_utf8(batch.pallet.clone()).unwrap(), + String::from_utf8(batch.benchmark.clone()).unwrap() + ).unwrap(); + + // params + let components = &batch.results[0].components; + for component in components { + write!(file, "{:?}: u32,", component.0).unwrap(); + } + // return value + write!(file, ") -> Weight {{\n").unwrap(); + + let extrinsic_time = Analysis::min_squares_iqr(&batch.results, BenchmarkSelector::ExtrinsicTime).unwrap(); + let reads = Analysis::min_squares_iqr(&batch.results, BenchmarkSelector::Reads).unwrap(); + let writes = Analysis::min_squares_iqr(&batch.results, BenchmarkSelector::Writes).unwrap(); + + // return value + write!(file, " ({} as Weight)\n", extrinsic_time.base).unwrap(); + extrinsic_time.slopes.iter().zip(extrinsic_time.names.iter()).for_each(|(name, slope)| { + write!(file, " .saturating_add(({} as Weight).saturating_mul({} as Weight))\n", + name, + slope, + ).unwrap(); + }); + + // close function + write!(file, "}}\n").unwrap(); + + + + }); + + Ok(()) +} From c3e96393fabd532071a6044169dbd5bf916d8d3d Mon Sep 17 00:00:00 2001 From: Shawn Tabrizi Date: Tue, 23 Jun 2020 10:59:07 +0200 Subject: [PATCH 21/36] use all benchmarks --- utils/frame/benchmarking-cli/src/writer.rs | 24 +++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/utils/frame/benchmarking-cli/src/writer.rs b/utils/frame/benchmarking-cli/src/writer.rs index 9ccf2b2463524..5db0aab3daada 100644 --- a/utils/frame/benchmarking-cli/src/writer.rs +++ b/utils/frame/benchmarking-cli/src/writer.rs @@ -19,16 +19,12 @@ pub fn write_results(file: &mut File, batches: Result, Strin // params let components = &batch.results[0].components; for component in components { - write!(file, "{:?}: u32,", component.0).unwrap(); + write!(file, "{:?}: u32, ", component.0).unwrap(); } // return value write!(file, ") -> Weight {{\n").unwrap(); let extrinsic_time = Analysis::min_squares_iqr(&batch.results, BenchmarkSelector::ExtrinsicTime).unwrap(); - let reads = Analysis::min_squares_iqr(&batch.results, BenchmarkSelector::Reads).unwrap(); - let writes = Analysis::min_squares_iqr(&batch.results, BenchmarkSelector::Writes).unwrap(); - - // return value write!(file, " ({} as Weight)\n", extrinsic_time.base).unwrap(); extrinsic_time.slopes.iter().zip(extrinsic_time.names.iter()).for_each(|(name, slope)| { write!(file, " .saturating_add(({} as Weight).saturating_mul({} as Weight))\n", @@ -37,6 +33,24 @@ pub fn write_results(file: &mut File, batches: Result, Strin ).unwrap(); }); + let reads = Analysis::min_squares_iqr(&batch.results, BenchmarkSelector::Reads).unwrap(); + write!(file, " .saturating_add(T::DbWeight::get().reads({}))\n", reads.base).unwrap(); + reads.slopes.iter().zip(reads.names.iter()).for_each(|(name, slope)| { + write!(file, " .saturating_add(T::DbWeight::get().reads(({} as Weight).saturating_mul({} as Weight))\n", + name, + slope, + ).unwrap(); + }); + + let writes = Analysis::min_squares_iqr(&batch.results, BenchmarkSelector::Writes).unwrap(); + write!(file, " .saturating_add(T::DbWeight::get().writes({}))\n", writes.base).unwrap(); + writes.slopes.iter().zip(writes.names.iter()).for_each(|(name, slope)| { + write!(file, " .saturating_add(T::DbWeight::get().writes(({} as Weight).saturating_mul({} as Weight))\n", + name, + slope, + ).unwrap(); + }); + // close function write!(file, "}}\n").unwrap(); From 87dfc1900b5efbab8c8bab9840aad8c5a4404fc0 Mon Sep 17 00:00:00 2001 From: Shawn Tabrizi Date: Tue, 23 Jun 2020 15:11:41 +0200 Subject: [PATCH 22/36] broken --- bin/node/runtime/src/benchmarks.rs | 64 +++++++++++++++++++++ bin/node/runtime/src/lib.rs | 3 + frame/balances/src/lib.rs | 15 +++++ utils/frame/benchmarking-cli/src/command.rs | 2 +- utils/frame/benchmarking-cli/src/writer.rs | 21 +++---- 5 files changed, 94 insertions(+), 11 deletions(-) create mode 100644 bin/node/runtime/src/benchmarks.rs diff --git a/bin/node/runtime/src/benchmarks.rs b/bin/node/runtime/src/benchmarks.rs new file mode 100644 index 0000000000000..baddd08edb4d0 --- /dev/null +++ b/bin/node/runtime/src/benchmarks.rs @@ -0,0 +1,64 @@ +use frame_support::weights::Weight; +use frame_support::traits::Get; + +use pallet_balances::BalancesWeight; + +pub struct WeightForBalances; + +impl BalancesWeight for WeightForBalances { + fn balances_transfer(u: u32, e: u32, ) -> Weight { + (998433000 as Weight) + .saturating_add((u as Weight).saturating_mul(0 as Weight)) + .saturating_add((e as Weight).saturating_mul(0 as Weight)) + .saturating_add(T::DbWeight::get().reads(2 as Weight)) + .saturating_add(T::DbWeight::get().reads((u as Weight).saturating_mul(0 as Weight))) + .saturating_add(T::DbWeight::get().reads((e as Weight).saturating_mul(0 as Weight))) + .saturating_add(T::DbWeight::get().writes(2 as Weight)) + .saturating_add(T::DbWeight::get().writes((u as Weight).saturating_mul(0 as Weight))) + .saturating_add(T::DbWeight::get().writes((e as Weight).saturating_mul(0 as Weight))) + } + fn balances_transfer_best_case(u: u32, e: u32, ) -> Weight { + (553917000 as Weight) + .saturating_add((u as Weight).saturating_mul(0 as Weight)) + .saturating_add((e as Weight).saturating_mul(0 as Weight)) + .saturating_add(T::DbWeight::get().reads(2 as Weight)) + .saturating_add(T::DbWeight::get().reads((u as Weight).saturating_mul(0 as Weight))) + .saturating_add(T::DbWeight::get().reads((e as Weight).saturating_mul(0 as Weight))) + .saturating_add(T::DbWeight::get().writes(2 as Weight)) + .saturating_add(T::DbWeight::get().writes((u as Weight).saturating_mul(0 as Weight))) + .saturating_add(T::DbWeight::get().writes((e as Weight).saturating_mul(0 as Weight))) + } + fn balances_transfer_keep_alive(u: u32, e: u32, ) -> Weight { + (483036000 as Weight) + .saturating_add((u as Weight).saturating_mul(0 as Weight)) + .saturating_add((e as Weight).saturating_mul(23000 as Weight)) + .saturating_add(T::DbWeight::get().reads(2 as Weight)) + .saturating_add(T::DbWeight::get().reads((u as Weight).saturating_mul(0 as Weight))) + .saturating_add(T::DbWeight::get().reads((e as Weight).saturating_mul(0 as Weight))) + .saturating_add(T::DbWeight::get().writes(2 as Weight)) + .saturating_add(T::DbWeight::get().writes((u as Weight).saturating_mul(0 as Weight))) + .saturating_add(T::DbWeight::get().writes((e as Weight).saturating_mul(0 as Weight))) + } + fn balances_set_balance(u: u32, e: u32, ) -> Weight { + (332225000 as Weight) + .saturating_add((u as Weight).saturating_mul(5000 as Weight)) + .saturating_add((e as Weight).saturating_mul(11000 as Weight)) + .saturating_add(T::DbWeight::get().reads(1 as Weight)) + .saturating_add(T::DbWeight::get().reads((u as Weight).saturating_mul(0 as Weight))) + .saturating_add(T::DbWeight::get().reads((e as Weight).saturating_mul(0 as Weight))) + .saturating_add(T::DbWeight::get().writes(1 as Weight)) + .saturating_add(T::DbWeight::get().writes((u as Weight).saturating_mul(0 as Weight))) + .saturating_add(T::DbWeight::get().writes((e as Weight).saturating_mul(0 as Weight))) + } + fn balances_set_balance_killing(u: u32, e: u32, ) -> Weight { + (434352000 as Weight) + .saturating_add((u as Weight).saturating_mul(0 as Weight)) + .saturating_add((e as Weight).saturating_mul(0 as Weight)) + .saturating_add(T::DbWeight::get().reads(1 as Weight)) + .saturating_add(T::DbWeight::get().reads((u as Weight).saturating_mul(0 as Weight))) + .saturating_add(T::DbWeight::get().reads((e as Weight).saturating_mul(0 as Weight))) + .saturating_add(T::DbWeight::get().writes(1 as Weight)) + .saturating_add(T::DbWeight::get().writes((u as Weight).saturating_mul(0 as Weight))) + .saturating_add(T::DbWeight::get().writes((e as Weight).saturating_mul(0 as Weight))) + } +} diff --git a/bin/node/runtime/src/lib.rs b/bin/node/runtime/src/lib.rs index d668eb62c638c..04cc572011a05 100644 --- a/bin/node/runtime/src/lib.rs +++ b/bin/node/runtime/src/lib.rs @@ -84,6 +84,8 @@ use impls::{CurrencyToVoteHandler, Author}; pub mod constants; use constants::{time::*, currency::*}; +pub mod benchmarks; + // Make the WASM binary available. #[cfg(feature = "std")] include!(concat!(env!("OUT_DIR"), "/wasm_binary.rs")); @@ -291,6 +293,7 @@ impl pallet_balances::Trait for Runtime { type Event = Event; type ExistentialDeposit = ExistentialDeposit; type AccountStore = frame_system::Module; + type Weights = benchmarks::WeightForBalances; } parameter_types! { diff --git a/frame/balances/src/lib.rs b/frame/balances/src/lib.rs index e882bdf349683..c49bfd5a23035 100644 --- a/frame/balances/src/lib.rs +++ b/frame/balances/src/lib.rs @@ -159,6 +159,7 @@ use sp_std::prelude::*; use sp_std::{cmp, result, mem, fmt::Debug, ops::BitOr, convert::Infallible}; use codec::{Codec, Encode, Decode}; use frame_support::{ + weights::Weight, StorageValue, Parameter, decl_event, decl_storage, decl_module, decl_error, ensure, traits::{ Currency, OnKilledAccount, OnUnbalanced, TryDrop, StoredMap, @@ -188,6 +189,16 @@ pub trait Subtrait: frame_system::Trait { /// The means of storing the balances of an account. type AccountStore: StoredMap>; + + type Weight: BalancesWeight; +} + +pub trait BalancesWeight { + fn balances_transfer(u: u32, e: u32, ) -> Weight; + fn balances_transfer_best_case(u: u32, e: u32, ) -> Weight; + fn balances_transfer_keep_alive(u: u32, e: u32, ) -> Weight; + fn balances_set_balance(u: u32, e: u32, ) -> Weight; + fn balances_set_balance_killing(u: u32, e: u32, ) -> Weight; } pub trait Trait: frame_system::Trait { @@ -206,12 +217,15 @@ pub trait Trait: frame_system::Trait { /// The means of storing the balances of an account. type AccountStore: StoredMap>; + + type Weight: BalancesWeight; } impl, I: Instance> Subtrait for T { type Balance = T::Balance; type ExistentialDeposit = T::ExistentialDeposit; type AccountStore = T::AccountStore; + type Weight = T::Weight; } decl_event!( @@ -879,6 +893,7 @@ impl, I: Instance> Trait for ElevatedTrait { type DustRemoval = (); type ExistentialDeposit = T::ExistentialDeposit; type AccountStore = T::AccountStore; + type Weight = T::Weight; } impl, I: Instance> Currency for Module where diff --git a/utils/frame/benchmarking-cli/src/command.rs b/utils/frame/benchmarking-cli/src/command.rs index 51ee33ec13a67..b299e9cb5d06b 100644 --- a/utils/frame/benchmarking-cli/src/command.rs +++ b/utils/frame/benchmarking-cli/src/command.rs @@ -98,7 +98,7 @@ impl BenchmarkCmd { println!( "Pallet: {:?}, Extrinsic: {:?}, Lowest values: {:?}, Highest values: {:?}, Steps: {:?}, Repeat: {:?}", String::from_utf8(batch.pallet).expect("Encoded from String; qed"), - String::from_utf8(batch.benchmark.clone()).expect("Encoded from String; qed"), + String::from_utf8(batch.benchmark).expect("Encoded from String; qed"), self.lowest_range_values, self.highest_range_values, self.steps, diff --git a/utils/frame/benchmarking-cli/src/writer.rs b/utils/frame/benchmarking-cli/src/writer.rs index 5db0aab3daada..55ca826b9f8b2 100644 --- a/utils/frame/benchmarking-cli/src/writer.rs +++ b/utils/frame/benchmarking-cli/src/writer.rs @@ -10,8 +10,9 @@ pub fn write_results(file: &mut File, batches: Result, Strin let batches = batches.unwrap(); //let f = &mut std::fmt::Formatter::new(); batches.iter().for_each(|batch| { + // function name - write!(file, "pub fn {}_{}(", + write!(file, "pub fn {}_{}(", String::from_utf8(batch.pallet.clone()).unwrap(), String::from_utf8(batch.benchmark.clone()).unwrap() ).unwrap(); @@ -25,27 +26,27 @@ pub fn write_results(file: &mut File, batches: Result, Strin write!(file, ") -> Weight {{\n").unwrap(); let extrinsic_time = Analysis::min_squares_iqr(&batch.results, BenchmarkSelector::ExtrinsicTime).unwrap(); - write!(file, " ({} as Weight)\n", extrinsic_time.base).unwrap(); - extrinsic_time.slopes.iter().zip(extrinsic_time.names.iter()).for_each(|(name, slope)| { + write!(file, " ({} as Weight)\n", extrinsic_time.base.saturating_mul(1000)).unwrap(); + extrinsic_time.slopes.iter().zip(extrinsic_time.names.iter()).for_each(|(slope, name)| { write!(file, " .saturating_add(({} as Weight).saturating_mul({} as Weight))\n", name, - slope, + slope.saturating_mul(1000), ).unwrap(); }); let reads = Analysis::min_squares_iqr(&batch.results, BenchmarkSelector::Reads).unwrap(); - write!(file, " .saturating_add(T::DbWeight::get().reads({}))\n", reads.base).unwrap(); - reads.slopes.iter().zip(reads.names.iter()).for_each(|(name, slope)| { - write!(file, " .saturating_add(T::DbWeight::get().reads(({} as Weight).saturating_mul({} as Weight))\n", + write!(file, " .saturating_add(T::DbWeight::get().reads({} as Weight))\n", reads.base).unwrap(); + reads.slopes.iter().zip(reads.names.iter()).for_each(|(slope, name)| { + write!(file, " .saturating_add(T::DbWeight::get().reads(({} as Weight).saturating_mul({} as Weight)))\n", name, slope, ).unwrap(); }); let writes = Analysis::min_squares_iqr(&batch.results, BenchmarkSelector::Writes).unwrap(); - write!(file, " .saturating_add(T::DbWeight::get().writes({}))\n", writes.base).unwrap(); - writes.slopes.iter().zip(writes.names.iter()).for_each(|(name, slope)| { - write!(file, " .saturating_add(T::DbWeight::get().writes(({} as Weight).saturating_mul({} as Weight))\n", + write!(file, " .saturating_add(T::DbWeight::get().writes({} as Weight))\n", writes.base).unwrap(); + writes.slopes.iter().zip(writes.names.iter()).for_each(|(slope, name)| { + write!(file, " .saturating_add(T::DbWeight::get().writes(({} as Weight).saturating_mul({} as Weight)))\n", name, slope, ).unwrap(); From 6f82848458b81cf7f62a129ce233b08f044486e0 Mon Sep 17 00:00:00 2001 From: Shawn Tabrizi Date: Tue, 23 Jun 2020 17:57:01 +0200 Subject: [PATCH 23/36] working without trait --- bin/node/runtime/src/benchmarks.rs | 102 ++++++++++----------- bin/node/runtime/src/lib.rs | 2 +- frame/balances/src/lib.rs | 25 ++--- utils/frame/benchmarking-cli/src/writer.rs | 68 ++++++++++---- 4 files changed, 115 insertions(+), 82 deletions(-) diff --git a/bin/node/runtime/src/benchmarks.rs b/bin/node/runtime/src/benchmarks.rs index baddd08edb4d0..dc7e666e695ab 100644 --- a/bin/node/runtime/src/benchmarks.rs +++ b/bin/node/runtime/src/benchmarks.rs @@ -1,64 +1,60 @@ -use frame_support::weights::Weight; -use frame_support::traits::Get; - +use frame_support::weights::{Weight, constants::RocksDbWeight}; use pallet_balances::BalancesWeight; - pub struct WeightForBalances; - -impl BalancesWeight for WeightForBalances { - fn balances_transfer(u: u32, e: u32, ) -> Weight { - (998433000 as Weight) - .saturating_add((u as Weight).saturating_mul(0 as Weight)) +impl BalancesWeight for WeightForBalances { + fn transfer(u: u32, e: u32, ) -> Weight { + (574875000 as Weight) + .saturating_add((u as Weight).saturating_mul(7000 as Weight)) .saturating_add((e as Weight).saturating_mul(0 as Weight)) - .saturating_add(T::DbWeight::get().reads(2 as Weight)) - .saturating_add(T::DbWeight::get().reads((u as Weight).saturating_mul(0 as Weight))) - .saturating_add(T::DbWeight::get().reads((e as Weight).saturating_mul(0 as Weight))) - .saturating_add(T::DbWeight::get().writes(2 as Weight)) - .saturating_add(T::DbWeight::get().writes((u as Weight).saturating_mul(0 as Weight))) - .saturating_add(T::DbWeight::get().writes((e as Weight).saturating_mul(0 as Weight))) + .saturating_add(RocksDbWeight::get().reads(2 as Weight)) + .saturating_add(RocksDbWeight::get().reads((u as Weight).saturating_mul(0 as Weight))) + .saturating_add(RocksDbWeight::get().reads((e as Weight).saturating_mul(0 as Weight))) + .saturating_add(RocksDbWeight::get().writes(2 as Weight)) + .saturating_add(RocksDbWeight::get().writes((u as Weight).saturating_mul(0 as Weight))) + .saturating_add(RocksDbWeight::get().writes((e as Weight).saturating_mul(0 as Weight))) } - fn balances_transfer_best_case(u: u32, e: u32, ) -> Weight { - (553917000 as Weight) - .saturating_add((u as Weight).saturating_mul(0 as Weight)) + fn transfer_best_case(u: u32, e: u32, ) -> Weight { + (368265000 as Weight) + .saturating_add((u as Weight).saturating_mul(26000 as Weight)) .saturating_add((e as Weight).saturating_mul(0 as Weight)) - .saturating_add(T::DbWeight::get().reads(2 as Weight)) - .saturating_add(T::DbWeight::get().reads((u as Weight).saturating_mul(0 as Weight))) - .saturating_add(T::DbWeight::get().reads((e as Weight).saturating_mul(0 as Weight))) - .saturating_add(T::DbWeight::get().writes(2 as Weight)) - .saturating_add(T::DbWeight::get().writes((u as Weight).saturating_mul(0 as Weight))) - .saturating_add(T::DbWeight::get().writes((e as Weight).saturating_mul(0 as Weight))) + .saturating_add(RocksDbWeight::get().reads(2 as Weight)) + .saturating_add(RocksDbWeight::get().reads((u as Weight).saturating_mul(0 as Weight))) + .saturating_add(RocksDbWeight::get().reads((e as Weight).saturating_mul(0 as Weight))) + .saturating_add(RocksDbWeight::get().writes(2 as Weight)) + .saturating_add(RocksDbWeight::get().writes((u as Weight).saturating_mul(0 as Weight))) + .saturating_add(RocksDbWeight::get().writes((e as Weight).saturating_mul(0 as Weight))) } - fn balances_transfer_keep_alive(u: u32, e: u32, ) -> Weight { - (483036000 as Weight) - .saturating_add((u as Weight).saturating_mul(0 as Weight)) - .saturating_add((e as Weight).saturating_mul(23000 as Weight)) - .saturating_add(T::DbWeight::get().reads(2 as Weight)) - .saturating_add(T::DbWeight::get().reads((u as Weight).saturating_mul(0 as Weight))) - .saturating_add(T::DbWeight::get().reads((e as Weight).saturating_mul(0 as Weight))) - .saturating_add(T::DbWeight::get().writes(2 as Weight)) - .saturating_add(T::DbWeight::get().writes((u as Weight).saturating_mul(0 as Weight))) - .saturating_add(T::DbWeight::get().writes((e as Weight).saturating_mul(0 as Weight))) + fn transfer_keep_alive(u: u32, e: u32, ) -> Weight { + (402076000 as Weight) + .saturating_add((u as Weight).saturating_mul(7000 as Weight)) + .saturating_add((e as Weight).saturating_mul(14000 as Weight)) + .saturating_add(RocksDbWeight::get().reads(2 as Weight)) + .saturating_add(RocksDbWeight::get().reads((u as Weight).saturating_mul(0 as Weight))) + .saturating_add(RocksDbWeight::get().reads((e as Weight).saturating_mul(0 as Weight))) + .saturating_add(RocksDbWeight::get().writes(2 as Weight)) + .saturating_add(RocksDbWeight::get().writes((u as Weight).saturating_mul(0 as Weight))) + .saturating_add(RocksDbWeight::get().writes((e as Weight).saturating_mul(0 as Weight))) } - fn balances_set_balance(u: u32, e: u32, ) -> Weight { - (332225000 as Weight) - .saturating_add((u as Weight).saturating_mul(5000 as Weight)) - .saturating_add((e as Weight).saturating_mul(11000 as Weight)) - .saturating_add(T::DbWeight::get().reads(1 as Weight)) - .saturating_add(T::DbWeight::get().reads((u as Weight).saturating_mul(0 as Weight))) - .saturating_add(T::DbWeight::get().reads((e as Weight).saturating_mul(0 as Weight))) - .saturating_add(T::DbWeight::get().writes(1 as Weight)) - .saturating_add(T::DbWeight::get().writes((u as Weight).saturating_mul(0 as Weight))) - .saturating_add(T::DbWeight::get().writes((e as Weight).saturating_mul(0 as Weight))) - } - fn balances_set_balance_killing(u: u32, e: u32, ) -> Weight { - (434352000 as Weight) + fn set_balance(u: u32, e: u32, ) -> Weight { + (341092000 as Weight) .saturating_add((u as Weight).saturating_mul(0 as Weight)) + .saturating_add((e as Weight).saturating_mul(12000 as Weight)) + .saturating_add(RocksDbWeight::get().reads(1 as Weight)) + .saturating_add(RocksDbWeight::get().reads((u as Weight).saturating_mul(0 as Weight))) + .saturating_add(RocksDbWeight::get().reads((e as Weight).saturating_mul(0 as Weight))) + .saturating_add(RocksDbWeight::get().writes(1 as Weight)) + .saturating_add(RocksDbWeight::get().writes((u as Weight).saturating_mul(0 as Weight))) + .saturating_add(RocksDbWeight::get().writes((e as Weight).saturating_mul(0 as Weight))) + } + fn set_balance_killing(u: u32, e: u32, ) -> Weight { + (412116000 as Weight) + .saturating_add((u as Weight).saturating_mul(28000 as Weight)) .saturating_add((e as Weight).saturating_mul(0 as Weight)) - .saturating_add(T::DbWeight::get().reads(1 as Weight)) - .saturating_add(T::DbWeight::get().reads((u as Weight).saturating_mul(0 as Weight))) - .saturating_add(T::DbWeight::get().reads((e as Weight).saturating_mul(0 as Weight))) - .saturating_add(T::DbWeight::get().writes(1 as Weight)) - .saturating_add(T::DbWeight::get().writes((u as Weight).saturating_mul(0 as Weight))) - .saturating_add(T::DbWeight::get().writes((e as Weight).saturating_mul(0 as Weight))) + .saturating_add(RocksDbWeight::get().reads(1 as Weight)) + .saturating_add(RocksDbWeight::get().reads((u as Weight).saturating_mul(0 as Weight))) + .saturating_add(RocksDbWeight::get().reads((e as Weight).saturating_mul(0 as Weight))) + .saturating_add(RocksDbWeight::get().writes(1 as Weight)) + .saturating_add(RocksDbWeight::get().writes((u as Weight).saturating_mul(0 as Weight))) + .saturating_add(RocksDbWeight::get().writes((e as Weight).saturating_mul(0 as Weight))) } } diff --git a/bin/node/runtime/src/lib.rs b/bin/node/runtime/src/lib.rs index 04cc572011a05..e510a2439819a 100644 --- a/bin/node/runtime/src/lib.rs +++ b/bin/node/runtime/src/lib.rs @@ -293,7 +293,7 @@ impl pallet_balances::Trait for Runtime { type Event = Event; type ExistentialDeposit = ExistentialDeposit; type AccountStore = frame_system::Module; - type Weights = benchmarks::WeightForBalances; + type Weight = benchmarks::WeightForBalances; } parameter_types! { diff --git a/frame/balances/src/lib.rs b/frame/balances/src/lib.rs index c49bfd5a23035..3d94cd4358263 100644 --- a/frame/balances/src/lib.rs +++ b/frame/balances/src/lib.rs @@ -190,15 +190,15 @@ pub trait Subtrait: frame_system::Trait { /// The means of storing the balances of an account. type AccountStore: StoredMap>; - type Weight: BalancesWeight; + type Weight: BalancesWeight; } -pub trait BalancesWeight { - fn balances_transfer(u: u32, e: u32, ) -> Weight; - fn balances_transfer_best_case(u: u32, e: u32, ) -> Weight; - fn balances_transfer_keep_alive(u: u32, e: u32, ) -> Weight; - fn balances_set_balance(u: u32, e: u32, ) -> Weight; - fn balances_set_balance_killing(u: u32, e: u32, ) -> Weight; +pub trait BalancesWeight { + fn transfer(u: u32, e: u32, ) -> Weight; + fn transfer_best_case(u: u32, e: u32, ) -> Weight; + fn transfer_keep_alive(u: u32, e: u32, ) -> Weight; + fn set_balance(u: u32, e: u32, ) -> Weight; + fn set_balance_killing(u: u32, e: u32, ) -> Weight; } pub trait Trait: frame_system::Trait { @@ -218,7 +218,7 @@ pub trait Trait: frame_system::Trait { /// The means of storing the balances of an account. type AccountStore: StoredMap>; - type Weight: BalancesWeight; + type Weight: BalancesWeight; } impl, I: Instance> Subtrait for T { @@ -451,7 +451,7 @@ decl_module! { /// - DB Weight: 1 Read and 1 Write to destination account /// - Origin account is already in memory, so no DB operations for them. /// # - #[weight = T::DbWeight::get().reads_writes(1, 1) + 70_000_000] + #[weight = T::Weight::transfer(0,0)] pub fn transfer( origin, dest: ::Source, @@ -480,7 +480,7 @@ decl_module! { /// - Killing: 35.11 µs /// - DB Weight: 1 Read, 1 Write to `who` /// # - #[weight = T::DbWeight::get().reads_writes(1, 1) + 35_000_000] + #[weight = T::Weight::set_balance(0, 0)] fn set_balance( origin, who: ::Source, @@ -522,7 +522,8 @@ decl_module! { /// - Same as transfer, but additional read and write because the source account is /// not assumed to be in the overlay. /// # - #[weight = T::DbWeight::get().reads_writes(2, 2) + 70_000_000] + // TODO: ADD NEW BENCHMARK + #[weight = T::Weight::transfer(0,0)] pub fn force_transfer( origin, source: ::Source, @@ -546,7 +547,7 @@ decl_module! { /// - Base Weight: 51.4 µs /// - DB Weight: 1 Read and 1 Write to dest (sender is in overlay already) /// # - #[weight = T::DbWeight::get().reads_writes(1, 1) + 50_000_000] + #[weight = T::Weight::transfer_keep_alive(0,0)] pub fn transfer_keep_alive( origin, dest: ::Source, diff --git a/utils/frame/benchmarking-cli/src/writer.rs b/utils/frame/benchmarking-cli/src/writer.rs index 55ca826b9f8b2..75ba69f49f1fe 100644 --- a/utils/frame/benchmarking-cli/src/writer.rs +++ b/utils/frame/benchmarking-cli/src/writer.rs @@ -1,21 +1,57 @@ -use std::fs::{File, OpenOptions}; +use std::fs::File; use std::io::prelude::*; use frame_benchmarking::{BenchmarkBatch, BenchmarkSelector, Analysis}; +fn uppercase_first_letter(s: &str) -> String { + let mut c = s.chars(); + match c.next() { + None => String::new(), + Some(f) => f.to_uppercase().collect::() + c.as_str(), + } +} + pub fn open_file() -> Result { File::create("benchmarks.rs") } pub fn write_results(file: &mut File, batches: Result, String>) -> Result<(), std::io::Error> { let batches = batches.unwrap(); - //let f = &mut std::fmt::Formatter::new(); + + let mut current_pallet = Vec::::new(); + batches.iter().for_each(|batch| { + let pallet_string = String::from_utf8(batch.pallet.clone()).unwrap(); + let benchmark_string = String::from_utf8(batch.benchmark.clone()).unwrap(); + + // only create new trait definitions when we go to a new pallet + if batch.pallet != current_pallet { + if !current_pallet.is_empty() { + // close trait + write!(file, "}}\n").unwrap(); + } + + // imports + write!(file, "use frame_support::weights::{{Weight, constants::RocksDbWeight}};\n").unwrap(); + write!(file, "use pallet_{}::{}Weight;\n", + pallet_string, + uppercase_first_letter(&pallet_string), + ).unwrap(); + write!(file, "pub struct WeightFor{};\n", + uppercase_first_letter(&pallet_string), + ).unwrap(); + + // trait wrapper + write!(file, "impl {}Weight for WeightFor{} {{\n", + uppercase_first_letter(&pallet_string), + uppercase_first_letter(&pallet_string), + ).unwrap(); + + current_pallet = batch.pallet.clone() + } + // function name - write!(file, "pub fn {}_{}(", - String::from_utf8(batch.pallet.clone()).unwrap(), - String::from_utf8(batch.benchmark.clone()).unwrap() - ).unwrap(); + write!(file, " fn {}(", benchmark_string).unwrap(); // params let components = &batch.results[0].components; @@ -26,38 +62,38 @@ pub fn write_results(file: &mut File, batches: Result, Strin write!(file, ") -> Weight {{\n").unwrap(); let extrinsic_time = Analysis::min_squares_iqr(&batch.results, BenchmarkSelector::ExtrinsicTime).unwrap(); - write!(file, " ({} as Weight)\n", extrinsic_time.base.saturating_mul(1000)).unwrap(); + write!(file, " ({} as Weight)\n", extrinsic_time.base.saturating_mul(1000)).unwrap(); extrinsic_time.slopes.iter().zip(extrinsic_time.names.iter()).for_each(|(slope, name)| { - write!(file, " .saturating_add(({} as Weight).saturating_mul({} as Weight))\n", + write!(file, " .saturating_add(({} as Weight).saturating_mul({} as Weight))\n", name, slope.saturating_mul(1000), ).unwrap(); }); let reads = Analysis::min_squares_iqr(&batch.results, BenchmarkSelector::Reads).unwrap(); - write!(file, " .saturating_add(T::DbWeight::get().reads({} as Weight))\n", reads.base).unwrap(); + write!(file, " .saturating_add(RocksDbWeight::get().reads({} as Weight))\n", reads.base).unwrap(); reads.slopes.iter().zip(reads.names.iter()).for_each(|(slope, name)| { - write!(file, " .saturating_add(T::DbWeight::get().reads(({} as Weight).saturating_mul({} as Weight)))\n", + write!(file, " .saturating_add(RocksDbWeight::get().reads(({} as Weight).saturating_mul({} as Weight)))\n", name, slope, ).unwrap(); }); let writes = Analysis::min_squares_iqr(&batch.results, BenchmarkSelector::Writes).unwrap(); - write!(file, " .saturating_add(T::DbWeight::get().writes({} as Weight))\n", writes.base).unwrap(); + write!(file, " .saturating_add(RocksDbWeight::get().writes({} as Weight))\n", writes.base).unwrap(); writes.slopes.iter().zip(writes.names.iter()).for_each(|(slope, name)| { - write!(file, " .saturating_add(T::DbWeight::get().writes(({} as Weight).saturating_mul({} as Weight)))\n", + write!(file, " .saturating_add(RocksDbWeight::get().writes(({} as Weight).saturating_mul({} as Weight)))\n", name, slope, ).unwrap(); }); // close function - write!(file, "}}\n").unwrap(); - - - + write!(file, " }}\n").unwrap(); }); + // final close trait + write!(file, "}}\n").unwrap(); + Ok(()) } From 78ca73eb0f1f75a943b0f4855569fa8f3ad25e12 Mon Sep 17 00:00:00 2001 From: Shawn Tabrizi Date: Tue, 23 Jun 2020 18:27:49 +0200 Subject: [PATCH 24/36] Make work for multiple pallets --- utils/frame/benchmarking-cli/src/writer.rs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/utils/frame/benchmarking-cli/src/writer.rs b/utils/frame/benchmarking-cli/src/writer.rs index 75ba69f49f1fe..3c29f79f65bed 100644 --- a/utils/frame/benchmarking-cli/src/writer.rs +++ b/utils/frame/benchmarking-cli/src/writer.rs @@ -19,6 +19,9 @@ pub fn write_results(file: &mut File, batches: Result, Strin let mut current_pallet = Vec::::new(); + // general imports + write!(file, "use frame_support::weights::{{Weight, constants::RocksDbWeight}};\n").unwrap(); + batches.iter().for_each(|batch| { let pallet_string = String::from_utf8(batch.pallet.clone()).unwrap(); @@ -31,12 +34,13 @@ pub fn write_results(file: &mut File, batches: Result, Strin write!(file, "}}\n").unwrap(); } - // imports - write!(file, "use frame_support::weights::{{Weight, constants::RocksDbWeight}};\n").unwrap(); + // pallet trait import write!(file, "use pallet_{}::{}Weight;\n", pallet_string, uppercase_first_letter(&pallet_string), ).unwrap(); + + // struct for weights write!(file, "pub struct WeightFor{};\n", uppercase_first_letter(&pallet_string), ).unwrap(); From b38d82ee5457d82f72312d4ef5843cff844a7d99 Mon Sep 17 00:00:00 2001 From: Shawn Tabrizi Date: Sat, 27 Jun 2020 17:50:13 +0200 Subject: [PATCH 25/36] Fix merge issues --- frame/benchmarking/src/analysis.rs | 7 ------- primitives/state-machine/src/ext.rs | 12 ------------ utils/frame/benchmarking-cli/src/writer.rs | 6 +++--- 3 files changed, 3 insertions(+), 22 deletions(-) diff --git a/frame/benchmarking/src/analysis.rs b/frame/benchmarking/src/analysis.rs index 42f2d90370a81..c17e206c34c6c 100644 --- a/frame/benchmarking/src/analysis.rs +++ b/frame/benchmarking/src/analysis.rs @@ -36,13 +36,6 @@ pub enum BenchmarkSelector { Writes, } -pub enum BenchmarkSelector { - ExtrinsicTime, - StorageRootTime, - Reads, - Writes, -} - impl Analysis { pub fn median_slopes(r: &Vec, selector: BenchmarkSelector) -> Option { let results = r[0].components.iter().enumerate().map(|(i, &(param, _))| { diff --git a/primitives/state-machine/src/ext.rs b/primitives/state-machine/src/ext.rs index f772a4e7229b4..cd4f83661b9e3 100644 --- a/primitives/state-machine/src/ext.rs +++ b/primitives/state-machine/src/ext.rs @@ -612,18 +612,6 @@ where fn set_whitelist(&mut self, new: Vec>) { self.backend.set_whitelist(new) } - - fn read_write_count(&self) -> (u32, u32, u32, u32) { - self.backend.read_write_count() - } - - fn reset_read_write_count(&mut self) { - self.backend.reset_read_write_count() - } - - fn set_whitelist(&mut self, new: Vec>) { - self.backend.set_whitelist(new) - } } diff --git a/utils/frame/benchmarking-cli/src/writer.rs b/utils/frame/benchmarking-cli/src/writer.rs index 3c29f79f65bed..5564dbfcd795e 100644 --- a/utils/frame/benchmarking-cli/src/writer.rs +++ b/utils/frame/benchmarking-cli/src/writer.rs @@ -69,8 +69,8 @@ pub fn write_results(file: &mut File, batches: Result, Strin write!(file, " ({} as Weight)\n", extrinsic_time.base.saturating_mul(1000)).unwrap(); extrinsic_time.slopes.iter().zip(extrinsic_time.names.iter()).for_each(|(slope, name)| { write!(file, " .saturating_add(({} as Weight).saturating_mul({} as Weight))\n", - name, slope.saturating_mul(1000), + name, ).unwrap(); }); @@ -78,8 +78,8 @@ pub fn write_results(file: &mut File, batches: Result, Strin write!(file, " .saturating_add(RocksDbWeight::get().reads({} as Weight))\n", reads.base).unwrap(); reads.slopes.iter().zip(reads.names.iter()).for_each(|(slope, name)| { write!(file, " .saturating_add(RocksDbWeight::get().reads(({} as Weight).saturating_mul({} as Weight)))\n", - name, slope, + name, ).unwrap(); }); @@ -87,8 +87,8 @@ pub fn write_results(file: &mut File, batches: Result, Strin write!(file, " .saturating_add(RocksDbWeight::get().writes({} as Weight))\n", writes.base).unwrap(); writes.slopes.iter().zip(writes.names.iter()).for_each(|(slope, name)| { write!(file, " .saturating_add(RocksDbWeight::get().writes(({} as Weight).saturating_mul({} as Weight)))\n", - name, slope, + name, ).unwrap(); }); From ccff2e11b8c4c40d24dcd1258544725d6b8c2bdb Mon Sep 17 00:00:00 2001 From: Shawn Tabrizi Date: Sun, 28 Jun 2020 18:14:38 +0200 Subject: [PATCH 26/36] writer appends to file --- utils/frame/benchmarking-cli/src/writer.rs | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/utils/frame/benchmarking-cli/src/writer.rs b/utils/frame/benchmarking-cli/src/writer.rs index 5564dbfcd795e..e7c3449b2adeb 100644 --- a/utils/frame/benchmarking-cli/src/writer.rs +++ b/utils/frame/benchmarking-cli/src/writer.rs @@ -1,17 +1,21 @@ -use std::fs::File; +use std::fs::{File, OpenOptions}; use std::io::prelude::*; use frame_benchmarking::{BenchmarkBatch, BenchmarkSelector, Analysis}; fn uppercase_first_letter(s: &str) -> String { - let mut c = s.chars(); - match c.next() { - None => String::new(), - Some(f) => f.to_uppercase().collect::() + c.as_str(), - } + let mut c = s.chars(); + match c.next() { + None => String::new(), + Some(f) => f.to_uppercase().collect::() + c.as_str(), + } } pub fn open_file() -> Result { - File::create("benchmarks.rs") + OpenOptions::new() + .create(true) + .write(true) + .append(true) + .open("benchmarks.rs") } pub fn write_results(file: &mut File, batches: Result, String>) -> Result<(), std::io::Error> { From 02f69ff44e6d8b7edc58ee1b3f294d0de506e2c6 Mon Sep 17 00:00:00 2001 From: Shawn Tabrizi Date: Thu, 2 Jul 2020 17:34:35 +0200 Subject: [PATCH 27/36] implement () for balances weight trait --- frame/balances/src/lib.rs | 8 ++++++++ frame/democracy/src/tests.rs | 1 + frame/multisig/src/tests.rs | 1 + frame/staking/src/mock.rs | 1 + 4 files changed, 11 insertions(+) diff --git a/frame/balances/src/lib.rs b/frame/balances/src/lib.rs index fdb7acb66abe4..75a5d9fa54c9a 100644 --- a/frame/balances/src/lib.rs +++ b/frame/balances/src/lib.rs @@ -201,6 +201,14 @@ pub trait BalancesWeight { fn set_balance_killing(u: u32, e: u32, ) -> Weight; } +impl BalancesWeight for () { + fn transfer(_u: u32, _e: u32, ) -> Weight { 100_000_000 } + fn transfer_best_case(_u: u32, _e: u32, ) -> Weight { 100_000_000 } + fn transfer_keep_alive(_u: u32, _e: u32, ) -> Weight { 100_000_000 } + fn set_balance(_u: u32, _e: u32, ) -> Weight { 100_000_000 } + fn set_balance_killing(_u: u32, _e: u32, ) -> Weight { 100_000_000 } +} + pub trait Trait: frame_system::Trait { /// The balance of an account. type Balance: Parameter + Member + AtLeast32BitUnsigned + Codec + Default + Copy + diff --git a/frame/democracy/src/tests.rs b/frame/democracy/src/tests.rs index b92f4bd0760f6..59d1504756266 100644 --- a/frame/democracy/src/tests.rs +++ b/frame/democracy/src/tests.rs @@ -135,6 +135,7 @@ impl pallet_balances::Trait for Test { type DustRemoval = (); type ExistentialDeposit = ExistentialDeposit; type AccountStore = System; + type Weight = (); } parameter_types! { pub const LaunchPeriod: u64 = 2; diff --git a/frame/multisig/src/tests.rs b/frame/multisig/src/tests.rs index 4911ca90cf33d..474399671d6b0 100644 --- a/frame/multisig/src/tests.rs +++ b/frame/multisig/src/tests.rs @@ -94,6 +94,7 @@ impl pallet_balances::Trait for Test { type DustRemoval = (); type ExistentialDeposit = ExistentialDeposit; type AccountStore = System; + type Weight = (); } parameter_types! { pub const DepositBase: u64 = 1; diff --git a/frame/staking/src/mock.rs b/frame/staking/src/mock.rs index 3860dba90f350..979a271b43d1c 100644 --- a/frame/staking/src/mock.rs +++ b/frame/staking/src/mock.rs @@ -231,6 +231,7 @@ impl pallet_balances::Trait for Test { type DustRemoval = (); type ExistentialDeposit = ExistentialDeposit; type AccountStore = System; + type Weight = (); } parameter_types! { pub const Offset: BlockNumber = 0; From aa17f5c86ef8105c0f11646af1f8543b9b7dcdaa Mon Sep 17 00:00:00 2001 From: Shawn Tabrizi Date: Thu, 2 Jul 2020 17:54:50 +0200 Subject: [PATCH 28/36] update name of trait --- bin/node/runtime/src/benchmarks.rs | 3 +-- bin/node/runtime/src/lib.rs | 2 +- frame/balances/src/lib.rs | 20 ++++++++++---------- frame/balances/src/tests_composite.rs | 1 + frame/balances/src/tests_local.rs | 1 + utils/frame/benchmarking-cli/src/writer.rs | 10 ++-------- 6 files changed, 16 insertions(+), 21 deletions(-) diff --git a/bin/node/runtime/src/benchmarks.rs b/bin/node/runtime/src/benchmarks.rs index dc7e666e695ab..5414314757152 100644 --- a/bin/node/runtime/src/benchmarks.rs +++ b/bin/node/runtime/src/benchmarks.rs @@ -1,7 +1,6 @@ use frame_support::weights::{Weight, constants::RocksDbWeight}; -use pallet_balances::BalancesWeight; pub struct WeightForBalances; -impl BalancesWeight for WeightForBalances { +impl pallet_balances::Weights for WeightForBalances { fn transfer(u: u32, e: u32, ) -> Weight { (574875000 as Weight) .saturating_add((u as Weight).saturating_mul(7000 as Weight)) diff --git a/bin/node/runtime/src/lib.rs b/bin/node/runtime/src/lib.rs index 9348cc9f38d30..0377b4011d88d 100644 --- a/bin/node/runtime/src/lib.rs +++ b/bin/node/runtime/src/lib.rs @@ -295,7 +295,7 @@ impl pallet_balances::Trait for Runtime { type Event = Event; type ExistentialDeposit = ExistentialDeposit; type AccountStore = frame_system::Module; - type Weight = benchmarks::WeightForBalances; + type Weights = benchmarks::WeightForBalances; } parameter_types! { diff --git a/frame/balances/src/lib.rs b/frame/balances/src/lib.rs index 75a5d9fa54c9a..f57542bfc6de0 100644 --- a/frame/balances/src/lib.rs +++ b/frame/balances/src/lib.rs @@ -190,10 +190,10 @@ pub trait Subtrait: frame_system::Trait { /// The means of storing the balances of an account. type AccountStore: StoredMap>; - type Weight: BalancesWeight; + type Weights: Weights; } -pub trait BalancesWeight { +pub trait Weights { fn transfer(u: u32, e: u32, ) -> Weight; fn transfer_best_case(u: u32, e: u32, ) -> Weight; fn transfer_keep_alive(u: u32, e: u32, ) -> Weight; @@ -201,7 +201,7 @@ pub trait BalancesWeight { fn set_balance_killing(u: u32, e: u32, ) -> Weight; } -impl BalancesWeight for () { +impl Weights for () { fn transfer(_u: u32, _e: u32, ) -> Weight { 100_000_000 } fn transfer_best_case(_u: u32, _e: u32, ) -> Weight { 100_000_000 } fn transfer_keep_alive(_u: u32, _e: u32, ) -> Weight { 100_000_000 } @@ -226,14 +226,14 @@ pub trait Trait: frame_system::Trait { /// The means of storing the balances of an account. type AccountStore: StoredMap>; - type Weight: BalancesWeight; + type Weights: Weights; } impl, I: Instance> Subtrait for T { type Balance = T::Balance; type ExistentialDeposit = T::ExistentialDeposit; type AccountStore = T::AccountStore; - type Weight = T::Weight; + type Weights = T::Weights; } decl_event!( @@ -459,7 +459,7 @@ decl_module! { /// - DB Weight: 1 Read and 1 Write to destination account /// - Origin account is already in memory, so no DB operations for them. /// # - #[weight = T::Weight::transfer(0,0)] + #[weight = T::Weights::transfer(0,0)] pub fn transfer( origin, dest: ::Source, @@ -488,7 +488,7 @@ decl_module! { /// - Killing: 35.11 µs /// - DB Weight: 1 Read, 1 Write to `who` /// # - #[weight = T::Weight::set_balance(0, 0)] + #[weight = T::Weights::set_balance(0, 0)] fn set_balance( origin, who: ::Source, @@ -531,7 +531,7 @@ decl_module! { /// not assumed to be in the overlay. /// # // TODO: ADD NEW BENCHMARK - #[weight = T::Weight::transfer(0,0)] + #[weight = T::Weights::transfer(0,0)] pub fn force_transfer( origin, source: ::Source, @@ -555,7 +555,7 @@ decl_module! { /// - Base Weight: 51.4 µs /// - DB Weight: 1 Read and 1 Write to dest (sender is in overlay already) /// # - #[weight = T::Weight::transfer_keep_alive(0,0)] + #[weight = T::Weights::transfer_keep_alive(0,0)] pub fn transfer_keep_alive( origin, dest: ::Source, @@ -902,7 +902,7 @@ impl, I: Instance> Trait for ElevatedTrait { type DustRemoval = (); type ExistentialDeposit = T::ExistentialDeposit; type AccountStore = T::AccountStore; - type Weight = T::Weight; + type Weights = T::Weights; } impl, I: Instance> Currency for Module where diff --git a/frame/balances/src/tests_composite.rs b/frame/balances/src/tests_composite.rs index 81cb3449a8237..1d54fb6aafff4 100644 --- a/frame/balances/src/tests_composite.rs +++ b/frame/balances/src/tests_composite.rs @@ -108,6 +108,7 @@ impl Trait for Test { type Event = Event; type ExistentialDeposit = ExistentialDeposit; type AccountStore = system::Module; + type Weights = (); } pub struct ExtBuilder { diff --git a/frame/balances/src/tests_local.rs b/frame/balances/src/tests_local.rs index 54ab22af33c6c..71ec0cdb3686d 100644 --- a/frame/balances/src/tests_local.rs +++ b/frame/balances/src/tests_local.rs @@ -113,6 +113,7 @@ impl Trait for Test { system::CallKillAccount, u64, super::AccountData >; + type Weights = (); } pub struct ExtBuilder { diff --git a/utils/frame/benchmarking-cli/src/writer.rs b/utils/frame/benchmarking-cli/src/writer.rs index e7c3449b2adeb..a9fa211e6558d 100644 --- a/utils/frame/benchmarking-cli/src/writer.rs +++ b/utils/frame/benchmarking-cli/src/writer.rs @@ -38,20 +38,14 @@ pub fn write_results(file: &mut File, batches: Result, Strin write!(file, "}}\n").unwrap(); } - // pallet trait import - write!(file, "use pallet_{}::{}Weight;\n", - pallet_string, - uppercase_first_letter(&pallet_string), - ).unwrap(); - // struct for weights write!(file, "pub struct WeightFor{};\n", uppercase_first_letter(&pallet_string), ).unwrap(); // trait wrapper - write!(file, "impl {}Weight for WeightFor{} {{\n", - uppercase_first_letter(&pallet_string), + write!(file, "impl pallet_{}::Weights for WeightFor{} {{\n", + pallet_string, uppercase_first_letter(&pallet_string), ).unwrap(); From c83c7fb7270d4dcae3743d5310e50b2fc88be6fa Mon Sep 17 00:00:00 2001 From: Shawn Tabrizi Date: Thu, 2 Jul 2020 22:59:14 +0200 Subject: [PATCH 29/36] Weights to WeightInfo --- bin/node/runtime/src/benchmarks.rs | 2 +- bin/node/runtime/src/lib.rs | 2 +- frame/balances/src/lib.rs | 20 ++++++++++---------- frame/balances/src/tests_composite.rs | 2 +- frame/balances/src/tests_local.rs | 2 +- utils/frame/benchmarking-cli/src/command.rs | 6 ++++-- utils/frame/benchmarking-cli/src/lib.rs | 4 ++++ utils/frame/benchmarking-cli/src/writer.rs | 2 +- 8 files changed, 23 insertions(+), 17 deletions(-) diff --git a/bin/node/runtime/src/benchmarks.rs b/bin/node/runtime/src/benchmarks.rs index 5414314757152..d0df6491ffda4 100644 --- a/bin/node/runtime/src/benchmarks.rs +++ b/bin/node/runtime/src/benchmarks.rs @@ -1,6 +1,6 @@ use frame_support::weights::{Weight, constants::RocksDbWeight}; pub struct WeightForBalances; -impl pallet_balances::Weights for WeightForBalances { +impl pallet_balances::WeightInfo for WeightForBalances { fn transfer(u: u32, e: u32, ) -> Weight { (574875000 as Weight) .saturating_add((u as Weight).saturating_mul(7000 as Weight)) diff --git a/bin/node/runtime/src/lib.rs b/bin/node/runtime/src/lib.rs index 0377b4011d88d..5bb534715716d 100644 --- a/bin/node/runtime/src/lib.rs +++ b/bin/node/runtime/src/lib.rs @@ -295,7 +295,7 @@ impl pallet_balances::Trait for Runtime { type Event = Event; type ExistentialDeposit = ExistentialDeposit; type AccountStore = frame_system::Module; - type Weights = benchmarks::WeightForBalances; + type WeightInfo = benchmarks::WeightForBalances; } parameter_types! { diff --git a/frame/balances/src/lib.rs b/frame/balances/src/lib.rs index f57542bfc6de0..1812dc69102b0 100644 --- a/frame/balances/src/lib.rs +++ b/frame/balances/src/lib.rs @@ -190,10 +190,10 @@ pub trait Subtrait: frame_system::Trait { /// The means of storing the balances of an account. type AccountStore: StoredMap>; - type Weights: Weights; + type WeightInfo: WeightInfo; } -pub trait Weights { +pub trait WeightInfo { fn transfer(u: u32, e: u32, ) -> Weight; fn transfer_best_case(u: u32, e: u32, ) -> Weight; fn transfer_keep_alive(u: u32, e: u32, ) -> Weight; @@ -201,7 +201,7 @@ pub trait Weights { fn set_balance_killing(u: u32, e: u32, ) -> Weight; } -impl Weights for () { +impl WeightInfo for () { fn transfer(_u: u32, _e: u32, ) -> Weight { 100_000_000 } fn transfer_best_case(_u: u32, _e: u32, ) -> Weight { 100_000_000 } fn transfer_keep_alive(_u: u32, _e: u32, ) -> Weight { 100_000_000 } @@ -226,14 +226,14 @@ pub trait Trait: frame_system::Trait { /// The means of storing the balances of an account. type AccountStore: StoredMap>; - type Weights: Weights; + type WeightInfo: WeightInfo; } impl, I: Instance> Subtrait for T { type Balance = T::Balance; type ExistentialDeposit = T::ExistentialDeposit; type AccountStore = T::AccountStore; - type Weights = T::Weights; + type WeightInfo = T::WeightInfo; } decl_event!( @@ -459,7 +459,7 @@ decl_module! { /// - DB Weight: 1 Read and 1 Write to destination account /// - Origin account is already in memory, so no DB operations for them. /// # - #[weight = T::Weights::transfer(0,0)] + #[weight = T::WeightInfo::transfer(0,0)] pub fn transfer( origin, dest: ::Source, @@ -488,7 +488,7 @@ decl_module! { /// - Killing: 35.11 µs /// - DB Weight: 1 Read, 1 Write to `who` /// # - #[weight = T::Weights::set_balance(0, 0)] + #[weight = T::WeightInfo::set_balance(0, 0)] fn set_balance( origin, who: ::Source, @@ -531,7 +531,7 @@ decl_module! { /// not assumed to be in the overlay. /// # // TODO: ADD NEW BENCHMARK - #[weight = T::Weights::transfer(0,0)] + #[weight = T::WeightInfo::transfer(0,0)] pub fn force_transfer( origin, source: ::Source, @@ -555,7 +555,7 @@ decl_module! { /// - Base Weight: 51.4 µs /// - DB Weight: 1 Read and 1 Write to dest (sender is in overlay already) /// # - #[weight = T::Weights::transfer_keep_alive(0,0)] + #[weight = T::WeightInfo::transfer_keep_alive(0,0)] pub fn transfer_keep_alive( origin, dest: ::Source, @@ -902,7 +902,7 @@ impl, I: Instance> Trait for ElevatedTrait { type DustRemoval = (); type ExistentialDeposit = T::ExistentialDeposit; type AccountStore = T::AccountStore; - type Weights = T::Weights; + type WeightInfo = T::WeightInfo; } impl, I: Instance> Currency for Module where diff --git a/frame/balances/src/tests_composite.rs b/frame/balances/src/tests_composite.rs index 1d54fb6aafff4..66badd078dbdb 100644 --- a/frame/balances/src/tests_composite.rs +++ b/frame/balances/src/tests_composite.rs @@ -108,7 +108,7 @@ impl Trait for Test { type Event = Event; type ExistentialDeposit = ExistentialDeposit; type AccountStore = system::Module; - type Weights = (); + type WeightInfo = (); } pub struct ExtBuilder { diff --git a/frame/balances/src/tests_local.rs b/frame/balances/src/tests_local.rs index 71ec0cdb3686d..ea125770537b6 100644 --- a/frame/balances/src/tests_local.rs +++ b/frame/balances/src/tests_local.rs @@ -113,7 +113,7 @@ impl Trait for Test { system::CallKillAccount, u64, super::AccountData >; - type Weights = (); + type WeightInfo = (); } pub struct ExtBuilder { diff --git a/utils/frame/benchmarking-cli/src/command.rs b/utils/frame/benchmarking-cli/src/command.rs index b299e9cb5d06b..769bf33a1e8b7 100644 --- a/utils/frame/benchmarking-cli/src/command.rs +++ b/utils/frame/benchmarking-cli/src/command.rs @@ -89,8 +89,10 @@ impl BenchmarkCmd { let results = , String> as Decode>::decode(&mut &result[..]) .map_err(|e| format!("Failed to decode benchmark results: {:?}", e))?; - let mut file = crate::writer::open_file()?; - crate::writer::write_results(&mut file, results.clone())?; + if self.output { + let mut file = crate::writer::open_file()?; + crate::writer::write_results(&mut file, results.clone())?; + } match results { Ok(batches) => for batch in batches.into_iter() { diff --git a/utils/frame/benchmarking-cli/src/lib.rs b/utils/frame/benchmarking-cli/src/lib.rs index 2772490b37e2c..dbf11e77fcf58 100644 --- a/utils/frame/benchmarking-cli/src/lib.rs +++ b/utils/frame/benchmarking-cli/src/lib.rs @@ -60,6 +60,10 @@ pub struct BenchmarkCmd { #[structopt(long)] pub no_min_squares: bool, + /// Output the benchmarks to a Rust file. + #[structopt(long)] + pub output: bool, + #[allow(missing_docs)] #[structopt(flatten)] pub shared_params: sc_cli::SharedParams, diff --git a/utils/frame/benchmarking-cli/src/writer.rs b/utils/frame/benchmarking-cli/src/writer.rs index a9fa211e6558d..9bd491b3b9735 100644 --- a/utils/frame/benchmarking-cli/src/writer.rs +++ b/utils/frame/benchmarking-cli/src/writer.rs @@ -44,7 +44,7 @@ pub fn write_results(file: &mut File, batches: Result, Strin ).unwrap(); // trait wrapper - write!(file, "impl pallet_{}::Weights for WeightFor{} {{\n", + write!(file, "impl pallet_{}::WeightInfo for WeightFor{} {{\n", pallet_string, uppercase_first_letter(&pallet_string), ).unwrap(); From b29eef17543bf0397f432af46f7ee1920d02bdfb Mon Sep 17 00:00:00 2001 From: Shawn Tabrizi Date: Thu, 2 Jul 2020 23:50:04 +0200 Subject: [PATCH 30/36] auto trait writer --- frame/balances/src/lib.rs | 10 +-- utils/frame/benchmarking-cli/src/command.rs | 9 ++- utils/frame/benchmarking-cli/src/lib.rs | 4 + utils/frame/benchmarking-cli/src/writer.rs | 82 ++++++++++++++++++++- 4 files changed, 96 insertions(+), 9 deletions(-) diff --git a/frame/balances/src/lib.rs b/frame/balances/src/lib.rs index 1812dc69102b0..d133691d8fc94 100644 --- a/frame/balances/src/lib.rs +++ b/frame/balances/src/lib.rs @@ -202,11 +202,11 @@ pub trait WeightInfo { } impl WeightInfo for () { - fn transfer(_u: u32, _e: u32, ) -> Weight { 100_000_000 } - fn transfer_best_case(_u: u32, _e: u32, ) -> Weight { 100_000_000 } - fn transfer_keep_alive(_u: u32, _e: u32, ) -> Weight { 100_000_000 } - fn set_balance(_u: u32, _e: u32, ) -> Weight { 100_000_000 } - fn set_balance_killing(_u: u32, _e: u32, ) -> Weight { 100_000_000 } + fn transfer(_u: u32, _e: u32, ) -> Weight { 1_000_000_000 } + fn transfer_best_case(_u: u32, _e: u32, ) -> Weight { 1_000_000_000 } + fn transfer_keep_alive(_u: u32, _e: u32, ) -> Weight { 1_000_000_000 } + fn set_balance(_u: u32, _e: u32, ) -> Weight { 1_000_000_000 } + fn set_balance_killing(_u: u32, _e: u32, ) -> Weight { 1_000_000_000 } } pub trait Trait: frame_system::Trait { diff --git a/utils/frame/benchmarking-cli/src/command.rs b/utils/frame/benchmarking-cli/src/command.rs index 769bf33a1e8b7..3c948da483aaf 100644 --- a/utils/frame/benchmarking-cli/src/command.rs +++ b/utils/frame/benchmarking-cli/src/command.rs @@ -90,8 +90,13 @@ impl BenchmarkCmd { .map_err(|e| format!("Failed to decode benchmark results: {:?}", e))?; if self.output { - let mut file = crate::writer::open_file()?; - crate::writer::write_results(&mut file, results.clone())?; + if self.weight_trait { + let mut file = crate::writer::open_file("traits.rs")?; + crate::writer::write_trait(&mut file, results.clone())?; + } else { + let mut file = crate::writer::open_file("benchmarks.rs")?; + crate::writer::write_results(&mut file, results.clone())?; + } } match results { diff --git a/utils/frame/benchmarking-cli/src/lib.rs b/utils/frame/benchmarking-cli/src/lib.rs index dbf11e77fcf58..2bef5457626a8 100644 --- a/utils/frame/benchmarking-cli/src/lib.rs +++ b/utils/frame/benchmarking-cli/src/lib.rs @@ -64,6 +64,10 @@ pub struct BenchmarkCmd { #[structopt(long)] pub output: bool, + /// Output the trait definition to a Rust file. + #[structopt(long)] + pub weight_trait: bool, + #[allow(missing_docs)] #[structopt(flatten)] pub shared_params: sc_cli::SharedParams, diff --git a/utils/frame/benchmarking-cli/src/writer.rs b/utils/frame/benchmarking-cli/src/writer.rs index 9bd491b3b9735..8b2d6683842e2 100644 --- a/utils/frame/benchmarking-cli/src/writer.rs +++ b/utils/frame/benchmarking-cli/src/writer.rs @@ -10,12 +10,90 @@ fn uppercase_first_letter(s: &str) -> String { } } -pub fn open_file() -> Result { +pub fn open_file(path: &str) -> Result { OpenOptions::new() .create(true) .write(true) .append(true) - .open("benchmarks.rs") + .open(path) +} + +pub fn write_trait(file: &mut File, batches: Result, String>) -> Result<(), std::io::Error> { + let batches = batches.unwrap(); + + let mut current_pallet = Vec::::new(); + + batches.iter().for_each(|batch| { + + let pallet_string = String::from_utf8(batch.pallet.clone()).unwrap(); + let benchmark_string = String::from_utf8(batch.benchmark.clone()).unwrap(); + + // only create new trait definitions when we go to a new pallet + if batch.pallet != current_pallet { + if !current_pallet.is_empty() { + // close trait + write!(file, "}}\n").unwrap(); + } + + // trait wrapper + write!(file, "// {}\n", pallet_string).unwrap(); + write!(file, "pub trait WeightInfo {{\n").unwrap(); + + current_pallet = batch.pallet.clone() + } + + // function name + write!(file, " fn {}(", benchmark_string).unwrap(); + + // params + let components = &batch.results[0].components; + for component in components { + write!(file, "{:?}: u32, ", component.0).unwrap(); + } + // return value + write!(file, ") -> Weight;\n").unwrap(); + }); + + // final close trait + write!(file, "}}\n").unwrap(); + + // Reset + current_pallet = Vec::::new(); + + batches.iter().for_each(|batch| { + + let benchmark_string = String::from_utf8(batch.benchmark.clone()).unwrap(); + + // only create new trait definitions when we go to a new pallet + if batch.pallet != current_pallet { + if !current_pallet.is_empty() { + // close trait + write!(file, "}}\n").unwrap(); + } + + // impl trait + write!(file, "\n").unwrap(); + write!(file, "impl WeightInfo for () {{\n").unwrap(); + + current_pallet = batch.pallet.clone() + } + + // function name + write!(file, " fn {}(", benchmark_string).unwrap(); + + // params + let components = &batch.results[0].components; + for component in components { + write!(file, "_{:?}: u32, ", component.0).unwrap(); + } + // return value + write!(file, ") -> Weight {{ 1_000_000_000 }}\n").unwrap(); + }); + + // final close trait + write!(file, "}}\n").unwrap(); + + Ok(()) } pub fn write_results(file: &mut File, batches: Result, String>) -> Result<(), std::io::Error> { From 102a4632ac12e3fb6e1f0546353249d8905151e9 Mon Sep 17 00:00:00 2001 From: Shawn Tabrizi Date: Fri, 3 Jul 2020 15:44:28 +0200 Subject: [PATCH 31/36] Heap pages are configurable --- utils/frame/benchmarking-cli/src/command.rs | 2 +- utils/frame/benchmarking-cli/src/lib.rs | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/utils/frame/benchmarking-cli/src/command.rs b/utils/frame/benchmarking-cli/src/command.rs index 3c948da483aaf..09b246e4766a3 100644 --- a/utils/frame/benchmarking-cli/src/command.rs +++ b/utils/frame/benchmarking-cli/src/command.rs @@ -55,7 +55,7 @@ impl BenchmarkCmd { let state = BenchmarkingState::::new(genesis_storage, cache_size)?; let executor = NativeExecutor::::new( wasm_method, - None, // heap pages + self.heap_pages, 2, // The runtime instances cache size. ); diff --git a/utils/frame/benchmarking-cli/src/lib.rs b/utils/frame/benchmarking-cli/src/lib.rs index 2bef5457626a8..8a53c9fd8b16b 100644 --- a/utils/frame/benchmarking-cli/src/lib.rs +++ b/utils/frame/benchmarking-cli/src/lib.rs @@ -68,6 +68,10 @@ pub struct BenchmarkCmd { #[structopt(long)] pub weight_trait: bool, + /// Set the heap pages while running benchmarks. + #[structopt(long)] + pub heap_pages: Option, + #[allow(missing_docs)] #[structopt(flatten)] pub shared_params: sc_cli::SharedParams, From 8456e1262632c1eb1b9841382d4e35811195b1eb Mon Sep 17 00:00:00 2001 From: Shawn Tabrizi Date: Fri, 3 Jul 2020 17:17:14 +0200 Subject: [PATCH 32/36] clean out runtime changes --- bin/node/runtime/src/benchmarks.rs | 59 --------------------------- bin/node/runtime/src/lib.rs | 3 -- frame/balances/src/lib.rs | 32 ++------------- frame/balances/src/tests_composite.rs | 1 - frame/balances/src/tests_local.rs | 1 - 5 files changed, 4 insertions(+), 92 deletions(-) delete mode 100644 bin/node/runtime/src/benchmarks.rs diff --git a/bin/node/runtime/src/benchmarks.rs b/bin/node/runtime/src/benchmarks.rs deleted file mode 100644 index d0df6491ffda4..0000000000000 --- a/bin/node/runtime/src/benchmarks.rs +++ /dev/null @@ -1,59 +0,0 @@ -use frame_support::weights::{Weight, constants::RocksDbWeight}; -pub struct WeightForBalances; -impl pallet_balances::WeightInfo for WeightForBalances { - fn transfer(u: u32, e: u32, ) -> Weight { - (574875000 as Weight) - .saturating_add((u as Weight).saturating_mul(7000 as Weight)) - .saturating_add((e as Weight).saturating_mul(0 as Weight)) - .saturating_add(RocksDbWeight::get().reads(2 as Weight)) - .saturating_add(RocksDbWeight::get().reads((u as Weight).saturating_mul(0 as Weight))) - .saturating_add(RocksDbWeight::get().reads((e as Weight).saturating_mul(0 as Weight))) - .saturating_add(RocksDbWeight::get().writes(2 as Weight)) - .saturating_add(RocksDbWeight::get().writes((u as Weight).saturating_mul(0 as Weight))) - .saturating_add(RocksDbWeight::get().writes((e as Weight).saturating_mul(0 as Weight))) - } - fn transfer_best_case(u: u32, e: u32, ) -> Weight { - (368265000 as Weight) - .saturating_add((u as Weight).saturating_mul(26000 as Weight)) - .saturating_add((e as Weight).saturating_mul(0 as Weight)) - .saturating_add(RocksDbWeight::get().reads(2 as Weight)) - .saturating_add(RocksDbWeight::get().reads((u as Weight).saturating_mul(0 as Weight))) - .saturating_add(RocksDbWeight::get().reads((e as Weight).saturating_mul(0 as Weight))) - .saturating_add(RocksDbWeight::get().writes(2 as Weight)) - .saturating_add(RocksDbWeight::get().writes((u as Weight).saturating_mul(0 as Weight))) - .saturating_add(RocksDbWeight::get().writes((e as Weight).saturating_mul(0 as Weight))) - } - fn transfer_keep_alive(u: u32, e: u32, ) -> Weight { - (402076000 as Weight) - .saturating_add((u as Weight).saturating_mul(7000 as Weight)) - .saturating_add((e as Weight).saturating_mul(14000 as Weight)) - .saturating_add(RocksDbWeight::get().reads(2 as Weight)) - .saturating_add(RocksDbWeight::get().reads((u as Weight).saturating_mul(0 as Weight))) - .saturating_add(RocksDbWeight::get().reads((e as Weight).saturating_mul(0 as Weight))) - .saturating_add(RocksDbWeight::get().writes(2 as Weight)) - .saturating_add(RocksDbWeight::get().writes((u as Weight).saturating_mul(0 as Weight))) - .saturating_add(RocksDbWeight::get().writes((e as Weight).saturating_mul(0 as Weight))) - } - fn set_balance(u: u32, e: u32, ) -> Weight { - (341092000 as Weight) - .saturating_add((u as Weight).saturating_mul(0 as Weight)) - .saturating_add((e as Weight).saturating_mul(12000 as Weight)) - .saturating_add(RocksDbWeight::get().reads(1 as Weight)) - .saturating_add(RocksDbWeight::get().reads((u as Weight).saturating_mul(0 as Weight))) - .saturating_add(RocksDbWeight::get().reads((e as Weight).saturating_mul(0 as Weight))) - .saturating_add(RocksDbWeight::get().writes(1 as Weight)) - .saturating_add(RocksDbWeight::get().writes((u as Weight).saturating_mul(0 as Weight))) - .saturating_add(RocksDbWeight::get().writes((e as Weight).saturating_mul(0 as Weight))) - } - fn set_balance_killing(u: u32, e: u32, ) -> Weight { - (412116000 as Weight) - .saturating_add((u as Weight).saturating_mul(28000 as Weight)) - .saturating_add((e as Weight).saturating_mul(0 as Weight)) - .saturating_add(RocksDbWeight::get().reads(1 as Weight)) - .saturating_add(RocksDbWeight::get().reads((u as Weight).saturating_mul(0 as Weight))) - .saturating_add(RocksDbWeight::get().reads((e as Weight).saturating_mul(0 as Weight))) - .saturating_add(RocksDbWeight::get().writes(1 as Weight)) - .saturating_add(RocksDbWeight::get().writes((u as Weight).saturating_mul(0 as Weight))) - .saturating_add(RocksDbWeight::get().writes((e as Weight).saturating_mul(0 as Weight))) - } -} diff --git a/bin/node/runtime/src/lib.rs b/bin/node/runtime/src/lib.rs index 5bb534715716d..7bec203f8c446 100644 --- a/bin/node/runtime/src/lib.rs +++ b/bin/node/runtime/src/lib.rs @@ -84,8 +84,6 @@ use impls::{CurrencyToVoteHandler, Author}; pub mod constants; use constants::{time::*, currency::*}; -pub mod benchmarks; - // Make the WASM binary available. #[cfg(feature = "std")] include!(concat!(env!("OUT_DIR"), "/wasm_binary.rs")); @@ -295,7 +293,6 @@ impl pallet_balances::Trait for Runtime { type Event = Event; type ExistentialDeposit = ExistentialDeposit; type AccountStore = frame_system::Module; - type WeightInfo = benchmarks::WeightForBalances; } parameter_types! { diff --git a/frame/balances/src/lib.rs b/frame/balances/src/lib.rs index d133691d8fc94..62402c78630c7 100644 --- a/frame/balances/src/lib.rs +++ b/frame/balances/src/lib.rs @@ -159,7 +159,6 @@ use sp_std::prelude::*; use sp_std::{cmp, result, mem, fmt::Debug, ops::BitOr, convert::Infallible}; use codec::{Codec, Encode, Decode}; use frame_support::{ - weights::Weight, StorageValue, Parameter, decl_event, decl_storage, decl_module, decl_error, ensure, traits::{ Currency, OnKilledAccount, OnUnbalanced, TryDrop, StoredMap, @@ -189,24 +188,6 @@ pub trait Subtrait: frame_system::Trait { /// The means of storing the balances of an account. type AccountStore: StoredMap>; - - type WeightInfo: WeightInfo; -} - -pub trait WeightInfo { - fn transfer(u: u32, e: u32, ) -> Weight; - fn transfer_best_case(u: u32, e: u32, ) -> Weight; - fn transfer_keep_alive(u: u32, e: u32, ) -> Weight; - fn set_balance(u: u32, e: u32, ) -> Weight; - fn set_balance_killing(u: u32, e: u32, ) -> Weight; -} - -impl WeightInfo for () { - fn transfer(_u: u32, _e: u32, ) -> Weight { 1_000_000_000 } - fn transfer_best_case(_u: u32, _e: u32, ) -> Weight { 1_000_000_000 } - fn transfer_keep_alive(_u: u32, _e: u32, ) -> Weight { 1_000_000_000 } - fn set_balance(_u: u32, _e: u32, ) -> Weight { 1_000_000_000 } - fn set_balance_killing(_u: u32, _e: u32, ) -> Weight { 1_000_000_000 } } pub trait Trait: frame_system::Trait { @@ -225,15 +206,12 @@ pub trait Trait: frame_system::Trait { /// The means of storing the balances of an account. type AccountStore: StoredMap>; - - type WeightInfo: WeightInfo; } impl, I: Instance> Subtrait for T { type Balance = T::Balance; type ExistentialDeposit = T::ExistentialDeposit; type AccountStore = T::AccountStore; - type WeightInfo = T::WeightInfo; } decl_event!( @@ -459,7 +437,7 @@ decl_module! { /// - DB Weight: 1 Read and 1 Write to destination account /// - Origin account is already in memory, so no DB operations for them. /// # - #[weight = T::WeightInfo::transfer(0,0)] + #[weight = T::DbWeight::get().reads_writes(1, 1) + 70_000_000] pub fn transfer( origin, dest: ::Source, @@ -488,7 +466,7 @@ decl_module! { /// - Killing: 35.11 µs /// - DB Weight: 1 Read, 1 Write to `who` /// # - #[weight = T::WeightInfo::set_balance(0, 0)] + #[weight = T::DbWeight::get().reads_writes(1, 1) + 35_000_000] fn set_balance( origin, who: ::Source, @@ -530,8 +508,7 @@ decl_module! { /// - Same as transfer, but additional read and write because the source account is /// not assumed to be in the overlay. /// # - // TODO: ADD NEW BENCHMARK - #[weight = T::WeightInfo::transfer(0,0)] + #[weight = T::DbWeight::get().reads_writes(2, 2) + 70_000_000] pub fn force_transfer( origin, source: ::Source, @@ -555,7 +532,7 @@ decl_module! { /// - Base Weight: 51.4 µs /// - DB Weight: 1 Read and 1 Write to dest (sender is in overlay already) /// # - #[weight = T::WeightInfo::transfer_keep_alive(0,0)] + #[weight = T::DbWeight::get().reads_writes(1, 1) + 50_000_000] pub fn transfer_keep_alive( origin, dest: ::Source, @@ -902,7 +879,6 @@ impl, I: Instance> Trait for ElevatedTrait { type DustRemoval = (); type ExistentialDeposit = T::ExistentialDeposit; type AccountStore = T::AccountStore; - type WeightInfo = T::WeightInfo; } impl, I: Instance> Currency for Module where diff --git a/frame/balances/src/tests_composite.rs b/frame/balances/src/tests_composite.rs index 66badd078dbdb..81cb3449a8237 100644 --- a/frame/balances/src/tests_composite.rs +++ b/frame/balances/src/tests_composite.rs @@ -108,7 +108,6 @@ impl Trait for Test { type Event = Event; type ExistentialDeposit = ExistentialDeposit; type AccountStore = system::Module; - type WeightInfo = (); } pub struct ExtBuilder { diff --git a/frame/balances/src/tests_local.rs b/frame/balances/src/tests_local.rs index ea125770537b6..54ab22af33c6c 100644 --- a/frame/balances/src/tests_local.rs +++ b/frame/balances/src/tests_local.rs @@ -113,7 +113,6 @@ impl Trait for Test { system::CallKillAccount, u64, super::AccountData >; - type WeightInfo = (); } pub struct ExtBuilder { From 0b22d503310e1e68f6f96f480ba2e88f1e3d7075 Mon Sep 17 00:00:00 2001 From: Shawn Tabrizi Date: Fri, 3 Jul 2020 17:18:16 +0200 Subject: [PATCH 33/36] more clean up --- frame/democracy/src/tests.rs | 1 - frame/multisig/src/tests.rs | 1 - frame/staking/src/mock.rs | 1 - 3 files changed, 3 deletions(-) diff --git a/frame/democracy/src/tests.rs b/frame/democracy/src/tests.rs index ea379808893d3..2f300ec8bc61d 100644 --- a/frame/democracy/src/tests.rs +++ b/frame/democracy/src/tests.rs @@ -137,7 +137,6 @@ impl pallet_balances::Trait for Test { type DustRemoval = (); type ExistentialDeposit = ExistentialDeposit; type AccountStore = System; - type Weight = (); } parameter_types! { pub const LaunchPeriod: u64 = 2; diff --git a/frame/multisig/src/tests.rs b/frame/multisig/src/tests.rs index 474399671d6b0..4911ca90cf33d 100644 --- a/frame/multisig/src/tests.rs +++ b/frame/multisig/src/tests.rs @@ -94,7 +94,6 @@ impl pallet_balances::Trait for Test { type DustRemoval = (); type ExistentialDeposit = ExistentialDeposit; type AccountStore = System; - type Weight = (); } parameter_types! { pub const DepositBase: u64 = 1; diff --git a/frame/staking/src/mock.rs b/frame/staking/src/mock.rs index 979a271b43d1c..3860dba90f350 100644 --- a/frame/staking/src/mock.rs +++ b/frame/staking/src/mock.rs @@ -231,7 +231,6 @@ impl pallet_balances::Trait for Test { type DustRemoval = (); type ExistentialDeposit = ExistentialDeposit; type AccountStore = System; - type Weight = (); } parameter_types! { pub const Offset: BlockNumber = 0; From ace2d6d66cea2b213540bd610190b6e71e1d7418 Mon Sep 17 00:00:00 2001 From: Shawn Tabrizi Date: Fri, 3 Jul 2020 19:18:49 +0200 Subject: [PATCH 34/36] Fix string generation --- Cargo.lock | 1 + bin/node/runtime/src/lib.rs | 36 +++++++++++----------- frame/benchmarking/src/lib.rs | 9 +++--- utils/frame/benchmarking-cli/Cargo.toml | 1 + utils/frame/benchmarking-cli/src/writer.rs | 25 ++++++--------- 5 files changed, 34 insertions(+), 38 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9ea9b890570c0..e565954feb2f8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1450,6 +1450,7 @@ dependencies = [ name = "frame-benchmarking-cli" version = "2.0.0-rc4" dependencies = [ + "Inflector", "frame-benchmarking", "parity-scale-codec", "sc-cli", diff --git a/bin/node/runtime/src/lib.rs b/bin/node/runtime/src/lib.rs index 7bec203f8c446..65b0bd187a81f 100644 --- a/bin/node/runtime/src/lib.rs +++ b/bin/node/runtime/src/lib.rs @@ -1099,24 +1099,24 @@ impl_runtime_apis! { let mut batches = Vec::::new(); let params = (&pallet, &benchmark, &lowest_range_values, &highest_range_values, &steps, repeat, &whitelist); - add_benchmark!(params, batches, b"balances", Balances); - add_benchmark!(params, batches, b"collective", Council); - add_benchmark!(params, batches, b"democracy", Democracy); - add_benchmark!(params, batches, b"elections", Elections); - add_benchmark!(params, batches, b"identity", Identity); - add_benchmark!(params, batches, b"im-online", ImOnline); - add_benchmark!(params, batches, b"indices", Indices); - add_benchmark!(params, batches, b"multisig", Multisig); - add_benchmark!(params, batches, b"offences", OffencesBench::); - add_benchmark!(params, batches, b"proxy", Proxy); - add_benchmark!(params, batches, b"scheduler", Scheduler); - add_benchmark!(params, batches, b"session", SessionBench::); - add_benchmark!(params, batches, b"staking", Staking); - add_benchmark!(params, batches, b"system", SystemBench::); - add_benchmark!(params, batches, b"timestamp", Timestamp); - add_benchmark!(params, batches, b"treasury", Treasury); - add_benchmark!(params, batches, b"utility", Utility); - add_benchmark!(params, batches, b"vesting", Vesting); + add_benchmark!(params, batches, pallet_balances, Balances); + add_benchmark!(params, batches, pallet_collective, Council); + add_benchmark!(params, batches, pallet_democracy, Democracy); + add_benchmark!(params, batches, pallet_elections_phragmen, Elections); + add_benchmark!(params, batches, pallet_identity, Identity); + add_benchmark!(params, batches, pallet_im_online, ImOnline); + add_benchmark!(params, batches, pallet_indices, Indices); + add_benchmark!(params, batches, pallet_multisig, Multisig); + add_benchmark!(params, batches, pallet_offences, OffencesBench::); + add_benchmark!(params, batches, pallet_proxy, Proxy); + add_benchmark!(params, batches, pallet_scheduler, Scheduler); + add_benchmark!(params, batches, pallet_session, SessionBench::); + add_benchmark!(params, batches, pallet_staking, Staking); + add_benchmark!(params, batches, frame_system, SystemBench::); + add_benchmark!(params, batches, pallet_timestamp, Timestamp); + add_benchmark!(params, batches, pallet_treasury, Treasury); + add_benchmark!(params, batches, pallet_utility, Utility); + add_benchmark!(params, batches, pallet_vesting, Vesting); if batches.is_empty() { return Err("Benchmark not found for this pallet.".into()) } Ok(batches) diff --git a/frame/benchmarking/src/lib.rs b/frame/benchmarking/src/lib.rs index 7a7848305a01c..e85ec6711ebc6 100644 --- a/frame/benchmarking/src/lib.rs +++ b/frame/benchmarking/src/lib.rs @@ -1180,9 +1180,10 @@ macro_rules! impl_benchmark_test { /// At the end of `dispatch_benchmark`, you should return this batches object. #[macro_export] macro_rules! add_benchmark { - ( $params:ident, $batches:ident, $name:literal, $( $location:tt )* ) => ( + ( $params:ident, $batches:ident, $name:ident, $( $location:tt )* ) => ( + let name_string = stringify!($name).as_bytes(); let (pallet, benchmark, lowest_range_values, highest_range_values, steps, repeat, whitelist) = $params; - if &pallet[..] == &$name[..] || &pallet[..] == &b"*"[..] { + if &pallet[..] == &name_string[..] || &pallet[..] == &b"*"[..] { if &pallet[..] == &b"*"[..] || &benchmark[..] == &b"*"[..] { for benchmark in $( $location )*::benchmarks().into_iter() { $batches.push($crate::BenchmarkBatch { @@ -1194,7 +1195,7 @@ macro_rules! add_benchmark { repeat, whitelist, )?, - pallet: $name.to_vec(), + pallet: name_string.to_vec(), benchmark: benchmark.to_vec(), }); } @@ -1208,7 +1209,7 @@ macro_rules! add_benchmark { repeat, whitelist, )?, - pallet: $name.to_vec(), + pallet: name_string.to_vec(), benchmark: benchmark.clone(), }); } diff --git a/utils/frame/benchmarking-cli/Cargo.toml b/utils/frame/benchmarking-cli/Cargo.toml index 003b4d9c05b3e..db620c86ca98e 100644 --- a/utils/frame/benchmarking-cli/Cargo.toml +++ b/utils/frame/benchmarking-cli/Cargo.toml @@ -12,6 +12,7 @@ description = "CLI for benchmarking FRAME" targets = ["x86_64-unknown-linux-gnu"] [dependencies] +Inflector = "0.11.4" frame-benchmarking = { version = "2.0.0-rc4", path = "../../../frame/benchmarking" } sp-core = { version = "2.0.0-rc4", path = "../../../primitives/core" } sc-service = { version = "0.8.0-rc4", default-features = false, path = "../../../client/service" } diff --git a/utils/frame/benchmarking-cli/src/writer.rs b/utils/frame/benchmarking-cli/src/writer.rs index 8b2d6683842e2..85d831b0f1101 100644 --- a/utils/frame/benchmarking-cli/src/writer.rs +++ b/utils/frame/benchmarking-cli/src/writer.rs @@ -1,14 +1,7 @@ use std::fs::{File, OpenOptions}; use std::io::prelude::*; use frame_benchmarking::{BenchmarkBatch, BenchmarkSelector, Analysis}; - -fn uppercase_first_letter(s: &str) -> String { - let mut c = s.chars(); - match c.next() { - None => String::new(), - Some(f) => f.to_uppercase().collect::() + c.as_str(), - } -} +use inflector::Inflector; pub fn open_file(path: &str) -> Result { OpenOptions::new() @@ -102,7 +95,7 @@ pub fn write_results(file: &mut File, batches: Result, Strin let mut current_pallet = Vec::::new(); // general imports - write!(file, "use frame_support::weights::{{Weight, constants::RocksDbWeight}};\n").unwrap(); + write!(file, "use frame_support::weights::{{Weight, constants::RocksDbWeight as DbWeight}};\n").unwrap(); batches.iter().for_each(|batch| { @@ -118,13 +111,13 @@ pub fn write_results(file: &mut File, batches: Result, Strin // struct for weights write!(file, "pub struct WeightFor{};\n", - uppercase_first_letter(&pallet_string), + pallet_string.to_pascal_case(), ).unwrap(); // trait wrapper - write!(file, "impl pallet_{}::WeightInfo for WeightFor{} {{\n", + write!(file, "impl {}::WeightInfo for WeightFor{} {{\n", pallet_string, - uppercase_first_letter(&pallet_string), + pallet_string.to_pascal_case(), ).unwrap(); current_pallet = batch.pallet.clone() @@ -151,18 +144,18 @@ pub fn write_results(file: &mut File, batches: Result, Strin }); let reads = Analysis::min_squares_iqr(&batch.results, BenchmarkSelector::Reads).unwrap(); - write!(file, " .saturating_add(RocksDbWeight::get().reads({} as Weight))\n", reads.base).unwrap(); + write!(file, " .saturating_add(DbWeight::get().reads({} as Weight))\n", reads.base).unwrap(); reads.slopes.iter().zip(reads.names.iter()).for_each(|(slope, name)| { - write!(file, " .saturating_add(RocksDbWeight::get().reads(({} as Weight).saturating_mul({} as Weight)))\n", + write!(file, " .saturating_add(DbWeight::get().reads(({} as Weight).saturating_mul({} as Weight)))\n", slope, name, ).unwrap(); }); let writes = Analysis::min_squares_iqr(&batch.results, BenchmarkSelector::Writes).unwrap(); - write!(file, " .saturating_add(RocksDbWeight::get().writes({} as Weight))\n", writes.base).unwrap(); + write!(file, " .saturating_add(DbWeight::get().writes({} as Weight))\n", writes.base).unwrap(); writes.slopes.iter().zip(writes.names.iter()).for_each(|(slope, name)| { - write!(file, " .saturating_add(RocksDbWeight::get().writes(({} as Weight).saturating_mul({} as Weight)))\n", + write!(file, " .saturating_add(DbWeight::get().writes(({} as Weight).saturating_mul({} as Weight)))\n", slope, name, ).unwrap(); From 761626b98e1036621dc09d214cca53de9ff3763e Mon Sep 17 00:00:00 2001 From: Shawn Tabrizi Date: Fri, 3 Jul 2020 19:29:28 +0200 Subject: [PATCH 35/36] Update comments --- bin/node/runtime/src/lib.rs | 1 - frame/benchmarking/src/lib.rs | 26 +++++++++++++++++----- utils/frame/benchmarking-cli/src/writer.rs | 19 ++++++++++++++++ 3 files changed, 39 insertions(+), 7 deletions(-) diff --git a/bin/node/runtime/src/lib.rs b/bin/node/runtime/src/lib.rs index 65b0bd187a81f..e127a70cc104a 100644 --- a/bin/node/runtime/src/lib.rs +++ b/bin/node/runtime/src/lib.rs @@ -1080,7 +1080,6 @@ impl_runtime_apis! { let whitelist: Vec> = vec![ // Block Number - // frame_system::Number::::hashed_key().to_vec(), hex_literal::hex!("26aa394eea5630e07c48ae0c9558cef702a5c1b19ab7a04f536c519aca4983ac").to_vec(), // Total Issuance hex_literal::hex!("c2261276cc9d1f8598ea4b6a74b15c2f57c875e4cff74148e4628f264b974c80").to_vec(), diff --git a/frame/benchmarking/src/lib.rs b/frame/benchmarking/src/lib.rs index e85ec6711ebc6..532cb273c989d 100644 --- a/frame/benchmarking/src/lib.rs +++ b/frame/benchmarking/src/lib.rs @@ -1158,22 +1158,36 @@ macro_rules! impl_benchmark_test { /// First create an object that holds in the input parameters for the benchmark: /// /// ```ignore -/// let params = (&pallet, &benchmark, &lowest_range_values, &highest_range_values, &steps, repeat); +/// let params = (&pallet, &benchmark, &lowest_range_values, &highest_range_values, &steps, repeat, &whitelist); /// ``` /// +/// The `whitelist` is a `Vec>` of storage keys that you would like to skip for DB tracking. For example: +/// +/// ```ignore +/// let whitelist: Vec> = vec![ +/// // Block Number +/// hex_literal::hex!("26aa394eea5630e07c48ae0c9558cef702a5c1b19ab7a04f536c519aca4983ac").to_vec(), +/// // Total Issuance +/// hex_literal::hex!("c2261276cc9d1f8598ea4b6a74b15c2f57c875e4cff74148e4628f264b974c80").to_vec(), +/// // Execution Phase +/// hex_literal::hex!("26aa394eea5630e07c48ae0c9558cef7ff553b5a9862a516939d82b3d3d8661a").to_vec(), +/// // Event Count +/// hex_literal::hex!("26aa394eea5630e07c48ae0c9558cef70a98fdbe9ce6c55837576c60c7af3850").to_vec(), +/// ]; +/// /// Then define a mutable local variable to hold your `BenchmarkBatch` object: /// /// ```ignore /// let mut batches = Vec::::new(); /// ```` /// -/// Then add the pallets you want to benchmark to this object, including the string -/// you want to use target a particular pallet: +/// Then add the pallets you want to benchmark to this object, using their crate name and generated +/// module struct: /// /// ```ignore -/// add_benchmark!(params, batches, b"balances", Balances); -/// add_benchmark!(params, batches, b"identity", Identity); -/// add_benchmark!(params, batches, b"session", SessionBench::); +/// add_benchmark!(params, batches, pallet_balances, Balances); +/// add_benchmark!(params, batches, pallet_session, SessionBench::); +/// add_benchmark!(params, batches, frame_system, SystemBench::); /// ... /// ``` /// diff --git a/utils/frame/benchmarking-cli/src/writer.rs b/utils/frame/benchmarking-cli/src/writer.rs index 85d831b0f1101..bd411b536a826 100644 --- a/utils/frame/benchmarking-cli/src/writer.rs +++ b/utils/frame/benchmarking-cli/src/writer.rs @@ -1,3 +1,22 @@ +// This file is part of Substrate. + +// Copyright (C) 2020 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Outputs benchmark results to Rust files that can be ingested by the runtime. + use std::fs::{File, OpenOptions}; use std::io::prelude::*; use frame_benchmarking::{BenchmarkBatch, BenchmarkSelector, Analysis}; From eb0ae88bf976b2e19e154b23d8bcabcbd6719858 Mon Sep 17 00:00:00 2001 From: Shawn Tabrizi Date: Mon, 6 Jul 2020 02:28:20 +0200 Subject: [PATCH 36/36] Update bin/node/runtime/src/lib.rs --- bin/node/runtime/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/node/runtime/src/lib.rs b/bin/node/runtime/src/lib.rs index 3c95e00b0c02f..baf5f12b2f54e 100644 --- a/bin/node/runtime/src/lib.rs +++ b/bin/node/runtime/src/lib.rs @@ -1136,7 +1136,7 @@ impl_runtime_apis! { let mut batches = Vec::::new(); let params = (&pallet, &benchmark, &lowest_range_values, &highest_range_values, &steps, repeat, &whitelist); - add_benchmark!(params, batches, pallet_babe, Babe); + add_benchmark!(params, batches, pallet_babe, Babe); add_benchmark!(params, batches, pallet_balances, Balances); add_benchmark!(params, batches, pallet_collective, Council); add_benchmark!(params, batches, pallet_democracy, Democracy);