From 0f8dbeb2f892796ba2d3c63b4f3dc608c1905af1 Mon Sep 17 00:00:00 2001 From: Szegoo Date: Sat, 5 Nov 2022 10:41:23 +0100 Subject: [PATCH 01/15] move Metric --- client/sysinfo/src/lib.rs | 2 +- client/sysinfo/src/sysinfo.rs | 39 +++++++++++++++++- .../benchmarking-cli/src/machine/hardware.rs | 41 +------------------ .../frame/benchmarking-cli/src/machine/mod.rs | 4 +- 4 files changed, 42 insertions(+), 44 deletions(-) diff --git a/client/sysinfo/src/lib.rs b/client/sysinfo/src/lib.rs index 372c7c1a6d219..2b32239ec9afd 100644 --- a/client/sysinfo/src/lib.rs +++ b/client/sysinfo/src/lib.rs @@ -29,7 +29,7 @@ mod sysinfo_linux; pub use sysinfo::{ benchmark_cpu, benchmark_disk_random_writes, benchmark_disk_sequential_writes, benchmark_memory, benchmark_sr25519_verify, gather_hwbench, gather_sysinfo, - serialize_throughput, serialize_throughput_option, Throughput, + serialize_throughput, serialize_throughput_option, Metric, Throughput, }; /// The operating system part of the current target triplet. diff --git a/client/sysinfo/src/sysinfo.rs b/client/sysinfo/src/sysinfo.rs index c66a6f6a62aed..93ae8854e1e43 100644 --- a/client/sysinfo/src/sysinfo.rs +++ b/client/sysinfo/src/sysinfo.rs @@ -24,7 +24,7 @@ use sp_io::crypto::sr25519_verify; use sp_std::{fmt, prelude::*}; use rand::{seq::SliceRandom, Rng, RngCore}; -use serde::Serializer; +use serde::{Deserialize, Serialize, Serializer}; use std::{ fs::File, io::{Seek, SeekFrom, Write}, @@ -33,6 +33,43 @@ use std::{ time::{Duration, Instant}, }; +/// A single hardware metric. +#[derive(Deserialize, Serialize, Debug, Clone, Copy, PartialEq)] +pub enum Metric { + /// SR25519 signature verification. + Sr25519Verify, + /// Blake2-256 hashing algorithm. + Blake2256, + /// Copying data in RAM. + MemCopy, + /// Disk sequential write. + DiskSeqWrite, + /// Disk random write. + DiskRndWrite, +} + +impl Metric { + /// The category of the metric. + pub fn category(&self) -> &'static str { + match self { + Self::Sr25519Verify | Self::Blake2256 => "CPU", + Self::MemCopy => "Memory", + Self::DiskSeqWrite | Self::DiskRndWrite => "Disk", + } + } + + /// The name of the metric. It is always prefixed by the [`self::category()`]. + pub fn name(&self) -> &'static str { + match self { + Self::Sr25519Verify => "SR25519-Verify", + Self::Blake2256 => "BLAKE2-256", + Self::MemCopy => "Copy", + Self::DiskSeqWrite => "Seq Write", + Self::DiskRndWrite => "Rnd Write", + } + } +} + /// The unit in which the [`Throughput`] (bytes per second) is denoted. pub enum Unit { GiBs, diff --git a/utils/frame/benchmarking-cli/src/machine/hardware.rs b/utils/frame/benchmarking-cli/src/machine/hardware.rs index 50c88ec74646c..11cf2240391c6 100644 --- a/utils/frame/benchmarking-cli/src/machine/hardware.rs +++ b/utils/frame/benchmarking-cli/src/machine/hardware.rs @@ -18,7 +18,7 @@ //! Contains types to define hardware requirements. use lazy_static::lazy_static; -use sc_sysinfo::Throughput; +use sc_sysinfo::{Metric, Throughput}; use serde::{de::Visitor, Deserialize, Deserializer, Serialize, Serializer}; use sp_std::{fmt, fmt::Formatter}; @@ -84,45 +84,6 @@ pub struct Requirement { pub minimum: Throughput, } -/// A single hardware metric. -/// -/// The implementation of these is in `sc-sysinfo`. -#[derive(Deserialize, Serialize, Debug, Clone, Copy, PartialEq)] -pub enum Metric { - /// SR25519 signature verification. - Sr25519Verify, - /// Blake2-256 hashing algorithm. - Blake2256, - /// Copying data in RAM. - MemCopy, - /// Disk sequential write. - DiskSeqWrite, - /// Disk random write. - DiskRndWrite, -} - -impl Metric { - /// The category of the metric. - pub fn category(&self) -> &'static str { - match self { - Self::Sr25519Verify | Self::Blake2256 => "CPU", - Self::MemCopy => "Memory", - Self::DiskSeqWrite | Self::DiskRndWrite => "Disk", - } - } - - /// The name of the metric. It is always prefixed by the [`self::category()`]. - pub fn name(&self) -> &'static str { - match self { - Self::Sr25519Verify => "SR25519-Verify", - Self::Blake2256 => "BLAKE2-256", - Self::MemCopy => "Copy", - Self::DiskSeqWrite => "Seq Write", - Self::DiskRndWrite => "Rnd Write", - } - } -} - #[cfg(test)] mod tests { use super::*; diff --git a/utils/frame/benchmarking-cli/src/machine/mod.rs b/utils/frame/benchmarking-cli/src/machine/mod.rs index 82b4e5be7358e..b2eae71ee8f9e 100644 --- a/utils/frame/benchmarking-cli/src/machine/mod.rs +++ b/utils/frame/benchmarking-cli/src/machine/mod.rs @@ -30,11 +30,11 @@ use sc_cli::{CliConfiguration, Result, SharedParams}; use sc_service::Configuration; use sc_sysinfo::{ benchmark_cpu, benchmark_disk_random_writes, benchmark_disk_sequential_writes, - benchmark_memory, benchmark_sr25519_verify, ExecutionLimit, Throughput, + benchmark_memory, benchmark_sr25519_verify, ExecutionLimit, Metric, Throughput, }; use crate::shared::check_build_profile; -pub use hardware::{Metric, Requirement, Requirements, SUBSTRATE_REFERENCE_HARDWARE}; +pub use hardware::{Requirement, Requirements, SUBSTRATE_REFERENCE_HARDWARE}; /// Command to benchmark the hardware. /// From f232e0c034e80057d8b51437c2a86980c5b4e873 Mon Sep 17 00:00:00 2001 From: Szegoo Date: Sun, 6 Nov 2022 19:07:27 +0100 Subject: [PATCH 02/15] run hardware bench if validiator flag is being used --- bin/node/cli/src/command.rs | 26 ++++++++++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/bin/node/cli/src/command.rs b/bin/node/cli/src/command.rs index 108d7743843b6..5a1909f147934 100644 --- a/bin/node/cli/src/command.rs +++ b/bin/node/cli/src/command.rs @@ -83,6 +83,22 @@ impl SubstrateCli for Cli { pub fn run() -> Result<()> { let cli = Cli::from_args(); + if cli.run.validator { + let cmd = MachineCmd { + shared_params: cli.run.shared_params.clone(), + allow_fail: true, + tolerance: 10.0, + verify_duration: 5.0, + disk_duration: 5.0, + hash_duration: 5.0, + memory_duration: 5.0, + }; + + let runner = cli.create_runner(&cmd)?; + + let _ = runner.sync_run(|config| cmd.run(&config, SUBSTRATE_REFERENCE_HARDWARE.clone())); + } + match &cli.subcommand { None => { let runner = cli.create_runner(&cli.run)?; @@ -166,8 +182,14 @@ pub fn run() -> Result<()> { &ext_factory, ) }, - BenchmarkCmd::Machine(cmd) => - cmd.run(&config, SUBSTRATE_REFERENCE_HARDWARE.clone()), + BenchmarkCmd::Machine(cmd) => { + // the hardware benchmark is run automatically if the + // validator flag is being used. + if !cli.run.validator { + return cmd.run(&config, SUBSTRATE_REFERENCE_HARDWARE.clone()) + } + Ok(()) + }, } }) }, From 41c00dba26c208fe641d2239cf98d38b96cadcfb Mon Sep 17 00:00:00 2001 From: Szegoo Date: Mon, 7 Nov 2022 07:53:07 +0100 Subject: [PATCH 03/15] fix rustdoc & update node-template --- bin/node-template/node/src/command.rs | 30 ++++++++++++++++++++++++--- client/sysinfo/src/sysinfo.rs | 2 +- 2 files changed, 28 insertions(+), 4 deletions(-) diff --git a/bin/node-template/node/src/command.rs b/bin/node-template/node/src/command.rs index 6d293b7b85fcc..e2b16ea5201ac 100644 --- a/bin/node-template/node/src/command.rs +++ b/bin/node-template/node/src/command.rs @@ -4,7 +4,9 @@ use crate::{ cli::{Cli, Subcommand}, service, }; -use frame_benchmarking_cli::{BenchmarkCmd, ExtrinsicFactory, SUBSTRATE_REFERENCE_HARDWARE}; +use frame_benchmarking_cli::{ + BenchmarkCmd, ExtrinsicFactory, MachineCmd, SUBSTRATE_REFERENCE_HARDWARE, +}; use node_template_runtime::{Block, EXISTENTIAL_DEPOSIT}; use sc_cli::{ChainSpec, RuntimeVersion, SubstrateCli}; use sc_service::PartialComponents; @@ -53,6 +55,22 @@ impl SubstrateCli for Cli { pub fn run() -> sc_cli::Result<()> { let cli = Cli::from_args(); + if cli.run.validator { + let cmd = MachineCmd { + shared_params: cli.run.shared_params.clone(), + allow_fail: true, + tolerance: 10.0, + verify_duration: 5.0, + disk_duration: 5.0, + hash_duration: 5.0, + memory_duration: 5.0, + }; + + let runner = cli.create_runner(&cmd)?; + + let _ = runner.sync_run(|config| cmd.run(&config, SUBSTRATE_REFERENCE_HARDWARE.clone())); + } + match &cli.subcommand { Some(Subcommand::Key(cmd)) => cmd.run(&cli), Some(Subcommand::BuildSpec(cmd)) => { @@ -167,8 +185,14 @@ pub fn run() -> sc_cli::Result<()> { cmd.run(client, inherent_benchmark_data()?, Vec::new(), &ext_factory) }, - BenchmarkCmd::Machine(cmd) => - cmd.run(&config, SUBSTRATE_REFERENCE_HARDWARE.clone()), + BenchmarkCmd::Machine(cmd) => { + // the hardware benchmark is run automatically if the + // validator flag is being used. + if !cli.run.validator { + return cmd.run(&config, SUBSTRATE_REFERENCE_HARDWARE.clone()) + } + Ok(()) + }, } }) }, diff --git a/client/sysinfo/src/sysinfo.rs b/client/sysinfo/src/sysinfo.rs index 93ae8854e1e43..1eaf7b3685e93 100644 --- a/client/sysinfo/src/sysinfo.rs +++ b/client/sysinfo/src/sysinfo.rs @@ -58,7 +58,7 @@ impl Metric { } } - /// The name of the metric. It is always prefixed by the [`self::category()`]. + /// The name of the metric. It is always prefixed by the [`self.category()`]. pub fn name(&self) -> &'static str { match self { Self::Sr25519Verify => "SR25519-Verify", From 2a8e8dff138a0b4c1dc1523015b1d47f3be3ce21 Mon Sep 17 00:00:00 2001 From: Szegoo Date: Mon, 7 Nov 2022 08:24:35 +0100 Subject: [PATCH 04/15] fix --- bin/node-template/node/src/command.rs | 26 ++------------------------ bin/node/cli/src/command.rs | 20 ++------------------ bin/node/cli/src/service.rs | 2 +- 3 files changed, 5 insertions(+), 43 deletions(-) diff --git a/bin/node-template/node/src/command.rs b/bin/node-template/node/src/command.rs index e2b16ea5201ac..4cdd450bdca36 100644 --- a/bin/node-template/node/src/command.rs +++ b/bin/node-template/node/src/command.rs @@ -55,22 +55,6 @@ impl SubstrateCli for Cli { pub fn run() -> sc_cli::Result<()> { let cli = Cli::from_args(); - if cli.run.validator { - let cmd = MachineCmd { - shared_params: cli.run.shared_params.clone(), - allow_fail: true, - tolerance: 10.0, - verify_duration: 5.0, - disk_duration: 5.0, - hash_duration: 5.0, - memory_duration: 5.0, - }; - - let runner = cli.create_runner(&cmd)?; - - let _ = runner.sync_run(|config| cmd.run(&config, SUBSTRATE_REFERENCE_HARDWARE.clone())); - } - match &cli.subcommand { Some(Subcommand::Key(cmd)) => cmd.run(&cli), Some(Subcommand::BuildSpec(cmd)) => { @@ -185,14 +169,8 @@ pub fn run() -> sc_cli::Result<()> { cmd.run(client, inherent_benchmark_data()?, Vec::new(), &ext_factory) }, - BenchmarkCmd::Machine(cmd) => { - // the hardware benchmark is run automatically if the - // validator flag is being used. - if !cli.run.validator { - return cmd.run(&config, SUBSTRATE_REFERENCE_HARDWARE.clone()) - } - Ok(()) - }, + BenchmarkCmd::Machine(cmd) => + cmd.run(&config, SUBSTRATE_REFERENCE_HARDWARE.clone()), } }) }, diff --git a/bin/node/cli/src/command.rs b/bin/node/cli/src/command.rs index 5a1909f147934..6594d157c2c6b 100644 --- a/bin/node/cli/src/command.rs +++ b/bin/node/cli/src/command.rs @@ -83,22 +83,6 @@ impl SubstrateCli for Cli { pub fn run() -> Result<()> { let cli = Cli::from_args(); - if cli.run.validator { - let cmd = MachineCmd { - shared_params: cli.run.shared_params.clone(), - allow_fail: true, - tolerance: 10.0, - verify_duration: 5.0, - disk_duration: 5.0, - hash_duration: 5.0, - memory_duration: 5.0, - }; - - let runner = cli.create_runner(&cmd)?; - - let _ = runner.sync_run(|config| cmd.run(&config, SUBSTRATE_REFERENCE_HARDWARE.clone())); - } - match &cli.subcommand { None => { let runner = cli.create_runner(&cli.run)?; @@ -183,8 +167,8 @@ pub fn run() -> Result<()> { ) }, BenchmarkCmd::Machine(cmd) => { - // the hardware benchmark is run automatically if the - // validator flag is being used. + // the hardware benchmark is run automatically if the validator flag + // is being used. if !cli.run.validator { return cmd.run(&config, SUBSTRATE_REFERENCE_HARDWARE.clone()) } diff --git a/bin/node/cli/src/service.rs b/bin/node/cli/src/service.rs index 6c29f0c08ee13..309db22ce40db 100644 --- a/bin/node/cli/src/service.rs +++ b/bin/node/cli/src/service.rs @@ -317,7 +317,7 @@ pub fn new_full_base( &sc_consensus_babe::BabeLink, ), ) -> Result { - let hwbench = if !disable_hardware_benchmarks { + let hwbench = if !disable_hardware_benchmarks || config.role.is_authority() { config.database.path().map(|database_path| { let _ = std::fs::create_dir_all(&database_path); sc_sysinfo::gather_hwbench(Some(database_path)) From adb5ca5eae423138cb18f33436371bffd91ca113 Mon Sep 17 00:00:00 2001 From: Szegoo Date: Mon, 7 Nov 2022 08:41:00 +0100 Subject: [PATCH 05/15] unused improt --- bin/node-template/node/src/command.rs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/bin/node-template/node/src/command.rs b/bin/node-template/node/src/command.rs index 4cdd450bdca36..6d293b7b85fcc 100644 --- a/bin/node-template/node/src/command.rs +++ b/bin/node-template/node/src/command.rs @@ -4,9 +4,7 @@ use crate::{ cli::{Cli, Subcommand}, service, }; -use frame_benchmarking_cli::{ - BenchmarkCmd, ExtrinsicFactory, MachineCmd, SUBSTRATE_REFERENCE_HARDWARE, -}; +use frame_benchmarking_cli::{BenchmarkCmd, ExtrinsicFactory, SUBSTRATE_REFERENCE_HARDWARE}; use node_template_runtime::{Block, EXISTENTIAL_DEPOSIT}; use sc_cli::{ChainSpec, RuntimeVersion, SubstrateCli}; use sc_service::PartialComponents; From 2ea691d06d97ba650d28f9bcc07354f2364b99fe Mon Sep 17 00:00:00 2001 From: Sergej Sakac Date: Fri, 11 Nov 2022 11:15:49 +0100 Subject: [PATCH 06/15] warn --- bin/node/cli/src/command.rs | 10 ++-------- bin/node/cli/src/service.rs | 2 +- utils/frame/benchmarking-cli/src/machine/mod.rs | 2 +- 3 files changed, 4 insertions(+), 10 deletions(-) diff --git a/bin/node/cli/src/command.rs b/bin/node/cli/src/command.rs index 6594d157c2c6b..108d7743843b6 100644 --- a/bin/node/cli/src/command.rs +++ b/bin/node/cli/src/command.rs @@ -166,14 +166,8 @@ pub fn run() -> Result<()> { &ext_factory, ) }, - BenchmarkCmd::Machine(cmd) => { - // the hardware benchmark is run automatically if the validator flag - // is being used. - if !cli.run.validator { - return cmd.run(&config, SUBSTRATE_REFERENCE_HARDWARE.clone()) - } - Ok(()) - }, + BenchmarkCmd::Machine(cmd) => + cmd.run(&config, SUBSTRATE_REFERENCE_HARDWARE.clone()), } }) }, diff --git a/bin/node/cli/src/service.rs b/bin/node/cli/src/service.rs index 309db22ce40db..6c29f0c08ee13 100644 --- a/bin/node/cli/src/service.rs +++ b/bin/node/cli/src/service.rs @@ -317,7 +317,7 @@ pub fn new_full_base( &sc_consensus_babe::BabeLink, ), ) -> Result { - let hwbench = if !disable_hardware_benchmarks || config.role.is_authority() { + let hwbench = if !disable_hardware_benchmarks { config.database.path().map(|database_path| { let _ = std::fs::create_dir_all(&database_path); sc_sysinfo::gather_hwbench(Some(database_path)) diff --git a/utils/frame/benchmarking-cli/src/machine/mod.rs b/utils/frame/benchmarking-cli/src/machine/mod.rs index b2eae71ee8f9e..e1eb64c963ebe 100644 --- a/utils/frame/benchmarking-cli/src/machine/mod.rs +++ b/utils/frame/benchmarking-cli/src/machine/mod.rs @@ -184,7 +184,7 @@ impl MachineCmd { ); // Print the final result. if failed != 0 { - info!("The hardware fails to meet the requirements"); + warn!("The hardware fails to meet the requirements"); self.check_failed(Error::UnmetRequirement)?; } else { info!("The hardware meets the requirements "); From 889e355bd394baeb5c250cd4d1b227863ce3c003 Mon Sep 17 00:00:00 2001 From: Sergej Sakac Date: Sat, 12 Nov 2022 15:20:21 +0100 Subject: [PATCH 07/15] move Requirement --- client/sysinfo/src/lib.rs | 3 +- client/sysinfo/src/sysinfo.rs | 52 ++++++++++++++++++- utils/frame/benchmarking-cli/src/lib.rs | 2 +- .../benchmarking-cli/src/machine/hardware.rs | 52 +------------------ .../frame/benchmarking-cli/src/machine/mod.rs | 7 +-- 5 files changed, 58 insertions(+), 58 deletions(-) diff --git a/client/sysinfo/src/lib.rs b/client/sysinfo/src/lib.rs index c227cc46fa57d..f623bdae53e66 100644 --- a/client/sysinfo/src/lib.rs +++ b/client/sysinfo/src/lib.rs @@ -29,7 +29,8 @@ mod sysinfo_linux; pub use sysinfo::{ benchmark_cpu, benchmark_disk_random_writes, benchmark_disk_sequential_writes, benchmark_memory, benchmark_sr25519_verify, gather_hwbench, gather_sysinfo, - serialize_throughput, serialize_throughput_option, Metric, Throughput, + serialize_throughput, serialize_throughput_option, Metric, Requirement, Requirements, + Throughput, }; /// The operating system part of the current target triplet. diff --git a/client/sysinfo/src/sysinfo.rs b/client/sysinfo/src/sysinfo.rs index 1eaf7b3685e93..655b1b45e5232 100644 --- a/client/sysinfo/src/sysinfo.rs +++ b/client/sysinfo/src/sysinfo.rs @@ -21,10 +21,10 @@ use crate::{ExecutionLimit, HwBench}; use sc_telemetry::SysInfo; use sp_core::{sr25519, Pair}; use sp_io::crypto::sr25519_verify; -use sp_std::{fmt, prelude::*}; +use sp_std::{fmt, fmt::Formatter, prelude::*}; use rand::{seq::SliceRandom, Rng, RngCore}; -use serde::{Deserialize, Serialize, Serializer}; +use serde::{de::Visitor, Deserialize, Deserializer, Serialize, Serializer}; use std::{ fs::File, io::{Seek, SeekFrom, Write}, @@ -174,6 +174,54 @@ where serializer.serialize_none() } +/// Serializes throughput into MiBs and represents it as `f64`. +fn serialize_throughput_as_f64(throughput: &Throughput, serializer: S) -> Result +where + S: Serializer, +{ + serializer.serialize_f64(throughput.as_mibs()) +} + +struct ThroughputVisitor; +impl<'de> Visitor<'de> for ThroughputVisitor { + type Value = Throughput; + + fn expecting(&self, formatter: &mut Formatter) -> fmt::Result { + formatter.write_str("A value that is a f64.") + } + + fn visit_f64(self, value: f64) -> Result + where + E: serde::de::Error, + { + Ok(Throughput::from_mibs(value)) + } +} + +fn deserialize_throughput<'de, D>(deserializer: D) -> Result +where + D: Deserializer<'de>, +{ + Ok(deserializer.deserialize_f64(ThroughputVisitor))? +} + +/// Multiple requirements for the hardware. +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq)] +pub struct Requirements(pub Vec); + +/// A single requirement for the hardware. +#[derive(Deserialize, Serialize, Debug, Clone, Copy, PartialEq)] +pub struct Requirement { + /// The metric to measure. + pub metric: Metric, + /// The minimal throughput that needs to be archived for this requirement. + #[serde( + serialize_with = "serialize_throughput_as_f64", + deserialize_with = "deserialize_throughput" + )] + pub minimum: Throughput, +} + #[inline(always)] pub(crate) fn benchmark( name: &str, diff --git a/utils/frame/benchmarking-cli/src/lib.rs b/utils/frame/benchmarking-cli/src/lib.rs index a44a208b16ae9..5723a8038fdbb 100644 --- a/utils/frame/benchmarking-cli/src/lib.rs +++ b/utils/frame/benchmarking-cli/src/lib.rs @@ -27,7 +27,7 @@ mod storage; pub use block::BlockCmd; pub use extrinsic::{ExtrinsicBuilder, ExtrinsicCmd, ExtrinsicFactory}; -pub use machine::{MachineCmd, Requirements, SUBSTRATE_REFERENCE_HARDWARE}; +pub use machine::{MachineCmd, SUBSTRATE_REFERENCE_HARDWARE}; pub use overhead::OverheadCmd; pub use pallet::PalletCmd; pub use sc_service::BasePath; diff --git a/utils/frame/benchmarking-cli/src/machine/hardware.rs b/utils/frame/benchmarking-cli/src/machine/hardware.rs index 11cf2240391c6..318c193d7c08c 100644 --- a/utils/frame/benchmarking-cli/src/machine/hardware.rs +++ b/utils/frame/benchmarking-cli/src/machine/hardware.rs @@ -18,40 +18,7 @@ //! Contains types to define hardware requirements. use lazy_static::lazy_static; -use sc_sysinfo::{Metric, Throughput}; -use serde::{de::Visitor, Deserialize, Deserializer, Serialize, Serializer}; -use sp_std::{fmt, fmt::Formatter}; - -/// Serializes throughput into MiBs and represents it as `f64`. -fn serialize_throughput_as_f64(throughput: &Throughput, serializer: S) -> Result -where - S: Serializer, -{ - serializer.serialize_f64(throughput.as_mibs()) -} - -struct ThroughputVisitor; -impl<'de> Visitor<'de> for ThroughputVisitor { - type Value = Throughput; - - fn expecting(&self, formatter: &mut Formatter) -> fmt::Result { - formatter.write_str("A value that is a f64.") - } - - fn visit_f64(self, value: f64) -> Result - where - E: serde::de::Error, - { - Ok(Throughput::from_mibs(value)) - } -} - -fn deserialize_throughput<'de, D>(deserializer: D) -> Result -where - D: Deserializer<'de>, -{ - Ok(deserializer.deserialize_f64(ThroughputVisitor))? -} +use sc_sysinfo::Requirements; lazy_static! { /// The hardware requirements as measured on reference hardware. @@ -67,23 +34,6 @@ lazy_static! { }; } -/// Multiple requirements for the hardware. -#[derive(Serialize, Deserialize, Debug, Clone, PartialEq)] -pub struct Requirements(pub Vec); - -/// A single requirement for the hardware. -#[derive(Deserialize, Serialize, Debug, Clone, Copy, PartialEq)] -pub struct Requirement { - /// The metric to measure. - pub metric: Metric, - /// The minimal throughput that needs to be archived for this requirement. - #[serde( - serialize_with = "serialize_throughput_as_f64", - deserialize_with = "deserialize_throughput" - )] - pub minimum: Throughput, -} - #[cfg(test)] mod tests { use super::*; diff --git a/utils/frame/benchmarking-cli/src/machine/mod.rs b/utils/frame/benchmarking-cli/src/machine/mod.rs index e1eb64c963ebe..bcffef255e5b5 100644 --- a/utils/frame/benchmarking-cli/src/machine/mod.rs +++ b/utils/frame/benchmarking-cli/src/machine/mod.rs @@ -30,11 +30,12 @@ use sc_cli::{CliConfiguration, Result, SharedParams}; use sc_service::Configuration; use sc_sysinfo::{ benchmark_cpu, benchmark_disk_random_writes, benchmark_disk_sequential_writes, - benchmark_memory, benchmark_sr25519_verify, ExecutionLimit, Metric, Throughput, + benchmark_memory, benchmark_sr25519_verify, ExecutionLimit, Metric, Requirement, Requirements, + Throughput, }; use crate::shared::check_build_profile; -pub use hardware::{Requirement, Requirements, SUBSTRATE_REFERENCE_HARDWARE}; +pub use hardware::SUBSTRATE_REFERENCE_HARDWARE; /// Command to benchmark the hardware. /// @@ -184,7 +185,7 @@ impl MachineCmd { ); // Print the final result. if failed != 0 { - warn!("The hardware fails to meet the requirements"); + info!("The hardware fails to meet the requirements"); self.check_failed(Error::UnmetRequirement)?; } else { info!("The hardware meets the requirements "); From 861ee8b4e6974d0523056561e39518bd789db935 Mon Sep 17 00:00:00 2001 From: Sergej Sakac Date: Sun, 13 Nov 2022 09:35:47 +0100 Subject: [PATCH 08/15] bench_result --- client/sysinfo/src/lib.rs | 2 +- client/sysinfo/src/sysinfo.rs | 13 +++++++++++++ utils/frame/benchmarking-cli/src/machine/mod.rs | 6 +++--- 3 files changed, 17 insertions(+), 4 deletions(-) diff --git a/client/sysinfo/src/lib.rs b/client/sysinfo/src/lib.rs index f623bdae53e66..2b9cfae163a69 100644 --- a/client/sysinfo/src/lib.rs +++ b/client/sysinfo/src/lib.rs @@ -27,7 +27,7 @@ mod sysinfo; mod sysinfo_linux; pub use sysinfo::{ - benchmark_cpu, benchmark_disk_random_writes, benchmark_disk_sequential_writes, + bench_result, benchmark_cpu, benchmark_disk_random_writes, benchmark_disk_sequential_writes, benchmark_memory, benchmark_sr25519_verify, gather_hwbench, gather_sysinfo, serialize_throughput, serialize_throughput_option, Metric, Requirement, Requirements, Throughput, diff --git a/client/sysinfo/src/sysinfo.rs b/client/sysinfo/src/sysinfo.rs index 655b1b45e5232..1f76151f3e773 100644 --- a/client/sysinfo/src/sysinfo.rs +++ b/client/sysinfo/src/sysinfo.rs @@ -622,6 +622,19 @@ pub fn gather_hwbench(scratch_directory: Option<&Path>) -> HwBench { hwbench } +/// Checks whether the benchmark passed and returns a tuple `(passed, rel_score)`. +pub fn bench_result(requirement: &Requirement, score: Throughput, tolerance: f64) -> (bool, f64) { + let rel_score = score.as_bytes() / requirement.minimum.as_bytes(); + + // Sanity check if the result is off by factor >100x. + if rel_score >= 100.0 || rel_score <= 0.01 { + log::warn!("Bad benchmark result."); + } + + let passed = rel_score == (1.0 - (tolerance / 100.0)); + (passed, rel_score) +} + #[cfg(test)] mod tests { use super::*; diff --git a/utils/frame/benchmarking-cli/src/machine/mod.rs b/utils/frame/benchmarking-cli/src/machine/mod.rs index bcffef255e5b5..c891e077ca79c 100644 --- a/utils/frame/benchmarking-cli/src/machine/mod.rs +++ b/utils/frame/benchmarking-cli/src/machine/mod.rs @@ -29,7 +29,7 @@ use log::{error, info, warn}; use sc_cli::{CliConfiguration, Result, SharedParams}; use sc_service::Configuration; use sc_sysinfo::{ - benchmark_cpu, benchmark_disk_random_writes, benchmark_disk_sequential_writes, + bench_result, benchmark_cpu, benchmark_disk_random_writes, benchmark_disk_sequential_writes, benchmark_memory, benchmark_sr25519_verify, ExecutionLimit, Metric, Requirement, Requirements, Throughput, }; @@ -131,13 +131,13 @@ impl MachineCmd { // Dispatch the concrete function from `sc-sysinfo`. let score = self.measure(&requirement.metric, dir)?; - let rel_score = score.as_bytes() / requirement.minimum.as_bytes(); + let (passed, rel_score) = bench_result(requirement, score, self.tolerance); // Sanity check if the result is off by factor >100x. if rel_score >= 100.0 || rel_score <= 0.01 { self.check_failed(Error::BadResults)?; } - let passed = rel_score >= (1.0 - (self.tolerance / 100.0)); + Ok(BenchResult { passed, score, rel_score }) } From b3b6333c79d3603d9dd6a0cdcc78818cefa27f9b Mon Sep 17 00:00:00 2001 From: Sergej Sakac Date: Sun, 13 Nov 2022 10:28:52 +0100 Subject: [PATCH 09/15] ensure_requirements --- client/sysinfo/src/lib.rs | 2 +- client/sysinfo/src/sysinfo.rs | 43 ++++++++++++++----- .../frame/benchmarking-cli/src/machine/mod.rs | 5 ++- 3 files changed, 36 insertions(+), 14 deletions(-) diff --git a/client/sysinfo/src/lib.rs b/client/sysinfo/src/lib.rs index 2b9cfae163a69..f623bdae53e66 100644 --- a/client/sysinfo/src/lib.rs +++ b/client/sysinfo/src/lib.rs @@ -27,7 +27,7 @@ mod sysinfo; mod sysinfo_linux; pub use sysinfo::{ - bench_result, benchmark_cpu, benchmark_disk_random_writes, benchmark_disk_sequential_writes, + benchmark_cpu, benchmark_disk_random_writes, benchmark_disk_sequential_writes, benchmark_memory, benchmark_sr25519_verify, gather_hwbench, gather_sysinfo, serialize_throughput, serialize_throughput_option, Metric, Requirement, Requirements, Throughput, diff --git a/client/sysinfo/src/sysinfo.rs b/client/sysinfo/src/sysinfo.rs index 1f76151f3e773..208a07a6e9c86 100644 --- a/client/sysinfo/src/sysinfo.rs +++ b/client/sysinfo/src/sysinfo.rs @@ -589,7 +589,7 @@ pub fn benchmark_sr25519_verify(limit: ExecutionLimit) -> Throughput { /// Benchmarks the hardware and returns the results of those benchmarks. /// /// Optionally accepts a path to a `scratch_directory` to use to benchmark the disk. -pub fn gather_hwbench(scratch_directory: Option<&Path>) -> HwBench { +pub fn gather_hwbench(scratch_directory: Option<&Path>, requirements: Requirements) -> HwBench { #[allow(unused_mut)] let mut hwbench = HwBench { cpu_hashrate_score: benchmark_cpu(DEFAULT_CPU_EXECUTION_LIMIT), @@ -618,21 +618,42 @@ pub fn gather_hwbench(scratch_directory: Option<&Path>) -> HwBench { }, }; } + // if validator + ensure_requirements(hwbench.clone(), requirements); hwbench } -/// Checks whether the benchmark passed and returns a tuple `(passed, rel_score)`. -pub fn bench_result(requirement: &Requirement, score: Throughput, tolerance: f64) -> (bool, f64) { - let rel_score = score.as_bytes() / requirement.minimum.as_bytes(); - - // Sanity check if the result is off by factor >100x. - if rel_score >= 100.0 || rel_score <= 0.01 { - log::warn!("Bad benchmark result."); +fn ensure_requirements(hwbench: HwBench, requirements: Requirements) { + let mut failed = 0; + for requirement in requirements.0.iter() { + match requirement.metric { + Metric::Blake2256 => + if requirement.minimum > hwbench.cpu_hashrate_score { + failed += 1; + }, + Metric::MemCopy => + if requirement.minimum > hwbench.memory_memcpy_score { + failed += 1; + }, + Metric::DiskSeqWrite => + if let Some(score) = hwbench.disk_sequential_write_score { + if requirement.minimum > score { + failed += 1; + } + }, + Metric::DiskRndWrite => + if let Some(score) = hwbench.disk_random_write_score { + if requirement.minimum > score { + failed += 1; + } + }, + _ => (), + } + } + if failed != 0 { + log::warn!("The hardware fails to meet the requirements"); } - - let passed = rel_score == (1.0 - (tolerance / 100.0)); - (passed, rel_score) } #[cfg(test)] diff --git a/utils/frame/benchmarking-cli/src/machine/mod.rs b/utils/frame/benchmarking-cli/src/machine/mod.rs index c891e077ca79c..e37311f72ae7b 100644 --- a/utils/frame/benchmarking-cli/src/machine/mod.rs +++ b/utils/frame/benchmarking-cli/src/machine/mod.rs @@ -29,7 +29,7 @@ use log::{error, info, warn}; use sc_cli::{CliConfiguration, Result, SharedParams}; use sc_service::Configuration; use sc_sysinfo::{ - bench_result, benchmark_cpu, benchmark_disk_random_writes, benchmark_disk_sequential_writes, + benchmark_cpu, benchmark_disk_random_writes, benchmark_disk_sequential_writes, benchmark_memory, benchmark_sr25519_verify, ExecutionLimit, Metric, Requirement, Requirements, Throughput, }; @@ -131,12 +131,13 @@ impl MachineCmd { // Dispatch the concrete function from `sc-sysinfo`. let score = self.measure(&requirement.metric, dir)?; - let (passed, rel_score) = bench_result(requirement, score, self.tolerance); + let rel_score = score.as_bytes() / requirement.minimum.as_bytes(); // Sanity check if the result is off by factor >100x. if rel_score >= 100.0 || rel_score <= 0.01 { self.check_failed(Error::BadResults)?; } + let passed = rel_score >= (1.0 - (self.tolerance / 100.0)); Ok(BenchResult { passed, score, rel_score }) } From fb5bef820cee46c0f11d85f4fc2320108559df3c Mon Sep 17 00:00:00 2001 From: Sergej Sakac Date: Sun, 13 Nov 2022 10:44:04 +0100 Subject: [PATCH 10/15] make the code compile --- bin/node/cli/src/service.rs | 4 +++- utils/frame/benchmarking-cli/src/machine/mod.rs | 1 - 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/bin/node/cli/src/service.rs b/bin/node/cli/src/service.rs index 6c29f0c08ee13..f99e740b39bd6 100644 --- a/bin/node/cli/src/service.rs +++ b/bin/node/cli/src/service.rs @@ -32,6 +32,7 @@ use sc_executor::NativeElseWasmExecutor; use sc_network::NetworkService; use sc_network_common::{protocol::event::Event, service::NetworkEventStream}; use sc_service::{config::Configuration, error::Error as ServiceError, RpcHandlers, TaskManager}; +use sc_sysinfo::Requirements; use sc_telemetry::{Telemetry, TelemetryWorker}; use sp_api::ProvideRuntimeApi; use sp_core::crypto::Pair; @@ -317,10 +318,11 @@ pub fn new_full_base( &sc_consensus_babe::BabeLink, ), ) -> Result { + let requirements: Requirements = Requirements(vec![]); let hwbench = if !disable_hardware_benchmarks { config.database.path().map(|database_path| { let _ = std::fs::create_dir_all(&database_path); - sc_sysinfo::gather_hwbench(Some(database_path)) + sc_sysinfo::gather_hwbench(Some(database_path), requirements) }) } else { None diff --git a/utils/frame/benchmarking-cli/src/machine/mod.rs b/utils/frame/benchmarking-cli/src/machine/mod.rs index e37311f72ae7b..bcffef255e5b5 100644 --- a/utils/frame/benchmarking-cli/src/machine/mod.rs +++ b/utils/frame/benchmarking-cli/src/machine/mod.rs @@ -138,7 +138,6 @@ impl MachineCmd { self.check_failed(Error::BadResults)?; } let passed = rel_score >= (1.0 - (self.tolerance / 100.0)); - Ok(BenchResult { passed, score, rel_score }) } From befdd842fd16e0404dd79d682657832d21f2e5e0 Mon Sep 17 00:00:00 2001 From: Sergej Sakac Date: Sun, 13 Nov 2022 10:53:22 +0100 Subject: [PATCH 11/15] check if authority --- bin/node/cli/src/service.rs | 9 ++++++--- client/sysinfo/src/sysinfo.rs | 12 +++++++++--- 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/bin/node/cli/src/service.rs b/bin/node/cli/src/service.rs index f99e740b39bd6..947979ea2725e 100644 --- a/bin/node/cli/src/service.rs +++ b/bin/node/cli/src/service.rs @@ -21,6 +21,7 @@ //! Service implementation. Specialized wrapper over substrate service. use codec::Encode; +use frame_benchmarking_cli::SUBSTRATE_REFERENCE_HARDWARE; use frame_system_rpc_runtime_api::AccountNonceApi; use futures::prelude::*; use kitchensink_runtime::RuntimeApi; @@ -32,7 +33,6 @@ use sc_executor::NativeElseWasmExecutor; use sc_network::NetworkService; use sc_network_common::{protocol::event::Event, service::NetworkEventStream}; use sc_service::{config::Configuration, error::Error as ServiceError, RpcHandlers, TaskManager}; -use sc_sysinfo::Requirements; use sc_telemetry::{Telemetry, TelemetryWorker}; use sp_api::ProvideRuntimeApi; use sp_core::crypto::Pair; @@ -318,11 +318,14 @@ pub fn new_full_base( &sc_consensus_babe::BabeLink, ), ) -> Result { - let requirements: Requirements = Requirements(vec![]); let hwbench = if !disable_hardware_benchmarks { config.database.path().map(|database_path| { let _ = std::fs::create_dir_all(&database_path); - sc_sysinfo::gather_hwbench(Some(database_path), requirements) + sc_sysinfo::gather_hwbench( + Some(database_path), + SUBSTRATE_REFERENCE_HARDWARE.clone(), + config.role.is_authority(), + ) }) } else { None diff --git a/client/sysinfo/src/sysinfo.rs b/client/sysinfo/src/sysinfo.rs index 208a07a6e9c86..c530c4380f66d 100644 --- a/client/sysinfo/src/sysinfo.rs +++ b/client/sysinfo/src/sysinfo.rs @@ -589,7 +589,11 @@ pub fn benchmark_sr25519_verify(limit: ExecutionLimit) -> Throughput { /// Benchmarks the hardware and returns the results of those benchmarks. /// /// Optionally accepts a path to a `scratch_directory` to use to benchmark the disk. -pub fn gather_hwbench(scratch_directory: Option<&Path>, requirements: Requirements) -> HwBench { +pub fn gather_hwbench( + scratch_directory: Option<&Path>, + requirements: Requirements, + is_authority: bool, +) -> HwBench { #[allow(unused_mut)] let mut hwbench = HwBench { cpu_hashrate_score: benchmark_cpu(DEFAULT_CPU_EXECUTION_LIMIT), @@ -618,8 +622,10 @@ pub fn gather_hwbench(scratch_directory: Option<&Path>, requirements: Requiremen }, }; } - // if validator - ensure_requirements(hwbench.clone(), requirements); + + if is_authority { + ensure_requirements(hwbench.clone(), requirements); + } hwbench } From f79582a916525b067e1f16cf948eed6f51642dcc Mon Sep 17 00:00:00 2001 From: Sergej Sakac <73715684+Szegoo@users.noreply.github.com> Date: Sun, 13 Nov 2022 22:28:39 +0100 Subject: [PATCH 12/15] Update client/sysinfo/src/sysinfo.rs Co-authored-by: Oliver Tale-Yazdi --- client/sysinfo/src/sysinfo.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/sysinfo/src/sysinfo.rs b/client/sysinfo/src/sysinfo.rs index c530c4380f66d..ab9b329525681 100644 --- a/client/sysinfo/src/sysinfo.rs +++ b/client/sysinfo/src/sysinfo.rs @@ -658,7 +658,7 @@ fn ensure_requirements(hwbench: HwBench, requirements: Requirements) { } } if failed != 0 { - log::warn!("The hardware fails to meet the requirements"); + log::warn!("Your hardware performance score was less than expected for role 'Authority'. See https://wiki.polkadot.network/docs/maintain-guides-how-to-validate-polkadot#reference-hardware"); } } From da40db951c1474fe323cc489a4108ee3f8ee253a Mon Sep 17 00:00:00 2001 From: Sergej Sakac Date: Mon, 14 Nov 2022 08:25:41 +0100 Subject: [PATCH 13/15] nit fixes --- client/sysinfo/src/sysinfo.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/client/sysinfo/src/sysinfo.rs b/client/sysinfo/src/sysinfo.rs index ab9b329525681..ec49647be6ea2 100644 --- a/client/sysinfo/src/sysinfo.rs +++ b/client/sysinfo/src/sysinfo.rs @@ -588,7 +588,9 @@ pub fn benchmark_sr25519_verify(limit: ExecutionLimit) -> Throughput { /// Benchmarks the hardware and returns the results of those benchmarks. /// -/// Optionally accepts a path to a `scratch_directory` to use to benchmark the disk. +/// Optionally accepts a path to a `scratch_directory` to use to benchmark the +/// disk. Also accepts the `requirements` for the hardware benchmark and a +/// boolean to specify if the node is an authority. pub fn gather_hwbench( scratch_directory: Option<&Path>, requirements: Requirements, @@ -654,7 +656,7 @@ fn ensure_requirements(hwbench: HwBench, requirements: Requirements) { failed += 1; } }, - _ => (), + Metric::Sr25519Verify => {}, } } if failed != 0 { From b4883931986c3f87daecf1e8eb188ea133a291b5 Mon Sep 17 00:00:00 2001 From: Sergej Sakac Date: Wed, 18 Jan 2023 14:56:17 +0100 Subject: [PATCH 14/15] warning signs --- client/sysinfo/src/sysinfo.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/sysinfo/src/sysinfo.rs b/client/sysinfo/src/sysinfo.rs index ec49647be6ea2..a2d475459d969 100644 --- a/client/sysinfo/src/sysinfo.rs +++ b/client/sysinfo/src/sysinfo.rs @@ -660,7 +660,7 @@ fn ensure_requirements(hwbench: HwBench, requirements: Requirements) { } } if failed != 0 { - log::warn!("Your hardware performance score was less than expected for role 'Authority'. See https://wiki.polkadot.network/docs/maintain-guides-how-to-validate-polkadot#reference-hardware"); + log::warn!("⚠️ ⚠️ ⚠️ Your hardware performance score was less than expected for role 'Authority'. See https://wiki.polkadot.network/docs/maintain-guides-how-to-validate-polkadot#reference-hardware"); } } From 117663d6e5b2c9944040218e2cd1b179dca914e3 Mon Sep 17 00:00:00 2001 From: Sergej Sakac <73715684+Szegoo@users.noreply.github.com> Date: Thu, 19 Jan 2023 19:41:55 +0100 Subject: [PATCH 15/15] Update client/sysinfo/src/sysinfo.rs Co-authored-by: Oliver Tale-Yazdi --- client/sysinfo/src/sysinfo.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/sysinfo/src/sysinfo.rs b/client/sysinfo/src/sysinfo.rs index a2d475459d969..800df4b8a41a9 100644 --- a/client/sysinfo/src/sysinfo.rs +++ b/client/sysinfo/src/sysinfo.rs @@ -660,7 +660,7 @@ fn ensure_requirements(hwbench: HwBench, requirements: Requirements) { } } if failed != 0 { - log::warn!("⚠️ ⚠️ ⚠️ Your hardware performance score was less than expected for role 'Authority'. See https://wiki.polkadot.network/docs/maintain-guides-how-to-validate-polkadot#reference-hardware"); + log::warn!("⚠️ Your hardware performance score was less than expected for role 'Authority'. See https://wiki.polkadot.network/docs/maintain-guides-how-to-validate-polkadot#reference-hardware"); } }