Skip to content
This repository has been archived by the owner on Nov 15, 2023. It is now read-only.

Move Throughput into sc-sysinfo #12368

Merged
merged 45 commits into from
Nov 4, 2022
Merged
Show file tree
Hide file tree
Changes from 4 commits
Commits
Show all changes
45 commits
Select commit Hold shift + click to select a range
ada0ee0
move Throughput to sc-sysinfo
Szegoo Sep 27, 2022
d25f110
replace u64
Szegoo Sep 27, 2022
8f7084e
Merge branch 'paritytech:master' into hardware-bench
Szegoo Sep 28, 2022
a564d4f
fix in tests
Szegoo Sep 29, 2022
7c30f8d
change Throughput
Szegoo Sep 29, 2022
d276670
refactored Throughput
Szegoo Sep 29, 2022
96ec3f0
fixes
Szegoo Sep 30, 2022
7226a57
moved tests & fixes
Szegoo Sep 30, 2022
f5530ac
custom serializer
Szegoo Sep 30, 2022
99b25a7
note
Szegoo Sep 30, 2022
cdd84d2
fix serializer
Szegoo Oct 1, 2022
9092ba0
forgot to remove
Szegoo Oct 1, 2022
621b56c
deserialize
Szegoo Oct 1, 2022
5f256b0
functioning deserialization :)
Szegoo Oct 1, 2022
5731506
try to make clipply happy
Szegoo Oct 1, 2022
8c61d1b
Serialize as function
Szegoo Oct 2, 2022
ed10eae
test HwBench
Szegoo Oct 2, 2022
0997131
rename
Szegoo Oct 2, 2022
3785a62
fix serialization
Szegoo Oct 3, 2022
7354ccb
deserialize as function
Szegoo Oct 3, 2022
c3d0995
unused import
Szegoo Oct 3, 2022
bab6f75
move serialize/deserialize
Szegoo Oct 3, 2022
51a71ac
don't serialize none
Szegoo Oct 3, 2022
a62ecb8
remove nonsense
Szegoo Oct 3, 2022
0ed3320
remove nonsense comment :P
Szegoo Oct 3, 2022
4f5dd61
fixes
Szegoo Oct 4, 2022
cd73e34
remove all the todos
Szegoo Oct 5, 2022
5028b9a
return enum
Szegoo Oct 5, 2022
1f08df9
fixes
Szegoo Oct 6, 2022
fee21a9
Merge branch 'paritytech:master' into hardware-bench
Szegoo Oct 6, 2022
f59280a
fix nit
Szegoo Oct 7, 2022
3173864
improve docs & readability
Szegoo Oct 7, 2022
2005c2b
Merge branch 'paritytech:master' into hardware-bench
Szegoo Oct 7, 2022
e63e6bd
Update client/sysinfo/src/sysinfo.rs
Szegoo Oct 12, 2022
1ee8772
fix all the nits
Szegoo Oct 12, 2022
38c584a
Merge branch 'paritytech:master' into hardware-bench
Szegoo Oct 12, 2022
02f3862
rename
Szegoo Oct 12, 2022
0679a4a
Merge branch 'paritytech:master' into hardware-bench
Szegoo Oct 15, 2022
cbca2c2
fix
Szegoo Oct 15, 2022
698d53e
Merge branch 'paritytech:master' into hardware-bench
Szegoo Oct 20, 2022
30fe5ee
Update client/sysinfo/src/sysinfo.rs
Szegoo Oct 20, 2022
7a3fe3e
remove unit from serialization
Szegoo Oct 29, 2022
70d48ba
Merge branch 'paritytech:master' into hardware-bench
Szegoo Oct 29, 2022
3590c83
Merge branch 'paritytech:master' into hardware-bench
Szegoo Oct 30, 2022
9211e14
Update utils/frame/benchmarking-cli/src/machine/hardware.rs
ggwpez Nov 4, 2022
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 5 additions & 5 deletions client/sysinfo/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ mod sysinfo_linux;

pub use sysinfo::{
benchmark_cpu, benchmark_disk_random_writes, benchmark_disk_sequential_writes,
benchmark_memory, benchmark_sr25519_verify, gather_hwbench, gather_sysinfo,
benchmark_memory, benchmark_sr25519_verify, gather_hwbench, gather_sysinfo, Throughput,
};

/// The operating system part of the current target triplet.
Expand All @@ -44,13 +44,13 @@ pub const TARGET_ENV: &str = include_str!(concat!(env!("OUT_DIR"), "/target_env.
#[derive(Clone, Debug, serde::Serialize)]
pub struct HwBench {
/// The CPU speed, as measured in how many MB/s it can hash using the BLAKE2b-256 hash.
pub cpu_hashrate_score: u64,
pub cpu_hashrate_score: Throughput,
/// Memory bandwidth in MB/s, calculated by measuring the throughput of `memcpy`.
pub memory_memcpy_score: u64,
pub memory_memcpy_score: Throughput,
/// Sequential disk write speed in MB/s.
pub disk_sequential_write_score: Option<u64>,
pub disk_sequential_write_score: Option<Throughput>,
/// Random disk write speed in MB/s.
pub disk_random_write_score: Option<u64>,
pub disk_random_write_score: Option<Throughput>,
Szegoo marked this conversation as resolved.
Show resolved Hide resolved
}
Szegoo marked this conversation as resolved.
Show resolved Hide resolved

/// Limit the execution time of a benchmark.
Expand Down
102 changes: 89 additions & 13 deletions client/sysinfo/src/sysinfo.rs
Original file line number Diff line number Diff line change
Expand Up @@ -21,9 +21,10 @@ use crate::{ExecutionLimit, HwBench};
use sc_telemetry::SysInfo;
use sp_core::{sr25519, Pair};
use sp_io::crypto::sr25519_verify;
use sp_std::prelude::*;
use sp_std::{fmt, prelude::*};

use rand::{seq::SliceRandom, Rng, RngCore};
use serde::{Deserialize, Serialize};
use std::{
fs::File,
io::{Seek, SeekFrom, Write},
Expand All @@ -32,6 +33,77 @@ use std::{
time::{Duration, Instant},
};

/// Throughput as measured in bytes per second.
#[derive(Deserialize, Serialize, Debug, Clone, Copy, PartialEq)]
pub enum Throughput {
/// KiB/s
KiBs(f64),
/// MiB/s
MiBs(f64),
/// GiB/s
GiBs(f64),
}
Szegoo marked this conversation as resolved.
Show resolved Hide resolved

const KIBIBYTE: f64 = 1024.0;
Szegoo marked this conversation as resolved.
Show resolved Hide resolved

impl Throughput {
/// The unit of the metric.
pub fn unit(&self) -> &'static str {
match self {
Self::KiBs(_) => "KiB/s",
Self::MiBs(_) => "MiB/s",
Self::GiBs(_) => "GiB/s",
}
}

/// [`Self`] as number of byte/s.
pub fn to_bs(&self) -> f64 {
self.to_kibs() * KIBIBYTE
}

/// [`Self`] as number of kibibyte/s.
pub fn to_kibs(&self) -> f64 {
self.to_mibs() * KIBIBYTE
}

/// [`Self`] as number of mebibyte/s.
pub fn to_mibs(&self) -> f64 {
self.to_gibs() * KIBIBYTE
}

/// [`Self`] as number of gibibyte/s.
pub fn to_gibs(&self) -> f64 {
match self {
Self::KiBs(k) => *k / (KIBIBYTE * KIBIBYTE),
Self::MiBs(m) => *m / KIBIBYTE,
Self::GiBs(g) => *g,
}
}

/// Normalizes [`Self`] to use the larges unit possible.
pub fn normalize(&self) -> Self {
let bs = self.to_bs();

if bs >= KIBIBYTE * KIBIBYTE * KIBIBYTE {
Self::GiBs(self.to_gibs())
} else if bs >= KIBIBYTE * KIBIBYTE {
Self::MiBs(self.to_mibs())
} else {
Self::KiBs(self.to_kibs())
}
}
}

impl fmt::Display for Throughput {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
let normalized = self.normalize();
match normalized {
Self::KiBs(s) | Self::MiBs(s) | Self::GiBs(s) =>
write!(f, "{:.2?} {}", s, normalized.unit()),
}
}
}

#[inline(always)]
pub(crate) fn benchmark<E>(
name: &str,
Expand Down Expand Up @@ -121,7 +193,7 @@ pub const DEFAULT_CPU_EXECUTION_LIMIT: ExecutionLimit =
ExecutionLimit::Both { max_iterations: 4 * 1024, max_duration: Duration::from_millis(100) };

// This benchmarks the CPU speed as measured by calculating BLAKE2b-256 hashes, in MB/s.
Szegoo marked this conversation as resolved.
Show resolved Hide resolved
pub fn benchmark_cpu(limit: ExecutionLimit) -> f64 {
pub fn benchmark_cpu(limit: ExecutionLimit) -> Throughput {
// In general the results of this benchmark are somewhat sensitive to how much
// data we hash at the time. The smaller this is the *less* MB/s we can hash,
// the bigger this is the *more* MB/s we can hash, up until a certain point
Expand All @@ -148,8 +220,10 @@ pub fn benchmark_cpu(limit: ExecutionLimit) -> f64 {
Ok(())
};

benchmark("CPU score", SIZE, limit.max_iterations(), limit.max_duration(), run)
.expect("benchmark cannot fail; qed")
Throughput::MiBs(
benchmark("CPU score", SIZE, limit.max_iterations(), limit.max_duration(), run)
.expect("benchmark cannot fail; qed"),
)
}

/// A default [`ExecutionLimit`] that can be used to call [`benchmark_memory`].
Expand All @@ -161,7 +235,7 @@ pub const DEFAULT_MEMORY_EXECUTION_LIMIT: ExecutionLimit =
// It doesn't technically measure the absolute maximum memory bandwidth available,
// but that's fine, because real code most of the time isn't optimized to take
// advantage of the full memory bandwidth either.
pub fn benchmark_memory(limit: ExecutionLimit) -> f64 {
pub fn benchmark_memory(limit: ExecutionLimit) -> Throughput {
// Ideally this should be at least as big as the CPU's L3 cache,
// and it should be big enough so that the `memcpy` takes enough
// time to be actually measurable.
Expand Down Expand Up @@ -196,8 +270,10 @@ pub fn benchmark_memory(limit: ExecutionLimit) -> f64 {
Ok(())
};

benchmark("memory score", SIZE, limit.max_iterations(), limit.max_duration(), run)
.expect("benchmark cannot fail; qed")
Throughput::MiBs(
benchmark("memory score", SIZE, limit.max_iterations(), limit.max_duration(), run)
.expect("benchmark cannot fail; qed"),
)
}

struct TemporaryFile {
Expand Down Expand Up @@ -402,8 +478,8 @@ pub fn benchmark_sr25519_verify(limit: ExecutionLimit) -> f64 {
pub fn gather_hwbench(scratch_directory: Option<&Path>) -> HwBench {
#[allow(unused_mut)]
let mut hwbench = HwBench {
cpu_hashrate_score: benchmark_cpu(DEFAULT_CPU_EXECUTION_LIMIT) as u64,
memory_memcpy_score: benchmark_memory(DEFAULT_MEMORY_EXECUTION_LIMIT) as u64,
cpu_hashrate_score: benchmark_cpu(DEFAULT_CPU_EXECUTION_LIMIT),
memory_memcpy_score: benchmark_memory(DEFAULT_MEMORY_EXECUTION_LIMIT),
disk_sequential_write_score: None,
disk_random_write_score: None,
};
Expand All @@ -412,7 +488,7 @@ pub fn gather_hwbench(scratch_directory: Option<&Path>) -> HwBench {
hwbench.disk_sequential_write_score =
match benchmark_disk_sequential_writes(DEFAULT_DISK_EXECUTION_LIMIT, scratch_directory)
{
Ok(score) => Some(score as u64),
Ok(score) => Some(Throughput::MiBs(score)),
Err(error) => {
log::warn!("Failed to run the sequential write disk benchmark: {}", error);
None
Expand All @@ -421,7 +497,7 @@ pub fn gather_hwbench(scratch_directory: Option<&Path>) -> HwBench {

hwbench.disk_random_write_score =
match benchmark_disk_random_writes(DEFAULT_DISK_EXECUTION_LIMIT, scratch_directory) {
Ok(score) => Some(score as u64),
Ok(score) => Some(Throughput::MiBs(score)),
Err(error) => {
log::warn!("Failed to run the random write disk benchmark: {}", error);
None
Expand Down Expand Up @@ -450,12 +526,12 @@ mod tests {

#[test]
fn test_benchmark_cpu() {
assert!(benchmark_cpu(DEFAULT_CPU_EXECUTION_LIMIT) > 0.0);
assert!(benchmark_cpu(DEFAULT_CPU_EXECUTION_LIMIT).to_bs() > 0.0);
}

#[test]
fn test_benchmark_memory() {
assert!(benchmark_memory(DEFAULT_MEMORY_EXECUTION_LIMIT) > 0.0);
assert!(benchmark_memory(DEFAULT_MEMORY_EXECUTION_LIMIT).to_bs() > 0.0);
}

#[test]
Expand Down
73 changes: 1 addition & 72 deletions utils/frame/benchmarking-cli/src/machine/hardware.rs
Original file line number Diff line number Diff line change
Expand Up @@ -18,8 +18,8 @@
//! Contains types to define hardware requirements.

use lazy_static::lazy_static;
use sc_sysinfo::Throughput;
use serde::{Deserialize, Serialize};
use std::fmt;

lazy_static! {
/// The hardware requirements as measured on reference hardware.
Expand Down Expand Up @@ -65,17 +65,6 @@ pub enum Metric {
DiskRndWrite,
}

/// Throughput as measured in bytes per second.
#[derive(Deserialize, Serialize, Debug, Clone, Copy, PartialEq)]
pub enum Throughput {
/// KiB/s
KiBs(f64),
/// MiB/s
MiBs(f64),
/// GiB/s
GiBs(f64),
}

impl Metric {
/// The category of the metric.
pub fn category(&self) -> &'static str {
Expand All @@ -98,66 +87,6 @@ impl Metric {
}
}

const KIBIBYTE: f64 = 1024.0;

impl Throughput {
/// The unit of the metric.
pub fn unit(&self) -> &'static str {
match self {
Self::KiBs(_) => "KiB/s",
Self::MiBs(_) => "MiB/s",
Self::GiBs(_) => "GiB/s",
}
}

/// [`Self`] as number of byte/s.
pub fn to_bs(&self) -> f64 {
self.to_kibs() * KIBIBYTE
}

/// [`Self`] as number of kibibyte/s.
pub fn to_kibs(&self) -> f64 {
self.to_mibs() * KIBIBYTE
}

/// [`Self`] as number of mebibyte/s.
pub fn to_mibs(&self) -> f64 {
self.to_gibs() * KIBIBYTE
}

/// [`Self`] as number of gibibyte/s.
pub fn to_gibs(&self) -> f64 {
match self {
Self::KiBs(k) => *k / (KIBIBYTE * KIBIBYTE),
Self::MiBs(m) => *m / KIBIBYTE,
Self::GiBs(g) => *g,
}
}

/// Normalizes [`Self`] to use the larges unit possible.
pub fn normalize(&self) -> Self {
let bs = self.to_bs();

if bs >= KIBIBYTE * KIBIBYTE * KIBIBYTE {
Self::GiBs(self.to_gibs())
} else if bs >= KIBIBYTE * KIBIBYTE {
Self::MiBs(self.to_mibs())
} else {
Self::KiBs(self.to_kibs())
}
}
}

impl fmt::Display for Throughput {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
let normalized = self.normalize();
match normalized {
Self::KiBs(s) | Self::MiBs(s) | Self::GiBs(s) =>
write!(f, "{:.2?} {}", s, normalized.unit()),
}
}
}

#[cfg(test)]
mod tests {
use super::*;
Expand Down
9 changes: 5 additions & 4 deletions utils/frame/benchmarking-cli/src/machine/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -30,11 +30,11 @@ use sc_cli::{CliConfiguration, Result, SharedParams};
use sc_service::Configuration;
use sc_sysinfo::{
benchmark_cpu, benchmark_disk_random_writes, benchmark_disk_sequential_writes,
benchmark_memory, benchmark_sr25519_verify, ExecutionLimit,
benchmark_memory, benchmark_sr25519_verify, ExecutionLimit, Throughput,
};

use crate::shared::check_build_profile;
pub use hardware::{Metric, Requirement, Requirements, Throughput, SUBSTRATE_REFERENCE_HARDWARE};
pub use hardware::{Metric, Requirement, Requirements, SUBSTRATE_REFERENCE_HARDWARE};

/// Command to benchmark the hardware.
///
Expand Down Expand Up @@ -128,6 +128,7 @@ impl MachineCmd {
/// Benchmarks a specific metric of the hardware and judges the resulting score.
fn run_benchmark(&self, requirement: &Requirement, dir: &Path) -> Result<BenchResult> {
// Dispatch the concrete function from `sc-sysinfo`.

let score = self.measure(&requirement.metric, dir)?;
let rel_score = score.to_bs() / requirement.minimum.to_bs();

Expand All @@ -147,9 +148,9 @@ impl MachineCmd {
let memory_limit = ExecutionLimit::from_secs_f32(self.memory_duration);

let score = match metric {
Metric::Blake2256 => Throughput::MiBs(benchmark_cpu(hash_limit) as f64),
Metric::Blake2256 => benchmark_cpu(hash_limit),
Metric::Sr25519Verify => Throughput::MiBs(benchmark_sr25519_verify(verify_limit)),
Metric::MemCopy => Throughput::MiBs(benchmark_memory(memory_limit) as f64),
Metric::MemCopy => benchmark_memory(memory_limit),
Metric::DiskSeqWrite =>
Throughput::MiBs(benchmark_disk_sequential_writes(disk_limit, dir)? as f64),
Metric::DiskRndWrite =>
Expand Down