[subsystem-benchmarks] Add statement-distribution benchmarks

.gitlab/pipeline/publish.yml

@@ -76,6 +76,8 @@ publish-subsystem-benchmarks:
       artifacts: true
     - job: subsystem-benchmark-approval-voting
       artifacts: true
+    - job: subsystem-benchmark-statement-distribution
+      artifacts: true
     - job: publish-rustdoc
       artifacts: false
   script:
@@ -119,6 +121,8 @@ trigger_workflow:
       artifacts: true
     - job: subsystem-benchmark-approval-voting
       artifacts: true
+    - job: subsystem-benchmark-statement-distribution
+      artifacts: true
   script:
     - echo "Triggering workflow"
     - >

.gitlab/pipeline/test.yml

@@ -619,3 +619,10 @@ subsystem-benchmark-approval-voting:
   script:
     - cargo bench -p polkadot-node-core-approval-voting --bench approval-voting-regression-bench --features subsystem-benchmarks
   allow_failure: true
+
+subsystem-benchmark-statement-distribution:
+  extends:
+    - .subsystem-benchmark-template
+  script:
+    - cargo bench -p polkadot-statement-distribution --bench statement-distribution-regression-bench --features subsystem-benchmarks
+  allow_failure: true

polkadot/node/network/statement-distribution/Cargo.toml

@@ -42,3 +42,13 @@ sc-network = { path = "../../../../substrate/client/network" }
 futures-timer = "3.0.2"
 polkadot-primitives-test-helpers = { path = "../../../primitives/test-helpers" }
 rand_chacha = "0.3"
+polkadot-subsystem-bench = { path = "../../subsystem-bench" }
+
+[[bench]]
+name = "statement-distribution-regression-bench"
+path = "benches/statement-distribution-regression-bench.rs"
+harness = false
+required-features = ["subsystem-benchmarks"]
+
+[features]
+subsystem-benchmarks = []

polkadot/node/network/statement-distribution/benches/statement-distribution-regression-bench.rs

@@ -0,0 +1,78 @@
+// Copyright (C) Parity Technologies (UK) Ltd.
+// This file is part of Polkadot.
+
+// Polkadot is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// Polkadot is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+
+// You should have received a copy of the GNU General Public License
+// along with Polkadot.  If not, see <http://www.gnu.org/licenses/>.
+
+//! statement-distribution regression tests
+//!
+//! Statement distribution benchmark based on Kusama parameters and scale.
+
+use polkadot_subsystem_bench::{
+	configuration::TestConfiguration,
+	statement::{benchmark_statement_distribution, prepare_test, TestState},
+	usage::BenchmarkUsage,
+	utils::save_to_file,
+};
+use std::io::Write;
+
+const BENCH_COUNT: usize = 50;
+
+fn main() -> Result<(), String> {
+	let mut messages = vec![];
+	let mut config = TestConfiguration::default();
+	config.n_cores = 100;
+	config.n_validators = 500;
+	config.num_blocks = 10;
+	config.connectivity = 100;
+	config.generate_pov_sizes();
+	let state = TestState::new(&config);
+
+	println!("Benchmarking...");
+	let usages: Vec<BenchmarkUsage> = (0..BENCH_COUNT)
+		.map(|n| {
+			print!("\r[{}{}]", "#".repeat(n), "_".repeat(BENCH_COUNT - n));
+			std::io::stdout().flush().unwrap();
+			let (mut env, _cfgs) = prepare_test(&state, false);
+			env.runtime().block_on(benchmark_statement_distribution(
+				"statement-distribution",
+				&mut env,
+				&state,
+			))
+		})
+		.collect();
+	println!("\rDone!{}", " ".repeat(BENCH_COUNT));
+
+	let average_usage = BenchmarkUsage::average(&usages);
+	save_to_file(
+		"charts/statement-distribution-regression-bench.json",
+		average_usage.to_chart_json().map_err(|e| e.to_string())?,
+	)
+	.map_err(|e| e.to_string())?;
+	println!("{}", average_usage);
+
+	// We expect no variance for received and sent
+	// but use 0.001 because we operate with floats
+	messages.extend(average_usage.check_network_usage(&[
+		("Received from peers", 106.4000, 0.001),
+		("Sent to peers", 127.9100, 0.001),
+	]));
+	messages.extend(average_usage.check_cpu_usage(&[("statement-distribution", 0.0390, 0.1)]));
+
+	if messages.is_empty() {
+		Ok(())
+	} else {
+		eprintln!("{}", messages.join("\n"));
+		Err("Regressions found".to_string())
+	}
+}

polkadot/node/network/statement-distribution/src/lib.rs

@@ -19,7 +19,6 @@
 //! This is responsible for distributing signed statements about candidate
 //! validity among validators.
 
-#![deny(unused_crate_dependencies)]
 #![warn(missing_docs)]
 
 use error::{log_error, FatalResult};

polkadot/node/subsystem-bench/Cargo.toml

@@ -28,6 +28,7 @@ polkadot-primitives = { path = "../../primitives" }
 polkadot-node-network-protocol = { path = "../network/protocol" }
 polkadot-availability-recovery = { path = "../network/availability-recovery", features = ["subsystem-benchmarks"] }
 polkadot-availability-distribution = { path = "../network/availability-distribution" }
+polkadot-statement-distribution = { path = "../network/statement-distribution" }
 polkadot-node-core-av-store = { path = "../core/av-store" }
 polkadot-node-core-chain-api = { path = "../core/chain-api" }
 polkadot-availability-bitfield-distribution = { path = "../network/bitfield-distribution" }

polkadot/node/subsystem-bench/examples/statement_distribution.yaml

@@ -0,0 +1,5 @@
+TestConfiguration:
+- objective: StatementDistribution
+  num_blocks: 10
+  n_cores: 100
+  n_validators: 500

polkadot/node/subsystem-bench/src/cli/subsystem-bench.rs

@@ -20,7 +20,7 @@
 use clap::Parser;
 use color_eyre::eyre;
 use colored::Colorize;
-use polkadot_subsystem_bench::{approval, availability, configuration};
+use polkadot_subsystem_bench::{approval, availability, configuration, statement};
 use pyroscope::PyroscopeAgent;
 use pyroscope_pprofrs::{pprof_backend, PprofConfig};
 use serde::{Deserialize, Serialize};
@@ -40,6 +40,8 @@ pub enum TestObjective {
 	DataAvailabilityWrite,
 	/// Benchmark the approval-voting and approval-distribution subsystems.
 	ApprovalVoting(approval::ApprovalsOptions),
+	// Benchmark the statement-distribution subsystem
+	StatementDistribution,
 }
 
 impl std::fmt::Display for TestObjective {
@@ -51,6 +53,7 @@ impl std::fmt::Display for TestObjective {
 				Self::DataAvailabilityRead(_) => "DataAvailabilityRead",
 				Self::DataAvailabilityWrite => "DataAvailabilityWrite",
 				Self::ApprovalVoting(_) => "ApprovalVoting",
+				Self::StatementDistribution => "StatementDistribution",
 			}
 		)
 	}
@@ -170,6 +173,15 @@ impl BenchCli {
 						state,
 					))
 				},
+				TestObjective::StatementDistribution => {
+					let state = statement::TestState::new(&test_config);
+					let (mut env, _protocol_config) = statement::prepare_test(&state, true);
+					env.runtime().block_on(statement::benchmark_statement_distribution(
+						&benchmark_name,
+						&mut env,
+						&state,
+					))
+				},
 			};
 			println!("{}", usage);
 		}

polkadot/node/subsystem-bench/src/lib/approval/mod.rs

@@ -30,7 +30,7 @@ use crate::{
 	mock::{
 		chain_api::{ChainApiState, MockChainApi},
 		network_bridge::{MockNetworkBridgeRx, MockNetworkBridgeTx},
-		runtime_api::MockRuntimeApi,
+		runtime_api::{MockRuntimeApi, MockRuntimeApiCoreState},
 		AlwaysSupportsParachains, TestSyncOracle,
 	},
 	network::{
@@ -465,8 +465,9 @@ impl ApprovalTestState {
 	}
 }
 
+#[async_trait::async_trait]
 impl HandleNetworkMessage for ApprovalTestState {
-	fn handle(
+	async fn handle(
 		&self,
 		_message: crate::network::NetworkMessage,
 		_node_sender: &mut futures::channel::mpsc::UnboundedSender<crate::network::NetworkMessage>,
@@ -807,6 +808,7 @@ fn build_overseer(
 		state.candidate_events_by_block(),
 		Some(state.babe_epoch.clone()),
 		1,
+		MockRuntimeApiCoreState::Occupied,
 	);
 	let mock_tx_bridge = MockNetworkBridgeTx::new(
 		network.clone(),
@@ -915,7 +917,9 @@ pub async fn bench_approvals_run(
 
 	// First create the initialization messages that make sure that then node under
 	// tests receives notifications about the topology used and the connected peers.
-	let mut initialization_messages = env.network().generate_peer_connected();
+	let mut initialization_messages = env.network().generate_peer_connected(|e| {
+		AllMessages::ApprovalDistribution(ApprovalDistributionMessage::NetworkBridgeUpdate(e))
+	});
 	initialization_messages.extend(generate_new_session_topology(
 		&state.test_authorities,
 		ValidatorIndex(NODE_UNDER_TEST),

polkadot/node/subsystem-bench/src/lib/availability/mod.rs

@@ -22,7 +22,7 @@ use crate::{
 		av_store::{self, MockAvailabilityStore, NetworkAvailabilityState},
 		chain_api::{ChainApiState, MockChainApi},
 		network_bridge::{self, MockNetworkBridgeRx, MockNetworkBridgeTx},
-		runtime_api::{self, MockRuntimeApi},
+		runtime_api::{self, MockRuntimeApi, MockRuntimeApiCoreState},
 		AlwaysSupportsParachains,
 	},
 	network::new_network,
@@ -189,6 +189,7 @@ pub fn prepare_test(
 		Default::default(),
 		Default::default(),
 		0,
+		MockRuntimeApiCoreState::Occupied,
 	);
 
 	let (overseer, overseer_handle) = match &mode {

polkadot/node/subsystem-bench/src/lib/configuration.rs

@@ -18,12 +18,13 @@
 
 use crate::keyring::Keyring;
 use itertools::Itertools;
-use polkadot_primitives::{AssignmentId, AuthorityDiscoveryId, ValidatorId};
+use polkadot_primitives::{AssignmentId, AuthorityDiscoveryId, ValidatorId, ValidatorPair};
 use rand::thread_rng;
 use rand_distr::{Distribution, Normal, Uniform};
 use sc_network_types::PeerId;
 use serde::{Deserialize, Serialize};
 use sp_consensus_babe::AuthorityId;
+use sp_core::Pair;
 use std::collections::HashMap;
 
 /// Peer networking latency configuration.
@@ -89,6 +90,15 @@ fn default_n_delay_tranches() -> usize {
 fn default_no_show_slots() -> usize {
 	3
 }
+fn default_minimum_backing_votes() -> u32 {
+	2
+}
+fn default_max_candidate_depth() -> u32 {
+	3
+}
+fn default_allowed_ancestry_len() -> u32 {
+	2
+}
 
 /// The test input parameters
 #[derive(Clone, Debug, Serialize, Deserialize)]
@@ -137,6 +147,15 @@ pub struct TestConfiguration {
 	pub connectivity: usize,
 	/// Number of blocks to run the test for
 	pub num_blocks: usize,
+	/// Number of minimum backing votes
+	#[serde(default = "default_minimum_backing_votes")]
+	pub minimum_backing_votes: u32,
+	/// Async Backing max_candidate_depth
+	#[serde(default = "default_max_candidate_depth")]
+	pub max_candidate_depth: u32,
+	/// Async Backing allowed_ancestry_len
+	#[serde(default = "default_allowed_ancestry_len")]
+	pub allowed_ancestry_len: u32,
 }
 
 impl Default for TestConfiguration {
@@ -158,6 +177,9 @@ impl Default for TestConfiguration {
 			latency: default_peer_latency(),
 			connectivity: default_connectivity(),
 			num_blocks: Default::default(),
+			minimum_backing_votes: default_minimum_backing_votes(),
+			max_candidate_depth: default_max_candidate_depth(),
+			allowed_ancestry_len: default_allowed_ancestry_len(),
 		}
 	}
 }
@@ -208,6 +230,11 @@ impl TestConfiguration {
 			.map(|(peer_id, authority_id)| (*peer_id, authority_id.clone()))
 			.collect();
 
+		let validator_pairs = key_seeds
+			.iter()
+			.map(|seed| ValidatorPair::from_string_with_seed(seed, None).unwrap().0)
+			.collect();
+
 		TestAuthorities {
 			keyring,
 			validator_public,
@@ -217,6 +244,7 @@ impl TestConfiguration {
 			validator_assignment_id,
 			key_seeds,
 			peer_id_to_authority,
+			validator_pairs,
 		}
 	}
 }
@@ -246,6 +274,7 @@ pub struct TestAuthorities {
 	pub key_seeds: Vec<String>,
 	pub peer_ids: Vec<PeerId>,
 	pub peer_id_to_authority: HashMap<PeerId, AuthorityDiscoveryId>,
+	pub validator_pairs: Vec<ValidatorPair>,
 }
 
 /// Sample latency (in milliseconds) from a normal distribution with parameters

polkadot/node/subsystem-bench/src/lib/lib.rs

@@ -14,7 +14,7 @@
 // You should have received a copy of the GNU General Public License
 // along with Polkadot.  If not, see <http://www.gnu.org/licenses/>.
 
-// The validator index that represent the node that is under test.
+// The validator index that represents the node that is under test.
 pub const NODE_UNDER_TEST: u32 = 0;
 
 pub mod approval;
@@ -25,5 +25,6 @@ pub(crate) mod environment;
 pub(crate) mod keyring;
 pub(crate) mod mock;
 pub(crate) mod network;
+pub mod statement;
 pub mod usage;
 pub mod utils;

polkadot/node/subsystem-bench/src/lib/mock/av_store.rs

@@ -49,8 +49,9 @@ pub struct NetworkAvailabilityState {
 }
 
 // Implement access to the state.
+#[async_trait::async_trait]
 impl HandleNetworkMessage for NetworkAvailabilityState {
-	fn handle(
+	async fn handle(
 		&self,
 		message: NetworkMessage,
 		_node_sender: &mut futures::channel::mpsc::UnboundedSender<NetworkMessage>,