Benchmark with main spec (paritytech#802)

* Benchmark with main spec

* Update benchmark.md

.github/workflows/parachain.yml

@@ -86,7 +86,6 @@ jobs:
             --manifest-path parachain/Cargo.toml
             --verbose
             --package snowbridge-ethereum-beacon-client
-            --features runtime-benchmarks
             --features minimal
           toolchain: nightly-2022-11-15
       - uses: actions-rs/install@v0.1.2

core/packages/test/.envrc-example

@@ -19,11 +19,11 @@ source_up_if_exists
 # export ETH_WS_ENDPOINT=wss://goerli.infura.io/ws/v3
 # export BEACON_HTTP_ENDPOINT=https://lodestar-goerli.chainsafe.io
 # export ETH_GAS_LIMIT=0  # For real network just uncomment and set as 0 here
+# export ACTIVE_SPEC=mainnet # For real network just uncomment and set as mainnet here
 
 # export DEPLOYER_ETH_KEY=           # Your private key to deploy contracts
 # export BEEFY_RELAY_ETH_KEY=         # Your Beefy relayer account private key
 # export PARACHAIN_RELAY_ETH_KEY=     # Your Parachain relayer account private key
 # export E2E_TEST_ETH_KEY=            # Your E2E test account private key
 
 # export CUMULUS_VER=snowbridge       # For dev setup change to some feature branch
-

core/packages/test/scripts/build-binary.sh

@@ -26,13 +26,17 @@ rebuild_cumulus(){
         --git https://github.com/Snowfork/cumulus \
         --branch "$cumulus_version" polkadot-parachain-bin \
         --locked \
-        --root $cumulus_dir #add version path to root to avoid recompiling when switch between versions 
+        --root $cumulus_dir #add version path to root to avoid recompiling when switch between versions
     popd
 }
 
 build_cumulus_from_source(){
     pushd $root_dir/cumulus
-    cargo build --release --bin polkadot-parachain
+    if [[ "$active_spec" == "minimal" ]]; then
+      cargo build --features "minimal" --release --bin polkadot-parachain
+    else
+      cargo build --release --bin polkadot-parachain
+    fi
     cp target/release/polkadot-parachain $output_bin_dir/polkadot-parachain
     popd
 }
@@ -44,7 +48,7 @@ rebuild_relaychain(){
         --git https://github.com/paritytech/polkadot \
         --tag "$relaychain_version" polkadot \
         --locked \
-        --root $relaychain_dir #add version path to root to avoid recompiling when switch between versions 
+        --root $relaychain_dir #add version path to root to avoid recompiling when switch between versions
     popd
 }
 

core/packages/test/scripts/set-env.sh

@@ -24,6 +24,9 @@ export output_bin_dir="$output_dir/bin"
 ethereum_data_dir="$output_dir/geth"
 export PATH="$output_bin_dir:$PATH"
 
+# Because we can run a local node for public network like goerli or mainnet, add active_spec as a separate config here
+# to decouple with eth_network, explicit set ACTIVE_SPEC to mainnet when test with public network
+active_spec="${ACTIVE_SPEC:-minimal}"
 eth_network="${ETH_NETWORK:-localhost}"
 eth_endpoint_http="${ETH_RPC_ENDPOINT:-http://127.0.0.1:8545}/${INFURA_PROJECT_ID:-}"
 eth_endpoint_ws="${ETH_WS_ENDPOINT:-ws://127.0.0.1:8546}/${INFURA_PROJECT_ID:-}"
@@ -114,4 +117,3 @@ check_tool() {
         exit
     fi
 }
-

core/packages/test/scripts/start-relayer.sh

@@ -35,11 +35,6 @@ config_relayer(){
     ' \
     config/parachain-relay.json > $output_dir/parachain-relay.json
 
-    active_spec="mainnet"
-    if [ "$eth_network" == "localhost" ]; then
-        active_spec="minimal"
-    fi
-
     # Configure beacon relay
     jq \
         --arg beacon_endpoint_http $beacon_endpoint_http \
@@ -141,4 +136,3 @@ if [ -z "${from_start_services:-}" ]; then
     check_tool && build_relayer && rm -rf *relay.log && start_relayer
     wait
 fi
-

parachain/pallets/ethereum-beacon-client/Cargo.toml

@@ -48,7 +48,7 @@ std = [
     "scale-info/std",
     "frame-support/std",
     "frame-system/std",
-    'frame-benchmarking?/std',
+    'frame-benchmarking/std',
     "sp-core/std",
     "sp-io/std",
     "sp-runtime/std",
@@ -62,9 +62,10 @@ std = [
     "pallet-timestamp/std"
 ]
 runtime-benchmarks = [
-    "frame-benchmarking",
+    "frame-benchmarking/runtime-benchmarks",
     "frame-support/runtime-benchmarks",
     "frame-system/runtime-benchmarks",
     "hex-literal"
 ]
+
 minimal = []

parachain/pallets/ethereum-beacon-client/benchmark.md

@@ -0,0 +1,77 @@
+# Motivation
+Demonstrate that [FastAggregateVerify](https://datatracker.ietf.org/doc/html/draft-irtf-cfrg-bls-signature-04#section-3.3.4) is the most expensive call in ethereum beacon light client, though in [#13031](https://github.com/paritytech/substrate/pull/13031) Parity team has wrapped some low level host functions for `bls-12381` but adding a high level host function specific for it is super helpful.
+
+# Benchmark
+We add several benchmarks [here](https://github.com/Snowfork/snowbridge/blob/8891ca3cdcf2e04d8118c206588c956541ae4710/parachain/pallets/ethereum-beacon-client/src/benchmarking/mod.rs#L98-L124) as following to demonstrate [bls_fast_aggregate_verify](https://github.com/Snowfork/snowbridge/blob/8891ca3cdcf2e04d8118c206588c956541ae4710/parachain/pallets/ethereum-beacon-client/src/lib.rs#L764) is the main bottleneck. Test data [here](https://github.com/Snowfork/snowbridge/blob/8891ca3cdcf2e04d8118c206588c956541ae4710/parachain/pallets/ethereum-beacon-client/src/benchmarking/data_mainnet.rs#L553-L1120) is real from goerli network which contains 512 public keys from sync committee.
+
+
+## sync_committee_period_update
+Base line benchmark for extrinsic [sync_committee_period_update](https://github.com/Snowfork/snowbridge/blob/8891ca3cdcf2e04d8118c206588c956541ae4710/parachain/pallets/ethereum-beacon-client/src/lib.rs#L233)
+
+
+## bls_fast_aggregate_verify
+Subfunction of extrinsic `sync_committee_period_update` which does what [FastAggregateVerify](https://datatracker.ietf.org/doc/html/draft-irtf-cfrg-bls-signature-04#section-3.3.4) requires.
+
+## bls_aggregate_pubkey
+Subfunction of `bls_fast_aggregate_verify` which decompress and instantiate G1 pubkeys only.
+
+## bls_verify_message
+Subfunction of `bls_fast_aggregate_verify` which verify the prepared signature only.
+
+
+# Result
+
+## hardware spec
+Run benchmark in a EC2 instance 
+```
+cargo run --release --bin polkadot-parachain --features runtime-benchmarks -- benchmark machine --base-path /mnt/scratch/benchmark
+
++----------+----------------+-------------+-------------+-------------------+
+| Category | Function       | Score       | Minimum     | Result            |
++===========================================================================+
+| CPU      | BLAKE2-256     | 1.08 GiBs   | 1.00 GiBs   | ✅ Pass (107.5 %) |
+|----------+----------------+-------------+-------------+-------------------|
+| CPU      | SR25519-Verify | 568.87 KiBs | 666.00 KiBs | ❌ Fail ( 85.4 %) |
+|----------+----------------+-------------+-------------+-------------------|
+| Memory   | Copy           | 13.67 GiBs  | 14.32 GiBs  | ✅ Pass ( 95.4 %) |
+|----------+----------------+-------------+-------------+-------------------|
+| Disk     | Seq Write      | 334.35 MiBs | 450.00 MiBs | ❌ Fail ( 74.3 %) |
+|----------+----------------+-------------+-------------+-------------------|
+| Disk     | Rnd Write      | 143.59 MiBs | 200.00 MiBs | ❌ Fail ( 71.8 %) |
++----------+----------------+-------------+-------------+-------------------+
+```
+
+## benchmark
+
+```
+cargo run --release --bin polkadot-parachain \
+--features runtime-benchmarks \
+-- \
+benchmark pallet \
+--base-path /mnt/scratch/benchmark \
+--chain=bridge-hub-rococo-dev \
+--pallet=snowbridge_ethereum_beacon_client \
+--extrinsic="*" \
+--execution=wasm --wasm-execution=compiled \
+--steps 50 --repeat 20 \
+--output ./parachains/runtimes/bridge-hubs/bridge-hub-rococo/src/weights/snowbridge_ethereum_beacon_client.rs
+```
+
+### [Weights](https://github.com/Snowfork/cumulus/blob/ron/benchmark-beacon-bridge/parachains/runtimes/bridge-hubs/bridge-hub-rococo/src/weights/snowbridge_ethereum_beacon_client.rs)
+
+|extrinsic       | minimum execution time benchmarked(us) |
+| --------------------------------------- |----------------------------------------|
+|sync_committee_period_update | 123_126                                |                              
+|bls_fast_aggregate_verify| 121_083                                |
+|bls_aggregate_pubkey | 90_306                                  |
+|bls_verify_message | 28_000                                  |
+
+- [bls_fast_aggregate_verify](#bls_fast_aggregate_verify) consumes 98% execution time of [sync_committee_period_update](#sync_committee_period_update)
+
+- [bls_aggregate_pubkey](#bls_aggregate_pubkey) consumes 75% execution time of [bls_fast_aggregate_verify](#bls_fast_aggregate_verify)
+
+- [bls_verify_message](#bls_verify_message) consumes 23% execution time of [bls_fast_aggregate_verify](#bls_fast_aggregate_verify)
+
+# Conclusion
+
+A high level host function specific for  [bls_fast_aggregate_verify](https://github.com/Snowfork/snowbridge/blob/8891ca3cdcf2e04d8118c206588c956541ae4710/parachain/pallets/ethereum-beacon-client/src/lib.rs#L764) is super helpful.

parachain/pallets/ethereum-beacon-client/src/benchmarking/mod.rs

@@ -4,36 +4,17 @@ use crate::Pallet as EthereumBeaconClient;
 use frame_benchmarking::{benchmarks, impl_benchmark_test_suite, whitelisted_caller};
 use frame_system::RawOrigin;
 
-#[cfg(feature = "minimal")]
-mod data_minimal;
-#[cfg(feature = "minimal")]
-use data_minimal::*;
-
-#[cfg(not(feature = "minimal"))]
+// For benchmark focus on main spec only
 mod data_mainnet;
-#[cfg(not(feature = "minimal"))]
 use data_mainnet::*;
+mod util;
+use util::*;
 
 benchmarks! {
 	sync_committee_period_update {
 		let caller: T::AccountId = whitelisted_caller();
 
-		let initial_sync_data = initial_sync();
-
-		EthereumBeaconClient::<T>::initial_sync(initial_sync_data.clone())?;
-
-		let sync_committee_update = sync_committee_update();
-
-		//initialize SyncCommittees with period in sync_committee_update
-		LatestSyncCommitteePeriod::<T>::set(EthereumBeaconClient::<T>::compute_current_sync_period(
-				sync_committee_update.attested_header.slot,
-			));
-		SyncCommittees::<T>::insert(
-			EthereumBeaconClient::<T>::compute_current_sync_period(
-				sync_committee_update.attested_header.slot,
-			),
-			initial_sync_data.current_sync_committee,
-		);
+		let sync_committee_update = initialize_sync_committee::<T>()?;
 
 	}: sync_committee_period_update(RawOrigin::Signed(caller.clone()), sync_committee_update.clone())
 	verify {
@@ -107,16 +88,42 @@ benchmarks! {
 		let header: ExecutionHeader = header_update.execution_header.try_into().unwrap();
 		<ExecutionHeaders<T>>::get(header.block_hash).unwrap();
 	}
-}
 
-#[cfg(feature = "minimal")]
-impl_benchmark_test_suite!(
-	EthereumBeaconClient,
-	crate::mock::new_tester::<crate::mock::mock_minimal::Test>(),
-	crate::mock::mock_minimal::Test,
-);
+	unblock_bridge {
+	}: _(RawOrigin::Root)
+	verify {
+		assert_eq!(<Blocked<T>>::get(),false);
+	}
+
+	bls_fast_aggregate_verify {
+		let update = initialize_sync_committee::<T>()?;
+		let participant_pubkeys = get_participant_pubkeys::<T>(&update)?;
+		let signing_root = get_signing_message::<T>(&update)?;
+	}:{
+		EthereumBeaconClient::<T>::bls_fast_aggregate_verify(participant_pubkeys,signing_root,update.sync_aggregate.sync_committee_signature)?;
+	}
+
+	bls_aggregate_pubkey {
+		let update = initialize_sync_committee::<T>()?;
+		let participant_pubkeys = get_participant_pubkeys::<T>(&update)?;
+	}:{
+		participant_pubkeys
+				.iter()
+				.map(|bytes| milagro_bls::PublicKey::from_bytes_unchecked(&bytes.0))
+				.collect::<Result<Vec<milagro_bls::PublicKey>, _>>().unwrap()
+	}
+
+	bls_verify_message {
+		let update = initialize_sync_committee::<T>()?;
+		let participant_pubkeys = get_participant_pubkeys::<T>(&update)?;
+		let signing_root = get_signing_message::<T>(&update)?;
+		let agg_sig = get_aggregate_signature::<T>(update.sync_aggregate.sync_committee_signature).unwrap();
+		let agg_pub_key = get_aggregate_pubkey::<T>(participant_pubkeys).unwrap();
+	}:{
+		agg_sig.fast_aggregate_verify_pre_aggregated(&signing_root.as_bytes(), &agg_pub_key)
+	}
+}
 
-#[cfg(not(feature = "minimal"))]
 impl_benchmark_test_suite!(
 	EthereumBeaconClient,
 	crate::mock::new_tester::<crate::mock::mock_mainnet::Test>(),

parachain/pallets/ethereum-beacon-client/src/benchmarking/util.rs

@@ -0,0 +1,83 @@
+use super::*;
+use crate::Pallet as EthereumBeaconClient;
+use milagro_bls::{AggregatePublicKey, AggregateSignature, Signature};
+
+pub fn initialize_sync_committee<T: Config>() -> Result<SyncCommitteePeriodUpdateOf<T>, &'static str>
+{
+	let initial_sync_data = initial_sync();
+
+	EthereumBeaconClient::<T>::initial_sync(initial_sync_data.clone())?;
+
+	let sync_committee_update: SyncCommitteePeriodUpdateOf<T> = sync_committee_update();
+
+	//initialize SyncCommittees with period in sync_committee_update
+	LatestSyncCommitteePeriod::<T>::set(EthereumBeaconClient::<T>::compute_current_sync_period(
+		sync_committee_update.attested_header.slot,
+	));
+	SyncCommittees::<T>::insert(
+		EthereumBeaconClient::<T>::compute_current_sync_period(
+			sync_committee_update.attested_header.slot,
+		),
+		initial_sync_data.current_sync_committee,
+	);
+	Ok(sync_committee_update)
+}
+
+pub fn get_participant_pubkeys<T: Config>(
+	update: &SyncCommitteePeriodUpdateOf<T>,
+) -> Result<Vec<PublicKey>, &'static str> {
+	let sync_committee_bits =
+		get_sync_committee_bits(update.sync_aggregate.sync_committee_bits.clone()).unwrap();
+	let current_period =
+		EthereumBeaconClient::<T>::compute_current_sync_period(update.attested_header.slot);
+	let current_sync_committee =
+		EthereumBeaconClient::<T>::get_sync_committee_for_period(current_period)?;
+	let sync_committee_pubkeys = current_sync_committee.pubkeys;
+	let mut participant_pubkeys: Vec<PublicKey> = Vec::new();
+	for (bit, pubkey) in sync_committee_bits.iter().zip(sync_committee_pubkeys.iter()) {
+		if *bit == 1 as u8 {
+			let pubk = pubkey.clone();
+			participant_pubkeys.push(pubk);
+		}
+	}
+	Ok(participant_pubkeys)
+}
+
+pub fn get_signing_message<T: Config>(
+	update: &SyncCommitteePeriodUpdateOf<T>,
+) -> Result<Root, &'static str> {
+	let validators_root = <ValidatorsRoot<T>>::get();
+	let fork_version = EthereumBeaconClient::<T>::compute_fork_version(
+		EthereumBeaconClient::<T>::compute_epoch_at_slot(
+			update.signature_slot,
+			config::SLOTS_PER_EPOCH,
+		),
+	);
+	let domain_type = config::DOMAIN_SYNC_COMMITTEE.to_vec();
+	let domain =
+		EthereumBeaconClient::<T>::compute_domain(domain_type, fork_version, validators_root)?;
+	let signing_root =
+		EthereumBeaconClient::<T>::compute_signing_root(update.attested_header.clone(), domain)?;
+	Ok(signing_root)
+}
+
+pub fn get_aggregate_signature<T: Config>(
+	signature: BoundedVec<u8, T::MaxSignatureSize>,
+) -> Result<AggregateSignature, Error<T>> {
+	let sig = Signature::from_bytes(&signature[..]).map_err(|_| Error::<T>::InvalidSignature)?;
+	let agg_sig = AggregateSignature::from_signature(&sig);
+	Ok(agg_sig)
+}
+
+pub fn get_aggregate_pubkey<T: Config>(
+	pubkeys: Vec<PublicKey>,
+) -> Result<AggregatePublicKey, Error<T>> {
+	let milagro_public_keys = pubkeys
+		.iter()
+		.map(|bytes| milagro_bls::PublicKey::from_bytes_unchecked(&bytes.0))
+		.collect::<Result<Vec<milagro_bls::PublicKey>, _>>()
+		.map_err(|_| Error::<T>::InvalidSignaturePoint)?;
+	let agg_pub_key = AggregatePublicKey::into_aggregate(&milagro_public_keys)
+		.map_err(|_| Error::<T>::InvalidAggregatePublicKeys)?;
+	Ok(agg_pub_key)
+}

parachain/pallets/ethereum-beacon-client/src/lib.rs

@@ -342,7 +342,7 @@ pub mod pallet {
 		}
 
 		#[pallet::call_index(3)]
-		#[pallet::weight({1000})]
+		#[pallet::weight(T::WeightInfo::unblock_bridge())]
 		#[transactional]
 		pub fn unblock_bridge(origin: OriginFor<T>) -> DispatchResult {
 			let _sender = ensure_root(origin)?;