forked from astriaorg/astria
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
perf(sequencer): add benchmark for prepare_proposal (ENG-660) (astria…
…org#1337) ## Summary Addition of a new benchmark target to assess the performance of the prepare_proposal method in sequencer's `App`. ## Background Previous perf work has indicated this is a bottleneck. However, making that determination was done via spamoor which is slightly convoluted to run. Before working to improve the performance, we want to have a faster feedback loop on the effects of updates, hence the need for a benchmark which is easy to run and which isolates the slow function. ## Changes - Added benchmark to `app` module. Currently this has only one case: a mempool filled with transactions containing exclusively transfers. This matches the shape of the data being sent when using spamoor. - Added `benchmark` feature to enable sharing some of the existing test utils. ## Testing This is a new test. Example of running `cargo bench --features=benchmark -qp astria-sequencer app` on my Ryzen 7900X: ``` Timer precision: 10 ns astria_sequencer fastest │ slowest │ median │ mean │ samples │ iters ╰─ app │ │ │ │ │ ╰─ benchmarks │ │ │ │ │ ╰─ execute_transactions_prepare_proposal 11.63 s │ 14.68 s │ 12.74 s │ 12.88 s │ 10 │ 10 ``` Since rebasing after astriaorg#1317 has merged, the same run shows (as expected) a slowdown: ``` astria_sequencer fastest │ slowest │ median │ mean │ samples │ iters ╰─ app │ │ │ │ │ ╰─ benchmarks │ │ │ │ │ ╰─ execute_transactions_prepare_proposal 14.49 s │ 17 s │ 16.52 s │ 15.98 s │ 8 │ 8 ``` ## Related Issues Closes astriaorg#1314.
- Loading branch information
Showing
9 changed files
with
288 additions
and
98 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,113 @@ | ||
| //! To run the benchmark, from the root of the monorepo, run: | ||
| //! ```sh | ||
| //! cargo bench --features=benchmark -qp astria-sequencer app | ||
| //! ``` | ||
| use std::time::Duration; | ||
|
|
||
| use astria_core::sequencer::{ | ||
| Account, | ||
| AddressPrefixes, | ||
| GenesisState, | ||
| UncheckedGenesisState, | ||
| }; | ||
| use cnidarium::Storage; | ||
| use penumbra_ibc::params::IBCParameters; | ||
|
|
||
| use crate::{ | ||
| app::{ | ||
| test_utils, | ||
| App, | ||
| }, | ||
| benchmark_utils::{ | ||
| self, | ||
| TxTypes, | ||
| SIGNER_COUNT, | ||
| }, | ||
| proposal::block_size_constraints::BlockSizeConstraints, | ||
| test_utils::{ | ||
| astria_address, | ||
| nria, | ||
| ASTRIA_PREFIX, | ||
| }, | ||
| }; | ||
|
|
||
| /// The max time for any benchmark. | ||
| const MAX_TIME: Duration = Duration::from_secs(120); | ||
| /// The value provided to `BlockSizeConstraints::new` to constrain block sizes. | ||
| /// | ||
| /// Taken from the actual value seen in `prepare_proposal.max_tx_bytes` when handling | ||
| /// `prepare_proposal` during stress testing using spamoor. | ||
| const COMETBFT_MAX_TX_BYTES: usize = 22_019_254; | ||
|
|
||
| struct Fixture { | ||
| app: App, | ||
| _storage: Storage, | ||
| } | ||
|
|
||
| impl Fixture { | ||
| /// Initializes a new `App` instance with the genesis accounts derived from the secret keys of | ||
| /// `benchmark_utils::signing_keys()`, and inserts transactions into the app mempool. | ||
| async fn new() -> Fixture { | ||
| let accounts = benchmark_utils::signing_keys() | ||
| .enumerate() | ||
| .take(usize::from(SIGNER_COUNT)) | ||
| .map(|(index, signing_key)| Account { | ||
| address: astria_address(&signing_key.address_bytes()), | ||
| balance: 10u128 | ||
| .pow(19) | ||
| .saturating_add(u128::try_from(index).unwrap()), | ||
| }) | ||
| .collect::<Vec<_>>(); | ||
| let address_prefixes = AddressPrefixes { | ||
| base: ASTRIA_PREFIX.into(), | ||
| }; | ||
| let first_address = accounts.first().unwrap().address; | ||
| let unchecked_genesis_state = UncheckedGenesisState { | ||
| accounts, | ||
| address_prefixes, | ||
| authority_sudo_address: first_address, | ||
| ibc_sudo_address: first_address, | ||
| ibc_relayer_addresses: vec![], | ||
| native_asset_base_denomination: nria(), | ||
| ibc_params: IBCParameters::default(), | ||
| allowed_fee_assets: vec![nria().into()], | ||
| fees: test_utils::default_fees(), | ||
| }; | ||
| let genesis_state = GenesisState::try_from(unchecked_genesis_state).unwrap(); | ||
|
|
||
| let (app, storage) = | ||
| test_utils::initialize_app_with_storage(Some(genesis_state), vec![]).await; | ||
|
|
||
| for tx in benchmark_utils::transactions(TxTypes::AllTransfers) { | ||
| app.mempool.insert(tx.clone(), 0).await.unwrap(); | ||
| } | ||
| Fixture { | ||
| app, | ||
| _storage: storage, | ||
| } | ||
| } | ||
| } | ||
|
|
||
| #[divan::bench(max_time = MAX_TIME)] | ||
| fn execute_transactions_prepare_proposal(bencher: divan::Bencher) { | ||
| let runtime = tokio::runtime::Builder::new_multi_thread() | ||
| .enable_all() | ||
| .build() | ||
| .unwrap(); | ||
| let mut fixture = runtime.block_on(async { Fixture::new().await }); | ||
| bencher | ||
| .with_inputs(|| BlockSizeConstraints::new(COMETBFT_MAX_TX_BYTES).unwrap()) | ||
| .bench_local_refs(|constraints| { | ||
| let (_tx_bytes, included_txs) = runtime.block_on(async { | ||
| fixture | ||
| .app | ||
| .execute_transactions_prepare_proposal(constraints) | ||
| .await | ||
| .unwrap() | ||
| }); | ||
| // Ensure we actually processed some txs. This will trip if execution fails for all | ||
| // txs, or more likely, if the mempool becomes exhausted of txs. | ||
| assert!(!included_txs.is_empty()); | ||
| }); | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,132 @@ | ||
| use std::{ | ||
| collections::HashMap, | ||
| sync::{ | ||
| Arc, | ||
| OnceLock, | ||
| }, | ||
| }; | ||
|
|
||
| use astria_core::{ | ||
| crypto::SigningKey, | ||
| primitive::v1::{ | ||
| asset::{ | ||
| Denom, | ||
| IbcPrefixed, | ||
| }, | ||
| RollupId, | ||
| }, | ||
| protocol::transaction::v1alpha1::{ | ||
| action::{ | ||
| Action, | ||
| SequenceAction, | ||
| TransferAction, | ||
| }, | ||
| SignedTransaction, | ||
| TransactionParams, | ||
| UnsignedTransaction, | ||
| }, | ||
| }; | ||
|
|
||
| use crate::test_utils::{ | ||
| astria_address, | ||
| nria, | ||
| }; | ||
|
|
||
| /// The number of different signers of transactions, and also the number of different chain IDs. | ||
| pub(crate) const SIGNER_COUNT: u8 = 10; | ||
| /// The number of transfers per transaction. | ||
| /// | ||
| /// 2866 chosen after experimentation of spamming composer. | ||
| pub(crate) const TRANSFERS_PER_TX: usize = 2866; | ||
|
|
||
| const SEQUENCE_ACTION_TX_COUNT: usize = 100_001; | ||
| const TRANSFERS_TX_COUNT: usize = 10_000; | ||
|
|
||
| #[derive(Copy, Clone, Eq, PartialEq, Hash, Debug)] | ||
| pub(crate) enum TxTypes { | ||
| AllSequenceActions, | ||
| AllTransfers, | ||
| } | ||
|
|
||
| /// Returns an endlessly-repeating iterator over `SIGNER_COUNT` separate signing keys. | ||
| pub(crate) fn signing_keys() -> impl Iterator<Item = &'static SigningKey> { | ||
| static SIGNING_KEYS: OnceLock<Vec<SigningKey>> = OnceLock::new(); | ||
| SIGNING_KEYS | ||
| .get_or_init(|| { | ||
| (0..SIGNER_COUNT) | ||
| .map(|i| SigningKey::from([i; 32])) | ||
| .collect() | ||
| }) | ||
| .iter() | ||
| .cycle() | ||
| } | ||
|
|
||
| /// Returns a static ref to a collection of `MAX_INITIAL_TXS + 1` transactions. | ||
| pub(crate) fn transactions(tx_types: TxTypes) -> &'static Vec<Arc<SignedTransaction>> { | ||
| static TXS: OnceLock<HashMap<TxTypes, Vec<Arc<SignedTransaction>>>> = OnceLock::new(); | ||
| TXS.get_or_init(|| { | ||
| let mut map = HashMap::new(); | ||
| map.insert(TxTypes::AllSequenceActions, sequence_actions()); | ||
| map.insert(TxTypes::AllTransfers, transfers()); | ||
| map | ||
| }) | ||
| .get(&tx_types) | ||
| .unwrap() | ||
| } | ||
|
|
||
| fn sequence_actions() -> Vec<Arc<SignedTransaction>> { | ||
| let mut nonces_and_chain_ids = HashMap::new(); | ||
| signing_keys() | ||
| .map(move |signing_key| { | ||
| let verification_key = signing_key.verification_key(); | ||
| let (nonce, chain_id) = nonces_and_chain_ids | ||
| .entry(verification_key) | ||
| .or_insert_with(|| (0_u32, format!("chain-{}", signing_key.verification_key()))); | ||
| *nonce = (*nonce).wrapping_add(1); | ||
| let params = TransactionParams::builder() | ||
| .nonce(*nonce) | ||
| .chain_id(chain_id.as_str()) | ||
| .build(); | ||
| let sequence_action = SequenceAction { | ||
| rollup_id: RollupId::new([1; 32]), | ||
| data: vec![2; 1000].into(), | ||
| fee_asset: Denom::IbcPrefixed(IbcPrefixed::new([3; 32])), | ||
| }; | ||
| let tx = UnsignedTransaction { | ||
| actions: vec![Action::Sequence(sequence_action)], | ||
| params, | ||
| } | ||
| .into_signed(signing_key); | ||
| Arc::new(tx) | ||
| }) | ||
| .take(SEQUENCE_ACTION_TX_COUNT) | ||
| .collect() | ||
| } | ||
|
|
||
| fn transfers() -> Vec<Arc<SignedTransaction>> { | ||
| let sender = signing_keys().next().unwrap(); | ||
| let receiver = signing_keys().nth(1).unwrap(); | ||
| let to = astria_address(&receiver.address_bytes()); | ||
| let action = Action::from(TransferAction { | ||
| to, | ||
| amount: 1, | ||
| asset: nria().into(), | ||
| fee_asset: nria().into(), | ||
| }); | ||
| (0..TRANSFERS_TX_COUNT) | ||
| .map(|nonce| { | ||
| let params = TransactionParams::builder() | ||
| .nonce(u32::try_from(nonce).unwrap()) | ||
| .chain_id("test") | ||
| .build(); | ||
| let tx = UnsignedTransaction { | ||
| actions: std::iter::repeat(action.clone()) | ||
| .take(TRANSFERS_PER_TX) | ||
| .collect(), | ||
| params, | ||
| } | ||
| .into_signed(sender); | ||
| Arc::new(tx) | ||
| }) | ||
| .collect() | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.