From 9e453eae3b96c7c51e59ecb6345053641da220c5 Mon Sep 17 00:00:00 2001 From: serejkaaa512 <5125402@mail.ru> Date: Tue, 10 Dec 2024 09:32:23 +0300 Subject: [PATCH] feat(collator): use prev data accounts extra total count for finalize wu calculation --- Cargo.lock | 6 +-- Cargo.toml | 3 +- block-util/src/dict.rs | 3 +- cli/src/cmd/tools/gen_zerostate.rs | 10 ++-- collator/src/collator/do_collate/finalize.rs | 52 +++++++++++++------- collator/src/collator/do_collate/mod.rs | 2 +- collator/src/collator/types.rs | 4 +- 7 files changed, 44 insertions(+), 36 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e380c7a92..f29a865f8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -890,8 +890,7 @@ dependencies = [ [[package]] name = "everscale-types" version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b73dfe88b44fb8dab157b67818568764a4e43f049313eecaf58e13c55e913af" +source = "git+https://github.com/broxus/everscale-types.git?branch=feature/depth-balance-update#9e44841ce4641fde3778157de2cf1d31bf45ec2a" dependencies = [ "ahash", "anyhow", @@ -920,8 +919,7 @@ dependencies = [ [[package]] name = "everscale-types-proc" version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "817dbaf10f56aa40db0f83452c51671bd9f0b8741d56f38459ebb911959437e7" +source = "git+https://github.com/broxus/everscale-types.git?branch=feature/depth-balance-update#9e44841ce4641fde3778157de2cf1d31bf45ec2a" dependencies = [ "proc-macro2", "quote", diff --git a/Cargo.toml b/Cargo.toml index 6f4d6aa62..46f29c584 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -46,7 +46,7 @@ dirs = "5.0.1" ed25519 = "2.0" ed25519-dalek = "2.1" everscale-crypto = { version = "0.2", features = ["tl-proto", "serde"] } -everscale-types = { version = "0.1.2", features = ["tycho"] } +everscale-types = { git = "https://github.com/broxus/everscale-types.git", features = ["tycho"], branch = "feature/depth-balance-update" } exponential-backoff = "1" fdlimit = "0.3.0" futures-util = "0.3" @@ -136,6 +136,7 @@ tycho-util = { path = "./util", version = "0.1.4" } [patch.crates-io] weedb = { version = "0.3.8", git = "https://github.com/broxus/weedb.git", branch = "next-rocksdb" } +everscale-types = { git = "https://github.com/broxus/everscale-types.git", features = ["tycho"], branch = "feature/depth-balance-update" } [workspace.lints.rust] future_incompatible = "warn" diff --git a/block-util/src/dict.rs b/block-util/src/dict.rs index c4d2fc8ca..eb24ab5db 100644 --- a/block-util/src/dict.rs +++ b/block-util/src/dict.rs @@ -55,8 +55,7 @@ where Ok(Self { dict_root: build_aug_dict_from_sorted_iter( iter.into_iter().map(|(k, a, v)| { - // SAFETY: We know that this cell is not a library cell. - let value = unsafe { v.inner().as_slice_unchecked() }; + let value = v.inner().as_slice_allow_pruned(); (k, a, value) }), K::BITS, diff --git a/cli/src/cmd/tools/gen_zerostate.rs b/cli/src/cmd/tools/gen_zerostate.rs index 79f7b41ad..387298b04 100644 --- a/cli/src/cmd/tools/gen_zerostate.rs +++ b/cli/src/cmd/tools/gen_zerostate.rs @@ -324,10 +324,7 @@ impl ZerostateConfig { accounts.set( account, - DepthBalanceInfo { - balance, - split_depth: 0, - }, + DepthBalanceInfo { balance, count: 1 }, ShardAccount { account: account_state_cell, last_trans_hash: HashBytes::ZERO, @@ -671,15 +668,14 @@ fn make_default_params() -> Result { }, finalize: WorkUnitsParamsFinalize { build_transactions: 1_000, // 1 mcs - build_accounts: 500, // 0.5 mcs + build_accounts: 1200, // 1.2 mcs build_in_msg: 500, // 0.5 mcs build_out_msg: 500, // 0.5 mcs serialize_min: 15_000_000, // 15 ms serialize_accounts: 1_000, // 1 mcs serialize_msg: 2_000, // 2 mcs state_update_min: 15_000_000, // 15 ms - state_update_accounts: 500, // 0.5 mcs - state_update_msg: 2_000, // 2 mcs + state_update_accounts: 2200, // 2.2 mcs }, }, })?; diff --git a/collator/src/collator/do_collate/finalize.rs b/collator/src/collator/do_collate/finalize.rs index 649645946..d66bb80d3 100644 --- a/collator/src/collator/do_collate/finalize.rs +++ b/collator/src/collator/do_collate/finalize.rs @@ -31,6 +31,8 @@ use crate::tracing_targets; use crate::types::{BlockCandidate, CollationSessionInfo, CollatorConfig, McData, ShardHashesExt}; use crate::utils::block::detect_top_processed_to_anchor; +pub const LOGARITHM_DELIMITER: u64 = 1000; + pub struct FinalizeState { pub execute_result: ExecuteResult, } @@ -218,7 +220,8 @@ impl Phase { histogram_build_account_blocks_and_messages.finish(); let processed_accounts = processed_accounts_res?; - self.state.collation_data.accounts_count = processed_accounts.accounts_len as u64; + self.state.collation_data.updated_accounts_count = + processed_accounts.updated_accounts_count as u64; let in_msgs = in_msgs_res?; let out_msgs = out_msgs_res?; @@ -343,21 +346,29 @@ impl Phase { labels, ); - let accounts_count = self.state.collation_data.accounts_count; + let updated_accounts_count = self.state.collation_data.updated_accounts_count; + let old_accounts_count = self + .state + .prev_shard_data + .observable_accounts() + .root_extra() + .count as u64; let in_msgs_len = self.state.collation_data.in_msgs.len() as u64; let out_msgs_len = self.state.collation_data.out_msgs.len() as u64; finalize_wu_total = Self::calc_finalize_wu_total( - accounts_count, + old_accounts_count, + updated_accounts_count, in_msgs_len, out_msgs_len, wu_params_finalize, ); tracing::debug!(target: tracing_targets::COLLATOR, - "finalize_wu_total: {}, accounts_count: {}, in_msgs: {}, out_msgs: {} ", + "finalize_wu_total: {}, old_state_accounts_count: {}, updated_accounts_count: {}, in_msgs: {}, out_msgs: {} ", finalize_wu_total, - accounts_count, + old_accounts_count, + updated_accounts_count, in_msgs_len, out_msgs_len, ); @@ -640,7 +651,8 @@ impl Phase { } fn calc_finalize_wu_total( - accounts_count: u64, + old_accounts_count: u64, + updated_accounts_count: u64, in_msgs_len: u64, out_msgs_len: u64, wu_params_finalize: WorkUnitsParamsFinalize, @@ -655,13 +667,15 @@ impl Phase { serialize_msg, state_update_min, state_update_accounts, - state_update_msg, } = wu_params_finalize; - let accounts_count_logarithm = accounts_count.checked_ilog2().unwrap_or_default() as u64; - let build = accounts_count - .saturating_mul(accounts_count_logarithm) - .saturating_mul(build_accounts as u64); + let old_accounts_count_logarithm = + ((old_accounts_count as f64).log2() * LOGARITHM_DELIMITER as f64) as u64; + + let build = updated_accounts_count + .saturating_mul(build_accounts as u64) + .saturating_mul(old_accounts_count_logarithm) + .saturating_div(LOGARITHM_DELIMITER); let build_in_msg = in_msgs_len .saturating_mul(in_msgs_len.checked_ilog2().unwrap_or_default() as u64) .saturating_mul(build_in_msg as u64); @@ -676,15 +690,15 @@ impl Phase { let merkle_calc = std::cmp::max( state_update_min as u64, - accounts_count - .saturating_mul(accounts_count_logarithm) + updated_accounts_count .saturating_mul(state_update_accounts as u64) - .saturating_add(out_msgs_len.saturating_mul(state_update_msg as u64)), + .saturating_mul(old_accounts_count_logarithm) + .saturating_div(LOGARITHM_DELIMITER), ); let serialize = std::cmp::max( serialize_min as u64, - accounts_count.saturating_mul(serialize_accounts as u64), + updated_accounts_count.saturating_mul(serialize_accounts as u64), ) .saturating_add((in_msgs_len + out_msgs_len).saturating_mul(serialize_msg as u64)); @@ -937,7 +951,7 @@ impl Phase { shard_accounts.set_any( &updated_account.account_addr, &DepthBalanceInfo { - split_depth: 0, // NOTE: will need to set when we implement accounts split/merge logic + count: 1, balance: updated_account.balance.clone(), }, &updated_account.shard_account, @@ -972,7 +986,7 @@ impl Phase { account_blocks.insert(updated_account.account_addr, account_block); } - let accounts_len = account_blocks.len(); + let updated_accounts_count = account_blocks.len(); // TODO: Somehow consume accounts inside an iterator let account_blocks = RelaxedAugDict::try_from_sorted_iter_any( @@ -985,7 +999,7 @@ impl Phase { account_blocks: account_blocks.build()?, shard_accounts: shard_accounts.build()?, new_config_params, - accounts_len, + updated_accounts_count, }) } @@ -1227,7 +1241,7 @@ struct ProcessedAccounts { account_blocks: AccountBlocks, shard_accounts: ShardAccounts, new_config_params: Option, - accounts_len: usize, + updated_accounts_count: usize, } fn create_merkle_update( diff --git a/collator/src/collator/do_collate/mod.rs b/collator/src/collator/do_collate/mod.rs index ddf0be459..c80217209 100644 --- a/collator/src/collator/do_collate/mod.rs +++ b/collator/src/collator/do_collate/mod.rs @@ -1127,7 +1127,7 @@ impl CollatorStdImpl { metrics::gauge!("tycho_do_collate_tx_per_block", &labels) .set(collation_data.tx_count as f64); metrics::gauge!("tycho_do_collate_accounts_per_block", &labels) - .set(collation_data.accounts_count as f64); + .set(collation_data.updated_accounts_count as f64); metrics::counter!("tycho_do_collate_int_enqueue_count") .increment(collation_data.int_enqueue_count); metrics::counter!("tycho_do_collate_int_dequeue_count") diff --git a/collator/src/collator/types.rs b/collator/src/collator/types.rs index 56103e635..0312b9481 100644 --- a/collator/src/collator/types.rs +++ b/collator/src/collator/types.rs @@ -306,7 +306,7 @@ impl BlockCollationDataBuilder { start_lt, next_lt: start_lt + 1, tx_count: 0, - accounts_count: 0, + updated_accounts_count: 0, total_execute_msgs_time_mc: 0, execute_count_all: 0, execute_count_ext: 0, @@ -339,7 +339,7 @@ pub(super) struct BlockCollationData { pub gen_utime_ms: u16, pub tx_count: u64, - pub accounts_count: u64, + pub updated_accounts_count: u64, pub block_limit: BlockLimitStats,