From bcc98af4c2042b33ccbdc1dec13fce4a272149c9 Mon Sep 17 00:00:00 2001 From: Wen <113942165+wen-coding@users.noreply.github.com> Date: Fri, 13 Sep 2024 10:07:31 -0700 Subject: [PATCH 01/25] wen_restart: Change HeaviestFork stage to a leader based approach. --- core/src/validator.rs | 5 +- local-cluster/src/validator_configs.rs | 1 + multinode-demo/bootstrap-validator.sh | 3 + multinode-demo/validator.sh | 3 + net/net.sh | 2 +- net/remote/remote-node.sh | 3 +- validator/src/cli.rs | 16 + validator/src/main.rs | 1 + wen-restart/proto/wen_restart.proto | 14 +- wen-restart/src/heaviest_fork_aggregate.rs | 665 ------------------- wen-restart/src/lib.rs | 1 - wen-restart/src/wen_restart.rs | 702 +++++++-------------- 12 files changed, 247 insertions(+), 1169 deletions(-) delete mode 100644 wen-restart/src/heaviest_fork_aggregate.rs diff --git a/core/src/validator.rs b/core/src/validator.rs index 555b1221df6e1c..f689dcc00aac27 100644 --- a/core/src/validator.rs +++ b/core/src/validator.rs @@ -276,6 +276,7 @@ pub struct ValidatorConfig { pub generator_config: Option, pub use_snapshot_archives_at_startup: UseSnapshotArchivesAtStartup, pub wen_restart_proto_path: Option, + pub wen_restart_leader: Option, pub unified_scheduler_handler_threads: Option, pub ip_echo_server_threads: NonZeroUsize, pub replay_forks_threads: NonZeroUsize, @@ -348,6 +349,7 @@ impl Default for ValidatorConfig { generator_config: None, use_snapshot_archives_at_startup: UseSnapshotArchivesAtStartup::default(), wen_restart_proto_path: None, + wen_restart_leader: None, unified_scheduler_handler_threads: None, ip_echo_server_threads: NonZeroUsize::new(1).expect("1 is non-zero"), replay_forks_threads: NonZeroUsize::new(1).expect("1 is non-zero"), @@ -1377,9 +1379,10 @@ impl Validator { .map_err(ValidatorError::Other)?; if in_wen_restart { - info!("Waiting for wen_restart phase one to finish"); + info!("Waiting for wen_restart to finish"); wait_for_wen_restart(WenRestartConfig { wen_restart_path: config.wen_restart_proto_path.clone().unwrap(), + wen_restart_leader: config.wen_restart_leader.unwrap(), last_vote, blockstore: blockstore.clone(), cluster_info: cluster_info.clone(), diff --git a/local-cluster/src/validator_configs.rs b/local-cluster/src/validator_configs.rs index a2366eb41489c8..cb33f44c274a58 100644 --- a/local-cluster/src/validator_configs.rs +++ b/local-cluster/src/validator_configs.rs @@ -68,6 +68,7 @@ pub fn safe_clone_config(config: &ValidatorConfig) -> ValidatorConfig { generator_config: config.generator_config.clone(), use_snapshot_archives_at_startup: config.use_snapshot_archives_at_startup, wen_restart_proto_path: config.wen_restart_proto_path.clone(), + wen_restart_leader: config.wen_restart_leader, unified_scheduler_handler_threads: config.unified_scheduler_handler_threads, ip_echo_server_threads: config.ip_echo_server_threads, replay_forks_threads: config.replay_forks_threads, diff --git a/multinode-demo/bootstrap-validator.sh b/multinode-demo/bootstrap-validator.sh index 471756254cb5db..3a0a6920a166bf 100755 --- a/multinode-demo/bootstrap-validator.sh +++ b/multinode-demo/bootstrap-validator.sh @@ -115,6 +115,9 @@ while [[ -n $1 ]]; do elif [[ $1 == --wen-restart ]]; then args+=("$1" "$2") shift 2 + elif [[ $1 == --wen-restart-leader ]]; then + args+=("$1" "$2") + shift 2 else echo "Unknown argument: $1" $program --help diff --git a/multinode-demo/validator.sh b/multinode-demo/validator.sh index d4e081c8893858..c1d445911a77b8 100755 --- a/multinode-demo/validator.sh +++ b/multinode-demo/validator.sh @@ -185,6 +185,9 @@ while [[ -n $1 ]]; do elif [[ $1 == --wen-restart ]]; then args+=("$1" "$2") shift 2 + elif [[ $1 == --wen-restart-leader ]]; then + args+=("$1" "$2") + shift 2 elif [[ $1 = -h ]]; then usage "$@" else diff --git a/net/net.sh b/net/net.sh index 94fa429ace5086..d9f1faea2a75e8 100755 --- a/net/net.sh +++ b/net/net.sh @@ -146,7 +146,7 @@ Operate a configured testnet -i [ip address] - IP Address of the node to start or stop startnode specific option: - --wen-restart [proto_file] - Use given proto file (create if non-exist) and apply wen_restat + --wen-restart [leader_pubkey] - Use given leader pubkey and apply wen_restat startclients-specific options: $CLIENT_OPTIONS diff --git a/net/remote/remote-node.sh b/net/remote/remote-node.sh index fe3f6a1d38dbca..9e3f57c5bc8f41 100755 --- a/net/remote/remote-node.sh +++ b/net/remote/remote-node.sh @@ -436,7 +436,8 @@ EOF fi if [[ -n "$maybeWenRestart" ]]; then - args+=(--wen-restart "$maybeWenRestart") + args+=(--wen-restart wen_restart.proto3) + args+=(--wen-restart-leader "$maybeWenRestart") fi cat >> ~/solana/on-reboot <(version: &'a str, default_args: &'a DefaultArgs) -> App<'a, 'a> { .takes_value(true) .required(false) .conflicts_with("wait_for_supermajority") + .requires("wen_restart_leader") .help( "Only used during coordinated cluster restarts.\ \n\n\ + Need to also specify the leader's pubkey in --wen-restart-leader.\ + \n\n\ When specified, the validator will enter Wen Restart mode which \ pauses normal activity. Validators in this mode will gossip their last \ vote to reach consensus on a safe restart slot and repair all blocks \ @@ -1604,6 +1607,19 @@ pub fn app<'a>(version: &'a str, default_args: &'a DefaultArgs) -> App<'a, 'a> { further debugging and watch the discord channel for instructions.", ), ) + .arg( + Arg::with_name("wen_restart_leader") + .long("wen-restart-leader") + .hidden(hidden_unless_forced()) + .value_name("PUBKEY") + .takes_value(true) + .required(false) + .requires("wen_restart") + .help( + "Specifies the pubkey of the leader used in wen restart. \ + May get stuck if the leader used is different from others.", + ), + ) .args(&thread_args(&default_args.thread_args)) .args(&get_deprecated_arguments()) .after_help("The default subcommand is run") diff --git a/validator/src/main.rs b/validator/src/main.rs index c3cedd49828d06..49b803d38529b2 100644 --- a/validator/src/main.rs +++ b/validator/src/main.rs @@ -1536,6 +1536,7 @@ pub fn main() { delay_leader_block_for_pending_fork: matches .is_present("delay_leader_block_for_pending_fork"), wen_restart_proto_path: value_t!(matches, "wen_restart", PathBuf).ok(), + wen_restart_leader: value_t!(matches, "wen_restart_leader", Pubkey).ok(), ..ValidatorConfig::default() }; diff --git a/wen-restart/proto/wen_restart.proto b/wen-restart/proto/wen_restart.proto index b32ca5f6537283..d95da208b4bbb9 100644 --- a/wen-restart/proto/wen_restart.proto +++ b/wen-restart/proto/wen_restart.proto @@ -39,17 +39,7 @@ message HeaviestForkRecord { uint64 total_active_stake = 3; uint32 shred_version = 4; uint64 wallclock = 5; -} - -message HeaviestForkAggregateFinal { - uint64 total_active_stake = 1; - uint64 total_active_stake_seen_supermajority = 2; - uint64 total_active_stake_agreed_with_me = 3; -} - -message HeaviestForkAggregateRecord { - map received = 1; - optional HeaviestForkAggregateFinal final_result = 2; + string from = 6; } message GenerateSnapshotRecord { @@ -69,7 +59,7 @@ message WenRestartProgress { optional LastVotedForkSlotsRecord my_last_voted_fork_slots = 2; optional LastVotedForkSlotsAggregateRecord last_voted_fork_slots_aggregate = 3; optional HeaviestForkRecord my_heaviest_fork = 4; - optional HeaviestForkAggregateRecord heaviest_fork_aggregate = 5; + optional HeaviestForkRecord leader_heaviest_fork = 5; optional GenerateSnapshotRecord my_snapshot = 6; map conflict_message = 7; } diff --git a/wen-restart/src/heaviest_fork_aggregate.rs b/wen-restart/src/heaviest_fork_aggregate.rs deleted file mode 100644 index 84a67426d89664..00000000000000 --- a/wen-restart/src/heaviest_fork_aggregate.rs +++ /dev/null @@ -1,665 +0,0 @@ -use { - crate::solana::wen_restart_proto::HeaviestForkRecord, - anyhow::Result, - log::*, - solana_gossip::restart_crds_values::RestartHeaviestFork, - solana_runtime::epoch_stakes::EpochStakes, - solana_sdk::{clock::Slot, hash::Hash, pubkey::Pubkey}, - std::{ - collections::{HashMap, HashSet}, - str::FromStr, - }, -}; - -pub(crate) struct HeaviestForkAggregate { - supermajority_threshold: f64, - my_shred_version: u16, - my_pubkey: Pubkey, - // We use the epoch_stakes of the Epoch our heaviest bank is in. Proceed and exit only if - // enough validator agree with me. - epoch_stakes: EpochStakes, - heaviest_forks: HashMap, - block_stake_map: HashMap<(Slot, Hash), u64>, - active_peers: HashSet, - active_peers_seen_supermajority: HashSet, -} - -#[derive(Clone, Debug, PartialEq)] -pub struct HeaviestForkFinalResult { - pub block_stake_map: HashMap<(Slot, Hash), u64>, - pub total_active_stake: u64, - pub total_active_stake_seen_supermajority: u64, -} - -#[derive(Debug, PartialEq)] -pub enum HeaviestForkAggregateResult { - AlreadyExists, - DifferentVersionExists(RestartHeaviestFork, RestartHeaviestFork), - Inserted(HeaviestForkRecord), - Malformed, - ZeroStakeIgnored, -} - -impl HeaviestForkAggregate { - pub(crate) fn new( - wait_for_supermajority_threshold_percent: u64, - my_shred_version: u16, - epoch_stakes: &EpochStakes, - my_heaviest_fork_slot: Slot, - my_heaviest_fork_hash: Hash, - my_pubkey: &Pubkey, - ) -> Self { - let mut active_peers = HashSet::new(); - active_peers.insert(*my_pubkey); - let mut block_stake_map = HashMap::new(); - block_stake_map.insert( - (my_heaviest_fork_slot, my_heaviest_fork_hash), - epoch_stakes.node_id_to_stake(my_pubkey).unwrap_or(0), - ); - Self { - supermajority_threshold: wait_for_supermajority_threshold_percent as f64 / 100.0, - my_shred_version, - my_pubkey: *my_pubkey, - epoch_stakes: epoch_stakes.clone(), - heaviest_forks: HashMap::new(), - block_stake_map, - active_peers, - active_peers_seen_supermajority: HashSet::new(), - } - } - - pub(crate) fn aggregate_from_record( - &mut self, - key_string: &str, - record: &HeaviestForkRecord, - ) -> Result { - let from = Pubkey::from_str(key_string)?; - let bankhash = Hash::from_str(&record.bankhash)?; - let restart_heaviest_fork = RestartHeaviestFork { - from, - wallclock: record.wallclock, - last_slot: record.slot, - last_slot_hash: bankhash, - observed_stake: record.total_active_stake, - shred_version: record.shred_version as u16, - }; - Ok(self.aggregate(restart_heaviest_fork)) - } - - fn is_valid_change( - current_heaviest_fork: &RestartHeaviestFork, - new_heaviest_fork: &RestartHeaviestFork, - ) -> HeaviestForkAggregateResult { - if current_heaviest_fork.last_slot != new_heaviest_fork.last_slot - || current_heaviest_fork.last_slot_hash != new_heaviest_fork.last_slot_hash - { - return HeaviestForkAggregateResult::DifferentVersionExists( - current_heaviest_fork.clone(), - new_heaviest_fork.clone(), - ); - } - if current_heaviest_fork == new_heaviest_fork - || current_heaviest_fork.wallclock > new_heaviest_fork.wallclock - || current_heaviest_fork.observed_stake == new_heaviest_fork.observed_stake - { - return HeaviestForkAggregateResult::AlreadyExists; - } - HeaviestForkAggregateResult::Inserted(HeaviestForkRecord { - slot: new_heaviest_fork.last_slot, - bankhash: new_heaviest_fork.last_slot_hash.to_string(), - total_active_stake: new_heaviest_fork.observed_stake, - shred_version: new_heaviest_fork.shred_version as u32, - wallclock: new_heaviest_fork.wallclock, - }) - } - - pub(crate) fn aggregate( - &mut self, - received_heaviest_fork: RestartHeaviestFork, - ) -> HeaviestForkAggregateResult { - let total_stake = self.epoch_stakes.total_stake(); - let from = &received_heaviest_fork.from; - let sender_stake = self.epoch_stakes.node_id_to_stake(from).unwrap_or(0); - if sender_stake == 0 { - warn!( - "Gossip should not accept zero-stake RestartLastVotedFork from {:?}", - from - ); - return HeaviestForkAggregateResult::ZeroStakeIgnored; - } - if from == &self.my_pubkey { - return HeaviestForkAggregateResult::AlreadyExists; - } - if received_heaviest_fork.shred_version != self.my_shred_version { - warn!( - "Gossip should not accept RestartLastVotedFork with different shred version {} from {:?}", - received_heaviest_fork.shred_version, from - ); - return HeaviestForkAggregateResult::Malformed; - } - let result = if let Some(old_heaviest_fork) = self.heaviest_forks.get(from) { - let result = Self::is_valid_change(old_heaviest_fork, &received_heaviest_fork); - if let HeaviestForkAggregateResult::Inserted(_) = result { - // continue following processing - } else { - return result; - } - result - } else { - let entry = self - .block_stake_map - .entry(( - received_heaviest_fork.last_slot, - received_heaviest_fork.last_slot_hash, - )) - .or_insert(0); - *entry = entry.saturating_add(sender_stake); - self.active_peers.insert(*from); - HeaviestForkAggregateResult::Inserted(HeaviestForkRecord { - slot: received_heaviest_fork.last_slot, - bankhash: received_heaviest_fork.last_slot_hash.to_string(), - total_active_stake: received_heaviest_fork.observed_stake, - shred_version: received_heaviest_fork.shred_version as u32, - wallclock: received_heaviest_fork.wallclock, - }) - }; - self.heaviest_forks - .insert(*from, received_heaviest_fork.clone()); - if received_heaviest_fork.observed_stake as f64 / total_stake as f64 - >= self.supermajority_threshold - { - self.active_peers_seen_supermajority.insert(*from); - } - if !self - .active_peers_seen_supermajority - .contains(&self.my_pubkey) - && self.total_active_stake() as f64 / total_stake as f64 >= self.supermajority_threshold - { - self.active_peers_seen_supermajority.insert(self.my_pubkey); - } - result - } - - pub(crate) fn total_active_stake(&self) -> u64 { - self.active_peers.iter().fold(0, |sum: u64, pubkey| { - sum.saturating_add(self.epoch_stakes.node_id_to_stake(pubkey).unwrap_or(0)) - }) - } - - pub(crate) fn total_active_stake_seen_supermajority(&self) -> u64 { - self.active_peers_seen_supermajority - .iter() - .fold(0, |sum: u64, pubkey| { - sum.saturating_add(self.epoch_stakes.node_id_to_stake(pubkey).unwrap_or(0)) - }) - } - - pub(crate) fn block_stake_map(self) -> HashMap<(Slot, Hash), u64> { - self.block_stake_map - } -} - -#[cfg(test)] -mod tests { - use { - crate::{ - heaviest_fork_aggregate::{HeaviestForkAggregate, HeaviestForkAggregateResult}, - solana::wen_restart_proto::HeaviestForkRecord, - }, - solana_gossip::restart_crds_values::RestartHeaviestFork, - solana_program::{clock::Slot, pubkey::Pubkey}, - solana_runtime::{ - bank::Bank, - genesis_utils::{ - create_genesis_config_with_vote_accounts, GenesisConfigInfo, ValidatorVoteKeypairs, - }, - }, - solana_sdk::{hash::Hash, signature::Signer, timing::timestamp}, - }; - - const TOTAL_VALIDATOR_COUNT: u16 = 20; - const MY_INDEX: usize = 19; - const SHRED_VERSION: u16 = 52; - - struct TestAggregateInitResult { - pub heaviest_fork_aggregate: HeaviestForkAggregate, - pub validator_voting_keypairs: Vec, - pub heaviest_slot: Slot, - pub heaviest_hash: Hash, - } - - fn test_aggregate_init() -> TestAggregateInitResult { - solana_logger::setup(); - let validator_voting_keypairs: Vec<_> = (0..TOTAL_VALIDATOR_COUNT) - .map(|_| ValidatorVoteKeypairs::new_rand()) - .collect(); - let GenesisConfigInfo { genesis_config, .. } = create_genesis_config_with_vote_accounts( - 10_000, - &validator_voting_keypairs, - vec![100; validator_voting_keypairs.len()], - ); - let (_, bank_forks) = Bank::new_with_bank_forks_for_tests(&genesis_config); - let root_bank = bank_forks.read().unwrap().root_bank(); - let heaviest_slot = root_bank.slot().saturating_add(3); - let heaviest_hash = Hash::new_unique(); - TestAggregateInitResult { - heaviest_fork_aggregate: HeaviestForkAggregate::new( - 75, - SHRED_VERSION, - root_bank.epoch_stakes(root_bank.epoch()).unwrap(), - heaviest_slot, - heaviest_hash, - &validator_voting_keypairs[MY_INDEX].node_keypair.pubkey(), - ), - validator_voting_keypairs, - heaviest_slot, - heaviest_hash, - } - } - - #[test] - fn test_aggregate_from_gossip() { - let mut test_state = test_aggregate_init(); - let initial_num_active_validators = 3; - let timestamp1 = timestamp(); - for validator_voting_keypair in test_state - .validator_voting_keypairs - .iter() - .take(initial_num_active_validators) - { - let pubkey = validator_voting_keypair.node_keypair.pubkey(); - assert_eq!( - test_state - .heaviest_fork_aggregate - .aggregate(RestartHeaviestFork { - from: pubkey, - wallclock: timestamp1, - last_slot: test_state.heaviest_slot, - last_slot_hash: test_state.heaviest_hash, - observed_stake: 100, - shred_version: SHRED_VERSION, - },), - HeaviestForkAggregateResult::Inserted(HeaviestForkRecord { - slot: test_state.heaviest_slot, - bankhash: test_state.heaviest_hash.to_string(), - total_active_stake: 100, - shred_version: SHRED_VERSION as u32, - wallclock: timestamp1, - }), - ); - } - assert_eq!( - test_state.heaviest_fork_aggregate.total_active_stake(), - (initial_num_active_validators + 1) as u64 * 100 - ); - - let new_active_validator = test_state.validator_voting_keypairs - [initial_num_active_validators + 1] - .node_keypair - .pubkey(); - let now = timestamp(); - let new_active_validator_last_voted_slots = RestartHeaviestFork { - from: new_active_validator, - wallclock: now, - last_slot: test_state.heaviest_slot, - last_slot_hash: test_state.heaviest_hash, - observed_stake: 100, - shred_version: SHRED_VERSION, - }; - assert_eq!( - test_state - .heaviest_fork_aggregate - .aggregate(new_active_validator_last_voted_slots), - HeaviestForkAggregateResult::Inserted(HeaviestForkRecord { - slot: test_state.heaviest_slot, - bankhash: test_state.heaviest_hash.to_string(), - total_active_stake: 100, - shred_version: SHRED_VERSION as u32, - wallclock: now, - }), - ); - let expected_total_active_stake = (initial_num_active_validators + 2) as u64 * 100; - assert_eq!( - test_state.heaviest_fork_aggregate.total_active_stake(), - expected_total_active_stake - ); - let replace_message_validator = test_state.validator_voting_keypairs[2] - .node_keypair - .pubkey(); - // If hash changes, it will be ignored. - let now = timestamp(); - let new_hash = Hash::new_unique(); - let replace_message_validator_last_fork = RestartHeaviestFork { - from: replace_message_validator, - wallclock: now, - last_slot: test_state.heaviest_slot + 1, - last_slot_hash: new_hash, - observed_stake: 100, - shred_version: SHRED_VERSION, - }; - assert_eq!( - test_state - .heaviest_fork_aggregate - .aggregate(replace_message_validator_last_fork.clone()), - HeaviestForkAggregateResult::DifferentVersionExists( - RestartHeaviestFork { - from: replace_message_validator, - wallclock: timestamp1, - last_slot: test_state.heaviest_slot, - last_slot_hash: test_state.heaviest_hash, - observed_stake: 100, - shred_version: SHRED_VERSION, - }, - replace_message_validator_last_fork, - ), - ); - assert_eq!( - test_state.heaviest_fork_aggregate.total_active_stake(), - expected_total_active_stake - ); - - // test that zero stake validator is ignored. - let random_pubkey = Pubkey::new_unique(); - assert_eq!( - test_state - .heaviest_fork_aggregate - .aggregate(RestartHeaviestFork { - from: random_pubkey, - wallclock: timestamp(), - last_slot: test_state.heaviest_slot, - last_slot_hash: test_state.heaviest_hash, - observed_stake: 100, - shred_version: SHRED_VERSION, - },), - HeaviestForkAggregateResult::ZeroStakeIgnored, - ); - assert_eq!( - test_state.heaviest_fork_aggregate.total_active_stake(), - expected_total_active_stake - ); - - // If everyone is seeing only 70%, the total active stake seeing supermajority is 0. - for validator_voting_keypair in test_state.validator_voting_keypairs.iter().take(13) { - let pubkey = validator_voting_keypair.node_keypair.pubkey(); - let now = timestamp(); - assert_eq!( - test_state - .heaviest_fork_aggregate - .aggregate(RestartHeaviestFork { - from: pubkey, - wallclock: now, - last_slot: test_state.heaviest_slot, - last_slot_hash: test_state.heaviest_hash, - observed_stake: 1400, - shred_version: SHRED_VERSION, - },), - HeaviestForkAggregateResult::Inserted(HeaviestForkRecord { - slot: test_state.heaviest_slot, - bankhash: test_state.heaviest_hash.to_string(), - total_active_stake: 1400, - shred_version: SHRED_VERSION as u32, - wallclock: now, - }), - ); - } - assert_eq!( - test_state.heaviest_fork_aggregate.total_active_stake(), - 1400 - ); - assert_eq!( - test_state - .heaviest_fork_aggregate - .total_active_stake_seen_supermajority(), - 0 - ); - - // test that when 75% of the stake is seeing supermajority, - // the active percent seeing supermajority is 75%. - for validator_voting_keypair in test_state.validator_voting_keypairs.iter().take(14) { - let pubkey = validator_voting_keypair.node_keypair.pubkey(); - let now = timestamp(); - assert_eq!( - test_state - .heaviest_fork_aggregate - .aggregate(RestartHeaviestFork { - from: pubkey, - wallclock: now, - last_slot: test_state.heaviest_slot, - last_slot_hash: test_state.heaviest_hash, - observed_stake: 1500, - shred_version: SHRED_VERSION, - },), - HeaviestForkAggregateResult::Inserted(HeaviestForkRecord { - slot: test_state.heaviest_slot, - bankhash: test_state.heaviest_hash.to_string(), - total_active_stake: 1500, - shred_version: SHRED_VERSION as u32, - wallclock: now, - }), - ); - } - - assert_eq!( - test_state.heaviest_fork_aggregate.total_active_stake(), - 1500 - ); - // I myself is seeing supermajority as well, with the 14 validators - // reporting 70%, the total active stake seeing supermajority is 1500 (75%). - assert_eq!( - test_state - .heaviest_fork_aggregate - .total_active_stake_seen_supermajority(), - 1500 - ); - - // test that message from my pubkey is ignored. - assert_eq!( - test_state - .heaviest_fork_aggregate - .aggregate(RestartHeaviestFork { - from: test_state.validator_voting_keypairs[MY_INDEX] - .node_keypair - .pubkey(), - wallclock: timestamp(), - last_slot: test_state.heaviest_slot, - last_slot_hash: test_state.heaviest_hash, - observed_stake: 100, - shred_version: SHRED_VERSION, - },), - HeaviestForkAggregateResult::AlreadyExists, - ); - } - - #[test] - fn test_aggregate_from_record() { - let mut test_state = test_aggregate_init(); - let time1 = timestamp(); - let from = test_state.validator_voting_keypairs[0] - .node_keypair - .pubkey(); - let record = HeaviestForkRecord { - wallclock: time1, - slot: test_state.heaviest_slot, - bankhash: test_state.heaviest_hash.to_string(), - shred_version: SHRED_VERSION as u32, - total_active_stake: 100, - }; - assert_eq!(test_state.heaviest_fork_aggregate.total_active_stake(), 100); - assert_eq!( - test_state - .heaviest_fork_aggregate - .aggregate_from_record(&from.to_string(), &record,) - .unwrap(), - HeaviestForkAggregateResult::Inserted(record.clone()), - ); - assert_eq!(test_state.heaviest_fork_aggregate.total_active_stake(), 200); - // Now if you get the same result from Gossip again, it should be ignored. - assert_eq!( - test_state - .heaviest_fork_aggregate - .aggregate(RestartHeaviestFork { - from, - wallclock: time1, - last_slot: test_state.heaviest_slot, - last_slot_hash: test_state.heaviest_hash, - observed_stake: 100, - shred_version: SHRED_VERSION, - },), - HeaviestForkAggregateResult::AlreadyExists, - ); - - // If only observed_stake changes, it will be replaced. - let time2 = timestamp(); - let old_heaviest_fork = RestartHeaviestFork { - from, - wallclock: time2, - last_slot: test_state.heaviest_slot, - last_slot_hash: test_state.heaviest_hash, - observed_stake: 200, - shred_version: SHRED_VERSION, - }; - assert_eq!( - test_state - .heaviest_fork_aggregate - .aggregate(old_heaviest_fork.clone()), - HeaviestForkAggregateResult::Inserted(HeaviestForkRecord { - wallclock: time2, - slot: test_state.heaviest_slot, - bankhash: test_state.heaviest_hash.to_string(), - shred_version: SHRED_VERSION as u32, - total_active_stake: 200, - }), - ); - - // If slot changes, it will be ignored. - let new_heaviest_fork = RestartHeaviestFork { - from: test_state.validator_voting_keypairs[0] - .node_keypair - .pubkey(), - wallclock: timestamp(), - last_slot: test_state.heaviest_slot + 1, - last_slot_hash: test_state.heaviest_hash, - observed_stake: 100, - shred_version: SHRED_VERSION, - }; - assert_eq!( - test_state - .heaviest_fork_aggregate - .aggregate(new_heaviest_fork.clone()), - HeaviestForkAggregateResult::DifferentVersionExists( - old_heaviest_fork.clone(), - new_heaviest_fork - ) - ); - // If hash changes, it will also be ignored. - let new_heaviest_fork = RestartHeaviestFork { - from: test_state.validator_voting_keypairs[0] - .node_keypair - .pubkey(), - wallclock: timestamp(), - last_slot: test_state.heaviest_slot, - last_slot_hash: Hash::new_unique(), - observed_stake: 100, - shred_version: SHRED_VERSION, - }; - assert_eq!( - test_state - .heaviest_fork_aggregate - .aggregate(new_heaviest_fork.clone()), - HeaviestForkAggregateResult::DifferentVersionExists( - old_heaviest_fork, - new_heaviest_fork - ) - ); - - // percentage doesn't change since it's a replace. - assert_eq!(test_state.heaviest_fork_aggregate.total_active_stake(), 200); - - // Record from validator with zero stake should be ignored. - assert_eq!( - test_state - .heaviest_fork_aggregate - .aggregate_from_record( - &Pubkey::new_unique().to_string(), - &HeaviestForkRecord { - wallclock: timestamp(), - slot: test_state.heaviest_slot, - bankhash: test_state.heaviest_hash.to_string(), - shred_version: SHRED_VERSION as u32, - total_active_stake: 100, - } - ) - .unwrap(), - HeaviestForkAggregateResult::ZeroStakeIgnored, - ); - // percentage doesn't change since the previous aggregate is ignored. - assert_eq!(test_state.heaviest_fork_aggregate.total_active_stake(), 200); - - // Record from my pubkey should be ignored. - assert_eq!( - test_state - .heaviest_fork_aggregate - .aggregate_from_record( - &test_state.validator_voting_keypairs[MY_INDEX] - .node_keypair - .pubkey() - .to_string(), - &HeaviestForkRecord { - wallclock: timestamp(), - slot: test_state.heaviest_slot, - bankhash: test_state.heaviest_hash.to_string(), - shred_version: SHRED_VERSION as u32, - total_active_stake: 100, - } - ) - .unwrap(), - HeaviestForkAggregateResult::AlreadyExists, - ); - } - - #[test] - fn test_aggregate_from_record_failures() { - let mut test_state = test_aggregate_init(); - let mut heaviest_fork_record = HeaviestForkRecord { - wallclock: timestamp(), - slot: test_state.heaviest_slot, - bankhash: test_state.heaviest_hash.to_string(), - shred_version: SHRED_VERSION as u32, - total_active_stake: 100, - }; - // First test that this is a valid record. - assert_eq!( - test_state - .heaviest_fork_aggregate - .aggregate_from_record( - &test_state.validator_voting_keypairs[0] - .node_keypair - .pubkey() - .to_string(), - &heaviest_fork_record, - ) - .unwrap(), - HeaviestForkAggregateResult::Inserted(heaviest_fork_record.clone()), - ); - // Then test that it fails if the record is invalid. - - // Invalid pubkey. - assert!(test_state - .heaviest_fork_aggregate - .aggregate_from_record("invalid_pubkey", &heaviest_fork_record,) - .is_err()); - - // Invalid hash. - heaviest_fork_record.bankhash.clear(); - assert!(test_state - .heaviest_fork_aggregate - .aggregate_from_record( - &test_state.validator_voting_keypairs[0] - .node_keypair - .pubkey() - .to_string(), - &heaviest_fork_record, - ) - .is_err()); - } -} diff --git a/wen-restart/src/lib.rs b/wen-restart/src/lib.rs index d798cae1313ef4..d389136bb13bcd 100644 --- a/wen-restart/src/lib.rs +++ b/wen-restart/src/lib.rs @@ -4,6 +4,5 @@ pub(crate) mod solana { } } -pub(crate) mod heaviest_fork_aggregate; pub(crate) mod last_voted_fork_slots_aggregate; pub mod wen_restart; diff --git a/wen-restart/src/wen_restart.rs b/wen-restart/src/wen_restart.rs index 924debb2adf226..67f3ed51e6b382 100644 --- a/wen-restart/src/wen_restart.rs +++ b/wen-restart/src/wen_restart.rs @@ -2,16 +2,15 @@ use { crate::{ - heaviest_fork_aggregate::{HeaviestForkAggregate, HeaviestForkAggregateResult}, last_voted_fork_slots_aggregate::{ LastVotedForkSlotsAggregate, LastVotedForkSlotsAggregateResult, LastVotedForkSlotsEpochInfo, LastVotedForkSlotsFinalResult, }, solana::wen_restart_proto::{ - self, ConflictMessage, GenerateSnapshotRecord, HeaviestForkAggregateFinal, - HeaviestForkAggregateRecord, HeaviestForkRecord, LastVotedForkSlotsAggregateFinal, - LastVotedForkSlotsAggregateRecord, LastVotedForkSlotsEpochInfoRecord, - LastVotedForkSlotsRecord, State as RestartState, WenRestartProgress, + self, ConflictMessage, GenerateSnapshotRecord, HeaviestForkRecord, + LastVotedForkSlotsAggregateFinal, LastVotedForkSlotsAggregateRecord, + LastVotedForkSlotsEpochInfoRecord, LastVotedForkSlotsRecord, State as RestartState, + WenRestartProgress, }, }, anyhow::Result, @@ -44,7 +43,7 @@ use { purge_all_bank_snapshots, }, }, - solana_sdk::{shred_version::compute_shred_version, timing::timestamp}, + solana_sdk::{pubkey::Pubkey, shred_version::compute_shred_version, timing::timestamp}, solana_timings::ExecuteTimings, solana_vote_program::vote_state::VoteTransaction, std::{ @@ -58,7 +57,7 @@ use { Arc, RwLock, }, thread::sleep, - time::{Duration, Instant}, + time::Duration, }, }; @@ -70,13 +69,12 @@ const REPAIR_THRESHOLD: f64 = 0.42; // made regarding how much non-conforming/offline validators the // algorithm can tolerate. const HEAVIEST_FORK_THRESHOLD_DELTA: f64 = 0.38; -// We allow at most 5% of the stake to disagree with us. -const HEAVIEST_FORK_DISAGREE_THRESHOLD_PERCENT: f64 = 5.0; // We update HeaviestFork every 5 minutes at least. const HEAVIEST_REFRESH_INTERVAL_IN_SECONDS: u64 = 300; #[derive(Debug, PartialEq)] pub enum WenRestartError { + BankHashMismatch(Slot, Hash, Hash), BlockNotFound(Slot), BlockNotFull(Slot), BlockNotFrozenAfterReplay(Slot, Option), @@ -85,6 +83,7 @@ pub enum WenRestartError { Exiting, FutureSnapshotExists(Slot, Slot, String), GenerateSnapshotWhenOneExists(Slot, String), + HeaviestForkOnLeaderOnDifferentFork(Slot, Slot), MalformedLastVotedForkSlotsProtobuf(Option), MalformedProgress(RestartState, String), MissingLastVotedForkSlots, @@ -97,6 +96,13 @@ pub enum WenRestartError { impl std::fmt::Display for WenRestartError { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { + WenRestartError::BankHashMismatch(slot, expected, actual) => { + write!( + f, + "Bank hash mismatch for slot: {} expected: {} actual: {}", + slot, expected, actual + ) + } WenRestartError::BlockNotFound(slot) => { write!(f, "Block not found: {}", slot) } @@ -138,6 +144,15 @@ impl std::fmt::Display for WenRestartError { "Generate snapshot when one exists for slot: {slot} in directory: {directory}", ) } + WenRestartError::HeaviestForkOnLeaderOnDifferentFork( + leader_heaviest_slot, + should_include_slot, + ) => { + write!( + f, + "Heaviest fork on leader on different fork: heaviest: {leader_heaviest_slot} does not include: {should_include_slot}", + ) + } WenRestartError::MalformedLastVotedForkSlotsProtobuf(record) => { write!(f, "Malformed last voted fork slots protobuf: {:?}", record) } @@ -637,16 +652,11 @@ pub(crate) fn find_bankhash_of_heaviest_fork( Ok(parent_bank.hash()) } -// Aggregate the heaviest fork and send updates to the cluster. -pub(crate) fn aggregate_restart_heaviest_fork( - wen_restart_path: &PathBuf, - wait_for_supermajority_threshold_percent: u64, +pub(crate) fn send_restart_heaviest_fork( cluster_info: Arc, - bank_forks: Arc>, exit: Arc, progress: &mut WenRestartProgress, ) -> Result<()> { - let root_bank = bank_forks.read().unwrap().root_bank(); if progress.my_heaviest_fork.is_none() { return Err(WenRestartError::MalformedProgress( RestartState::HeaviestFork, @@ -657,223 +667,150 @@ pub(crate) fn aggregate_restart_heaviest_fork( let my_heaviest_fork = progress.my_heaviest_fork.clone().unwrap(); let heaviest_fork_slot = my_heaviest_fork.slot; let heaviest_fork_hash = Hash::from_str(&my_heaviest_fork.bankhash)?; - // When checking whether to exit aggregate_restart_heaviest_fork, use the epoch_stakes - // associated with the heaviest fork slot we picked. This ensures that everyone agreeing - // with me use the same EpochStakes to calculate the supermajority threshold. - let epoch_stakes = root_bank - .epoch_stakes(root_bank.epoch_schedule().get_epoch(heaviest_fork_slot)) - .unwrap(); - let total_stake = epoch_stakes.total_stake(); - let adjusted_threshold_percent = wait_for_supermajority_threshold_percent - .saturating_sub(HEAVIEST_FORK_DISAGREE_THRESHOLD_PERCENT.round() as u64); - // The threshold for supermajority should definitely be higher than 67%. - assert!( - adjusted_threshold_percent > 67, - "Majority threshold too low" - ); - let mut heaviest_fork_aggregate = HeaviestForkAggregate::new( - adjusted_threshold_percent, - cluster_info.my_shred_version(), - epoch_stakes, - heaviest_fork_slot, - heaviest_fork_hash, - &cluster_info.id(), - ); - if let Some(aggregate_record) = &progress.heaviest_fork_aggregate { - for (key_string, message) in &aggregate_record.received { - if let Err(e) = heaviest_fork_aggregate.aggregate_from_record(key_string, message) { - // Do not abort wen_restart if we got one malformed message. - error!("Failed to aggregate from record: {:?}", e); - } + loop { + if exit.load(Ordering::Relaxed) { + break; } - } else { - progress.heaviest_fork_aggregate = Some(HeaviestForkAggregateRecord { - received: HashMap::new(), - final_result: None, - }); + cluster_info.push_restart_heaviest_fork(heaviest_fork_slot, heaviest_fork_hash, 0); + sleep(Duration::from_secs(HEAVIEST_REFRESH_INTERVAL_IN_SECONDS)); } + Ok(()) +} - let mut total_active_stake = heaviest_fork_aggregate.total_active_stake(); - progress - .my_heaviest_fork - .as_mut() - .unwrap() - .total_active_stake = total_active_stake; - +pub(crate) fn receive_restart_heaviest_fork( + wen_restart_leader: Pubkey, + cluster_info: Arc, + exit: Arc, + progress: &mut WenRestartProgress, +) -> Result<(Slot, Hash)> { let mut cursor = solana_gossip::crds::Cursor::default(); - // Init progress_changed to true and progress_last_sent to old time so we can send out the first Gossip message. - let mut progress_changed = true; - let mut progress_last_sent = Instant::now() - .checked_sub(Duration::from_secs(HEAVIEST_REFRESH_INTERVAL_IN_SECONDS)) - .unwrap(); - let majority_stake_required = - (total_stake as f64 / 100.0 * adjusted_threshold_percent as f64).round() as u64; - let mut total_active_stake_higher_than_supermajority = false; loop { if exit.load(Ordering::Relaxed) { return Err(WenRestartError::Exiting.into()); } - let start = timestamp(); for new_heaviest_fork in cluster_info.get_restart_heaviest_fork(&mut cursor) { - info!("Received new heaviest fork: {:?}", new_heaviest_fork); - let from = new_heaviest_fork.from.to_string(); - match heaviest_fork_aggregate.aggregate(new_heaviest_fork) { - HeaviestForkAggregateResult::Inserted(record) => { - info!("Successfully aggregated new heaviest fork: {:?}", record); - progress - .heaviest_fork_aggregate - .as_mut() - .unwrap() - .received - .insert(from, record); - progress_changed = true; - } - HeaviestForkAggregateResult::DifferentVersionExists(old_record, new_record) => { - warn!("Different version from {from} exists old {old_record:#?} vs new {new_record:#?}"); - progress.conflict_message.insert( - from, - ConflictMessage { - old_message: format!("{:?}", old_record), - new_message: format!("{:?}", new_record), - }, - ); - } - HeaviestForkAggregateResult::ZeroStakeIgnored => (), - HeaviestForkAggregateResult::AlreadyExists => (), - HeaviestForkAggregateResult::Malformed => (), + if new_heaviest_fork.from != wen_restart_leader { + warn!( + "Received heaviest fork from non leader: {}", + new_heaviest_fork.from + ); + continue; } - } - let current_total_active_stake = heaviest_fork_aggregate.total_active_stake(); - if current_total_active_stake > total_active_stake { - total_active_stake = current_total_active_stake; - progress - .my_heaviest_fork - .as_mut() - .unwrap() - .total_active_stake = current_total_active_stake; - progress_changed = true; - } - if progress_changed { - progress_changed = false; - let total_active_stake_seen_supermajority = - heaviest_fork_aggregate.total_active_stake_seen_supermajority(); info!( - "Total active stake seeing supermajority: {} Total active stake: {} Required to exit {} Total stake {}", - total_active_stake_seen_supermajority, - heaviest_fork_aggregate.total_active_stake(), - majority_stake_required, - total_stake + "Received new heaviest fork from leader: {} {:?}", + wen_restart_leader, new_heaviest_fork ); - let can_exit = total_active_stake_seen_supermajority >= majority_stake_required; - let saw_supermajority_first_time = current_total_active_stake - >= majority_stake_required - && !total_active_stake_higher_than_supermajority - && { - total_active_stake_higher_than_supermajority = true; - true - }; - // Only send out updates every 5 minutes or when we can exit or active stake passes supermajority - // the first time. - if progress_last_sent.elapsed().as_secs() >= HEAVIEST_REFRESH_INTERVAL_IN_SECONDS - || can_exit - || saw_supermajority_first_time - { - cluster_info.push_restart_heaviest_fork( - heaviest_fork_slot, - heaviest_fork_hash, - current_total_active_stake, - ); - write_wen_restart_records(wen_restart_path, progress)?; - progress_last_sent = Instant::now(); - } - if can_exit { - break; - } - } - let elapsed = timestamp().saturating_sub(start); - let time_left = GOSSIP_SLEEP_MILLIS.saturating_sub(elapsed); - if time_left > 0 { - sleep(Duration::from_millis(time_left)); + let heaviest_slot = new_heaviest_fork.last_slot; + let heaviest_hash = new_heaviest_fork.last_slot_hash; + progress.leader_heaviest_fork = Some(HeaviestForkRecord { + slot: heaviest_slot, + bankhash: heaviest_hash.to_string(), + total_active_stake: 0, + wallclock: new_heaviest_fork.wallclock, + shred_version: new_heaviest_fork.shred_version as u32, + from: new_heaviest_fork.from.to_string(), + }); + return Ok((heaviest_slot, heaviest_hash)); } + sleep(Duration::from_millis(GOSSIP_SLEEP_MILLIS)); } +} - // Final check to see if supermajority agrees with us. - let total_active_stake = heaviest_fork_aggregate.total_active_stake(); - let total_active_stake_seen_supermajority = - heaviest_fork_aggregate.total_active_stake_seen_supermajority(); - let block_stake_map = heaviest_fork_aggregate.block_stake_map(); - let total_active_stake_agreed_with_me = *block_stake_map - .get(&(heaviest_fork_slot, heaviest_fork_hash)) - .unwrap_or(&0); - // It doesn't matter if 5% disagrees with us. - let success_threshold = - wait_for_supermajority_threshold_percent as f64 - HEAVIEST_FORK_DISAGREE_THRESHOLD_PERCENT; - if total_active_stake_agreed_with_me as f64 * 100.0 / total_stake as f64 >= success_threshold { - info!( - "Heaviest fork agreed upon by supermajority: slot: {}, bankhash: {}", - heaviest_fork_slot, heaviest_fork_hash - ); - progress - .heaviest_fork_aggregate - .as_mut() - .unwrap() - .final_result = Some(HeaviestForkAggregateFinal { - total_active_stake, - total_active_stake_seen_supermajority, - total_active_stake_agreed_with_me, - }); - Ok(()) - } else { - info!( - "Not enough stake agreeing with our heaviest fork: slot: {}, - bankhash: {}, stake aggreeing with us {} out of {}", - heaviest_fork_slot, - heaviest_fork_hash, - total_active_stake_agreed_with_me, - total_active_stake - ); - let mut max_slot_hash = (0, Hash::default()); - let mut max_stake = 0; - for (slot, hash) in block_stake_map.keys() { - let stake = block_stake_map[&(*slot, *hash)]; - if stake > max_stake { - max_stake = stake; - max_slot_hash = (*slot, *hash); - } - info!( - "Slot: {}, Hash: {}, Stake: {}", - slot, - hash, - block_stake_map[&(*slot, *hash)] - ); +fn repair_heaviest_fork( + my_heaviest_fork_slot: Slot, + heaviest_slot: Slot, + exit: Arc, + blockstore: Arc, + wen_restart_repair_slots: Arc>>, +) -> Result<()> { + loop { + if exit.load(Ordering::Relaxed) { + return Err(WenRestartError::Exiting.into()); } - let max_stake_percent = max_stake as f64 * 100.0 / total_stake as f64; - if max_stake_percent >= success_threshold { - warn!( - "Max stake slot: {}, hash: {}, stake: {:.2}% does not agree with my - choice, please go to discord to download the snapshot and restart - the validator with --wait-for-supermajority.", - max_slot_hash.0, max_slot_hash.1, max_stake_percent - ); + let to_repair = if blockstore.meta(heaviest_slot).is_ok_and(|x| x.is_some()) { + AncestorIterator::new_inclusive(heaviest_slot, &blockstore) + .take_while(|slot| *slot > my_heaviest_fork_slot && !blockstore.is_full(*slot)) + .collect() } else { - warn!( - "Cluster consensus slot: {}, hash: {}, stake: {:.2}% does not agree, - please go to discord for next steps.", - max_slot_hash.0, max_slot_hash.1, max_stake_percent - ); + vec![heaviest_slot] + }; + if to_repair.is_empty() { + return Ok(()); // All blocks are full } - Err(WenRestartError::NotEnoughStakeAgreeingWithUs( - heaviest_fork_slot, - heaviest_fork_hash, - block_stake_map, + *wen_restart_repair_slots.write().unwrap() = to_repair; + sleep(Duration::from_millis(GOSSIP_SLEEP_MILLIS)); + } +} + +pub(crate) fn verify_restart_heaviest_fork( + my_heaviest_fork_slot: Slot, + heaviest_slot: Slot, + heaviest_hash: &Hash, + bank_forks: Arc>, + blockstore: Arc, + exit: Arc, + wen_restart_repair_slots: Arc>>, +) -> Result<()> { + repair_heaviest_fork( + my_heaviest_fork_slot, + heaviest_slot, + exit.clone(), + blockstore.clone(), + wen_restart_repair_slots.clone(), + )?; + let root_bank = bank_forks.read().unwrap().root_bank(); + let root_slot = root_bank.slot(); + let mut slots: Vec = + AncestorIterator::new_inclusive(heaviest_slot, &blockstore).collect(); + slots.sort(); + if !slots.contains(&root_slot) { + return Err( + WenRestartError::HeaviestForkOnLeaderOnDifferentFork(heaviest_slot, root_slot).into(), + ); + } + if heaviest_slot > my_heaviest_fork_slot && !slots.contains(&my_heaviest_fork_slot) { + return Err(WenRestartError::HeaviestForkOnLeaderOnDifferentFork( + heaviest_slot, + my_heaviest_fork_slot, ) - .into()) + .into()); } + if heaviest_slot < my_heaviest_fork_slot + && !AncestorIterator::new(my_heaviest_fork_slot, &blockstore) + .any(|slot| slot == heaviest_slot) + { + return Err(WenRestartError::HeaviestForkOnLeaderOnDifferentFork( + heaviest_slot, + my_heaviest_fork_slot, + ) + .into()); + } + let my_bankhash = if !slots.is_empty() { + find_bankhash_of_heaviest_fork( + heaviest_slot, + slots, + blockstore.clone(), + bank_forks.clone(), + bank_forks.read().unwrap().root_bank(), + &exit, + )? + } else if let Some(bank) = bank_forks.read().unwrap().get(heaviest_slot) { + bank.hash() + } else { + return Err(WenRestartError::BlockNotFound(heaviest_slot).into()); + }; + if my_bankhash != *heaviest_hash { + return Err( + WenRestartError::BankHashMismatch(heaviest_slot, my_bankhash, *heaviest_hash).into(), + ); + } + Ok(()) } #[derive(Clone)] pub struct WenRestartConfig { pub wen_restart_path: PathBuf, + pub wen_restart_leader: Pubkey, pub last_vote: VoteTransaction, pub blockstore: Arc, pub cluster_info: Arc, @@ -954,6 +891,7 @@ pub fn wait_for_wen_restart(config: WenRestartConfig) -> Result<()> { total_active_stake: 0, wallclock: 0, shred_version: config.cluster_info.my_shred_version() as u32, + from: config.cluster_info.id().to_string(), } } }; @@ -963,14 +901,29 @@ pub fn wait_for_wen_restart(config: WenRestartConfig) -> Result<()> { } } WenRestartProgressInternalState::HeaviestFork { new_root_slot } => { - aggregate_restart_heaviest_fork( - &config.wen_restart_path, - config.wait_for_supermajority_threshold_percent, - config.cluster_info.clone(), - config.bank_forks.clone(), - config.exit.clone(), - &mut progress, - )?; + if config.cluster_info.id() == config.wen_restart_leader { + send_restart_heaviest_fork( + config.cluster_info.clone(), + config.exit.clone(), + &mut progress, + )?; + } else { + let (leader_slot, leader_hash) = receive_restart_heaviest_fork( + config.wen_restart_leader, + config.cluster_info.clone(), + config.exit.clone(), + &mut progress, + )?; + verify_restart_heaviest_fork( + new_root_slot, + leader_slot, + &leader_hash, + config.bank_forks.clone(), + config.blockstore.clone(), + config.exit.clone(), + config.wen_restart_repair_slots.clone().unwrap(), + )?; + } WenRestartProgressInternalState::HeaviestFork { new_root_slot } } WenRestartProgressInternalState::GenerateSnapshot { @@ -1333,6 +1286,7 @@ mod tests { const TICKS_PER_SLOT: u64 = 2; const TOTAL_VALIDATOR_COUNT: u16 = 20; const MY_INDEX: usize = TOTAL_VALIDATOR_COUNT as usize - 1; + const LEADER_INDEX: usize = 0; const WAIT_FOR_THREAD_TIMEOUT: u64 = 10_000; const WAIT_FOR_SUPERMAJORITY_THRESHOLD_PERCENT: u64 = 80; const NON_CONFORMING_VALIDATOR_PERCENT: u64 = 5; @@ -1404,6 +1358,7 @@ mod tests { pub bank_forks: Arc>, pub last_voted_fork_slots: Vec, pub wen_restart_proto_path: PathBuf, + pub wen_restart_leader: Pubkey, pub last_blockhash: Hash, pub genesis_config_hash: Hash, } @@ -1493,6 +1448,9 @@ mod tests { let mut wen_restart_proto_path = ledger_path.path().to_path_buf(); wen_restart_proto_path.push("wen_restart_status.proto"); let _ = remove_file(&wen_restart_proto_path); + let wen_restart_leader = validator_voting_keypairs[LEADER_INDEX] + .node_keypair + .pubkey(); WenRestartTestInitResult { validator_voting_keypairs, blockstore, @@ -1500,6 +1458,7 @@ mod tests { bank_forks, last_voted_fork_slots, wen_restart_proto_path, + wen_restart_leader, last_blockhash, genesis_config_hash: genesis_config.hash(), } @@ -1556,6 +1515,7 @@ mod tests { let last_vote_slot: Slot = test_state.last_voted_fork_slots[0]; let wen_restart_config = WenRestartConfig { wen_restart_path: test_state.wen_restart_proto_path.clone(), + wen_restart_leader: test_state.wen_restart_leader, last_vote: VoteTransaction::from(Vote::new(vec![last_vote_slot], last_vote_bankhash)), blockstore: test_state.blockstore.clone(), cluster_info: test_state.cluster_info.clone(), @@ -1623,6 +1583,7 @@ mod tests { let exit = Arc::new(AtomicBool::new(false)); let wen_restart_config = WenRestartConfig { wen_restart_path: test_state.wen_restart_proto_path.clone(), + wen_restart_leader: test_state.wen_restart_leader, last_vote: VoteTransaction::from(Vote::new(vec![last_vote_slot], last_vote_bankhash)), blockstore: test_state.blockstore.clone(), cluster_info: test_state.cluster_info.clone(), @@ -1701,7 +1662,6 @@ mod tests { sleep(Duration::from_millis(100)); } // Now simulate receiving HeaviestFork messages. - let mut expected_received_heaviest_fork = HashMap::new(); let validators_to_take: usize = ((WAIT_FOR_SUPERMAJORITY_THRESHOLD_PERCENT - NON_CONFORMING_VALIDATOR_PERCENT) * TOTAL_VALIDATOR_COUNT as u64 @@ -1728,16 +1688,6 @@ mod tests { &keypairs.node_keypair, now, ); - expected_received_heaviest_fork.insert( - node_pubkey.to_string(), - HeaviestForkRecord { - slot: expected_heaviest_fork_slot, - bankhash: expected_heaviest_fork_bankhash.to_string(), - total_active_stake: total_active_stake_during_heaviest_fork, - shred_version: SHRED_VERSION as u32, - wallclock: now, - }, - ); } assert!(wen_restart_thread_handle.join().is_ok()); @@ -1765,6 +1715,12 @@ mod tests { ); // We are simulating 5% joined LastVotedForkSlots but not HeaviestFork. let voted_stake = total_active_stake_during_heaviest_fork + 100; + let my_pubkey = test_state.validator_voting_keypairs[MY_INDEX] + .node_keypair + .pubkey(); + let leader_pubkey = test_state.validator_voting_keypairs[LEADER_INDEX] + .node_keypair + .pubkey(); assert_eq!( progress, WenRestartProgress { @@ -1797,24 +1753,19 @@ mod tests { }), my_heaviest_fork: Some(HeaviestForkRecord { slot: expected_heaviest_fork_slot, - bankhash: progress - .my_heaviest_fork - .as_ref() - .unwrap() - .bankhash - .to_string(), - total_active_stake: total_active_stake_during_heaviest_fork, + bankhash: expected_heaviest_fork_bankhash.to_string(), + total_active_stake: 0, shred_version: SHRED_VERSION as u32, wallclock: 0, + from: my_pubkey.to_string(), }), - heaviest_fork_aggregate: Some(HeaviestForkAggregateRecord { - received: expected_received_heaviest_fork, - final_result: Some(HeaviestForkAggregateFinal { - total_active_stake: total_active_stake_during_heaviest_fork, - total_active_stake_seen_supermajority: - total_active_stake_during_heaviest_fork, - total_active_stake_agreed_with_me: total_active_stake_during_heaviest_fork, - }), + leader_heaviest_fork: Some(HeaviestForkRecord { + slot: expected_heaviest_fork_slot, + bankhash: expected_heaviest_fork_bankhash.to_string(), + total_active_stake: 0, + shred_version: SHRED_VERSION as u32, + wallclock: progress.leader_heaviest_fork.as_ref().unwrap().wallclock, + from: leader_pubkey.to_string(), }), my_snapshot: Some(GenerateSnapshotRecord { slot: expected_heaviest_fork_slot, @@ -1984,6 +1935,7 @@ mod tests { assert_eq!( wait_for_wen_restart(WenRestartConfig { wen_restart_path: test_state.wen_restart_proto_path, + wen_restart_leader: test_state.wen_restart_leader, last_vote: VoteTransaction::from(Vote::new( vec![new_root_slot], last_vote_bankhash @@ -2071,6 +2023,7 @@ mod tests { total_active_stake: 0, shred_version: SHRED_VERSION as u32, wallclock: 0, + from: Pubkey::default().to_string(), }), ..Default::default() }; @@ -2200,6 +2153,7 @@ mod tests { total_active_stake: 0, shred_version: SHRED_VERSION as u32, wallclock: 0, + from: Pubkey::default().to_string(), }), last_voted_fork_slots_aggregate: Some(LastVotedForkSlotsAggregateRecord { received: HashMap::new(), @@ -2263,6 +2217,7 @@ mod tests { total_active_stake: 0, shred_version: SHRED_VERSION as u32, wallclock: 0, + from: Pubkey::default().to_string(), }), my_snapshot: Some(GenerateSnapshotRecord { slot: 0, @@ -2304,6 +2259,7 @@ mod tests { total_active_stake: 0, shred_version: SHRED_VERSION as u32, wallclock: 0, + from: Pubkey::default().to_string(), }), my_snapshot: Some(GenerateSnapshotRecord { slot: last_vote_slot, @@ -2590,12 +2546,23 @@ mod tests { }], }), }); + let my_pubkey = Pubkey::new_unique(); let my_heaviest_fork = Some(HeaviestForkRecord { slot: 1, bankhash: Hash::default().to_string(), total_active_stake: 900, shred_version: SHRED_VERSION as u32, wallclock: 0, + from: my_pubkey.to_string(), + }); + let leader_pubkey = Pubkey::new_unique(); + let leader_heaviest_fork = Some(HeaviestForkRecord { + slot: 2, + bankhash: Hash::default().to_string(), + total_active_stake: 800, + shred_version: SHRED_VERSION as u32, + wallclock: 0, + from: leader_pubkey.to_string(), }); let my_bankhash = Hash::new_unique(); let new_shred_version = SHRED_VERSION + 57; @@ -2605,14 +2572,6 @@ mod tests { path: "snapshot_1".to_string(), shred_version: new_shred_version as u32, }); - let heaviest_fork_aggregate = Some(HeaviestForkAggregateRecord { - received: HashMap::new(), - final_result: Some(HeaviestForkAggregateFinal { - total_active_stake: 900, - total_active_stake_seen_supermajority: 900, - total_active_stake_agreed_with_me: 900, - }), - }); let expected_slots_stake_map: HashMap = vec![(0, 900), (1, 800)].into_iter().collect(); for (entrance_state, exit_state, entrance_progress, exit_progress) in [ @@ -2691,6 +2650,7 @@ mod tests { total_active_stake: 900, shred_version: SHRED_VERSION as u32, wallclock: 0, + from: my_pubkey.to_string(), }), }, WenRestartProgressInternalState::HeaviestFork { new_root_slot: 1 }, @@ -2719,7 +2679,7 @@ mod tests { my_last_voted_fork_slots: my_last_voted_fork_slots.clone(), last_voted_fork_slots_aggregate: last_voted_fork_slots_aggregate.clone(), my_heaviest_fork: my_heaviest_fork.clone(), - heaviest_fork_aggregate: heaviest_fork_aggregate.clone(), + leader_heaviest_fork: leader_heaviest_fork.clone(), ..Default::default() }, WenRestartProgress { @@ -2727,7 +2687,7 @@ mod tests { my_last_voted_fork_slots: my_last_voted_fork_slots.clone(), last_voted_fork_slots_aggregate: last_voted_fork_slots_aggregate.clone(), my_heaviest_fork: my_heaviest_fork.clone(), - heaviest_fork_aggregate: heaviest_fork_aggregate.clone(), + leader_heaviest_fork: leader_heaviest_fork.clone(), ..Default::default() }, ), @@ -2746,7 +2706,7 @@ mod tests { my_last_voted_fork_slots: my_last_voted_fork_slots.clone(), last_voted_fork_slots_aggregate: last_voted_fork_slots_aggregate.clone(), my_heaviest_fork: my_heaviest_fork.clone(), - heaviest_fork_aggregate: heaviest_fork_aggregate.clone(), + leader_heaviest_fork: leader_heaviest_fork.clone(), ..Default::default() }, WenRestartProgress { @@ -2754,7 +2714,7 @@ mod tests { my_last_voted_fork_slots: my_last_voted_fork_slots.clone(), last_voted_fork_slots_aggregate: last_voted_fork_slots_aggregate.clone(), my_heaviest_fork: my_heaviest_fork.clone(), - heaviest_fork_aggregate, + leader_heaviest_fork, my_snapshot: my_snapshot.clone(), ..Default::default() }, @@ -2979,241 +2939,6 @@ mod tests { ); } - fn start_aggregate_heaviest_fork_thread( - test_state: &WenRestartTestInitResult, - heaviest_fork_slot: Slot, - heaviest_fork_bankhash: Hash, - exit: Arc, - expected_error: Option, - ) -> std::thread::JoinHandle<()> { - let progress = wen_restart_proto::WenRestartProgress { - state: RestartState::HeaviestFork.into(), - my_heaviest_fork: Some(HeaviestForkRecord { - slot: heaviest_fork_slot, - bankhash: heaviest_fork_bankhash.to_string(), - total_active_stake: WAIT_FOR_SUPERMAJORITY_THRESHOLD_PERCENT - .saturating_mul(TOTAL_VALIDATOR_COUNT as u64), - shred_version: SHRED_VERSION as u32, - wallclock: 0, - }), - ..Default::default() - }; - let wen_restart_path = test_state.wen_restart_proto_path.clone(); - let cluster_info = test_state.cluster_info.clone(); - let bank_forks = test_state.bank_forks.clone(); - Builder::new() - .name("solana-wen-restart-aggregate-heaviest-fork".to_string()) - .spawn(move || { - let result = aggregate_restart_heaviest_fork( - &wen_restart_path, - WAIT_FOR_SUPERMAJORITY_THRESHOLD_PERCENT, - cluster_info, - bank_forks, - exit, - &mut progress.clone(), - ); - if let Some(expected_error) = expected_error { - assert_eq!( - result.unwrap_err().downcast::().unwrap(), - expected_error - ); - } else { - assert!(result.is_ok()); - } - }) - .unwrap() - } - - #[test] - fn test_aggregate_heaviest_fork_send_gossip_early() { - let ledger_path = get_tmp_ledger_path_auto_delete!(); - let test_state = wen_restart_test_init(&ledger_path); - let heaviest_fork_slot = test_state.last_voted_fork_slots[0] + 3; - let heaviest_fork_bankhash = Hash::new_unique(); - - let mut cursor = solana_gossip::crds::Cursor::default(); - // clear the heaviest fork queue so we make sure a new HeaviestFork is sent out later. - let _ = test_state - .cluster_info - .get_restart_heaviest_fork(&mut cursor); - - let exit = Arc::new(AtomicBool::new(false)); - let thread = start_aggregate_heaviest_fork_thread( - &test_state, - heaviest_fork_slot, - heaviest_fork_bankhash, - exit.clone(), - Some(WenRestartError::Exiting), - ); - // Find the first HeaviestFork message sent out entering the loop. - let my_pubkey = test_state.cluster_info.id(); - let mut found_myself = false; - while !found_myself { - sleep(Duration::from_millis(100)); - test_state.cluster_info.flush_push_queue(); - for gossip_record in test_state - .cluster_info - .get_restart_heaviest_fork(&mut cursor) - { - if gossip_record.from == my_pubkey && gossip_record.observed_stake > 0 { - found_myself = true; - break; - } - } - } - // Simulating everyone sending out the first RestartHeaviestFork message, Gossip propagation takes - // time, so the observed_stake is probably smaller than actual active stake. We should send out - // heaviest fork indicating we have active stake exceeding supermajority. - let validators_to_take: usize = ((WAIT_FOR_SUPERMAJORITY_THRESHOLD_PERCENT - - NON_CONFORMING_VALIDATOR_PERCENT) - * TOTAL_VALIDATOR_COUNT as u64 - / 100 - - 1) - .try_into() - .unwrap(); - for keypair in test_state - .validator_voting_keypairs - .iter() - .take(validators_to_take) - { - let node_pubkey = keypair.node_keypair.pubkey(); - let node = ContactInfo::new_rand(&mut rand::thread_rng(), Some(node_pubkey)); - let now = timestamp(); - push_restart_heaviest_fork( - test_state.cluster_info.clone(), - &node, - heaviest_fork_slot, - &heaviest_fork_bankhash, - 100, - &keypair.node_keypair, - now, - ); - } - let mut found_myself = false; - let expected_active_stake = (WAIT_FOR_SUPERMAJORITY_THRESHOLD_PERCENT - - NON_CONFORMING_VALIDATOR_PERCENT) - * TOTAL_VALIDATOR_COUNT as u64; - while !found_myself { - sleep(Duration::from_millis(100)); - test_state.cluster_info.flush_push_queue(); - for gossip_record in test_state - .cluster_info - .get_restart_heaviest_fork(&mut cursor) - { - if gossip_record.from == my_pubkey - && gossip_record.observed_stake == expected_active_stake - { - found_myself = true; - break; - } - } - } - exit.store(true, Ordering::Relaxed); - assert!(thread.join().is_ok()); - } - - #[test] - fn test_aggregate_heaviest_fork() { - let ledger_path = get_tmp_ledger_path_auto_delete!(); - let test_state = wen_restart_test_init(&ledger_path); - let heaviest_fork_slot = test_state.last_voted_fork_slots[0] + 3; - let heaviest_fork_bankhash = Hash::new_unique(); - let expected_active_stake = (WAIT_FOR_SUPERMAJORITY_THRESHOLD_PERCENT - - NON_CONFORMING_VALIDATOR_PERCENT) - * TOTAL_VALIDATOR_COUNT as u64; - let progress = wen_restart_proto::WenRestartProgress { - state: RestartState::HeaviestFork.into(), - my_heaviest_fork: Some(HeaviestForkRecord { - slot: heaviest_fork_slot, - bankhash: heaviest_fork_bankhash.to_string(), - total_active_stake: expected_active_stake, - shred_version: SHRED_VERSION as u32, - wallclock: 0, - }), - ..Default::default() - }; - - let different_bankhash = Hash::new_unique(); - let validators_to_take: usize = ((WAIT_FOR_SUPERMAJORITY_THRESHOLD_PERCENT - - NON_CONFORMING_VALIDATOR_PERCENT) - * TOTAL_VALIDATOR_COUNT as u64 - / 100 - - 1) - .try_into() - .unwrap(); - for keypair in test_state - .validator_voting_keypairs - .iter() - .take(validators_to_take) - { - let node_pubkey = keypair.node_keypair.pubkey(); - let node = ContactInfo::new_rand(&mut rand::thread_rng(), Some(node_pubkey)); - let now = timestamp(); - push_restart_heaviest_fork( - test_state.cluster_info.clone(), - &node, - heaviest_fork_slot, - &different_bankhash, - expected_active_stake, - &keypair.node_keypair, - now, - ); - } - let mut expected_block_stake_map = HashMap::new(); - expected_block_stake_map.insert((heaviest_fork_slot, heaviest_fork_bankhash), 100); - expected_block_stake_map.insert((heaviest_fork_slot, different_bankhash), 1400); - assert_eq!( - aggregate_restart_heaviest_fork( - &test_state.wen_restart_proto_path, - WAIT_FOR_SUPERMAJORITY_THRESHOLD_PERCENT, - test_state.cluster_info.clone(), - test_state.bank_forks.clone(), - Arc::new(AtomicBool::new(false)), - &mut progress.clone(), - ) - .unwrap_err() - .downcast::() - .unwrap(), - WenRestartError::NotEnoughStakeAgreeingWithUs( - heaviest_fork_slot, - heaviest_fork_bankhash, - expected_block_stake_map - ), - ); - // If we have enough stake agreeing with us, we should be able to aggregate the heaviest fork. - let validators_to_take: usize = - (WAIT_FOR_SUPERMAJORITY_THRESHOLD_PERCENT * TOTAL_VALIDATOR_COUNT as u64 / 100 - 1) - .try_into() - .unwrap(); - for keypair in test_state - .validator_voting_keypairs - .iter() - .take(validators_to_take) - { - let node_pubkey = keypair.node_keypair.pubkey(); - let node = ContactInfo::new_rand(&mut rand::thread_rng(), Some(node_pubkey)); - let now = timestamp(); - push_restart_heaviest_fork( - test_state.cluster_info.clone(), - &node, - heaviest_fork_slot, - &heaviest_fork_bankhash, - expected_active_stake, - &keypair.node_keypair, - now, - ); - } - assert!(aggregate_restart_heaviest_fork( - &test_state.wen_restart_proto_path, - WAIT_FOR_SUPERMAJORITY_THRESHOLD_PERCENT, - test_state.cluster_info.clone(), - test_state.bank_forks.clone(), - Arc::new(AtomicBool::new(false)), - &mut progress.clone(), - ) - .is_ok()); - } - #[test] fn test_generate_snapshot() { solana_logger::setup(); @@ -3375,6 +3100,7 @@ mod tests { let last_vote_bankhash = Hash::new_unique(); let config = WenRestartConfig { wen_restart_path: test_state.wen_restart_proto_path.clone(), + wen_restart_leader: test_state.wen_restart_leader, last_vote: VoteTransaction::from(Vote::new(vec![last_vote_slot], last_vote_bankhash)), blockstore: test_state.blockstore.clone(), cluster_info: test_state.cluster_info.clone(), From 0f929b37ab193d66874934573b6f59e4bcf55410 Mon Sep 17 00:00:00 2001 From: Wen <113942165+wen-coding@users.noreply.github.com> Date: Fri, 13 Sep 2024 14:52:36 -0700 Subject: [PATCH 02/25] Fix logic to calculate slots to repair, add more logs. --- wen-restart/src/wen_restart.rs | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/wen-restart/src/wen_restart.rs b/wen-restart/src/wen_restart.rs index 67f3ed51e6b382..fe659619f3d458 100644 --- a/wen-restart/src/wen_restart.rs +++ b/wen-restart/src/wen_restart.rs @@ -647,6 +647,11 @@ pub(crate) fn find_bankhash_of_heaviest_fork( cur_bank } }; + info!( + "wen_restart find bankhash slot: {} hash: {}", + slot, + bank.hash() + ); parent_bank = bank; } Ok(parent_bank.hash()) @@ -729,11 +734,13 @@ fn repair_heaviest_fork( } let to_repair = if blockstore.meta(heaviest_slot).is_ok_and(|x| x.is_some()) { AncestorIterator::new_inclusive(heaviest_slot, &blockstore) - .take_while(|slot| *slot > my_heaviest_fork_slot && !blockstore.is_full(*slot)) + .take_while(|slot| *slot > my_heaviest_fork_slot) + .filter(|slot| !blockstore.is_full(*slot)) .collect() } else { vec![heaviest_slot] }; + info!("wen_restart repair slots: {:?}", to_repair); if to_repair.is_empty() { return Ok(()); // All blocks are full } From 63f2bb1c85090e0fbb3699d37ba6512aa695f099 Mon Sep 17 00:00:00 2001 From: Wen <113942165+wen-coding@users.noreply.github.com> Date: Fri, 13 Sep 2024 15:18:25 -0700 Subject: [PATCH 03/25] Let the leader generate snapshot and print error log before continuing to send out heaviest fork. --- wen-restart/src/wen_restart.rs | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/wen-restart/src/wen_restart.rs b/wen-restart/src/wen_restart.rs index fe659619f3d458..0ffac69a5fffa2 100644 --- a/wen-restart/src/wen_restart.rs +++ b/wen-restart/src/wen_restart.rs @@ -660,6 +660,7 @@ pub(crate) fn find_bankhash_of_heaviest_fork( pub(crate) fn send_restart_heaviest_fork( cluster_info: Arc, exit: Arc, + only_loop_once: bool, progress: &mut WenRestartProgress, ) -> Result<()> { if progress.my_heaviest_fork.is_none() { @@ -677,6 +678,9 @@ pub(crate) fn send_restart_heaviest_fork( break; } cluster_info.push_restart_heaviest_fork(heaviest_fork_slot, heaviest_fork_hash, 0); + if only_loop_once { + break; + } sleep(Duration::from_secs(HEAVIEST_REFRESH_INTERVAL_IN_SECONDS)); } Ok(()) @@ -912,6 +916,7 @@ pub fn wait_for_wen_restart(config: WenRestartConfig) -> Result<()> { send_restart_heaviest_fork( config.cluster_info.clone(), config.exit.clone(), + true, // only_loop_once &mut progress, )?; } else { @@ -964,6 +969,14 @@ pub fn wait_for_wen_restart(config: WenRestartConfig) -> Result<()> { --no-snapshot-fetch", slot, hash, shred_version, ); + if config.cluster_info.id() == config.wen_restart_leader { + send_restart_heaviest_fork( + config.cluster_info.clone(), + config.exit.clone(), + false, // only_loop_once + &mut progress, + )?; + } return Ok(()); } }; From 0590c8e77b5e0beacd95249d79b4c5495a24e77c Mon Sep 17 00:00:00 2001 From: Wen <113942165+wen-coding@users.noreply.github.com> Date: Fri, 13 Sep 2024 22:20:34 -0700 Subject: [PATCH 04/25] Filter ancestors older than root. --- wen-restart/src/wen_restart.rs | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/wen-restart/src/wen_restart.rs b/wen-restart/src/wen_restart.rs index 0ffac69a5fffa2..5f4df3d32cd23c 100644 --- a/wen-restart/src/wen_restart.rs +++ b/wen-restart/src/wen_restart.rs @@ -769,10 +769,14 @@ pub(crate) fn verify_restart_heaviest_fork( blockstore.clone(), wen_restart_repair_slots.clone(), )?; - let root_bank = bank_forks.read().unwrap().root_bank(); + let root_bank; + { + root_bank = bank_forks.read().unwrap().root_bank(); + } let root_slot = root_bank.slot(); - let mut slots: Vec = - AncestorIterator::new_inclusive(heaviest_slot, &blockstore).collect(); + let mut slots: Vec = AncestorIterator::new_inclusive(heaviest_slot, &blockstore) + .take_while(|slot| slot >= &root_slot) + .collect(); slots.sort(); if !slots.contains(&root_slot) { return Err( @@ -802,7 +806,7 @@ pub(crate) fn verify_restart_heaviest_fork( slots, blockstore.clone(), bank_forks.clone(), - bank_forks.read().unwrap().root_bank(), + root_bank, &exit, )? } else if let Some(bank) = bank_forks.read().unwrap().get(heaviest_slot) { From ac5c6b120e950a2f71ea9e6876d80dc8b82c159e Mon Sep 17 00:00:00 2001 From: Wen <113942165+wen-coding@users.noreply.github.com> Date: Fri, 13 Sep 2024 23:12:39 -0700 Subject: [PATCH 05/25] Reduce lock scope. --- wen-restart/src/wen_restart.rs | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/wen-restart/src/wen_restart.rs b/wen-restart/src/wen_restart.rs index 5f4df3d32cd23c..def1be686093a5 100644 --- a/wen-restart/src/wen_restart.rs +++ b/wen-restart/src/wen_restart.rs @@ -570,15 +570,16 @@ pub(crate) fn find_bankhash_of_heaviest_fork( root_bank: Arc, exit: &AtomicBool, ) -> Result { - let heaviest_fork_bankhash = bank_forks - .read() - .unwrap() - .get(heaviest_fork_slot) - .map(|bank| bank.hash()); - if let Some(hash) = heaviest_fork_bankhash { - return Ok(hash); + { + if let Some(hash) = bank_forks + .read() + .unwrap() + .get(heaviest_fork_slot) + .map(|bank| bank.hash()) + { + return Ok(hash); + } } - let leader_schedule_cache = LeaderScheduleCache::new_from_bank(&root_bank); let replay_tx_thread_pool = rayon::ThreadPoolBuilder::new() .thread_name(|i| format!("solReplayTx{i:02}")) From 0a5efe8a0e3c5a0ba4b130ea95bda20568963515 Mon Sep 17 00:00:00 2001 From: Wen <113942165+wen-coding@users.noreply.github.com> Date: Sat, 14 Sep 2024 14:55:09 -0700 Subject: [PATCH 06/25] Rename variables and functions. --- wen-restart/src/wen_restart.rs | 57 ++++++++++++++++++---------------- 1 file changed, 31 insertions(+), 26 deletions(-) diff --git a/wen-restart/src/wen_restart.rs b/wen-restart/src/wen_restart.rs index def1be686093a5..bb5eb98db2e31c 100644 --- a/wen-restart/src/wen_restart.rs +++ b/wen-restart/src/wen_restart.rs @@ -710,17 +710,17 @@ pub(crate) fn receive_restart_heaviest_fork( "Received new heaviest fork from leader: {} {:?}", wen_restart_leader, new_heaviest_fork ); - let heaviest_slot = new_heaviest_fork.last_slot; - let heaviest_hash = new_heaviest_fork.last_slot_hash; + let leader_heaviest_slot = new_heaviest_fork.last_slot; + let leader_heaviest_hash = new_heaviest_fork.last_slot_hash; progress.leader_heaviest_fork = Some(HeaviestForkRecord { - slot: heaviest_slot, - bankhash: heaviest_hash.to_string(), + slot: leader_heaviest_slot, + bankhash: leader_heaviest_hash.to_string(), total_active_stake: 0, wallclock: new_heaviest_fork.wallclock, shred_version: new_heaviest_fork.shred_version as u32, from: new_heaviest_fork.from.to_string(), }); - return Ok((heaviest_slot, heaviest_hash)); + return Ok((leader_heaviest_slot, leader_heaviest_hash)); } sleep(Duration::from_millis(GOSSIP_SLEEP_MILLIS)); } @@ -754,10 +754,10 @@ fn repair_heaviest_fork( } } -pub(crate) fn verify_restart_heaviest_fork( +pub(crate) fn verify_leader_heaviest_fork( my_heaviest_fork_slot: Slot, - heaviest_slot: Slot, - heaviest_hash: &Hash, + leader_heaviest_slot: Slot, + leader_heaviest_hash: &Hash, bank_forks: Arc>, blockstore: Arc, exit: Arc, @@ -765,7 +765,7 @@ pub(crate) fn verify_restart_heaviest_fork( ) -> Result<()> { repair_heaviest_fork( my_heaviest_fork_slot, - heaviest_slot, + leader_heaviest_slot, exit.clone(), blockstore.clone(), wen_restart_repair_slots.clone(), @@ -775,50 +775,55 @@ pub(crate) fn verify_restart_heaviest_fork( root_bank = bank_forks.read().unwrap().root_bank(); } let root_slot = root_bank.slot(); - let mut slots: Vec = AncestorIterator::new_inclusive(heaviest_slot, &blockstore) + let mut slots: Vec = AncestorIterator::new_inclusive(leader_heaviest_slot, &blockstore) .take_while(|slot| slot >= &root_slot) .collect(); slots.sort(); if !slots.contains(&root_slot) { - return Err( - WenRestartError::HeaviestForkOnLeaderOnDifferentFork(heaviest_slot, root_slot).into(), - ); + return Err(WenRestartError::HeaviestForkOnLeaderOnDifferentFork( + leader_heaviest_slot, + root_slot, + ) + .into()); } - if heaviest_slot > my_heaviest_fork_slot && !slots.contains(&my_heaviest_fork_slot) { + if leader_heaviest_slot > my_heaviest_fork_slot && !slots.contains(&my_heaviest_fork_slot) { return Err(WenRestartError::HeaviestForkOnLeaderOnDifferentFork( - heaviest_slot, + leader_heaviest_slot, my_heaviest_fork_slot, ) .into()); } - if heaviest_slot < my_heaviest_fork_slot + if leader_heaviest_slot < my_heaviest_fork_slot && !AncestorIterator::new(my_heaviest_fork_slot, &blockstore) - .any(|slot| slot == heaviest_slot) + .any(|slot| slot == leader_heaviest_slot) { return Err(WenRestartError::HeaviestForkOnLeaderOnDifferentFork( - heaviest_slot, + leader_heaviest_slot, my_heaviest_fork_slot, ) .into()); } let my_bankhash = if !slots.is_empty() { find_bankhash_of_heaviest_fork( - heaviest_slot, + leader_heaviest_slot, slots, blockstore.clone(), bank_forks.clone(), root_bank, &exit, )? - } else if let Some(bank) = bank_forks.read().unwrap().get(heaviest_slot) { + } else if let Some(bank) = bank_forks.read().unwrap().get(leader_heaviest_slot) { bank.hash() } else { - return Err(WenRestartError::BlockNotFound(heaviest_slot).into()); + return Err(WenRestartError::BlockNotFound(leader_heaviest_slot).into()); }; - if my_bankhash != *heaviest_hash { - return Err( - WenRestartError::BankHashMismatch(heaviest_slot, my_bankhash, *heaviest_hash).into(), - ); + if my_bankhash != *leader_heaviest_hash { + return Err(WenRestartError::BankHashMismatch( + leader_heaviest_slot, + my_bankhash, + *leader_heaviest_hash, + ) + .into()); } Ok(()) } @@ -931,7 +936,7 @@ pub fn wait_for_wen_restart(config: WenRestartConfig) -> Result<()> { config.exit.clone(), &mut progress, )?; - verify_restart_heaviest_fork( + verify_leader_heaviest_fork( new_root_slot, leader_slot, &leader_hash, From 05453339056333e6f6b6aa8d994851645d88f60c Mon Sep 17 00:00:00 2001 From: Wen <113942165+wen-coding@users.noreply.github.com> Date: Mon, 16 Sep 2024 14:02:58 -0700 Subject: [PATCH 07/25] Make leader aggregate the heaviest fork of everyone. --- wen-restart/proto/wen_restart.proto | 10 +- wen-restart/src/heaviest_fork_aggregate.rs | 614 +++++++++++++++++++++ wen-restart/src/lib.rs | 1 + wen-restart/src/wen_restart.rs | 166 ++++-- 4 files changed, 758 insertions(+), 33 deletions(-) create mode 100644 wen-restart/src/heaviest_fork_aggregate.rs diff --git a/wen-restart/proto/wen_restart.proto b/wen-restart/proto/wen_restart.proto index d95da208b4bbb9..e99d20207ed770 100644 --- a/wen-restart/proto/wen_restart.proto +++ b/wen-restart/proto/wen_restart.proto @@ -42,6 +42,11 @@ message HeaviestForkRecord { string from = 6; } +message HeaviestForkAggregateRecord { + repeated HeaviestForkRecord received = 1; + uint64 total_active_stake = 2; +} + message GenerateSnapshotRecord { string path = 1; uint64 slot = 2; @@ -60,6 +65,7 @@ message WenRestartProgress { optional LastVotedForkSlotsAggregateRecord last_voted_fork_slots_aggregate = 3; optional HeaviestForkRecord my_heaviest_fork = 4; optional HeaviestForkRecord leader_heaviest_fork = 5; - optional GenerateSnapshotRecord my_snapshot = 6; - map conflict_message = 7; + optional HeaviestForkAggregateRecord heaviest_fork_aggregate = 6; + optional GenerateSnapshotRecord my_snapshot = 7; + map conflict_message = 8; } diff --git a/wen-restart/src/heaviest_fork_aggregate.rs b/wen-restart/src/heaviest_fork_aggregate.rs new file mode 100644 index 00000000000000..63bca761a93b87 --- /dev/null +++ b/wen-restart/src/heaviest_fork_aggregate.rs @@ -0,0 +1,614 @@ +use { + crate::solana::wen_restart_proto::HeaviestForkRecord, + anyhow::Result, + log::*, + solana_gossip::restart_crds_values::RestartHeaviestFork, + solana_runtime::epoch_stakes::EpochStakes, + solana_sdk::{clock::Slot, hash::Hash, pubkey::Pubkey}, + std::{ + collections::{HashMap, HashSet}, + str::FromStr, + }, +}; + +pub(crate) struct HeaviestForkAggregate { + my_shred_version: u16, + my_pubkey: Pubkey, + // We use the epoch_stakes of the Epoch our heaviest bank is in. Proceed and exit only if + // enough validator agree with me. + epoch_stakes: EpochStakes, + heaviest_forks: HashMap, + block_stake_map: HashMap<(Slot, Hash), u64>, + active_peers: HashSet, +} + +#[derive(Debug, PartialEq)] +pub enum HeaviestForkAggregateResult { + AlreadyExists, + DifferentVersionExists(RestartHeaviestFork, RestartHeaviestFork), + Inserted(HeaviestForkRecord), + Malformed, + ZeroStakeIgnored, +} + +impl HeaviestForkAggregate { + pub(crate) fn new( + my_shred_version: u16, + epoch_stakes: &EpochStakes, + my_heaviest_fork_slot: Slot, + my_heaviest_fork_hash: Hash, + my_pubkey: &Pubkey, + ) -> Self { + let mut active_peers = HashSet::new(); + active_peers.insert(*my_pubkey); + let mut block_stake_map = HashMap::new(); + block_stake_map.insert( + (my_heaviest_fork_slot, my_heaviest_fork_hash), + epoch_stakes.node_id_to_stake(my_pubkey).unwrap_or(0), + ); + Self { + my_shred_version, + my_pubkey: *my_pubkey, + epoch_stakes: epoch_stakes.clone(), + heaviest_forks: HashMap::new(), + block_stake_map, + active_peers, + } + } + + pub(crate) fn aggregate_from_record( + &mut self, + record: &HeaviestForkRecord, + ) -> Result { + let from = Pubkey::from_str(&record.from)?; + let bankhash = Hash::from_str(&record.bankhash)?; + let restart_heaviest_fork = RestartHeaviestFork { + from, + wallclock: record.wallclock, + last_slot: record.slot, + last_slot_hash: bankhash, + observed_stake: record.total_active_stake, + shred_version: record.shred_version as u16, + }; + Ok(self.aggregate(restart_heaviest_fork)) + } + + fn is_valid_change( + current_heaviest_fork: &RestartHeaviestFork, + new_heaviest_fork: &RestartHeaviestFork, + ) -> HeaviestForkAggregateResult { + if current_heaviest_fork.last_slot != new_heaviest_fork.last_slot + || current_heaviest_fork.last_slot_hash != new_heaviest_fork.last_slot_hash + { + return HeaviestForkAggregateResult::DifferentVersionExists( + current_heaviest_fork.clone(), + new_heaviest_fork.clone(), + ); + } + if current_heaviest_fork == new_heaviest_fork + || current_heaviest_fork.wallclock > new_heaviest_fork.wallclock + { + return HeaviestForkAggregateResult::AlreadyExists; + } + HeaviestForkAggregateResult::Inserted(HeaviestForkRecord { + slot: new_heaviest_fork.last_slot, + bankhash: new_heaviest_fork.last_slot_hash.to_string(), + total_active_stake: new_heaviest_fork.observed_stake, + shred_version: new_heaviest_fork.shred_version as u32, + wallclock: new_heaviest_fork.wallclock, + from: new_heaviest_fork.from.to_string(), + }) + } + + pub(crate) fn aggregate( + &mut self, + received_heaviest_fork: RestartHeaviestFork, + ) -> HeaviestForkAggregateResult { + let from = &received_heaviest_fork.from; + let sender_stake = self.epoch_stakes.node_id_to_stake(from).unwrap_or(0); + if sender_stake == 0 { + warn!( + "Gossip should not accept zero-stake RestartLastVotedFork from {:?}", + from + ); + return HeaviestForkAggregateResult::ZeroStakeIgnored; + } + if from == &self.my_pubkey { + return HeaviestForkAggregateResult::AlreadyExists; + } + if received_heaviest_fork.shred_version != self.my_shred_version { + warn!( + "Gossip should not accept RestartLastVotedFork with different shred version {} from {:?}", + received_heaviest_fork.shred_version, from + ); + return HeaviestForkAggregateResult::Malformed; + } + let result = if let Some(old_heaviest_fork) = self.heaviest_forks.get(from) { + let result = Self::is_valid_change(old_heaviest_fork, &received_heaviest_fork); + if let HeaviestForkAggregateResult::Inserted(_) = result { + // continue following processing + } else { + return result; + } + result + } else { + let entry = self + .block_stake_map + .entry(( + received_heaviest_fork.last_slot, + received_heaviest_fork.last_slot_hash, + )) + .or_insert(0); + *entry = entry.saturating_add(sender_stake); + self.active_peers.insert(*from); + HeaviestForkAggregateResult::Inserted(HeaviestForkRecord { + slot: received_heaviest_fork.last_slot, + bankhash: received_heaviest_fork.last_slot_hash.to_string(), + total_active_stake: received_heaviest_fork.observed_stake, + shred_version: received_heaviest_fork.shred_version as u32, + wallclock: received_heaviest_fork.wallclock, + from: from.to_string(), + }) + }; + self.heaviest_forks + .insert(*from, received_heaviest_fork.clone()); + result + } + + pub(crate) fn total_active_stake(&self) -> u64 { + self.active_peers.iter().fold(0, |sum: u64, pubkey| { + sum.saturating_add(self.epoch_stakes.node_id_to_stake(pubkey).unwrap_or(0)) + }) + } + + pub(crate) fn block_stake_map(self) -> HashMap<(Slot, Hash), u64> { + self.block_stake_map + } +} + +#[cfg(test)] +mod tests { + use { + crate::{ + heaviest_fork_aggregate::{HeaviestForkAggregate, HeaviestForkAggregateResult}, + solana::wen_restart_proto::HeaviestForkRecord, + }, + solana_gossip::restart_crds_values::RestartHeaviestFork, + solana_program::{clock::Slot, pubkey::Pubkey}, + solana_runtime::{ + bank::Bank, + genesis_utils::{ + create_genesis_config_with_vote_accounts, GenesisConfigInfo, ValidatorVoteKeypairs, + }, + }, + solana_sdk::{hash::Hash, signature::Signer, timing::timestamp}, + }; + + const TOTAL_VALIDATOR_COUNT: u16 = 20; + const MY_INDEX: usize = 19; + const SHRED_VERSION: u16 = 52; + + struct TestAggregateInitResult { + pub heaviest_fork_aggregate: HeaviestForkAggregate, + pub validator_voting_keypairs: Vec, + pub heaviest_slot: Slot, + pub heaviest_hash: Hash, + } + + fn test_aggregate_init() -> TestAggregateInitResult { + solana_logger::setup(); + let validator_voting_keypairs: Vec<_> = (0..TOTAL_VALIDATOR_COUNT) + .map(|_| ValidatorVoteKeypairs::new_rand()) + .collect(); + let GenesisConfigInfo { genesis_config, .. } = create_genesis_config_with_vote_accounts( + 10_000, + &validator_voting_keypairs, + vec![100; validator_voting_keypairs.len()], + ); + let (_, bank_forks) = Bank::new_with_bank_forks_for_tests(&genesis_config); + let root_bank = bank_forks.read().unwrap().root_bank(); + let heaviest_slot = root_bank.slot().saturating_add(3); + let heaviest_hash = Hash::new_unique(); + TestAggregateInitResult { + heaviest_fork_aggregate: HeaviestForkAggregate::new( + SHRED_VERSION, + root_bank.epoch_stakes(root_bank.epoch()).unwrap(), + heaviest_slot, + heaviest_hash, + &validator_voting_keypairs[MY_INDEX].node_keypair.pubkey(), + ), + validator_voting_keypairs, + heaviest_slot, + heaviest_hash, + } + } + + #[test] + fn test_aggregate_from_gossip() { + let mut test_state = test_aggregate_init(); + let initial_num_active_validators = 3; + let timestamp1 = timestamp(); + for validator_voting_keypair in test_state + .validator_voting_keypairs + .iter() + .take(initial_num_active_validators) + { + let pubkey = validator_voting_keypair.node_keypair.pubkey(); + assert_eq!( + test_state + .heaviest_fork_aggregate + .aggregate(RestartHeaviestFork { + from: pubkey, + wallclock: timestamp1, + last_slot: test_state.heaviest_slot, + last_slot_hash: test_state.heaviest_hash, + observed_stake: 100, + shred_version: SHRED_VERSION, + },), + HeaviestForkAggregateResult::Inserted(HeaviestForkRecord { + slot: test_state.heaviest_slot, + bankhash: test_state.heaviest_hash.to_string(), + total_active_stake: 100, + shred_version: SHRED_VERSION as u32, + wallclock: timestamp1, + from: pubkey.to_string(), + }), + ); + } + assert_eq!( + test_state.heaviest_fork_aggregate.total_active_stake(), + (initial_num_active_validators + 1) as u64 * 100 + ); + + let new_active_validator = test_state.validator_voting_keypairs + [initial_num_active_validators + 1] + .node_keypair + .pubkey(); + let now = timestamp(); + let new_active_validator_last_voted_slots = RestartHeaviestFork { + from: new_active_validator, + wallclock: now, + last_slot: test_state.heaviest_slot, + last_slot_hash: test_state.heaviest_hash, + observed_stake: 100, + shred_version: SHRED_VERSION, + }; + assert_eq!( + test_state + .heaviest_fork_aggregate + .aggregate(new_active_validator_last_voted_slots), + HeaviestForkAggregateResult::Inserted(HeaviestForkRecord { + slot: test_state.heaviest_slot, + bankhash: test_state.heaviest_hash.to_string(), + total_active_stake: 100, + shred_version: SHRED_VERSION as u32, + wallclock: now, + from: new_active_validator.to_string(), + }), + ); + let expected_total_active_stake = (initial_num_active_validators + 2) as u64 * 100; + assert_eq!( + test_state.heaviest_fork_aggregate.total_active_stake(), + expected_total_active_stake + ); + let replace_message_validator = test_state.validator_voting_keypairs[2] + .node_keypair + .pubkey(); + // If hash changes, it will be ignored. + let now = timestamp(); + let new_hash = Hash::new_unique(); + let replace_message_validator_last_fork = RestartHeaviestFork { + from: replace_message_validator, + wallclock: now, + last_slot: test_state.heaviest_slot + 1, + last_slot_hash: new_hash, + observed_stake: 100, + shred_version: SHRED_VERSION, + }; + assert_eq!( + test_state + .heaviest_fork_aggregate + .aggregate(replace_message_validator_last_fork.clone()), + HeaviestForkAggregateResult::DifferentVersionExists( + RestartHeaviestFork { + from: replace_message_validator, + wallclock: timestamp1, + last_slot: test_state.heaviest_slot, + last_slot_hash: test_state.heaviest_hash, + observed_stake: 100, + shred_version: SHRED_VERSION, + }, + replace_message_validator_last_fork, + ), + ); + assert_eq!( + test_state.heaviest_fork_aggregate.total_active_stake(), + expected_total_active_stake + ); + + // test that zero stake validator is ignored. + let random_pubkey = Pubkey::new_unique(); + assert_eq!( + test_state + .heaviest_fork_aggregate + .aggregate(RestartHeaviestFork { + from: random_pubkey, + wallclock: timestamp(), + last_slot: test_state.heaviest_slot, + last_slot_hash: test_state.heaviest_hash, + observed_stake: 100, + shred_version: SHRED_VERSION, + },), + HeaviestForkAggregateResult::ZeroStakeIgnored, + ); + assert_eq!( + test_state.heaviest_fork_aggregate.total_active_stake(), + expected_total_active_stake + ); + + // If everyone is seeing only 70%, the total active stake seeing supermajority is 0. + for validator_voting_keypair in test_state.validator_voting_keypairs.iter().take(13) { + let pubkey = validator_voting_keypair.node_keypair.pubkey(); + let now = timestamp(); + assert_eq!( + test_state + .heaviest_fork_aggregate + .aggregate(RestartHeaviestFork { + from: pubkey, + wallclock: now, + last_slot: test_state.heaviest_slot, + last_slot_hash: test_state.heaviest_hash, + observed_stake: 1400, + shred_version: SHRED_VERSION, + },), + HeaviestForkAggregateResult::Inserted(HeaviestForkRecord { + slot: test_state.heaviest_slot, + bankhash: test_state.heaviest_hash.to_string(), + total_active_stake: 1400, + shred_version: SHRED_VERSION as u32, + wallclock: now, + from: pubkey.to_string(), + }), + ); + } + assert_eq!( + test_state.heaviest_fork_aggregate.total_active_stake(), + 1400 + ); + + // test that when 75% of the stake is seeing supermajority, + // the active percent seeing supermajority is 75%. + for validator_voting_keypair in test_state.validator_voting_keypairs.iter().take(14) { + let pubkey = validator_voting_keypair.node_keypair.pubkey(); + let now = timestamp(); + assert_eq!( + test_state + .heaviest_fork_aggregate + .aggregate(RestartHeaviestFork { + from: pubkey, + wallclock: now, + last_slot: test_state.heaviest_slot, + last_slot_hash: test_state.heaviest_hash, + observed_stake: 1500, + shred_version: SHRED_VERSION, + },), + HeaviestForkAggregateResult::Inserted(HeaviestForkRecord { + slot: test_state.heaviest_slot, + bankhash: test_state.heaviest_hash.to_string(), + total_active_stake: 1500, + shred_version: SHRED_VERSION as u32, + wallclock: now, + from: pubkey.to_string(), + }), + ); + } + + assert_eq!( + test_state.heaviest_fork_aggregate.total_active_stake(), + 1500 + ); + + // test that message from my pubkey is ignored. + assert_eq!( + test_state + .heaviest_fork_aggregate + .aggregate(RestartHeaviestFork { + from: test_state.validator_voting_keypairs[MY_INDEX] + .node_keypair + .pubkey(), + wallclock: timestamp(), + last_slot: test_state.heaviest_slot, + last_slot_hash: test_state.heaviest_hash, + observed_stake: 100, + shred_version: SHRED_VERSION, + },), + HeaviestForkAggregateResult::AlreadyExists, + ); + } + + #[test] + fn test_aggregate_from_record() { + let mut test_state = test_aggregate_init(); + let time1 = timestamp(); + let from = test_state.validator_voting_keypairs[0] + .node_keypair + .pubkey(); + let record = HeaviestForkRecord { + wallclock: time1, + slot: test_state.heaviest_slot, + bankhash: test_state.heaviest_hash.to_string(), + shred_version: SHRED_VERSION as u32, + total_active_stake: 100, + from: from.to_string(), + }; + assert_eq!(test_state.heaviest_fork_aggregate.total_active_stake(), 100); + assert_eq!( + test_state + .heaviest_fork_aggregate + .aggregate_from_record(&record) + .unwrap(), + HeaviestForkAggregateResult::Inserted(record.clone()), + ); + assert_eq!(test_state.heaviest_fork_aggregate.total_active_stake(), 200); + // Now if you get the same result from Gossip again, it should be ignored. + assert_eq!( + test_state + .heaviest_fork_aggregate + .aggregate(RestartHeaviestFork { + from, + wallclock: time1, + last_slot: test_state.heaviest_slot, + last_slot_hash: test_state.heaviest_hash, + observed_stake: 100, + shred_version: SHRED_VERSION, + },), + HeaviestForkAggregateResult::AlreadyExists, + ); + + // If only observed_stake changes, it will be replaced. + let time2 = timestamp(); + let old_heaviest_fork = RestartHeaviestFork { + from, + wallclock: time2, + last_slot: test_state.heaviest_slot, + last_slot_hash: test_state.heaviest_hash, + observed_stake: 200, + shred_version: SHRED_VERSION, + }; + assert_eq!( + test_state + .heaviest_fork_aggregate + .aggregate(old_heaviest_fork.clone()), + HeaviestForkAggregateResult::Inserted(HeaviestForkRecord { + wallclock: time2, + slot: test_state.heaviest_slot, + bankhash: test_state.heaviest_hash.to_string(), + shred_version: SHRED_VERSION as u32, + total_active_stake: 200, + from: from.to_string(), + }), + ); + + // If slot changes, it will be ignored. + let new_heaviest_fork = RestartHeaviestFork { + from: test_state.validator_voting_keypairs[0] + .node_keypair + .pubkey(), + wallclock: timestamp(), + last_slot: test_state.heaviest_slot + 1, + last_slot_hash: test_state.heaviest_hash, + observed_stake: 100, + shred_version: SHRED_VERSION, + }; + assert_eq!( + test_state + .heaviest_fork_aggregate + .aggregate(new_heaviest_fork.clone()), + HeaviestForkAggregateResult::DifferentVersionExists( + old_heaviest_fork.clone(), + new_heaviest_fork + ) + ); + // If hash changes, it will also be ignored. + let new_heaviest_fork = RestartHeaviestFork { + from: test_state.validator_voting_keypairs[0] + .node_keypair + .pubkey(), + wallclock: timestamp(), + last_slot: test_state.heaviest_slot, + last_slot_hash: Hash::new_unique(), + observed_stake: 100, + shred_version: SHRED_VERSION, + }; + assert_eq!( + test_state + .heaviest_fork_aggregate + .aggregate(new_heaviest_fork.clone()), + HeaviestForkAggregateResult::DifferentVersionExists( + old_heaviest_fork, + new_heaviest_fork + ) + ); + + // percentage doesn't change since it's a replace. + assert_eq!(test_state.heaviest_fork_aggregate.total_active_stake(), 200); + + // Record from validator with zero stake should be ignored. + let zero_stake_validator = Pubkey::new_unique(); + assert_eq!( + test_state + .heaviest_fork_aggregate + .aggregate_from_record(&HeaviestForkRecord { + wallclock: timestamp(), + slot: test_state.heaviest_slot, + bankhash: test_state.heaviest_hash.to_string(), + shred_version: SHRED_VERSION as u32, + total_active_stake: 100, + from: zero_stake_validator.to_string(), + }) + .unwrap(), + HeaviestForkAggregateResult::ZeroStakeIgnored, + ); + // percentage doesn't change since the previous aggregate is ignored. + assert_eq!(test_state.heaviest_fork_aggregate.total_active_stake(), 200); + + // Record from my pubkey should be ignored. + let my_pubkey = test_state.validator_voting_keypairs[MY_INDEX] + .node_keypair + .pubkey(); + assert_eq!( + test_state + .heaviest_fork_aggregate + .aggregate_from_record(&HeaviestForkRecord { + wallclock: timestamp(), + slot: test_state.heaviest_slot, + bankhash: test_state.heaviest_hash.to_string(), + shred_version: SHRED_VERSION as u32, + total_active_stake: 100, + from: my_pubkey.to_string(), + }) + .unwrap(), + HeaviestForkAggregateResult::AlreadyExists, + ); + } + + #[test] + fn test_aggregate_from_record_failures() { + let mut test_state = test_aggregate_init(); + let from = test_state.validator_voting_keypairs[0] + .node_keypair + .pubkey(); + let mut heaviest_fork_record = HeaviestForkRecord { + wallclock: timestamp(), + slot: test_state.heaviest_slot, + bankhash: test_state.heaviest_hash.to_string(), + shred_version: SHRED_VERSION as u32, + total_active_stake: 100, + from: from.to_string(), + }; + // First test that this is a valid record. + assert_eq!( + test_state + .heaviest_fork_aggregate + .aggregate_from_record(&heaviest_fork_record,) + .unwrap(), + HeaviestForkAggregateResult::Inserted(heaviest_fork_record.clone()), + ); + // Then test that it fails if the record is invalid. + + // Invalid pubkey. + heaviest_fork_record.from = "invalid_pubkey".to_string(); + assert!(test_state + .heaviest_fork_aggregate + .aggregate_from_record(&heaviest_fork_record,) + .is_err()); + + // Invalid hash. + heaviest_fork_record.from = from.to_string(); + heaviest_fork_record.bankhash.clear(); + assert!(test_state + .heaviest_fork_aggregate + .aggregate_from_record(&heaviest_fork_record,) + .is_err()); + } +} diff --git a/wen-restart/src/lib.rs b/wen-restart/src/lib.rs index d389136bb13bcd..d798cae1313ef4 100644 --- a/wen-restart/src/lib.rs +++ b/wen-restart/src/lib.rs @@ -4,5 +4,6 @@ pub(crate) mod solana { } } +pub(crate) mod heaviest_fork_aggregate; pub(crate) mod last_voted_fork_slots_aggregate; pub mod wen_restart; diff --git a/wen-restart/src/wen_restart.rs b/wen-restart/src/wen_restart.rs index bb5eb98db2e31c..9f88c2bd8d9fa5 100644 --- a/wen-restart/src/wen_restart.rs +++ b/wen-restart/src/wen_restart.rs @@ -2,15 +2,16 @@ use { crate::{ + heaviest_fork_aggregate::{HeaviestForkAggregate, HeaviestForkAggregateResult}, last_voted_fork_slots_aggregate::{ LastVotedForkSlotsAggregate, LastVotedForkSlotsAggregateResult, LastVotedForkSlotsEpochInfo, LastVotedForkSlotsFinalResult, }, solana::wen_restart_proto::{ - self, ConflictMessage, GenerateSnapshotRecord, HeaviestForkRecord, - LastVotedForkSlotsAggregateFinal, LastVotedForkSlotsAggregateRecord, - LastVotedForkSlotsEpochInfoRecord, LastVotedForkSlotsRecord, State as RestartState, - WenRestartProgress, + self, ConflictMessage, GenerateSnapshotRecord, HeaviestForkAggregateRecord, + HeaviestForkRecord, LastVotedForkSlotsAggregateFinal, + LastVotedForkSlotsAggregateRecord, LastVotedForkSlotsEpochInfoRecord, + LastVotedForkSlotsRecord, State as RestartState, WenRestartProgress, }, }, anyhow::Result, @@ -69,8 +70,6 @@ const REPAIR_THRESHOLD: f64 = 0.42; // made regarding how much non-conforming/offline validators the // algorithm can tolerate. const HEAVIEST_FORK_THRESHOLD_DELTA: f64 = 0.38; -// We update HeaviestFork every 5 minutes at least. -const HEAVIEST_REFRESH_INTERVAL_IN_SECONDS: u64 = 300; #[derive(Debug, PartialEq)] pub enum WenRestartError { @@ -203,6 +202,7 @@ pub(crate) enum WenRestartProgressInternalState { }, HeaviestFork { new_root_slot: Slot, + new_root_hash: Hash, }, GenerateSnapshot { new_root_slot: Slot, @@ -658,12 +658,15 @@ pub(crate) fn find_bankhash_of_heaviest_fork( Ok(parent_bank.hash()) } -pub(crate) fn send_restart_heaviest_fork( +// Aggregate the heaviest fork and send updates to the cluster. +pub(crate) fn aggregate_restart_heaviest_fork( + wen_restart_path: &PathBuf, cluster_info: Arc, + bank_forks: Arc>, exit: Arc, - only_loop_once: bool, progress: &mut WenRestartProgress, ) -> Result<()> { + let root_bank = bank_forks.read().unwrap().root_bank(); if progress.my_heaviest_fork.is_none() { return Err(WenRestartError::MalformedProgress( RestartState::HeaviestFork, @@ -674,17 +677,92 @@ pub(crate) fn send_restart_heaviest_fork( let my_heaviest_fork = progress.my_heaviest_fork.clone().unwrap(); let heaviest_fork_slot = my_heaviest_fork.slot; let heaviest_fork_hash = Hash::from_str(&my_heaviest_fork.bankhash)?; + // Use the epoch_stakes associated with the heaviest fork slot we picked. + let epoch_stakes = root_bank + .epoch_stakes(root_bank.epoch_schedule().get_epoch(heaviest_fork_slot)) + .unwrap(); + let total_stake = epoch_stakes.total_stake(); + let mut heaviest_fork_aggregate = HeaviestForkAggregate::new( + cluster_info.my_shred_version(), + epoch_stakes, + heaviest_fork_slot, + heaviest_fork_hash, + &cluster_info.id(), + ); + if let Some(aggregate_record) = &progress.heaviest_fork_aggregate { + for message in &aggregate_record.received { + if let Err(e) = heaviest_fork_aggregate.aggregate_from_record(message) { + // Do not abort wen_restart if we got one malformed message. + error!("Failed to aggregate from record: {:?}", e); + } + } + } else { + progress.heaviest_fork_aggregate = Some(HeaviestForkAggregateRecord { + received: Vec::new(), + total_active_stake: 0, + }); + } + + let mut cursor = solana_gossip::crds::Cursor::default(); + let mut total_active_stake = 0; loop { if exit.load(Ordering::Relaxed) { - break; + for ((slot, hash), stake) in heaviest_fork_aggregate.block_stake_map().iter() { + info!("Slot: {}, Hash: {}, Stake: {}", slot, hash, stake,); + } + return Ok(()); } - cluster_info.push_restart_heaviest_fork(heaviest_fork_slot, heaviest_fork_hash, 0); - if only_loop_once { - break; + let start = timestamp(); + for new_heaviest_fork in cluster_info.get_restart_heaviest_fork(&mut cursor) { + info!("Received new heaviest fork: {:?}", new_heaviest_fork); + let from = new_heaviest_fork.from.to_string(); + match heaviest_fork_aggregate.aggregate(new_heaviest_fork) { + HeaviestForkAggregateResult::Inserted(record) => { + info!("Successfully aggregated new heaviest fork: {:?}", record); + progress + .heaviest_fork_aggregate + .as_mut() + .unwrap() + .received + .push(record); + } + HeaviestForkAggregateResult::DifferentVersionExists(old_record, new_record) => { + warn!("Different version from {from} exists old {old_record:#?} vs new {new_record:#?}"); + progress.conflict_message.insert( + from, + ConflictMessage { + old_message: format!("{:?}", old_record), + new_message: format!("{:?}", new_record), + }, + ); + } + HeaviestForkAggregateResult::ZeroStakeIgnored => (), + HeaviestForkAggregateResult::AlreadyExists => (), + HeaviestForkAggregateResult::Malformed => (), + } + } + let current_total_active_stake = heaviest_fork_aggregate.total_active_stake(); + if current_total_active_stake > total_active_stake { + total_active_stake = current_total_active_stake; + progress + .heaviest_fork_aggregate + .as_mut() + .unwrap() + .total_active_stake = current_total_active_stake; + info!( + "Total active stake: {} Total stake {} Percentage {}%", + total_active_stake, + total_stake, + total_active_stake as f64 * 100.0 / total_stake as f64, + ); + write_wen_restart_records(wen_restart_path, progress)?; + } + let elapsed = timestamp().saturating_sub(start); + let time_left = GOSSIP_SLEEP_MILLIS.saturating_sub(elapsed); + if time_left > 0 { + sleep(Duration::from_millis(time_left)); } - sleep(Duration::from_secs(HEAVIEST_REFRESH_INTERVAL_IN_SECONDS)); } - Ok(()) } pub(crate) fn receive_restart_heaviest_fork( @@ -921,12 +999,19 @@ pub fn wait_for_wen_restart(config: WenRestartConfig) -> Result<()> { my_heaviest_fork: Some(heaviest_fork), } } - WenRestartProgressInternalState::HeaviestFork { new_root_slot } => { + WenRestartProgressInternalState::HeaviestFork { + new_root_slot, + new_root_hash, + } => { if config.cluster_info.id() == config.wen_restart_leader { - send_restart_heaviest_fork( + config + .cluster_info + .push_restart_heaviest_fork(new_root_slot, new_root_hash, 0); + aggregate_restart_heaviest_fork( + &config.wen_restart_path, config.cluster_info.clone(), + config.bank_forks.clone(), config.exit.clone(), - true, // only_loop_once &mut progress, )?; } else { @@ -936,7 +1021,7 @@ pub fn wait_for_wen_restart(config: WenRestartConfig) -> Result<()> { config.exit.clone(), &mut progress, )?; - verify_leader_heaviest_fork( + match verify_leader_heaviest_fork( new_root_slot, leader_slot, &leader_hash, @@ -944,9 +1029,26 @@ pub fn wait_for_wen_restart(config: WenRestartConfig) -> Result<()> { config.blockstore.clone(), config.exit.clone(), config.wen_restart_repair_slots.clone().unwrap(), - )?; + ) { + Ok(()) => config.cluster_info.push_restart_heaviest_fork( + leader_slot, + leader_hash, + 0, + ), + Err(e) => { + config.cluster_info.push_restart_heaviest_fork( + new_root_slot, + new_root_hash, + 0, + ); + return Err(e); + } + } + } + WenRestartProgressInternalState::HeaviestFork { + new_root_slot, + new_root_hash, } - WenRestartProgressInternalState::HeaviestFork { new_root_slot } } WenRestartProgressInternalState::GenerateSnapshot { new_root_slot, @@ -979,14 +1081,6 @@ pub fn wait_for_wen_restart(config: WenRestartConfig) -> Result<()> { --no-snapshot-fetch", slot, hash, shred_version, ); - if config.cluster_info.id() == config.wen_restart_leader { - send_restart_heaviest_fork( - config.cluster_info.clone(), - config.exit.clone(), - false, // only_loop_once - &mut progress, - )?; - } return Ok(()); } }; @@ -1054,12 +1148,16 @@ pub(crate) fn increment_and_write_wen_restart_records( progress.my_heaviest_fork = Some(my_heaviest_fork.clone()); WenRestartProgressInternalState::HeaviestFork { new_root_slot: my_heaviest_fork.slot, + new_root_hash: Hash::from_str(&my_heaviest_fork.bankhash)?, } } else { return Err(WenRestartError::UnexpectedState(RestartState::HeaviestFork).into()); } } - WenRestartProgressInternalState::HeaviestFork { new_root_slot } => { + WenRestartProgressInternalState::HeaviestFork { + new_root_slot, + new_root_hash: _, + } => { progress.set_state(RestartState::GenerateSnapshot); WenRestartProgressInternalState::GenerateSnapshot { new_root_slot, @@ -2683,7 +2781,10 @@ mod tests { from: my_pubkey.to_string(), }), }, - WenRestartProgressInternalState::HeaviestFork { new_root_slot: 1 }, + WenRestartProgressInternalState::HeaviestFork { + new_root_slot: 1, + new_root_hash: Hash::default(), + }, WenRestartProgress { state: RestartState::HeaviestFork.into(), my_last_voted_fork_slots: my_last_voted_fork_slots.clone(), @@ -2699,7 +2800,10 @@ mod tests { }, ), ( - WenRestartProgressInternalState::HeaviestFork { new_root_slot: 1 }, + WenRestartProgressInternalState::HeaviestFork { + new_root_slot: 1, + new_root_hash: Hash::default(), + }, WenRestartProgressInternalState::GenerateSnapshot { new_root_slot: 1, my_snapshot: None, From f719c5bfb37e5eaaf14a6cf40065bfecad238d63 Mon Sep 17 00:00:00 2001 From: Wen <113942165+wen-coding@users.noreply.github.com> Date: Mon, 16 Sep 2024 19:28:33 -0700 Subject: [PATCH 08/25] Move heaviest fork aggregation to DONE stage. --- wen-restart/proto/wen_restart.proto | 4 ++-- wen-restart/src/wen_restart.rs | 16 +++++++++------- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/wen-restart/proto/wen_restart.proto b/wen-restart/proto/wen_restart.proto index e99d20207ed770..108473aacb0db2 100644 --- a/wen-restart/proto/wen_restart.proto +++ b/wen-restart/proto/wen_restart.proto @@ -64,8 +64,8 @@ message WenRestartProgress { optional LastVotedForkSlotsRecord my_last_voted_fork_slots = 2; optional LastVotedForkSlotsAggregateRecord last_voted_fork_slots_aggregate = 3; optional HeaviestForkRecord my_heaviest_fork = 4; - optional HeaviestForkRecord leader_heaviest_fork = 5; - optional HeaviestForkAggregateRecord heaviest_fork_aggregate = 6; + optional HeaviestForkAggregateRecord heaviest_fork_aggregate = 5; + optional HeaviestForkRecord leader_heaviest_fork = 6; optional GenerateSnapshotRecord my_snapshot = 7; map conflict_message = 8; } diff --git a/wen-restart/src/wen_restart.rs b/wen-restart/src/wen_restart.rs index 9f88c2bd8d9fa5..dfb24ff23347a8 100644 --- a/wen-restart/src/wen_restart.rs +++ b/wen-restart/src/wen_restart.rs @@ -1007,13 +1007,6 @@ pub fn wait_for_wen_restart(config: WenRestartConfig) -> Result<()> { config .cluster_info .push_restart_heaviest_fork(new_root_slot, new_root_hash, 0); - aggregate_restart_heaviest_fork( - &config.wen_restart_path, - config.cluster_info.clone(), - config.bank_forks.clone(), - config.exit.clone(), - &mut progress, - )?; } else { let (leader_slot, leader_hash) = receive_restart_heaviest_fork( config.wen_restart_leader, @@ -1081,6 +1074,15 @@ pub fn wait_for_wen_restart(config: WenRestartConfig) -> Result<()> { --no-snapshot-fetch", slot, hash, shred_version, ); + if config.cluster_info.id() == config.wen_restart_leader { + aggregate_restart_heaviest_fork( + &config.wen_restart_path, + config.cluster_info.clone(), + config.bank_forks.clone(), + config.exit.clone(), + &mut progress, + )?; + } return Ok(()); } }; From 6f82a36a9b11dfb6f07d3dbac4ab7022b8e9cb5a Mon Sep 17 00:00:00 2001 From: Wen <113942165+wen-coding@users.noreply.github.com> Date: Tue, 17 Sep 2024 11:06:55 +0800 Subject: [PATCH 09/25] No warning when receiving HeaviestFork from non-leader. --- wen-restart/src/wen_restart.rs | 39 +++++++++++++++------------------- 1 file changed, 17 insertions(+), 22 deletions(-) diff --git a/wen-restart/src/wen_restart.rs b/wen-restart/src/wen_restart.rs index dfb24ff23347a8..08d95f425d08ba 100644 --- a/wen-restart/src/wen_restart.rs +++ b/wen-restart/src/wen_restart.rs @@ -750,7 +750,7 @@ pub(crate) fn aggregate_restart_heaviest_fork( .unwrap() .total_active_stake = current_total_active_stake; info!( - "Total active stake: {} Total stake {} Percentage {}%", + "Total active stake: {} Total stake {} Percentage {:.2}%", total_active_stake, total_stake, total_active_stake as f64 * 100.0 / total_stake as f64, @@ -777,34 +777,29 @@ pub(crate) fn receive_restart_heaviest_fork( return Err(WenRestartError::Exiting.into()); } for new_heaviest_fork in cluster_info.get_restart_heaviest_fork(&mut cursor) { - if new_heaviest_fork.from != wen_restart_leader { - warn!( - "Received heaviest fork from non leader: {}", - new_heaviest_fork.from + if new_heaviest_fork.from == wen_restart_leader { + info!( + "Received new heaviest fork from leader: {} {:?}", + wen_restart_leader, new_heaviest_fork ); - continue; + let leader_heaviest_slot = new_heaviest_fork.last_slot; + let leader_heaviest_hash = new_heaviest_fork.last_slot_hash; + progress.leader_heaviest_fork = Some(HeaviestForkRecord { + slot: leader_heaviest_slot, + bankhash: leader_heaviest_hash.to_string(), + total_active_stake: 0, + wallclock: new_heaviest_fork.wallclock, + shred_version: new_heaviest_fork.shred_version as u32, + from: new_heaviest_fork.from.to_string(), + }); + return Ok((leader_heaviest_slot, leader_heaviest_hash)); } - info!( - "Received new heaviest fork from leader: {} {:?}", - wen_restart_leader, new_heaviest_fork - ); - let leader_heaviest_slot = new_heaviest_fork.last_slot; - let leader_heaviest_hash = new_heaviest_fork.last_slot_hash; - progress.leader_heaviest_fork = Some(HeaviestForkRecord { - slot: leader_heaviest_slot, - bankhash: leader_heaviest_hash.to_string(), - total_active_stake: 0, - wallclock: new_heaviest_fork.wallclock, - shred_version: new_heaviest_fork.shred_version as u32, - from: new_heaviest_fork.from.to_string(), - }); - return Ok((leader_heaviest_slot, leader_heaviest_hash)); } sleep(Duration::from_millis(GOSSIP_SLEEP_MILLIS)); } } -fn repair_heaviest_fork( +pub(crate) fn repair_heaviest_fork( my_heaviest_fork_slot: Slot, heaviest_slot: Slot, exit: Arc, From 4fdc2285262a9da4bba0e4660684770f04c11a8a Mon Sep 17 00:00:00 2001 From: Wen <113942165+wen-coding@users.noreply.github.com> Date: Tue, 17 Sep 2024 12:22:41 +0800 Subject: [PATCH 10/25] Add test for receive_restart_heaviest_fork. --- wen-restart/src/wen_restart.rs | 57 ++++++++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) diff --git a/wen-restart/src/wen_restart.rs b/wen-restart/src/wen_restart.rs index 08d95f425d08ba..302576d59ef483 100644 --- a/wen-restart/src/wen_restart.rs +++ b/wen-restart/src/wen_restart.rs @@ -3274,4 +3274,61 @@ mod tests { .is_ok()); assert!(wait_for_wen_restart(config).is_ok()); } + + #[test] + fn test_receive_restart_heaviest_fork() { + let mut rng = rand::thread_rng(); + let leader_keypair = Keypair::new(); + let node_keypair = Arc::new(Keypair::new()); + let cluster_info = Arc::new(ClusterInfo::new( + { + let mut contact_info = + ContactInfo::new_localhost(&node_keypair.pubkey(), timestamp()); + contact_info.set_shred_version(SHRED_VERSION); + contact_info + }, + node_keypair.clone(), + SocketAddrSpace::Unspecified, + )); + let exit = Arc::new(AtomicBool::new(false)); + let random_keypair = Keypair::new(); + let random_node = ContactInfo::new_rand(&mut rng, Some(random_keypair.pubkey())); + let random_slot = 3; + let random_hash = Hash::new_unique(); + push_restart_heaviest_fork( + cluster_info.clone(), + &random_node, + random_slot, + &random_hash, + 0, + &random_keypair, + timestamp(), + ); + let leader_node = ContactInfo::new_rand(&mut rng, Some(leader_keypair.pubkey())); + let leader_slot = 6; + let leader_hash = Hash::new_unique(); + push_restart_heaviest_fork( + cluster_info.clone(), + &leader_node, + leader_slot, + &leader_hash, + 0, + &leader_keypair, + timestamp(), + ); + let mut progress = WenRestartProgress { + state: RestartState::HeaviestFork.into(), + ..Default::default() + }; + assert_eq!( + receive_restart_heaviest_fork( + leader_keypair.pubkey(), + cluster_info, + exit, + &mut progress + ) + .unwrap(), + (leader_slot, leader_hash) + ); + } } From f91580377d80282926ebb53810e6b8b267ac3ea1 Mon Sep 17 00:00:00 2001 From: Wen <113942165+wen-coding@users.noreply.github.com> Date: Tue, 17 Sep 2024 12:52:22 +0800 Subject: [PATCH 11/25] Add test for repair_heaviest_fork. --- wen-restart/src/wen_restart.rs | 55 ++++++++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/wen-restart/src/wen_restart.rs b/wen-restart/src/wen_restart.rs index 302576d59ef483..11212c0e20364f 100644 --- a/wen-restart/src/wen_restart.rs +++ b/wen-restart/src/wen_restart.rs @@ -3331,4 +3331,59 @@ mod tests { (leader_slot, leader_hash) ); } + + #[test] + fn test_repair_heaviest_fork() { + let ledger_path = get_tmp_ledger_path_auto_delete!(); + let my_heaviest_fork_slot = 1; + let leader_heaviest_slot_parent = 2; + let leader_heaviest_slot = 3; + let exit = Arc::new(AtomicBool::new(false)); + let blockstore = Arc::new(Blockstore::open(ledger_path.path()).unwrap()); + let wen_restart_repair_slots = Arc::new(RwLock::new(Vec::new())); + let exit_clone = exit.clone(); + let blockstore_clone = blockstore.clone(); + let wen_restart_repair_slots_clone = wen_restart_repair_slots.clone(); + let repair_heaviest_fork_thread_handle = Builder::new() + .name("solana-repair-heaviest-fork".to_string()) + .spawn(move || { + assert!(repair_heaviest_fork( + my_heaviest_fork_slot, + leader_heaviest_slot, + exit_clone, + blockstore_clone, + wen_restart_repair_slots_clone + ) + .is_ok()); + }) + .unwrap(); + sleep(Duration::from_millis(GOSSIP_SLEEP_MILLIS)); + // When there is nothing in blockstore, should repair the heaviest slot. + assert_eq!( + *wen_restart_repair_slots.read().unwrap(), + vec![leader_heaviest_slot] + ); + // Now add block 3, 3's parent is 2, should repair 2. + let _ = insert_slots_into_blockstore( + blockstore.clone(), + leader_heaviest_slot_parent, + &[leader_heaviest_slot], + TICKS_PER_SLOT, + Hash::default(), + ); + sleep(Duration::from_millis(GOSSIP_SLEEP_MILLIS)); + assert_eq!( + *wen_restart_repair_slots.read().unwrap(), + vec![leader_heaviest_slot_parent] + ); + // Insert 2 which links to 1, should exit now. + let _ = insert_slots_into_blockstore( + blockstore.clone(), + my_heaviest_fork_slot, + &[leader_heaviest_slot_parent], + TICKS_PER_SLOT, + Hash::default(), + ); + repair_heaviest_fork_thread_handle.join().unwrap(); + } } From 31395a79889abca0025fbe4a925cbb4f1b047a08 Mon Sep 17 00:00:00 2001 From: Wen <113942165+wen-coding@users.noreply.github.com> Date: Tue, 17 Sep 2024 13:31:45 +0800 Subject: [PATCH 12/25] Add test for verify_leader_heaviest_fork. --- wen-restart/src/wen_restart.rs | 72 ++++++++++++++++++++++++++++++++-- 1 file changed, 69 insertions(+), 3 deletions(-) diff --git a/wen-restart/src/wen_restart.rs b/wen-restart/src/wen_restart.rs index 11212c0e20364f..76cbb594273ffc 100644 --- a/wen-restart/src/wen_restart.rs +++ b/wen-restart/src/wen_restart.rs @@ -885,10 +885,13 @@ pub(crate) fn verify_leader_heaviest_fork( root_bank, &exit, )? - } else if let Some(bank) = bank_forks.read().unwrap().get(leader_heaviest_slot) { - bank.hash() } else { - return Err(WenRestartError::BlockNotFound(leader_heaviest_slot).into()); + bank_forks + .read() + .unwrap() + .get(leader_heaviest_slot) + .unwrap() + .hash() }; if my_bankhash != *leader_heaviest_hash { return Err(WenRestartError::BankHashMismatch( @@ -3386,4 +3389,67 @@ mod tests { ); repair_heaviest_fork_thread_handle.join().unwrap(); } + + #[test] + fn test_verify_leader_heaviest_fork() { + let ledger_path = get_tmp_ledger_path_auto_delete!(); + let test_state = wen_restart_test_init(&ledger_path); + let last_vote = test_state.last_voted_fork_slots[0]; + let exit = Arc::new(AtomicBool::new(false)); + // Create two forks: last_vote -> last_vote+1 and last_vote -> last_vote+2 + let root_bank; + { + root_bank = test_state.bank_forks.read().unwrap().root_bank().clone(); + } + let leader_slot = last_vote + 1; + let my_slot = last_vote + 2; + let _ = insert_slots_into_blockstore( + test_state.blockstore.clone(), + last_vote, + &[leader_slot], + TICKS_PER_SLOT, + test_state.last_blockhash, + ); + let _ = insert_slots_into_blockstore( + test_state.blockstore.clone(), + last_vote, + &[my_slot], + TICKS_PER_SLOT, + test_state.last_blockhash, + ); + let wen_restart_repair_slots = Arc::new(RwLock::new(Vec::new())); + assert_eq!( + verify_leader_heaviest_fork( + my_slot, + leader_slot, + &Hash::default(), + test_state.bank_forks.clone(), + test_state.blockstore.clone(), + exit.clone(), + wen_restart_repair_slots.clone() + ) + .unwrap_err() + .downcast::() + .unwrap(), + WenRestartError::HeaviestForkOnLeaderOnDifferentFork(leader_slot, my_slot) + ); + let leader_hash = Hash::new_unique(); + let my_hash = root_bank.hash(); + let root_slot = root_bank.slot(); + assert_eq!( + verify_leader_heaviest_fork( + root_slot, + root_slot, + &leader_hash, + test_state.bank_forks.clone(), + test_state.blockstore.clone(), + exit.clone(), + wen_restart_repair_slots.clone() + ) + .unwrap_err() + .downcast::() + .unwrap(), + WenRestartError::BankHashMismatch(root_slot, my_hash, leader_hash) + ); + } } From 9147dc956820198613fdeb21da663295aa211c57 Mon Sep 17 00:00:00 2001 From: Wen <113942165+wen-coding@users.noreply.github.com> Date: Fri, 20 Sep 2024 23:21:22 +0800 Subject: [PATCH 13/25] Rename wen_restart_leader to wen_restart_coordinator. --- core/src/validator.rs | 6 +- local-cluster/src/validator_configs.rs | 2 +- multinode-demo/bootstrap-validator.sh | 2 +- multinode-demo/validator.sh | 2 +- net/net.sh | 2 +- net/remote/remote-node.sh | 2 +- validator/src/cli.rs | 6 +- validator/src/main.rs | 2 +- wen-restart/proto/wen_restart.proto | 2 +- wen-restart/src/wen_restart.rs | 178 +++++++++++++------------ 10 files changed, 105 insertions(+), 99 deletions(-) diff --git a/core/src/validator.rs b/core/src/validator.rs index f689dcc00aac27..b6c9298388f2e3 100644 --- a/core/src/validator.rs +++ b/core/src/validator.rs @@ -276,7 +276,7 @@ pub struct ValidatorConfig { pub generator_config: Option, pub use_snapshot_archives_at_startup: UseSnapshotArchivesAtStartup, pub wen_restart_proto_path: Option, - pub wen_restart_leader: Option, + pub wen_restart_coordinator: Option, pub unified_scheduler_handler_threads: Option, pub ip_echo_server_threads: NonZeroUsize, pub replay_forks_threads: NonZeroUsize, @@ -349,7 +349,7 @@ impl Default for ValidatorConfig { generator_config: None, use_snapshot_archives_at_startup: UseSnapshotArchivesAtStartup::default(), wen_restart_proto_path: None, - wen_restart_leader: None, + wen_restart_coordinator: None, unified_scheduler_handler_threads: None, ip_echo_server_threads: NonZeroUsize::new(1).expect("1 is non-zero"), replay_forks_threads: NonZeroUsize::new(1).expect("1 is non-zero"), @@ -1382,7 +1382,7 @@ impl Validator { info!("Waiting for wen_restart to finish"); wait_for_wen_restart(WenRestartConfig { wen_restart_path: config.wen_restart_proto_path.clone().unwrap(), - wen_restart_leader: config.wen_restart_leader.unwrap(), + wen_restart_coordinator: config.wen_restart_coordinator.unwrap(), last_vote, blockstore: blockstore.clone(), cluster_info: cluster_info.clone(), diff --git a/local-cluster/src/validator_configs.rs b/local-cluster/src/validator_configs.rs index cb33f44c274a58..bd4ec4c10596c5 100644 --- a/local-cluster/src/validator_configs.rs +++ b/local-cluster/src/validator_configs.rs @@ -68,7 +68,7 @@ pub fn safe_clone_config(config: &ValidatorConfig) -> ValidatorConfig { generator_config: config.generator_config.clone(), use_snapshot_archives_at_startup: config.use_snapshot_archives_at_startup, wen_restart_proto_path: config.wen_restart_proto_path.clone(), - wen_restart_leader: config.wen_restart_leader, + wen_restart_coordinator: config.wen_restart_coordinator, unified_scheduler_handler_threads: config.unified_scheduler_handler_threads, ip_echo_server_threads: config.ip_echo_server_threads, replay_forks_threads: config.replay_forks_threads, diff --git a/multinode-demo/bootstrap-validator.sh b/multinode-demo/bootstrap-validator.sh index 3a0a6920a166bf..d21ee1aaa8b73f 100755 --- a/multinode-demo/bootstrap-validator.sh +++ b/multinode-demo/bootstrap-validator.sh @@ -115,7 +115,7 @@ while [[ -n $1 ]]; do elif [[ $1 == --wen-restart ]]; then args+=("$1" "$2") shift 2 - elif [[ $1 == --wen-restart-leader ]]; then + elif [[ $1 == --wen-restart-coordinator ]]; then args+=("$1" "$2") shift 2 else diff --git a/multinode-demo/validator.sh b/multinode-demo/validator.sh index c1d445911a77b8..c97812c6cbb910 100755 --- a/multinode-demo/validator.sh +++ b/multinode-demo/validator.sh @@ -185,7 +185,7 @@ while [[ -n $1 ]]; do elif [[ $1 == --wen-restart ]]; then args+=("$1" "$2") shift 2 - elif [[ $1 == --wen-restart-leader ]]; then + elif [[ $1 == --wen-restart-coordinator ]]; then args+=("$1" "$2") shift 2 elif [[ $1 = -h ]]; then diff --git a/net/net.sh b/net/net.sh index d9f1faea2a75e8..3ef7430ebd54d6 100755 --- a/net/net.sh +++ b/net/net.sh @@ -146,7 +146,7 @@ Operate a configured testnet -i [ip address] - IP Address of the node to start or stop startnode specific option: - --wen-restart [leader_pubkey] - Use given leader pubkey and apply wen_restat + --wen-restart [coordinator_pubkey] - Use given coordinator pubkey and apply wen_restat startclients-specific options: $CLIENT_OPTIONS diff --git a/net/remote/remote-node.sh b/net/remote/remote-node.sh index 9e3f57c5bc8f41..edd21ba73145b4 100755 --- a/net/remote/remote-node.sh +++ b/net/remote/remote-node.sh @@ -437,7 +437,7 @@ EOF if [[ -n "$maybeWenRestart" ]]; then args+=(--wen-restart wen_restart.proto3) - args+=(--wen-restart-leader "$maybeWenRestart") + args+=(--wen-restart-coordinator "$maybeWenRestart") fi cat >> ~/solana/on-reboot <(version: &'a str, default_args: &'a DefaultArgs) -> App<'a, 'a> { .takes_value(true) .required(false) .conflicts_with("wait_for_supermajority") - .requires("wen_restart_leader") + .requires("wen_restart_coordinator") .help( "Only used during coordinated cluster restarts.\ \n\n\ @@ -1608,8 +1608,8 @@ pub fn app<'a>(version: &'a str, default_args: &'a DefaultArgs) -> App<'a, 'a> { ), ) .arg( - Arg::with_name("wen_restart_leader") - .long("wen-restart-leader") + Arg::with_name("wen_restart_coordinator") + .long("wen-restart-coordinator") .hidden(hidden_unless_forced()) .value_name("PUBKEY") .takes_value(true) diff --git a/validator/src/main.rs b/validator/src/main.rs index 49b803d38529b2..579a4f08bfe30c 100644 --- a/validator/src/main.rs +++ b/validator/src/main.rs @@ -1536,7 +1536,7 @@ pub fn main() { delay_leader_block_for_pending_fork: matches .is_present("delay_leader_block_for_pending_fork"), wen_restart_proto_path: value_t!(matches, "wen_restart", PathBuf).ok(), - wen_restart_leader: value_t!(matches, "wen_restart_leader", Pubkey).ok(), + wen_restart_coordinator: value_t!(matches, "wen_restart_coordinator", Pubkey).ok(), ..ValidatorConfig::default() }; diff --git a/wen-restart/proto/wen_restart.proto b/wen-restart/proto/wen_restart.proto index 108473aacb0db2..e88eecd88485d8 100644 --- a/wen-restart/proto/wen_restart.proto +++ b/wen-restart/proto/wen_restart.proto @@ -65,7 +65,7 @@ message WenRestartProgress { optional LastVotedForkSlotsAggregateRecord last_voted_fork_slots_aggregate = 3; optional HeaviestForkRecord my_heaviest_fork = 4; optional HeaviestForkAggregateRecord heaviest_fork_aggregate = 5; - optional HeaviestForkRecord leader_heaviest_fork = 6; + optional HeaviestForkRecord coordinator_heaviest_fork = 6; optional GenerateSnapshotRecord my_snapshot = 7; map conflict_message = 8; } diff --git a/wen-restart/src/wen_restart.rs b/wen-restart/src/wen_restart.rs index 76cbb594273ffc..654df6ad9490ef 100644 --- a/wen-restart/src/wen_restart.rs +++ b/wen-restart/src/wen_restart.rs @@ -144,12 +144,12 @@ impl std::fmt::Display for WenRestartError { ) } WenRestartError::HeaviestForkOnLeaderOnDifferentFork( - leader_heaviest_slot, + coordinator_heaviest_slot, should_include_slot, ) => { write!( f, - "Heaviest fork on leader on different fork: heaviest: {leader_heaviest_slot} does not include: {should_include_slot}", + "Heaviest fork on coordinator on different fork: heaviest: {coordinator_heaviest_slot} does not include: {should_include_slot}", ) } WenRestartError::MalformedLastVotedForkSlotsProtobuf(record) => { @@ -766,7 +766,7 @@ pub(crate) fn aggregate_restart_heaviest_fork( } pub(crate) fn receive_restart_heaviest_fork( - wen_restart_leader: Pubkey, + wen_restart_coordinator: Pubkey, cluster_info: Arc, exit: Arc, progress: &mut WenRestartProgress, @@ -777,22 +777,22 @@ pub(crate) fn receive_restart_heaviest_fork( return Err(WenRestartError::Exiting.into()); } for new_heaviest_fork in cluster_info.get_restart_heaviest_fork(&mut cursor) { - if new_heaviest_fork.from == wen_restart_leader { + if new_heaviest_fork.from == wen_restart_coordinator { info!( - "Received new heaviest fork from leader: {} {:?}", - wen_restart_leader, new_heaviest_fork + "Received new heaviest fork from coordinator: {} {:?}", + wen_restart_coordinator, new_heaviest_fork ); - let leader_heaviest_slot = new_heaviest_fork.last_slot; - let leader_heaviest_hash = new_heaviest_fork.last_slot_hash; - progress.leader_heaviest_fork = Some(HeaviestForkRecord { - slot: leader_heaviest_slot, - bankhash: leader_heaviest_hash.to_string(), + let coordinator_heaviest_slot = new_heaviest_fork.last_slot; + let coordinator_heaviest_hash = new_heaviest_fork.last_slot_hash; + progress.coordinator_heaviest_fork = Some(HeaviestForkRecord { + slot: coordinator_heaviest_slot, + bankhash: coordinator_heaviest_hash.to_string(), total_active_stake: 0, wallclock: new_heaviest_fork.wallclock, shred_version: new_heaviest_fork.shred_version as u32, from: new_heaviest_fork.from.to_string(), }); - return Ok((leader_heaviest_slot, leader_heaviest_hash)); + return Ok((coordinator_heaviest_slot, coordinator_heaviest_hash)); } } sleep(Duration::from_millis(GOSSIP_SLEEP_MILLIS)); @@ -827,10 +827,10 @@ pub(crate) fn repair_heaviest_fork( } } -pub(crate) fn verify_leader_heaviest_fork( +pub(crate) fn verify_coordinator_heaviest_fork( my_heaviest_fork_slot: Slot, - leader_heaviest_slot: Slot, - leader_heaviest_hash: &Hash, + coordinator_heaviest_slot: Slot, + coordinator_heaviest_hash: &Hash, bank_forks: Arc>, blockstore: Arc, exit: Arc, @@ -838,7 +838,7 @@ pub(crate) fn verify_leader_heaviest_fork( ) -> Result<()> { repair_heaviest_fork( my_heaviest_fork_slot, - leader_heaviest_slot, + coordinator_heaviest_slot, exit.clone(), blockstore.clone(), wen_restart_repair_slots.clone(), @@ -848,37 +848,39 @@ pub(crate) fn verify_leader_heaviest_fork( root_bank = bank_forks.read().unwrap().root_bank(); } let root_slot = root_bank.slot(); - let mut slots: Vec = AncestorIterator::new_inclusive(leader_heaviest_slot, &blockstore) - .take_while(|slot| slot >= &root_slot) - .collect(); + let mut slots: Vec = + AncestorIterator::new_inclusive(coordinator_heaviest_slot, &blockstore) + .take_while(|slot| slot >= &root_slot) + .collect(); slots.sort(); if !slots.contains(&root_slot) { return Err(WenRestartError::HeaviestForkOnLeaderOnDifferentFork( - leader_heaviest_slot, + coordinator_heaviest_slot, root_slot, ) .into()); } - if leader_heaviest_slot > my_heaviest_fork_slot && !slots.contains(&my_heaviest_fork_slot) { + if coordinator_heaviest_slot > my_heaviest_fork_slot && !slots.contains(&my_heaviest_fork_slot) + { return Err(WenRestartError::HeaviestForkOnLeaderOnDifferentFork( - leader_heaviest_slot, + coordinator_heaviest_slot, my_heaviest_fork_slot, ) .into()); } - if leader_heaviest_slot < my_heaviest_fork_slot + if coordinator_heaviest_slot < my_heaviest_fork_slot && !AncestorIterator::new(my_heaviest_fork_slot, &blockstore) - .any(|slot| slot == leader_heaviest_slot) + .any(|slot| slot == coordinator_heaviest_slot) { return Err(WenRestartError::HeaviestForkOnLeaderOnDifferentFork( - leader_heaviest_slot, + coordinator_heaviest_slot, my_heaviest_fork_slot, ) .into()); } let my_bankhash = if !slots.is_empty() { find_bankhash_of_heaviest_fork( - leader_heaviest_slot, + coordinator_heaviest_slot, slots, blockstore.clone(), bank_forks.clone(), @@ -889,15 +891,15 @@ pub(crate) fn verify_leader_heaviest_fork( bank_forks .read() .unwrap() - .get(leader_heaviest_slot) + .get(coordinator_heaviest_slot) .unwrap() .hash() }; - if my_bankhash != *leader_heaviest_hash { + if my_bankhash != *coordinator_heaviest_hash { return Err(WenRestartError::BankHashMismatch( - leader_heaviest_slot, + coordinator_heaviest_slot, my_bankhash, - *leader_heaviest_hash, + *coordinator_heaviest_hash, ) .into()); } @@ -907,7 +909,7 @@ pub(crate) fn verify_leader_heaviest_fork( #[derive(Clone)] pub struct WenRestartConfig { pub wen_restart_path: PathBuf, - pub wen_restart_leader: Pubkey, + pub wen_restart_coordinator: Pubkey, pub last_vote: VoteTransaction, pub blockstore: Arc, pub cluster_info: Arc, @@ -1001,29 +1003,29 @@ pub fn wait_for_wen_restart(config: WenRestartConfig) -> Result<()> { new_root_slot, new_root_hash, } => { - if config.cluster_info.id() == config.wen_restart_leader { + if config.cluster_info.id() == config.wen_restart_coordinator { config .cluster_info .push_restart_heaviest_fork(new_root_slot, new_root_hash, 0); } else { - let (leader_slot, leader_hash) = receive_restart_heaviest_fork( - config.wen_restart_leader, + let (coordinator_slot, coordinator_hash) = receive_restart_heaviest_fork( + config.wen_restart_coordinator, config.cluster_info.clone(), config.exit.clone(), &mut progress, )?; - match verify_leader_heaviest_fork( + match verify_coordinator_heaviest_fork( new_root_slot, - leader_slot, - &leader_hash, + coordinator_slot, + &coordinator_hash, config.bank_forks.clone(), config.blockstore.clone(), config.exit.clone(), config.wen_restart_repair_slots.clone().unwrap(), ) { Ok(()) => config.cluster_info.push_restart_heaviest_fork( - leader_slot, - leader_hash, + coordinator_slot, + coordinator_hash, 0, ), Err(e) => { @@ -1072,7 +1074,7 @@ pub fn wait_for_wen_restart(config: WenRestartConfig) -> Result<()> { --no-snapshot-fetch", slot, hash, shred_version, ); - if config.cluster_info.id() == config.wen_restart_leader { + if config.cluster_info.id() == config.wen_restart_coordinator { aggregate_restart_heaviest_fork( &config.wen_restart_path, config.cluster_info.clone(), @@ -1486,7 +1488,7 @@ mod tests { pub bank_forks: Arc>, pub last_voted_fork_slots: Vec, pub wen_restart_proto_path: PathBuf, - pub wen_restart_leader: Pubkey, + pub wen_restart_coordinator: Pubkey, pub last_blockhash: Hash, pub genesis_config_hash: Hash, } @@ -1576,7 +1578,7 @@ mod tests { let mut wen_restart_proto_path = ledger_path.path().to_path_buf(); wen_restart_proto_path.push("wen_restart_status.proto"); let _ = remove_file(&wen_restart_proto_path); - let wen_restart_leader = validator_voting_keypairs[LEADER_INDEX] + let wen_restart_coordinator = validator_voting_keypairs[LEADER_INDEX] .node_keypair .pubkey(); WenRestartTestInitResult { @@ -1586,7 +1588,7 @@ mod tests { bank_forks, last_voted_fork_slots, wen_restart_proto_path, - wen_restart_leader, + wen_restart_coordinator, last_blockhash, genesis_config_hash: genesis_config.hash(), } @@ -1643,7 +1645,7 @@ mod tests { let last_vote_slot: Slot = test_state.last_voted_fork_slots[0]; let wen_restart_config = WenRestartConfig { wen_restart_path: test_state.wen_restart_proto_path.clone(), - wen_restart_leader: test_state.wen_restart_leader, + wen_restart_coordinator: test_state.wen_restart_coordinator, last_vote: VoteTransaction::from(Vote::new(vec![last_vote_slot], last_vote_bankhash)), blockstore: test_state.blockstore.clone(), cluster_info: test_state.cluster_info.clone(), @@ -1711,7 +1713,7 @@ mod tests { let exit = Arc::new(AtomicBool::new(false)); let wen_restart_config = WenRestartConfig { wen_restart_path: test_state.wen_restart_proto_path.clone(), - wen_restart_leader: test_state.wen_restart_leader, + wen_restart_coordinator: test_state.wen_restart_coordinator, last_vote: VoteTransaction::from(Vote::new(vec![last_vote_slot], last_vote_bankhash)), blockstore: test_state.blockstore.clone(), cluster_info: test_state.cluster_info.clone(), @@ -1846,7 +1848,7 @@ mod tests { let my_pubkey = test_state.validator_voting_keypairs[MY_INDEX] .node_keypair .pubkey(); - let leader_pubkey = test_state.validator_voting_keypairs[LEADER_INDEX] + let coordinator_pubkey = test_state.validator_voting_keypairs[LEADER_INDEX] .node_keypair .pubkey(); assert_eq!( @@ -1887,13 +1889,17 @@ mod tests { wallclock: 0, from: my_pubkey.to_string(), }), - leader_heaviest_fork: Some(HeaviestForkRecord { + coordinator_heaviest_fork: Some(HeaviestForkRecord { slot: expected_heaviest_fork_slot, bankhash: expected_heaviest_fork_bankhash.to_string(), total_active_stake: 0, shred_version: SHRED_VERSION as u32, - wallclock: progress.leader_heaviest_fork.as_ref().unwrap().wallclock, - from: leader_pubkey.to_string(), + wallclock: progress + .coordinator_heaviest_fork + .as_ref() + .unwrap() + .wallclock, + from: coordinator_pubkey.to_string(), }), my_snapshot: Some(GenerateSnapshotRecord { slot: expected_heaviest_fork_slot, @@ -2063,7 +2069,7 @@ mod tests { assert_eq!( wait_for_wen_restart(WenRestartConfig { wen_restart_path: test_state.wen_restart_proto_path, - wen_restart_leader: test_state.wen_restart_leader, + wen_restart_coordinator: test_state.wen_restart_coordinator, last_vote: VoteTransaction::from(Vote::new( vec![new_root_slot], last_vote_bankhash @@ -2683,14 +2689,14 @@ mod tests { wallclock: 0, from: my_pubkey.to_string(), }); - let leader_pubkey = Pubkey::new_unique(); - let leader_heaviest_fork = Some(HeaviestForkRecord { + let coordinator_pubkey = Pubkey::new_unique(); + let coordinator_heaviest_fork = Some(HeaviestForkRecord { slot: 2, bankhash: Hash::default().to_string(), total_active_stake: 800, shred_version: SHRED_VERSION as u32, wallclock: 0, - from: leader_pubkey.to_string(), + from: coordinator_pubkey.to_string(), }); let my_bankhash = Hash::new_unique(); let new_shred_version = SHRED_VERSION + 57; @@ -2813,7 +2819,7 @@ mod tests { my_last_voted_fork_slots: my_last_voted_fork_slots.clone(), last_voted_fork_slots_aggregate: last_voted_fork_slots_aggregate.clone(), my_heaviest_fork: my_heaviest_fork.clone(), - leader_heaviest_fork: leader_heaviest_fork.clone(), + coordinator_heaviest_fork: coordinator_heaviest_fork.clone(), ..Default::default() }, WenRestartProgress { @@ -2821,7 +2827,7 @@ mod tests { my_last_voted_fork_slots: my_last_voted_fork_slots.clone(), last_voted_fork_slots_aggregate: last_voted_fork_slots_aggregate.clone(), my_heaviest_fork: my_heaviest_fork.clone(), - leader_heaviest_fork: leader_heaviest_fork.clone(), + coordinator_heaviest_fork: coordinator_heaviest_fork.clone(), ..Default::default() }, ), @@ -2840,7 +2846,7 @@ mod tests { my_last_voted_fork_slots: my_last_voted_fork_slots.clone(), last_voted_fork_slots_aggregate: last_voted_fork_slots_aggregate.clone(), my_heaviest_fork: my_heaviest_fork.clone(), - leader_heaviest_fork: leader_heaviest_fork.clone(), + coordinator_heaviest_fork: coordinator_heaviest_fork.clone(), ..Default::default() }, WenRestartProgress { @@ -2848,7 +2854,7 @@ mod tests { my_last_voted_fork_slots: my_last_voted_fork_slots.clone(), last_voted_fork_slots_aggregate: last_voted_fork_slots_aggregate.clone(), my_heaviest_fork: my_heaviest_fork.clone(), - leader_heaviest_fork, + coordinator_heaviest_fork, my_snapshot: my_snapshot.clone(), ..Default::default() }, @@ -3234,7 +3240,7 @@ mod tests { let last_vote_bankhash = Hash::new_unique(); let config = WenRestartConfig { wen_restart_path: test_state.wen_restart_proto_path.clone(), - wen_restart_leader: test_state.wen_restart_leader, + wen_restart_coordinator: test_state.wen_restart_coordinator, last_vote: VoteTransaction::from(Vote::new(vec![last_vote_slot], last_vote_bankhash)), blockstore: test_state.blockstore.clone(), cluster_info: test_state.cluster_info.clone(), @@ -3281,7 +3287,7 @@ mod tests { #[test] fn test_receive_restart_heaviest_fork() { let mut rng = rand::thread_rng(); - let leader_keypair = Keypair::new(); + let coordinator_keypair = Keypair::new(); let node_keypair = Arc::new(Keypair::new()); let cluster_info = Arc::new(ClusterInfo::new( { @@ -3307,16 +3313,16 @@ mod tests { &random_keypair, timestamp(), ); - let leader_node = ContactInfo::new_rand(&mut rng, Some(leader_keypair.pubkey())); - let leader_slot = 6; - let leader_hash = Hash::new_unique(); + let coordinator_node = ContactInfo::new_rand(&mut rng, Some(coordinator_keypair.pubkey())); + let coordinator_slot = 6; + let coordinator_hash = Hash::new_unique(); push_restart_heaviest_fork( cluster_info.clone(), - &leader_node, - leader_slot, - &leader_hash, + &coordinator_node, + coordinator_slot, + &coordinator_hash, 0, - &leader_keypair, + &coordinator_keypair, timestamp(), ); let mut progress = WenRestartProgress { @@ -3325,13 +3331,13 @@ mod tests { }; assert_eq!( receive_restart_heaviest_fork( - leader_keypair.pubkey(), + coordinator_keypair.pubkey(), cluster_info, exit, &mut progress ) .unwrap(), - (leader_slot, leader_hash) + (coordinator_slot, coordinator_hash) ); } @@ -3339,8 +3345,8 @@ mod tests { fn test_repair_heaviest_fork() { let ledger_path = get_tmp_ledger_path_auto_delete!(); let my_heaviest_fork_slot = 1; - let leader_heaviest_slot_parent = 2; - let leader_heaviest_slot = 3; + let coordinator_heaviest_slot_parent = 2; + let coordinator_heaviest_slot = 3; let exit = Arc::new(AtomicBool::new(false)); let blockstore = Arc::new(Blockstore::open(ledger_path.path()).unwrap()); let wen_restart_repair_slots = Arc::new(RwLock::new(Vec::new())); @@ -3352,7 +3358,7 @@ mod tests { .spawn(move || { assert!(repair_heaviest_fork( my_heaviest_fork_slot, - leader_heaviest_slot, + coordinator_heaviest_slot, exit_clone, blockstore_clone, wen_restart_repair_slots_clone @@ -3364,26 +3370,26 @@ mod tests { // When there is nothing in blockstore, should repair the heaviest slot. assert_eq!( *wen_restart_repair_slots.read().unwrap(), - vec![leader_heaviest_slot] + vec![coordinator_heaviest_slot] ); // Now add block 3, 3's parent is 2, should repair 2. let _ = insert_slots_into_blockstore( blockstore.clone(), - leader_heaviest_slot_parent, - &[leader_heaviest_slot], + coordinator_heaviest_slot_parent, + &[coordinator_heaviest_slot], TICKS_PER_SLOT, Hash::default(), ); sleep(Duration::from_millis(GOSSIP_SLEEP_MILLIS)); assert_eq!( *wen_restart_repair_slots.read().unwrap(), - vec![leader_heaviest_slot_parent] + vec![coordinator_heaviest_slot_parent] ); // Insert 2 which links to 1, should exit now. let _ = insert_slots_into_blockstore( blockstore.clone(), my_heaviest_fork_slot, - &[leader_heaviest_slot_parent], + &[coordinator_heaviest_slot_parent], TICKS_PER_SLOT, Hash::default(), ); @@ -3391,7 +3397,7 @@ mod tests { } #[test] - fn test_verify_leader_heaviest_fork() { + fn test_verify_coordinator_heaviest_fork() { let ledger_path = get_tmp_ledger_path_auto_delete!(); let test_state = wen_restart_test_init(&ledger_path); let last_vote = test_state.last_voted_fork_slots[0]; @@ -3401,12 +3407,12 @@ mod tests { { root_bank = test_state.bank_forks.read().unwrap().root_bank().clone(); } - let leader_slot = last_vote + 1; + let coordinator_slot = last_vote + 1; let my_slot = last_vote + 2; let _ = insert_slots_into_blockstore( test_state.blockstore.clone(), last_vote, - &[leader_slot], + &[coordinator_slot], TICKS_PER_SLOT, test_state.last_blockhash, ); @@ -3419,9 +3425,9 @@ mod tests { ); let wen_restart_repair_slots = Arc::new(RwLock::new(Vec::new())); assert_eq!( - verify_leader_heaviest_fork( + verify_coordinator_heaviest_fork( my_slot, - leader_slot, + coordinator_slot, &Hash::default(), test_state.bank_forks.clone(), test_state.blockstore.clone(), @@ -3431,16 +3437,16 @@ mod tests { .unwrap_err() .downcast::() .unwrap(), - WenRestartError::HeaviestForkOnLeaderOnDifferentFork(leader_slot, my_slot) + WenRestartError::HeaviestForkOnLeaderOnDifferentFork(coordinator_slot, my_slot) ); - let leader_hash = Hash::new_unique(); + let coordinator_hash = Hash::new_unique(); let my_hash = root_bank.hash(); let root_slot = root_bank.slot(); assert_eq!( - verify_leader_heaviest_fork( + verify_coordinator_heaviest_fork( root_slot, root_slot, - &leader_hash, + &coordinator_hash, test_state.bank_forks.clone(), test_state.blockstore.clone(), exit.clone(), @@ -3449,7 +3455,7 @@ mod tests { .unwrap_err() .downcast::() .unwrap(), - WenRestartError::BankHashMismatch(root_slot, my_hash, leader_hash) + WenRestartError::BankHashMismatch(root_slot, my_hash, coordinator_hash) ); } } From 3142433ec8c4a2dbc0a92477887c3d07ce822afe Mon Sep 17 00:00:00 2001 From: Wen <113942165+wen-coding@users.noreply.github.com> Date: Tue, 8 Oct 2024 14:07:28 -0700 Subject: [PATCH 14/25] Fix a bad merge --- wen-restart/src/wen_restart.rs | 3 --- 1 file changed, 3 deletions(-) diff --git a/wen-restart/src/wen_restart.rs b/wen-restart/src/wen_restart.rs index f818b234c208ba..9675b3a2a740e7 100644 --- a/wen-restart/src/wen_restart.rs +++ b/wen-restart/src/wen_restart.rs @@ -1581,9 +1581,6 @@ mod tests { let mut wen_restart_proto_path = ledger_path.path().to_path_buf(); wen_restart_proto_path.push("wen_restart_status.proto"); let _ = remove_file(&wen_restart_proto_path); - let wen_restart_coordinator = validator_voting_keypairs[COORDINATOR_INDEX] - .node_keypair - .pubkey(); WenRestartTestInitResult { validator_voting_keypairs, blockstore, From 8dfa0a3a15d41690f72ed10c4a66d079e28b3eb0 Mon Sep 17 00:00:00 2001 From: Wen <113942165+wen-coding@users.noreply.github.com> Date: Wed, 9 Oct 2024 11:20:43 -0700 Subject: [PATCH 15/25] Non-coordinator should use coodinator's (slot, hash) after verification. --- wen-restart/src/wen_restart.rs | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/wen-restart/src/wen_restart.rs b/wen-restart/src/wen_restart.rs index 9675b3a2a740e7..02d1bff5d63eb6 100644 --- a/wen-restart/src/wen_restart.rs +++ b/wen-restart/src/wen_restart.rs @@ -1007,6 +1007,10 @@ pub fn wait_for_wen_restart(config: WenRestartConfig) -> Result<()> { config .cluster_info .push_restart_heaviest_fork(new_root_slot, new_root_hash, 0); + WenRestartProgressInternalState::HeaviestFork { + new_root_slot, + new_root_hash, + } } else { let (coordinator_slot, coordinator_hash) = receive_restart_heaviest_fork( config.wen_restart_coordinator, @@ -1037,10 +1041,10 @@ pub fn wait_for_wen_restart(config: WenRestartConfig) -> Result<()> { return Err(e); } } - } - WenRestartProgressInternalState::HeaviestFork { - new_root_slot, - new_root_hash, + WenRestartProgressInternalState::HeaviestFork { + new_root_slot: coordinator_slot, + new_root_hash: coordinator_hash, + } } } WenRestartProgressInternalState::GenerateSnapshot { From 85ed8d1f1b90bbde966fa04abe8a3dc48d67a5bb Mon Sep 17 00:00:00 2001 From: Wen <113942165+wen-coding@users.noreply.github.com> Date: Wed, 9 Oct 2024 13:08:41 -0700 Subject: [PATCH 16/25] Remove trailing whitespace. --- wen-restart/src/wen_restart.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/wen-restart/src/wen_restart.rs b/wen-restart/src/wen_restart.rs index 02d1bff5d63eb6..b0ac2cf4cd59fb 100644 --- a/wen-restart/src/wen_restart.rs +++ b/wen-restart/src/wen_restart.rs @@ -1010,7 +1010,7 @@ pub fn wait_for_wen_restart(config: WenRestartConfig) -> Result<()> { WenRestartProgressInternalState::HeaviestFork { new_root_slot, new_root_hash, - } + } } else { let (coordinator_slot, coordinator_hash) = receive_restart_heaviest_fork( config.wen_restart_coordinator, @@ -1044,7 +1044,7 @@ pub fn wait_for_wen_restart(config: WenRestartConfig) -> Result<()> { WenRestartProgressInternalState::HeaviestFork { new_root_slot: coordinator_slot, new_root_hash: coordinator_hash, - } + } } } WenRestartProgressInternalState::GenerateSnapshot { From ba540c0f9e5edae5ebb593bafabf8dfc696f1bb4 Mon Sep 17 00:00:00 2001 From: Wen <113942165+wen-coding@users.noreply.github.com> Date: Wed, 16 Oct 2024 19:50:56 -0700 Subject: [PATCH 17/25] Fix a bad merge --- wen-restart/src/wen_restart.rs | 46 +--------------------------------- 1 file changed, 1 insertion(+), 45 deletions(-) diff --git a/wen-restart/src/wen_restart.rs b/wen-restart/src/wen_restart.rs index bf81c0c0bf6a9b..c87c52a328f24e 100644 --- a/wen-restart/src/wen_restart.rs +++ b/wen-restart/src/wen_restart.rs @@ -765,40 +765,6 @@ pub(crate) fn aggregate_restart_heaviest_fork( } } -pub(crate) fn receive_restart_heaviest_fork( - wen_restart_coordinator: Pubkey, - cluster_info: Arc, - exit: Arc, - progress: &mut WenRestartProgress, -) -> Result<(Slot, Hash)> { - let mut cursor = solana_gossip::crds::Cursor::default(); - loop { - if exit.load(Ordering::Relaxed) { - return Err(WenRestartError::Exiting.into()); - } - for new_heaviest_fork in cluster_info.get_restart_heaviest_fork(&mut cursor) { - if new_heaviest_fork.from == wen_restart_coordinator { - info!( - "Received new heaviest fork from coordinator: {} {:?}", - wen_restart_coordinator, new_heaviest_fork - ); - let coordinator_heaviest_slot = new_heaviest_fork.last_slot; - let coordinator_heaviest_hash = new_heaviest_fork.last_slot_hash; - progress.coordinator_heaviest_fork = Some(HeaviestForkRecord { - slot: coordinator_heaviest_slot, - bankhash: coordinator_heaviest_hash.to_string(), - total_active_stake: 0, - wallclock: new_heaviest_fork.wallclock, - shred_version: new_heaviest_fork.shred_version as u32, - from: new_heaviest_fork.from.to_string(), - }); - return Ok((coordinator_heaviest_slot, coordinator_heaviest_hash)); - } - } - sleep(Duration::from_millis(GOSSIP_SLEEP_MILLIS)); - } -} - pub(crate) fn repair_heaviest_fork( my_heaviest_fork_slot: Slot, heaviest_slot: Slot, @@ -931,6 +897,7 @@ pub(crate) fn receive_restart_heaviest_fork( total_active_stake: 0, wallclock: new_heaviest_fork.wallclock, shred_version: new_heaviest_fork.shred_version as u32, + from: new_heaviest_fork.from.to_string(), }); return Ok((coordinator_heaviest_slot, coordinator_heaviest_hash)); } @@ -1941,17 +1908,6 @@ mod tests { shred_version: progress.my_snapshot.as_ref().unwrap().shred_version, path: progress.my_snapshot.as_ref().unwrap().path.clone(), }), - coordinator_heaviest_fork: Some(HeaviestForkRecord { - slot: expected_heaviest_fork_slot, - bankhash: expected_heaviest_fork_bankhash.to_string(), - total_active_stake: 0, - shred_version: SHRED_VERSION as u32, - wallclock: progress - .coordinator_heaviest_fork - .as_ref() - .unwrap() - .wallclock, - }), ..Default::default() } ); From 21f77b64705977c0bb5a09ab21885cc294184b2e Mon Sep 17 00:00:00 2001 From: Wen <113942165+wen-coding@users.noreply.github.com> Date: Thu, 17 Oct 2024 17:01:09 -0700 Subject: [PATCH 18/25] Fix a bad merge. --- wen-restart/src/wen_restart.rs | 4 ---- 1 file changed, 4 deletions(-) diff --git a/wen-restart/src/wen_restart.rs b/wen-restart/src/wen_restart.rs index 96b39477049277..b44a2a3eab6c7c 100644 --- a/wen-restart/src/wen_restart.rs +++ b/wen-restart/src/wen_restart.rs @@ -2690,10 +2690,6 @@ mod tests { path: "snapshot_1".to_string(), shred_version: new_shred_version as u32, }); - let heaviest_fork_aggregate = Some(HeaviestForkAggregateRecord { - received: Vec::new(), - total_active_stake: 0, - }); let expected_slots_stake_map: HashMap = vec![(0, 900), (1, 800)].into_iter().collect(); for (entrance_state, exit_state, entrance_progress, exit_progress) in [ From cda571e919838d58ee1c43f825cb70f71afb3603 Mon Sep 17 00:00:00 2001 From: Wen <113942165+wen-coding@users.noreply.github.com> Date: Thu, 17 Oct 2024 19:07:06 -0700 Subject: [PATCH 19/25] Remove unnecessary changes. --- wen-restart/src/wen_restart.rs | 26 +++++++++----------------- 1 file changed, 9 insertions(+), 17 deletions(-) diff --git a/wen-restart/src/wen_restart.rs b/wen-restart/src/wen_restart.rs index b44a2a3eab6c7c..1dea0bfbcb89cc 100644 --- a/wen-restart/src/wen_restart.rs +++ b/wen-restart/src/wen_restart.rs @@ -570,15 +570,13 @@ pub(crate) fn find_bankhash_of_heaviest_fork( root_bank: Arc, exit: &AtomicBool, ) -> Result { + if let Some(hash) = bank_forks + .read() + .unwrap() + .get(heaviest_fork_slot) + .map(|bank| bank.hash()) { - if let Some(hash) = bank_forks - .read() - .unwrap() - .get(heaviest_fork_slot) - .map(|bank| bank.hash()) - { - return Ok(hash); - } + return Ok(hash); } let leader_schedule_cache = LeaderScheduleCache::new_from_bank(&root_bank); let replay_tx_thread_pool = rayon::ThreadPoolBuilder::new() @@ -648,11 +646,6 @@ pub(crate) fn find_bankhash_of_heaviest_fork( cur_bank } }; - info!( - "wen_restart find bankhash slot: {} hash: {}", - slot, - bank.hash() - ); parent_bank = bank; } Ok(parent_bank.hash()) @@ -1153,7 +1146,7 @@ pub(crate) fn increment_and_write_wen_restart_records( progress.my_heaviest_fork = Some(my_heaviest_fork.clone()); WenRestartProgressInternalState::HeaviestFork { new_root_slot: my_heaviest_fork.slot, - new_root_hash: Hash::from_str(&my_heaviest_fork.bankhash)?, + new_root_hash: Hash::from_str(&my_heaviest_fork.bankhash).unwrap(), } } else { return Err(WenRestartError::UnexpectedState(RestartState::HeaviestFork).into()); @@ -2673,14 +2666,13 @@ mod tests { wallclock: 0, from: my_pubkey.to_string(), }); - let coordinator_pubkey = Pubkey::new_unique(); let coordinator_heaviest_fork = Some(HeaviestForkRecord { slot: 2, - bankhash: Hash::default().to_string(), + bankhash: Hash::new_unique().to_string(), total_active_stake: 800, shred_version: SHRED_VERSION as u32, wallclock: 0, - from: coordinator_pubkey.to_string(), + from: Pubkey::new_unique().to_string(), }); let my_bankhash = Hash::new_unique(); let new_shred_version = SHRED_VERSION + 57; From 0ec3fc6c1c7bd5db874eebef3749bf3a4d0b20f9 Mon Sep 17 00:00:00 2001 From: Wen <113942165+wen-coding@users.noreply.github.com> Date: Thu, 17 Oct 2024 19:32:28 -0700 Subject: [PATCH 20/25] Make coordinator select different slot in test, wait before exiting with error. --- wen-restart/src/wen_restart.rs | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/wen-restart/src/wen_restart.rs b/wen-restart/src/wen_restart.rs index 1dea0bfbcb89cc..ec708ffff657bc 100644 --- a/wen-restart/src/wen_restart.rs +++ b/wen-restart/src/wen_restart.rs @@ -1025,11 +1025,14 @@ pub fn wait_for_wen_restart(config: WenRestartConfig) -> Result<()> { 0, ), Err(e) => { + warn!("Failed to verify coordinator heaviest fork: {:?}, exit in 10 seconds", e); config.cluster_info.push_restart_heaviest_fork( new_root_slot, new_root_hash, 0, ); + // Wait for 10 seconds so the heaviest fork gets out. + sleep(Duration::from_secs(10)); return Err(e); } } @@ -1773,21 +1776,23 @@ mod tests { test_state.last_blockhash, ); - let expected_heaviest_fork_slot = last_vote_slot + 2; - let expected_heaviest_fork_bankhash; + let my_heaviest_fork_slot = last_vote_slot + 2; + let my_heaviest_fork_bankhash; loop { if let Some(bank) = test_state .bank_forks .read() .unwrap() - .get(expected_heaviest_fork_slot) + .get(my_heaviest_fork_slot) { - expected_heaviest_fork_bankhash = bank.hash(); + my_heaviest_fork_bankhash = bank.hash(); break; } sleep(Duration::from_millis(100)); } // Now simulate receiving HeaviestFork messages from coordinator. + let coordinator_heaviest_fork_slot = my_heaviest_fork_slot - 1; + let coordinator_heaviest_fork_bankhash = test_state.bank_forks.read().unwrap().get(coordinator_heaviest_fork_slot).unwrap().hash(); let coordinator_keypair = &test_state.validator_voting_keypairs[COORDINATOR_INDEX].node_keypair; let node = ContactInfo::new_rand(&mut rng, Some(coordinator_keypair.pubkey())); @@ -1795,8 +1800,8 @@ mod tests { push_restart_heaviest_fork( test_state.cluster_info.clone(), &node, - expected_heaviest_fork_slot, - &expected_heaviest_fork_bankhash, + coordinator_heaviest_fork_slot, + &coordinator_heaviest_fork_bankhash, 0, coordinator_keypair, now, @@ -1858,8 +1863,8 @@ mod tests { }), }), my_heaviest_fork: Some(HeaviestForkRecord { - slot: expected_heaviest_fork_slot, - bankhash: expected_heaviest_fork_bankhash.to_string(), + slot: my_heaviest_fork_slot, + bankhash: my_heaviest_fork_bankhash.to_string(), total_active_stake: 0, shred_version: SHRED_VERSION as u32, wallclock: 0, @@ -1867,14 +1872,14 @@ mod tests { }), heaviest_fork_aggregate: None, my_snapshot: Some(GenerateSnapshotRecord { - slot: expected_heaviest_fork_slot, + slot: coordinator_heaviest_fork_slot, bankhash: progress.my_snapshot.as_ref().unwrap().bankhash.clone(), shred_version: progress.my_snapshot.as_ref().unwrap().shred_version, path: progress.my_snapshot.as_ref().unwrap().path.clone(), }), coordinator_heaviest_fork: Some(HeaviestForkRecord { - slot: expected_heaviest_fork_slot, - bankhash: expected_heaviest_fork_bankhash.to_string(), + slot: coordinator_heaviest_fork_slot, + bankhash: coordinator_heaviest_fork_bankhash.to_string(), total_active_stake: 0, shred_version: SHRED_VERSION as u32, wallclock: progress From e18e8f752d74507855e0a50c167487c9e726f80e Mon Sep 17 00:00:00 2001 From: Wen <113942165+wen-coding@users.noreply.github.com> Date: Thu, 17 Oct 2024 22:10:46 -0700 Subject: [PATCH 21/25] Add send_and_receive_heaviest_fork and test. --- wen-restart/src/wen_restart.rs | 203 +++++++++++++++++++++++++-------- 1 file changed, 156 insertions(+), 47 deletions(-) diff --git a/wen-restart/src/wen_restart.rs b/wen-restart/src/wen_restart.rs index ec708ffff657bc..2a0c33a7eec203 100644 --- a/wen-restart/src/wen_restart.rs +++ b/wen-restart/src/wen_restart.rs @@ -898,6 +898,45 @@ pub(crate) fn receive_restart_heaviest_fork( } } +pub(crate) fn send_and_receive_heaviest_fork( + new_root_slot: Slot, + new_root_hash: Hash, + config: &WenRestartConfig, + progress: &mut WenRestartProgress, + pushfn: impl FnOnce(Slot, Hash) -> (), +) -> Result<(Slot, Hash)> { + if config.cluster_info.id() == config.wen_restart_coordinator { + pushfn(new_root_slot, new_root_hash); + Ok((new_root_slot, new_root_hash)) + } else { + let (coordinator_slot, coordinator_hash) = receive_restart_heaviest_fork( + config.wen_restart_coordinator, + config.cluster_info.clone(), + config.exit.clone(), + progress, + )?; + match verify_coordinator_heaviest_fork( + new_root_slot, + coordinator_slot, + &coordinator_hash, + config.bank_forks.clone(), + config.blockstore.clone(), + config.exit.clone(), + config.wen_restart_repair_slots.clone().unwrap(), + ) { + Ok(()) => pushfn(coordinator_slot, coordinator_hash), + Err(e) => { + warn!("Failed to verify coordinator heaviest fork: {:?}, exit in 10 seconds", e); + pushfn(new_root_slot, new_root_hash); + // Wait for 10 seconds so the heaviest fork gets out. + sleep(Duration::from_secs(10)); + return Err(e); + } + } + Ok((coordinator_slot, coordinator_hash)) + } +} + #[derive(Clone)] pub struct WenRestartConfig { pub wen_restart_path: PathBuf, @@ -995,52 +1034,19 @@ pub fn wait_for_wen_restart(config: WenRestartConfig) -> Result<()> { new_root_slot, new_root_hash, } => { - if config.cluster_info.id() == config.wen_restart_coordinator { - config - .cluster_info - .push_restart_heaviest_fork(new_root_slot, new_root_hash, 0); - WenRestartProgressInternalState::HeaviestFork { - new_root_slot, - new_root_hash, - } - } else { - let (coordinator_slot, coordinator_hash) = receive_restart_heaviest_fork( - config.wen_restart_coordinator, - config.cluster_info.clone(), - config.exit.clone(), - &mut progress, - )?; - match verify_coordinator_heaviest_fork( - new_root_slot, - coordinator_slot, - &coordinator_hash, - config.bank_forks.clone(), - config.blockstore.clone(), - config.exit.clone(), - config.wen_restart_repair_slots.clone().unwrap(), - ) { - Ok(()) => config.cluster_info.push_restart_heaviest_fork( - coordinator_slot, - coordinator_hash, - 0, - ), - Err(e) => { - warn!("Failed to verify coordinator heaviest fork: {:?}, exit in 10 seconds", e); - config.cluster_info.push_restart_heaviest_fork( - new_root_slot, - new_root_hash, - 0, - ); - // Wait for 10 seconds so the heaviest fork gets out. - sleep(Duration::from_secs(10)); - return Err(e); - } - } - WenRestartProgressInternalState::HeaviestFork { - new_root_slot: coordinator_slot, - new_root_hash: coordinator_hash, - } - } + let (slot, hash) = send_and_receive_heaviest_fork( + new_root_slot, + new_root_hash, + &config, + &mut progress, + |slot, hash| { + config.cluster_info.push_restart_heaviest_fork(slot, hash, 0); + }, + )?; + WenRestartProgressInternalState::HeaviestFork { + new_root_slot: slot, + new_root_hash: hash, + } } WenRestartProgressInternalState::GenerateSnapshot { new_root_slot, @@ -1792,7 +1798,13 @@ mod tests { } // Now simulate receiving HeaviestFork messages from coordinator. let coordinator_heaviest_fork_slot = my_heaviest_fork_slot - 1; - let coordinator_heaviest_fork_bankhash = test_state.bank_forks.read().unwrap().get(coordinator_heaviest_fork_slot).unwrap().hash(); + let coordinator_heaviest_fork_bankhash = test_state + .bank_forks + .read() + .unwrap() + .get(coordinator_heaviest_fork_slot) + .unwrap() + .hash(); let coordinator_keypair = &test_state.validator_voting_keypairs[COORDINATOR_INDEX].node_keypair; let node = ContactInfo::new_rand(&mut rng, Some(coordinator_keypair.pubkey())); @@ -3527,4 +3539,101 @@ mod tests { WenRestartError::BankHashMismatch(root_slot, my_hash, coordinator_hash) ); } + + #[test] + fn test_send_and_receive_heaviest_fork() { + let ledger_path = get_tmp_ledger_path_auto_delete!(); + let test_state = wen_restart_test_init(&ledger_path); + let last_vote = test_state.last_voted_fork_slots[0]; + let exit = Arc::new(AtomicBool::new(false)); + let mut pushed_slot = 0; + let mut pushed_hash = Hash::default(); + // The coordinator always sends its own choice. + let coordinator_slot = last_vote; + let old_root_bank = test_state.bank_forks.read().unwrap().root_bank(); + let mut slots = test_state.last_voted_fork_slots.clone(); + slots.reverse(); + let coordinator_hash = find_bankhash_of_heaviest_fork( + coordinator_slot, slots, test_state.blockstore.clone(), test_state.bank_forks.clone(), + old_root_bank, &exit).unwrap(); + let mut progress = WenRestartProgress { + state: RestartState::HeaviestFork.into(), + ..Default::default() + }; + // Set coordinator to myself, should return my choice. + let mut config = WenRestartConfig { + wen_restart_path: test_state.wen_restart_proto_path.clone(), + wen_restart_coordinator: test_state.cluster_info.id(), + last_vote: VoteTransaction::from(Vote::new(vec![last_vote], Hash::default())), + blockstore: test_state.blockstore.clone(), + cluster_info: test_state.cluster_info.clone(), + bank_forks: test_state.bank_forks.clone(), + wen_restart_repair_slots: Some(Arc::new(RwLock::new(Vec::new()))), + wait_for_supermajority_threshold_percent: 80, + snapshot_config: SnapshotConfig::default(), + accounts_background_request_sender: AbsRequestSender::default(), + genesis_config_hash: test_state.genesis_config_hash, + exit: exit.clone(), + }; + assert_eq!( + send_and_receive_heaviest_fork(coordinator_slot, coordinator_hash, &config, &mut progress, |slot, hash| { + pushed_slot = slot; + pushed_hash = hash; + }) + .unwrap(), + (coordinator_slot, coordinator_hash) + ); + assert_eq!(pushed_slot, coordinator_slot); + assert_eq!(pushed_hash, coordinator_hash); + // Now set the coordinator the someone else, need to return their choice. + let coordinator_keypair = + &test_state.validator_voting_keypairs[COORDINATOR_INDEX].node_keypair; + config.wen_restart_coordinator = coordinator_keypair.pubkey(); + let mut rng = rand::thread_rng(); + let node = ContactInfo::new_rand(&mut rng, Some(coordinator_keypair.pubkey())); + let now = timestamp(); + push_restart_heaviest_fork( + test_state.cluster_info.clone(), + &node, + coordinator_slot, + &coordinator_hash, + 0, + coordinator_keypair, + now, + ); + let my_slot = test_state.last_voted_fork_slots[1]; + let my_hash = test_state.bank_forks.read().unwrap().get(my_slot).unwrap().hash(); + assert_eq!( + send_and_receive_heaviest_fork(my_slot, my_hash, &config, &mut progress, |slot, hash| { + pushed_slot = slot; + pushed_hash = hash; + }) + .unwrap(), + (coordinator_slot, coordinator_hash) + ); + assert_eq!(pushed_slot, coordinator_slot); + assert_eq!(pushed_hash, coordinator_hash); + // my slot on a different fork, should exit with error but still push heaviest fork. + let my_slot = coordinator_slot + 1; + let _ = insert_slots_into_blockstore( + test_state.blockstore.clone(), + 0, + &[coordinator_slot], + TICKS_PER_SLOT, + test_state.last_blockhash, + ); + let my_hash = Hash::new_unique(); + assert_eq!( + send_and_receive_heaviest_fork(my_slot, my_hash, &config, &mut progress, |slot, hash| { + pushed_slot = slot; + pushed_hash = hash; + }) + .unwrap_err() + .downcast::() + .unwrap(), + WenRestartError::HeaviestForkOnLeaderOnDifferentFork(coordinator_slot, my_slot) + ); + assert_eq!(pushed_slot, my_slot); + assert_eq!(pushed_hash, my_hash); + } } From 677228791ce83f741aa19e402c1186a34b1a5b50 Mon Sep 17 00:00:00 2001 From: Wen <113942165+wen-coding@users.noreply.github.com> Date: Thu, 17 Oct 2024 22:25:28 -0700 Subject: [PATCH 22/25] Make the coordinator print stats every 10 seconds. --- wen-restart/src/heaviest_fork_aggregate.rs | 6 +- wen-restart/src/wen_restart.rs | 89 ++++++++++++++++------ 2 files changed, 69 insertions(+), 26 deletions(-) diff --git a/wen-restart/src/heaviest_fork_aggregate.rs b/wen-restart/src/heaviest_fork_aggregate.rs index 63bca761a93b87..710a6010292daf 100644 --- a/wen-restart/src/heaviest_fork_aggregate.rs +++ b/wen-restart/src/heaviest_fork_aggregate.rs @@ -161,8 +161,10 @@ impl HeaviestForkAggregate { }) } - pub(crate) fn block_stake_map(self) -> HashMap<(Slot, Hash), u64> { - self.block_stake_map + pub(crate) fn print_block_stake_map(&self) { + for ((slot, hash), stake) in self.block_stake_map.iter() { + info!("Slot: {}, Hash: {}, Stake: {}", slot, hash, stake,); + } } } diff --git a/wen-restart/src/wen_restart.rs b/wen-restart/src/wen_restart.rs index 2a0c33a7eec203..2d7b4fe74acf4f 100644 --- a/wen-restart/src/wen_restart.rs +++ b/wen-restart/src/wen_restart.rs @@ -58,7 +58,7 @@ use { Arc, RwLock, }, thread::sleep, - time::Duration, + time::{Duration, Instant}, }, }; @@ -70,6 +70,8 @@ const REPAIR_THRESHOLD: f64 = 0.42; // made regarding how much non-conforming/offline validators the // algorithm can tolerate. const HEAVIEST_FORK_THRESHOLD_DELTA: f64 = 0.38; +// The coordinator print new stats every 10 seconds. +const COORDINATOR_STAT_PRINT_INTERVAL_SECONDS: u64 = 10; #[derive(Debug, PartialEq)] pub enum WenRestartError { @@ -651,7 +653,7 @@ pub(crate) fn find_bankhash_of_heaviest_fork( Ok(parent_bank.hash()) } -// Aggregate the heaviest fork and send updates to the cluster. +// Aggregate the heaviest fork at the coordinator. pub(crate) fn aggregate_restart_heaviest_fork( wen_restart_path: &PathBuf, cluster_info: Arc, @@ -698,11 +700,9 @@ pub(crate) fn aggregate_restart_heaviest_fork( let mut cursor = solana_gossip::crds::Cursor::default(); let mut total_active_stake = 0; + let mut stat_printed_at = Instant::now(); loop { if exit.load(Ordering::Relaxed) { - for ((slot, hash), stake) in heaviest_fork_aggregate.block_stake_map().iter() { - info!("Slot: {}, Hash: {}, Stake: {}", slot, hash, stake,); - } return Ok(()); } let start = timestamp(); @@ -754,6 +754,12 @@ pub(crate) fn aggregate_restart_heaviest_fork( if time_left > 0 { sleep(Duration::from_millis(time_left)); } + // Print the block stake map after a while. + if stat_printed_at.elapsed() > Duration::from_secs(COORDINATOR_STAT_PRINT_INTERVAL_SECONDS) + { + heaviest_fork_aggregate.print_block_stake_map(); + stat_printed_at = Instant::now(); + } } } @@ -903,7 +909,7 @@ pub(crate) fn send_and_receive_heaviest_fork( new_root_hash: Hash, config: &WenRestartConfig, progress: &mut WenRestartProgress, - pushfn: impl FnOnce(Slot, Hash) -> (), + pushfn: impl FnOnce(Slot, Hash), ) -> Result<(Slot, Hash)> { if config.cluster_info.id() == config.wen_restart_coordinator { pushfn(new_root_slot, new_root_hash); @@ -926,7 +932,10 @@ pub(crate) fn send_and_receive_heaviest_fork( ) { Ok(()) => pushfn(coordinator_slot, coordinator_hash), Err(e) => { - warn!("Failed to verify coordinator heaviest fork: {:?}, exit in 10 seconds", e); + warn!( + "Failed to verify coordinator heaviest fork: {:?}, exit in 10 seconds", + e + ); pushfn(new_root_slot, new_root_hash); // Wait for 10 seconds so the heaviest fork gets out. sleep(Duration::from_secs(10)); @@ -1040,13 +1049,15 @@ pub fn wait_for_wen_restart(config: WenRestartConfig) -> Result<()> { &config, &mut progress, |slot, hash| { - config.cluster_info.push_restart_heaviest_fork(slot, hash, 0); + config + .cluster_info + .push_restart_heaviest_fork(slot, hash, 0); }, )?; WenRestartProgressInternalState::HeaviestFork { new_root_slot: slot, new_root_hash: hash, - } + } } WenRestartProgressInternalState::GenerateSnapshot { new_root_slot, @@ -3554,8 +3565,14 @@ mod tests { let mut slots = test_state.last_voted_fork_slots.clone(); slots.reverse(); let coordinator_hash = find_bankhash_of_heaviest_fork( - coordinator_slot, slots, test_state.blockstore.clone(), test_state.bank_forks.clone(), - old_root_bank, &exit).unwrap(); + coordinator_slot, + slots, + test_state.blockstore.clone(), + test_state.bank_forks.clone(), + old_root_bank, + &exit, + ) + .unwrap(); let mut progress = WenRestartProgress { state: RestartState::HeaviestFork.into(), ..Default::default() @@ -3576,10 +3593,16 @@ mod tests { exit: exit.clone(), }; assert_eq!( - send_and_receive_heaviest_fork(coordinator_slot, coordinator_hash, &config, &mut progress, |slot, hash| { - pushed_slot = slot; - pushed_hash = hash; - }) + send_and_receive_heaviest_fork( + coordinator_slot, + coordinator_hash, + &config, + &mut progress, + |slot, hash| { + pushed_slot = slot; + pushed_hash = hash; + } + ) .unwrap(), (coordinator_slot, coordinator_hash) ); @@ -3602,12 +3625,24 @@ mod tests { now, ); let my_slot = test_state.last_voted_fork_slots[1]; - let my_hash = test_state.bank_forks.read().unwrap().get(my_slot).unwrap().hash(); + let my_hash = test_state + .bank_forks + .read() + .unwrap() + .get(my_slot) + .unwrap() + .hash(); assert_eq!( - send_and_receive_heaviest_fork(my_slot, my_hash, &config, &mut progress, |slot, hash| { - pushed_slot = slot; - pushed_hash = hash; - }) + send_and_receive_heaviest_fork( + my_slot, + my_hash, + &config, + &mut progress, + |slot, hash| { + pushed_slot = slot; + pushed_hash = hash; + } + ) .unwrap(), (coordinator_slot, coordinator_hash) ); @@ -3624,10 +3659,16 @@ mod tests { ); let my_hash = Hash::new_unique(); assert_eq!( - send_and_receive_heaviest_fork(my_slot, my_hash, &config, &mut progress, |slot, hash| { - pushed_slot = slot; - pushed_hash = hash; - }) + send_and_receive_heaviest_fork( + my_slot, + my_hash, + &config, + &mut progress, + |slot, hash| { + pushed_slot = slot; + pushed_hash = hash; + } + ) .unwrap_err() .downcast::() .unwrap(), From a34c297d15f341f92c22ea10fbae067dc7840f00 Mon Sep 17 00:00:00 2001 From: Wen <113942165+wen-coding@users.noreply.github.com> Date: Sat, 19 Oct 2024 15:02:06 -0700 Subject: [PATCH 23/25] Rename variables and add comments. --- wen-restart/src/wen_restart.rs | 137 ++++++++++++++++++--------------- 1 file changed, 74 insertions(+), 63 deletions(-) diff --git a/wen-restart/src/wen_restart.rs b/wen-restart/src/wen_restart.rs index 2d7b4fe74acf4f..0fff527e85e42c 100644 --- a/wen-restart/src/wen_restart.rs +++ b/wen-restart/src/wen_restart.rs @@ -203,11 +203,11 @@ pub(crate) enum WenRestartProgressInternalState { my_heaviest_fork: Option, }, HeaviestFork { - new_root_slot: Slot, - new_root_hash: Hash, + final_restart_slot: Slot, + final_restart_hash: Hash, }, GenerateSnapshot { - new_root_slot: Slot, + final_restart_slot: Slot, my_snapshot: Option, }, Done { @@ -466,13 +466,13 @@ fn check_slot_smaller_than_intended_snapshot_slot( // We don't use set_root() explicitly, because it may kick off snapshot requests, we // can't have multiple snapshot requests in progress. In bank_to_snapshot_archive() // everything set_root() does will be done (without bank_forks setting root). So -// when we restart from the snapshot bank on new_root_slot will become root. +// when we restart from the snapshot bank on final_restart_slot will become root. pub(crate) fn generate_snapshot( bank_forks: Arc>, snapshot_config: &SnapshotConfig, accounts_background_request_sender: &AbsRequestSender, genesis_config_hash: Hash, - new_root_slot: Slot, + final_restart_slot: Slot, ) -> Result { let new_root_bank; { @@ -481,16 +481,16 @@ pub(crate) fn generate_snapshot( if !old_root_bank .hard_forks() .iter() - .any(|(slot, _)| slot == &new_root_slot) + .any(|(slot, _)| slot == &final_restart_slot) { - old_root_bank.register_hard_fork(new_root_slot); + old_root_bank.register_hard_fork(final_restart_slot); } - // new_root_slot is guaranteed to have a bank in bank_forks, it's checked in + // final_restart_slot is guaranteed to have a bank in bank_forks, it's checked in // find_bankhash_of_heaviest_fork(). - match my_bank_forks.get(new_root_slot) { + match my_bank_forks.get(final_restart_slot) { Some(bank) => new_root_bank = bank.clone(), None => { - return Err(WenRestartError::BlockNotFound(new_root_slot).into()); + return Err(WenRestartError::BlockNotFound(final_restart_slot).into()); } } let mut banks = vec![&new_root_bank]; @@ -498,7 +498,7 @@ pub(crate) fn generate_snapshot( banks.extend(parents.iter()); let _ = my_bank_forks.send_eah_request_if_needed( - new_root_slot, + final_restart_slot, &banks, accounts_background_request_sender, )?; @@ -523,20 +523,24 @@ pub(crate) fn generate_snapshot( }; // In very rare cases it's possible that the local root is not on the heaviest fork, so the // validator generated snapshot for slots > local root. If the cluster agreed upon restart - // slot new_root_slot is less than the the current highest full_snapshot_slot, that means the + // slot final_restart_slot is less than the the current highest full_snapshot_slot, that means the // locally rooted full_snapshot_slot will be rolled back. this requires human inspection。 // // In even rarer cases, the selected slot might be the latest full snapshot slot. We could // just re-generate a new snapshot to make sure the snapshot is up to date after hard fork, // but for now we just return an error to keep the code simple. - check_slot_smaller_than_intended_snapshot_slot(full_snapshot_slot, new_root_slot, directory)?; + check_slot_smaller_than_intended_snapshot_slot( + full_snapshot_slot, + final_restart_slot, + directory, + )?; directory = &snapshot_config.incremental_snapshot_archives_dir; if let Some(incremental_snapshot_slot) = get_highest_incremental_snapshot_archive_slot(directory, full_snapshot_slot) { check_slot_smaller_than_intended_snapshot_slot( incremental_snapshot_slot, - new_root_slot, + final_restart_slot, directory, )?; } @@ -553,11 +557,11 @@ pub(crate) fn generate_snapshot( compute_shred_version(&genesis_config_hash, Some(&new_root_bank.hard_forks())); let new_snapshot_path = archive_info.path().display().to_string(); info!("wen_restart incremental snapshot generated on {new_snapshot_path} base slot {full_snapshot_slot}"); - // We might have bank snapshots past the new_root_slot, we need to purge them. + // We might have bank snapshots past the final_restart_slot, we need to purge them. purge_all_bank_snapshots(&snapshot_config.bank_snapshots_dir); Ok(GenerateSnapshotRecord { path: new_snapshot_path, - slot: new_root_slot, + slot: final_restart_slot, bankhash: new_root_bank.hash().to_string(), shred_version: new_shred_version as u32, }) @@ -774,6 +778,8 @@ pub(crate) fn repair_heaviest_fork( if exit.load(Ordering::Relaxed) { return Err(WenRestartError::Exiting.into()); } + // Repair all ancestors of heaviest_slot (including itself) which are larger than + // my_heaviest_fork_slot. let to_repair = if blockstore.meta(heaviest_slot).is_ok_and(|x| x.is_some()) { AncestorIterator::new_inclusive(heaviest_slot, &blockstore) .take_while(|slot| *slot > my_heaviest_fork_slot) @@ -812,19 +818,20 @@ pub(crate) fn verify_coordinator_heaviest_fork( root_bank = bank_forks.read().unwrap().root_bank(); } let root_slot = root_bank.slot(); - let mut slots: Vec = + let mut coordinator_heaviest_slot_ancestors: Vec = AncestorIterator::new_inclusive(coordinator_heaviest_slot, &blockstore) .take_while(|slot| slot >= &root_slot) .collect(); - slots.sort(); - if !slots.contains(&root_slot) { + coordinator_heaviest_slot_ancestors.sort(); + if !coordinator_heaviest_slot_ancestors.contains(&root_slot) { return Err(WenRestartError::HeaviestForkOnLeaderOnDifferentFork( coordinator_heaviest_slot, root_slot, ) .into()); } - if coordinator_heaviest_slot > my_heaviest_fork_slot && !slots.contains(&my_heaviest_fork_slot) + if coordinator_heaviest_slot > my_heaviest_fork_slot + && !coordinator_heaviest_slot_ancestors.contains(&my_heaviest_fork_slot) { return Err(WenRestartError::HeaviestForkOnLeaderOnDifferentFork( coordinator_heaviest_slot, @@ -842,10 +849,10 @@ pub(crate) fn verify_coordinator_heaviest_fork( ) .into()); } - let my_bankhash = if !slots.is_empty() { + let my_bankhash = if !coordinator_heaviest_slot_ancestors.is_empty() { find_bankhash_of_heaviest_fork( coordinator_heaviest_slot, - slots, + coordinator_heaviest_slot_ancestors, blockstore.clone(), bank_forks.clone(), root_bank, @@ -905,15 +912,15 @@ pub(crate) fn receive_restart_heaviest_fork( } pub(crate) fn send_and_receive_heaviest_fork( - new_root_slot: Slot, - new_root_hash: Hash, + final_restart_slot: Slot, + final_restart_hash: Hash, config: &WenRestartConfig, progress: &mut WenRestartProgress, pushfn: impl FnOnce(Slot, Hash), ) -> Result<(Slot, Hash)> { if config.cluster_info.id() == config.wen_restart_coordinator { - pushfn(new_root_slot, new_root_hash); - Ok((new_root_slot, new_root_hash)) + pushfn(final_restart_slot, final_restart_hash); + Ok((final_restart_slot, final_restart_hash)) } else { let (coordinator_slot, coordinator_hash) = receive_restart_heaviest_fork( config.wen_restart_coordinator, @@ -922,7 +929,7 @@ pub(crate) fn send_and_receive_heaviest_fork( progress, )?; match verify_coordinator_heaviest_fork( - new_root_slot, + final_restart_slot, coordinator_slot, &coordinator_hash, config.bank_forks.clone(), @@ -933,12 +940,11 @@ pub(crate) fn send_and_receive_heaviest_fork( Ok(()) => pushfn(coordinator_slot, coordinator_hash), Err(e) => { warn!( - "Failed to verify coordinator heaviest fork: {:?}, exit in 10 seconds", + "Failed to verify coordinator heaviest fork: {:?}, exit soon", e ); - pushfn(new_root_slot, new_root_hash); - // Wait for 10 seconds so the heaviest fork gets out. - sleep(Duration::from_secs(10)); + pushfn(final_restart_slot, final_restart_hash); + sleep(Duration::from_millis(GOSSIP_SLEEP_MILLIS)); return Err(e); } } @@ -1040,12 +1046,12 @@ pub fn wait_for_wen_restart(config: WenRestartConfig) -> Result<()> { } } WenRestartProgressInternalState::HeaviestFork { - new_root_slot, - new_root_hash, + final_restart_slot, + final_restart_hash, } => { let (slot, hash) = send_and_receive_heaviest_fork( - new_root_slot, - new_root_hash, + final_restart_slot, + final_restart_hash, &config, &mut progress, |slot, hash| { @@ -1055,12 +1061,12 @@ pub fn wait_for_wen_restart(config: WenRestartConfig) -> Result<()> { }, )?; WenRestartProgressInternalState::HeaviestFork { - new_root_slot: slot, - new_root_hash: hash, + final_restart_slot: slot, + final_restart_hash: hash, } } WenRestartProgressInternalState::GenerateSnapshot { - new_root_slot, + final_restart_slot, my_snapshot, } => { let snapshot_record = match my_snapshot { @@ -1070,11 +1076,11 @@ pub fn wait_for_wen_restart(config: WenRestartConfig) -> Result<()> { &config.snapshot_config, &config.accounts_background_request_sender, config.genesis_config_hash, - new_root_slot, + final_restart_slot, )?, }; WenRestartProgressInternalState::GenerateSnapshot { - new_root_slot, + final_restart_slot, my_snapshot: Some(snapshot_record), } } @@ -1165,22 +1171,24 @@ pub(crate) fn increment_and_write_wen_restart_records( if let Some(my_heaviest_fork) = my_heaviest_fork { progress.my_heaviest_fork = Some(my_heaviest_fork.clone()); WenRestartProgressInternalState::HeaviestFork { - new_root_slot: my_heaviest_fork.slot, - new_root_hash: Hash::from_str(&my_heaviest_fork.bankhash).unwrap(), + final_restart_slot: my_heaviest_fork.slot, + final_restart_hash: Hash::from_str(&my_heaviest_fork.bankhash).unwrap(), } } else { return Err(WenRestartError::UnexpectedState(RestartState::HeaviestFork).into()); } } - WenRestartProgressInternalState::HeaviestFork { new_root_slot, .. } => { + WenRestartProgressInternalState::HeaviestFork { + final_restart_slot, .. + } => { progress.set_state(RestartState::GenerateSnapshot); WenRestartProgressInternalState::GenerateSnapshot { - new_root_slot, + final_restart_slot, my_snapshot: None, } } WenRestartProgressInternalState::GenerateSnapshot { - new_root_slot: _, + final_restart_slot: _, my_snapshot, } => { if let Some(my_snapshot) = my_snapshot { @@ -1345,7 +1353,7 @@ pub(crate) fn initialize( )), RestartState::GenerateSnapshot => Ok(( WenRestartProgressInternalState::GenerateSnapshot { - new_root_slot: progress + final_restart_slot: progress .my_heaviest_fork .as_ref() .ok_or(WenRestartError::MalformedProgress( @@ -1935,9 +1943,12 @@ mod tests { let old_root_bank = test_state.bank_forks.read().unwrap().root_bank(); // Add bank last_vote + 1 linking directly to 0, tweak its epoch_stakes, and then add it to bank_forks. - let new_root_slot = last_vote_slot + 1; - let mut new_root_bank = - Bank::new_from_parent(old_root_bank.clone(), &Pubkey::default(), new_root_slot); + let final_restart_slot = last_vote_slot + 1; + let mut new_root_bank = Bank::new_from_parent( + old_root_bank.clone(), + &Pubkey::default(), + final_restart_slot, + ); assert_eq!(new_root_bank.epoch(), 1); // For epoch 2, make validator 0 have 90% of the stake. @@ -1974,7 +1985,7 @@ mod tests { let _ = insert_slots_into_blockstore( test_state.blockstore.clone(), 0, - &[new_root_slot], + &[final_restart_slot], TICKS_PER_SLOT, old_root_bank.last_blockhash(), ); @@ -2027,7 +2038,7 @@ mod tests { // new_epoch_bank (slot = first slot in epoch 2). They both link to last_vote_slot + 1. // old_epoch_bank has everyone's votes except 0, so it has > 66% stake in the old epoch. // new_epoch_bank has 0's vote, so it has > 66% stake in the new epoch. - let old_epoch_slot = new_root_slot + 1; + let old_epoch_slot = final_restart_slot + 1; let _ = insert_slots_into_blockstore( test_state.blockstore.clone(), new_root_bank.slot(), @@ -2038,7 +2049,7 @@ mod tests { let new_epoch_slot = new_root_bank.epoch_schedule().get_first_slot_in_epoch(2); let _ = insert_slots_into_blockstore( test_state.blockstore.clone(), - new_root_slot, + final_restart_slot, &[new_epoch_slot], TICKS_PER_SLOT, new_root_bank.last_blockhash(), @@ -2057,9 +2068,9 @@ mod tests { let now = timestamp(); // Validator 0 votes for the new_epoch_bank while everyone elese vote for old_epoch_bank. let last_voted_fork_slots = if index == 0 { - vec![new_epoch_slot, new_root_slot, 0] + vec![new_epoch_slot, final_restart_slot, 0] } else { - vec![old_epoch_slot, new_root_slot, 0] + vec![old_epoch_slot, final_restart_slot, 0] }; push_restart_last_voted_fork_slots( test_state.cluster_info.clone(), @@ -2076,7 +2087,7 @@ mod tests { wen_restart_path: test_state.wen_restart_proto_path, wen_restart_coordinator: test_state.wen_restart_coordinator, last_vote: VoteTransaction::from(Vote::new( - vec![new_root_slot], + vec![final_restart_slot], last_vote_bankhash )), blockstore: test_state.blockstore, @@ -2094,7 +2105,7 @@ mod tests { .unwrap(), WenRestartError::BlockNotLinkedToExpectedParent( new_epoch_slot, - Some(new_root_slot), + Some(final_restart_slot), old_epoch_slot ) ); @@ -2376,7 +2387,7 @@ mod tests { .unwrap(), ( WenRestartProgressInternalState::GenerateSnapshot { - new_root_slot: 0, + final_restart_slot: 0, my_snapshot: progress.my_snapshot.clone(), }, progress, @@ -2792,8 +2803,8 @@ mod tests { }), }, WenRestartProgressInternalState::HeaviestFork { - new_root_slot: 1, - new_root_hash: Hash::default(), + final_restart_slot: 1, + final_restart_hash: Hash::default(), }, WenRestartProgress { state: RestartState::HeaviestFork.into(), @@ -2811,11 +2822,11 @@ mod tests { ), ( WenRestartProgressInternalState::HeaviestFork { - new_root_slot: 1, - new_root_hash: Hash::default(), + final_restart_slot: 1, + final_restart_hash: Hash::default(), }, WenRestartProgressInternalState::GenerateSnapshot { - new_root_slot: 1, + final_restart_slot: 1, my_snapshot: None, }, WenRestartProgress { @@ -2837,7 +2848,7 @@ mod tests { ), ( WenRestartProgressInternalState::GenerateSnapshot { - new_root_slot: 1, + final_restart_slot: 1, my_snapshot: my_snapshot.clone(), }, WenRestartProgressInternalState::Done { From be7c1ed40a0482eb1670f77c580f7cc09631d5a0 Mon Sep 17 00:00:00 2001 From: Wen <113942165+wen-coding@users.noreply.github.com> Date: Tue, 29 Oct 2024 10:50:36 -0700 Subject: [PATCH 24/25] Rename final_restart_slot/hash with my_heaviest_fork_slot/hash --- wen-restart/src/wen_restart.rs | 107 +++++++++++++++++---------------- 1 file changed, 54 insertions(+), 53 deletions(-) diff --git a/wen-restart/src/wen_restart.rs b/wen-restart/src/wen_restart.rs index 0fff527e85e42c..b2e4316847a218 100644 --- a/wen-restart/src/wen_restart.rs +++ b/wen-restart/src/wen_restart.rs @@ -203,11 +203,11 @@ pub(crate) enum WenRestartProgressInternalState { my_heaviest_fork: Option, }, HeaviestFork { - final_restart_slot: Slot, - final_restart_hash: Hash, + my_heaviest_fork_slot: Slot, + my_heaviest_fork_hash: Hash, }, GenerateSnapshot { - final_restart_slot: Slot, + my_heaviest_fork_slot: Slot, my_snapshot: Option, }, Done { @@ -466,13 +466,13 @@ fn check_slot_smaller_than_intended_snapshot_slot( // We don't use set_root() explicitly, because it may kick off snapshot requests, we // can't have multiple snapshot requests in progress. In bank_to_snapshot_archive() // everything set_root() does will be done (without bank_forks setting root). So -// when we restart from the snapshot bank on final_restart_slot will become root. +// when we restart from the snapshot bank on my_heaviest_fork_slot will become root. pub(crate) fn generate_snapshot( bank_forks: Arc>, snapshot_config: &SnapshotConfig, accounts_background_request_sender: &AbsRequestSender, genesis_config_hash: Hash, - final_restart_slot: Slot, + my_heaviest_fork_slot: Slot, ) -> Result { let new_root_bank; { @@ -481,16 +481,16 @@ pub(crate) fn generate_snapshot( if !old_root_bank .hard_forks() .iter() - .any(|(slot, _)| slot == &final_restart_slot) + .any(|(slot, _)| slot == &my_heaviest_fork_slot) { - old_root_bank.register_hard_fork(final_restart_slot); + old_root_bank.register_hard_fork(my_heaviest_fork_slot); } - // final_restart_slot is guaranteed to have a bank in bank_forks, it's checked in + // my_heaviest_fork_slot is guaranteed to have a bank in bank_forks, it's checked in // find_bankhash_of_heaviest_fork(). - match my_bank_forks.get(final_restart_slot) { + match my_bank_forks.get(my_heaviest_fork_slot) { Some(bank) => new_root_bank = bank.clone(), None => { - return Err(WenRestartError::BlockNotFound(final_restart_slot).into()); + return Err(WenRestartError::BlockNotFound(my_heaviest_fork_slot).into()); } } let mut banks = vec![&new_root_bank]; @@ -498,7 +498,7 @@ pub(crate) fn generate_snapshot( banks.extend(parents.iter()); let _ = my_bank_forks.send_eah_request_if_needed( - final_restart_slot, + my_heaviest_fork_slot, &banks, accounts_background_request_sender, )?; @@ -523,7 +523,7 @@ pub(crate) fn generate_snapshot( }; // In very rare cases it's possible that the local root is not on the heaviest fork, so the // validator generated snapshot for slots > local root. If the cluster agreed upon restart - // slot final_restart_slot is less than the the current highest full_snapshot_slot, that means the + // slot my_heaviest_fork_slot is less than the the current highest full_snapshot_slot, that means the // locally rooted full_snapshot_slot will be rolled back. this requires human inspection。 // // In even rarer cases, the selected slot might be the latest full snapshot slot. We could @@ -531,7 +531,7 @@ pub(crate) fn generate_snapshot( // but for now we just return an error to keep the code simple. check_slot_smaller_than_intended_snapshot_slot( full_snapshot_slot, - final_restart_slot, + my_heaviest_fork_slot, directory, )?; directory = &snapshot_config.incremental_snapshot_archives_dir; @@ -540,7 +540,7 @@ pub(crate) fn generate_snapshot( { check_slot_smaller_than_intended_snapshot_slot( incremental_snapshot_slot, - final_restart_slot, + my_heaviest_fork_slot, directory, )?; } @@ -557,11 +557,11 @@ pub(crate) fn generate_snapshot( compute_shred_version(&genesis_config_hash, Some(&new_root_bank.hard_forks())); let new_snapshot_path = archive_info.path().display().to_string(); info!("wen_restart incremental snapshot generated on {new_snapshot_path} base slot {full_snapshot_slot}"); - // We might have bank snapshots past the final_restart_slot, we need to purge them. + // We might have bank snapshots past the my_heaviest_fork_slot, we need to purge them. purge_all_bank_snapshots(&snapshot_config.bank_snapshots_dir); Ok(GenerateSnapshotRecord { path: new_snapshot_path, - slot: final_restart_slot, + slot: my_heaviest_fork_slot, bankhash: new_root_bank.hash().to_string(), shred_version: new_shred_version as u32, }) @@ -912,15 +912,15 @@ pub(crate) fn receive_restart_heaviest_fork( } pub(crate) fn send_and_receive_heaviest_fork( - final_restart_slot: Slot, - final_restart_hash: Hash, + my_heaviest_fork_slot: Slot, + my_heaviest_fork_hash: Hash, config: &WenRestartConfig, progress: &mut WenRestartProgress, pushfn: impl FnOnce(Slot, Hash), ) -> Result<(Slot, Hash)> { if config.cluster_info.id() == config.wen_restart_coordinator { - pushfn(final_restart_slot, final_restart_hash); - Ok((final_restart_slot, final_restart_hash)) + pushfn(my_heaviest_fork_slot, my_heaviest_fork_hash); + Ok((my_heaviest_fork_slot, my_heaviest_fork_hash)) } else { let (coordinator_slot, coordinator_hash) = receive_restart_heaviest_fork( config.wen_restart_coordinator, @@ -929,7 +929,7 @@ pub(crate) fn send_and_receive_heaviest_fork( progress, )?; match verify_coordinator_heaviest_fork( - final_restart_slot, + my_heaviest_fork_slot, coordinator_slot, &coordinator_hash, config.bank_forks.clone(), @@ -943,7 +943,7 @@ pub(crate) fn send_and_receive_heaviest_fork( "Failed to verify coordinator heaviest fork: {:?}, exit soon", e ); - pushfn(final_restart_slot, final_restart_hash); + pushfn(my_heaviest_fork_slot, my_heaviest_fork_hash); sleep(Duration::from_millis(GOSSIP_SLEEP_MILLIS)); return Err(e); } @@ -1046,12 +1046,12 @@ pub fn wait_for_wen_restart(config: WenRestartConfig) -> Result<()> { } } WenRestartProgressInternalState::HeaviestFork { - final_restart_slot, - final_restart_hash, + my_heaviest_fork_slot, + my_heaviest_fork_hash, } => { let (slot, hash) = send_and_receive_heaviest_fork( - final_restart_slot, - final_restart_hash, + my_heaviest_fork_slot, + my_heaviest_fork_hash, &config, &mut progress, |slot, hash| { @@ -1061,12 +1061,12 @@ pub fn wait_for_wen_restart(config: WenRestartConfig) -> Result<()> { }, )?; WenRestartProgressInternalState::HeaviestFork { - final_restart_slot: slot, - final_restart_hash: hash, + my_heaviest_fork_slot: slot, + my_heaviest_fork_hash: hash, } } WenRestartProgressInternalState::GenerateSnapshot { - final_restart_slot, + my_heaviest_fork_slot, my_snapshot, } => { let snapshot_record = match my_snapshot { @@ -1076,11 +1076,11 @@ pub fn wait_for_wen_restart(config: WenRestartConfig) -> Result<()> { &config.snapshot_config, &config.accounts_background_request_sender, config.genesis_config_hash, - final_restart_slot, + my_heaviest_fork_slot, )?, }; WenRestartProgressInternalState::GenerateSnapshot { - final_restart_slot, + my_heaviest_fork_slot, my_snapshot: Some(snapshot_record), } } @@ -1171,24 +1171,25 @@ pub(crate) fn increment_and_write_wen_restart_records( if let Some(my_heaviest_fork) = my_heaviest_fork { progress.my_heaviest_fork = Some(my_heaviest_fork.clone()); WenRestartProgressInternalState::HeaviestFork { - final_restart_slot: my_heaviest_fork.slot, - final_restart_hash: Hash::from_str(&my_heaviest_fork.bankhash).unwrap(), + my_heaviest_fork_slot: my_heaviest_fork.slot, + my_heaviest_fork_hash: Hash::from_str(&my_heaviest_fork.bankhash).unwrap(), } } else { return Err(WenRestartError::UnexpectedState(RestartState::HeaviestFork).into()); } } WenRestartProgressInternalState::HeaviestFork { - final_restart_slot, .. + my_heaviest_fork_slot, + .. } => { progress.set_state(RestartState::GenerateSnapshot); WenRestartProgressInternalState::GenerateSnapshot { - final_restart_slot, + my_heaviest_fork_slot, my_snapshot: None, } } WenRestartProgressInternalState::GenerateSnapshot { - final_restart_slot: _, + my_heaviest_fork_slot: _, my_snapshot, } => { if let Some(my_snapshot) = my_snapshot { @@ -1353,7 +1354,7 @@ pub(crate) fn initialize( )), RestartState::GenerateSnapshot => Ok(( WenRestartProgressInternalState::GenerateSnapshot { - final_restart_slot: progress + my_heaviest_fork_slot: progress .my_heaviest_fork .as_ref() .ok_or(WenRestartError::MalformedProgress( @@ -1943,11 +1944,11 @@ mod tests { let old_root_bank = test_state.bank_forks.read().unwrap().root_bank(); // Add bank last_vote + 1 linking directly to 0, tweak its epoch_stakes, and then add it to bank_forks. - let final_restart_slot = last_vote_slot + 1; + let my_heaviest_fork_slot = last_vote_slot + 1; let mut new_root_bank = Bank::new_from_parent( old_root_bank.clone(), &Pubkey::default(), - final_restart_slot, + my_heaviest_fork_slot, ); assert_eq!(new_root_bank.epoch(), 1); @@ -1985,7 +1986,7 @@ mod tests { let _ = insert_slots_into_blockstore( test_state.blockstore.clone(), 0, - &[final_restart_slot], + &[my_heaviest_fork_slot], TICKS_PER_SLOT, old_root_bank.last_blockhash(), ); @@ -2038,7 +2039,7 @@ mod tests { // new_epoch_bank (slot = first slot in epoch 2). They both link to last_vote_slot + 1. // old_epoch_bank has everyone's votes except 0, so it has > 66% stake in the old epoch. // new_epoch_bank has 0's vote, so it has > 66% stake in the new epoch. - let old_epoch_slot = final_restart_slot + 1; + let old_epoch_slot = my_heaviest_fork_slot + 1; let _ = insert_slots_into_blockstore( test_state.blockstore.clone(), new_root_bank.slot(), @@ -2049,7 +2050,7 @@ mod tests { let new_epoch_slot = new_root_bank.epoch_schedule().get_first_slot_in_epoch(2); let _ = insert_slots_into_blockstore( test_state.blockstore.clone(), - final_restart_slot, + my_heaviest_fork_slot, &[new_epoch_slot], TICKS_PER_SLOT, new_root_bank.last_blockhash(), @@ -2068,9 +2069,9 @@ mod tests { let now = timestamp(); // Validator 0 votes for the new_epoch_bank while everyone elese vote for old_epoch_bank. let last_voted_fork_slots = if index == 0 { - vec![new_epoch_slot, final_restart_slot, 0] + vec![new_epoch_slot, my_heaviest_fork_slot, 0] } else { - vec![old_epoch_slot, final_restart_slot, 0] + vec![old_epoch_slot, my_heaviest_fork_slot, 0] }; push_restart_last_voted_fork_slots( test_state.cluster_info.clone(), @@ -2087,7 +2088,7 @@ mod tests { wen_restart_path: test_state.wen_restart_proto_path, wen_restart_coordinator: test_state.wen_restart_coordinator, last_vote: VoteTransaction::from(Vote::new( - vec![final_restart_slot], + vec![my_heaviest_fork_slot], last_vote_bankhash )), blockstore: test_state.blockstore, @@ -2105,7 +2106,7 @@ mod tests { .unwrap(), WenRestartError::BlockNotLinkedToExpectedParent( new_epoch_slot, - Some(final_restart_slot), + Some(my_heaviest_fork_slot), old_epoch_slot ) ); @@ -2387,7 +2388,7 @@ mod tests { .unwrap(), ( WenRestartProgressInternalState::GenerateSnapshot { - final_restart_slot: 0, + my_heaviest_fork_slot: 0, my_snapshot: progress.my_snapshot.clone(), }, progress, @@ -2803,8 +2804,8 @@ mod tests { }), }, WenRestartProgressInternalState::HeaviestFork { - final_restart_slot: 1, - final_restart_hash: Hash::default(), + my_heaviest_fork_slot: 1, + my_heaviest_fork_hash: Hash::default(), }, WenRestartProgress { state: RestartState::HeaviestFork.into(), @@ -2822,11 +2823,11 @@ mod tests { ), ( WenRestartProgressInternalState::HeaviestFork { - final_restart_slot: 1, - final_restart_hash: Hash::default(), + my_heaviest_fork_slot: 1, + my_heaviest_fork_hash: Hash::default(), }, WenRestartProgressInternalState::GenerateSnapshot { - final_restart_slot: 1, + my_heaviest_fork_slot: 1, my_snapshot: None, }, WenRestartProgress { @@ -2848,7 +2849,7 @@ mod tests { ), ( WenRestartProgressInternalState::GenerateSnapshot { - final_restart_slot: 1, + my_heaviest_fork_slot: 1, my_snapshot: my_snapshot.clone(), }, WenRestartProgressInternalState::Done { From e7efe5c83bcba4b216c6d801c081ffd530c3dea7 Mon Sep 17 00:00:00 2001 From: Wen <113942165+wen-coding@users.noreply.github.com> Date: Tue, 29 Oct 2024 16:50:14 -0700 Subject: [PATCH 25/25] flush_push_queue before waiting to speed things up. --- wen-restart/src/wen_restart.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/wen-restart/src/wen_restart.rs b/wen-restart/src/wen_restart.rs index b2e4316847a218..9296ba8e82bd42 100644 --- a/wen-restart/src/wen_restart.rs +++ b/wen-restart/src/wen_restart.rs @@ -944,6 +944,9 @@ pub(crate) fn send_and_receive_heaviest_fork( e ); pushfn(my_heaviest_fork_slot, my_heaviest_fork_hash); + // flush_push_queue only flushes the messages to crds, doesn't guarantee + // sending them out, so we still need to wait for a while before exiting. + config.cluster_info.flush_push_queue(); sleep(Duration::from_millis(GOSSIP_SLEEP_MILLIS)); return Err(e); }