From 894a3b295186c207af92692803f707f5cc08cdcb Mon Sep 17 00:00:00 2001 From: Aaron Blankstein Date: Thu, 17 Oct 2024 17:30:20 -0500 Subject: [PATCH 1/5] get replay blocks working with nakamoto blocks --- stackslib/src/chainstate/nakamoto/mod.rs | 43 +++++ stackslib/src/chainstate/nakamoto/tenure.rs | 4 + stackslib/src/cli.rs | 172 ++++++++++++++++++-- stackslib/src/main.rs | 5 + 4 files changed, 208 insertions(+), 16 deletions(-) diff --git a/stackslib/src/chainstate/nakamoto/mod.rs b/stackslib/src/chainstate/nakamoto/mod.rs index 8abbe058f5..2c3396a34b 100644 --- a/stackslib/src/chainstate/nakamoto/mod.rs +++ b/stackslib/src/chainstate/nakamoto/mod.rs @@ -2012,6 +2012,7 @@ impl NakamotoChainState { commit_burn, sortition_burn, &active_reward_set, + false, ) { Ok(next_chain_tip_info) => (Some(next_chain_tip_info), None), Err(e) => (None, Some(e)), @@ -3901,6 +3902,7 @@ impl NakamotoChainState { burnchain_commit_burn: u64, burnchain_sortition_burn: u64, active_reward_set: &RewardSet, + do_not_advance: bool, ) -> Result< ( StacksEpochReceipt, @@ -4095,6 +4097,7 @@ impl NakamotoChainState { burn_dbconn, block, parent_coinbase_height, + do_not_advance, )?; if new_tenure { // tenure height must have advanced @@ -4275,6 +4278,46 @@ impl NakamotoChainState { .as_ref() .map(|rewards| rewards.reward_info.clone()); + if do_not_advance { + // get burn block stats, for the transaction receipt + let (parent_burn_block_hash, parent_burn_block_height, parent_burn_block_timestamp) = + if block.is_first_mined() { + (BurnchainHeaderHash([0; 32]), 0, 0) + } else { + let sn = SortitionDB::get_block_snapshot_consensus(burn_dbconn, &parent_ch)? + .ok_or_else(|| { + // shouldn't happen + warn!( + "CORRUPTION: {} does not correspond to a burn block", + &parent_ch + ); + ChainstateError::InvalidStacksBlock("No parent consensus hash".into()) + })?; + ( + sn.burn_header_hash, + sn.block_height, + sn.burn_header_timestamp, + ) + }; + + let epoch_receipt = StacksEpochReceipt { + header: StacksHeaderInfo::regtest_genesis(), + tx_receipts, + matured_rewards, + matured_rewards_info: matured_rewards_info_opt, + parent_microblocks_cost: ExecutionCost::zero(), + anchored_block_cost: block_execution_cost, + parent_burn_block_hash, + parent_burn_block_height: u32::try_from(parent_burn_block_height).unwrap_or(0), // shouldn't be fatal + parent_burn_block_timestamp, + evaluated_epoch, + epoch_transition: applied_epoch_transition, + signers_updated: signer_set_calc.is_some(), + }; + + return Ok((epoch_receipt, clarity_commit, None)); + } + let new_tip = Self::advance_tip( &mut chainstate_tx.tx, &parent_chain_tip.anchored_header, diff --git a/stackslib/src/chainstate/nakamoto/tenure.rs b/stackslib/src/chainstate/nakamoto/tenure.rs index 4b7734653c..5c729d845d 100644 --- a/stackslib/src/chainstate/nakamoto/tenure.rs +++ b/stackslib/src/chainstate/nakamoto/tenure.rs @@ -840,6 +840,7 @@ impl NakamotoChainState { handle: &mut SH, block: &NakamotoBlock, parent_coinbase_height: u64, + do_not_advance: bool, ) -> Result { let Some(tenure_payload) = block.get_tenure_tx_payload() else { // no new tenure @@ -867,6 +868,9 @@ impl NakamotoChainState { )); }; + if do_not_advance { + return Ok(coinbase_height); + } Self::insert_nakamoto_tenure(headers_tx, &block.header, coinbase_height, tenure_payload)?; return Ok(coinbase_height); } diff --git a/stackslib/src/cli.rs b/stackslib/src/cli.rs index 9ff6e55644..14e06c6864 100644 --- a/stackslib/src/cli.rs +++ b/stackslib/src/cli.rs @@ -47,6 +47,7 @@ use crate::util_lib::db::IndexDBTx; /// Can be used with CLI commands to support non-mainnet chainstate /// Allows integration testing of these functions +#[derive(Deserialize)] pub struct StacksChainConfig { pub chain_id: u32, pub first_block_height: u64, @@ -68,10 +69,48 @@ impl StacksChainConfig { epochs: STACKS_EPOCHS_MAINNET.to_vec(), } } + + pub fn default_testnet() -> Self { + let mut pox_constants = PoxConstants::regtest_default(); + pox_constants.prepare_length = 100; + pox_constants.reward_cycle_length = 900; + pox_constants.v1_unlock_height = 3; + pox_constants.v2_unlock_height = 5; + pox_constants.pox_3_activation_height = 5; + pox_constants.pox_4_activation_height = 6; + pox_constants.v3_unlock_height = 7; + let mut epochs = STACKS_EPOCHS_REGTEST.to_vec(); + epochs[0].start_height = 0; + epochs[0].end_height = 0; + epochs[1].start_height = 0; + epochs[1].end_height = 1; + epochs[2].start_height = 1; + epochs[2].end_height = 2; + epochs[3].start_height = 2; + epochs[3].end_height = 3; + epochs[4].start_height = 3; + epochs[4].end_height = 4; + epochs[5].start_height = 4; + epochs[5].end_height = 5; + epochs[6].start_height = 5; + epochs[6].end_height = 6; + epochs[7].start_height = 6; + epochs[7].end_height = 56_457; + epochs[8].start_height = 56_457; + Self { + chain_id: CHAIN_ID_TESTNET, + first_block_height: 0, + first_burn_header_hash: BurnchainHeaderHash::from_hex(BITCOIN_REGTEST_FIRST_BLOCK_HASH) + .unwrap(), + first_burn_header_timestamp: BITCOIN_REGTEST_FIRST_BLOCK_TIMESTAMP.into(), + pox_constants, + epochs, + } + } } const STACKS_CHAIN_CONFIG_DEFAULT_MAINNET: LazyCell = - LazyCell::new(StacksChainConfig::default_mainnet); + LazyCell::new(StacksChainConfig::default_testnet); /// Replay blocks from chainstate database /// Terminates on error using `process::exit()` @@ -151,6 +190,91 @@ pub fn command_replay_block(argv: &[String], conf: Option<&StacksChainConfig>) { println!("Finished. run_time_seconds = {}", start.elapsed().as_secs()); } +/// Replay blocks from chainstate database +/// Terminates on error using `process::exit()` +/// +/// Arguments: +/// - `argv`: Args in CLI format: ` [args...]` +pub fn command_replay_block_nakamoto(argv: &[String], conf: Option<&StacksChainConfig>) { + let print_help_and_exit = || -> ! { + let n = &argv[0]; + eprintln!("Usage:"); + eprintln!(" {n} "); + eprintln!(" {n} prefix "); + eprintln!(" {n} index-range "); + eprintln!(" {n} range "); + eprintln!(" {n} "); + process::exit(1); + }; + let start = Instant::now(); + let db_path = argv.get(1).unwrap_or_else(|| print_help_and_exit()); + let mode = argv.get(2).map(String::as_str); + + let chain_state_path = format!("{db_path}/chainstate/"); + + let default_conf = STACKS_CHAIN_CONFIG_DEFAULT_MAINNET; + let conf = conf.unwrap_or(&default_conf); + + let mainnet = conf.chain_id == CHAIN_ID_MAINNET; + let (chainstate, _) = + StacksChainState::open(mainnet, conf.chain_id, &chain_state_path, None).unwrap(); + + let conn = chainstate.nakamoto_blocks_db(); + + let query = match mode { + Some("prefix") => format!( + "SELECT index_block_hash FROM nakamoto_staging_blocks WHERE orphaned = 0 AND index_block_hash LIKE \"{}%\"", + argv[3] + ), + Some("first") => format!( + "SELECT index_block_hash FROM nakamoto_staging_blocks WHERE orphaned = 0 ORDER BY height ASC LIMIT {}", + argv[3] + ), + Some("range") => { + let arg4 = argv[3] + .parse::() + .expect(" not a valid u64"); + let arg5 = argv[4].parse::().expect(" not a valid u64"); + let start = arg4.saturating_sub(1); + let blocks = arg5.saturating_sub(arg4); + format!("SELECT index_block_hash FROM nakamoto_staging_blocks WHERE orphaned = 0 ORDER BY height ASC LIMIT {start}, {blocks}") + } + Some("index-range") => { + let start = argv[3] + .parse::() + .expect(" not a valid u64"); + let end = argv[4].parse::().expect(" not a valid u64"); + let blocks = end.saturating_sub(start); + format!("SELECT index_block_hash FROM nakamoto_staging_blocks WHERE orphaned = 0 ORDER BY index_block_hash ASC LIMIT {start}, {blocks}") + } + Some("last") => format!( + "SELECT index_block_hash FROM nakamoto_staging_blocks WHERE orphaned = 0 ORDER BY height DESC LIMIT {}", + argv[3] + ), + Some(_) => print_help_and_exit(), + // Default to ALL blocks + None => "SELECT index_block_hash FROM nakamoto_staging_blocks WHERE orphaned = 0".into(), + }; + + let mut stmt = conn.prepare(&query).unwrap(); + let mut hashes_set = stmt.query(NO_PARAMS).unwrap(); + + let mut index_block_hashes: Vec = vec![]; + while let Ok(Some(row)) = hashes_set.next() { + index_block_hashes.push(row.get(0).unwrap()); + } + + let total = index_block_hashes.len(); + println!("Will check {total} blocks"); + for (i, index_block_hash) in index_block_hashes.iter().enumerate() { + if i % 100 == 0 { + println!("Checked {i}..."); + } + replay_naka_staging_block(db_path, index_block_hash, &conf); + } + println!("Finished. run_time_seconds = {}", start.elapsed().as_secs()); +} + /// Replay mock mined blocks from JSON files /// Terminates on error using `process::exit()` /// @@ -525,11 +649,39 @@ fn replay_block( }; } +/// Fetch and process a NakamotoBlock from database and call `replay_block_nakamoto()` to validate +fn replay_naka_staging_block(db_path: &str, index_block_hash_hex: &str, conf: &StacksChainConfig) { + let block_id = StacksBlockId::from_hex(index_block_hash_hex).unwrap(); + let chain_state_path = format!("{db_path}/chainstate/"); + let sort_db_path = format!("{db_path}/burnchain/sortition"); + + let mainnet = conf.chain_id == CHAIN_ID_MAINNET; + let (mut chainstate, _) = + StacksChainState::open(mainnet, conf.chain_id, &chain_state_path, None).unwrap(); + + let mut sortdb = SortitionDB::connect( + &sort_db_path, + conf.first_block_height, + &conf.first_burn_header_hash, + conf.first_burn_header_timestamp, + &conf.epochs, + conf.pox_constants.clone(), + None, + true, + ) + .unwrap(); + + let (block, block_size) = chainstate + .nakamoto_blocks_db() + .get_nakamoto_block(&block_id) + .unwrap() + .unwrap(); + replay_block_nakamoto(&mut sortdb, &mut chainstate, &block, block_size).unwrap(); +} + fn replay_block_nakamoto( sort_db: &mut SortitionDB, stacks_chain_state: &mut StacksChainState, - mut chainstate_tx: ChainstateTx, - clarity_instance: &mut ClarityInstance, block: &NakamotoBlock, block_size: u64, ) -> Result<(), ChainstateError> { @@ -758,6 +910,7 @@ fn replay_block_nakamoto( commit_burn, sortition_burn, &active_reward_set, + true, ) { Ok(next_chain_tip_info) => (Some(next_chain_tip_info), None), Err(e) => (None, Some(e)), @@ -785,18 +938,5 @@ fn replay_block_nakamoto( return Err(e); }; - let (receipt, clarity_commit, reward_set_data) = ok_opt.expect("FATAL: unreachable"); - - assert_eq!( - receipt.header.anchored_header.block_hash(), - block.header.block_hash() - ); - assert_eq!(receipt.header.consensus_hash, block.header.consensus_hash); - - info!( - "Advanced to new tip! {}/{}", - &receipt.header.consensus_hash, - &receipt.header.anchored_header.block_hash() - ); Ok(()) } diff --git a/stackslib/src/main.rs b/stackslib/src/main.rs index 98315cffa8..ab83505825 100644 --- a/stackslib/src/main.rs +++ b/stackslib/src/main.rs @@ -1470,6 +1470,11 @@ simulating a miner. process::exit(0); } + if argv[1] == "replay-naka-block" { + cli::command_replay_block_nakamoto(&argv[1..], None); + process::exit(0); + } + if argv[1] == "replay-mock-mining" { cli::command_replay_mock_mining(&argv[1..], None); process::exit(0); From 87ae30f26d8c64d032bf6bccb4013fa7fbfb7463 Mon Sep 17 00:00:00 2001 From: Aaron Blankstein Date: Mon, 21 Oct 2024 10:47:47 -0500 Subject: [PATCH 2/5] network flags for nakamoto replay block --- stackslib/src/cli.rs | 2 +- stackslib/src/main.rs | 26 +++++++++++++++++++++++++- 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/stackslib/src/cli.rs b/stackslib/src/cli.rs index 14e06c6864..34fee2a9be 100644 --- a/stackslib/src/cli.rs +++ b/stackslib/src/cli.rs @@ -110,7 +110,7 @@ impl StacksChainConfig { } const STACKS_CHAIN_CONFIG_DEFAULT_MAINNET: LazyCell = - LazyCell::new(StacksChainConfig::default_testnet); + LazyCell::new(StacksChainConfig::default_mainnet); /// Replay blocks from chainstate database /// Terminates on error using `process::exit()` diff --git a/stackslib/src/main.rs b/stackslib/src/main.rs index ab83505825..7cc5b24426 100644 --- a/stackslib/src/main.rs +++ b/stackslib/src/main.rs @@ -1471,7 +1471,31 @@ simulating a miner. } if argv[1] == "replay-naka-block" { - cli::command_replay_block_nakamoto(&argv[1..], None); + let chain_config = + if let Some(network_flag_ix) = argv.iter().position(|arg| arg == "--network") { + let Some(network_choice) = argv.get(network_flag_ix + 1) else { + eprintln!("Must supply network choice after `--network` option"); + process::exit(1); + }; + + let network_config = match network_choice.as_str() { + "testnet" => cli::StacksChainConfig::default_testnet(), + "mainnet" => cli::StacksChainConfig::default_mainnet(), + other => { + eprintln!("Unknown network choice `{other}`"); + process::exit(1); + } + }; + + argv.remove(network_flag_ix + 1); + argv.remove(network_flag_ix); + + Some(network_config) + } else { + None + }; + + cli::command_replay_block_nakamoto(&argv[1..], chain_config.as_ref()); process::exit(0); } From 84950edfa160e6662b3cbd41a14522fcf2fb9ba7 Mon Sep 17 00:00:00 2001 From: Aaron Blankstein Date: Wed, 23 Oct 2024 10:17:25 -0500 Subject: [PATCH 3/5] address PR reviews --- stackslib/src/chainstate/nakamoto/mod.rs | 103 +++++++++++++++-------- stackslib/src/main.rs | 2 +- 2 files changed, 68 insertions(+), 37 deletions(-) diff --git a/stackslib/src/chainstate/nakamoto/mod.rs b/stackslib/src/chainstate/nakamoto/mod.rs index 2c3396a34b..3cf61397bf 100644 --- a/stackslib/src/chainstate/nakamoto/mod.rs +++ b/stackslib/src/chainstate/nakamoto/mod.rs @@ -3885,6 +3885,60 @@ impl NakamotoChainState { Ok(()) } + pub(crate) fn make_non_advancing_receipt<'a>( + clarity_commit: PreCommitClarityBlock<'a>, + burn_dbconn: &SortitionHandleConn, + parent_ch: &ConsensusHash, + evaluated_epoch: StacksEpochId, + matured_rewards: Vec, + tx_receipts: Vec, + matured_rewards_info_opt: Option, + block_execution_cost: ExecutionCost, + applied_epoch_transition: bool, + signers_updated: bool, + ) -> Result< + ( + StacksEpochReceipt, + PreCommitClarityBlock<'a>, + Option, + ), + ChainstateError, + > { + // get burn block stats, for the transaction receipt + + let parent_sn = SortitionDB::get_block_snapshot_consensus(burn_dbconn, &parent_ch)? + .ok_or_else(|| { + // shouldn't happen + warn!( + "CORRUPTION: {} does not correspond to a burn block", + &parent_ch + ); + ChainstateError::InvalidStacksBlock("No parent consensus hash".into()) + })?; + let (parent_burn_block_hash, parent_burn_block_height, parent_burn_block_timestamp) = ( + parent_sn.burn_header_hash, + parent_sn.block_height, + parent_sn.burn_header_timestamp, + ); + + let epoch_receipt = StacksEpochReceipt { + header: StacksHeaderInfo::regtest_genesis(), + tx_receipts, + matured_rewards, + matured_rewards_info: matured_rewards_info_opt, + parent_microblocks_cost: ExecutionCost::zero(), + anchored_block_cost: block_execution_cost, + parent_burn_block_hash, + parent_burn_block_height: u32::try_from(parent_burn_block_height).unwrap_or(0), // shouldn't be fatal + parent_burn_block_timestamp, + evaluated_epoch, + epoch_transition: applied_epoch_transition, + signers_updated, + }; + + return Ok((epoch_receipt, clarity_commit, None)); + } + /// Append a Nakamoto Stacks block to the Stacks chain state. /// NOTE: This does _not_ set the block as processed! The caller must do this. pub(crate) fn append_block<'a>( @@ -4279,43 +4333,20 @@ impl NakamotoChainState { .map(|rewards| rewards.reward_info.clone()); if do_not_advance { - // get burn block stats, for the transaction receipt - let (parent_burn_block_hash, parent_burn_block_height, parent_burn_block_timestamp) = - if block.is_first_mined() { - (BurnchainHeaderHash([0; 32]), 0, 0) - } else { - let sn = SortitionDB::get_block_snapshot_consensus(burn_dbconn, &parent_ch)? - .ok_or_else(|| { - // shouldn't happen - warn!( - "CORRUPTION: {} does not correspond to a burn block", - &parent_ch - ); - ChainstateError::InvalidStacksBlock("No parent consensus hash".into()) - })?; - ( - sn.burn_header_hash, - sn.block_height, - sn.burn_header_timestamp, - ) - }; - - let epoch_receipt = StacksEpochReceipt { - header: StacksHeaderInfo::regtest_genesis(), - tx_receipts, - matured_rewards, - matured_rewards_info: matured_rewards_info_opt, - parent_microblocks_cost: ExecutionCost::zero(), - anchored_block_cost: block_execution_cost, - parent_burn_block_hash, - parent_burn_block_height: u32::try_from(parent_burn_block_height).unwrap_or(0), // shouldn't be fatal - parent_burn_block_timestamp, + // if we're performing a block replay, and we don't want to advance any + // of the db state, return a fake receipt + return Self::make_non_advancing_receipt( + clarity_commit, + burn_dbconn, + &parent_ch, evaluated_epoch, - epoch_transition: applied_epoch_transition, - signers_updated: signer_set_calc.is_some(), - }; - - return Ok((epoch_receipt, clarity_commit, None)); + matured_rewards, + tx_receipts, + matured_rewards_info_opt, + block_execution_cost, + applied_epoch_transition, + signer_set_calc.is_some(), + ); } let new_tip = Self::advance_tip( diff --git a/stackslib/src/main.rs b/stackslib/src/main.rs index 7cc5b24426..bcb7dfc964 100644 --- a/stackslib/src/main.rs +++ b/stackslib/src/main.rs @@ -1478,7 +1478,7 @@ simulating a miner. process::exit(1); }; - let network_config = match network_choice.as_str() { + let network_config = match network_choice.to_lowercase().as_str() { "testnet" => cli::StacksChainConfig::default_testnet(), "mainnet" => cli::StacksChainConfig::default_mainnet(), other => { From eac2e5e25c1bdddffdc35c8c6527c05abfcd84c3 Mon Sep 17 00:00:00 2001 From: Aaron Blankstein Date: Thu, 24 Oct 2024 13:09:15 -0500 Subject: [PATCH 4/5] fix: merge artifact --- stackslib/src/chainstate/nakamoto/mod.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/stackslib/src/chainstate/nakamoto/mod.rs b/stackslib/src/chainstate/nakamoto/mod.rs index c504f52c62..0bd51ddb80 100644 --- a/stackslib/src/chainstate/nakamoto/mod.rs +++ b/stackslib/src/chainstate/nakamoto/mod.rs @@ -3897,6 +3897,7 @@ impl NakamotoChainState { block_execution_cost: ExecutionCost, applied_epoch_transition: bool, signers_updated: bool, + coinbase_height: u64, ) -> Result< ( StacksEpochReceipt, @@ -3935,6 +3936,7 @@ impl NakamotoChainState { evaluated_epoch, epoch_transition: applied_epoch_transition, signers_updated, + coinbase_height, }; return Ok((epoch_receipt, clarity_commit, None)); @@ -4347,6 +4349,7 @@ impl NakamotoChainState { block_execution_cost, applied_epoch_transition, signer_set_calc.is_some(), + coinbase_height, ); } From 98d3638e3f9da383c86a1efbb85fda87c87514a8 Mon Sep 17 00:00:00 2001 From: Brice Dobry Date: Tue, 19 Nov 2024 11:08:40 -0500 Subject: [PATCH 5/5] chore: resolve merge conflict --- stackslib/src/cli.rs | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/stackslib/src/cli.rs b/stackslib/src/cli.rs index 3a53306682..f703f8a367 100644 --- a/stackslib/src/cli.rs +++ b/stackslib/src/cli.rs @@ -79,24 +79,24 @@ impl StacksChainConfig { pox_constants.pox_3_activation_height = 5; pox_constants.pox_4_activation_height = 6; pox_constants.v3_unlock_height = 7; - let mut epochs = STACKS_EPOCHS_REGTEST.to_vec(); - epochs[0].start_height = 0; - epochs[0].end_height = 0; - epochs[1].start_height = 0; - epochs[1].end_height = 1; - epochs[2].start_height = 1; - epochs[2].end_height = 2; - epochs[3].start_height = 2; - epochs[3].end_height = 3; - epochs[4].start_height = 3; - epochs[4].end_height = 4; - epochs[5].start_height = 4; - epochs[5].end_height = 5; - epochs[6].start_height = 5; - epochs[6].end_height = 6; - epochs[7].start_height = 6; - epochs[7].end_height = 56_457; - epochs[8].start_height = 56_457; + let mut epochs = EpochList::new(&*STACKS_EPOCHS_REGTEST); + epochs[StacksEpochId::Epoch10].start_height = 0; + epochs[StacksEpochId::Epoch10].end_height = 0; + epochs[StacksEpochId::Epoch20].start_height = 0; + epochs[StacksEpochId::Epoch20].end_height = 1; + epochs[StacksEpochId::Epoch2_05].start_height = 1; + epochs[StacksEpochId::Epoch2_05].end_height = 2; + epochs[StacksEpochId::Epoch21].start_height = 2; + epochs[StacksEpochId::Epoch21].end_height = 3; + epochs[StacksEpochId::Epoch22].start_height = 3; + epochs[StacksEpochId::Epoch22].end_height = 4; + epochs[StacksEpochId::Epoch23].start_height = 4; + epochs[StacksEpochId::Epoch23].end_height = 5; + epochs[StacksEpochId::Epoch24].start_height = 5; + epochs[StacksEpochId::Epoch24].end_height = 6; + epochs[StacksEpochId::Epoch25].start_height = 6; + epochs[StacksEpochId::Epoch25].end_height = 56_457; + epochs[StacksEpochId::Epoch30].start_height = 56_457; Self { chain_id: CHAIN_ID_TESTNET, first_block_height: 0,