Merge pull request #5346 from stacks-network/test/replay-block-naka

Test: Add replay block command for nakamoto blocks

stackslib/src/chainstate/nakamoto/mod.rs

@@ -2020,6 +2020,7 @@ impl NakamotoChainState {
             commit_burn,
             sortition_burn,
             &active_reward_set,
+            false,
         ) {
             Ok(next_chain_tip_info) => (Some(next_chain_tip_info), None),
             Err(e) => (None, Some(e)),
@@ -3893,6 +3894,62 @@ impl NakamotoChainState {
         Ok(())
     }
 
+    pub(crate) fn make_non_advancing_receipt<'a>(
+        clarity_commit: PreCommitClarityBlock<'a>,
+        burn_dbconn: &SortitionHandleConn,
+        parent_ch: &ConsensusHash,
+        evaluated_epoch: StacksEpochId,
+        matured_rewards: Vec<MinerReward>,
+        tx_receipts: Vec<StacksTransactionReceipt>,
+        matured_rewards_info_opt: Option<MinerRewardInfo>,
+        block_execution_cost: ExecutionCost,
+        applied_epoch_transition: bool,
+        signers_updated: bool,
+        coinbase_height: u64,
+    ) -> Result<
+        (
+            StacksEpochReceipt,
+            PreCommitClarityBlock<'a>,
+            Option<RewardSetData>,
+        ),
+        ChainstateError,
+    > {
+        // get burn block stats, for the transaction receipt
+
+        let parent_sn = SortitionDB::get_block_snapshot_consensus(burn_dbconn, &parent_ch)?
+            .ok_or_else(|| {
+                // shouldn't happen
+                warn!(
+                    "CORRUPTION: {} does not correspond to a burn block",
+                    &parent_ch
+                );
+                ChainstateError::InvalidStacksBlock("No parent consensus hash".into())
+            })?;
+        let (parent_burn_block_hash, parent_burn_block_height, parent_burn_block_timestamp) = (
+            parent_sn.burn_header_hash,
+            parent_sn.block_height,
+            parent_sn.burn_header_timestamp,
+        );
+
+        let epoch_receipt = StacksEpochReceipt {
+            header: StacksHeaderInfo::regtest_genesis(),
+            tx_receipts,
+            matured_rewards,
+            matured_rewards_info: matured_rewards_info_opt,
+            parent_microblocks_cost: ExecutionCost::zero(),
+            anchored_block_cost: block_execution_cost,
+            parent_burn_block_hash,
+            parent_burn_block_height: u32::try_from(parent_burn_block_height).unwrap_or(0), // shouldn't be fatal
+            parent_burn_block_timestamp,
+            evaluated_epoch,
+            epoch_transition: applied_epoch_transition,
+            signers_updated,
+            coinbase_height,
+        };
+
+        return Ok((epoch_receipt, clarity_commit, None));
+    }
+
     /// Append a Nakamoto Stacks block to the Stacks chain state.
     /// NOTE: This does _not_ set the block as processed!  The caller must do this.
     pub(crate) fn append_block<'a>(
@@ -3910,6 +3967,7 @@ impl NakamotoChainState {
         burnchain_commit_burn: u64,
         burnchain_sortition_burn: u64,
         active_reward_set: &RewardSet,
+        do_not_advance: bool,
     ) -> Result<
         (
             StacksEpochReceipt,
@@ -4104,6 +4162,7 @@ impl NakamotoChainState {
             burn_dbconn,
             block,
             parent_coinbase_height,
+            do_not_advance,
         )?;
         if new_tenure {
             // tenure height must have advanced
@@ -4284,6 +4343,24 @@ impl NakamotoChainState {
             .as_ref()
             .map(|rewards| rewards.reward_info.clone());
 
+        if do_not_advance {
+            // if we're performing a block replay, and we don't want to advance any
+            //  of the db state, return a fake receipt
+            return Self::make_non_advancing_receipt(
+                clarity_commit,
+                burn_dbconn,
+                &parent_ch,
+                evaluated_epoch,
+                matured_rewards,
+                tx_receipts,
+                matured_rewards_info_opt,
+                block_execution_cost,
+                applied_epoch_transition,
+                signer_set_calc.is_some(),
+                coinbase_height,
+            );
+        }
+
         let new_tip = Self::advance_tip(
             &mut chainstate_tx.tx,
             &parent_chain_tip.anchored_header,

stackslib/src/chainstate/nakamoto/tenure.rs

@@ -840,6 +840,7 @@ impl NakamotoChainState {
         handle: &mut SH,
         block: &NakamotoBlock,
         parent_coinbase_height: u64,
+        do_not_advance: bool,
     ) -> Result<u64, ChainstateError> {
         let Some(tenure_payload) = block.get_tenure_tx_payload() else {
             // no new tenure
@@ -867,6 +868,9 @@ impl NakamotoChainState {
             ));
         };
 
+        if do_not_advance {
+            return Ok(coinbase_height);
+        }
         Self::insert_nakamoto_tenure(headers_tx, &block.header, coinbase_height, tenure_payload)?;
         return Ok(coinbase_height);
     }

stackslib/src/cli.rs

@@ -47,6 +47,7 @@ use crate::util_lib::db::IndexDBTx;
 
 /// Can be used with CLI commands to support non-mainnet chainstate
 /// Allows integration testing of these functions
+#[derive(Deserialize)]
 pub struct StacksChainConfig {
     pub chain_id: u32,
     pub first_block_height: u64,
@@ -68,6 +69,44 @@ impl StacksChainConfig {
             epochs: (*STACKS_EPOCHS_MAINNET).clone(),
         }
     }
+
+    pub fn default_testnet() -> Self {
+        let mut pox_constants = PoxConstants::regtest_default();
+        pox_constants.prepare_length = 100;
+        pox_constants.reward_cycle_length = 900;
+        pox_constants.v1_unlock_height = 3;
+        pox_constants.v2_unlock_height = 5;
+        pox_constants.pox_3_activation_height = 5;
+        pox_constants.pox_4_activation_height = 6;
+        pox_constants.v3_unlock_height = 7;
+        let mut epochs = EpochList::new(&*STACKS_EPOCHS_REGTEST);
+        epochs[StacksEpochId::Epoch10].start_height = 0;
+        epochs[StacksEpochId::Epoch10].end_height = 0;
+        epochs[StacksEpochId::Epoch20].start_height = 0;
+        epochs[StacksEpochId::Epoch20].end_height = 1;
+        epochs[StacksEpochId::Epoch2_05].start_height = 1;
+        epochs[StacksEpochId::Epoch2_05].end_height = 2;
+        epochs[StacksEpochId::Epoch21].start_height = 2;
+        epochs[StacksEpochId::Epoch21].end_height = 3;
+        epochs[StacksEpochId::Epoch22].start_height = 3;
+        epochs[StacksEpochId::Epoch22].end_height = 4;
+        epochs[StacksEpochId::Epoch23].start_height = 4;
+        epochs[StacksEpochId::Epoch23].end_height = 5;
+        epochs[StacksEpochId::Epoch24].start_height = 5;
+        epochs[StacksEpochId::Epoch24].end_height = 6;
+        epochs[StacksEpochId::Epoch25].start_height = 6;
+        epochs[StacksEpochId::Epoch25].end_height = 56_457;
+        epochs[StacksEpochId::Epoch30].start_height = 56_457;
+        Self {
+            chain_id: CHAIN_ID_TESTNET,
+            first_block_height: 0,
+            first_burn_header_hash: BurnchainHeaderHash::from_hex(BITCOIN_REGTEST_FIRST_BLOCK_HASH)
+                .unwrap(),
+            first_burn_header_timestamp: BITCOIN_REGTEST_FIRST_BLOCK_TIMESTAMP.into(),
+            pox_constants,
+            epochs,
+        }
+    }
 }
 
 const STACKS_CHAIN_CONFIG_DEFAULT_MAINNET: LazyCell<StacksChainConfig> =
@@ -151,6 +190,91 @@ pub fn command_replay_block(argv: &[String], conf: Option<&StacksChainConfig>) {
     println!("Finished. run_time_seconds = {}", start.elapsed().as_secs());
 }
 
+/// Replay blocks from chainstate database
+/// Terminates on error using `process::exit()`
+///
+/// Arguments:
+///  - `argv`: Args in CLI format: `<command-name> [args...]`
+pub fn command_replay_block_nakamoto(argv: &[String], conf: Option<&StacksChainConfig>) {
+    let print_help_and_exit = || -> ! {
+        let n = &argv[0];
+        eprintln!("Usage:");
+        eprintln!("  {n} <database-path>");
+        eprintln!("  {n} <database-path> prefix <index-block-hash-prefix>");
+        eprintln!("  {n} <database-path> index-range <start-block> <end-block>");
+        eprintln!("  {n} <database-path> range <start-block> <end-block>");
+        eprintln!("  {n} <database-path> <first|last> <block-count>");
+        process::exit(1);
+    };
+    let start = Instant::now();
+    let db_path = argv.get(1).unwrap_or_else(|| print_help_and_exit());
+    let mode = argv.get(2).map(String::as_str);
+
+    let chain_state_path = format!("{db_path}/chainstate/");
+
+    let default_conf = STACKS_CHAIN_CONFIG_DEFAULT_MAINNET;
+    let conf = conf.unwrap_or(&default_conf);
+
+    let mainnet = conf.chain_id == CHAIN_ID_MAINNET;
+    let (chainstate, _) =
+        StacksChainState::open(mainnet, conf.chain_id, &chain_state_path, None).unwrap();
+
+    let conn = chainstate.nakamoto_blocks_db();
+
+    let query = match mode {
+        Some("prefix") => format!(
+			"SELECT index_block_hash FROM nakamoto_staging_blocks WHERE orphaned = 0 AND index_block_hash LIKE \"{}%\"",
+			argv[3]
+		),
+        Some("first") => format!(
+			"SELECT index_block_hash FROM nakamoto_staging_blocks WHERE orphaned = 0 ORDER BY height ASC LIMIT {}",
+			argv[3]
+		),
+        Some("range") => {
+            let arg4 = argv[3]
+                .parse::<u64>()
+                .expect("<start_block> not a valid u64");
+            let arg5 = argv[4].parse::<u64>().expect("<end-block> not a valid u64");
+            let start = arg4.saturating_sub(1);
+            let blocks = arg5.saturating_sub(arg4);
+            format!("SELECT index_block_hash FROM nakamoto_staging_blocks WHERE orphaned = 0 ORDER BY height ASC LIMIT {start}, {blocks}")
+        }
+        Some("index-range") => {
+            let start = argv[3]
+                .parse::<u64>()
+                .expect("<start_block> not a valid u64");
+            let end = argv[4].parse::<u64>().expect("<end-block> not a valid u64");
+            let blocks = end.saturating_sub(start);
+            format!("SELECT index_block_hash FROM nakamoto_staging_blocks WHERE orphaned = 0 ORDER BY index_block_hash ASC LIMIT {start}, {blocks}")
+        }
+        Some("last") => format!(
+			"SELECT index_block_hash FROM nakamoto_staging_blocks WHERE orphaned = 0 ORDER BY height DESC LIMIT {}",
+			argv[3]
+		),
+        Some(_) => print_help_and_exit(),
+        // Default to ALL blocks
+        None => "SELECT index_block_hash FROM nakamoto_staging_blocks WHERE orphaned = 0".into(),
+    };
+
+    let mut stmt = conn.prepare(&query).unwrap();
+    let mut hashes_set = stmt.query(NO_PARAMS).unwrap();
+
+    let mut index_block_hashes: Vec<String> = vec![];
+    while let Ok(Some(row)) = hashes_set.next() {
+        index_block_hashes.push(row.get(0).unwrap());
+    }
+
+    let total = index_block_hashes.len();
+    println!("Will check {total} blocks");
+    for (i, index_block_hash) in index_block_hashes.iter().enumerate() {
+        if i % 100 == 0 {
+            println!("Checked {i}...");
+        }
+        replay_naka_staging_block(db_path, index_block_hash, &conf);
+    }
+    println!("Finished. run_time_seconds = {}", start.elapsed().as_secs());
+}
+
 /// Replay mock mined blocks from JSON files
 /// Terminates on error using `process::exit()`
 ///
@@ -525,6 +649,36 @@ fn replay_block(
     };
 }
 
+/// Fetch and process a NakamotoBlock from database and call `replay_block_nakamoto()` to validate
+fn replay_naka_staging_block(db_path: &str, index_block_hash_hex: &str, conf: &StacksChainConfig) {
+    let block_id = StacksBlockId::from_hex(index_block_hash_hex).unwrap();
+    let chain_state_path = format!("{db_path}/chainstate/");
+    let sort_db_path = format!("{db_path}/burnchain/sortition");
+
+    let mainnet = conf.chain_id == CHAIN_ID_MAINNET;
+    let (mut chainstate, _) =
+        StacksChainState::open(mainnet, conf.chain_id, &chain_state_path, None).unwrap();
+
+    let mut sortdb = SortitionDB::connect(
+        &sort_db_path,
+        conf.first_block_height,
+        &conf.first_burn_header_hash,
+        conf.first_burn_header_timestamp,
+        &conf.epochs,
+        conf.pox_constants.clone(),
+        None,
+        true,
+    )
+    .unwrap();
+
+    let (block, block_size) = chainstate
+        .nakamoto_blocks_db()
+        .get_nakamoto_block(&block_id)
+        .unwrap()
+        .unwrap();
+    replay_block_nakamoto(&mut sortdb, &mut chainstate, &block, block_size).unwrap();
+}
+
 fn replay_block_nakamoto(
     sort_db: &mut SortitionDB,
     stacks_chain_state: &mut StacksChainState,
@@ -756,6 +910,7 @@ fn replay_block_nakamoto(
         commit_burn,
         sortition_burn,
         &active_reward_set,
+        true,
     ) {
         Ok(next_chain_tip_info) => (Some(next_chain_tip_info), None),
         Err(e) => (None, Some(e)),
@@ -783,18 +938,5 @@ fn replay_block_nakamoto(
         return Err(e);
     };
 
-    let (receipt, _clarity_commit, _reward_set_data) = ok_opt.expect("FATAL: unreachable");
-
-    assert_eq!(
-        receipt.header.anchored_header.block_hash(),
-        block.header.block_hash()
-    );
-    assert_eq!(receipt.header.consensus_hash, block.header.consensus_hash);
-
-    info!(
-        "Advanced to new tip! {}/{}",
-        &receipt.header.consensus_hash,
-        &receipt.header.anchored_header.block_hash()
-    );
     Ok(())
 }

stackslib/src/main.rs

@@ -1470,6 +1470,35 @@ simulating a miner.
         process::exit(0);
     }
 
+    if argv[1] == "replay-naka-block" {
+        let chain_config =
+            if let Some(network_flag_ix) = argv.iter().position(|arg| arg == "--network") {
+                let Some(network_choice) = argv.get(network_flag_ix + 1) else {
+                    eprintln!("Must supply network choice after `--network` option");
+                    process::exit(1);
+                };
+
+                let network_config = match network_choice.to_lowercase().as_str() {
+                    "testnet" => cli::StacksChainConfig::default_testnet(),
+                    "mainnet" => cli::StacksChainConfig::default_mainnet(),
+                    other => {
+                        eprintln!("Unknown network choice `{other}`");
+                        process::exit(1);
+                    }
+                };
+
+                argv.remove(network_flag_ix + 1);
+                argv.remove(network_flag_ix);
+
+                Some(network_config)
+            } else {
+                None
+            };
+
+        cli::command_replay_block_nakamoto(&argv[1..], chain_config.as_ref());
+        process::exit(0);
+    }
+
     if argv[1] == "replay-mock-mining" {
         cli::command_replay_mock_mining(&argv[1..], None);
         process::exit(0);