From 6ef41a52f140fc99e8e6c01b70c655eaeb06e3ee Mon Sep 17 00:00:00 2001 From: Xiangyi Zheng Date: Thu, 20 Apr 2023 11:00:52 +0100 Subject: [PATCH] (ND-322)feat: undo-block tool - revert the current head of the chain to its previous block --- CHANGELOG.md | 1 + Cargo.lock | 17 +++ Cargo.toml | 2 + chain/chain/src/store.rs | 134 ++++++++++++++++++ integration-tests/Cargo.toml | 1 + integration-tests/src/tests/client/mod.rs | 1 + .../src/tests/client/undo_block.rs | 77 ++++++++++ neard/Cargo.toml | 1 + neard/src/cli.rs | 7 + tools/undo-block/Cargo.toml | 18 +++ tools/undo-block/src/cli.rs | 40 ++++++ tools/undo-block/src/lib.rs | 49 +++++++ 12 files changed, 348 insertions(+) create mode 100644 integration-tests/src/tests/client/undo_block.rs create mode 100644 tools/undo-block/Cargo.toml create mode 100644 tools/undo-block/src/cli.rs create mode 100644 tools/undo-block/src/lib.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 5c411adf237..d1cdd9a9beb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ * Node can sync State from S3. [#8789](https://github.com/near/nearcore/pull/8789) * The contract runtime switched to using our fork of wasmer, with various improvements. +* undo-block tool to reset the chain head from current head to its prev block. Use the tool by running: `./target/release/neard --home {path_to_config_directory} undo-block`. [#8681](https://github.com/near/nearcore/pull/8681) ## 1.33.0 diff --git a/Cargo.lock b/Cargo.lock index c5fd7fb6d9e..d2dfef941e6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2542,6 +2542,7 @@ dependencies = [ "near-store", "near-telemetry", "near-test-contracts", + "near-undo-block", "near-vm-errors", "near-vm-runner", "nearcore", @@ -3929,6 +3930,21 @@ dependencies = [ "wat", ] +[[package]] +name = "near-undo-block" +version = "0.0.0" +dependencies = [ + "anyhow", + "chrono", + "clap 3.1.18", + "near-chain", + "near-chain-configs", + "near-primitives", + "near-store", + "nearcore", + "tracing", +] + [[package]] name = "near-vm" version = "0.0.0" @@ -4276,6 +4292,7 @@ dependencies = [ "near-primitives", "near-state-parts", "near-store", + "near-undo-block", "nearcore", "once_cell", "openssl-probe", diff --git a/Cargo.toml b/Cargo.toml index f36cd73f038..5c4657da11b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -67,6 +67,7 @@ members = [ "tools/state-viewer", "tools/storage-usage-delta-calculator", "tools/themis", + "tools/undo-block", "utils/config", "utils/fmt", "utils/mainnet-res", @@ -204,6 +205,7 @@ near-state-viewer = { path = "tools/state-viewer", package = "state-viewer" } near-store = { path = "core/store" } near-telemetry = { path = "chain/telemetry" } near-test-contracts = { path = "runtime/near-test-contracts" } +near-undo-block = { path = "tools/undo-block" } near-vm-compiler = { path = "runtime/near-vm/lib/compiler"} near-vm-compiler-singlepass = { path = "runtime/near-vm/lib/compiler-singlepass" } near-vm-engine = { path = "runtime/near-vm/lib/engine" } diff --git a/chain/chain/src/store.rs b/chain/chain/src/store.rs index 22ec2b5d0ea..c382a0d48d9 100644 --- a/chain/chain/src/store.rs +++ b/chain/chain/src/store.rs @@ -47,6 +47,7 @@ use crate::chunks_store::ReadOnlyChunksStore; use crate::types::{Block, BlockHeader, LatestKnown}; use crate::{byzantine_assert, RuntimeWithEpochManagerAdapter}; use near_store::db::StoreStatistics; +use near_store::flat::store_helper; use std::sync::Arc; /// lru cache size @@ -1956,6 +1957,27 @@ impl<'a> ChainStoreUpdate<'a> { self.chunk_tail = Some(height); } + fn clear_header_data_for_heights( + &mut self, + start: BlockHeight, + end: BlockHeight, + ) -> Result<(), Error> { + for height in start..=end { + let header_hashes = self.chain_store.get_all_header_hashes_by_height(height)?; + for header_hash in header_hashes { + // Delete header_hash-indexed data: block header + let mut store_update = self.store().store_update(); + let key: &[u8] = header_hash.as_bytes(); + store_update.delete(DBCol::BlockHeader, key); + self.chain_store.headers.pop(key); + self.merge(store_update); + } + let key = index_to_bytes(height); + self.gc_col(DBCol::HeaderHashesByHeight, &key); + } + Ok(()) + } + pub fn clear_chunk_data_and_headers( &mut self, min_chunk_height: BlockHeight, @@ -2205,6 +2227,118 @@ impl<'a> ChainStoreUpdate<'a> { Ok(()) } + // Delete all data in rocksdb that are partially or wholly indexed and can be looked up by hash of the current head of the chain + // and that indicates a link between current head and its prev block + pub fn clear_head_block_data( + &mut self, + runtime_adapter: &dyn RuntimeWithEpochManagerAdapter, + ) -> Result<(), Error> { + let header_head = self.header_head().unwrap(); + let header_head_height = header_head.height; + let block_hash = self.head().unwrap().last_block_hash; + + let block = + self.get_block(&block_hash).expect("block data is not expected to be already cleaned"); + + let epoch_id = block.header().epoch_id(); + + let head_height = block.header().height(); + + // 1. Delete shard_id-indexed data (TrieChanges, Receipts, ChunkExtra, State Headers and Parts, FlatStorage data) + for shard_id in 0..block.header().chunk_mask().len() as ShardId { + let shard_uid = runtime_adapter.shard_id_to_uid(shard_id, epoch_id).unwrap(); + let block_shard_id = get_block_shard_uid(&block_hash, &shard_uid); + + // delete TrieChanges + self.gc_col(DBCol::TrieChanges, &block_shard_id); + + // delete Receipts + self.gc_outgoing_receipts(&block_hash, shard_id); + self.gc_col(DBCol::IncomingReceipts, &block_shard_id); + + // delete DBCol::ChunkExtra based on shard_uid since it's indexed by shard_uid in the storage + self.gc_col(DBCol::ChunkExtra, &block_shard_id); + + // delete state parts and state headers + if let Ok(shard_state_header) = self.chain_store.get_state_header(shard_id, block_hash) + { + let state_num_parts = + get_num_state_parts(shard_state_header.state_root_node().memory_usage); + self.gc_col_state_parts(block_hash, shard_id, state_num_parts)?; + let state_header_key = StateHeaderKey(shard_id, block_hash).try_to_vec()?; + self.gc_col(DBCol::StateHeaders, &state_header_key); + } + + // delete flat storage columns: FlatStateChanges and FlatStateDeltaMetadata + if cfg!(feature = "protocol_feature_flat_state") { + let mut store_update = self.store().store_update(); + store_helper::remove_delta(&mut store_update, shard_uid, block_hash); + self.merge(store_update); + } + } + + // 2. Delete block_hash-indexed data + self.gc_col(DBCol::Block, block_hash.as_bytes()); + self.gc_col(DBCol::BlockExtra, block_hash.as_bytes()); + self.gc_col(DBCol::NextBlockHashes, block_hash.as_bytes()); + self.gc_col(DBCol::ChallengedBlocks, block_hash.as_bytes()); + self.gc_col(DBCol::BlocksToCatchup, block_hash.as_bytes()); + let storage_key = KeyForStateChanges::for_block(&block_hash); + let stored_state_changes: Vec> = self + .chain_store + .store() + .iter_prefix(DBCol::StateChanges, storage_key.as_ref()) + .map(|item| item.map(|(key, _)| key)) + .collect::>>()?; + for key in stored_state_changes { + self.gc_col(DBCol::StateChanges, &key); + } + self.gc_col(DBCol::BlockRefCount, block_hash.as_bytes()); + self.gc_outcomes(&block)?; + self.gc_col(DBCol::BlockInfo, block_hash.as_bytes()); + self.gc_col(DBCol::StateDlInfos, block_hash.as_bytes()); + + // 3. update columns related to prev block (block refcount and NextBlockHashes) + self.dec_block_refcount(block.header().prev_hash())?; + self.gc_col(DBCol::NextBlockHashes, block.header().prev_hash().as_bytes()); + + // 4. Update or delete block_hash_per_height + self.gc_col_block_per_height(&block_hash, head_height, block.header().epoch_id())?; + + self.clear_chunk_data_at_height(head_height)?; + + self.clear_header_data_for_heights(head_height, header_head_height)?; + + Ok(()) + } + + fn clear_chunk_data_at_height(&mut self, height: BlockHeight) -> Result<(), Error> { + let chunk_hashes = self.chain_store.get_all_chunk_hashes_by_height(height)?; + for chunk_hash in chunk_hashes { + // 1. Delete chunk-related data + let chunk = self.get_chunk(&chunk_hash)?.clone(); + debug_assert_eq!(chunk.cloned_header().height_created(), height); + for transaction in chunk.transactions() { + self.gc_col(DBCol::Transactions, transaction.get_hash().as_bytes()); + } + for receipt in chunk.receipts() { + self.gc_col(DBCol::Receipts, receipt.get_hash().as_bytes()); + } + + // 2. Delete chunk_hash-indexed data + let chunk_hash = chunk_hash.as_bytes(); + self.gc_col(DBCol::Chunks, chunk_hash); + self.gc_col(DBCol::PartialChunks, chunk_hash); + self.gc_col(DBCol::InvalidChunks, chunk_hash); + } + + // 4. Delete chunk hashes per height + let key = index_to_bytes(height); + self.gc_col(DBCol::ChunkHashesByHeight, &key); + + Ok(()) + } + pub fn gc_col_block_per_height( &mut self, block_hash: &CryptoHash, diff --git a/integration-tests/Cargo.toml b/integration-tests/Cargo.toml index 5f525495c5e..ba18788be29 100644 --- a/integration-tests/Cargo.toml +++ b/integration-tests/Cargo.toml @@ -49,6 +49,7 @@ near-o11y.workspace = true near-telemetry.workspace = true near-test-contracts.workspace = true near-performance-metrics.workspace = true +near-undo-block.workspace = true near-vm-errors.workspace = true near-vm-runner.workspace = true nearcore.workspace = true diff --git a/integration-tests/src/tests/client/mod.rs b/integration-tests/src/tests/client/mod.rs index 194be7aac8c..3bb5464d08c 100644 --- a/integration-tests/src/tests/client/mod.rs +++ b/integration-tests/src/tests/client/mod.rs @@ -9,3 +9,4 @@ mod runtimes; #[cfg(feature = "sandbox")] mod sandbox; mod sharding_upgrade; +mod undo_block; diff --git a/integration-tests/src/tests/client/undo_block.rs b/integration-tests/src/tests/client/undo_block.rs new file mode 100644 index 00000000000..754df5db81e --- /dev/null +++ b/integration-tests/src/tests/client/undo_block.rs @@ -0,0 +1,77 @@ +use near_chain::{ + ChainGenesis, ChainStore, ChainStoreAccess, Provenance, RuntimeWithEpochManagerAdapter, +}; +use near_chain_configs::Genesis; +use near_client::test_utils::TestEnv; +use near_o11y::testonly::init_test_logger; +use near_store::test_utils::create_test_store; +use near_store::Store; +use near_undo_block::undo_block; +use nearcore::config::GenesisExt; +use std::path::Path; +use std::sync::Arc; + +/// Setup environment with one Near client for testing. +fn setup_env( + genesis: &Genesis, + store: Store, +) -> (TestEnv, Arc) { + let chain_genesis = ChainGenesis::new(genesis); + let runtime: Arc = + nearcore::NightshadeRuntime::test(Path::new("../../../.."), store, genesis); + (TestEnv::builder(chain_genesis).runtime_adapters(vec![runtime.clone()]).build(), runtime) +} + +// Checks that Near client can successfully undo block on given height and then produce and process block normally after restart +fn test_undo_block(epoch_length: u64, stop_height: u64) { + init_test_logger(); + + let save_trie_changes = true; + + let mut genesis = Genesis::test(vec!["test0".parse().unwrap(), "test1".parse().unwrap()], 1); + genesis.config.epoch_length = epoch_length; + + let store = create_test_store(); + let (mut env, runtime) = setup_env(&genesis, store.clone()); + + for i in 1..=stop_height { + let block = env.clients[0].produce_block(i).unwrap().unwrap(); + env.process_block(0, block, Provenance::PRODUCED); + } + + let mut chain_store = + ChainStore::new(store.clone(), genesis.config.genesis_height, save_trie_changes); + + let current_head = chain_store.head().unwrap(); + let prev_block_hash = current_head.prev_block_hash; + + undo_block(&mut chain_store, &*runtime).unwrap(); + + // after undo, the current head should be the prev_block_hash + assert_eq!(chain_store.head().unwrap().last_block_hash.as_bytes(), prev_block_hash.as_bytes()); + assert_eq!(chain_store.head().unwrap().height, stop_height - 1); + + // set up an environment again with the same store + let (mut env, _) = setup_env(&genesis, store.clone()); + // the new env should be able to produce block normally + let block = env.clients[0].produce_block(stop_height).unwrap().unwrap(); + env.process_block(0, block, Provenance::PRODUCED); + + // after processing the new block, the head should now be at stop_height + assert_eq!(chain_store.head().unwrap().height, stop_height); +} + +#[test] +fn test_undo_block_middle_of_epoch() { + test_undo_block(5, 3) +} + +#[test] +fn test_undo_block_end_of_epoch() { + test_undo_block(5, 5) +} + +#[test] +fn test_undo_block_start_of_epoch() { + test_undo_block(5, 6) +} diff --git a/neard/Cargo.toml b/neard/Cargo.toml index 0040ada6305..5d071dc4c7d 100644 --- a/neard/Cargo.toml +++ b/neard/Cargo.toml @@ -48,6 +48,7 @@ near-primitives.workspace = true near-state-parts.workspace = true near-state-viewer.workspace = true near-store.workspace = true +near-undo-block.workspace = true [build-dependencies] anyhow.workspace = true diff --git a/neard/src/cli.rs b/neard/src/cli.rs index db1b4c73559..0c1ebbe3b22 100644 --- a/neard/src/cli.rs +++ b/neard/src/cli.rs @@ -22,6 +22,7 @@ use near_state_parts::cli::StatePartsCommand; use near_state_viewer::StateViewerSubCommand; use near_store::db::RocksDB; use near_store::Mode; +use near_undo_block::cli::UndoBlockCommand; use serde_json::Value; use std::fs::File; use std::io::BufReader; @@ -123,6 +124,9 @@ impl NeardCmd { NeardSubCommand::ValidateConfig(cmd) => { cmd.run(&home_dir)?; } + NeardSubCommand::UndoBlock(cmd) => { + cmd.run(&home_dir, genesis_validation)?; + } }; Ok(()) } @@ -239,6 +243,9 @@ pub(super) enum NeardSubCommand { /// validate config files including genesis.json and config.json ValidateConfig(ValidateConfigCommand), + + // reset the head of the chain locally to the prev block of current head + UndoBlock(UndoBlockCommand), } #[derive(clap::Parser)] diff --git a/tools/undo-block/Cargo.toml b/tools/undo-block/Cargo.toml new file mode 100644 index 00000000000..b29cffb9b2e --- /dev/null +++ b/tools/undo-block/Cargo.toml @@ -0,0 +1,18 @@ +[package] +name = "near-undo-block" +version = "0.0.0" +authors.workspace = true +publish = false +edition.workspace = true + +[dependencies] +anyhow.workspace = true +clap.workspace = true +tracing.workspace = true +chrono.workspace = true + +near-chain.workspace = true +near-chain-configs.workspace = true +near-store.workspace = true +nearcore.workspace = true +near-primitives.workspace = true diff --git a/tools/undo-block/src/cli.rs b/tools/undo-block/src/cli.rs new file mode 100644 index 00000000000..5b7abe5f016 --- /dev/null +++ b/tools/undo-block/src/cli.rs @@ -0,0 +1,40 @@ +use near_chain::ChainStore; +use near_chain_configs::GenesisValidationMode; +use near_store::{Mode, NodeStorage}; +use nearcore::load_config; +use nearcore::NightshadeRuntime; +use std::path::Path; + +#[derive(clap::Parser)] +pub struct UndoBlockCommand {} + +impl UndoBlockCommand { + pub fn run( + self, + home_dir: &Path, + genesis_validation: GenesisValidationMode, + ) -> anyhow::Result<()> { + let near_config = load_config(home_dir, genesis_validation) + .unwrap_or_else(|e| panic!("Error loading config: {:#}", e)); + + let store_opener = NodeStorage::opener( + home_dir, + near_config.config.archive, + &near_config.config.store, + None, + ); + + let storage = store_opener.open_in_mode(Mode::ReadWrite).unwrap(); + let store = storage.get_hot_store(); + + let runtime = NightshadeRuntime::from_config(home_dir, store.clone(), &near_config); + + let mut chain_store = ChainStore::new( + store, + near_config.genesis.config.genesis_height, + near_config.client_config.save_trie_changes, + ); + + crate::undo_block(&mut chain_store, &*runtime) + } +} diff --git a/tools/undo-block/src/lib.rs b/tools/undo-block/src/lib.rs new file mode 100644 index 00000000000..c6429dc95cf --- /dev/null +++ b/tools/undo-block/src/lib.rs @@ -0,0 +1,49 @@ +use chrono::Utc; +use near_chain::types::LatestKnown; +use near_chain::RuntimeWithEpochManagerAdapter; +use near_chain::{ChainStore, ChainStoreAccess, ChainStoreUpdate}; +use near_primitives::block::Tip; +use near_primitives::utils::to_timestamp; + +pub mod cli; + +pub fn undo_block( + chain_store: &mut ChainStore, + runtime: &dyn RuntimeWithEpochManagerAdapter, +) -> anyhow::Result<()> { + let current_head = chain_store.head()?; + let current_head_hash = current_head.last_block_hash; + let prev_block_hash = current_head.prev_block_hash; + let prev_header = chain_store.get_block_header(&prev_block_hash)?; + let prev_tip = Tip::from_header(&prev_header); + let current_head_height = current_head.height; + let prev_block_height = prev_tip.height; + + tracing::info!(target: "neard", ?prev_block_hash, ?current_head_hash, ?prev_block_height, ?current_head_height, "Trying to update head"); + + // stop if it's already the final block + if chain_store.final_head()?.height >= current_head.height { + return Err(anyhow::anyhow!("Cannot revert past final block")); + } + + let mut chain_store_update = ChainStoreUpdate::new(chain_store); + + chain_store_update.clear_head_block_data(runtime)?; + + chain_store_update.save_head(&prev_tip)?; + + chain_store_update.commit()?; + + chain_store.save_latest_known(LatestKnown { + height: prev_tip.height, + seen: to_timestamp(Utc::now()), + })?; + + let new_chain_store_head = chain_store.head()?; + let new_chain_store_header_head = chain_store.header_head()?; + let new_head_height = new_chain_store_head.height; + let new_header_height = new_chain_store_header_head.height; + + tracing::info!(target: "neard", ?new_head_height, ?new_header_height, "The current chain store shows"); + Ok(()) +}