From 1ab5d1bbefe00e16da45121d8b35379afe09dc85 Mon Sep 17 00:00:00 2001 From: Anton Puhach Date: Wed, 28 Feb 2024 18:54:55 +0100 Subject: [PATCH] refactor: move NightshadeRuntime to chain (#10675) This PR is a part of getting rid of `KeyValueRuntime` (#10678) and `MockEpochManager` (#10634). Currently `NightshadeRuntime` is part of `nearcore` which makes it impossible to use it in `chain` since `chain` depends on `nearcore`. We need to use `NightshadeRuntime` as a replacement for `KeyValueRuntime` in `chain` tests. Ideally it would be great to move it to `node-runtime` crate, but currently it has a lot of dependencies from `near_chain` which makes it very challenging, so as the first step let's move it to `chain`. This PR includes: * move `nearcore::runtime` to `near_chain::runtime` * introduce `NightshadeRuntimeExt` to expose `NightshadeRuntime::from_config` separately since `NearConfig` is part of `nearcore` * move `GenesisExt` test methods directly to `Genesis`, also duplicate some constants for now (will be refactored separately to reduce size of this PR). * move the necessary migrations code to `chain` --- Cargo.lock | 8 + chain/chain/Cargo.toml | 27 ++- chain/chain/src/lib.rs | 1 + .../chain}/src/runtime/errors.rs | 2 +- chain/chain/src/runtime/metrics.rs | 67 ++++++ chain/chain/src/runtime/migrations.rs | 62 ++++++ {nearcore => chain/chain}/src/runtime/mod.rs | 95 +++----- .../chain}/src/runtime/tests.rs | 14 +- core/chain-configs/src/test_utils.rs | 210 +++++++++++++++++- genesis-tools/genesis-populate/src/lib.rs | 2 +- integration-tests/src/nearcore_utils.rs | 2 +- integration-tests/src/node/mod.rs | 4 +- integration-tests/src/node/runtime_node.rs | 1 - integration-tests/src/runtime_utils.rs | 1 - .../src/tests/client/benchmarks.rs | 1 - .../src/tests/client/block_corruption.rs | 1 - .../src/tests/client/challenges.rs | 1 - .../src/tests/client/cold_storage.rs | 1 - .../src/tests/client/epoch_sync.rs | 1 - .../access_key_nonce_for_implicit_accounts.rs | 1 - .../account_id_in_function_call_permission.rs | 1 - .../client/features/adversarial_behaviors.rs | 1 - .../client/features/chunk_nodes_cache.rs | 1 - .../tests/client/features/delegate_action.rs | 1 - .../features/fix_contract_loading_cost.rs | 1 - .../client/features/fix_storage_usage.rs | 1 - .../src/tests/client/features/flat_storage.rs | 1 - .../features/increase_deployment_cost.rs | 1 - .../features/increase_storage_compute_cost.rs | 1 - .../limit_contract_functions_number.rs | 1 - .../features/lower_storage_key_limit.rs | 1 - .../src/tests/client/features/nearvm.rs | 1 - .../client/features/nonrefundable_transfer.rs | 1 - .../features/orphan_chunk_state_witness.rs | 1 - ...restore_receipts_after_fix_apply_chunks.rs | 3 +- .../src/tests/client/features/restrict_tla.rs | 1 - .../client/features/stateless_validation.rs | 1 - .../tests/client/features/wallet_contract.rs | 2 +- .../client/features/zero_balance_account.rs | 1 - .../src/tests/client/flat_storage.rs | 1 - .../src/tests/client/process_blocks.rs | 2 +- .../src/tests/client/resharding.rs | 1 - .../src/tests/client/runtimes.rs | 1 - integration-tests/src/tests/client/sandbox.rs | 1 - .../src/tests/client/state_dump.rs | 1 - .../src/tests/client/state_snapshot.rs | 1 - .../src/tests/client/sync_state_nodes.rs | 2 +- .../src/tests/client/undo_block.rs | 1 - .../src/tests/nearcore/node_cluster.rs | 2 +- .../src/tests/nearcore/stake_nodes.rs | 2 +- .../src/tests/nearcore/sync_nodes.rs | 2 +- .../src/tests/runtime/deployment.rs | 1 - .../src/tests/runtime/sanity_checks.rs | 1 - .../src/tests/standard_cases/runtime.rs | 2 +- integration-tests/src/tests/test_errors.rs | 2 +- nearcore/benches/store.rs | 2 +- nearcore/src/config.rs | 155 +++++-------- nearcore/src/lib.rs | 4 +- nearcore/src/metrics.rs | 67 +----- nearcore/src/migrations.rs | 62 ------ nearcore/tests/economics.rs | 2 +- test-utils/runtime-tester/src/run_test.rs | 2 +- test-utils/store-validator/src/main.rs | 2 +- tools/epoch-sync/src/cli.rs | 2 +- tools/flat-storage/src/commands.rs | 2 +- tools/fork-network/src/cli.rs | 4 +- tools/mirror/src/offline.rs | 2 +- tools/mock-node/src/setup.rs | 3 +- tools/speedy_sync/src/main.rs | 2 +- tools/state-viewer/src/apply_chain_range.rs | 1 - tools/state-viewer/src/apply_chunk.rs | 1 - tools/state-viewer/src/commands.rs | 2 +- tools/state-viewer/src/state_changes.rs | 2 +- tools/state-viewer/src/state_dump.rs | 1 - tools/state-viewer/src/state_parts.rs | 2 +- 75 files changed, 489 insertions(+), 378 deletions(-) rename {nearcore => chain/chain}/src/runtime/errors.rs (98%) create mode 100644 chain/chain/src/runtime/metrics.rs create mode 100644 chain/chain/src/runtime/migrations.rs rename {nearcore => chain/chain}/src/runtime/mod.rs (94%) rename {nearcore => chain/chain}/src/runtime/tests.rs (99%) diff --git a/Cargo.lock b/Cargo.lock index 24c65e01c83..a013b903252 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3459,6 +3459,7 @@ dependencies = [ "bytesize", "chrono", "crossbeam-channel", + "easy-ext", "enum-map", "insta", "itertools", @@ -3471,19 +3472,26 @@ dependencies = [ "near-client-primitives", "near-crypto", "near-epoch-manager", + "near-mainnet-res", "near-network", "near-o11y", + "near-parameters", "near-performance-metrics", "near-performance-metrics-macros", "near-pool", "near-primitives", "near-store", + "near-vm-runner", + "node-runtime", "num-rational", "once_cell", + "primitive-types", "rand", "rand_chacha", "rayon", + "serde_json", "strum", + "tempfile", "thiserror", "tracing", "yansi", diff --git a/chain/chain/Cargo.toml b/chain/chain/Cargo.toml index 7fa93215d78..8dfdcb831d2 100644 --- a/chain/chain/Cargo.toml +++ b/chain/chain/Cargo.toml @@ -28,6 +28,7 @@ strum.workspace = true thiserror.workspace = true tracing.workspace = true yansi.workspace = true +easy-ext.workspace = true near-async.workspace = true near-cache.workspace = true @@ -43,8 +44,15 @@ near-performance-metrics-macros.workspace = true near-pool.workspace = true near-primitives.workspace = true near-store.workspace = true +node-runtime.workspace = true +near-parameters.workspace = true +near-vm-runner.workspace = true +near-mainnet-res.workspace = true [dev-dependencies] +serde_json.workspace = true +primitive-types.workspace = true +tempfile.workspace = true insta.workspace = true assert_matches.workspace = true @@ -55,7 +63,12 @@ expensive_tests = [] test_features = [] shadow_chunk_validation = [] no_cache = ["near-store/no_cache"] -new_epoch_sync = ["near-store/new_epoch_sync", "near-primitives/new_epoch_sync", "near-epoch-manager/new_epoch_sync", "near-chain-primitives/new_epoch_sync"] +new_epoch_sync = [ + "near-store/new_epoch_sync", + "near-primitives/new_epoch_sync", + "near-epoch-manager/new_epoch_sync", + "near-chain-primitives/new_epoch_sync", +] protocol_feature_reject_blocks_with_outdated_protocol_version = [ "near-primitives/protocol_feature_reject_blocks_with_outdated_protocol_version", @@ -66,12 +79,16 @@ nightly = [ "near-chain-configs/nightly", "near-client-primitives/nightly", "near-epoch-manager/nightly", + "near-mainnet-res/nightly", "near-network/nightly", "near-o11y/nightly", + "near-parameters/nightly", "near-pool/nightly", "near-primitives/nightly", "near-store/nightly", + "near-vm-runner/nightly", "nightly_protocol", + "node-runtime/nightly", "protocol_feature_reject_blocks_with_outdated_protocol_version", ] nightly_protocol = [ @@ -79,13 +96,15 @@ nightly_protocol = [ "near-chain-configs/nightly_protocol", "near-client-primitives/nightly_protocol", "near-epoch-manager/nightly_protocol", + "near-mainnet-res/nightly_protocol", "near-network/nightly_protocol", "near-o11y/nightly_protocol", + "near-parameters/nightly_protocol", "near-pool/nightly_protocol", "near-primitives/nightly_protocol", "near-store/nightly_protocol", + "near-vm-runner/nightly_protocol", + "node-runtime/nightly_protocol", ] -statelessnet_protocol = [ - "near-primitives/statelessnet_protocol", -] +statelessnet_protocol = ["near-primitives/statelessnet_protocol"] sandbox = ["near-primitives/sandbox"] diff --git a/chain/chain/src/lib.rs b/chain/chain/src/lib.rs index 6b1404aedc4..52babc8fdc4 100644 --- a/chain/chain/src/lib.rs +++ b/chain/chain/src/lib.rs @@ -24,6 +24,7 @@ pub mod migrations; pub mod missing_chunks; pub mod orphan; pub mod resharding; +pub mod runtime; mod state_request_tracker; pub mod state_snapshot_actor; mod stateless_validation; diff --git a/nearcore/src/runtime/errors.rs b/chain/chain/src/runtime/errors.rs similarity index 98% rename from nearcore/src/runtime/errors.rs rename to chain/chain/src/runtime/errors.rs index 6d36bb75c4d..34321a1e468 100644 --- a/nearcore/src/runtime/errors.rs +++ b/chain/chain/src/runtime/errors.rs @@ -1,4 +1,4 @@ -use near_chain::near_chain_primitives::error::QueryError; +use crate::near_chain_primitives::error::QueryError; #[easy_ext::ext(FromStateViewerErrors)] impl QueryError { diff --git a/chain/chain/src/runtime/metrics.rs b/chain/chain/src/runtime/metrics.rs new file mode 100644 index 00000000000..6dbf49b08a7 --- /dev/null +++ b/chain/chain/src/runtime/metrics.rs @@ -0,0 +1,67 @@ +use near_o11y::metrics::{ + exponential_buckets, linear_buckets, processing_time_buckets, try_create_histogram_vec, + try_create_int_gauge_vec, HistogramVec, IntGaugeVec, +}; + +use once_cell::sync::Lazy; + +pub(crate) static APPLY_CHUNK_DELAY: Lazy = Lazy::new(|| { + try_create_histogram_vec( + "near_apply_chunk_delay_seconds", + "Time to process a chunk. Gas used by the chunk is a metric label, rounded up to 100 teragas.", + &["tgas_ceiling"], + Some(linear_buckets(0.0, 0.05, 50).unwrap()), + ) + .unwrap() +}); + +pub(crate) static DELAYED_RECEIPTS_COUNT: Lazy = Lazy::new(|| { + try_create_int_gauge_vec( + "near_delayed_receipts_count", + "The count of the delayed receipts. Indicator of congestion.", + &["shard_id"], + ) + .unwrap() +}); + +pub(crate) static PREPARE_TX_SIZE: Lazy = Lazy::new(|| { + try_create_histogram_vec( + "near_prepare_tx_size", + "Sum of transaction sizes per produced chunk, as a histogram", + &["shard_id"], + // Maximum is < 14MB, typical values are unknown right now so buckets + // might need to be adjusted later when we have collected data + Some(vec![1_000.0, 10_000., 100_000., 500_000., 1e6, 2e6, 4e6, 8e6, 12e6]), + ) + .unwrap() +}); + +pub static APPLYING_CHUNKS_TIME: Lazy = Lazy::new(|| { + try_create_histogram_vec( + "near_applying_chunks_time", + "Time taken to apply chunks per shard", + &["shard_id"], + Some(processing_time_buckets()), + ) + .unwrap() +}); + +pub(crate) static STATE_SYNC_OBTAIN_PART_DELAY: Lazy = Lazy::new(|| { + try_create_histogram_vec( + "near_state_sync_obtain_part_delay_sec", + "Latency of applying a state part", + &["shard_id", "result"], + Some(exponential_buckets(0.001, 2.0, 20).unwrap()), + ) + .unwrap() +}); + +pub(crate) static STATE_SYNC_APPLY_PART_DELAY: Lazy = Lazy::new(|| { + try_create_histogram_vec( + "near_state_sync_apply_part_delay_sec", + "Latency of applying a state part", + &["shard_id"], + Some(exponential_buckets(0.001, 2.0, 20).unwrap()), + ) + .unwrap() +}); diff --git a/chain/chain/src/runtime/migrations.rs b/chain/chain/src/runtime/migrations.rs new file mode 100644 index 00000000000..c2855ac3a59 --- /dev/null +++ b/chain/chain/src/runtime/migrations.rs @@ -0,0 +1,62 @@ +use near_primitives::receipt::ReceiptResult; +use near_primitives::runtime::migration_data::MigrationData; +use near_primitives::types::Gas; + +/// In test runs reads and writes here used 442 TGas, but in test on live net migration take +/// between 4 and 4.5s. We do not want to process any receipts in this block +const GAS_USED_FOR_STORAGE_USAGE_DELTA_MIGRATION: Gas = 1_000_000_000_000_000; + +pub fn load_migration_data(chain_id: &str) -> MigrationData { + let is_mainnet = chain_id == near_primitives::chains::MAINNET; + MigrationData { + storage_usage_delta: if is_mainnet { + near_mainnet_res::mainnet_storage_usage_delta() + } else { + Vec::new() + }, + storage_usage_fix_gas: if is_mainnet { + GAS_USED_FOR_STORAGE_USAGE_DELTA_MIGRATION + } else { + 0 + }, + restored_receipts: if is_mainnet { + near_mainnet_res::mainnet_restored_receipts() + } else { + ReceiptResult::default() + }, + } +} + +#[cfg(test)] +mod tests { + use super::*; + use near_mainnet_res::mainnet_restored_receipts; + use near_mainnet_res::mainnet_storage_usage_delta; + use near_primitives::hash::hash; + + #[test] + fn test_migration_data() { + assert_eq!( + hash(serde_json::to_string(&mainnet_storage_usage_delta()).unwrap().as_bytes()) + .to_string(), + "2fEgaLFBBJZqgLQEvHPsck4NS3sFzsgyKaMDqTw5HVvQ" + ); + let mainnet_migration_data = load_migration_data(near_primitives::chains::MAINNET); + assert_eq!(mainnet_migration_data.storage_usage_delta.len(), 3112); + let testnet_migration_data = load_migration_data(near_primitives::chains::TESTNET); + assert_eq!(testnet_migration_data.storage_usage_delta.len(), 0); + } + + #[test] + fn test_restored_receipts_data() { + assert_eq!( + hash(serde_json::to_string(&mainnet_restored_receipts()).unwrap().as_bytes()) + .to_string(), + "48ZMJukN7RzvyJSW9MJ5XmyQkQFfjy2ZxPRaDMMHqUcT" + ); + let mainnet_migration_data = load_migration_data(near_primitives::chains::MAINNET); + assert_eq!(mainnet_migration_data.restored_receipts.get(&0u64).unwrap().len(), 383); + let testnet_migration_data = load_migration_data(near_primitives::chains::TESTNET); + assert!(testnet_migration_data.restored_receipts.is_empty()); + } +} diff --git a/nearcore/src/runtime/mod.rs b/chain/chain/src/runtime/mod.rs similarity index 94% rename from nearcore/src/runtime/mod.rs rename to chain/chain/src/runtime/mod.rs index 6d91d532822..6b2cba8cb45 100644 --- a/nearcore/src/runtime/mod.rs +++ b/chain/chain/src/runtime/mod.rs @@ -1,15 +1,11 @@ -use crate::metrics; -use crate::migrations::load_migration_data; -use crate::NearConfig; - -use borsh::BorshDeserialize; -use errors::FromStateViewerErrors; -use near_chain::types::{ +use crate::types::{ ApplyChunkBlockContext, ApplyChunkResult, ApplyChunkShardContext, ApplyResultForResharding, PrepareTransactionsBlockContext, PrepareTransactionsChunkContext, PrepareTransactionsLimit, PreparedTransactions, RuntimeAdapter, RuntimeStorageConfig, StorageDataSource, Tip, }; -use near_chain::Error; +use crate::Error; +use borsh::BorshDeserialize; +use errors::FromStateViewerErrors; use near_chain_configs::{ GenesisConfig, ProtocolConfig, DEFAULT_GC_NUM_EPOCHS_TO_KEEP, MIN_GC_NUM_EPOCHS_TO_KEEP, }; @@ -62,6 +58,8 @@ use std::time::Instant; use tracing::{debug, error, info}; pub mod errors; +mod metrics; +pub mod migrations; #[cfg(test)] mod tests; @@ -81,47 +79,7 @@ pub struct NightshadeRuntime { } impl NightshadeRuntime { - pub fn from_config( - home_dir: &Path, - store: Store, - config: &NearConfig, - epoch_manager: Arc, - ) -> Arc { - // TODO (#9989): directly use the new state snapshot config once the migration is done. - let mut state_snapshot_type = - config.config.store.state_snapshot_config.state_snapshot_type.clone(); - if config.config.store.state_snapshot_enabled { - state_snapshot_type = StateSnapshotType::EveryEpoch; - } - // TODO (#9989): directly use the new state snapshot config once the migration is done. - let compaction_enabled = config.config.store.state_snapshot_compaction_enabled - || config.config.store.state_snapshot_config.compaction_enabled; - let state_snapshot_config = StateSnapshotConfig { - state_snapshot_type, - home_dir: home_dir.to_path_buf(), - hot_store_path: config - .config - .store - .path - .clone() - .unwrap_or_else(|| PathBuf::from("data")), - state_snapshot_subdir: PathBuf::from("state_snapshot"), - compaction_enabled, - }; - Self::new( - store, - &config.genesis.config, - epoch_manager, - config.client_config.trie_viewer_state_size_limit, - config.client_config.max_gas_burnt_view, - None, - config.config.gc.gc_num_epochs_to_keep(), - TrieConfig::from_store_config(&config.config.store), - state_snapshot_config, - ) - } - - fn new( + pub fn new( store: Store, genesis_config: &GenesisConfig, epoch_manager: Arc, @@ -157,7 +115,7 @@ impl NightshadeRuntime { tracing::error!(target: "runtime", ?err, "Failed to check if a state snapshot exists"); } - let migration_data = Arc::new(load_migration_data(&genesis_config.chain_id)); + let migration_data = Arc::new(migrations::load_migration_data(&genesis_config.chain_id)); Arc::new(NightshadeRuntime { genesis_config: genesis_config.clone(), runtime_config_store, @@ -935,18 +893,17 @@ impl RuntimeAdapter for NightshadeRuntime { block_hash: &CryptoHash, epoch_id: &EpochId, request: &QueryRequest, - ) -> Result { + ) -> Result { match request { QueryRequest::ViewAccount { account_id } => { - let account = self - .view_account(&shard_uid, *state_root, account_id) - .map_err(|err| { - near_chain::near_chain_primitives::error::QueryError::from_view_account_error( - err, - block_height, - *block_hash, - ) - })?; + let account = + self.view_account(&shard_uid, *state_root, account_id).map_err(|err| { + crate::near_chain_primitives::error::QueryError::from_view_account_error( + err, + block_height, + *block_hash, + ) + })?; Ok(QueryResponse { kind: QueryResponseKind::ViewAccount(account.into()), block_height, @@ -956,7 +913,7 @@ impl RuntimeAdapter for NightshadeRuntime { QueryRequest::ViewCode { account_id } => { let contract_code = self .view_contract_code(&shard_uid, *state_root, account_id) - .map_err(|err| near_chain::near_chain_primitives::error::QueryError::from_view_contract_code_error(err, block_height, *block_hash))?; + .map_err(|err| crate::near_chain_primitives::error::QueryError::from_view_contract_code_error(err, block_height, *block_hash))?; let hash = *contract_code.hash(); let contract_code_view = ContractCodeView { hash, code: contract_code.into_code() }; Ok(QueryResponse { @@ -970,7 +927,7 @@ impl RuntimeAdapter for NightshadeRuntime { let (epoch_height, current_protocol_version) = { let epoch_manager = self.epoch_manager.read(); let epoch_info = epoch_manager.get_epoch_info(epoch_id).map_err(|err| { - near_chain::near_chain_primitives::error::QueryError::from_epoch_error( + crate::near_chain_primitives::error::QueryError::from_epoch_error( err, block_height, *block_hash, @@ -996,7 +953,13 @@ impl RuntimeAdapter for NightshadeRuntime { self.epoch_manager.as_ref(), current_protocol_version, ) - .map_err(|err| near_chain::near_chain_primitives::error::QueryError::from_call_function_error(err, block_height, *block_hash))?; + .map_err(|err| { + crate::near_chain_primitives::error::QueryError::from_call_function_error( + err, + block_height, + *block_hash, + ) + })?; Ok(QueryResponse { kind: QueryResponseKind::CallResult(CallResult { result: call_function_result, @@ -1016,7 +979,7 @@ impl RuntimeAdapter for NightshadeRuntime { *include_proof, ) .map_err(|err| { - near_chain::near_chain_primitives::error::QueryError::from_view_state_error( + crate::near_chain_primitives::error::QueryError::from_view_state_error( err, block_height, *block_hash, @@ -1031,7 +994,7 @@ impl RuntimeAdapter for NightshadeRuntime { QueryRequest::ViewAccessKeyList { account_id } => { let access_key_list = self.view_access_keys(&shard_uid, *state_root, account_id).map_err(|err| { - near_chain::near_chain_primitives::error::QueryError::from_view_access_key_error( + crate::near_chain_primitives::error::QueryError::from_view_access_key_error( err, block_height, *block_hash, @@ -1055,7 +1018,7 @@ impl RuntimeAdapter for NightshadeRuntime { let access_key = self .view_access_key(&shard_uid, *state_root, account_id, public_key) .map_err(|err| { - near_chain::near_chain_primitives::error::QueryError::from_view_access_key_error( + crate::near_chain_primitives::error::QueryError::from_view_access_key_error( err, block_height, *block_hash, diff --git a/nearcore/src/runtime/tests.rs b/chain/chain/src/runtime/tests.rs similarity index 99% rename from nearcore/src/runtime/tests.rs rename to chain/chain/src/runtime/tests.rs index 1880943dbbd..e89ebd6f9b1 100644 --- a/nearcore/src/runtime/tests.rs +++ b/chain/chain/src/runtime/tests.rs @@ -1,7 +1,8 @@ use std::collections::BTreeSet; -use near_chain::types::{ChainConfig, RuntimeStorageConfig}; -use near_chain::{Chain, ChainGenesis, ChainStoreAccess, DoomslugThresholdMode}; +use crate::types::{ChainConfig, RuntimeStorageConfig}; +use crate::{Chain, ChainGenesis, ChainStoreAccess, DoomslugThresholdMode}; +use near_chain_configs::test_utils::{NEAR_BASE, TESTING_INIT_BALANCE, TESTING_INIT_STAKE}; use near_epoch_manager::shard_tracker::ShardTracker; use near_epoch_manager::types::BlockHeaderInfo; use near_epoch_manager::{EpochManager, RngSeed}; @@ -15,7 +16,6 @@ use near_store::genesis::initialize_genesis_state; use num_rational::Ratio; use rand::{rngs::StdRng, seq::SliceRandom, SeedableRng}; -use crate::config::{GenesisExt, TESTING_INIT_BALANCE, TESTING_INIT_STAKE}; use near_chain_configs::{ default_produce_chunk_add_transactions_time_limit, Genesis, DEFAULT_GC_NUM_EPOCHS_TO_KEEP, }; @@ -421,7 +421,7 @@ fn test_validator_rotation() { &signer, CryptoHash::default(), ); - let test2_stake_amount = 3600 * crate::NEAR_BASE; + let test2_stake_amount = 3600 * NEAR_BASE; let transactions = { // With the new validator selection algorithm, test2 needs to have less stake to // become a fisherman. @@ -1144,7 +1144,7 @@ fn test_fishermen_stake() { .iter() .map(|id| InMemorySigner::from_seed(id.clone(), KeyType::ED25519, id.as_ref())) .collect(); - let fishermen_stake = 3300 * crate::NEAR_BASE + 1; + let fishermen_stake = 3300 * NEAR_BASE + 1; let staking_transaction = stake(1, &signers[0], &block_producers[0], fishermen_stake); let staking_transaction1 = stake(1, &signers[1], &block_producers[1], fishermen_stake); @@ -1218,7 +1218,7 @@ fn test_fishermen_unstake() { .iter() .map(|id| InMemorySigner::from_seed(id.clone(), KeyType::ED25519, id.as_ref())) .collect(); - let fishermen_stake = 3300 * crate::NEAR_BASE + 1; + let fishermen_stake = 3300 * NEAR_BASE + 1; let staking_transaction = stake(1, &signers[0], &block_producers[0], fishermen_stake); env.step_default(vec![staking_transaction]); @@ -1369,7 +1369,7 @@ fn test_insufficient_stake() { .collect(); let staking_transaction1 = stake(1, &signers[1], &block_producers[1], 100); - let staking_transaction2 = stake(2, &signers[1], &block_producers[1], 100 * crate::NEAR_BASE); + let staking_transaction2 = stake(2, &signers[1], &block_producers[1], 100 * NEAR_BASE); env.step_default(vec![staking_transaction1, staking_transaction2]); assert!(env.last_proposals.is_empty()); let staking_transaction3 = stake(3, &signers[1], &block_producers[1], 0); diff --git a/core/chain-configs/src/test_utils.rs b/core/chain-configs/src/test_utils.rs index 9411a092b92..ad5117fb039 100644 --- a/core/chain-configs/src/test_utils.rs +++ b/core/chain-configs/src/test_utils.rs @@ -1,8 +1,17 @@ +use near_crypto::{InMemorySigner, KeyType, PublicKey}; +use near_primitives::account::{AccessKey, Account}; +use near_primitives::hash::CryptoHash; +use near_primitives::shard_layout::ShardLayout; +use near_primitives::state_record::StateRecord; use near_primitives::static_clock::StaticClock; +use near_primitives::types::{ + AccountId, AccountInfo, Balance, BlockHeightDelta, Gas, NumBlocks, NumSeats, NumShards, +}; +use near_primitives::utils::generate_random_string; use near_primitives::version::PROTOCOL_VERSION; -use num_rational::Ratio; +use num_rational::{Ratio, Rational32}; -use crate::GenesisConfig; +use crate::{Genesis, GenesisConfig}; impl GenesisConfig { pub fn test() -> Self { @@ -21,3 +30,200 @@ impl GenesisConfig { } } } + +/// Protocol treasury account +pub const PROTOCOL_TREASURY_ACCOUNT: &str = "near"; + +/// Initial balance used in tests. +pub const TESTING_INIT_BALANCE: Balance = 1_000_000_000 * NEAR_BASE; + +/// Validator's stake used in tests. +pub const TESTING_INIT_STAKE: Balance = 50_000_000 * NEAR_BASE; + +/// One NEAR, divisible by 10^24. +pub const NEAR_BASE: Balance = 1_000_000_000_000_000_000_000_000; + +/// Protocol upgrade stake threshold. +pub const PROTOCOL_UPGRADE_STAKE_THRESHOLD: Rational32 = Rational32::new_raw(4, 5); + +pub const FAST_EPOCH_LENGTH: BlockHeightDelta = 60; + +/// Initial gas limit. +pub const INITIAL_GAS_LIMIT: Gas = 1_000_000_000_000_000; + +/// The rate at which the gas price can be adjusted (alpha in the formula). +/// The formula is +/// gas_price_t = gas_price_{t-1} * (1 + (gas_used/gas_limit - 1/2) * alpha)) +pub const GAS_PRICE_ADJUSTMENT_RATE: Rational32 = Rational32::new_raw(1, 100); + +/// Criterion for kicking out block producers. +pub const BLOCK_PRODUCER_KICKOUT_THRESHOLD: u8 = 90; + +/// Protocol treasury reward +pub const PROTOCOL_REWARD_RATE: Rational32 = Rational32::new_raw(1, 10); + +/// Maximum inflation rate per year +pub const MAX_INFLATION_RATE: Rational32 = Rational32::new_raw(1, 20); + +/// Expected number of blocks per year +pub const NUM_BLOCKS_PER_YEAR: u64 = 365 * 24 * 60 * 60; + +/// Number of blocks for which a given transaction is valid +pub const TRANSACTION_VALIDITY_PERIOD: NumBlocks = 100; + +/// Criterion for kicking out chunk producers. +pub const CHUNK_PRODUCER_KICKOUT_THRESHOLD: u8 = 90; + +/// Fishermen stake threshold. +pub const FISHERMEN_THRESHOLD: Balance = 10 * NEAR_BASE; + +/// Initial and minimum gas price. +pub const MIN_GAS_PRICE: Balance = 100_000_000; + +impl Genesis { + // Creates new genesis with a given set of accounts and shard layout. + // The first num_validator_seats from accounts will be treated as 'validators'. + pub fn test_with_seeds( + accounts: Vec, + num_validator_seats: NumSeats, + num_validator_seats_per_shard: Vec, + shard_layout: ShardLayout, + ) -> Self { + let mut validators = vec![]; + let mut records = vec![]; + for (i, account) in accounts.into_iter().enumerate() { + let signer = + InMemorySigner::from_seed(account.clone(), KeyType::ED25519, account.as_ref()); + let i = i as u64; + if i < num_validator_seats { + validators.push(AccountInfo { + account_id: account.clone(), + public_key: signer.public_key.clone(), + amount: TESTING_INIT_STAKE, + }); + } + add_account_with_key( + &mut records, + account, + &signer.public_key.clone(), + TESTING_INIT_BALANCE - if i < num_validator_seats { TESTING_INIT_STAKE } else { 0 }, + if i < num_validator_seats { TESTING_INIT_STAKE } else { 0 }, + CryptoHash::default(), + ); + } + add_protocol_account(&mut records); + let config = GenesisConfig { + protocol_version: PROTOCOL_VERSION, + genesis_time: StaticClock::utc(), + chain_id: random_chain_id(), + num_block_producer_seats: num_validator_seats, + num_block_producer_seats_per_shard: num_validator_seats_per_shard.clone(), + avg_hidden_validator_seats_per_shard: vec![0; num_validator_seats_per_shard.len()], + dynamic_resharding: false, + protocol_upgrade_stake_threshold: PROTOCOL_UPGRADE_STAKE_THRESHOLD, + epoch_length: FAST_EPOCH_LENGTH, + gas_limit: INITIAL_GAS_LIMIT, + gas_price_adjustment_rate: GAS_PRICE_ADJUSTMENT_RATE, + block_producer_kickout_threshold: BLOCK_PRODUCER_KICKOUT_THRESHOLD, + validators, + protocol_reward_rate: PROTOCOL_REWARD_RATE, + total_supply: get_initial_supply(&records), + max_inflation_rate: MAX_INFLATION_RATE, + num_blocks_per_year: NUM_BLOCKS_PER_YEAR, + protocol_treasury_account: PROTOCOL_TREASURY_ACCOUNT.parse().unwrap(), + transaction_validity_period: TRANSACTION_VALIDITY_PERIOD, + chunk_producer_kickout_threshold: CHUNK_PRODUCER_KICKOUT_THRESHOLD, + fishermen_threshold: FISHERMEN_THRESHOLD, + min_gas_price: MIN_GAS_PRICE, + shard_layout, + ..Default::default() + }; + Genesis::new(config, records.into()).unwrap() + } + + pub fn test(accounts: Vec, num_validator_seats: NumSeats) -> Self { + Self::test_with_seeds( + accounts, + num_validator_seats, + vec![num_validator_seats], + ShardLayout::v0_single_shard(), + ) + } + + pub fn test_sharded( + accounts: Vec, + num_validator_seats: NumSeats, + num_validator_seats_per_shard: Vec, + ) -> Self { + let num_shards = num_validator_seats_per_shard.len() as NumShards; + Self::test_with_seeds( + accounts, + num_validator_seats, + num_validator_seats_per_shard, + ShardLayout::v0(num_shards, 0), + ) + } + + pub fn test_sharded_new_version( + accounts: Vec, + num_validator_seats: NumSeats, + num_validator_seats_per_shard: Vec, + ) -> Self { + let num_shards = num_validator_seats_per_shard.len() as NumShards; + Self::test_with_seeds( + accounts, + num_validator_seats, + num_validator_seats_per_shard, + ShardLayout::v0(num_shards, 1), + ) + } +} + +fn add_protocol_account(records: &mut Vec) { + let signer = InMemorySigner::from_seed( + PROTOCOL_TREASURY_ACCOUNT.parse().unwrap(), + KeyType::ED25519, + PROTOCOL_TREASURY_ACCOUNT, + ); + add_account_with_key( + records, + PROTOCOL_TREASURY_ACCOUNT.parse().unwrap(), + &signer.public_key, + TESTING_INIT_BALANCE, + 0, + CryptoHash::default(), + ); +} + +fn add_account_with_key( + records: &mut Vec, + account_id: AccountId, + public_key: &PublicKey, + amount: u128, + staked: u128, + code_hash: CryptoHash, +) { + records.push(StateRecord::Account { + account_id: account_id.clone(), + account: Account::new(amount, staked, 0, code_hash, 0, PROTOCOL_VERSION), + }); + records.push(StateRecord::AccessKey { + account_id, + public_key: public_key.clone(), + access_key: AccessKey::full_access(), + }); +} + +fn random_chain_id() -> String { + format!("test-chain-{}", generate_random_string(5)) +} + +pub fn get_initial_supply(records: &[StateRecord]) -> Balance { + let mut total_supply = 0; + for record in records { + if let StateRecord::Account { account, .. } = record { + total_supply += account.amount() + account.locked(); + } + } + total_supply +} diff --git a/genesis-tools/genesis-populate/src/lib.rs b/genesis-tools/genesis-populate/src/lib.rs index 3c2d7651152..74e87956548 100644 --- a/genesis-tools/genesis-populate/src/lib.rs +++ b/genesis-tools/genesis-populate/src/lib.rs @@ -22,7 +22,7 @@ use near_store::{ get_account, get_genesis_state_roots, set_access_key, set_account, set_code, Store, TrieUpdate, }; use near_vm_runner::ContractCode; -use nearcore::{NearConfig, NightshadeRuntime}; +use nearcore::{NearConfig, NightshadeRuntime, NightshadeRuntimeExt}; use std::collections::BTreeMap; use std::hash::{Hash, Hasher}; use std::path::{Path, PathBuf}; diff --git a/integration-tests/src/nearcore_utils.rs b/integration-tests/src/nearcore_utils.rs index 9590a09dede..92412eefe5e 100644 --- a/integration-tests/src/nearcore_utils.rs +++ b/integration-tests/src/nearcore_utils.rs @@ -15,7 +15,7 @@ use near_primitives::types::validator_stake::ValidatorStake; use near_primitives::types::{BlockHeightDelta, EpochId}; use near_primitives::validator_signer::ValidatorSigner; use near_primitives::version::PROTOCOL_VERSION; -use nearcore::config::{GenesisExt, TESTING_INIT_STAKE}; +use nearcore::config::TESTING_INIT_STAKE; use nearcore::{load_test_config, NearConfig}; // This assumes that there is no height skipped. Otherwise epoch hash calculation will be wrong. diff --git a/integration-tests/src/node/mod.rs b/integration-tests/src/node/mod.rs index 62c6ab2001a..3b7439323da 100644 --- a/integration-tests/src/node/mod.rs +++ b/integration-tests/src/node/mod.rs @@ -15,9 +15,7 @@ use near_primitives::types::{AccountId, Balance, NumSeats}; use near_primitives::validator_signer::InMemoryValidatorSigner; use near_primitives::views::AccountView; use near_vm_runner::ContractCode; -use nearcore::config::{ - create_testnet_configs, create_testnet_configs_from_seeds, Config, GenesisExt, -}; +use nearcore::config::{create_testnet_configs, create_testnet_configs_from_seeds, Config}; use nearcore::NearConfig; use testlib::runtime_utils::{alice_account, bob_account}; diff --git a/integration-tests/src/node/runtime_node.rs b/integration-tests/src/node/runtime_node.rs index 02f058e62f6..f3f99a32a3f 100644 --- a/integration-tests/src/node/runtime_node.rs +++ b/integration-tests/src/node/runtime_node.rs @@ -4,7 +4,6 @@ use near_chain_configs::Genesis; use near_crypto::{InMemorySigner, KeyType, Signer}; use near_parameters::RuntimeConfig; use near_primitives::types::AccountId; -use nearcore::config::GenesisExt; use testlib::runtime_utils::{add_test_contract, alice_account, bob_account, carol_account}; use crate::node::Node; diff --git a/integration-tests/src/runtime_utils.rs b/integration-tests/src/runtime_utils.rs index f75aa2ca45a..d41932ebac2 100644 --- a/integration-tests/src/runtime_utils.rs +++ b/integration-tests/src/runtime_utils.rs @@ -10,7 +10,6 @@ use near_primitives_core::types::NumShards; use near_store::genesis::GenesisStateApplier; use near_store::test_utils::TestTriesBuilder; use near_store::{ShardTries, TrieUpdate}; -use nearcore::config::GenesisExt; use node_runtime::{state_viewer::TrieViewer, Runtime}; use testlib::runtime_utils::{add_test_contract, alice_account, bob_account}; diff --git a/integration-tests/src/tests/client/benchmarks.rs b/integration-tests/src/tests/client/benchmarks.rs index e32b0b4011f..1d865d4c6ca 100644 --- a/integration-tests/src/tests/client/benchmarks.rs +++ b/integration-tests/src/tests/client/benchmarks.rs @@ -9,7 +9,6 @@ use near_client::test_utils::{create_chunk_on_height, TestEnv}; use near_client::{ProcessTxResponse, ProduceChunkResult}; use near_crypto::{InMemorySigner, KeyType}; use near_primitives::transaction::{Action, DeployContractAction, SignedTransaction}; -use nearcore::config::GenesisExt; use nearcore::test_utils::TestEnvNightshadeSetupExt; /// How long does it take to produce a large chunk? diff --git a/integration-tests/src/tests/client/block_corruption.rs b/integration-tests/src/tests/client/block_corruption.rs index 0f49648c116..f41b8f56bd5 100644 --- a/integration-tests/src/tests/client/block_corruption.rs +++ b/integration-tests/src/tests/client/block_corruption.rs @@ -10,7 +10,6 @@ use near_primitives::sharding::{ShardChunkHeader, ShardChunkHeaderInner}; use near_primitives::transaction::SignedTransaction; use near_primitives::validator_signer::InMemoryValidatorSigner; use near_primitives_core::types::BlockHeight; -use nearcore::config::GenesisExt; use nearcore::test_utils::TestEnvNightshadeSetupExt; const NOT_BREAKING_CHANGE_MSG: &str = "Not a breaking change"; diff --git a/integration-tests/src/tests/client/challenges.rs b/integration-tests/src/tests/client/challenges.rs index d921e8d9d28..3a589cc862f 100644 --- a/integration-tests/src/tests/client/challenges.rs +++ b/integration-tests/src/tests/client/challenges.rs @@ -24,7 +24,6 @@ use near_primitives::types::AccountId; use near_primitives::validator_signer::ValidatorSigner; use near_primitives::version::PROTOCOL_VERSION; use near_store::Trie; -use nearcore::config::GenesisExt; use nearcore::test_utils::TestEnvNightshadeSetupExt; /// Check that block containing a challenge is rejected. diff --git a/integration-tests/src/tests/client/cold_storage.rs b/integration-tests/src/tests/client/cold_storage.rs index 67028513690..57655f5c68a 100644 --- a/integration-tests/src/tests/client/cold_storage.rs +++ b/integration-tests/src/tests/client/cold_storage.rs @@ -21,7 +21,6 @@ use near_store::metadata::DbKind; use near_store::metadata::DB_VERSION; use near_store::test_utils::create_test_node_storage_with_cold; use near_store::{DBCol, Store, COLD_HEAD_KEY, HEAD_KEY}; -use nearcore::config::GenesisExt; use nearcore::test_utils::TestEnvNightshadeSetupExt; use nearcore::{cold_storage::spawn_cold_store_loop, NearConfig}; use std::collections::HashSet; diff --git a/integration-tests/src/tests/client/epoch_sync.rs b/integration-tests/src/tests/client/epoch_sync.rs index 004b4c6e853..02d4cc8a17a 100644 --- a/integration-tests/src/tests/client/epoch_sync.rs +++ b/integration-tests/src/tests/client/epoch_sync.rs @@ -28,7 +28,6 @@ use near_primitives_core::hash::CryptoHash; use near_primitives_core::types::BlockHeight; use near_store::Mode::ReadOnly; use near_store::{DBCol, NodeStorage}; -use nearcore::config::GenesisExt; use nearcore::test_utils::TestEnvNightshadeSetupExt; use nearcore::{start_with_config, NearConfig}; use std::collections::HashSet; diff --git a/integration-tests/src/tests/client/features/access_key_nonce_for_implicit_accounts.rs b/integration-tests/src/tests/client/features/access_key_nonce_for_implicit_accounts.rs index 1c17f1f9fb7..a9946eb4b80 100644 --- a/integration-tests/src/tests/client/features/access_key_nonce_for_implicit_accounts.rs +++ b/integration-tests/src/tests/client/features/access_key_nonce_for_implicit_accounts.rs @@ -21,7 +21,6 @@ use near_primitives::types::{AccountId, BlockHeight}; use near_primitives::utils::derive_near_implicit_account_id; use near_primitives::version::{ProtocolFeature, ProtocolVersion}; use near_primitives::views::FinalExecutionStatus; -use nearcore::config::GenesisExt; use nearcore::test_utils::TestEnvNightshadeSetupExt; use nearcore::NEAR_BASE; use rand::seq::SliceRandom; diff --git a/integration-tests/src/tests/client/features/account_id_in_function_call_permission.rs b/integration-tests/src/tests/client/features/account_id_in_function_call_permission.rs index c41e27aa12b..75fe9f9df91 100644 --- a/integration-tests/src/tests/client/features/account_id_in_function_call_permission.rs +++ b/integration-tests/src/tests/client/features/account_id_in_function_call_permission.rs @@ -7,7 +7,6 @@ use near_primitives::account::{AccessKey, AccessKeyPermission, FunctionCallPermi use near_primitives::errors::{ActionsValidationError, InvalidTxError}; use near_primitives::hash::CryptoHash; use near_primitives::transaction::{Action, AddKeyAction, Transaction}; -use nearcore::config::GenesisExt; use nearcore::test_utils::TestEnvNightshadeSetupExt; #[test] diff --git a/integration-tests/src/tests/client/features/adversarial_behaviors.rs b/integration-tests/src/tests/client/features/adversarial_behaviors.rs index 96442b736b1..ee75148d878 100644 --- a/integration-tests/src/tests/client/features/adversarial_behaviors.rs +++ b/integration-tests/src/tests/client/features/adversarial_behaviors.rs @@ -20,7 +20,6 @@ use near_primitives::{ types::{AccountId, EpochId, ShardId}, }; use near_primitives_core::{checked_feature, version::PROTOCOL_VERSION}; -use nearcore::config::GenesisExt; use nearcore::test_utils::TestEnvNightshadeSetupExt; use tracing::log::debug; diff --git a/integration-tests/src/tests/client/features/chunk_nodes_cache.rs b/integration-tests/src/tests/client/features/chunk_nodes_cache.rs index 77d19d467b5..c94b46c2629 100644 --- a/integration-tests/src/tests/client/features/chunk_nodes_cache.rs +++ b/integration-tests/src/tests/client/features/chunk_nodes_cache.rs @@ -15,7 +15,6 @@ use near_primitives::types::{BlockHeightDelta, Gas}; use near_primitives::version::{ProtocolFeature, ProtocolVersion}; use near_primitives::views::FinalExecutionStatus; use near_vm_runner::logic::TrieNodesCount; -use nearcore::config::GenesisExt; use nearcore::test_utils::TestEnvNightshadeSetupExt; fn process_transaction( diff --git a/integration-tests/src/tests/client/features/delegate_action.rs b/integration-tests/src/tests/client/features/delegate_action.rs index 567843e6122..8f956ce58e9 100644 --- a/integration-tests/src/tests/client/features/delegate_action.rs +++ b/integration-tests/src/tests/client/features/delegate_action.rs @@ -30,7 +30,6 @@ use near_primitives::views::{ AccessKeyPermissionView, ExecutionStatusView, FinalExecutionOutcomeView, FinalExecutionStatus, }; use near_test_contracts::{ft_contract, smallest_rs_contract}; -use nearcore::config::GenesisExt; use nearcore::test_utils::TestEnvNightshadeSetupExt; use nearcore::NEAR_BASE; use testlib::runtime_utils::{ diff --git a/integration-tests/src/tests/client/features/fix_contract_loading_cost.rs b/integration-tests/src/tests/client/features/fix_contract_loading_cost.rs index 38abe02a965..a2a64996902 100644 --- a/integration-tests/src/tests/client/features/fix_contract_loading_cost.rs +++ b/integration-tests/src/tests/client/features/fix_contract_loading_cost.rs @@ -4,7 +4,6 @@ use near_chain_configs::Genesis; use near_client::test_utils::TestEnv; use near_primitives::types::{AccountId, BlockHeight}; use near_primitives::views::FinalExecutionStatus; -use nearcore::config::GenesisExt; use nearcore::test_utils::TestEnvNightshadeSetupExt; /// Create a `TestEnv` with an account and a contract deployed to that account. diff --git a/integration-tests/src/tests/client/features/fix_storage_usage.rs b/integration-tests/src/tests/client/features/fix_storage_usage.rs index 01351cc55b2..729c57a83fb 100644 --- a/integration-tests/src/tests/client/features/fix_storage_usage.rs +++ b/integration-tests/src/tests/client/features/fix_storage_usage.rs @@ -6,7 +6,6 @@ use near_o11y::testonly::init_test_logger; use near_primitives::version::ProtocolFeature; use near_primitives::{trie_key::TrieKey, types::AccountId}; use near_store::{ShardUId, TrieUpdate}; -use nearcore::config::GenesisExt; use nearcore::test_utils::TestEnvNightshadeSetupExt; use crate::tests::client::process_blocks::set_block_protocol_version; diff --git a/integration-tests/src/tests/client/features/flat_storage.rs b/integration-tests/src/tests/client/features/flat_storage.rs index 9e8d5065525..15282191c10 100644 --- a/integration-tests/src/tests/client/features/flat_storage.rs +++ b/integration-tests/src/tests/client/features/flat_storage.rs @@ -9,7 +9,6 @@ use near_primitives::transaction::{Action, ExecutionMetadata, FunctionCallAction use near_primitives::version::ProtocolFeature; use near_primitives_core::hash::CryptoHash; use near_primitives_core::types::Gas; -use nearcore::config::GenesisExt; use nearcore::test_utils::TestEnvNightshadeSetupExt; /// Check that after flat storage upgrade: diff --git a/integration-tests/src/tests/client/features/increase_deployment_cost.rs b/integration-tests/src/tests/client/features/increase_deployment_cost.rs index e2bbe9a000e..4c8bec93994 100644 --- a/integration-tests/src/tests/client/features/increase_deployment_cost.rs +++ b/integration-tests/src/tests/client/features/increase_deployment_cost.rs @@ -8,7 +8,6 @@ use near_primitives::transaction::{Action, DeployContractAction}; use near_primitives::version::ProtocolFeature; use near_primitives::views::FinalExecutionStatus; use near_primitives_core::version::PROTOCOL_VERSION; -use nearcore::config::GenesisExt; use nearcore::test_utils::TestEnvNightshadeSetupExt; /// Tests if the cost of deployment is higher after the protocol update 53 diff --git a/integration-tests/src/tests/client/features/increase_storage_compute_cost.rs b/integration-tests/src/tests/client/features/increase_storage_compute_cost.rs index 8cc10ffdfd8..0664efe3fc4 100644 --- a/integration-tests/src/tests/client/features/increase_storage_compute_cost.rs +++ b/integration-tests/src/tests/client/features/increase_storage_compute_cost.rs @@ -22,7 +22,6 @@ use near_primitives::transaction::{ }; use near_primitives::types::AccountId; use near_primitives::version::ProtocolFeature; -use nearcore::config::GenesisExt; use nearcore::test_utils::TestEnvNightshadeSetupExt; /// Tracked in https://github.com/near/nearcore/issues/8938 diff --git a/integration-tests/src/tests/client/features/limit_contract_functions_number.rs b/integration-tests/src/tests/client/features/limit_contract_functions_number.rs index 8a04ee400a6..82cddd4a345 100644 --- a/integration-tests/src/tests/client/features/limit_contract_functions_number.rs +++ b/integration-tests/src/tests/client/features/limit_contract_functions_number.rs @@ -8,7 +8,6 @@ use near_primitives::errors::{ }; use near_primitives::version::ProtocolFeature; use near_primitives::views::FinalExecutionStatus; -use nearcore::config::GenesisExt; use nearcore::test_utils::TestEnvNightshadeSetupExt; #[allow(dead_code)] diff --git a/integration-tests/src/tests/client/features/lower_storage_key_limit.rs b/integration-tests/src/tests/client/features/lower_storage_key_limit.rs index 1eb4c54c5b9..89c1b8eff76 100644 --- a/integration-tests/src/tests/client/features/lower_storage_key_limit.rs +++ b/integration-tests/src/tests/client/features/lower_storage_key_limit.rs @@ -11,7 +11,6 @@ use near_primitives::hash::CryptoHash; use near_primitives::transaction::{Action, FunctionCallAction, Transaction}; use near_primitives::types::BlockHeight; use near_primitives::views::FinalExecutionStatus; -use nearcore::config::GenesisExt; use nearcore::test_utils::TestEnvNightshadeSetupExt; use crate::tests::client::process_blocks::{ diff --git a/integration-tests/src/tests/client/features/nearvm.rs b/integration-tests/src/tests/client/features/nearvm.rs index 44f88255ae8..3a2bbc1b1d1 100644 --- a/integration-tests/src/tests/client/features/nearvm.rs +++ b/integration-tests/src/tests/client/features/nearvm.rs @@ -8,7 +8,6 @@ use near_crypto::{InMemorySigner, KeyType, Signer}; use near_parameters::RuntimeConfigStore; use near_primitives::hash::CryptoHash; use near_primitives::transaction::{Action, FunctionCallAction, Transaction}; -use nearcore::config::GenesisExt; use nearcore::test_utils::TestEnvNightshadeSetupExt; #[cfg_attr(all(target_arch = "aarch64", target_vendor = "apple"), ignore)] diff --git a/integration-tests/src/tests/client/features/nonrefundable_transfer.rs b/integration-tests/src/tests/client/features/nonrefundable_transfer.rs index 66ceb6c5469..c26a88062ab 100644 --- a/integration-tests/src/tests/client/features/nonrefundable_transfer.rs +++ b/integration-tests/src/tests/client/features/nonrefundable_transfer.rs @@ -23,7 +23,6 @@ use near_primitives::views::{ ExecutionStatusView, FinalExecutionOutcomeView, QueryRequest, QueryResponseKind, }; use near_primitives_core::account::{AccessKey, AccessKeyPermission}; -use nearcore::config::GenesisExt; use nearcore::test_utils::TestEnvNightshadeSetupExt; use nearcore::NEAR_BASE; use testlib::fees_utils::FeeHelper; diff --git a/integration-tests/src/tests/client/features/orphan_chunk_state_witness.rs b/integration-tests/src/tests/client/features/orphan_chunk_state_witness.rs index 06c404d5d1e..0adf094e541 100644 --- a/integration-tests/src/tests/client/features/orphan_chunk_state_witness.rs +++ b/integration-tests/src/tests/client/features/orphan_chunk_state_witness.rs @@ -18,7 +18,6 @@ use near_primitives_core::checked_feature; use near_primitives_core::hash::CryptoHash; use near_primitives_core::types::AccountId; use near_primitives_core::version::PROTOCOL_VERSION; -use nearcore::config::GenesisExt; use nearcore::test_utils::TestEnvNightshadeSetupExt; struct OrphanWitnessTestEnv { diff --git a/integration-tests/src/tests/client/features/restore_receipts_after_fix_apply_chunks.rs b/integration-tests/src/tests/client/features/restore_receipts_after_fix_apply_chunks.rs index 44238eed0d2..f3aa02df8ef 100644 --- a/integration-tests/src/tests/client/features/restore_receipts_after_fix_apply_chunks.rs +++ b/integration-tests/src/tests/client/features/restore_receipts_after_fix_apply_chunks.rs @@ -1,4 +1,5 @@ use crate::tests::client::process_blocks::set_block_protocol_version; +use near_chain::runtime::migrations::load_migration_data; use near_chain::Provenance; use near_chain_configs::Genesis; use near_client::test_utils::TestEnv; @@ -7,8 +8,6 @@ use near_primitives::hash::CryptoHash; use near_primitives::runtime::migration_data::MigrationData; use near_primitives::types::BlockHeight; use near_primitives::version::ProtocolFeature; -use nearcore::config::GenesisExt; -use nearcore::migrations::load_migration_data; use nearcore::test_utils::TestEnvNightshadeSetupExt; use std::collections::HashSet; diff --git a/integration-tests/src/tests/client/features/restrict_tla.rs b/integration-tests/src/tests/client/features/restrict_tla.rs index 45c633107b5..06485f2bde8 100644 --- a/integration-tests/src/tests/client/features/restrict_tla.rs +++ b/integration-tests/src/tests/client/features/restrict_tla.rs @@ -5,7 +5,6 @@ use near_primitives::errors::{ActionError, ActionErrorKind}; use near_primitives::types::{AccountId, BlockHeight}; use near_primitives::views::FinalExecutionStatus; use near_primitives_core::version::PROTOCOL_VERSION; -use nearcore::config::GenesisExt; use nearcore::test_utils::TestEnvNightshadeSetupExt; #[test] diff --git a/integration-tests/src/tests/client/features/stateless_validation.rs b/integration-tests/src/tests/client/features/stateless_validation.rs index a58647766fb..0491e706354 100644 --- a/integration-tests/src/tests/client/features/stateless_validation.rs +++ b/integration-tests/src/tests/client/features/stateless_validation.rs @@ -1,7 +1,6 @@ use near_epoch_manager::{EpochManager, EpochManagerAdapter}; use near_primitives::stateless_validation::ChunkStateWitness; use near_store::test_utils::create_test_store; -use nearcore::config::GenesisExt; use rand::rngs::StdRng; use rand::{Rng, SeedableRng}; use std::collections::HashSet; diff --git a/integration-tests/src/tests/client/features/wallet_contract.rs b/integration-tests/src/tests/client/features/wallet_contract.rs index dc9ac33b1bf..fce2ddb0de6 100644 --- a/integration-tests/src/tests/client/features/wallet_contract.rs +++ b/integration-tests/src/tests/client/features/wallet_contract.rs @@ -21,7 +21,7 @@ use near_primitives_core::{ use near_store::ShardUId; use near_vm_runner::ContractCode; use near_wallet_contract::{wallet_contract, wallet_contract_magic_bytes}; -use nearcore::{config::GenesisExt, test_utils::TestEnvNightshadeSetupExt, NEAR_BASE}; +use nearcore::{test_utils::TestEnvNightshadeSetupExt, NEAR_BASE}; use node_runtime::ZERO_BALANCE_ACCOUNT_STORAGE_LIMIT; use rlp::RlpStream; use testlib::runtime_utils::{alice_account, bob_account, carol_account}; diff --git a/integration-tests/src/tests/client/features/zero_balance_account.rs b/integration-tests/src/tests/client/features/zero_balance_account.rs index 20baa2cd506..dd5ed8da9ca 100644 --- a/integration-tests/src/tests/client/features/zero_balance_account.rs +++ b/integration-tests/src/tests/client/features/zero_balance_account.rs @@ -13,7 +13,6 @@ use near_primitives::transaction::Action::AddKey; use near_primitives::transaction::{Action, AddKeyAction, DeleteKeyAction, SignedTransaction}; use near_primitives::version::{ProtocolFeature, PROTOCOL_VERSION}; use near_primitives::views::{FinalExecutionStatus, QueryRequest, QueryResponseKind}; -use nearcore::config::GenesisExt; use nearcore::test_utils::TestEnvNightshadeSetupExt; use node_runtime::ZERO_BALANCE_ACCOUNT_STORAGE_LIMIT; diff --git a/integration-tests/src/tests/client/flat_storage.rs b/integration-tests/src/tests/client/flat_storage.rs index d865a26af7c..74cd8e41a36 100644 --- a/integration-tests/src/tests/client/flat_storage.rs +++ b/integration-tests/src/tests/client/flat_storage.rs @@ -19,7 +19,6 @@ use near_store::flat::{ use near_store::test_utils::create_test_store; use near_store::trie::TrieNodesCount; use near_store::{KeyLookupMode, Store, TrieTraversalItem}; -use nearcore::config::GenesisExt; use nearcore::test_utils::TestEnvNightshadeSetupExt; use std::str::FromStr; use std::thread; diff --git a/integration-tests/src/tests/client/process_blocks.rs b/integration-tests/src/tests/client/process_blocks.rs index 08b05fa7ac9..493bbed9dd3 100644 --- a/integration-tests/src/tests/client/process_blocks.rs +++ b/integration-tests/src/tests/client/process_blocks.rs @@ -70,7 +70,7 @@ use near_store::metadata::DB_VERSION; use near_store::test_utils::create_test_node_storage_with_cold; use near_store::NodeStorage; use near_store::{get, DBCol, TrieChanges}; -use nearcore::config::{GenesisExt, TESTING_INIT_BALANCE, TESTING_INIT_STAKE}; +use nearcore::config::{TESTING_INIT_BALANCE, TESTING_INIT_STAKE}; use nearcore::test_utils::TestEnvNightshadeSetupExt; use nearcore::NEAR_BASE; use rand::prelude::StdRng; diff --git a/integration-tests/src/tests/client/resharding.rs b/integration-tests/src/tests/client/resharding.rs index c88d46936bb..51e862ea676 100644 --- a/integration-tests/src/tests/client/resharding.rs +++ b/integration-tests/src/tests/client/resharding.rs @@ -28,7 +28,6 @@ use near_store::metadata::DbKind; use near_store::test_utils::{gen_account, gen_unique_accounts}; use near_store::trie::SnapshotError; use near_store::{DBCol, ShardUId}; -use nearcore::config::GenesisExt; use nearcore::test_utils::TestEnvNightshadeSetupExt; use nearcore::NEAR_BASE; use rand::rngs::StdRng; diff --git a/integration-tests/src/tests/client/runtimes.rs b/integration-tests/src/tests/client/runtimes.rs index af415351ed1..ece190ae587 100644 --- a/integration-tests/src/tests/client/runtimes.rs +++ b/integration-tests/src/tests/client/runtimes.rs @@ -11,7 +11,6 @@ use near_primitives::hash::hash; use near_primitives::network::PeerId; use near_primitives::test_utils::create_test_signer; use near_primitives::validator_signer::InMemoryValidatorSigner; -use nearcore::config::GenesisExt; use nearcore::test_utils::TestEnvNightshadeSetupExt; use std::collections::HashMap; use std::sync::Arc; diff --git a/integration-tests/src/tests/client/sandbox.rs b/integration-tests/src/tests/client/sandbox.rs index c40eb225340..20c960065b1 100644 --- a/integration-tests/src/tests/client/sandbox.rs +++ b/integration-tests/src/tests/client/sandbox.rs @@ -10,7 +10,6 @@ use near_primitives::transaction::{ Action, DeployContractAction, FunctionCallAction, SignedTransaction, }; use near_primitives::types::{AccountId, BlockHeight, Nonce}; -use nearcore::config::GenesisExt; use nearcore::test_utils::TestEnvNightshadeSetupExt; fn test_setup() -> (TestEnv, InMemorySigner) { diff --git a/integration-tests/src/tests/client/state_dump.rs b/integration-tests/src/tests/client/state_dump.rs index 6b7d96f66f1..47067accfaf 100644 --- a/integration-tests/src/tests/client/state_dump.rs +++ b/integration-tests/src/tests/client/state_dump.rs @@ -21,7 +21,6 @@ use near_primitives::views::{QueryRequest, QueryResponseKind}; use near_store::flat::store_helper; use near_store::DBCol; use near_store::Store; -use nearcore::config::GenesisExt; use nearcore::state_sync::spawn_state_sync_dump; use nearcore::test_utils::TestEnvNightshadeSetupExt; use nearcore::NEAR_BASE; diff --git a/integration-tests/src/tests/client/state_snapshot.rs b/integration-tests/src/tests/client/state_snapshot.rs index c4089875925..69caebe2aca 100644 --- a/integration-tests/src/tests/client/state_snapshot.rs +++ b/integration-tests/src/tests/client/state_snapshot.rs @@ -15,7 +15,6 @@ use near_store::{ StoreConfig, TrieConfig, }; use near_store::{NodeStorage, Store}; -use nearcore::config::GenesisExt; use nearcore::test_utils::TestEnvNightshadeSetupExt; use nearcore::NEAR_BASE; use std::path::PathBuf; diff --git a/integration-tests/src/tests/client/sync_state_nodes.rs b/integration-tests/src/tests/client/sync_state_nodes.rs index f7abda5eab1..f757a420920 100644 --- a/integration-tests/src/tests/client/sync_state_nodes.rs +++ b/integration-tests/src/tests/client/sync_state_nodes.rs @@ -25,7 +25,7 @@ use near_primitives::utils::MaybeValidated; use near_primitives_core::types::ShardId; use near_store::DBCol; use nearcore::test_utils::TestEnvNightshadeSetupExt; -use nearcore::{config::GenesisExt, load_test_config, start_with_config}; +use nearcore::{load_test_config, start_with_config}; use std::ops::ControlFlow; use std::sync::{Arc, RwLock}; use std::time::Duration; diff --git a/integration-tests/src/tests/client/undo_block.rs b/integration-tests/src/tests/client/undo_block.rs index 920ac244fe1..8c7e0603997 100644 --- a/integration-tests/src/tests/client/undo_block.rs +++ b/integration-tests/src/tests/client/undo_block.rs @@ -6,7 +6,6 @@ use near_o11y::testonly::init_test_logger; use near_store::test_utils::create_test_store; use near_store::Store; use near_undo_block::undo_block; -use nearcore::config::GenesisExt; use nearcore::test_utils::TestEnvNightshadeSetupExt; use std::sync::Arc; diff --git a/integration-tests/src/tests/nearcore/node_cluster.rs b/integration-tests/src/tests/nearcore/node_cluster.rs index c30516d3a0d..739ddbc813c 100644 --- a/integration-tests/src/tests/nearcore/node_cluster.rs +++ b/integration-tests/src/tests/nearcore/node_cluster.rs @@ -9,7 +9,7 @@ use near_network::tcp; use near_network::test_utils::convert_boot_nodes; use near_o11y::testonly::init_integration_logger; use near_primitives::types::{BlockHeight, BlockHeightDelta, NumSeats, NumShards}; -use nearcore::{config::GenesisExt, load_test_config, start_with_config}; +use nearcore::{load_test_config, start_with_config}; fn start_nodes( temp_dir: &std::path::Path, diff --git a/integration-tests/src/tests/nearcore/stake_nodes.rs b/integration-tests/src/tests/nearcore/stake_nodes.rs index 50b1dcff60b..c26fe51ad54 100644 --- a/integration-tests/src/tests/nearcore/stake_nodes.rs +++ b/integration-tests/src/tests/nearcore/stake_nodes.rs @@ -20,7 +20,7 @@ use near_primitives::hash::CryptoHash; use near_primitives::transaction::SignedTransaction; use near_primitives::types::{AccountId, BlockHeightDelta, BlockReference, NumSeats}; use near_primitives::views::{QueryRequest, QueryResponseKind, ValidatorInfo}; -use nearcore::config::{GenesisExt, TESTING_INIT_BALANCE, TESTING_INIT_STAKE}; +use nearcore::config::{TESTING_INIT_BALANCE, TESTING_INIT_STAKE}; use nearcore::{load_test_config, start_with_config, NearConfig, NEAR_BASE}; use near_o11y::WithSpanContextExt; diff --git a/integration-tests/src/tests/nearcore/sync_nodes.rs b/integration-tests/src/tests/nearcore/sync_nodes.rs index 15587c1f887..8e9c99554e6 100644 --- a/integration-tests/src/tests/nearcore/sync_nodes.rs +++ b/integration-tests/src/tests/nearcore/sync_nodes.rs @@ -13,7 +13,7 @@ use near_o11y::testonly::init_integration_logger; use near_o11y::WithSpanContextExt; use near_primitives::test_utils::create_test_signer; use near_primitives::transaction::SignedTransaction; -use nearcore::config::{GenesisExt, TESTING_INIT_STAKE}; +use nearcore::config::TESTING_INIT_STAKE; use nearcore::{load_test_config, start_with_config}; use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::{Arc, RwLock}; diff --git a/integration-tests/src/tests/runtime/deployment.rs b/integration-tests/src/tests/runtime/deployment.rs index 2eccdcdf216..850e869c601 100644 --- a/integration-tests/src/tests/runtime/deployment.rs +++ b/integration-tests/src/tests/runtime/deployment.rs @@ -5,7 +5,6 @@ use near_primitives::transaction::{Action, DeployContractAction, SignedTransacti use near_primitives::types::AccountId; use near_primitives::version::PROTOCOL_VERSION; use near_primitives::views::FinalExecutionStatus; -use nearcore::config::GenesisExt; const ONE_NEAR: u128 = 10u128.pow(24); diff --git a/integration-tests/src/tests/runtime/sanity_checks.rs b/integration-tests/src/tests/runtime/sanity_checks.rs index 22fd37a9939..516517777a1 100644 --- a/integration-tests/src/tests/runtime/sanity_checks.rs +++ b/integration-tests/src/tests/runtime/sanity_checks.rs @@ -7,7 +7,6 @@ use near_primitives::version::PROTOCOL_VERSION; use near_primitives::views::{ CostGasUsed, ExecutionOutcomeWithIdView, ExecutionStatusView, FinalExecutionStatus, }; -use nearcore::config::GenesisExt; use std::collections::HashSet; use std::mem::size_of; use testlib::runtime_utils::{add_test_contract, alice_account, bob_account}; diff --git a/integration-tests/src/tests/standard_cases/runtime.rs b/integration-tests/src/tests/standard_cases/runtime.rs index 3fa8e5d8e68..bb170ae658d 100644 --- a/integration-tests/src/tests/standard_cases/runtime.rs +++ b/integration-tests/src/tests/standard_cases/runtime.rs @@ -5,7 +5,7 @@ use near_crypto::SecretKey; use near_primitives::checked_feature; use near_primitives::state_record::StateRecord; use near_primitives::version::PROTOCOL_VERSION; -use nearcore::config::{GenesisExt, TESTING_INIT_BALANCE}; +use nearcore::config::TESTING_INIT_BALANCE; use testlib::runtime_utils::{add_test_contract, alice_account, bob_account}; fn create_runtime_node() -> RuntimeNode { diff --git a/integration-tests/src/tests/test_errors.rs b/integration-tests/src/tests/test_errors.rs index bf45db0249a..91e5397529b 100644 --- a/integration-tests/src/tests/test_errors.rs +++ b/integration-tests/src/tests/test_errors.rs @@ -13,7 +13,7 @@ use near_primitives::transaction::{ Action, AddKeyAction, CreateAccountAction, SignedTransaction, TransferAction, }; use near_primitives::version::PROTOCOL_VERSION; -use nearcore::config::{GenesisExt, TESTING_INIT_BALANCE, TESTING_INIT_STAKE}; +use nearcore::config::{TESTING_INIT_BALANCE, TESTING_INIT_STAKE}; use nearcore::load_test_config; use testlib::runtime_utils::{alice_account, bob_account}; diff --git a/nearcore/benches/store.rs b/nearcore/benches/store.rs index 5d5f55b9527..2332ea62366 100644 --- a/nearcore/benches/store.rs +++ b/nearcore/benches/store.rs @@ -8,7 +8,7 @@ use near_epoch_manager::EpochManager; use near_o11y::testonly::init_integration_logger; use near_primitives::types::StateRoot; use near_store::Mode; -use nearcore::{get_default_home, load_config, NightshadeRuntime}; +use nearcore::{get_default_home, load_config, NightshadeRuntime, NightshadeRuntimeExt}; use std::time::{Duration, Instant}; /// Read `TrieItem`s - nodes containing values - using Trie iterator, stop when 10k items were read. diff --git a/nearcore/src/config.rs b/nearcore/src/config.rs index 358e4a1c323..172c87ebffe 100644 --- a/nearcore/src/config.rs +++ b/nearcore/src/config.rs @@ -1,6 +1,7 @@ use crate::download_file::{run_download_file, FileDownloadError}; use crate::dyn_config::LOG_CONFIG_FILENAME; use anyhow::{anyhow, bail, Context}; +use near_chain::runtime::NightshadeRuntime; use near_chain_configs::{ default_enable_multiline_logging, default_epoch_sync_enabled, default_header_sync_expected_height_per_second, default_header_sync_initial_timeout, @@ -16,6 +17,7 @@ use near_chain_configs::{ }; use near_config_utils::{ValidationError, ValidationErrors}; use near_crypto::{InMemorySigner, KeyFile, KeyType, PublicKey, Signer}; +use near_epoch_manager::EpochManagerHandle; #[cfg(feature = "json_rpc")] use near_jsonrpc::RpcConfig; use near_network::config::NetworkConfig; @@ -38,12 +40,14 @@ use near_primitives::validator_signer::{InMemoryValidatorSigner, ValidatorSigner use near_primitives::version::PROTOCOL_VERSION; #[cfg(feature = "rosetta_rpc")] use near_rosetta_rpc::RosettaRpcConfig; +use near_store::config::StateSnapshotType; +use near_store::{StateSnapshotConfig, Store, TrieConfig}; use near_telemetry::TelemetryConfig; use num_rational::Rational32; use std::fs; use std::fs::File; use std::io::{Read, Write}; -use std::path::Path; +use std::path::{Path, PathBuf}; use std::str::FromStr; use std::sync::Arc; use std::time::Duration; @@ -51,15 +55,15 @@ use std::time::Duration; use tempfile::tempdir; use tracing::{info, warn}; +/// One NEAR, divisible by 10^24. +pub const NEAR_BASE: Balance = 1_000_000_000_000_000_000_000_000; + /// Initial balance used in tests. pub const TESTING_INIT_BALANCE: Balance = 1_000_000_000 * NEAR_BASE; /// Validator's stake used in tests. pub const TESTING_INIT_STAKE: Balance = 50_000_000 * NEAR_BASE; -/// One NEAR, divisible by 10^24. -pub const NEAR_BASE: Balance = 1_000_000_000_000_000_000_000_000; - /// Millinear, 1/1000 of NEAR. pub const MILLI_NEAR: Balance = NEAR_BASE / 1000; @@ -498,106 +502,6 @@ impl Config { } } -#[easy_ext::ext(GenesisExt)] -impl Genesis { - // Creates new genesis with a given set of accounts and shard layout. - // The first num_validator_seats from accounts will be treated as 'validators'. - pub fn test_with_seeds( - accounts: Vec, - num_validator_seats: NumSeats, - num_validator_seats_per_shard: Vec, - shard_layout: ShardLayout, - ) -> Self { - let mut validators = vec![]; - let mut records = vec![]; - for (i, account) in accounts.into_iter().enumerate() { - let signer = - InMemorySigner::from_seed(account.clone(), KeyType::ED25519, account.as_ref()); - let i = i as u64; - if i < num_validator_seats { - validators.push(AccountInfo { - account_id: account.clone(), - public_key: signer.public_key.clone(), - amount: TESTING_INIT_STAKE, - }); - } - add_account_with_key( - &mut records, - account, - &signer.public_key.clone(), - TESTING_INIT_BALANCE - if i < num_validator_seats { TESTING_INIT_STAKE } else { 0 }, - if i < num_validator_seats { TESTING_INIT_STAKE } else { 0 }, - CryptoHash::default(), - ); - } - add_protocol_account(&mut records); - let config = GenesisConfig { - protocol_version: PROTOCOL_VERSION, - genesis_time: StaticClock::utc(), - chain_id: random_chain_id(), - num_block_producer_seats: num_validator_seats, - num_block_producer_seats_per_shard: num_validator_seats_per_shard.clone(), - avg_hidden_validator_seats_per_shard: vec![0; num_validator_seats_per_shard.len()], - dynamic_resharding: false, - protocol_upgrade_stake_threshold: PROTOCOL_UPGRADE_STAKE_THRESHOLD, - epoch_length: FAST_EPOCH_LENGTH, - gas_limit: INITIAL_GAS_LIMIT, - gas_price_adjustment_rate: GAS_PRICE_ADJUSTMENT_RATE, - block_producer_kickout_threshold: BLOCK_PRODUCER_KICKOUT_THRESHOLD, - validators, - protocol_reward_rate: PROTOCOL_REWARD_RATE, - total_supply: get_initial_supply(&records), - max_inflation_rate: MAX_INFLATION_RATE, - num_blocks_per_year: NUM_BLOCKS_PER_YEAR, - protocol_treasury_account: PROTOCOL_TREASURY_ACCOUNT.parse().unwrap(), - transaction_validity_period: TRANSACTION_VALIDITY_PERIOD, - chunk_producer_kickout_threshold: CHUNK_PRODUCER_KICKOUT_THRESHOLD, - fishermen_threshold: FISHERMEN_THRESHOLD, - min_gas_price: MIN_GAS_PRICE, - shard_layout, - ..Default::default() - }; - Genesis::new(config, records.into()).unwrap() - } - - pub fn test(accounts: Vec, num_validator_seats: NumSeats) -> Self { - Self::test_with_seeds( - accounts, - num_validator_seats, - vec![num_validator_seats], - ShardLayout::v0_single_shard(), - ) - } - - pub fn test_sharded( - accounts: Vec, - num_validator_seats: NumSeats, - num_validator_seats_per_shard: Vec, - ) -> Self { - let num_shards = num_validator_seats_per_shard.len() as NumShards; - Self::test_with_seeds( - accounts, - num_validator_seats, - num_validator_seats_per_shard, - ShardLayout::v0(num_shards, 0), - ) - } - - pub fn test_sharded_new_version( - accounts: Vec, - num_validator_seats: NumSeats, - num_validator_seats_per_shard: Vec, - ) -> Self { - let num_shards = num_validator_seats_per_shard.len() as NumShards; - Self::test_with_seeds( - accounts, - num_validator_seats, - num_validator_seats_per_shard, - ShardLayout::v0(num_shards, 1), - ) - } -} - #[derive(Clone)] pub struct NearConfig { pub config: Config, @@ -739,6 +643,49 @@ impl NearConfig { } } +#[easy_ext::ext(NightshadeRuntimeExt)] +impl NightshadeRuntime { + pub fn from_config( + home_dir: &Path, + store: Store, + config: &NearConfig, + epoch_manager: Arc, + ) -> Arc { + // TODO (#9989): directly use the new state snapshot config once the migration is done. + let mut state_snapshot_type = + config.config.store.state_snapshot_config.state_snapshot_type.clone(); + if config.config.store.state_snapshot_enabled { + state_snapshot_type = StateSnapshotType::EveryEpoch; + } + // TODO (#9989): directly use the new state snapshot config once the migration is done. + let compaction_enabled = config.config.store.state_snapshot_compaction_enabled + || config.config.store.state_snapshot_config.compaction_enabled; + let state_snapshot_config = StateSnapshotConfig { + state_snapshot_type, + home_dir: home_dir.to_path_buf(), + hot_store_path: config + .config + .store + .path + .clone() + .unwrap_or_else(|| PathBuf::from("data")), + state_snapshot_subdir: PathBuf::from("state_snapshot"), + compaction_enabled, + }; + NightshadeRuntime::new( + store, + &config.genesis.config, + epoch_manager, + config.client_config.trie_viewer_state_size_limit, + config.client_config.max_gas_burnt_view, + None, + config.config.gc.gc_num_epochs_to_keep(), + TrieConfig::from_store_config(&config.config.store), + state_snapshot_config, + ) + } +} + fn add_protocol_account(records: &mut Vec) { let signer = InMemorySigner::from_seed( PROTOCOL_TREASURY_ACCOUNT.parse().unwrap(), diff --git a/nearcore/src/lib.rs b/nearcore/src/lib.rs index dae92b19eab..a22399d1134 100644 --- a/nearcore/src/lib.rs +++ b/nearcore/src/lib.rs @@ -1,7 +1,7 @@ +pub use crate::config::NightshadeRuntimeExt; pub use crate::config::{init_configs, load_config, load_test_config, NearConfig, NEAR_BASE}; use crate::entity_debug::EntityDebugHandlerImpl; use crate::metrics::spawn_trie_metrics_loop; -pub use crate::runtime::NightshadeRuntime; use crate::cold_storage::spawn_cold_store_loop; use crate::state_sync::{spawn_state_sync_dump, StateSyncDumpHandle}; @@ -12,6 +12,7 @@ use cold_storage::ColdStoreLoopHandle; use near_async::actix::AddrWithAutoSpanContextExt; use near_async::messaging::{IntoMultiSender, IntoSender, LateBoundSender}; use near_async::time::{self, Clock}; +pub use near_chain::runtime::NightshadeRuntime; use near_chain::state_snapshot_actor::{ get_delete_snapshot_callback, get_make_snapshot_callback, SnapshotCallbacks, StateSnapshotActor, }; @@ -49,7 +50,6 @@ mod entity_debug; mod entity_debug_serializer; mod metrics; pub mod migrations; -mod runtime; pub mod state_sync; pub mod test_utils; diff --git a/nearcore/src/metrics.rs b/nearcore/src/metrics.rs index 9503f287451..8c528929730 100644 --- a/nearcore/src/metrics.rs +++ b/nearcore/src/metrics.rs @@ -4,9 +4,9 @@ use actix_rt::ArbiterHandle; use near_chain::{Block, ChainStore, ChainStoreAccess}; use near_epoch_manager::EpochManager; use near_o11y::metrics::{ - exponential_buckets, linear_buckets, processing_time_buckets, try_create_histogram_vec, - try_create_int_counter_vec, try_create_int_gauge, try_create_int_gauge_vec, HistogramVec, - IntCounterVec, IntGauge, IntGaugeVec, + exponential_buckets, try_create_histogram_vec, try_create_int_counter_vec, + try_create_int_gauge, try_create_int_gauge_vec, HistogramVec, IntCounterVec, IntGauge, + IntGaugeVec, }; use near_primitives::{shard_layout::ShardLayout, state_record::StateRecord, trie_key}; @@ -15,35 +15,6 @@ use once_cell::sync::Lazy; use crate::NearConfig; -pub static APPLYING_CHUNKS_TIME: Lazy = Lazy::new(|| { - try_create_histogram_vec( - "near_applying_chunks_time", - "Time taken to apply chunks per shard", - &["shard_id"], - Some(processing_time_buckets()), - ) - .unwrap() -}); - -pub(crate) static APPLY_CHUNK_DELAY: Lazy = Lazy::new(|| { - try_create_histogram_vec( - "near_apply_chunk_delay_seconds", - "Time to process a chunk. Gas used by the chunk is a metric label, rounded up to 100 teragas.", - &["tgas_ceiling"], - Some(linear_buckets(0.0, 0.05, 50).unwrap()), - ) - .unwrap() -}); - -pub(crate) static DELAYED_RECEIPTS_COUNT: Lazy = Lazy::new(|| { - try_create_int_gauge_vec( - "near_delayed_receipts_count", - "The count of the delayed receipts. Indicator of congestion.", - &["shard_id"], - ) - .unwrap() -}); - pub(crate) static POSTPONED_RECEIPTS_COUNT: Lazy = Lazy::new(|| { try_create_int_gauge_vec( "near_postponed_receipts_count", @@ -53,18 +24,6 @@ pub(crate) static POSTPONED_RECEIPTS_COUNT: Lazy = Lazy::new(|| { .unwrap() }); -pub(crate) static PREPARE_TX_SIZE: Lazy = Lazy::new(|| { - try_create_histogram_vec( - "near_prepare_tx_size", - "Sum of transaction sizes per produced chunk, as a histogram", - &["shard_id"], - // Maximum is < 14MB, typical values are unknown right now so buckets - // might need to be adjusted later when we have collected data - Some(vec![1_000.0, 10_000., 100_000., 500_000., 1e6, 2e6, 4e6, 8e6, 12e6]), - ) - .unwrap() -}); - pub(crate) static CONFIG_CORRECT: Lazy = Lazy::new(|| { try_create_int_gauge( "near_config_correct", @@ -128,26 +87,6 @@ pub(crate) static STATE_SYNC_DUMP_EPOCH_HEIGHT: Lazy = Lazy::new(|| .unwrap() }); -pub(crate) static STATE_SYNC_APPLY_PART_DELAY: Lazy = Lazy::new(|| { - try_create_histogram_vec( - "near_state_sync_apply_part_delay_sec", - "Latency of applying a state part", - &["shard_id"], - Some(exponential_buckets(0.001, 2.0, 20).unwrap()), - ) - .unwrap() -}); - -pub(crate) static STATE_SYNC_OBTAIN_PART_DELAY: Lazy = Lazy::new(|| { - try_create_histogram_vec( - "near_state_sync_obtain_part_delay_sec", - "Latency of applying a state part", - &["shard_id", "result"], - Some(exponential_buckets(0.001, 2.0, 20).unwrap()), - ) - .unwrap() -}); - fn log_trie_item(key: Vec, value: Vec) { if !tracing::level_enabled!(tracing::Level::TRACE) { return; diff --git a/nearcore/src/migrations.rs b/nearcore/src/migrations.rs index 2e7478b7c9e..a63803fc0f0 100644 --- a/nearcore/src/migrations.rs +++ b/nearcore/src/migrations.rs @@ -1,7 +1,4 @@ use near_chain::{ChainStore, ChainStoreAccess}; -use near_primitives::receipt::ReceiptResult; -use near_primitives::runtime::migration_data::MigrationData; -use near_primitives::types::Gas; use near_primitives::utils::index_to_bytes; use near_store::metadata::{DbVersion, DB_VERSION}; use near_store::migrations::BatchedStoreUpdate; @@ -53,31 +50,6 @@ pub fn do_migrate_30_to_31( Ok(()) } -/// In test runs reads and writes here used 442 TGas, but in test on live net migration take -/// between 4 and 4.5s. We do not want to process any receipts in this block -const GAS_USED_FOR_STORAGE_USAGE_DELTA_MIGRATION: Gas = 1_000_000_000_000_000; - -pub fn load_migration_data(chain_id: &str) -> MigrationData { - let is_mainnet = chain_id == near_primitives::chains::MAINNET; - MigrationData { - storage_usage_delta: if is_mainnet { - near_mainnet_res::mainnet_storage_usage_delta() - } else { - Vec::new() - }, - storage_usage_fix_gas: if is_mainnet { - GAS_USED_FOR_STORAGE_USAGE_DELTA_MIGRATION - } else { - 0 - }, - restored_receipts: if is_mainnet { - near_mainnet_res::mainnet_restored_receipts() - } else { - ReceiptResult::default() - }, - } -} - pub(super) struct Migrator<'a> { config: &'a crate::config::NearConfig, } @@ -119,37 +91,3 @@ impl<'a> near_store::StoreMigrator for Migrator<'a> { } } } - -#[cfg(test)] -mod tests { - use super::*; - use near_mainnet_res::mainnet_restored_receipts; - use near_mainnet_res::mainnet_storage_usage_delta; - use near_primitives::hash::hash; - - #[test] - fn test_migration_data() { - assert_eq!( - hash(serde_json::to_string(&mainnet_storage_usage_delta()).unwrap().as_bytes()) - .to_string(), - "2fEgaLFBBJZqgLQEvHPsck4NS3sFzsgyKaMDqTw5HVvQ" - ); - let mainnet_migration_data = load_migration_data(near_primitives::chains::MAINNET); - assert_eq!(mainnet_migration_data.storage_usage_delta.len(), 3112); - let testnet_migration_data = load_migration_data(near_primitives::chains::TESTNET); - assert_eq!(testnet_migration_data.storage_usage_delta.len(), 0); - } - - #[test] - fn test_restored_receipts_data() { - assert_eq!( - hash(serde_json::to_string(&mainnet_restored_receipts()).unwrap().as_bytes()) - .to_string(), - "48ZMJukN7RzvyJSW9MJ5XmyQkQFfjy2ZxPRaDMMHqUcT" - ); - let mainnet_migration_data = load_migration_data(near_primitives::chains::MAINNET); - assert_eq!(mainnet_migration_data.restored_receipts.get(&0u64).unwrap().len(), 383); - let testnet_migration_data = load_migration_data(near_primitives::chains::TESTNET); - assert!(testnet_migration_data.restored_receipts.is_empty()); - } -} diff --git a/nearcore/tests/economics.rs b/nearcore/tests/economics.rs index 7f36ba6d820..8d0f5fb5e8c 100644 --- a/nearcore/tests/economics.rs +++ b/nearcore/tests/economics.rs @@ -11,7 +11,7 @@ use near_crypto::{InMemorySigner, KeyType}; use near_o11y::testonly::init_integration_logger; use near_primitives::transaction::SignedTransaction; use near_store::{genesis::initialize_genesis_state, test_utils::create_test_store}; -use nearcore::{config::GenesisExt, NightshadeRuntime}; +use nearcore::NightshadeRuntime; use testlib::fees_utils::FeeHelper; use near_primitives::types::EpochId; diff --git a/test-utils/runtime-tester/src/run_test.rs b/test-utils/runtime-tester/src/run_test.rs index b035dfd05dd..2670cf1a835 100644 --- a/test-utils/runtime-tester/src/run_test.rs +++ b/test-utils/runtime-tester/src/run_test.rs @@ -12,7 +12,7 @@ use near_primitives::types::{AccountId, BlockHeight, BlockHeightDelta, Gas, Nonc use near_store::config::StateSnapshotType; use near_store::genesis::initialize_genesis_state; use near_store::test_utils::create_test_store; -use nearcore::{config::GenesisExt, NightshadeRuntime}; +use nearcore::NightshadeRuntime; use std::io; use std::path::Path; use std::time::Duration; diff --git a/test-utils/store-validator/src/main.rs b/test-utils/store-validator/src/main.rs index 2c5603342be..f307e348fa7 100644 --- a/test-utils/store-validator/src/main.rs +++ b/test-utils/store-validator/src/main.rs @@ -6,7 +6,7 @@ use near_epoch_manager::{ EpochManager, }; use near_o11y::testonly::init_integration_logger; -use nearcore::{get_default_home, load_config}; +use nearcore::{get_default_home, load_config, NightshadeRuntimeExt}; use std::path::PathBuf; use std::process; use yansi::Color::{Green, Red, White, Yellow}; diff --git a/tools/epoch-sync/src/cli.rs b/tools/epoch-sync/src/cli.rs index 2b37eeb1074..b53ef2d721b 100644 --- a/tools/epoch-sync/src/cli.rs +++ b/tools/epoch-sync/src/cli.rs @@ -9,7 +9,7 @@ use near_primitives::epoch_manager::block_info::BlockInfo; use near_primitives::epoch_manager::AGGREGATOR_KEY; use near_primitives::hash::CryptoHash; use near_store::{checkpoint_hot_storage_and_cleanup_columns, DBCol, NodeStorage}; -use nearcore::NightshadeRuntime; +use nearcore::{NightshadeRuntime, NightshadeRuntimeExt}; use std::collections::{HashMap, HashSet}; use std::path::{Path, PathBuf}; diff --git a/tools/flat-storage/src/commands.rs b/tools/flat-storage/src/commands.rs index 08c468fc334..bc0e8aff545 100644 --- a/tools/flat-storage/src/commands.rs +++ b/tools/flat-storage/src/commands.rs @@ -13,7 +13,7 @@ use near_store::flat::{ FlatStorageManager, FlatStorageStatus, }; use near_store::{DBCol, Mode, NodeStorage, ShardUId, Store, StoreOpener}; -use nearcore::{load_config, NearConfig, NightshadeRuntime}; +use nearcore::{load_config, NearConfig, NightshadeRuntime, NightshadeRuntimeExt}; use std::sync::atomic::AtomicBool; use std::{path::PathBuf, sync::Arc, time::Duration}; use tqdm::tqdm; diff --git a/tools/fork-network/src/cli.rs b/tools/fork-network/src/cli.rs index 9b4e9516e85..18f83f26ceb 100644 --- a/tools/fork-network/src/cli.rs +++ b/tools/fork-network/src/cli.rs @@ -30,7 +30,9 @@ use near_store::{ checkpoint_hot_storage_and_cleanup_columns, DBCol, Store, TrieDBStorage, TrieStorage, FINAL_HEAD_KEY, }; -use nearcore::{load_config, open_storage, NearConfig, NightshadeRuntime, NEAR_BASE}; +use nearcore::{ + load_config, open_storage, NearConfig, NightshadeRuntime, NightshadeRuntimeExt, NEAR_BASE, +}; use rayon::iter::{IntoParallelIterator, ParallelIterator}; use serde::Deserialize; use std::collections::HashSet; diff --git a/tools/mirror/src/offline.rs b/tools/mirror/src/offline.rs index 3b7f5cc10db..4749c5f2558 100644 --- a/tools/mirror/src/offline.rs +++ b/tools/mirror/src/offline.rs @@ -15,7 +15,7 @@ use near_primitives::views::{ AccessKeyPermissionView, ExecutionOutcomeWithIdView, QueryRequest, QueryResponseKind, }; use near_primitives_core::types::ShardId; -use nearcore::NightshadeRuntime; +use nearcore::{NightshadeRuntime, NightshadeRuntimeExt}; use std::path::Path; use std::sync::Arc; diff --git a/tools/mock-node/src/setup.rs b/tools/mock-node/src/setup.rs index f236660c20d..3a32560ee3e 100644 --- a/tools/mock-node/src/setup.rs +++ b/tools/mock-node/src/setup.rs @@ -16,7 +16,7 @@ use near_primitives::state_part::PartId; use near_primitives::state_sync::get_num_state_parts; use near_primitives::types::{BlockHeight, NumShards, ShardId}; use near_store::test_utils::create_test_store; -use nearcore::{NearConfig, NightshadeRuntime}; +use nearcore::{NearConfig, NightshadeRuntime, NightshadeRuntimeExt}; use rayon::iter::{IntoParallelIterator, ParallelIterator}; use std::cmp::min; use std::path::Path; @@ -289,7 +289,6 @@ mod tests { use near_o11y::WithSpanContextExt; use near_primitives::transaction::SignedTransaction; use near_store::test_utils::gen_account_from_alphabet; - use nearcore::config::GenesisExt; use nearcore::{load_test_config, start_with_config, NEAR_BASE}; use rand::thread_rng; use std::ops::ControlFlow; diff --git a/tools/speedy_sync/src/main.rs b/tools/speedy_sync/src/main.rs index 88ed636735f..c2e6329b7e4 100644 --- a/tools/speedy_sync/src/main.rs +++ b/tools/speedy_sync/src/main.rs @@ -16,7 +16,7 @@ use near_primitives::types::EpochId; use near_primitives::utils::index_to_bytes; use near_store::HEADER_HEAD_KEY; use near_store::{DBCol, Mode, NodeStorage, Store, StoreUpdate}; -use nearcore::NightshadeRuntime; +use nearcore::{NightshadeRuntime, NightshadeRuntimeExt}; use std::fs; use std::path::Path; diff --git a/tools/state-viewer/src/apply_chain_range.rs b/tools/state-viewer/src/apply_chain_range.rs index 7df5f50f9cb..e95e7d2e9bd 100644 --- a/tools/state-viewer/src/apply_chain_range.rs +++ b/tools/state-viewer/src/apply_chain_range.rs @@ -461,7 +461,6 @@ mod test { use near_store::genesis::initialize_genesis_state; use near_store::test_utils::create_test_store; use near_store::Store; - use nearcore::config::GenesisExt; use nearcore::config::TESTING_INIT_STAKE; use nearcore::NightshadeRuntime; diff --git a/tools/state-viewer/src/apply_chunk.rs b/tools/state-viewer/src/apply_chunk.rs index 5b5f21090b5..c80f0634674 100644 --- a/tools/state-viewer/src/apply_chunk.rs +++ b/tools/state-viewer/src/apply_chunk.rs @@ -491,7 +491,6 @@ mod test { use near_primitives::utils::get_num_seats_per_shard; use near_store::genesis::initialize_genesis_state; use near_store::test_utils::create_test_store; - use nearcore::config::GenesisExt; use nearcore::NightshadeRuntime; use rand::rngs::StdRng; use rand::SeedableRng; diff --git a/tools/state-viewer/src/commands.rs b/tools/state-viewer/src/commands.rs index 544f014b8f4..65e63d936e6 100644 --- a/tools/state-viewer/src/commands.rs +++ b/tools/state-viewer/src/commands.rs @@ -39,6 +39,7 @@ use near_store::flat::FlatStorageManager; use near_store::test_utils::create_test_store; use near_store::TrieStorage; use near_store::{DBCol, Store, Trie, TrieCache, TrieCachingStorage, TrieConfig, TrieDBStorage}; +use nearcore::NightshadeRuntimeExt; use nearcore::{NearConfig, NightshadeRuntime}; use node_runtime::adapter::ViewRuntimeAdapter; use serde_json::json; @@ -1329,7 +1330,6 @@ mod tests { use near_primitives::types::AccountId; use near_store::genesis::initialize_genesis_state; use nearcore::config::Config; - use nearcore::config::GenesisExt; use nearcore::{NearConfig, NightshadeRuntime}; use std::sync::Arc; diff --git a/tools/state-viewer/src/state_changes.rs b/tools/state-viewer/src/state_changes.rs index bbdf8b6b450..e00b36bf08d 100644 --- a/tools/state-viewer/src/state_changes.rs +++ b/tools/state-viewer/src/state_changes.rs @@ -10,7 +10,7 @@ use near_primitives::types::{ }; use near_primitives_core::types::BlockHeight; use near_store::{KeyForStateChanges, Store, WrappedTrieChanges}; -use nearcore::{NearConfig, NightshadeRuntime}; +use nearcore::{NearConfig, NightshadeRuntime, NightshadeRuntimeExt}; use std::path::{Path, PathBuf}; #[derive(clap::Subcommand, Debug, Clone)] diff --git a/tools/state-viewer/src/state_dump.rs b/tools/state-viewer/src/state_dump.rs index d1bc76ac9de..119e85f222f 100644 --- a/tools/state-viewer/src/state_dump.rs +++ b/tools/state-viewer/src/state_dump.rs @@ -311,7 +311,6 @@ mod test { use near_store::genesis::initialize_genesis_state; use near_store::test_utils::create_test_store; use near_store::Store; - use nearcore::config::GenesisExt; use nearcore::config::TESTING_INIT_STAKE; use nearcore::config::{Config, NearConfig}; use nearcore::test_utils::TestEnvNightshadeSetupExt; diff --git a/tools/state-viewer/src/state_parts.rs b/tools/state-viewer/src/state_parts.rs index 9fc5c25239d..a364bb8ae7d 100644 --- a/tools/state-viewer/src/state_parts.rs +++ b/tools/state-viewer/src/state_parts.rs @@ -17,7 +17,7 @@ use near_primitives::types::{EpochId, StateRoot}; use near_primitives_core::hash::CryptoHash; use near_primitives_core::types::{BlockHeight, EpochHeight, ShardId}; use near_store::{PartialStorage, Store, Trie}; -use nearcore::{NearConfig, NightshadeRuntime}; +use nearcore::{NearConfig, NightshadeRuntime, NightshadeRuntimeExt}; use std::ops::Range; use std::path::{Path, PathBuf}; use std::str::FromStr;