Lightweight testlib

Cargo.toml

@@ -27,11 +27,11 @@ members = [
     "chain/jsonrpc-primitives",
     "chain/rosetta-rpc",
     "test-utils/actix-test-utils",
-    "test-utils/testlib",
     "test-utils/loadtester",
     "test-utils/runtime-tester",
     "test-utils/state-viewer",
     "test-utils/store-validator",
+    "test-utils/testlib",
     "neard",
     "nearcore",
     "tools/rpctypegen/core",

integration-tests/Cargo.toml

@@ -14,10 +14,13 @@ chrono = { version = "0.4.4", features = ["serde"] }
 funty = "=1.1.0" # Pin dependency to avoid compilation errors: https://github.com/myrrlyn/funty/issues/3
 futures = "0.3"
 hex = "0.4"
+lazy_static = "1.4"
 log = "0.4"
+primitive-types = "0.9"
 rand = "0.7"
 serde = { version = "1", features = ["derive"] }
 serde_json = "1"
+tempfile = "3"
 
 near-actix-test-utils = { path = "../test-utils/actix-test-utils" }
 near-chain = { path = "../chain/chain" }
@@ -26,6 +29,8 @@ near-chunks = { path = "../chain/chunks" }
 near-client = { path = "../chain/client" }
 near-client-primitives = { path = "../chain/client-primitives" }
 near-crypto = { path = "../core/crypto" }
+near-jsonrpc = { path = "../chain/jsonrpc" }
+near-jsonrpc-client = { path = "../chain/jsonrpc/client" }
 near-jsonrpc-primitives = { path = "../chain/jsonrpc-primitives" }
 near-logger-utils = { path = "../test-utils/logger" }
 near-metrics = { path = "../core/metrics" }
@@ -47,7 +52,7 @@ adversarial = ["nearcore/adversarial"]
 metric_recorder = ["near-client-primitives/metric_recorder"]
 protocol_feature_alt_bn128 = ["nearcore/protocol_feature_alt_bn128"]
 protocol_feature_block_header_v3 = ["near-primitives/protocol_feature_block_header_v3", "near-chain/protocol_feature_block_header_v3", "near-store/protocol_feature_block_header_v3"]
-nightly_protocol_features = ["nearcore/nightly_protocol_features", "testlib/nightly_protocol_features", "protocol_feature_alt_bn128", "protocol_feature_block_header_v3", "protocol_feature_restore_receipts_after_fix"]
-nightly_protocol = ["nearcore/nightly_protocol", "testlib/nightly_protocol"]
+nightly_protocol_features = ["nearcore/nightly_protocol_features", "protocol_feature_alt_bn128", "protocol_feature_block_header_v3", "protocol_feature_restore_receipts_after_fix"]
+nightly_protocol = ["nearcore/nightly_protocol"]
 protocol_feature_restore_receipts_after_fix = ["nearcore/protocol_feature_restore_receipts_after_fix"]
 sandbox = ["near-network/sandbox", "near-chain/sandbox", "node-runtime/sandbox", "near-client/sandbox"]

integration-tests/src/genesis_helpers.rs

@@ -0,0 +1,37 @@
+use std::sync::Arc;
+
+use tempfile::tempdir;
+
+use near_chain::{Chain, ChainGenesis, DoomslugThresholdMode};
+use near_chain_configs::Genesis;
+use near_primitives::block::{Block, BlockHeader};
+use near_primitives::hash::CryptoHash;
+use near_store::test_utils::create_test_store;
+use nearcore::NightshadeRuntime;
+
+/// Compute genesis hash from genesis.
+pub fn genesis_hash(genesis: &Genesis) -> CryptoHash {
+    *genesis_header(genesis).hash()
+}
+
+/// Utility to generate genesis header from config for testing purposes.
+pub fn genesis_header(genesis: &Genesis) -> BlockHeader {
+    let dir = tempdir().unwrap();
+    let store = create_test_store();
+    let chain_genesis = ChainGenesis::from(genesis);
+    let runtime =
+        Arc::new(NightshadeRuntime::new(dir.path(), store, genesis, vec![], vec![], None, None));
+    let chain = Chain::new(runtime, &chain_genesis, DoomslugThresholdMode::TwoThirds).unwrap();
+    chain.genesis().clone()
+}
+
+/// Utility to generate genesis header from config for testing purposes.
+pub fn genesis_block(genesis: &Genesis) -> Block {
+    let dir = tempdir().unwrap();
+    let store = create_test_store();
+    let chain_genesis = ChainGenesis::from(genesis);
+    let runtime =
+        Arc::new(NightshadeRuntime::new(dir.path(), store, genesis, vec![], vec![], None, None));
+    let mut chain = Chain::new(runtime, &chain_genesis, DoomslugThresholdMode::TwoThirds).unwrap();
+    chain.get_block(&chain.genesis().hash().clone()).unwrap().clone()
+}

integration-tests/src/lib.rs

@@ -0,0 +1,5 @@
+pub mod genesis_helpers;
+pub mod node;
+pub mod runtime_utils;
+pub mod test_helpers;
+pub mod user;

test-utils/testlib/src/node/mod.rs → integration-tests/src/node/mod.rs

@@ -1,9 +1,15 @@
 use std::sync::Arc;
 use std::sync::RwLock;
 
+pub use crate::node::process_node::ProcessNode;
+pub use crate::node::runtime_node::RuntimeNode;
+pub use crate::node::thread_node::ThreadNode;
+use crate::user::{AsyncUser, User};
 use near_chain_configs::Genesis;
 use near_crypto::{InMemorySigner, Signer};
 use near_jsonrpc_primitives::errors::ServerError;
+use near_primitives::contract::ContractCode;
+use near_primitives::num_rational::Rational;
 use near_primitives::state_record::StateRecord;
 use near_primitives::transaction::SignedTransaction;
 use near_primitives::types::{AccountId, Balance, NumSeats};
@@ -13,13 +19,7 @@ use nearcore::config::{
     create_testnet_configs, create_testnet_configs_from_seeds, Config, GenesisExt,
 };
 use nearcore::NearConfig;
-
-pub use crate::node::process_node::ProcessNode;
-pub use crate::node::runtime_node::RuntimeNode;
-pub use crate::node::thread_node::ThreadNode;
-use crate::user::{AsyncUser, User};
-use near_primitives::contract::ContractCode;
-use num_rational::Rational;
+use testlib::runtime_utils::{alice_account, bob_account};
 
 mod process_node;
 mod runtime_node;
@@ -29,7 +29,6 @@ pub const TEST_BLOCK_FETCH_LIMIT: u64 = 5;
 pub const TEST_BLOCK_MAX_SIZE: u32 = 1000;
 
 pub fn configure_chain_spec() -> Genesis {
-    use super::runtime_utils::{alice_account, bob_account};
     Genesis::test(vec![alice_account(), bob_account()], 2)
 }
 

test-utils/testlib/src/node/runtime_node.rs → integration-tests/src/node/runtime_node.rs

@@ -6,11 +6,11 @@ use near_primitives::types::AccountId;
 use nearcore::config::GenesisExt;
 
 use crate::node::Node;
-use crate::runtime_utils::{
-    add_test_contract, alice_account, bob_account, get_runtime_and_trie_from_genesis,
-};
 use crate::user::runtime_user::MockClient;
 use crate::user::{RuntimeUser, User};
+use testlib::runtime_utils::{
+    add_test_contract, alice_account, bob_account, get_runtime_and_trie_from_genesis,
+};
 
 pub struct RuntimeNode {
     pub client: Arc<RwLock<MockClient>>,
@@ -90,10 +90,10 @@ impl Node for RuntimeNode {
 
 #[cfg(test)]
 mod tests {
-    use crate::fees_utils::FeeHelper;
     use crate::node::runtime_node::RuntimeNode;
     use crate::node::Node;
-    use crate::runtime_utils::{alice_account, bob_account};
+    use testlib::fees_utils::FeeHelper;
+    use testlib::runtime_utils::{alice_account, bob_account};
 
     #[test]
     pub fn test_send_money() {

integration-tests/src/runtime_utils.rs

@@ -0,0 +1,22 @@
+use near_chain_configs::Genesis;
+use near_primitives::types::StateRoot;
+use near_store::{ShardTries, TrieUpdate};
+use nearcore::config::GenesisExt;
+use node_runtime::{state_viewer::TrieViewer, Runtime};
+use testlib::runtime_utils::{
+    add_test_contract, alice_account, bob_account, get_runtime_and_trie_from_genesis,
+};
+
+pub fn get_runtime_and_trie() -> (Runtime, ShardTries, StateRoot) {
+    let mut genesis =
+        Genesis::test(vec![alice_account(), bob_account(), "carol.near".parse().unwrap()], 3);
+    add_test_contract(&mut genesis, &"test.contract".parse().unwrap());
+    get_runtime_and_trie_from_genesis(&genesis)
+}
+
+pub fn get_test_trie_viewer() -> (TrieViewer, TrieUpdate) {
+    let (_, tries, root) = get_runtime_and_trie();
+    let trie_viewer = TrieViewer::default();
+    let state_update = tries.new_trie_update(0, root);
+    (trie_viewer, state_update)
+}

integration-tests/tests/client/chunks_management.rs

@@ -7,6 +7,7 @@ use actix::{Addr, System};
 use futures::{future, FutureExt};
 use log::info;
 
+use integration_tests::test_helpers::heavy_test;
 use near_actix_test_utils::run_actix;
 use near_chunks::{
     CHUNK_REQUEST_RETRY_MS, CHUNK_REQUEST_SWITCH_TO_FULL_FETCH_MS,
@@ -20,7 +21,6 @@ use near_network::{NetworkClientMessages, NetworkRequests, NetworkResponses, Pee
 use near_primitives::hash::CryptoHash;
 use near_primitives::transaction::SignedTransaction;
 use near_primitives::types::AccountId;
-use testlib::test_helpers::heavy_test;
 
 /// Runs block producing client and stops after network mock received seven blocks
 /// Confirms that the blocks form a chain (which implies the chunks are distributed).

integration-tests/tests/nearcore/main.rs

@@ -0,0 +1,8 @@
+mod node_cluster;
+mod rpc_error_structs;
+mod rpc_nodes;
+mod run_nodes;
+mod stake_nodes;
+mod sync_nodes;
+mod sync_state_nodes;
+mod track_shards;

nearcore/tests/node_cluster.rs → integration-tests/tests/nearcore/node_cluster.rs

@@ -1,9 +1,74 @@
+use actix::Addr;
+use actix_rt::ArbiterHandle;
 use futures::future;
-
+use integration_tests::test_helpers::heavy_test;
 use near_actix_test_utils::{run_actix, spawn_interruptible};
+use near_chain_configs::Genesis;
 use near_client::{ClientActor, ViewClientActor};
-use near_primitives::types::{BlockHeight, BlockHeightDelta, NumSeats, NumShards};
-use testlib::{start_nodes, test_helpers::heavy_test};
+use near_logger_utils::init_integration_logger;
+use near_network::test_utils::{convert_boot_nodes, open_port};
+use near_primitives::types::{BlockHeight, BlockHeightDelta, NumSeats, NumShards, ShardId};
+use nearcore::{config::GenesisExt, load_test_config, start_with_config};
+use tempfile::TempDir;
+
+pub fn start_nodes(
+    num_shards: NumShards,
+    dirs: &[TempDir],
+    num_validator_seats: NumSeats,
+    num_lightclient: usize,
+    epoch_length: BlockHeightDelta,
+    genesis_height: BlockHeight,
+) -> (Genesis, Vec<String>, Vec<(Addr<ClientActor>, Addr<ViewClientActor>, Vec<ArbiterHandle>)>) {
+    init_integration_logger();
+
+    let num_nodes = dirs.len();
+    let num_tracking_nodes = num_nodes - num_lightclient;
+    let seeds = (0..num_nodes).map(|i| format!("near.{}", i)).collect::<Vec<_>>();
+    let mut genesis = Genesis::test_sharded(
+        seeds.iter().map(|s| s.parse().unwrap()).collect(),
+        num_validator_seats,
+        (0..num_shards).map(|_| num_validator_seats).collect(),
+    );
+    genesis.config.epoch_length = epoch_length;
+    genesis.config.genesis_height = genesis_height;
+
+    let validators = (0..num_validator_seats).map(|i| format!("near.{}", i)).collect::<Vec<_>>();
+    let mut near_configs = vec![];
+    let first_node = open_port();
+    let mut rpc_addrs = vec![];
+    for i in 0..num_nodes {
+        let mut near_config = load_test_config(
+            if i < num_validator_seats as usize { &validators[i] } else { "" },
+            if i == 0 { first_node } else { open_port() },
+            genesis.clone(),
+        );
+        rpc_addrs.push(near_config.rpc_addr().unwrap().clone());
+        near_config.client_config.min_num_peers = num_nodes - 1;
+        if i > 0 {
+            near_config.network_config.boot_nodes =
+                convert_boot_nodes(vec![("near.0", first_node)]);
+        }
+        // if non validator, add some shards to track.
+        if i >= (num_validator_seats as usize) && i < num_tracking_nodes {
+            let shards_per_node =
+                num_shards as usize / (num_tracking_nodes - num_validator_seats as usize);
+            let (from, to) = (
+                ((i - num_validator_seats as usize) * shards_per_node) as ShardId,
+                ((i - (num_validator_seats as usize) + 1) * shards_per_node) as ShardId,
+            );
+            near_config.client_config.tracked_shards.extend(&(from..to).collect::<Vec<_>>());
+        }
+        near_config.client_config.epoch_sync_enabled = false;
+        near_configs.push(near_config);
+    }
+
+    let mut res = vec![];
+    for (i, near_config) in near_configs.into_iter().enumerate() {
+        let (client, view_client, arbiters) = start_with_config(dirs[i].path(), near_config);
+        res.push((client, view_client, arbiters))
+    }
+    (genesis, rpc_addrs, res)
+}
 
 #[derive(Debug, Default)]
 pub struct NodeCluster {

nearcore/tests/rpc_error_structs.rs → integration-tests/tests/nearcore/rpc_error_structs.rs

@@ -2,26 +2,21 @@ use std::str::FromStr;
 
 use actix::{Actor, System};
 use borsh::BorshSerialize;
-
 use futures::{future, FutureExt, TryFutureExt};
 
+use crate::node_cluster::NodeCluster;
+use integration_tests::genesis_helpers::genesis_block;
 use near_actix_test_utils::spawn_interruptible;
 use near_client::GetBlock;
 use near_crypto::{InMemorySigner, KeyType};
 use near_jsonrpc::client::new_client;
 use near_logger_utils::init_integration_logger;
 use near_network::test_utils::WaitOrTimeout;
 use near_primitives::hash::CryptoHash;
-
 use near_primitives::serialize::to_base64;
 use near_primitives::transaction::SignedTransaction;
 use near_primitives::types::BlockId;
 
-use testlib::genesis_block;
-
-mod node_cluster;
-use node_cluster::NodeCluster;
-
 // Queries json-rpc block that doesn't exists
 // Checks if the struct is expected and contains the proper data
 #[test]