From 974827a174df70d13d76783c8f86fea4ace78f3d Mon Sep 17 00:00:00 2001 From: Sagar Dhawan Date: Tue, 10 Sep 2019 14:42:21 -0700 Subject: [PATCH 01/23] Limit threadpool threads and remove serial_test --- Cargo.lock | 39 +++++++++------------------- bench-tps/Cargo.toml | 4 --- bench-tps/src/bench.rs | 2 -- ci/test-stable.sh | 2 +- core/Cargo.toml | 4 +-- core/src/blocktree_processor.rs | 11 ++++---- core/src/entry.rs | 5 ++-- core/src/sigverify.rs | 9 +++---- core/src/window_service.rs | 3 ++- core/tests/cluster_info.rs | 4 --- local_cluster/Cargo.toml | 5 +--- local_cluster/src/local_cluster.rs | 3 +++ local_cluster/tests/local_cluster.rs | 31 +++------------------- local_cluster/tests/replicator.rs | 5 ---- metrics/Cargo.toml | 2 -- metrics/src/counter.rs | 6 ----- runtime/Cargo.toml | 1 + runtime/src/accounts_db.rs | 4 +-- 18 files changed, 40 insertions(+), 100 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index c8f05f7d3f34ac..13ba09c310b085 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2628,23 +2628,6 @@ dependencies = [ "yaml-rust 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)", ] -[[package]] -name = "serial_test" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "serial_test_derive" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "quote 0.6.13 (registry+https://github.com/rust-lang/crates.io-index)", - "syn 0.15.42 (registry+https://github.com/rust-lang/crates.io-index)", -] - [[package]] name = "sha-1" version = "0.8.1" @@ -2849,8 +2832,6 @@ dependencies = [ "serde_derive 1.0.100 (registry+https://github.com/rust-lang/crates.io-index)", "serde_json 1.0.40 (registry+https://github.com/rust-lang/crates.io-index)", "serde_yaml 0.8.9 (registry+https://github.com/rust-lang/crates.io-index)", - "serial_test 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", - "serial_test_derive 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", "solana-client 0.19.0-pre0", "solana-core 0.19.0-pre0", "solana-drone 0.19.0-pre0", @@ -3084,6 +3065,7 @@ dependencies = [ "jsonrpc-http-server 13.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "jsonrpc-pubsub 13.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "jsonrpc-ws-server 13.1.0 (registry+https://github.com/rust-lang/crates.io-index)", + "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)", "libc 0.2.62 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", "matches 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)", @@ -3098,8 +3080,6 @@ dependencies = [ "serde 1.0.100 (registry+https://github.com/rust-lang/crates.io-index)", "serde_derive 1.0.100 (registry+https://github.com/rust-lang/crates.io-index)", "serde_json 1.0.40 (registry+https://github.com/rust-lang/crates.io-index)", - "serial_test 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", - "serial_test_derive 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", "solana-budget-api 0.19.0-pre0", "solana-budget-program 0.19.0-pre0", "solana-chacha-sys 0.19.0-pre0", @@ -3112,6 +3092,7 @@ dependencies = [ "solana-merkle-tree 0.19.0-pre0", "solana-metrics 0.19.0-pre0", "solana-netutil 0.19.0-pre0", + "solana-rayon-threadlimit 0.19.0-pre0", "solana-runtime 0.19.0-pre0", "solana-sdk 0.19.0-pre0", "solana-stake-api 0.19.0-pre0", @@ -3355,11 +3336,10 @@ version = "0.19.0-pre0" dependencies = [ "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", "rand 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)", - "serial_test 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", - "serial_test_derive 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", "solana-client 0.19.0-pre0", "solana-core 0.19.0-pre0", "solana-logger 0.19.0-pre0", + "solana-rayon-threadlimit 0.19.0-pre0", "solana-runtime 0.19.0-pre0", "solana-sdk 0.19.0-pre0", "solana-stake-api 0.19.0-pre0", @@ -3402,8 +3382,6 @@ dependencies = [ "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", "rand 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)", - "serial_test 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", - "serial_test_derive 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", "solana-sdk 0.19.0-pre0", "sys-info 0.5.7 (registry+https://github.com/rust-lang/crates.io-index)", "ureq 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)", @@ -3472,6 +3450,14 @@ dependencies = [ "solana-sdk 0.19.0-pre0", ] +[[package]] +name = "solana-rayon-threadlimit" +version = "0.19.0-pre0" +dependencies = [ + "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)", + "sys-info 0.5.7 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "solana-replicator" version = "0.19.0-pre0" @@ -3510,6 +3496,7 @@ dependencies = [ "solana-measure 0.19.0-pre0", "solana-metrics 0.19.0-pre0", "solana-noop-program 0.19.0-pre0", + "solana-rayon-threadlimit 0.19.0-pre0", "solana-sdk 0.19.0-pre0", "solana-stake-api 0.19.0-pre0", "solana-stake-program 0.19.0-pre0", @@ -5323,8 +5310,6 @@ dependencies = [ "checksum serde_derive 1.0.100 (registry+https://github.com/rust-lang/crates.io-index)" = "11e410fde43e157d789fc290d26bc940778ad0fdd47836426fbac36573710dbb" "checksum serde_json 1.0.40 (registry+https://github.com/rust-lang/crates.io-index)" = "051c49229f282f7c6f3813f8286cc1e3323e8051823fce42c7ea80fe13521704" "checksum serde_yaml 0.8.9 (registry+https://github.com/rust-lang/crates.io-index)" = "38b08a9a90e5260fe01c6480ec7c811606df6d3a660415808c3c3fa8ed95b582" -"checksum serial_test 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "50bfbc39343545618d97869d77f38ed43e48dd77432717dbc7ed39d797f3ecbe" -"checksum serial_test_derive 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "89dd85be2e2ad75b041c9df2892ac078fa6e0b90024028b2b9fb4125b7530f01" "checksum sha-1 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)" = "23962131a91661d643c98940b20fcaffe62d776a823247be80a48fcb8b6fce68" "checksum sha2 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)" = "9eb6be24e4c23a84d7184280d2722f7f2731fcdd4a9d886efbfe4413e4847ea0" "checksum sha2 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)" = "7b4d8bfd0e469f417657573d8451fb33d16cfe0989359b93baf3a1ffc639543d" diff --git a/bench-tps/Cargo.toml b/bench-tps/Cargo.toml index 62f9d3afdd5ff4..7e92b8161c95a7 100644 --- a/bench-tps/Cargo.toml +++ b/bench-tps/Cargo.toml @@ -31,10 +31,6 @@ solana-sdk = { path = "../sdk", version = "0.19.0-pre0" } solana-move-loader-program = { path = "../programs/move_loader_program", version = "0.19.0-pre0" } solana-move-loader-api = { path = "../programs/move_loader_api", version = "0.19.0-pre0" } -[dev-dependencies] -serial_test = "0.2.0" -serial_test_derive = "0.2.0" - [features] cuda = ["solana-core/cuda"] diff --git a/bench-tps/src/bench.rs b/bench-tps/src/bench.rs index fc0072c97123c8..16ec27c0efe124 100644 --- a/bench-tps/src/bench.rs +++ b/bench-tps/src/bench.rs @@ -1028,7 +1028,6 @@ mod tests { } #[test] - #[serial] fn test_bench_tps_local_cluster_solana() { let mut config = Config::default(); config.tx_count = 100; @@ -1038,7 +1037,6 @@ mod tests { } #[test] - #[serial] fn test_bench_tps_local_cluster_move() { let mut config = Config::default(); config.tx_count = 100; diff --git a/ci/test-stable.sh b/ci/test-stable.sh index a82b3aa83f2a65..bee4e9642e0ef2 100755 --- a/ci/test-stable.sh +++ b/ci/test-stable.sh @@ -77,7 +77,7 @@ test-stable-perf) # Run root package library tests _ cargo +"$rust_stable" build --all --tests --bins ${V:+--verbose} --features="$ROOT_FEATURES" - _ cargo +"$rust_stable" test --manifest-path=core/Cargo.toml ${V:+--verbose} --features="$ROOT_FEATURES" -- --nocapture + _ cargo +"$rust_stable" test --manifest-path=core/Cargo.toml ${V:+--verbose} --features="$ROOT_FEATURES" -- --nocapture --test-threads=1 ;; *) echo "Error: Unknown test: $testName" diff --git a/core/Cargo.toml b/core/Cargo.toml index 0b3f5e977ff4e3..91f1ffb7499ebd 100644 --- a/core/Cargo.toml +++ b/core/Cargo.toml @@ -75,6 +75,8 @@ tokio-codec = "0.1" tokio-fs = "0.1" tokio-io = "0.1" untrusted = "0.7.0" +lazy_static = "1.4.0" +solana-rayon-threadlimit = { path = "../rayon-threadlimit", version = "0.19.0-pre0" } # reed-solomon-erasure's simd_c feature fails to build for x86_64-pc-windows-msvc, use pure-rust [target.'cfg(windows)'.dependencies] @@ -92,8 +94,6 @@ features = ["lz4"] [dev-dependencies] hex-literal = "0.2.1" matches = "0.1.6" -serial_test = "0.2.0" -serial_test_derive = "0.2.0" ureq = { version = "0.11.0", default-features = false, features = ["json"] } [[bench]] diff --git a/core/src/blocktree_processor.rs b/core/src/blocktree_processor.rs index 272fcc2c5a08f5..9c374d8e337c3c 100644 --- a/core/src/blocktree_processor.rs +++ b/core/src/blocktree_processor.rs @@ -2,6 +2,8 @@ use crate::bank_forks::BankForks; use crate::blocktree::{Blocktree, SlotMeta}; use crate::entry::{Entry, EntrySlice}; use crate::leader_schedule_cache::LeaderScheduleCache; +use rand::seq::SliceRandom; +use rand::thread_rng; use rayon::prelude::*; use rayon::ThreadPool; use solana_metrics::{datapoint, datapoint_error, inc_new_counter_debug}; @@ -16,16 +18,15 @@ use std::result; use std::sync::Arc; use std::time::{Duration, Instant}; -use rand::seq::SliceRandom; -use rand::thread_rng; - pub const NUM_THREADS: u32 = 10; +use solana_rayon_threadlimit::thread_count::get_thread_count; use std::cell::RefCell; thread_local!(static PAR_THREAD_POOL: RefCell = RefCell::new(rayon::ThreadPoolBuilder::new() - .num_threads(sys_info::cpu_num().unwrap_or(NUM_THREADS) as usize) + .num_threads(get_thread_count()) .build() - .unwrap())); + .unwrap()) +); fn first_err(results: &[Result<()>]) -> Result<()> { for r in results { diff --git a/core/src/entry.rs b/core/src/entry.rs index e156bf4504a409..94d9d7cd3ae996 100644 --- a/core/src/entry.rs +++ b/core/src/entry.rs @@ -14,6 +14,7 @@ use solana_merkle_tree::MerkleTree; use solana_metrics::inc_new_counter_warn; use solana_sdk::hash::Hash; use solana_sdk::signature::{Keypair, KeypairUtil}; +use solana_sdk::timing; use solana_sdk::transaction::Transaction; use std::borrow::Borrow; use std::cell::RefCell; @@ -22,7 +23,7 @@ use std::sync::{Arc, RwLock}; #[cfg(feature = "cuda")] use crate::sigverify::poh_verify_many; -use solana_sdk::timing; +use solana_rayon_threadlimit::thread_count::get_thread_count; #[cfg(feature = "cuda")] use std::sync::Mutex; #[cfg(feature = "cuda")] @@ -32,7 +33,7 @@ use std::time::Instant; pub const NUM_THREADS: u32 = 10; thread_local!(static PAR_THREAD_POOL: RefCell = RefCell::new(rayon::ThreadPoolBuilder::new() - .num_threads(sys_info::cpu_num().unwrap_or(NUM_THREADS) as usize) + .num_threads(get_thread_count()) .build() .unwrap())); diff --git a/core/src/sigverify.rs b/core/src/sigverify.rs index 6a7a997baaae34..10b24bfaafa287 100644 --- a/core/src/sigverify.rs +++ b/core/src/sigverify.rs @@ -19,17 +19,16 @@ use solana_sdk::signature::Signature; use solana_sdk::transaction::Transaction; use std::mem::size_of; -#[cfg(feature = "cuda")] -use std::os::raw::{c_int, c_uint}; - #[cfg(feature = "cuda")] use core::ffi::c_void; - +use solana_rayon_threadlimit::thread_count::get_thread_count; +#[cfg(feature = "cuda")] +use std::os::raw::{c_int, c_uint}; pub const NUM_THREADS: u32 = 10; use std::cell::RefCell; thread_local!(static PAR_THREAD_POOL: RefCell = RefCell::new(rayon::ThreadPoolBuilder::new() - .num_threads(sys_info::cpu_num().unwrap_or(NUM_THREADS) as usize) + .num_threads(get_thread_count()) .build() .unwrap())); diff --git a/core/src/window_service.rs b/core/src/window_service.rs index 153d762a596522..b9827803b9fe86 100644 --- a/core/src/window_service.rs +++ b/core/src/window_service.rs @@ -12,6 +12,7 @@ use crate::streamer::{PacketReceiver, PacketSender}; use rayon::iter::{IndexedParallelIterator, IntoParallelRefMutIterator, ParallelIterator}; use rayon::ThreadPool; use solana_metrics::{inc_new_counter_debug, inc_new_counter_error}; +use solana_rayon_threadlimit::thread_count::get_thread_count; use solana_runtime::bank::Bank; use solana_sdk::pubkey::Pubkey; use solana_sdk::timing::duration_as_ms; @@ -205,7 +206,7 @@ impl WindowService { trace!("{}: RECV_WINDOW started", id); let mut now = Instant::now(); let thread_pool = rayon::ThreadPoolBuilder::new() - .num_threads(sys_info::cpu_num().unwrap_or(NUM_THREADS) as usize) + .num_threads(get_thread_count()) .build() .unwrap(); loop { diff --git a/core/tests/cluster_info.rs b/core/tests/cluster_info.rs index 1cd9407bb235cd..f3466a007d20c4 100644 --- a/core/tests/cluster_info.rs +++ b/core/tests/cluster_info.rs @@ -191,7 +191,6 @@ fn run_simulation(stakes: &[u64], fanout: usize) { // Run with a single layer #[test] -#[serial] fn test_retransmit_small() { let stakes: Vec<_> = (0..200).map(|i| i).collect(); run_simulation(&stakes, 200); @@ -199,7 +198,6 @@ fn test_retransmit_small() { // Make sure at least 2 layers are used #[test] -#[serial] fn test_retransmit_medium() { let num_nodes = 2000; let stakes: Vec<_> = (0..num_nodes).map(|i| i).collect(); @@ -208,7 +206,6 @@ fn test_retransmit_medium() { // Make sure at least 2 layers are used but with equal stakes #[test] -#[serial] fn test_retransmit_medium_equal_stakes() { let num_nodes = 2000; let stakes: Vec<_> = (0..num_nodes).map(|_| 10).collect(); @@ -217,7 +214,6 @@ fn test_retransmit_medium_equal_stakes() { // Scale down the network and make sure many layers are used #[test] -#[serial] fn test_retransmit_large() { let num_nodes = 4000; let stakes: Vec<_> = (0..num_nodes).map(|i| i).collect(); diff --git a/local_cluster/Cargo.toml b/local_cluster/Cargo.toml index f9d3649576167c..f90ef4d8fc4f45 100644 --- a/local_cluster/Cargo.toml +++ b/local_cluster/Cargo.toml @@ -22,10 +22,7 @@ solana-storage-program = { path = "../programs/storage_program", version = "0.19 solana-vote-api = { path = "../programs/vote_api", version = "0.19.0-pre0" } symlink = "0.1.0" tempfile = "3.1.0" - -[dev-dependencies] -serial_test = "0.2.0" -serial_test_derive = "0.2.0" +solana-rayon-threadlimit = { path = "../rayon-threadlimit", version = "0.19.0-pre0" } [features] cuda = ["solana-core/cuda"] diff --git a/local_cluster/src/local_cluster.rs b/local_cluster/src/local_cluster.rs index fdd8ddef844ba8..7ee28c2017d707 100644 --- a/local_cluster/src/local_cluster.rs +++ b/local_cluster/src/local_cluster.rs @@ -10,6 +10,7 @@ use solana_core::{ service::Service, validator::{Validator, ValidatorConfig}, }; +use solana_rayon_threadlimit::thread_count::init_test_thread_count; use solana_sdk::{ client::SyncClient, clock::DEFAULT_TICKS_PER_SLOT, @@ -116,6 +117,8 @@ impl LocalCluster { } pub fn new(config: &ClusterConfig) -> Self { + init_test_thread_count(); + assert_eq!(config.validator_configs.len(), config.node_stakes.len()); let leader_keypair = Arc::new(Keypair::new()); let leader_pubkey = leader_keypair.pubkey(); diff --git a/local_cluster/tests/local_cluster.rs b/local_cluster/tests/local_cluster.rs index 4033d29e4067b4..16656a9ce48ef4 100644 --- a/local_cluster/tests/local_cluster.rs +++ b/local_cluster/tests/local_cluster.rs @@ -26,9 +26,6 @@ use std::{ use tempfile::TempDir; #[test] -#[serial] -#[allow(unused_attributes)] -#[ignore] fn test_ledger_cleanup_service() { solana_logger::setup(); error!("test_ledger_cleanup_service"); @@ -67,7 +64,6 @@ fn test_ledger_cleanup_service() { } #[test] -#[serial] fn test_spend_and_verify_all_nodes_1() { solana_logger::setup(); error!("test_spend_and_verify_all_nodes_1"); @@ -82,7 +78,6 @@ fn test_spend_and_verify_all_nodes_1() { } #[test] -#[serial] fn test_spend_and_verify_all_nodes_2() { solana_logger::setup(); error!("test_spend_and_verify_all_nodes_2"); @@ -97,7 +92,6 @@ fn test_spend_and_verify_all_nodes_2() { } #[test] -#[serial] fn test_spend_and_verify_all_nodes_3() { solana_logger::setup(); error!("test_spend_and_verify_all_nodes_3"); @@ -111,9 +105,8 @@ fn test_spend_and_verify_all_nodes_3() { ); } -#[allow(unused_attributes)] #[test] -#[serial] +#[allow(unused_attributes)] #[ignore] fn test_spend_and_verify_all_nodes_env_num_nodes() { solana_logger::setup(); @@ -132,7 +125,6 @@ fn test_spend_and_verify_all_nodes_env_num_nodes() { #[allow(unused_attributes)] #[test] -#[serial] #[should_panic] fn test_fullnode_exit_default_config_should_panic() { solana_logger::setup(); @@ -143,7 +135,6 @@ fn test_fullnode_exit_default_config_should_panic() { } #[test] -#[serial] fn test_fullnode_exit_2() { solana_logger::setup(); error!("test_fullnode_exit_2"); @@ -161,10 +152,7 @@ fn test_fullnode_exit_2() { } // Cluster needs a supermajority to remain, so the minimum size for this test is 4 -#[allow(unused_attributes)] #[test] -#[serial] -#[ignore] fn test_leader_failure_4() { solana_logger::setup(); error!("test_leader_failure_4"); @@ -186,7 +174,6 @@ fn test_leader_failure_4() { ); } #[test] -#[serial] fn test_two_unbalanced_stakes() { solana_logger::setup(); error!("test_two_unbalanced_stakes"); @@ -222,14 +209,13 @@ fn test_two_unbalanced_stakes() { } #[test] -#[ignore] fn test_forwarding() { // Set up a cluster where one node is never the leader, so all txs sent to this node // will be have to be forwarded in order to be confirmed let config = ClusterConfig { node_stakes: vec![999_990, 3], cluster_lamports: 2_000_000, - validator_configs: vec![ValidatorConfig::default(); 3], + validator_configs: vec![ValidatorConfig::default(); 2], ..ClusterConfig::default() }; let cluster = LocalCluster::new(&config); @@ -249,7 +235,6 @@ fn test_forwarding() { } #[test] -#[serial] fn test_restart_node() { solana_logger::setup(); error!("test_restart_node"); @@ -287,7 +272,6 @@ fn test_restart_node() { } #[test] -#[serial] fn test_listener_startup() { let config = ClusterConfig { node_stakes: vec![100; 1], @@ -303,7 +287,6 @@ fn test_listener_startup() { #[allow(unused_attributes)] #[test] -#[serial] fn test_snapshot_restart_locktower() { // First set up the cluster with 2 nodes let snapshot_interval_slots = 10; @@ -363,9 +346,7 @@ fn test_snapshot_restart_locktower() { ); } -#[allow(unused_attributes)] #[test] -#[serial] fn test_snapshots_blocktree_floor() { // First set up the cluster with 1 snapshotting leader let snapshot_interval_slots = 10; @@ -447,7 +428,6 @@ fn test_snapshots_blocktree_floor() { } #[test] -#[serial] fn test_snapshots_restart_validity() { solana_logger::setup(); let snapshot_interval_slots = 10; @@ -524,15 +504,13 @@ fn test_snapshots_restart_validity() { } } -#[allow(unused_attributes)] #[test] -#[serial] -#[ignore] fn test_fail_entry_verification_leader() { test_faulty_node(BroadcastStageType::FailEntryVerification); } #[test] +#[allow(unused_attributes)] #[ignore] fn test_fake_blobs_broadcast_leader() { test_faulty_node(BroadcastStageType::BroadcastFakeBlobs); @@ -591,10 +569,7 @@ fn test_faulty_node(faulty_node_type: BroadcastStageType) { ); } -#[allow(unused_attributes)] #[test] -#[serial] -#[ignore] fn test_repairman_catchup() { solana_logger::setup(); error!("test_repairman_catchup"); diff --git a/local_cluster/tests/replicator.rs b/local_cluster/tests/replicator.rs index e1e5537e2f293d..9ee6ed37724b8a 100644 --- a/local_cluster/tests/replicator.rs +++ b/local_cluster/tests/replicator.rs @@ -74,19 +74,16 @@ fn run_replicator_startup_basic(num_nodes: usize, num_replicators: usize) { } #[test] -#[serial] fn test_replicator_startup_1_node() { run_replicator_startup_basic(1, 1); } #[test] -#[serial] fn test_replicator_startup_2_nodes() { run_replicator_startup_basic(2, 1); } #[test] -#[serial] fn test_replicator_startup_leader_hang() { use std::net::{IpAddr, Ipv4Addr, SocketAddr}; @@ -125,7 +122,6 @@ fn test_replicator_startup_leader_hang() { } #[test] -#[serial] fn test_replicator_startup_ledger_hang() { solana_logger::setup(); info!("starting replicator test"); @@ -154,7 +150,6 @@ fn test_replicator_startup_ledger_hang() { } #[test] -#[serial] fn test_account_setup() { let num_nodes = 1; let num_replicators = 1; diff --git a/metrics/Cargo.toml b/metrics/Cargo.toml index 9906c8915da685..6c526f0d455303 100644 --- a/metrics/Cargo.toml +++ b/metrics/Cargo.toml @@ -18,8 +18,6 @@ ureq = "0.11.0" [dev-dependencies] rand = "0.6.5" -serial_test = "0.2.0" -serial_test_derive = "0.2.0" [lib] name = "solana_metrics" diff --git a/metrics/src/counter.rs b/metrics/src/counter.rs index 661d00be4b18de..a630b6b4fc4e89 100644 --- a/metrics/src/counter.rs +++ b/metrics/src/counter.rs @@ -225,7 +225,6 @@ mod tests { } #[test] - #[serial] fn test_counter() { env_logger::Builder::from_env(env_logger::Env::new().default_filter_or("solana=info")) .try_init() @@ -257,7 +256,6 @@ mod tests { } #[test] - #[serial] fn test_metricsrate() { env_logger::Builder::from_env(env_logger::Env::new().default_filter_or("solana=info")) .try_init() @@ -275,7 +273,6 @@ mod tests { } #[test] - #[serial] fn test_metricsrate_env() { env_logger::Builder::from_env(env_logger::Env::new().default_filter_or("solana=info")) .try_init() @@ -290,7 +287,6 @@ mod tests { } #[test] - #[serial] fn test_inc_new_counter() { let _readlock = get_env_lock().read(); //make sure that macros are syntactically correct @@ -301,7 +297,6 @@ mod tests { } #[test] - #[serial] fn test_lograte() { env_logger::Builder::from_env(env_logger::Env::new().default_filter_or("solana=info")) .try_init() @@ -322,7 +317,6 @@ mod tests { } #[test] - #[serial] fn test_lograte_env() { env_logger::Builder::from_env(env_logger::Env::new().default_filter_or("solana=info")) .try_init() diff --git a/runtime/Cargo.toml b/runtime/Cargo.toml index ac6cfccf690b8d..537b61e2747473 100644 --- a/runtime/Cargo.toml +++ b/runtime/Cargo.toml @@ -37,6 +37,7 @@ solana-vote-api = { path = "../programs/vote_api", version = "0.19.0-pre0" } solana-vote-program = { path = "../programs/vote_program", version = "0.19.0-pre0" } sys-info = "0.5.7" tempfile = "3.1.0" +solana-rayon-threadlimit = { path = "../rayon-threadlimit", version = "0.19.0-pre0" } [lib] crate-type = ["lib"] diff --git a/runtime/src/accounts_db.rs b/runtime/src/accounts_db.rs index 17094ba74dd20a..7d66f06766c1e7 100644 --- a/runtime/src/accounts_db.rs +++ b/runtime/src/accounts_db.rs @@ -30,6 +30,7 @@ use serde::de::{MapAccess, Visitor}; use serde::ser::{SerializeMap, Serializer}; use serde::{Deserialize, Serialize}; use solana_measure::measure::Measure; +use solana_rayon_threadlimit::thread_count::get_thread_count; use solana_sdk::account::Account; use solana_sdk::pubkey::Pubkey; use std::collections::{HashMap, HashSet}; @@ -39,7 +40,6 @@ use std::path::Path; use std::path::PathBuf; use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::{Arc, RwLock}; -use sys_info; use tempfile::TempDir; pub const DEFAULT_FILE_SIZE: u64 = 4 * 1024 * 1024; @@ -370,7 +370,7 @@ pub struct AccountsDB { impl Default for AccountsDB { fn default() -> Self { - let num_threads = sys_info::cpu_num().unwrap_or(DEFAULT_NUM_THREADS) as usize; + let num_threads = get_thread_count(); AccountsDB { accounts_index: RwLock::new(AccountsIndex::default()), From 3d893c4ba605732afd1c3f6c0919362d62e372b0 Mon Sep 17 00:00:00 2001 From: Sagar Dhawan Date: Tue, 10 Sep 2019 14:52:05 -0700 Subject: [PATCH 02/23] Remove remaining serial_derives --- bench-tps/src/bench.rs | 1 - core/tests/cluster_info.rs | 1 - local_cluster/tests/local_cluster.rs | 1 - local_cluster/tests/replicator.rs | 1 - metrics/src/counter.rs | 1 - 5 files changed, 5 deletions(-) diff --git a/bench-tps/src/bench.rs b/bench-tps/src/bench.rs index 16ec27c0efe124..0df9ad7f5d6de2 100644 --- a/bench-tps/src/bench.rs +++ b/bench-tps/src/bench.rs @@ -956,7 +956,6 @@ pub fn generate_and_fund_keypairs( mod tests { use super::*; - use serial_test_derive::serial; use solana_client::thin_client::create_client; use solana_core::cluster_info::FULLNODE_PORT_RANGE; use solana_core::validator::ValidatorConfig; diff --git a/core/tests/cluster_info.rs b/core/tests/cluster_info.rs index f3466a007d20c4..89bc941c71ef08 100644 --- a/core/tests/cluster_info.rs +++ b/core/tests/cluster_info.rs @@ -2,7 +2,6 @@ use rand::SeedableRng; use rand_chacha::ChaChaRng; use rayon::iter::ParallelIterator; use rayon::prelude::*; -use serial_test_derive::serial; use solana_core::cluster_info::{compute_retransmit_peers, ClusterInfo}; use solana_core::contact_info::ContactInfo; use solana_sdk::pubkey::Pubkey; diff --git a/local_cluster/tests/local_cluster.rs b/local_cluster/tests/local_cluster.rs index 16656a9ce48ef4..a0cc491be7eccb 100644 --- a/local_cluster/tests/local_cluster.rs +++ b/local_cluster/tests/local_cluster.rs @@ -1,7 +1,6 @@ extern crate solana_core; use log::*; -use serial_test_derive::serial; use solana_core::{ bank_forks::SnapshotConfig, blocktree::Blocktree, broadcast_stage::BroadcastStageType, gossip_service::discover_cluster, snapshot_utils, validator::ValidatorConfig, diff --git a/local_cluster/tests/replicator.rs b/local_cluster/tests/replicator.rs index 9ee6ed37724b8a..1a77480fc1aec9 100644 --- a/local_cluster/tests/replicator.rs +++ b/local_cluster/tests/replicator.rs @@ -4,7 +4,6 @@ extern crate log; #[macro_use] extern crate solana_core; -use serial_test_derive::serial; use solana_client::thin_client::create_client; use solana_core::blocktree::{create_new_tmp_ledger, get_tmp_ledger_path, Blocktree}; use solana_core::cluster_info::{ClusterInfo, Node, FULLNODE_PORT_RANGE}; diff --git a/metrics/src/counter.rs b/metrics/src/counter.rs index a630b6b4fc4e89..52865c381d8e43 100644 --- a/metrics/src/counter.rs +++ b/metrics/src/counter.rs @@ -207,7 +207,6 @@ mod tests { use crate::counter::{Counter, DEFAULT_LOG_RATE, DEFAULT_METRICS_RATE}; use log::Level; use log::*; - use serial_test_derive::serial; use std::env; use std::sync::atomic::Ordering; use std::sync::{Once, RwLock}; From 53b61f82439057ecc6619f86c7293666a332fb9e Mon Sep 17 00:00:00 2001 From: Sagar Dhawan Date: Tue, 10 Sep 2019 15:30:35 -0700 Subject: [PATCH 03/23] Try handle checks --- Cargo.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/Cargo.toml b/Cargo.toml index 66dacf5d1c8ec1..9820dcffe3b3fc 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -57,6 +57,7 @@ members = [ "fixed-buf", "vote-signer", "cli", + "raypon-threadlimit", ] exclude = [ From 9b8dd136442d76166353df7d955dfcafa233776a Mon Sep 17 00:00:00 2001 From: Sagar Dhawan Date: Tue, 10 Sep 2019 15:35:11 -0700 Subject: [PATCH 04/23] Fix typo --- Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 9820dcffe3b3fc..6a31f9da9b4e90 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -57,7 +57,7 @@ members = [ "fixed-buf", "vote-signer", "cli", - "raypon-threadlimit", + "rayon-threadlimit", ] exclude = [ From 7d32278b8124ffdb8ad90d8a71600dc989e9eb2a Mon Sep 17 00:00:00 2001 From: Sagar Dhawan Date: Tue, 10 Sep 2019 15:36:10 -0700 Subject: [PATCH 05/23] Add rayon-threadlimit --- rayon-threadlimit/Cargo.toml | 13 +++++++++++++ rayon-threadlimit/src/lib.rs | 4 ++++ rayon-threadlimit/src/thread_count.rs | 15 +++++++++++++++ 3 files changed, 32 insertions(+) create mode 100644 rayon-threadlimit/Cargo.toml create mode 100644 rayon-threadlimit/src/lib.rs create mode 100644 rayon-threadlimit/src/thread_count.rs diff --git a/rayon-threadlimit/Cargo.toml b/rayon-threadlimit/Cargo.toml new file mode 100644 index 00000000000000..a726940d00d764 --- /dev/null +++ b/rayon-threadlimit/Cargo.toml @@ -0,0 +1,13 @@ +[package] +name = "solana-rayon-threadlimit" +version = "0.19.0-pre0" +homepage = "https://solana.com/" +readme = "../README.md" +repository = "https://github.com/solana-labs/solana" +authors = ["Solana Maintainers "] +license = "Apache-2.0" +edition = "2018" + +[dependencies] +lazy_static = "1.4.0" +sys-info = "0.5.7" diff --git a/rayon-threadlimit/src/lib.rs b/rayon-threadlimit/src/lib.rs new file mode 100644 index 00000000000000..6676b1133057b1 --- /dev/null +++ b/rayon-threadlimit/src/lib.rs @@ -0,0 +1,4 @@ +pub mod thread_count; + +#[macro_use] +extern crate lazy_static; diff --git a/rayon-threadlimit/src/thread_count.rs b/rayon-threadlimit/src/thread_count.rs new file mode 100644 index 00000000000000..02a3e2fed66e89 --- /dev/null +++ b/rayon-threadlimit/src/thread_count.rs @@ -0,0 +1,15 @@ +use std::sync::RwLock; + +//TODO remove this hack when rayon fixes itself +lazy_static! { + static ref MAX_RAYON_THREADS: RwLock = + RwLock::new(sys_info::cpu_num().unwrap() as usize); +} + +pub fn get_thread_count() -> usize { + *MAX_RAYON_THREADS.read().unwrap() +} + +pub fn init_test_thread_count() { + *MAX_RAYON_THREADS.write().unwrap() = 1; +} From a41d071d5108cc8d5d551dfeb41eee45558dead5 Mon Sep 17 00:00:00 2001 From: Sagar Dhawan Date: Tue, 10 Sep 2019 15:47:35 -0700 Subject: [PATCH 06/23] Add gitignore --- rayon-threadlimit/.gitignore | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 rayon-threadlimit/.gitignore diff --git a/rayon-threadlimit/.gitignore b/rayon-threadlimit/.gitignore new file mode 100644 index 00000000000000..5404b132dba6e1 --- /dev/null +++ b/rayon-threadlimit/.gitignore @@ -0,0 +1,2 @@ +/target/ +/farf/ From b607857a0fc11be6fd0d1a452b950e4238187e9a Mon Sep 17 00:00:00 2001 From: Sagar Dhawan Date: Tue, 10 Sep 2019 16:15:59 -0700 Subject: [PATCH 07/23] Reintroduce --test-threads=1 --- ci/test-stable.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/test-stable.sh b/ci/test-stable.sh index bee4e9642e0ef2..9788aed6da9b23 100755 --- a/ci/test-stable.sh +++ b/ci/test-stable.sh @@ -33,7 +33,7 @@ test-stable) echo "Executing $testName" _ cargo +"$rust_stable" build --all --tests --bins ${V:+--verbose} - _ cargo +"$rust_stable" test --all ${V:+--verbose} -- --nocapture + _ cargo +"$rust_stable" test --all ${V:+--verbose} -- --nocapture --test-threads=1 ;; test-stable-perf) echo "Executing $testName" From 9a20f1d4457e590bc841200cbb0abf65b3af075e Mon Sep 17 00:00:00 2001 From: Sagar Dhawan Date: Tue, 10 Sep 2019 17:26:01 -0700 Subject: [PATCH 08/23] Bump timeout to 60min and ignore repairman catchup --- ci/buildkite.yml | 4 ++-- local_cluster/tests/local_cluster.rs | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/ci/buildkite.yml b/ci/buildkite.yml index 6a06a34d65a030..d72ab5e9f811c1 100644 --- a/ci/buildkite.yml +++ b/ci/buildkite.yml @@ -11,7 +11,7 @@ steps: - wait - command: "ci/test-stable-perf.sh" name: "stable-perf" - timeout_in_minutes: 30 + timeout_in_minutes: 60 artifact_paths: "log-*.txt" agents: - "queue=cuda" @@ -20,7 +20,7 @@ steps: timeout_in_minutes: 60 - command: ". ci/rust-version.sh; ci/docker-run.sh $$rust_stable_docker_image ci/test-stable.sh" name: "stable" - timeout_in_minutes: 40 + timeout_in_minutes: 60 artifact_paths: "log-*.txt" - command: ". ci/rust-version.sh; ci/docker-run.sh $$rust_nightly_docker_image ci/test-coverage.sh" name: "coverage" diff --git a/local_cluster/tests/local_cluster.rs b/local_cluster/tests/local_cluster.rs index a0cc491be7eccb..c61003d0d59157 100644 --- a/local_cluster/tests/local_cluster.rs +++ b/local_cluster/tests/local_cluster.rs @@ -569,6 +569,7 @@ fn test_faulty_node(faulty_node_type: BroadcastStageType) { } #[test] +#[ignore] fn test_repairman_catchup() { solana_logger::setup(); error!("test_repairman_catchup"); From 81ff788b90676e1f843ff720a14947a7cb77df50 Mon Sep 17 00:00:00 2001 From: Sagar Dhawan Date: Tue, 10 Sep 2019 18:03:00 -0700 Subject: [PATCH 09/23] ci priority --- ci/buildkite.yml | 40 +++++++++++++++++++++------------------- 1 file changed, 21 insertions(+), 19 deletions(-) diff --git a/ci/buildkite.yml b/ci/buildkite.yml index d72ab5e9f811c1..5a5ce88aea5e9b 100644 --- a/ci/buildkite.yml +++ b/ci/buildkite.yml @@ -2,29 +2,31 @@ # # Release tags use buildkite-release.yml instead steps: - - command: "ci/shellcheck.sh" - name: "shellcheck" - timeout_in_minutes: 5 - - command: ". ci/rust-version.sh; ci/docker-run.sh $$rust_nightly_docker_image ci/test-checks.sh" - name: "checks" - timeout_in_minutes: 35 - - wait - - command: "ci/test-stable-perf.sh" - name: "stable-perf" - timeout_in_minutes: 60 - artifact_paths: "log-*.txt" - agents: - - "queue=cuda" - - command: "ci/test-bench.sh" - name: "bench" - timeout_in_minutes: 60 +# - command: "ci/shellcheck.sh" +# name: "shellcheck" +# timeout_in_minutes: 5 +# - command: ". ci/rust-version.sh; ci/docker-run.sh $$rust_nightly_docker_image ci/test-checks.sh" +# name: "checks" +# timeout_in_minutes: 35 +# - wait +# - command: "ci/test-stable-perf.sh" +# name: "stable-perf" +# timeout_in_minutes: 60 +# artifact_paths: "log-*.txt" +# agents: +# - "queue=cuda" +# - command: "ci/test-bench.sh" +# name: "bench" +# timeout_in_minutes: 60 - command: ". ci/rust-version.sh; ci/docker-run.sh $$rust_stable_docker_image ci/test-stable.sh" name: "stable" timeout_in_minutes: 60 artifact_paths: "log-*.txt" - - command: ". ci/rust-version.sh; ci/docker-run.sh $$rust_nightly_docker_image ci/test-coverage.sh" - name: "coverage" - timeout_in_minutes: 40 + agents: + - "queue=high-core-default" +# - command: ". ci/rust-version.sh; ci/docker-run.sh $$rust_nightly_docker_image ci/test-coverage.sh" +# name: "coverage" +# timeout_in_minutes: 40 # TODO: Fix and re-enable test-large-network.sh # - command: "ci/test-large-network.sh || true" # name: "large-network [ignored]" From e88023da8f530b59ef9527aa40441cb28341c866 Mon Sep 17 00:00:00 2001 From: Sagar Dhawan Date: Wed, 11 Sep 2019 09:26:59 -0700 Subject: [PATCH 10/23] restrict ci to local_cluster only --- ci/test-stable.sh | 6 ++++-- local_cluster/tests/local_cluster.rs | 1 - 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/ci/test-stable.sh b/ci/test-stable.sh index 9788aed6da9b23..b35ce8032704b5 100755 --- a/ci/test-stable.sh +++ b/ci/test-stable.sh @@ -33,7 +33,7 @@ test-stable) echo "Executing $testName" _ cargo +"$rust_stable" build --all --tests --bins ${V:+--verbose} - _ cargo +"$rust_stable" test --all ${V:+--verbose} -- --nocapture --test-threads=1 + _ cargo +"$rust_stable" test --package solana-local-cluster --test local_cluster ${V:+--verbose} -- --nocapture --test-threads=1 ;; test-stable-perf) echo "Executing $testName" @@ -77,7 +77,9 @@ test-stable-perf) # Run root package library tests _ cargo +"$rust_stable" build --all --tests --bins ${V:+--verbose} --features="$ROOT_FEATURES" - _ cargo +"$rust_stable" test --manifest-path=core/Cargo.toml ${V:+--verbose} --features="$ROOT_FEATURES" -- --nocapture --test-threads=1 + _ cargo +"$rust_stable" test --package solana-local-cluster --test local_cluster --manifest-path=core/Cargo.toml ${V:+--verbose} --features="$ROOT_FEATURES" -- --nocapture --test-threads=1 +# _ cargo +"$rust_stable" test --manifest-path=core/Cargo.toml ${V:+--verbose} --features="$ROOT_FEATURES" -- --nocapture --test-threads=1 +# _ cargo +"$rust_stable" test --manifest-path=core/Cargo.toml ${V:+--verbose} --features="$ROOT_FEATURES" -- --nocapture --test-threads=1 ;; *) echo "Error: Unknown test: $testName" diff --git a/local_cluster/tests/local_cluster.rs b/local_cluster/tests/local_cluster.rs index c61003d0d59157..a0cc491be7eccb 100644 --- a/local_cluster/tests/local_cluster.rs +++ b/local_cluster/tests/local_cluster.rs @@ -569,7 +569,6 @@ fn test_faulty_node(faulty_node_type: BroadcastStageType) { } #[test] -#[ignore] fn test_repairman_catchup() { solana_logger::setup(); error!("test_repairman_catchup"); From 1ab2b5148702cbe152fba0d2ddb21c7280454f9a Mon Sep 17 00:00:00 2001 From: Sagar Dhawan Date: Wed, 11 Sep 2019 09:50:56 -0700 Subject: [PATCH 11/23] repairman catchup is flaky --- local_cluster/tests/local_cluster.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/local_cluster/tests/local_cluster.rs b/local_cluster/tests/local_cluster.rs index a0cc491be7eccb..c61003d0d59157 100644 --- a/local_cluster/tests/local_cluster.rs +++ b/local_cluster/tests/local_cluster.rs @@ -569,6 +569,7 @@ fn test_faulty_node(faulty_node_type: BroadcastStageType) { } #[test] +#[ignore] fn test_repairman_catchup() { solana_logger::setup(); error!("test_repairman_catchup"); From 3af10ec31bc56131ea6bb36c6424e3e2bee439cb Mon Sep 17 00:00:00 2001 From: Sagar Dhawan Date: Wed, 11 Sep 2019 10:16:12 -0700 Subject: [PATCH 12/23] give serial another shot --- Cargo.lock | 21 +++++++++++++++++++++ ci/test-stable.sh | 2 +- local_cluster/Cargo.toml | 4 ++++ local_cluster/tests/local_cluster.rs | 13 +++++++++++++ 4 files changed, 39 insertions(+), 1 deletion(-) diff --git a/Cargo.lock b/Cargo.lock index 13ba09c310b085..03da82211a15d8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2628,6 +2628,23 @@ dependencies = [ "yaml-rust 0.4.3 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "serial_test" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "serial_test_derive" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "quote 0.6.13 (registry+https://github.com/rust-lang/crates.io-index)", + "syn 0.15.42 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "sha-1" version = "0.8.1" @@ -3336,6 +3353,8 @@ version = "0.19.0-pre0" dependencies = [ "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", "rand 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)", + "serial_test 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", + "serial_test_derive 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", "solana-client 0.19.0-pre0", "solana-core 0.19.0-pre0", "solana-logger 0.19.0-pre0", @@ -5310,6 +5329,8 @@ dependencies = [ "checksum serde_derive 1.0.100 (registry+https://github.com/rust-lang/crates.io-index)" = "11e410fde43e157d789fc290d26bc940778ad0fdd47836426fbac36573710dbb" "checksum serde_json 1.0.40 (registry+https://github.com/rust-lang/crates.io-index)" = "051c49229f282f7c6f3813f8286cc1e3323e8051823fce42c7ea80fe13521704" "checksum serde_yaml 0.8.9 (registry+https://github.com/rust-lang/crates.io-index)" = "38b08a9a90e5260fe01c6480ec7c811606df6d3a660415808c3c3fa8ed95b582" +"checksum serial_test 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "50bfbc39343545618d97869d77f38ed43e48dd77432717dbc7ed39d797f3ecbe" +"checksum serial_test_derive 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "89dd85be2e2ad75b041c9df2892ac078fa6e0b90024028b2b9fb4125b7530f01" "checksum sha-1 0.8.1 (registry+https://github.com/rust-lang/crates.io-index)" = "23962131a91661d643c98940b20fcaffe62d776a823247be80a48fcb8b6fce68" "checksum sha2 0.7.1 (registry+https://github.com/rust-lang/crates.io-index)" = "9eb6be24e4c23a84d7184280d2722f7f2731fcdd4a9d886efbfe4413e4847ea0" "checksum sha2 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)" = "7b4d8bfd0e469f417657573d8451fb33d16cfe0989359b93baf3a1ffc639543d" diff --git a/ci/test-stable.sh b/ci/test-stable.sh index b35ce8032704b5..bcf3daadf9c320 100755 --- a/ci/test-stable.sh +++ b/ci/test-stable.sh @@ -77,7 +77,7 @@ test-stable-perf) # Run root package library tests _ cargo +"$rust_stable" build --all --tests --bins ${V:+--verbose} --features="$ROOT_FEATURES" - _ cargo +"$rust_stable" test --package solana-local-cluster --test local_cluster --manifest-path=core/Cargo.toml ${V:+--verbose} --features="$ROOT_FEATURES" -- --nocapture --test-threads=1 + _ cargo +"$rust_stable" test --package solana-local-cluster --test local_cluster --manifest-path=core/Cargo.toml ${V:+--verbose} --features="$ROOT_FEATURES" -- --nocapture # _ cargo +"$rust_stable" test --manifest-path=core/Cargo.toml ${V:+--verbose} --features="$ROOT_FEATURES" -- --nocapture --test-threads=1 # _ cargo +"$rust_stable" test --manifest-path=core/Cargo.toml ${V:+--verbose} --features="$ROOT_FEATURES" -- --nocapture --test-threads=1 ;; diff --git a/local_cluster/Cargo.toml b/local_cluster/Cargo.toml index f90ef4d8fc4f45..7f529de633abd7 100644 --- a/local_cluster/Cargo.toml +++ b/local_cluster/Cargo.toml @@ -24,6 +24,10 @@ symlink = "0.1.0" tempfile = "3.1.0" solana-rayon-threadlimit = { path = "../rayon-threadlimit", version = "0.19.0-pre0" } +[dev-dependencies] +serial_test = "0.2.0" +serial_test_derive = "0.2.0" + [features] cuda = ["solana-core/cuda"] diff --git a/local_cluster/tests/local_cluster.rs b/local_cluster/tests/local_cluster.rs index c61003d0d59157..30acb223d2658e 100644 --- a/local_cluster/tests/local_cluster.rs +++ b/local_cluster/tests/local_cluster.rs @@ -1,6 +1,7 @@ extern crate solana_core; use log::*; +use serial_test_derive::serial; use solana_core::{ bank_forks::SnapshotConfig, blocktree::Blocktree, broadcast_stage::BroadcastStageType, gossip_service::discover_cluster, snapshot_utils, validator::ValidatorConfig, @@ -25,6 +26,7 @@ use std::{ use tempfile::TempDir; #[test] +#[serial] fn test_ledger_cleanup_service() { solana_logger::setup(); error!("test_ledger_cleanup_service"); @@ -63,6 +65,7 @@ fn test_ledger_cleanup_service() { } #[test] +#[serial] fn test_spend_and_verify_all_nodes_1() { solana_logger::setup(); error!("test_spend_and_verify_all_nodes_1"); @@ -77,6 +80,7 @@ fn test_spend_and_verify_all_nodes_1() { } #[test] +#[serial] fn test_spend_and_verify_all_nodes_2() { solana_logger::setup(); error!("test_spend_and_verify_all_nodes_2"); @@ -134,6 +138,7 @@ fn test_fullnode_exit_default_config_should_panic() { } #[test] +#[serial] fn test_fullnode_exit_2() { solana_logger::setup(); error!("test_fullnode_exit_2"); @@ -173,6 +178,7 @@ fn test_leader_failure_4() { ); } #[test] +#[serial] fn test_two_unbalanced_stakes() { solana_logger::setup(); error!("test_two_unbalanced_stakes"); @@ -208,6 +214,7 @@ fn test_two_unbalanced_stakes() { } #[test] +#[serial] fn test_forwarding() { // Set up a cluster where one node is never the leader, so all txs sent to this node // will be have to be forwarded in order to be confirmed @@ -234,6 +241,7 @@ fn test_forwarding() { } #[test] +#[serial] fn test_restart_node() { solana_logger::setup(); error!("test_restart_node"); @@ -271,6 +279,7 @@ fn test_restart_node() { } #[test] +#[serial] fn test_listener_startup() { let config = ClusterConfig { node_stakes: vec![100; 1], @@ -286,6 +295,7 @@ fn test_listener_startup() { #[allow(unused_attributes)] #[test] +#[serial] fn test_snapshot_restart_locktower() { // First set up the cluster with 2 nodes let snapshot_interval_slots = 10; @@ -346,6 +356,7 @@ fn test_snapshot_restart_locktower() { } #[test] +#[serial] fn test_snapshots_blocktree_floor() { // First set up the cluster with 1 snapshotting leader let snapshot_interval_slots = 10; @@ -427,6 +438,7 @@ fn test_snapshots_blocktree_floor() { } #[test] +#[serial] fn test_snapshots_restart_validity() { solana_logger::setup(); let snapshot_interval_slots = 10; @@ -504,6 +516,7 @@ fn test_snapshots_restart_validity() { } #[test] +#[serial] fn test_fail_entry_verification_leader() { test_faulty_node(BroadcastStageType::FailEntryVerification); } From 2943e1240cd44d19b3df99d41f44e1e1fb387bc6 Mon Sep 17 00:00:00 2001 From: Sagar Dhawan Date: Wed, 11 Sep 2019 10:29:44 -0700 Subject: [PATCH 13/23] Remove --test-threads=1 for local_cluster --- ci/test-stable.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/test-stable.sh b/ci/test-stable.sh index bcf3daadf9c320..54104223ad9c1b 100755 --- a/ci/test-stable.sh +++ b/ci/test-stable.sh @@ -33,7 +33,7 @@ test-stable) echo "Executing $testName" _ cargo +"$rust_stable" build --all --tests --bins ${V:+--verbose} - _ cargo +"$rust_stable" test --package solana-local-cluster --test local_cluster ${V:+--verbose} -- --nocapture --test-threads=1 + _ cargo +"$rust_stable" test --package solana-local-cluster --test local_cluster ${V:+--verbose} -- --nocapture ;; test-stable-perf) echo "Executing $testName" From f3cbf8198419be1cb0f43ae91e204a8482fcfea9 Mon Sep 17 00:00:00 2001 From: Sagar Dhawan Date: Wed, 11 Sep 2019 10:34:51 -0700 Subject: [PATCH 14/23] Make a couple more tests serial --- local_cluster/tests/local_cluster.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/local_cluster/tests/local_cluster.rs b/local_cluster/tests/local_cluster.rs index 30acb223d2658e..727790055a987e 100644 --- a/local_cluster/tests/local_cluster.rs +++ b/local_cluster/tests/local_cluster.rs @@ -95,6 +95,7 @@ fn test_spend_and_verify_all_nodes_2() { } #[test] +#[serial] fn test_spend_and_verify_all_nodes_3() { solana_logger::setup(); error!("test_spend_and_verify_all_nodes_3"); @@ -157,6 +158,7 @@ fn test_fullnode_exit_2() { // Cluster needs a supermajority to remain, so the minimum size for this test is 4 #[test] +#[serial] fn test_leader_failure_4() { solana_logger::setup(); error!("test_leader_failure_4"); From c59654fe024e385a2b6f4633a9df084377187094 Mon Sep 17 00:00:00 2001 From: Sagar Dhawan Date: Wed, 11 Sep 2019 11:02:01 -0700 Subject: [PATCH 15/23] Give serial a proper second chance --- Cargo.lock | 6 ++++++ bench-tps/Cargo.toml | 4 ++++ bench-tps/src/bench.rs | 3 +++ core/Cargo.toml | 2 ++ core/tests/cluster_info.rs | 5 +++++ local_cluster/tests/replicator.rs | 5 +++++ metrics/Cargo.toml | 2 ++ metrics/src/counter.rs | 7 +++++++ 8 files changed, 34 insertions(+) diff --git a/Cargo.lock b/Cargo.lock index 03da82211a15d8..dc6d86cad04011 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2849,6 +2849,8 @@ dependencies = [ "serde_derive 1.0.100 (registry+https://github.com/rust-lang/crates.io-index)", "serde_json 1.0.40 (registry+https://github.com/rust-lang/crates.io-index)", "serde_yaml 0.8.9 (registry+https://github.com/rust-lang/crates.io-index)", + "serial_test 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", + "serial_test_derive 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", "solana-client 0.19.0-pre0", "solana-core 0.19.0-pre0", "solana-drone 0.19.0-pre0", @@ -3097,6 +3099,8 @@ dependencies = [ "serde 1.0.100 (registry+https://github.com/rust-lang/crates.io-index)", "serde_derive 1.0.100 (registry+https://github.com/rust-lang/crates.io-index)", "serde_json 1.0.40 (registry+https://github.com/rust-lang/crates.io-index)", + "serial_test 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", + "serial_test_derive 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", "solana-budget-api 0.19.0-pre0", "solana-budget-program 0.19.0-pre0", "solana-chacha-sys 0.19.0-pre0", @@ -3401,6 +3405,8 @@ dependencies = [ "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", "rand 0.6.5 (registry+https://github.com/rust-lang/crates.io-index)", + "serial_test 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", + "serial_test_derive 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", "solana-sdk 0.19.0-pre0", "sys-info 0.5.7 (registry+https://github.com/rust-lang/crates.io-index)", "ureq 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)", diff --git a/bench-tps/Cargo.toml b/bench-tps/Cargo.toml index 7e92b8161c95a7..62f9d3afdd5ff4 100644 --- a/bench-tps/Cargo.toml +++ b/bench-tps/Cargo.toml @@ -31,6 +31,10 @@ solana-sdk = { path = "../sdk", version = "0.19.0-pre0" } solana-move-loader-program = { path = "../programs/move_loader_program", version = "0.19.0-pre0" } solana-move-loader-api = { path = "../programs/move_loader_api", version = "0.19.0-pre0" } +[dev-dependencies] +serial_test = "0.2.0" +serial_test_derive = "0.2.0" + [features] cuda = ["solana-core/cuda"] diff --git a/bench-tps/src/bench.rs b/bench-tps/src/bench.rs index 0df9ad7f5d6de2..fc0072c97123c8 100644 --- a/bench-tps/src/bench.rs +++ b/bench-tps/src/bench.rs @@ -956,6 +956,7 @@ pub fn generate_and_fund_keypairs( mod tests { use super::*; + use serial_test_derive::serial; use solana_client::thin_client::create_client; use solana_core::cluster_info::FULLNODE_PORT_RANGE; use solana_core::validator::ValidatorConfig; @@ -1027,6 +1028,7 @@ mod tests { } #[test] + #[serial] fn test_bench_tps_local_cluster_solana() { let mut config = Config::default(); config.tx_count = 100; @@ -1036,6 +1038,7 @@ mod tests { } #[test] + #[serial] fn test_bench_tps_local_cluster_move() { let mut config = Config::default(); config.tx_count = 100; diff --git a/core/Cargo.toml b/core/Cargo.toml index 91f1ffb7499ebd..eb9f4a7a941292 100644 --- a/core/Cargo.toml +++ b/core/Cargo.toml @@ -94,6 +94,8 @@ features = ["lz4"] [dev-dependencies] hex-literal = "0.2.1" matches = "0.1.6" +serial_test = "0.2.0" +serial_test_derive = "0.2.0" ureq = { version = "0.11.0", default-features = false, features = ["json"] } [[bench]] diff --git a/core/tests/cluster_info.rs b/core/tests/cluster_info.rs index 89bc941c71ef08..1cd9407bb235cd 100644 --- a/core/tests/cluster_info.rs +++ b/core/tests/cluster_info.rs @@ -2,6 +2,7 @@ use rand::SeedableRng; use rand_chacha::ChaChaRng; use rayon::iter::ParallelIterator; use rayon::prelude::*; +use serial_test_derive::serial; use solana_core::cluster_info::{compute_retransmit_peers, ClusterInfo}; use solana_core::contact_info::ContactInfo; use solana_sdk::pubkey::Pubkey; @@ -190,6 +191,7 @@ fn run_simulation(stakes: &[u64], fanout: usize) { // Run with a single layer #[test] +#[serial] fn test_retransmit_small() { let stakes: Vec<_> = (0..200).map(|i| i).collect(); run_simulation(&stakes, 200); @@ -197,6 +199,7 @@ fn test_retransmit_small() { // Make sure at least 2 layers are used #[test] +#[serial] fn test_retransmit_medium() { let num_nodes = 2000; let stakes: Vec<_> = (0..num_nodes).map(|i| i).collect(); @@ -205,6 +208,7 @@ fn test_retransmit_medium() { // Make sure at least 2 layers are used but with equal stakes #[test] +#[serial] fn test_retransmit_medium_equal_stakes() { let num_nodes = 2000; let stakes: Vec<_> = (0..num_nodes).map(|_| 10).collect(); @@ -213,6 +217,7 @@ fn test_retransmit_medium_equal_stakes() { // Scale down the network and make sure many layers are used #[test] +#[serial] fn test_retransmit_large() { let num_nodes = 4000; let stakes: Vec<_> = (0..num_nodes).map(|i| i).collect(); diff --git a/local_cluster/tests/replicator.rs b/local_cluster/tests/replicator.rs index 1a77480fc1aec9..faed4b9405fd9b 100644 --- a/local_cluster/tests/replicator.rs +++ b/local_cluster/tests/replicator.rs @@ -73,16 +73,19 @@ fn run_replicator_startup_basic(num_nodes: usize, num_replicators: usize) { } #[test] +#[serial] fn test_replicator_startup_1_node() { run_replicator_startup_basic(1, 1); } #[test] +#[serial] fn test_replicator_startup_2_nodes() { run_replicator_startup_basic(2, 1); } #[test] +#[serial] fn test_replicator_startup_leader_hang() { use std::net::{IpAddr, Ipv4Addr, SocketAddr}; @@ -121,6 +124,7 @@ fn test_replicator_startup_leader_hang() { } #[test] +#[serial] fn test_replicator_startup_ledger_hang() { solana_logger::setup(); info!("starting replicator test"); @@ -149,6 +153,7 @@ fn test_replicator_startup_ledger_hang() { } #[test] +#[serial] fn test_account_setup() { let num_nodes = 1; let num_replicators = 1; diff --git a/metrics/Cargo.toml b/metrics/Cargo.toml index 6c526f0d455303..9906c8915da685 100644 --- a/metrics/Cargo.toml +++ b/metrics/Cargo.toml @@ -18,6 +18,8 @@ ureq = "0.11.0" [dev-dependencies] rand = "0.6.5" +serial_test = "0.2.0" +serial_test_derive = "0.2.0" [lib] name = "solana_metrics" diff --git a/metrics/src/counter.rs b/metrics/src/counter.rs index 52865c381d8e43..d457e8e81f232a 100644 --- a/metrics/src/counter.rs +++ b/metrics/src/counter.rs @@ -1,5 +1,6 @@ use crate::metrics::submit_counter; use log::*; +use serial_test_derive::serial; use solana_sdk::timing; use std::env; use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering}; @@ -224,6 +225,7 @@ mod tests { } #[test] + #[serial] fn test_counter() { env_logger::Builder::from_env(env_logger::Env::new().default_filter_or("solana=info")) .try_init() @@ -255,6 +257,7 @@ mod tests { } #[test] + #[serial] fn test_metricsrate() { env_logger::Builder::from_env(env_logger::Env::new().default_filter_or("solana=info")) .try_init() @@ -272,6 +275,7 @@ mod tests { } #[test] + #[serial] fn test_metricsrate_env() { env_logger::Builder::from_env(env_logger::Env::new().default_filter_or("solana=info")) .try_init() @@ -286,6 +290,7 @@ mod tests { } #[test] + #[serial] fn test_inc_new_counter() { let _readlock = get_env_lock().read(); //make sure that macros are syntactically correct @@ -296,6 +301,7 @@ mod tests { } #[test] + #[serial] fn test_lograte() { env_logger::Builder::from_env(env_logger::Env::new().default_filter_or("solana=info")) .try_init() @@ -316,6 +322,7 @@ mod tests { } #[test] + #[serial] fn test_lograte_env() { env_logger::Builder::from_env(env_logger::Env::new().default_filter_or("solana=info")) .try_init() From ef69c7929bc5f9a6b4468a84926ab130165698a4 Mon Sep 17 00:00:00 2001 From: Sagar Dhawan Date: Wed, 11 Sep 2019 11:02:33 -0700 Subject: [PATCH 16/23] Run all tests --- ci/test-stable.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/test-stable.sh b/ci/test-stable.sh index 54104223ad9c1b..bdcb2f003db8ce 100755 --- a/ci/test-stable.sh +++ b/ci/test-stable.sh @@ -33,7 +33,7 @@ test-stable) echo "Executing $testName" _ cargo +"$rust_stable" build --all --tests --bins ${V:+--verbose} - _ cargo +"$rust_stable" test --package solana-local-cluster --test local_cluster ${V:+--verbose} -- --nocapture + _ cargo +"$rust_stable" test ${V:+--verbose} -- --nocapture ;; test-stable-perf) echo "Executing $testName" @@ -77,7 +77,7 @@ test-stable-perf) # Run root package library tests _ cargo +"$rust_stable" build --all --tests --bins ${V:+--verbose} --features="$ROOT_FEATURES" - _ cargo +"$rust_stable" test --package solana-local-cluster --test local_cluster --manifest-path=core/Cargo.toml ${V:+--verbose} --features="$ROOT_FEATURES" -- --nocapture + _ cargo +"$rust_stable" test --manifest-path=core/Cargo.toml ${V:+--verbose} --features="$ROOT_FEATURES" -- --nocapture # _ cargo +"$rust_stable" test --manifest-path=core/Cargo.toml ${V:+--verbose} --features="$ROOT_FEATURES" -- --nocapture --test-threads=1 # _ cargo +"$rust_stable" test --manifest-path=core/Cargo.toml ${V:+--verbose} --features="$ROOT_FEATURES" -- --nocapture --test-threads=1 ;; From 16392cc214dc9389df9615f5ea52a01f80d9b159 Mon Sep 17 00:00:00 2001 From: Sagar Dhawan Date: Wed, 11 Sep 2019 11:21:12 -0700 Subject: [PATCH 17/23] fix metrics serial --- metrics/src/counter.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metrics/src/counter.rs b/metrics/src/counter.rs index d457e8e81f232a..661d00be4b18de 100644 --- a/metrics/src/counter.rs +++ b/metrics/src/counter.rs @@ -1,6 +1,5 @@ use crate::metrics::submit_counter; use log::*; -use serial_test_derive::serial; use solana_sdk::timing; use std::env; use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering}; @@ -208,6 +207,7 @@ mod tests { use crate::counter::{Counter, DEFAULT_LOG_RATE, DEFAULT_METRICS_RATE}; use log::Level; use log::*; + use serial_test_derive::serial; use std::env; use std::sync::atomic::Ordering; use std::sync::{Once, RwLock}; From 065c85507e064e5e0f9e9fc2453b0f200583ffad Mon Sep 17 00:00:00 2001 From: Sagar Dhawan Date: Wed, 11 Sep 2019 11:23:50 -0700 Subject: [PATCH 18/23] fix replicator_serial --- local_cluster/tests/replicator.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/local_cluster/tests/replicator.rs b/local_cluster/tests/replicator.rs index faed4b9405fd9b..e1e5537e2f293d 100644 --- a/local_cluster/tests/replicator.rs +++ b/local_cluster/tests/replicator.rs @@ -4,6 +4,7 @@ extern crate log; #[macro_use] extern crate solana_core; +use serial_test_derive::serial; use solana_client::thin_client::create_client; use solana_core::blocktree::{create_new_tmp_ledger, get_tmp_ledger_path, Blocktree}; use solana_core::cluster_info::{ClusterInfo, Node, FULLNODE_PORT_RANGE}; From b7f6c08c24febb85db65e0874d0646fdc8cee828 Mon Sep 17 00:00:00 2001 From: Sagar Dhawan Date: Wed, 11 Sep 2019 12:23:48 -0700 Subject: [PATCH 19/23] reenable everything and reduce timeouts --- ci/buildkite.yml | 52 +++++++++++++++++++++++------------------------- 1 file changed, 25 insertions(+), 27 deletions(-) diff --git a/ci/buildkite.yml b/ci/buildkite.yml index 5a5ce88aea5e9b..636660cb095585 100644 --- a/ci/buildkite.yml +++ b/ci/buildkite.yml @@ -2,37 +2,35 @@ # # Release tags use buildkite-release.yml instead steps: -# - command: "ci/shellcheck.sh" -# name: "shellcheck" -# timeout_in_minutes: 5 -# - command: ". ci/rust-version.sh; ci/docker-run.sh $$rust_nightly_docker_image ci/test-checks.sh" -# name: "checks" -# timeout_in_minutes: 35 -# - wait -# - command: "ci/test-stable-perf.sh" -# name: "stable-perf" -# timeout_in_minutes: 60 -# artifact_paths: "log-*.txt" -# agents: -# - "queue=cuda" -# - command: "ci/test-bench.sh" -# name: "bench" -# timeout_in_minutes: 60 + - command: "ci/shellcheck.sh" + name: "shellcheck" + timeout_in_minutes: 5 + - command: ". ci/rust-version.sh; ci/docker-run.sh $$rust_nightly_docker_image ci/test-checks.sh" + name: "checks" + timeout_in_minutes: 35 + - wait + - command: "ci/test-stable-perf.sh" + name: "stable-perf" + timeout_in_minutes: 40 + artifact_paths: "log-*.txt" + agents: + - "queue=cuda" + - command: "ci/test-bench.sh" + name: "bench" + timeout_in_minutes: 60 - command: ". ci/rust-version.sh; ci/docker-run.sh $$rust_stable_docker_image ci/test-stable.sh" name: "stable" - timeout_in_minutes: 60 + timeout_in_minutes: 40 artifact_paths: "log-*.txt" - agents: - - "queue=high-core-default" -# - command: ". ci/rust-version.sh; ci/docker-run.sh $$rust_nightly_docker_image ci/test-coverage.sh" -# name: "coverage" -# timeout_in_minutes: 40 + - command: ". ci/rust-version.sh; ci/docker-run.sh $$rust_nightly_docker_image ci/test-coverage.sh" + name: "coverage" + timeout_in_minutes: 40 # TODO: Fix and re-enable test-large-network.sh - # - command: "ci/test-large-network.sh || true" - # name: "large-network [ignored]" - # timeout_in_minutes: 20 - # agents: - # - "queue=large" + - command: "ci/test-large-network.sh || true" + name: "large-network [ignored]" + timeout_in_minutes: 20 + agents: + - "queue=large" - wait - trigger: "solana-secondary" branches: "!pull/*" From 49ecf3d7fdb6f51cc040fafeccfd7543e92434e1 Mon Sep 17 00:00:00 2001 From: Sagar Dhawan Date: Wed, 11 Sep 2019 12:26:27 -0700 Subject: [PATCH 20/23] fix the yaml --- ci/buildkite.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/ci/buildkite.yml b/ci/buildkite.yml index 636660cb095585..3bdf0286a51d3e 100644 --- a/ci/buildkite.yml +++ b/ci/buildkite.yml @@ -26,11 +26,11 @@ steps: name: "coverage" timeout_in_minutes: 40 # TODO: Fix and re-enable test-large-network.sh - - command: "ci/test-large-network.sh || true" - name: "large-network [ignored]" - timeout_in_minutes: 20 - agents: - - "queue=large" + # - command: "ci/test-large-network.sh || true" + # name: "large-network [ignored]" + # timeout_in_minutes: 20 + # agents: + # - "queue=large" - wait - trigger: "solana-secondary" branches: "!pull/*" From 78223f05148a9583ccc83862ed382471c46d32f1 Mon Sep 17 00:00:00 2001 From: Sagar Dhawan Date: Wed, 11 Sep 2019 13:23:58 -0700 Subject: [PATCH 21/23] Remove unnecessary diff --- ci/buildkite.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/ci/buildkite.yml b/ci/buildkite.yml index 3bdf0286a51d3e..b2709af8503328 100644 --- a/ci/buildkite.yml +++ b/ci/buildkite.yml @@ -26,11 +26,11 @@ steps: name: "coverage" timeout_in_minutes: 40 # TODO: Fix and re-enable test-large-network.sh - # - command: "ci/test-large-network.sh || true" - # name: "large-network [ignored]" - # timeout_in_minutes: 20 - # agents: - # - "queue=large" + # - command: "ci/test-large-network.sh || true" + # name: "large-network [ignored]" + # timeout_in_minutes: 20 + # agents: + # - "queue=large" - wait - trigger: "solana-secondary" branches: "!pull/*" From 7ddf930dbeaa2f50ba780f2479f0d0e654711e94 Mon Sep 17 00:00:00 2001 From: Sagar Dhawan Date: Wed, 11 Sep 2019 13:25:04 -0700 Subject: [PATCH 22/23] Remove commented out dev sections --- ci/test-stable.sh | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/ci/test-stable.sh b/ci/test-stable.sh index bdcb2f003db8ce..f05ba4729545e2 100755 --- a/ci/test-stable.sh +++ b/ci/test-stable.sh @@ -33,7 +33,7 @@ test-stable) echo "Executing $testName" _ cargo +"$rust_stable" build --all --tests --bins ${V:+--verbose} - _ cargo +"$rust_stable" test ${V:+--verbose} -- --nocapture + _ cargo +"$rust_stable" test --all ${V:+--verbose} -- --nocapture ;; test-stable-perf) echo "Executing $testName" @@ -78,8 +78,6 @@ test-stable-perf) # Run root package library tests _ cargo +"$rust_stable" build --all --tests --bins ${V:+--verbose} --features="$ROOT_FEATURES" _ cargo +"$rust_stable" test --manifest-path=core/Cargo.toml ${V:+--verbose} --features="$ROOT_FEATURES" -- --nocapture -# _ cargo +"$rust_stable" test --manifest-path=core/Cargo.toml ${V:+--verbose} --features="$ROOT_FEATURES" -- --nocapture --test-threads=1 -# _ cargo +"$rust_stable" test --manifest-path=core/Cargo.toml ${V:+--verbose} --features="$ROOT_FEATURES" -- --nocapture --test-threads=1 ;; *) echo "Error: Unknown test: $testName" From 3390921c392d1df8810531d78f4fb60353478199 Mon Sep 17 00:00:00 2001 From: Sagar Dhawan Date: Wed, 11 Sep 2019 14:21:52 -0700 Subject: [PATCH 23/23] Address review comments --- Cargo.lock | 1 - ci/test-stable.sh | 2 +- core/Cargo.toml | 1 - core/src/blocktree_processor.rs | 2 +- core/src/entry.rs | 2 +- core/src/sigverify.rs | 2 +- core/src/window_service.rs | 2 +- local_cluster/src/local_cluster.rs | 4 ++-- rayon-threadlimit/src/lib.rs | 18 ++++++++++++++++-- rayon-threadlimit/src/thread_count.rs | 15 --------------- runtime/src/accounts_db.rs | 2 +- 11 files changed, 24 insertions(+), 27 deletions(-) delete mode 100644 rayon-threadlimit/src/thread_count.rs diff --git a/Cargo.lock b/Cargo.lock index dc6d86cad04011..9f601563fabb5e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3084,7 +3084,6 @@ dependencies = [ "jsonrpc-http-server 13.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "jsonrpc-pubsub 13.1.0 (registry+https://github.com/rust-lang/crates.io-index)", "jsonrpc-ws-server 13.1.0 (registry+https://github.com/rust-lang/crates.io-index)", - "lazy_static 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)", "libc 0.2.62 (registry+https://github.com/rust-lang/crates.io-index)", "log 0.4.8 (registry+https://github.com/rust-lang/crates.io-index)", "matches 0.1.8 (registry+https://github.com/rust-lang/crates.io-index)", diff --git a/ci/test-stable.sh b/ci/test-stable.sh index f05ba4729545e2..a82b3aa83f2a65 100755 --- a/ci/test-stable.sh +++ b/ci/test-stable.sh @@ -77,7 +77,7 @@ test-stable-perf) # Run root package library tests _ cargo +"$rust_stable" build --all --tests --bins ${V:+--verbose} --features="$ROOT_FEATURES" - _ cargo +"$rust_stable" test --manifest-path=core/Cargo.toml ${V:+--verbose} --features="$ROOT_FEATURES" -- --nocapture + _ cargo +"$rust_stable" test --manifest-path=core/Cargo.toml ${V:+--verbose} --features="$ROOT_FEATURES" -- --nocapture ;; *) echo "Error: Unknown test: $testName" diff --git a/core/Cargo.toml b/core/Cargo.toml index eb9f4a7a941292..464c7c215ae819 100644 --- a/core/Cargo.toml +++ b/core/Cargo.toml @@ -75,7 +75,6 @@ tokio-codec = "0.1" tokio-fs = "0.1" tokio-io = "0.1" untrusted = "0.7.0" -lazy_static = "1.4.0" solana-rayon-threadlimit = { path = "../rayon-threadlimit", version = "0.19.0-pre0" } # reed-solomon-erasure's simd_c feature fails to build for x86_64-pc-windows-msvc, use pure-rust diff --git a/core/src/blocktree_processor.rs b/core/src/blocktree_processor.rs index 9c374d8e337c3c..02a71f019fdc79 100644 --- a/core/src/blocktree_processor.rs +++ b/core/src/blocktree_processor.rs @@ -19,7 +19,7 @@ use std::sync::Arc; use std::time::{Duration, Instant}; pub const NUM_THREADS: u32 = 10; -use solana_rayon_threadlimit::thread_count::get_thread_count; +use solana_rayon_threadlimit::get_thread_count; use std::cell::RefCell; thread_local!(static PAR_THREAD_POOL: RefCell = RefCell::new(rayon::ThreadPoolBuilder::new() diff --git a/core/src/entry.rs b/core/src/entry.rs index 94d9d7cd3ae996..f5863a186a4cdf 100644 --- a/core/src/entry.rs +++ b/core/src/entry.rs @@ -23,7 +23,7 @@ use std::sync::{Arc, RwLock}; #[cfg(feature = "cuda")] use crate::sigverify::poh_verify_many; -use solana_rayon_threadlimit::thread_count::get_thread_count; +use solana_rayon_threadlimit::get_thread_count; #[cfg(feature = "cuda")] use std::sync::Mutex; #[cfg(feature = "cuda")] diff --git a/core/src/sigverify.rs b/core/src/sigverify.rs index 10b24bfaafa287..579f9c8302b966 100644 --- a/core/src/sigverify.rs +++ b/core/src/sigverify.rs @@ -21,7 +21,7 @@ use std::mem::size_of; #[cfg(feature = "cuda")] use core::ffi::c_void; -use solana_rayon_threadlimit::thread_count::get_thread_count; +use solana_rayon_threadlimit::get_thread_count; #[cfg(feature = "cuda")] use std::os::raw::{c_int, c_uint}; pub const NUM_THREADS: u32 = 10; diff --git a/core/src/window_service.rs b/core/src/window_service.rs index b9827803b9fe86..15c67b1c4cfa8f 100644 --- a/core/src/window_service.rs +++ b/core/src/window_service.rs @@ -12,7 +12,7 @@ use crate::streamer::{PacketReceiver, PacketSender}; use rayon::iter::{IndexedParallelIterator, IntoParallelRefMutIterator, ParallelIterator}; use rayon::ThreadPool; use solana_metrics::{inc_new_counter_debug, inc_new_counter_error}; -use solana_rayon_threadlimit::thread_count::get_thread_count; +use solana_rayon_threadlimit::get_thread_count; use solana_runtime::bank::Bank; use solana_sdk::pubkey::Pubkey; use solana_sdk::timing::duration_as_ms; diff --git a/local_cluster/src/local_cluster.rs b/local_cluster/src/local_cluster.rs index 7ee28c2017d707..649ad4b393eecf 100644 --- a/local_cluster/src/local_cluster.rs +++ b/local_cluster/src/local_cluster.rs @@ -10,7 +10,7 @@ use solana_core::{ service::Service, validator::{Validator, ValidatorConfig}, }; -use solana_rayon_threadlimit::thread_count::init_test_thread_count; +use solana_rayon_threadlimit::set_thread_count; use solana_sdk::{ client::SyncClient, clock::DEFAULT_TICKS_PER_SLOT, @@ -117,7 +117,7 @@ impl LocalCluster { } pub fn new(config: &ClusterConfig) -> Self { - init_test_thread_count(); + set_thread_count(1); assert_eq!(config.validator_configs.len(), config.node_stakes.len()); let leader_keypair = Arc::new(Keypair::new()); diff --git a/rayon-threadlimit/src/lib.rs b/rayon-threadlimit/src/lib.rs index 6676b1133057b1..341c462642e65a 100644 --- a/rayon-threadlimit/src/lib.rs +++ b/rayon-threadlimit/src/lib.rs @@ -1,4 +1,18 @@ -pub mod thread_count; - #[macro_use] extern crate lazy_static; + +use std::sync::RwLock; + +//TODO remove this hack when rayon fixes itself +lazy_static! { + static ref MAX_RAYON_THREADS: RwLock = + RwLock::new(sys_info::cpu_num().unwrap() as usize); +} + +pub fn get_thread_count() -> usize { + *MAX_RAYON_THREADS.read().unwrap() +} + +pub fn set_thread_count(count: usize) { + *MAX_RAYON_THREADS.write().unwrap() = count; +} diff --git a/rayon-threadlimit/src/thread_count.rs b/rayon-threadlimit/src/thread_count.rs deleted file mode 100644 index 02a3e2fed66e89..00000000000000 --- a/rayon-threadlimit/src/thread_count.rs +++ /dev/null @@ -1,15 +0,0 @@ -use std::sync::RwLock; - -//TODO remove this hack when rayon fixes itself -lazy_static! { - static ref MAX_RAYON_THREADS: RwLock = - RwLock::new(sys_info::cpu_num().unwrap() as usize); -} - -pub fn get_thread_count() -> usize { - *MAX_RAYON_THREADS.read().unwrap() -} - -pub fn init_test_thread_count() { - *MAX_RAYON_THREADS.write().unwrap() = 1; -} diff --git a/runtime/src/accounts_db.rs b/runtime/src/accounts_db.rs index 7d66f06766c1e7..f25ecee1f7fe05 100644 --- a/runtime/src/accounts_db.rs +++ b/runtime/src/accounts_db.rs @@ -30,7 +30,7 @@ use serde::de::{MapAccess, Visitor}; use serde::ser::{SerializeMap, Serializer}; use serde::{Deserialize, Serialize}; use solana_measure::measure::Measure; -use solana_rayon_threadlimit::thread_count::get_thread_count; +use solana_rayon_threadlimit::get_thread_count; use solana_sdk::account::Account; use solana_sdk::pubkey::Pubkey; use std::collections::{HashMap, HashSet};