From 25ed92ca96247549faf09964be4a32860330e4f3 Mon Sep 17 00:00:00 2001 From: Lucas Date: Thu, 6 Jun 2024 15:23:36 -0300 Subject: [PATCH] Add shuttle test infrastructure --- Cargo.lock | 207 ++++++++++++++++++--- Cargo.toml | 3 + ledger-tool/Cargo.toml | 1 + program-runtime/Cargo.toml | 2 + program-runtime/src/invoke_context.rs | 4 +- program-runtime/src/loaded_programs.rs | 17 +- program-runtime/src/sysvar_cache.rs | 2 +- programs/bpf_loader/Cargo.toml | 4 + programs/bpf_loader/src/lib.rs | 10 +- programs/bpf_loader/src/syscalls/mod.rs | 2 +- programs/loader-v4/Cargo.toml | 4 + programs/loader-v4/src/lib.rs | 7 +- programs/sbf/Cargo.lock | 16 ++ programs/sbf/Cargo.toml | 2 + programs/stake/Cargo.toml | 1 + programs/system/Cargo.toml | 1 + svm/Cargo.toml | 7 + svm/src/message_processor.rs | 2 +- svm/src/program_loader.rs | 2 +- svm/src/transaction_processing_callback.rs | 2 +- svm/src/transaction_processor.rs | 23 ++- type-overrides/Cargo.toml | 19 ++ type-overrides/src/lib.rs | 49 +++++ 23 files changed, 323 insertions(+), 64 deletions(-) create mode 100644 type-overrides/Cargo.toml create mode 100644 type-overrides/src/lib.rs diff --git a/Cargo.lock b/Cargo.lock index 5a5217888ac20c..08fc3efd6069d1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -184,6 +184,7 @@ dependencies = [ "solana-storage-bigtable", "solana-streamer", "solana-transaction-status", + "solana-type-overrides", "solana-unified-scheduler-pool", "solana-version", "solana-vote-program", @@ -624,6 +625,12 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b34d609dfbaf33d6889b2b7106d3ca345eacad44200913df5ba02bfd31d2ba9" +[[package]] +name = "assoc" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfdc70193dadb9d7287fa4b633f15f90c876915b31f6af17da307fc59c9859a8" + [[package]] name = "async-channel" version = "1.9.0" @@ -882,6 +889,18 @@ dependencies = [ "typenum", ] +[[package]] +name = "bitvec" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bc2832c24239b0141d5674bb9174f9d68a8b5b3f2753311927c172ca46f7e9c" +dependencies = [ + "funty", + "radium", + "tap", + "wyz", +] + [[package]] name = "blake3" version = "1.5.1" @@ -1271,7 +1290,7 @@ dependencies = [ "num-traits", "serde", "wasm-bindgen", - "windows-targets 0.52.0", + "windows-targets 0.52.5", ] [[package]] @@ -2238,6 +2257,12 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a06f77d526c1a601b7c4cdd98f54b5eaabffc14d5f2f0296febdc7f357c6d3ba" +[[package]] +name = "funty" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c" + [[package]] name = "futures" version = "0.1.31" @@ -2374,6 +2399,20 @@ dependencies = [ "regex", ] +[[package]] +name = "generator" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "186014d53bc231d0090ef8d6f03e0920c54d85a5ed22f4f2f74315ec56cf83fb" +dependencies = [ + "cc", + "cfg-if 1.0.0", + "libc", + "log", + "rustversion", + "windows", +] + [[package]] name = "generic-array" version = "0.12.4" @@ -3789,6 +3828,12 @@ version = "6.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8e22443d1643a904602595ba1cd8f7d896afe56d26712531c5ff73a15b2fbf64" +[[package]] +name = "owo-colors" +version = "3.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1b04fb49957986fdce4d6ee7a65027d55d4b6d2265e5848bbb507b58ccfdb6f" + [[package]] name = "parity-tokio-ipc" version = "0.9.0" @@ -4341,6 +4386,12 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "radium" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc33ff2d4973d518d823d61aa239014831e521c75da58e3df4840d3f47749d09" + [[package]] name = "rand" version = "0.4.6" @@ -4440,6 +4491,15 @@ dependencies = [ "rand_core 0.5.1", ] +[[package]] +name = "rand_pcg" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59cad018caf63deb318e5a4586d99a24424a364f40f1e5778c29aca23f4fc73e" +dependencies = [ + "rand_core 0.6.4", +] + [[package]] name = "rand_xorshift" version = "0.3.0" @@ -4846,6 +4906,12 @@ dependencies = [ "winapi 0.3.9", ] +[[package]] +name = "scoped-tls" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1cf6437eb19a8f4a6cc0f7dca544973b0b78843adbfeb3683d1a94a0024a294" + [[package]] name = "scopeguard" version = "1.2.0" @@ -5188,6 +5254,25 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" +[[package]] +name = "shuttle" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d9a8db61a44e2b663f169a08206a789bcbd22ba32011e14951562848e7b9c98" +dependencies = [ + "assoc", + "bitvec", + "generator", + "hex", + "owo-colors", + "rand 0.8.5", + "rand_core 0.6.4", + "rand_pcg", + "scoped-tls", + "smallvec", + "tracing", +] + [[package]] name = "signal-hook" version = "0.3.17" @@ -5615,6 +5700,7 @@ dependencies = [ "solana-poseidon", "solana-program-runtime", "solana-sdk", + "solana-type-overrides", "solana-zk-token-sdk", "solana_rbpf", "test-case", @@ -6409,6 +6495,7 @@ dependencies = [ "solana-measure", "solana-program-runtime", "solana-sdk", + "solana-type-overrides", "solana_rbpf", ] @@ -6729,6 +6816,7 @@ dependencies = [ "solana-measure", "solana-metrics", "solana-sdk", + "solana-type-overrides", "solana_rbpf", "test-case", "thiserror", @@ -7220,6 +7308,7 @@ dependencies = [ "solana-logger", "solana-program-runtime", "solana-sdk", + "solana-type-overrides", "solana-vote-program", "test-case", ] @@ -7334,6 +7423,7 @@ dependencies = [ "solana-program-runtime", "solana-sdk", "solana-system-program", + "solana-type-overrides", ] [[package]] @@ -7349,6 +7439,7 @@ dependencies = [ "solana-logger", "solana-program-runtime", "solana-sdk", + "solana-type-overrides", ] [[package]] @@ -7575,6 +7666,16 @@ dependencies = [ "tokio", ] +[[package]] +name = "solana-type-overrides" +version = "2.0.0" +dependencies = [ + "futures 0.3.30", + "lazy_static", + "rand 0.8.5", + "shuttle", +] + [[package]] name = "solana-udp-client" version = "2.0.0" @@ -8232,6 +8333,12 @@ dependencies = [ "winapi 0.3.9", ] +[[package]] +name = "tap" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" + [[package]] name = "tar" version = "0.4.40" @@ -8745,11 +8852,10 @@ checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52" [[package]] name = "tracing" -version = "0.1.35" +version = "0.1.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a400e31aa60b9d44a52a8ee0343b5b18566b03a8321e0d321f695cf56e940160" +checksum = "c3523ab5a71916ccf420eebdf5521fcef02141234bbc0b8a49f2fdc4544364ef" dependencies = [ - "cfg-if 1.0.0", "log", "pin-project-lite", "tracing-attributes", @@ -9191,6 +9297,35 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" +[[package]] +name = "windows" +version = "0.54.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9252e5725dbed82865af151df558e754e4a3c2c30818359eb17465f1346a1b49" +dependencies = [ + "windows-core", + "windows-targets 0.52.5", +] + +[[package]] +name = "windows-core" +version = "0.54.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12661b9c89351d684a50a8a643ce5f608e20243b9fb84687800163429f161d65" +dependencies = [ + "windows-result", + "windows-targets 0.52.5", +] + +[[package]] +name = "windows-result" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "749f0da9cc72d82e600d8d2e44cadd0b9eedb9038f71a1c58556ac1c5791813b" +dependencies = [ + "windows-targets 0.52.5", +] + [[package]] name = "windows-sys" version = "0.48.0" @@ -9206,7 +9341,7 @@ version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" dependencies = [ - "windows-targets 0.52.0", + "windows-targets 0.52.5", ] [[package]] @@ -9226,17 +9361,18 @@ dependencies = [ [[package]] name = "windows-targets" -version = "0.52.0" +version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a18201040b24831fbb9e4eb208f8892e1f50a37feb53cc7ff887feb8f50e7cd" +checksum = "6f0713a46559409d202e70e28227288446bf7841d3211583a4b53e3f6d96e7eb" dependencies = [ - "windows_aarch64_gnullvm 0.52.0", - "windows_aarch64_msvc 0.52.0", - "windows_i686_gnu 0.52.0", - "windows_i686_msvc 0.52.0", - "windows_x86_64_gnu 0.52.0", - "windows_x86_64_gnullvm 0.52.0", - "windows_x86_64_msvc 0.52.0", + "windows_aarch64_gnullvm 0.52.5", + "windows_aarch64_msvc 0.52.5", + "windows_i686_gnu 0.52.5", + "windows_i686_gnullvm", + "windows_i686_msvc 0.52.5", + "windows_x86_64_gnu 0.52.5", + "windows_x86_64_gnullvm 0.52.5", + "windows_x86_64_msvc 0.52.5", ] [[package]] @@ -9247,9 +9383,9 @@ checksum = "91ae572e1b79dba883e0d315474df7305d12f569b400fcf90581b06062f7e1bc" [[package]] name = "windows_aarch64_gnullvm" -version = "0.52.0" +version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb7764e35d4db8a7921e09562a0304bf2f93e0a51bfccee0bd0bb0b666b015ea" +checksum = "7088eed71e8b8dda258ecc8bac5fb1153c5cffaf2578fc8ff5d61e23578d3263" [[package]] name = "windows_aarch64_msvc" @@ -9259,9 +9395,9 @@ checksum = "b2ef27e0d7bdfcfc7b868b317c1d32c641a6fe4629c171b8928c7b08d98d7cf3" [[package]] name = "windows_aarch64_msvc" -version = "0.52.0" +version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbaa0368d4f1d2aaefc55b6fcfee13f41544ddf36801e793edbbfd7d7df075ef" +checksum = "9985fd1504e250c615ca5f281c3f7a6da76213ebd5ccc9561496568a2752afb6" [[package]] name = "windows_i686_gnu" @@ -9271,9 +9407,15 @@ checksum = "622a1962a7db830d6fd0a69683c80a18fda201879f0f447f065a3b7467daa241" [[package]] name = "windows_i686_gnu" -version = "0.52.0" +version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a28637cb1fa3560a16915793afb20081aba2c92ee8af57b4d5f28e4b3e7df313" +checksum = "88ba073cf16d5372720ec942a8ccbf61626074c6d4dd2e745299726ce8b89670" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87f4261229030a858f36b459e748ae97545d6f1ec60e5e0d6a3d32e0dc232ee9" [[package]] name = "windows_i686_msvc" @@ -9283,9 +9425,9 @@ checksum = "4542c6e364ce21bf45d69fdd2a8e455fa38d316158cfd43b3ac1c5b1b19f8e00" [[package]] name = "windows_i686_msvc" -version = "0.52.0" +version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ffe5e8e31046ce6230cc7215707b816e339ff4d4d67c65dffa206fd0f7aa7b9a" +checksum = "db3c2bf3d13d5b658be73463284eaf12830ac9a26a90c717b7f771dfe97487bf" [[package]] name = "windows_x86_64_gnu" @@ -9295,9 +9437,9 @@ checksum = "ca2b8a661f7628cbd23440e50b05d705db3686f894fc9580820623656af974b1" [[package]] name = "windows_x86_64_gnu" -version = "0.52.0" +version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d6fa32db2bc4a2f5abeacf2b69f7992cd09dca97498da74a151a3132c26befd" +checksum = "4e4246f76bdeff09eb48875a0fd3e2af6aada79d409d33011886d3e1581517d9" [[package]] name = "windows_x86_64_gnullvm" @@ -9307,9 +9449,9 @@ checksum = "7896dbc1f41e08872e9d5e8f8baa8fdd2677f29468c4e156210174edc7f7b953" [[package]] name = "windows_x86_64_gnullvm" -version = "0.52.0" +version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a657e1e9d3f514745a572a6846d3c7aa7dbe1658c056ed9c3344c4109a6949e" +checksum = "852298e482cd67c356ddd9570386e2862b5673c85bd5f88df9ab6802b334c596" [[package]] name = "windows_x86_64_msvc" @@ -9319,9 +9461,9 @@ checksum = "1a515f5799fe4961cb532f983ce2b23082366b898e52ffbce459c86f67c8378a" [[package]] name = "windows_x86_64_msvc" -version = "0.52.0" +version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dff9641d1cd4be8d1a070daf9e3773c5f67e78b4d9d42263020c057706765c04" +checksum = "bec47e5bfd1bff0eeaf6d8b485cc1074891a197ab4225d504cb7a1ab88b02bf0" [[package]] name = "winnow" @@ -9342,6 +9484,15 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "wyz" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05f360fc0b24296329c78fda852a1e9ae82de9cf7b27dae4b7f62f118f77b9ed" +dependencies = [ + "tap", +] + [[package]] name = "x509-parser" version = "0.14.0" diff --git a/Cargo.toml b/Cargo.toml index 9cece9212d520c..7fd5aebf01afc5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -113,6 +113,7 @@ members = [ "transaction-metrics-tracker", "transaction-status", "turbine", + "type-overrides", "udp-client", "unified-scheduler-logic", "unified-scheduler-pool", @@ -307,6 +308,7 @@ serde_yaml = "0.9.34" serial_test = "2.0.0" sha2 = "0.10.8" sha3 = "0.10.8" +shuttle = "0.7.1" signal-hook = "0.3.17" siphasher = "0.3.11" smallvec = "1.13.2" @@ -391,6 +393,7 @@ solana-tpu-client = { path = "tpu-client", version = "=2.0.0", default-features solana-transaction-status = { path = "transaction-status", version = "=2.0.0" } solana-transaction-metrics-tracker = { path = "transaction-metrics-tracker", version = "=2.0.0" } solana-turbine = { path = "turbine", version = "=2.0.0" } +solana-type-overrides = { path = "type-overrides", version = "=2.0.0" } solana-udp-client = { path = "udp-client", version = "=2.0.0" } solana-version = { path = "version", version = "=2.0.0" } solana-vote = { path = "vote", version = "=2.0.0" } diff --git a/ledger-tool/Cargo.toml b/ledger-tool/Cargo.toml index 351aa44462cf1e..d9e55d9771b36c 100644 --- a/ledger-tool/Cargo.toml +++ b/ledger-tool/Cargo.toml @@ -48,6 +48,7 @@ solana-stake-program = { workspace = true } solana-storage-bigtable = { workspace = true } solana-streamer = { workspace = true } solana-transaction-status = { workspace = true } +solana-type-overrides = { workspace = true } solana-unified-scheduler-pool = { workspace = true } solana-version = { workspace = true } solana-vote-program = { workspace = true } diff --git a/program-runtime/Cargo.toml b/program-runtime/Cargo.toml index a3d25a9cb37f03..56809c02e072fd 100644 --- a/program-runtime/Cargo.toml +++ b/program-runtime/Cargo.toml @@ -28,6 +28,7 @@ solana-frozen-abi-macro = { workspace = true, optional = true } solana-measure = { workspace = true } solana-metrics = { workspace = true } solana-sdk = { workspace = true } +solana-type-overrides = { workspace = true } solana_rbpf = { workspace = true } thiserror = { workspace = true } @@ -54,3 +55,4 @@ frozen-abi = [ "solana-compute-budget/frozen-abi", "solana-sdk/frozen-abi", ] +shuttle-test = ["solana-type-overrides/shuttle-test"] diff --git a/program-runtime/src/invoke_context.rs b/program-runtime/src/invoke_context.rs index ace5e0de11475f..73934953dcb972 100644 --- a/program-runtime/src/invoke_context.rs +++ b/program-runtime/src/invoke_context.rs @@ -36,12 +36,12 @@ use { IndexOfAccount, InstructionAccount, TransactionAccount, TransactionContext, }, }, + solana_type_overrides::sync::{atomic::Ordering, Arc}, std::{ alloc::Layout, cell::RefCell, fmt::{self, Debug}, rc::Rc, - sync::{atomic::Ordering, Arc}, }, }; @@ -675,7 +675,7 @@ macro_rules! with_mock_invoke_context { account::ReadableAccount, feature_set::FeatureSet, hash::Hash, sysvar::rent::Rent, transaction_context::TransactionContext, }, - std::sync::Arc, + solana_type_overrides::sync::Arc, $crate::{ invoke_context::{EnvironmentConfig, InvokeContext}, loaded_programs::ProgramCacheForTxBatch, diff --git a/program-runtime/src/loaded_programs.rs b/program-runtime/src/loaded_programs.rs index 9728fe274e65e0..d113a7b56d9a20 100644 --- a/program-runtime/src/loaded_programs.rs +++ b/program-runtime/src/loaded_programs.rs @@ -5,7 +5,6 @@ use { }, log::{debug, error, log_enabled, trace}, percentage::PercentageInteger, - rand::{thread_rng, Rng}, solana_measure::measure::Measure, solana_rbpf::{ elf::Executable, @@ -20,13 +19,17 @@ use { pubkey::Pubkey, saturating_add_assign, }, - std::{ - collections::{hash_map::Entry, HashMap}, - fmt::{Debug, Formatter}, + solana_type_overrides::{ + rand::{thread_rng, Rng}, sync::{ atomic::{AtomicU64, Ordering}, Arc, Condvar, Mutex, RwLock, }, + thread, + }, + std::{ + collections::{hash_map::Entry, HashMap}, + fmt::{Debug, Formatter}, }, }; @@ -598,7 +601,7 @@ enum IndexImplementation { /// It is possible that multiple TX batches from different slots need different versions of a /// program. The deployment slot of a program is only known after load tho, /// so all loads for a given program key are serialized. - loading_entries: Mutex>, + loading_entries: Mutex>, }, } @@ -1100,7 +1103,7 @@ impl ProgramCache { if let Entry::Vacant(entry) = entry { entry.insert(( loaded_programs_for_tx_batch.slot, - std::thread::current().id(), + thread::current().id(), )); cooperative_loading_task = Some((*key, *usage_count)); } @@ -1134,7 +1137,7 @@ impl ProgramCache { loading_entries, .. } => { let loading_thread = loading_entries.get_mut().unwrap().remove(&key); - debug_assert_eq!(loading_thread, Some((slot, std::thread::current().id()))); + debug_assert_eq!(loading_thread, Some((slot, thread::current().id()))); // Check that it will be visible to our own fork once inserted if loaded_program.deployment_slot > self.latest_root_slot && !matches!( diff --git a/program-runtime/src/sysvar_cache.rs b/program-runtime/src/sysvar_cache.rs index 313faec6e11b02..1a270484410531 100644 --- a/program-runtime/src/sysvar_cache.rs +++ b/program-runtime/src/sysvar_cache.rs @@ -13,7 +13,7 @@ use { }, transaction_context::{IndexOfAccount, InstructionContext, TransactionContext}, }, - std::sync::Arc, + solana_type_overrides::sync::Arc, }; #[cfg(all(RUSTC_WITH_SPECIALIZATION, feature = "frozen-abi"))] diff --git a/programs/bpf_loader/Cargo.toml b/programs/bpf_loader/Cargo.toml index 6d3147d1e6bf03..102836486c91ef 100644 --- a/programs/bpf_loader/Cargo.toml +++ b/programs/bpf_loader/Cargo.toml @@ -20,6 +20,7 @@ solana-measure = { workspace = true } solana-poseidon = { workspace = true } solana-program-runtime = { workspace = true } solana-sdk = { workspace = true } +solana-type-overrides = { workspace = true } solana-zk-token-sdk = { workspace = true } solana_rbpf = { workspace = true } thiserror = { workspace = true } @@ -37,3 +38,6 @@ name = "solana_bpf_loader_program" [package.metadata.docs.rs] targets = ["x86_64-unknown-linux-gnu"] + +[features] +shuttle-test = ["solana-type-overrides/shuttle-test", "solana-program-runtime/shuttle-test"] diff --git a/programs/bpf_loader/src/lib.rs b/programs/bpf_loader/src/lib.rs index 8f14544ffcf0b3..4ae7717fafa7b6 100644 --- a/programs/bpf_loader/src/lib.rs +++ b/programs/bpf_loader/src/lib.rs @@ -46,12 +46,8 @@ use { system_instruction::{self, MAX_PERMITTED_DATA_LENGTH}, transaction_context::{IndexOfAccount, InstructionContext, TransactionContext}, }, - std::{ - cell::RefCell, - mem, - rc::Rc, - sync::{atomic::Ordering, Arc}, - }, + solana_type_overrides::sync::{atomic::Ordering, Arc}, + std::{cell::RefCell, mem, rc::Rc}, syscalls::{create_program_runtime_environment_v1, morph_into_deployment_environment_v1}, }; @@ -324,7 +320,7 @@ macro_rules! create_vm { #[macro_export] macro_rules! mock_create_vm { ($vm:ident, $additional_regions:expr, $accounts_metadata:expr, $invoke_context:expr $(,)?) => { - let loader = std::sync::Arc::new(BuiltinProgram::new_mock()); + let loader = solana_type_overrides::sync::Arc::new(BuiltinProgram::new_mock()); let function_registry = solana_rbpf::program::FunctionRegistry::default(); let executable = solana_rbpf::elf::Executable::::from_text_bytes( &[0x95, 0, 0, 0, 0, 0, 0, 0], diff --git a/programs/bpf_loader/src/syscalls/mod.rs b/programs/bpf_loader/src/syscalls/mod.rs index 425cbf8d56e820..5a6249295977ba 100644 --- a/programs/bpf_loader/src/syscalls/mod.rs +++ b/programs/bpf_loader/src/syscalls/mod.rs @@ -57,12 +57,12 @@ use { sysvar::{Sysvar, SysvarId}, transaction_context::{IndexOfAccount, InstructionAccount}, }, + solana_type_overrides::sync::Arc, std::{ alloc::Layout, mem::{align_of, size_of}, slice::from_raw_parts_mut, str::{from_utf8, Utf8Error}, - sync::Arc, }, thiserror::Error as ThisError, }; diff --git a/programs/loader-v4/Cargo.toml b/programs/loader-v4/Cargo.toml index da7d5c10c72a51..79c078f29785b5 100644 --- a/programs/loader-v4/Cargo.toml +++ b/programs/loader-v4/Cargo.toml @@ -14,6 +14,7 @@ solana-compute-budget = { workspace = true } solana-measure = { workspace = true } solana-program-runtime = { workspace = true } solana-sdk = { workspace = true } +solana-type-overrides = { workspace = true } solana_rbpf = { workspace = true } [dev-dependencies] @@ -25,3 +26,6 @@ name = "solana_loader_v4_program" [package.metadata.docs.rs] targets = ["x86_64-unknown-linux-gnu"] + +[features] +shuttle-test = ["solana-type-overrides/shuttle-test", "solana-program-runtime/shuttle-test"] diff --git a/programs/loader-v4/src/lib.rs b/programs/loader-v4/src/lib.rs index b564e84c43a9d4..681e8fa80d5061 100644 --- a/programs/loader-v4/src/lib.rs +++ b/programs/loader-v4/src/lib.rs @@ -30,11 +30,8 @@ use { saturating_add_assign, transaction_context::{BorrowedAccount, InstructionContext}, }, - std::{ - cell::RefCell, - rc::Rc, - sync::{atomic::Ordering, Arc}, - }, + solana_type_overrides::sync::{atomic::Ordering, Arc}, + std::{cell::RefCell, rc::Rc}, }; pub const DEFAULT_COMPUTE_UNITS: u64 = 2_000; diff --git a/programs/sbf/Cargo.lock b/programs/sbf/Cargo.lock index 090090f8c16e98..6658e64ed3d1b6 100644 --- a/programs/sbf/Cargo.lock +++ b/programs/sbf/Cargo.lock @@ -4737,6 +4737,7 @@ dependencies = [ "solana-poseidon", "solana-program-runtime", "solana-sdk", + "solana-type-overrides", "solana-zk-token-sdk", "solana_rbpf", "thiserror", @@ -5216,6 +5217,7 @@ dependencies = [ "solana-measure", "solana-program-runtime", "solana-sdk", + "solana-type-overrides", "solana_rbpf", ] @@ -5400,6 +5402,7 @@ dependencies = [ "solana-measure", "solana-metrics", "solana-sdk", + "solana-type-overrides", "solana_rbpf", "thiserror", ] @@ -5723,6 +5726,7 @@ dependencies = [ "solana-sdk", "solana-svm", "solana-transaction-status", + "solana-type-overrides", "solana_rbpf", "walkdir", ] @@ -6210,6 +6214,7 @@ dependencies = [ "solana-config-program", "solana-program-runtime", "solana-sdk", + "solana-type-overrides", "solana-vote-program", ] @@ -6311,6 +6316,7 @@ dependencies = [ "solana-program-runtime", "solana-sdk", "solana-system-program", + "solana-type-overrides", ] [[package]] @@ -6323,6 +6329,7 @@ dependencies = [ "serde_derive", "solana-program-runtime", "solana-sdk", + "solana-type-overrides", ] [[package]] @@ -6463,6 +6470,15 @@ dependencies = [ "tokio", ] +[[package]] +name = "solana-type-overrides" +version = "2.0.0" +dependencies = [ + "futures 0.3.30", + "lazy_static", + "rand 0.8.5", +] + [[package]] name = "solana-udp-client" version = "2.0.0" diff --git a/programs/sbf/Cargo.toml b/programs/sbf/Cargo.toml index 87ab32c025b5f3..da1dce0c320ac4 100644 --- a/programs/sbf/Cargo.toml +++ b/programs/sbf/Cargo.toml @@ -50,6 +50,7 @@ solana-sbf-rust-realloc-invoke = { path = "rust/realloc_invoke", version = "=2.0 solana-sdk = { path = "../../sdk", version = "=2.0.0" } solana-svm = { path = "../../svm", version = "=2.0.0" } solana-transaction-status = { path = "../../transaction-status", version = "=2.0.0" } +solana-type-overrides = { path = "../../type-overrides", version = "=2.0.0" } agave-validator = { path = "../../validator", version = "=2.0.0" } solana-zk-token-sdk = { path = "../../zk-token-sdk", version = "=2.0.0" } solana_rbpf = "=0.8.1" @@ -98,6 +99,7 @@ solana-sbf-rust-realloc-invoke = { workspace = true } solana-sdk = { workspace = true } solana-svm = { workspace = true } solana-transaction-status = { workspace = true } +solana-type-overrides = { workspace = true } solana_rbpf = { workspace = true } [dev-dependencies] diff --git a/programs/stake/Cargo.toml b/programs/stake/Cargo.toml index 1ccd2e857c7521..b55c904b4b17a6 100644 --- a/programs/stake/Cargo.toml +++ b/programs/stake/Cargo.toml @@ -15,6 +15,7 @@ log = { workspace = true } solana-config-program = { workspace = true } solana-program-runtime = { workspace = true } solana-sdk = { workspace = true } +solana-type-overrides = { workspace = true } solana-vote-program = { workspace = true } [dev-dependencies] diff --git a/programs/system/Cargo.toml b/programs/system/Cargo.toml index a3366d5c8f3f4f..1e5643587c8f88 100644 --- a/programs/system/Cargo.toml +++ b/programs/system/Cargo.toml @@ -16,6 +16,7 @@ serde = { workspace = true } serde_derive = { workspace = true } solana-program-runtime = { workspace = true } solana-sdk = { workspace = true } +solana-type-overrides = { workspace = true } [dev-dependencies] assert_matches = { workspace = true } diff --git a/svm/Cargo.toml b/svm/Cargo.toml index ebe845cd130837..bd84a1ffc59e1e 100644 --- a/svm/Cargo.toml +++ b/svm/Cargo.toml @@ -25,6 +25,7 @@ solana-metrics = { workspace = true } solana-program-runtime = { workspace = true } solana-sdk = { workspace = true } solana-system-program = { workspace = true } +solana-type-overrides = { workspace = true } [lib] crate-type = ["lib"] @@ -57,3 +58,9 @@ frozen-abi = [ "solana-program-runtime/frozen-abi", "solana-sdk/frozen-abi", ] +shuttle-test = [ + "solana-type-overrides/shuttle-test", + "solana-program-runtime/shuttle-test", + "solana-bpf-loader-program/shuttle-test", + "solana-loader-v4-program/shuttle-test", +] diff --git a/svm/src/message_processor.rs b/svm/src/message_processor.rs index 7b366e82304975..2948f5416285f5 100644 --- a/svm/src/message_processor.rs +++ b/svm/src/message_processor.rs @@ -654,7 +654,7 @@ mod tests { // copies the `random` implementation at: // https://docs.rs/libsecp256k1/latest/src/libsecp256k1/lib.rs.html#430 let secret_key = { - use rand::RngCore; + use solana_type_overrides::rand::RngCore; let mut rng = rand::thread_rng(); loop { let mut ret = [0u8; libsecp256k1::util::SECRET_KEY_SIZE]; diff --git a/svm/src/program_loader.rs b/svm/src/program_loader.rs index 8920340d4a2544..e947488c563402 100644 --- a/svm/src/program_loader.rs +++ b/svm/src/program_loader.rs @@ -18,7 +18,7 @@ use { loader_v4::{self, LoaderV4State, LoaderV4Status}, pubkey::Pubkey, }, - std::sync::Arc, + solana_type_overrides::sync::Arc, }; #[derive(Debug)] diff --git a/svm/src/transaction_processing_callback.rs b/svm/src/transaction_processing_callback.rs index ded481eda8ea5f..75d37634c8f093 100644 --- a/svm/src/transaction_processing_callback.rs +++ b/svm/src/transaction_processing_callback.rs @@ -4,7 +4,7 @@ use { account::AccountSharedData, feature_set::FeatureSet, hash::Hash, pubkey::Pubkey, rent_collector::RentCollector, }, - std::sync::Arc, + solana_type_overrides::sync::Arc, }; /// Runtime callbacks for transaction processing. diff --git a/svm/src/transaction_processor.rs b/svm/src/transaction_processor.rs index e7da3efce95602..b61b2bc6ae2f5d 100644 --- a/svm/src/transaction_processor.rs +++ b/svm/src/transaction_processor.rs @@ -42,15 +42,12 @@ use { transaction::{SanitizedTransaction, TransactionError}, transaction_context::{ExecutionRecord, TransactionContext}, }, + solana_type_overrides::sync::{atomic::Ordering, Arc, RwLock}, std::{ cell::RefCell, collections::{hash_map::Entry, HashMap, HashSet}, fmt::{Debug, Formatter}, rc::Rc, - sync::{ - atomic::Ordering::{self, Relaxed}, - Arc, RwLock, - }, }, }; @@ -505,12 +502,18 @@ impl TransactionBatchProcessor { &self.epoch_schedule, false, ) { - recompiled - .tx_usage_counter - .fetch_add(program_to_recompile.tx_usage_counter.load(Relaxed), Relaxed); - recompiled - .ix_usage_counter - .fetch_add(program_to_recompile.ix_usage_counter.load(Relaxed), Relaxed); + recompiled.tx_usage_counter.fetch_add( + program_to_recompile + .tx_usage_counter + .load(Ordering::Relaxed), + Ordering::Relaxed, + ); + recompiled.ix_usage_counter.fetch_add( + program_to_recompile + .ix_usage_counter + .load(Ordering::Relaxed), + Ordering::Relaxed, + ); let mut program_cache = self.program_cache.write().unwrap(); program_cache.assign_program(key, recompiled); } diff --git a/type-overrides/Cargo.toml b/type-overrides/Cargo.toml new file mode 100644 index 00000000000000..07b13fb6effa7f --- /dev/null +++ b/type-overrides/Cargo.toml @@ -0,0 +1,19 @@ +[package] +name = "solana-type-overrides" +description = "Type overrides for specialized testing" +publish = false +version = { workspace = true } +authors = { workspace = true } +repository = { workspace = true } +homepage = { workspace = true } +license = { workspace = true } +edition = { workspace = true } + +[dependencies] +futures = { workspace = true } +lazy_static = { workspace = true } +rand = { workspace = true } +shuttle = { workspace = true, optional = true } + +[features] +shuttle-test = ["dep:shuttle"] diff --git a/type-overrides/src/lib.rs b/type-overrides/src/lib.rs new file mode 100644 index 00000000000000..89855e18e19ddc --- /dev/null +++ b/type-overrides/src/lib.rs @@ -0,0 +1,49 @@ +/// +/// This lib contains both standard imports and imports shuttle. +/// Shuttle is a Rust crate that facilitates multithreaded testing. It has its own scheduler +/// and can efficiently detect bugs in concurrent code. The downside is that we need to replace +/// all imports by those from Shuttle. +/// +/// Instead of importing from std, rand, and so on, import the following from solana-type-override, +/// and include the 'shuttle-test' feature in your crate to use shuttle. +pub mod executor { + #[cfg(not(feature = "shuttle-test"))] + pub use futures::executor::*; + #[cfg(feature = "shuttle-test")] + pub use shuttle::future::*; +} + +pub mod hint { + #[cfg(feature = "shuttle-test")] + pub use shuttle::hint::*; + #[cfg(not(feature = "shuttle-test"))] + pub use std::hint::*; +} + +pub mod lazy_static { + #[cfg(not(feature = "shuttle-test"))] + pub use lazy_static::*; + #[cfg(feature = "shuttle-test")] + pub use shuttle::lazy_static::*; +} + +pub mod rand { + #[cfg(not(feature = "shuttle-test"))] + pub use rand::{thread_rng, Rng, RngCore}; + #[cfg(feature = "shuttle-test")] + pub use shuttle::rand::{thread_rng, Rng, RngCore}; +} + +pub mod sync { + #[cfg(feature = "shuttle-test")] + pub use shuttle::sync::*; + #[cfg(not(feature = "shuttle-test"))] + pub use std::sync::*; +} + +pub mod thread { + #[cfg(feature = "shuttle-test")] + pub use shuttle::thread::*; + #[cfg(not(feature = "shuttle-test"))] + pub use std::thread::*; +}