diff --git a/crates/fuzzing/src/oracles/engine.rs b/crates/fuzzing/src/oracles/engine.rs index 686858ea667d..bcb76b49403c 100644 --- a/crates/fuzzing/src/oracles/engine.rs +++ b/crates/fuzzing/src/oracles/engine.rs @@ -6,36 +6,52 @@ use anyhow::Error; use arbitrary::Unstructured; use wasmtime::Trap; -/// Pick one of the engines implemented in this module that is compatible with -/// the Wasm features passed in `features` and, when fuzzing Wasmtime against -/// itself, an existing `wasmtime_engine`. +/// Pick one of the engines implemented in this module that is: +/// - in the list of `allowed` engines +/// - can evaluate Wasm modules compatible with `existing_config`. pub fn choose( u: &mut Unstructured<'_>, existing_config: &Config, + allowed: &[&str], ) -> arbitrary::Result> { - // Filter out any engines that cannot match the given configuration. + // Filter out any engines that cannot match the `existing_config` or are not + // `allowed`. let mut engines: Vec> = vec![]; - let mut config2: WasmtimeConfig = u.arbitrary()?; // TODO change to WasmtimeConfig - config2.make_compatible_with(&existing_config.wasmtime); - let config2 = Config { - wasmtime: config2, - module_config: existing_config.module_config.clone(), - }; - if let Result::Ok(e) = WasmtimeEngine::new(config2) { - engines.push(Box::new(e)) + + if allowed.contains(&"wasmtime") { + let mut new_wasmtime_config: WasmtimeConfig = u.arbitrary()?; + new_wasmtime_config.make_compatible_with(&existing_config.wasmtime); + let new_config = Config { + wasmtime: new_wasmtime_config, + module_config: existing_config.module_config.clone(), + }; + if let Result::Ok(e) = WasmtimeEngine::new(new_config) { + engines.push(Box::new(e)) + } } - if let Result::Ok(e) = WasmiEngine::new(&existing_config.module_config) { - engines.push(Box::new(e)) + + if allowed.contains(&"wasmi") { + if let Result::Ok(e) = WasmiEngine::new(&existing_config.module_config) { + engines.push(Box::new(e)) + } } + #[cfg(feature = "fuzz-spec-interpreter")] - if let Result::Ok(e) = - crate::oracles::diff_spec::SpecInterpreter::new(&existing_config.module_config) - { - engines.push(Box::new(e)) + if allowed.contains(&"spec") { + if let Result::Ok(e) = + crate::oracles::diff_spec::SpecInterpreter::new(&existing_config.module_config) + { + engines.push(Box::new(e)) + } } + #[cfg(not(any(windows, target_arch = "s390x")))] - if let Result::Ok(e) = crate::oracles::diff_v8::V8Engine::new(&existing_config.module_config) { - engines.push(Box::new(e)) + if allowed.contains(&"v8") { + if let Result::Ok(e) = + crate::oracles::diff_v8::V8Engine::new(&existing_config.module_config) + { + engines.push(Box::new(e)) + } } // Use the input of the fuzzer to pick an engine that we'll be fuzzing @@ -93,6 +109,76 @@ pub fn setup_engine_runtimes() { crate::oracles::diff_spec::setup_ocaml_runtime(); } +/// Build a list of allowed values from the given `defaults` using the +/// `env_list`. +/// +/// ``` +/// # use wasmtime_fuzzing::oracles::engine::build_allowed_env_list; +/// // Passing no `env_list` returns the defaults: +/// assert_eq!(build_allowed_env_list(None, &["a"]), vec!["a"]); +/// // We can build up a subset of the defaults: +/// assert_eq!(build_allowed_env_list(Some(vec!["b".to_string()]), &["a","b"]), vec!["b"]); +/// // Alternately we can subtract from the defaults: +/// assert_eq!(build_allowed_env_list(Some(vec!["-a".to_string()]), &["a","b"]), vec!["b"]); +/// ``` +/// ```should_panic +/// # use wasmtime_fuzzing::oracles::engine::build_allowed_env_list; +/// // We are not allowed to mix set "addition" and "subtraction"; the following +/// // will panic: +/// build_allowed_env_list(Some(vec!["-a".to_string(), "b".to_string()]), &["a", "b"]); +/// ``` +/// ```should_panic +/// # use wasmtime_fuzzing::oracles::engine::build_allowed_env_list; +/// // This will also panic if invalid values are used: +/// build_allowed_env_list(Some(vec!["c".to_string()]), &["a", "b"]); +/// ``` +pub fn build_allowed_env_list<'a>( + env_list: Option>, + defaults: &[&'a str], +) -> Vec<&'a str> { + if let Some(configured) = &env_list { + // Check that the names are either all additions or all subtractions. + let subtract_from_defaults = configured.iter().all(|c| c.starts_with("-")); + let add_from_defaults = configured.iter().all(|c| !c.starts_with("-")); + let start = if subtract_from_defaults { 1 } else { 0 }; + if !subtract_from_defaults && !add_from_defaults { + panic!( + "all configured values must either subtract or add from defaults; found mixed values: {:?}", + &env_list + ); + } + + // Check that the configured names are valid ones. + for c in configured { + if !defaults.contains(&&c[start..]) { + panic!( + "invalid environment configuration `{}`; must be one of: {:?}", + c, defaults + ); + } + } + + // Select only the allowed names. + let mut allowed = Vec::with_capacity(defaults.len()); + for &d in defaults { + let mentioned = configured.iter().any(|c| &c[start..] == d); + if (add_from_defaults && mentioned) || (subtract_from_defaults && !mentioned) { + allowed.push(d); + } + } + allowed + } else { + defaults.to_vec() + } +} + +/// Retrieve a comma-delimited list of values from an environment variable. +pub fn parse_env_list(env_variable: &str) -> Option> { + std::env::var(env_variable) + .ok() + .map(|l| l.split(",").map(|s| s.to_owned()).collect()) +} + #[cfg(test)] pub fn smoke_test_engine(mk_engine: impl Fn(Config) -> anyhow::Result) where diff --git a/fuzz/fuzz_targets/differential.rs b/fuzz/fuzz_targets/differential.rs index c80883543b68..0d24eb94b0f4 100644 --- a/fuzz/fuzz_targets/differential.rs +++ b/fuzz/fuzz_targets/differential.rs @@ -8,22 +8,48 @@ use std::sync::Once; use wasmtime::Trap; use wasmtime_fuzzing::generators::{Config, DiffValue, DiffValueType, SingleInstModule}; use wasmtime_fuzzing::oracles::diff_wasmtime::WasmtimeInstance; +use wasmtime_fuzzing::oracles::engine::{build_allowed_env_list, parse_env_list}; use wasmtime_fuzzing::oracles::{differential, engine, log_wasm}; // Upper limit on the number of invocations for each WebAssembly function // executed by this fuzz target. const NUM_INVOCATIONS: usize = 5; +// Only run once when the fuzz target loads. +static SETUP: Once = Once::new(); + +// Environment-specified configuration for controlling the kinds of engines and +// modules used by this fuzz target. E.g.: +// - ALLOWED_ENGINES=wasmi,spec cargo +nightly fuzz run ... +// - ALLOWED_ENGINES=-v8 cargo +nightly fuzz run ... +// - ALLOWED_MODULES=single-inst cargo +nightly fuzz run ... +static mut ALLOWED_ENGINES: Vec<&str> = vec![]; +static mut ALLOWED_MODULES: Vec<&str> = vec![]; + // Statistics about what's actually getting executed during fuzzing static STATS: RuntimeStats = RuntimeStats::new(); -// The spec interpreter requires special one-time setup. -static SETUP: Once = Once::new(); - fuzz_target!(|data: &[u8]| { - // To avoid a uncaught `SIGSEGV` due to signal handlers; see comments on - // `setup_ocaml_runtime`. - SETUP.call_once(|| engine::setup_engine_runtimes()); + SETUP.call_once(|| { + // To avoid a uncaught `SIGSEGV` due to signal handlers; see comments on + // `setup_ocaml_runtime`. + engine::setup_engine_runtimes(); + + // Retrieve the configuration for this fuzz target from `ALLOWED_*` + // environment variables. + let allowed_engines = build_allowed_env_list( + parse_env_list("ALLOWED_ENGINES"), + &["wasmtime", "wasmi", "spec", "v8"], + ); + let allowed_modules = build_allowed_env_list( + parse_env_list("ALLOWED_MODULES"), + &["wasm-smith", "single-inst"], + ); + unsafe { + ALLOWED_ENGINES = allowed_engines; + ALLOWED_MODULES = allowed_modules; + } + }); // Errors in `run` have to do with not enough input in `data`, which we // ignore here since it doesn't affect how we'd like to fuzz. @@ -37,20 +63,31 @@ fn run(data: &[u8]) -> Result<()> { let mut config: Config = u.arbitrary()?; config.set_differential_config(); - // Generate the Wasm module. - let wasm = if u.arbitrary()? { + // Generate the Wasm module; this is specified by either the ALLOWED_MODULES + // environment variable or a random selection between wasm-smith and + // single-inst. + let build_wasm_smith_module = |u: &mut Unstructured, config: &mut Config| -> Result<_> { STATS.wasm_smith_modules.fetch_add(1, SeqCst); - let module = config.generate(&mut u, Some(1000))?; - module.to_bytes() - } else { + let module = config.generate(u, Some(1000))?; + Ok(module.to_bytes()) + }; + let build_single_inst_module = |u: &mut Unstructured, config: &mut Config| -> Result<_> { STATS.single_instruction_modules.fetch_add(1, SeqCst); - let module = SingleInstModule::new(&mut u, &mut config.module_config)?; - module.to_bytes() + let module = SingleInstModule::new(u, &mut config.module_config)?; + Ok(module.to_bytes()) + }; + if unsafe { ALLOWED_MODULES.is_empty() } { + panic!("unable to generate a module to fuzz against; check `ALLOWED_MODULES`") + } + let wasm = match *u.choose(unsafe { ALLOWED_MODULES.as_slice() })? { + "wasm-smith" => build_wasm_smith_module(&mut u, &mut config)?, + "single-inst" => build_single_inst_module(&mut u, &mut config)?, + _ => unreachable!(), }; log_wasm(&wasm); // Choose a left-hand side Wasm engine. - let mut lhs = engine::choose(&mut u, &config)?; + let mut lhs = engine::choose(&mut u, &config, unsafe { &ALLOWED_ENGINES })?; let lhs_instance = lhs.instantiate(&wasm); STATS.bump_engine(lhs.name());