Skip to content

Commit

Permalink
[fuzz] Remove some differential fuzz targets (#4735)
Browse files Browse the repository at this point in the history 
* [fuzz] Remove some differential fuzz targets

The changes in #4515 do everything the `differential_spec` and
`differential_wasmi` fuzz target already do. These fuzz targets are now
redundant and this PR removes them. It also updates the fuzz
documentation slightly.
  • Loading branch information
@abrown
abrown authored Aug 19, 2022
1 parent 80c77da commit 8b7fb19
Show file tree
Hide file tree
Showing 7 changed files with 40 additions and 338 deletions.
3 changes: 2 additions & 1 deletion .github/workflows/main.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -196,7 +196,8 @@ jobs:
with:
toolchain: nightly-2022-04-27
- run: cargo install cargo-fuzz --vers "^0.11"
# Install OCaml packages necessary for 'differential_spec' fuzz target.
# Install the OCaml packages necessary for fuzz targets that use the
# `wasm-spec-interpreter`.
- run: sudo apt install -y ocaml-nox ocamlbuild ocaml-findlib libzarith-ocaml-dev
- run: cargo fetch
working-directory: ./fuzz
Expand Down
249 changes: 0 additions & 249 deletions crates/fuzzing/src/oracles.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,6 @@ use self::diff_wasmtime::WasmtimeInstance;
use self::engine::DiffInstance;
use crate::generators::{self, DiffValue};
use arbitrary::Arbitrary;
use log::debug;
pub use stacks::check_stacks;
use std::cell::Cell;
use std::collections::hash_map::DefaultHasher;
Expand Down Expand Up @@ -832,254 +831,6 @@ fn table_ops_eventually_gcs() {
panic!("after {n} runs nothing ever gc'd, something is probably wrong");
}

/// Perform differential execution between Cranelift and wasmi, diffing the
/// resulting memory image when execution terminates. This relies on the
/// module-under-test to be instrumented to bound the execution time. Invoke
/// with a module generated by `wasm-smith` using the
/// `SingleFunctionModuleConfig` configuration type for best results.
///
/// May return `None` if we early-out due to a rejected fuzz config; these
/// should be rare if modules are generated appropriately.
pub fn differential_wasmi_execution(wasm: &[u8], config: &generators::Config) -> Option<()> {
crate::init_fuzzing();
log_wasm(wasm);

// Instantiate wasmi module and instance.
let wasmi_module = wasmi::Module::from_buffer(&wasm[..]).ok()?;
let wasmi_instance =
wasmi::ModuleInstance::new(&wasmi_module, &wasmi::ImportsBuilder::default()).ok()?;
let wasmi_instance = wasmi_instance.assert_no_start();

// If wasmi succeeded then we assert that wasmtime will also succeed.
let (wasmtime_module, mut wasmtime_store) = differential_store(wasm, config);
let wasmtime_module = wasmtime_module?;
let wasmtime_instance = Instance::new(&mut wasmtime_store, &wasmtime_module, &[])
.expect("Wasmtime can instantiate module");

// Introspect wasmtime module to find name of an exported function and of an
// exported memory.
let (func_name, ty) = first_exported_function(&wasmtime_module)?;

let wasmi_main_export = wasmi_instance.export_by_name(func_name).unwrap();
let wasmi_main = wasmi_main_export.as_func().unwrap();
let wasmi_val = wasmi::FuncInstance::invoke(&wasmi_main, &[], &mut wasmi::NopExternals);

let wasmtime_main = wasmtime_instance
.get_func(&mut wasmtime_store, func_name)
.expect("function export is present");
let mut wasmtime_results = vec![Val::I32(0); ty.results().len()];
let wasmtime_val = wasmtime_main
.call(&mut wasmtime_store, &[], &mut wasmtime_results)
.map(|()| wasmtime_results.get(0).cloned());

debug!(
"Successful execution: wasmi returned {:?}, wasmtime returned {:?}",
wasmi_val, wasmtime_val
);

match (&wasmi_val, &wasmtime_val) {
(&Ok(Some(wasmi::RuntimeValue::I32(a))), &Ok(Some(Val::I32(b)))) if a == b => {}
(&Ok(Some(wasmi::RuntimeValue::F32(a))), &Ok(Some(Val::F32(b))))
if f32_equal(a.to_bits(), b) => {}
(&Ok(Some(wasmi::RuntimeValue::I64(a))), &Ok(Some(Val::I64(b)))) if a == b => {}
(&Ok(Some(wasmi::RuntimeValue::F64(a))), &Ok(Some(Val::F64(b))))
if f64_equal(a.to_bits(), b) => {}
(&Ok(None), &Ok(None)) => {}
(&Err(_), &Err(_)) => {}
_ => {
panic!(
"Values do not match: wasmi returned {:?}; wasmtime returned {:?}",
wasmi_val, wasmtime_val
);
}
}

// Compare linear memories if there's an exported linear memory
let memory_name = match first_exported_memory(&wasmtime_module) {
Some(name) => name,
None => return Some(()),
};
let wasmi_mem_export = wasmi_instance.export_by_name(memory_name).unwrap();
let wasmi_mem = wasmi_mem_export.as_memory().unwrap();
let wasmtime_mem = wasmtime_instance
.get_memory(&mut wasmtime_store, memory_name)
.expect("memory export is present");

if wasmi_mem.current_size().0 != wasmtime_mem.size(&wasmtime_store) as usize {
panic!("resulting memories are not the same size");
}

// Wasmi memory may be stored non-contiguously; copy it out to a contiguous chunk.
let mut wasmi_buf: Vec<u8> = vec![0; wasmtime_mem.data_size(&wasmtime_store)];
wasmi_mem
.get_into(0, &mut wasmi_buf[..])
.expect("can access wasmi memory");

let wasmtime_slice = wasmtime_mem.data(&wasmtime_store);

if wasmi_buf.len() >= 64 {
debug!("-> First 64 bytes of wasmi heap: {:?}", &wasmi_buf[0..64]);
debug!(
"-> First 64 bytes of Wasmtime heap: {:?}",
&wasmtime_slice[0..64]
);
}

if &wasmi_buf[..] != &wasmtime_slice[..] {
panic!("memory contents are not equal");
}

Some(())
}

/// Perform differential execution between Wasmtime and the official WebAssembly
/// specification interpreter.
///
/// May return `None` if we early-out due to a rejected fuzz config.
#[cfg(feature = "fuzz-spec-interpreter")]
pub fn differential_spec_execution(wasm: &[u8], config: &generators::Config) -> Option<()> {
use anyhow::Context;

crate::init_fuzzing();
debug!("config: {:#?}", config);
log_wasm(wasm);

// Run the spec interpreter first, then Wasmtime. The order is important
// because both sides (OCaml runtime and Wasmtime) register signal handlers;
// Wasmtime uses these signal handlers for catching various WebAssembly
// failures. On certain OSes (e.g. Linux x86_64), the signal handlers
// interfere, observable as an uncaught `SIGSEGV`--not even caught by
// libFuzzer. By running Wasmtime second, its signal handlers are registered
// most recently and they catch failures appropriately.
//
// For now, execute with dummy (zeroed) function arguments.
let spec_vals = wasm_spec_interpreter::interpret(wasm, None);
debug!("spec interpreter returned: {:?}", &spec_vals);

let (wasmtime_module, mut wasmtime_store) = differential_store(wasm, config);
let wasmtime_module = match wasmtime_module {
Some(m) => m,
None => return None,
};

let wasmtime_vals =
Instance::new(&mut wasmtime_store, &wasmtime_module, &[]).and_then(|wasmtime_instance| {
// Find the first exported function.
let (func_name, ty) = first_exported_function(&wasmtime_module)
.context("Cannot find exported function")?;
let wasmtime_main = wasmtime_instance
.get_func(&mut wasmtime_store, &func_name[..])
.expect("function export is present");

let dummy_params = dummy::dummy_values(ty.params());

// Execute the function and return the values.
let mut results = vec![Val::I32(0); ty.results().len()];
wasmtime_main
.call(&mut wasmtime_store, &dummy_params, &mut results)
.map(|()| Some(results))
});

// Match a spec interpreter value against a Wasmtime value. Eventually this
// should support references and `v128` (TODO).
fn matches(spec_val: &wasm_spec_interpreter::Value, wasmtime_val: &wasmtime::Val) -> bool {
match (spec_val, wasmtime_val) {
(wasm_spec_interpreter::Value::I32(a), wasmtime::Val::I32(b)) => a == b,
(wasm_spec_interpreter::Value::I64(a), wasmtime::Val::I64(b)) => a == b,
(wasm_spec_interpreter::Value::F32(a), wasmtime::Val::F32(b)) => {
f32_equal(*a as u32, *b)
}
(wasm_spec_interpreter::Value::F64(a), wasmtime::Val::F64(b)) => {
f64_equal(*a as u64, *b)
}
(wasm_spec_interpreter::Value::V128(a), wasmtime::Val::V128(b)) => {
assert_eq!(a.len(), 16);
let a_num = u128::from_le_bytes(a.as_slice().try_into().unwrap());
a_num == *b
}
(_, _) => {
unreachable!("TODO: only fuzzing of scalar and vector value types is supported")
}
}
}

match (&spec_vals, &wasmtime_vals) {
// Compare the returned values, failing if they do not match.
(Ok(spec_vals), Ok(Some(wasmtime_vals))) => {
let all_match = spec_vals
.iter()
.zip(wasmtime_vals)
.all(|(s, w)| matches(s, w));
if !all_match {
panic!(
"Values do not match: spec returned {:?}; wasmtime returned {:?}",
spec_vals, wasmtime_vals
);
}
}
(_, Ok(None)) => {
// `run_in_wasmtime` rejected the config
return None;
}
// If both sides fail, skip this fuzz execution.
(Err(spec_error), Err(wasmtime_error)) => {
// The `None` value returned here indicates that both sides
// failed--if we see too many of these we might be failing too often
// to check instruction semantics. At some point it would be
// beneficial to compare the error messages from both sides (TODO).
// It would also be good to keep track of statistics about the
// ratios of the kinds of errors the fuzzer sees (TODO).
log::warn!(
"Both sides failed: spec returned '{}'; wasmtime returned {:?}",
spec_error,
wasmtime_error
);
return None;
}
// If only one side fails, fail the fuzz the test.
_ => {
panic!(
"Only one side failed: spec returned {:?}; wasmtime returned {:?}",
&spec_vals, &wasmtime_vals
);
}
}

// TODO Compare memory contents.

Some(())
}

fn differential_store(
wasm: &[u8],
fuzz_config: &generators::Config,
) -> (Option<Module>, Store<StoreLimits>) {
let store = fuzz_config.to_store();
let module = compile_module(store.engine(), wasm, true, fuzz_config);
(module, store)
}

// Introspect wasmtime module to find the name of the first exported function.
fn first_exported_function(module: &wasmtime::Module) -> Option<(&str, FuncType)> {
for e in module.exports() {
match e.ty() {
wasmtime::ExternType::Func(ty) => return Some((e.name(), ty)),
_ => {}
}
}
None
}

fn first_exported_memory(module: &Module) -> Option<&str> {
for e in module.exports() {
match e.ty() {
wasmtime::ExternType::Memory(..) => return Some(e.name()),
_ => {}
}
}
None
}

#[derive(Default)]
struct SignalOnDrop {
state: Arc<(Mutex<bool>, Condvar)>,
Expand Down
35 changes: 33 additions & 2 deletions crates/fuzzing/src/oracles/v8.rs
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
use super::{first_exported_function, first_exported_memory, log_wasm};
use super::{compile_module, log_wasm};
use crate::generators;
use std::convert::TryFrom;
use std::sync::Once;
use wasmtime::*;
Expand All @@ -16,7 +17,7 @@ use wasmtime::*;
pub fn differential_v8_execution(wasm: &[u8], config: &crate::generators::Config) -> Option<()> {
// Wasmtime setup
log_wasm(wasm);
let (wasmtime_module, mut wasmtime_store) = super::differential_store(wasm, config);
let (wasmtime_module, mut wasmtime_store) = differential_store(wasm, config);
let wasmtime_module = wasmtime_module?;
log::trace!("compiled module with wasmtime");

Expand Down Expand Up @@ -339,3 +340,33 @@ fn assert_error_matches(wasmtime: &anyhow::Error, v8: &str) {
}
verify_wasmtime("not possibly present in an error, just panic please");
}

fn differential_store(
wasm: &[u8],
fuzz_config: &generators::Config,
) -> (Option<Module>, Store<super::StoreLimits>) {
let store = fuzz_config.to_store();
let module = compile_module(store.engine(), wasm, true, fuzz_config);
(module, store)
}

// Introspect wasmtime module to find the name of the first exported function.
fn first_exported_function(module: &wasmtime::Module) -> Option<(&str, FuncType)> {
for e in module.exports() {
match e.ty() {
wasmtime::ExternType::Func(ty) => return Some((e.name(), ty)),
_ => {}
}
}
None
}

fn first_exported_memory(module: &Module) -> Option<&str> {
for e in module.exports() {
match e.ty() {
wasmtime::ExternType::Memory(..) => return Some(e.name()),
_ => {}
}
}
None
}
14 changes: 0 additions & 14 deletions fuzz/Cargo.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -66,20 +66,6 @@ path = "fuzz_targets/differential_meta.rs"
test = false
doc = false


[[bin]]
name = "differential_spec"
path = "fuzz_targets/differential_spec.rs"
test = false
doc = false
required-features = ['fuzz-spec-interpreter']

[[bin]]
name = "differential_wasmi"
path = "fuzz_targets/differential_wasmi.rs"
test = false
doc = false

[[bin]]
name = "differential_v8"
path = "fuzz_targets/differential_v8.rs"
Expand Down
10 changes: 5 additions & 5 deletions fuzz/README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,13 +36,13 @@ At the time of writing, we have the following fuzz targets:
from scratch.
* `differential`: Generate a Wasm module and check that Wasmtime returns
the same results when run with two different configurations.
* `differential_spec`: Generate a Wasm module and check that Wasmtime returns
the same results as the Wasm spec interpreter (see the `wasm-spec-interpreter`
crate).
* `differential_meta`: Generate a Wasm module, evaluate each exported function
with random inputs, and check that Wasmtime returns the same results as a
choice of another engine: the Wasm spec interpreter (see the
`wasm-spec-interpreter` crate), the `wasmi` interpreter, or Wasmtime itself
run with a different configuration.
* `differential_v8`: Generate a Wasm module and check that Wasmtime returns
the same results as V8.
* `differential_wasmi`: Generate a Wasm module and check that Wasmtime returns
the same results as the `wasmi` interpreter.
* `instantiate`: Generate a Wasm module and Wasmtime configuration and attempt
to compile and instantiate with them.
* `instantiate-many`: Generate many Wasm modules and attempt to compile and
Expand Down
Loading

0 comments on commit 8b7fb19

Please sign in to comment.