VM Fuzzer (#2401)

This Pull Request offers a basic VM fuzzer which relies on implied oracles (namely, "does it crash or timeout?").

It changes the following:

- Adds an insns_remaining field to Context, denoting the number of instructions remaining to execute (only available when fuzzing)
- Adds a JsNativeError variant, denoting when the number of instructions has been exceeded (only available when fuzzing)
- Adds a VM fuzzer which looks for cases where Boa may crash on an input

This offers no guarantees about correctness, only assertion violations. Depends on #2400.

Any issues I raise in association with this fuzzer will link back to this fuzzer.

You may run the fuzzer using the following commands:
```bash
$ cd boa_engine
$ cargo +nightly fuzz run -s none vm-implied
```

Co-authored-by: Addison Crump <addison.crump@cispa.de>

boa_engine/src/context/mod.rs

@@ -97,6 +97,10 @@ pub struct Context {
     #[cfg(feature = "intl")]
     icu: icu::Icu,
 
+    /// Number of instructions remaining before a forced exit
+    #[cfg(feature = "fuzz")]
+    pub(crate) instructions_remaining: usize,
+
     pub(crate) vm: Vm,
 
     pub(crate) promise_job_queue: VecDeque<JobCallback>,
@@ -593,6 +597,8 @@ pub struct ContextBuilder {
     interner: Option<Interner>,
     #[cfg(feature = "intl")]
     icu: Option<icu::Icu>,
+    #[cfg(feature = "fuzz")]
+    instructions_remaining: usize,
 }
 
 impl ContextBuilder {
@@ -615,6 +621,15 @@ impl ContextBuilder {
         Ok(self)
     }
 
+    /// Specifies the number of instructions remaining to the [`Context`].
+    ///
+    /// This function is only available if the `fuzz` feature is enabled.
+    #[cfg(feature = "fuzz")]
+    pub fn instructions_remaining(mut self, instructions_remaining: usize) -> Self {
+        self.instructions_remaining = instructions_remaining;
+        self
+    }
+
     /// Creates a new [`ContextBuilder`] with a default empty [`Interner`]
     /// and a default [`BoaProvider`] if the `intl` feature is enabled.
     pub fn new() -> Self {
@@ -643,6 +658,8 @@ impl ContextBuilder {
                 icu::Icu::new(Box::new(icu_testdata::get_provider()))
                     .expect("Failed to initialize default icu data.")
             }),
+            #[cfg(feature = "fuzz")]
+            instructions_remaining: self.instructions_remaining,
             promise_job_queue: VecDeque::new(),
         };
 

boa_engine/src/error.rs

@@ -503,6 +503,17 @@ impl JsNativeError {
         Self::new(JsNativeErrorKind::Uri, Box::default(), None)
     }
 
+    /// Creates a new `JsNativeError` that indicates that the context hit its execution limit. This
+    /// is only used in a fuzzing context.
+    #[cfg(feature = "fuzz")]
+    pub fn no_instructions_remain() -> Self {
+        Self::new(
+            JsNativeErrorKind::NoInstructionsRemain,
+            Box::default(),
+            None,
+        )
+    }
+
     /// Sets the message of this error.
     ///
     /// # Examples
@@ -619,6 +630,12 @@ impl JsNativeError {
             }
             JsNativeErrorKind::Type => (constructors.type_error().prototype(), ErrorKind::Type),
             JsNativeErrorKind::Uri => (constructors.uri_error().prototype(), ErrorKind::Uri),
+            #[cfg(feature = "fuzz")]
+            JsNativeErrorKind::NoInstructionsRemain => {
+                unreachable!(
+                    "The NoInstructionsRemain native error cannot be converted to an opaque type."
+                )
+            }
         };
 
         let o = JsObject::from_proto_and_data(prototype, ObjectData::error(tag));
@@ -747,6 +764,10 @@ pub enum JsNativeErrorKind {
     /// [e_uri]: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/encodeURI
     /// [d_uri]: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/decodeURI
     Uri,
+    /// Error thrown when no instructions remain. Only used in a fuzzing context; not a valid JS
+    /// error variant.
+    #[cfg(feature = "fuzz")]
+    NoInstructionsRemain,
 }
 
 impl std::fmt::Display for JsNativeErrorKind {
@@ -760,6 +781,8 @@ impl std::fmt::Display for JsNativeErrorKind {
             JsNativeErrorKind::Syntax => "SyntaxError",
             JsNativeErrorKind::Type => "TypeError",
             JsNativeErrorKind::Uri => "UriError",
+            #[cfg(feature = "fuzz")]
+            JsNativeErrorKind::NoInstructionsRemain => "NoInstructionsRemain",
         }
         .fmt(f)
     }

boa_engine/src/vm/mod.rs

@@ -7,6 +7,8 @@ use crate::{
     vm::{call_frame::CatchAddresses, code_block::Readable},
     Context, JsResult, JsValue,
 };
+#[cfg(feature = "fuzz")]
+use crate::{JsError, JsNativeError};
 use boa_interner::ToInternedString;
 use boa_profiler::Profiler;
 use std::{convert::TryInto, mem::size_of, time::Instant};
@@ -179,6 +181,13 @@ impl Context {
             });
 
         while self.vm.frame().pc < self.vm.frame().code.code.len() {
+            #[cfg(feature = "fuzz")]
+            if self.instructions_remaining == 0 {
+                return Err(JsError::from_native(JsNativeError::no_instructions_remain()));
+            } else {
+                self.instructions_remaining -= 1;
+            }
+
             let result = if self.vm.trace {
                 let mut pc = self.vm.frame().pc;
                 let opcode: Opcode = self

fuzz/Cargo.toml

@@ -27,3 +27,15 @@ name = "parser-idempotency"
 path = "fuzz_targets/parser-idempotency.rs"
 test = false
 doc = false
+
+[[bin]]
+name = "vm-implied"
+path = "fuzz_targets/vm-implied.rs"
+test = false
+doc = false
+
+[[bin]]
+name = "bytecompiler-implied"
+path = "fuzz_targets/bytecompiler-implied.rs"
+test = false
+doc = false

fuzz/README.md

@@ -35,3 +35,22 @@ following:
      information, as the inputs parsed between the two should be the same.
 
 In this way, this fuzzer can identify correctness issues present in the parser.
+
+## Bytecompiler Fuzzer
+
+The bytecompiler fuzzer, located in [bytecompiler-implied.rs](fuzz_targets/bytecompiler-implied.rs), identifies cases
+which cause an assertion failure in the bytecompiler. These crashes can cause denial of service issues and may block the
+discovery of crash cases in the VM fuzzer.
+
+## VM Fuzzer
+
+The VM fuzzer, located in [vm-implied.rs](fuzz_targets/vm-implied.rs), identifies crash cases in the VM. It does so by
+generating an arbitrary AST, converting it to source code (to remove invalid inputs), then executing that source code.
+Because we are not comparing against any invariants other than "does it crash", this fuzzer will only discover faults
+which cause the VM to terminate unexpectedly, e.g. as a result of a panic. It will not discover logic errors present in
+the VM.
+
+To ensure that the VM does not attempt to execute an infinite loop, Boa is restricted to a finite number of instructions
+before the VM is terminated. If a program takes more than a second or so to execute, it likely indicates an issue in the
+VM (as we expect the fuzzer to execute only a certain amount of instructions, which should take significantly less
+time).

fuzz/fuzz_targets/bytecompiler-implied.rs

@@ -0,0 +1,25 @@
+#![no_main]
+
+mod common;
+
+use crate::common::FuzzSource;
+use boa_engine::Context;
+use boa_parser::Parser;
+use libfuzzer_sys::{fuzz_target, Corpus};
+use std::io::Cursor;
+
+fn do_fuzz(original: FuzzSource) -> Corpus {
+    let mut ctx = Context::builder()
+        .interner(original.interner)
+        .instructions_remaining(0)
+        .build();
+    let mut parser = Parser::new(Cursor::new(&original.source));
+    if let Ok(parsed) = parser.parse_all(ctx.interner_mut()) {
+        let _ = ctx.compile(&parsed);
+        Corpus::Keep
+    } else {
+        Corpus::Reject
+    }
+}
+
+fuzz_target!(|original: FuzzSource| -> Corpus { do_fuzz(original) });

fuzz/fuzz_targets/common.rs

@@ -2,7 +2,7 @@ use boa_ast::{
     visitor::{VisitWith, VisitorMut},
     Expression, StatementList,
 };
-use boa_interner::{Interner, Sym};
+use boa_interner::{Interner, Sym, ToInternedString};
 use libfuzzer_sys::arbitrary;
 use libfuzzer_sys::arbitrary::{Arbitrary, Unstructured};
 use std::fmt::{Debug, Formatter};
@@ -72,3 +72,25 @@ impl Debug for FuzzData {
             .finish_non_exhaustive()
     }
 }
+
+pub struct FuzzSource {
+    pub interner: Interner,
+    pub source: String,
+}
+
+impl<'a> Arbitrary<'a> for FuzzSource {
+    fn arbitrary(u: &mut Unstructured<'a>) -> arbitrary::Result<Self> {
+        let data = FuzzData::arbitrary(u)?;
+        let source = data.ast.to_interned_string(&data.interner);
+        Ok(Self {
+            interner: data.interner,
+            source,
+        })
+    }
+}
+
+impl Debug for FuzzSource {
+    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+        f.write_fmt(format_args!("Fuzzed source:\n{}", self.source))
+    }
+}

fuzz/fuzz_targets/vm-implied.rs

@@ -0,0 +1,19 @@
+#![no_main]
+
+mod common;
+
+use crate::common::FuzzSource;
+use boa_engine::{Context, JsResult, JsValue};
+use libfuzzer_sys::fuzz_target;
+
+fn do_fuzz(original: FuzzSource) -> JsResult<JsValue> {
+    let mut ctx = Context::builder()
+        .interner(original.interner)
+        .instructions_remaining(1 << 16)
+        .build();
+    ctx.eval(&original.source)
+}
+
+fuzz_target!(|original: FuzzSource| {
+    let _ = do_fuzz(original);
+});