test: add JS fuzz/invariant integration tests (#658)

Co-authored-by: Franco Victorio <victorio.franco@gmail.com>

crates/edr_napi/index.d.ts

@@ -515,8 +515,8 @@ export interface SolidityTestRunnerConfigArgs {
 }
 /** Fuzz testing configuration */
 export interface FuzzConfigArgs {
-  /** Path where fuzz failures are recorded and replayed. */
-  failurePersistDir: string
+  /** Path where fuzz failures are recorded and replayed if set. */
+  failurePersistDir?: string
   /** Name of the file to record fuzz failures, defaults to `failures`. */
   failurePersistFile?: string
   /**
@@ -560,7 +560,7 @@ export interface FuzzConfigArgs {
 }
 /** Invariant testing configuration. */
 export interface InvariantConfigArgs {
-  /** Path where invariant failures are recorded and replayed. */
+  /** Path where invariant failures are recorded and replayed if set. */
   failurePersistDir?: string
   /**
    * The number of runs that must execute for each invariant test group.

crates/edr_napi/src/solidity_tests/config.rs

@@ -182,7 +182,8 @@ impl TryFrom<SolidityTestRunnerConfigArgs> for SolidityTestRunnerConfig {
 
         let invariant: InvariantConfig = fuzz
             .as_ref()
-            .map(|f| invariant.unwrap_or_default().defaults_from_fuzz(f))
+            .map(|f| invariant.clone().unwrap_or_default().defaults_from_fuzz(f))
+            .or(invariant)
             .map(TryFrom::try_from)
             .transpose()?
             .unwrap_or_default();
@@ -301,8 +302,8 @@ impl TryFrom<SolidityTestRunnerConfigArgs> for SolidityTestRunnerConfig {
 #[napi(object)]
 #[derive(Clone, Default, Debug)]
 pub struct FuzzConfigArgs {
-    /// Path where fuzz failures are recorded and replayed.
-    pub failure_persist_dir: String,
+    /// Path where fuzz failures are recorded and replayed if set.
+    pub failure_persist_dir: Option<String>,
     /// Name of the file to record fuzz failures, defaults to `failures`.
     pub failure_persist_file: Option<String>,
     /// The amount of fuzz runs to perform for each fuzz test case. Higher
@@ -348,8 +349,8 @@ impl TryFrom<FuzzConfigArgs> for FuzzConfig {
             include_push_bytes,
         } = value;
 
-        let failure_persist_dir = Some(failure_persist_dir.into());
-        let failure_persist_file = Some(failure_persist_file.unwrap_or("failures".to_string()));
+        let failure_persist_dir = failure_persist_dir.map(PathBuf::from);
+        let failure_persist_file = failure_persist_file.unwrap_or("failures".to_string());
         let seed = seed
             .map(|s| {
                 s.parse().map_err(|_err| {
@@ -362,6 +363,8 @@ impl TryFrom<FuzzConfigArgs> for FuzzConfig {
             seed,
             failure_persist_dir,
             failure_persist_file,
+            // TODO https://github.com/NomicFoundation/edr/issues/657
+            gas_report_samples: 0,
             ..FuzzConfig::default()
         };
 
@@ -393,7 +396,7 @@ impl TryFrom<FuzzConfigArgs> for FuzzConfig {
 #[napi(object)]
 #[derive(Clone, Default, Debug)]
 pub struct InvariantConfigArgs {
-    /// Path where invariant failures are recorded and replayed.
+    /// Path where invariant failures are recorded and replayed if set.
     pub failure_persist_dir: Option<String>,
     /// The number of runs that must execute for each invariant test group.
     /// Defaults to 256.
@@ -442,7 +445,7 @@ impl InvariantConfigArgs {
         } = fuzz;
 
         if self.failure_persist_dir.is_none() {
-            self.failure_persist_dir = Some(failure_persist_dir.clone());
+            self.failure_persist_dir.clone_from(failure_persist_dir);
         }
 
         if self.runs.is_none() {
@@ -483,6 +486,8 @@ impl From<InvariantConfigArgs> for InvariantConfig {
 
         let mut invariant = InvariantConfig {
             failure_persist_dir,
+            // TODO https://github.com/NomicFoundation/edr/issues/657
+            gas_report_samples: 0,
             ..InvariantConfig::default()
         };
 

crates/foundry/config/src/fuzz.rs

@@ -24,7 +24,7 @@ pub struct FuzzConfig {
     /// Path where fuzz failures are recorded and replayed.
     pub failure_persist_dir: Option<PathBuf>,
     /// Name of the file to record fuzz failures, defaults to `failures`.
-    pub failure_persist_file: Option<String>,
+    pub failure_persist_file: String,
 }
 
 impl Default for FuzzConfig {
@@ -36,7 +36,7 @@ impl Default for FuzzConfig {
             dictionary: FuzzDictionaryConfig::default(),
             gas_report_samples: 0,
             failure_persist_dir: None,
-            failure_persist_file: None,
+            failure_persist_file: "failures".into(),
         }
     }
 }
@@ -46,13 +46,8 @@ impl FuzzConfig {
     /// `{PROJECT_ROOT}/cache/fuzz` dir.
     pub fn new(cache_dir: PathBuf) -> Self {
         FuzzConfig {
-            runs: 256,
-            max_test_rejects: 65536,
-            seed: None,
-            dictionary: FuzzDictionaryConfig::default(),
-            gas_report_samples: 0,
             failure_persist_dir: Some(cache_dir),
-            failure_persist_file: Some("failures".to_string()),
+            ..FuzzConfig::default()
         }
     }
 }

crates/foundry/config/src/invariant.rs

@@ -69,10 +69,16 @@ impl InvariantConfig {
     }
 
     /// Returns path to failure dir of given invariant test contract.
-    pub fn failure_dir(self, contract_name: &str) -> PathBuf {
+    pub fn failure_dir(&self, contract_name: &str) -> Option<PathBuf> {
         self.failure_persist_dir
-            .unwrap()
-            .join("failures")
-            .join(contract_name.split(':').last().unwrap())
+            .as_ref()
+            .map(|failure_persist_dir| {
+                failure_persist_dir.join("failures").join(
+                    contract_name
+                        .split(':')
+                        .last()
+                        .expect("contract name should have solc version suffix"),
+                )
+            })
     }
 }

crates/foundry/forge/src/lib.rs

@@ -1,6 +1,12 @@
 #[macro_use]
 extern crate tracing;
 
+use std::{
+    collections::HashSet,
+    fmt::Debug,
+    sync::{OnceLock, RwLock},
+};
+
 use foundry_config::{FuzzConfig, InvariantConfig};
 use proptest::test_runner::{
     FailurePersistence, FileFailurePersistence, RngAlgorithm, TestRng, TestRunner,
@@ -25,6 +31,8 @@ pub mod result;
 pub use foundry_common::traits::TestFilter;
 pub use foundry_evm::*;
 
+static FAILURE_PATHS: OnceLock<RwLock<HashSet<&'static str>>> = OnceLock::new();
+
 /// Metadata on how to run fuzz/invariant tests
 #[derive(Clone, Debug, Default)]
 pub struct TestOptions {
@@ -38,19 +46,46 @@ impl TestOptions {
     /// Returns a "fuzz" test runner instance.
     pub fn fuzz_runner(&self) -> TestRunner {
         let fuzz_config = self.fuzz_config().clone();
-        let failure_persist_path = fuzz_config
-            .failure_persist_dir
-            .unwrap()
-            .join(fuzz_config.failure_persist_file.unwrap())
-            .into_os_string()
-            .into_string()
-            .unwrap();
-        self.fuzzer_with_cases(
-            fuzz_config.runs,
-            Some(Box::new(FileFailurePersistence::Direct(
-                failure_persist_path.leak(),
-            ))),
-        )
+
+        if let Some(failure_persist_dir) = fuzz_config.failure_persist_dir {
+            let failure_persist_path = failure_persist_dir
+                .join(fuzz_config.failure_persist_file)
+                .into_os_string()
+                .into_string()
+                .expect("path should be valid UTF-8");
+
+            // HACK: We need to leak the path as
+            // `proptest::test_runner::FileFailurePersistence` requires a
+            // `&'static str`. We mitigate this by making sure that one particular path
+            // is only leaked once.
+            let failure_paths = FAILURE_PATHS.get_or_init(RwLock::default);
+            // Need to be in a block to ensure that the read lock is dropped before we try
+            // to insert.
+            {
+                let failure_paths_guard = failure_paths.read().expect("lock is not poisoned");
+                if let Some(static_path) = failure_paths_guard.get(&*failure_persist_path) {
+                    return self.fuzzer_with_cases(
+                        fuzz_config.runs,
+                        Some(Box::new(FileFailurePersistence::Direct(static_path))),
+                    );
+                }
+            }
+            // Write block
+            {
+                let mut failure_paths_guard = failure_paths.write().expect("lock is not poisoned");
+                failure_paths_guard.insert(failure_persist_path.clone().leak());
+                let static_path = failure_paths_guard
+                    .get(&*failure_persist_path)
+                    .expect("must exist since we just inserted it");
+
+                self.fuzzer_with_cases(
+                    fuzz_config.runs,
+                    Some(Box::new(FileFailurePersistence::Direct(static_path))),
+                )
+            }
+        } else {
+            self.fuzzer_with_cases(fuzz_config.runs, None)
+        }
     }
 
     /// Returns an "invariant" test runner instance.