diff --git a/Cargo.lock b/Cargo.lock index 3135dc300bebd..6c020c9f76255 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4864,11 +4864,13 @@ dependencies = [ "indicatif", "parking_lot", "proptest", + "rayon", "revm", "revm-inspectors", "serde", "serde_json", "thiserror 2.0.17", + "tokio", "tracing", "uuid 1.19.0", ] diff --git a/crates/config/src/fuzz.rs b/crates/config/src/fuzz.rs index 9f2ba141897a8..cb964f97f68be 100644 --- a/crates/config/src/fuzz.rs +++ b/crates/config/src/fuzz.rs @@ -12,11 +12,8 @@ pub struct FuzzConfig { pub runs: u32, /// Fails the fuzzed test if a revert occurs. pub fail_on_revert: bool, - /// The maximum number of test case rejections allowed by proptest, to be - /// encountered during usage of `vm.assume` cheatcode. This will be used - /// to set the `max_global_rejects` value in proptest test runner config. - /// `max_local_rejects` option isn't exposed here since we're not using - /// `prop_filter`. + /// The maximum number of test case rejections allowed, + /// encountered during usage of `vm.assume` cheatcode. pub max_test_rejects: u32, /// Optional seed for the fuzzing RNG algorithm pub seed: Option, diff --git a/crates/evm/evm/Cargo.toml b/crates/evm/evm/Cargo.toml index b3e9553f40304..a69c19cbdb2bd 100644 --- a/crates/evm/evm/Cargo.toml +++ b/crates/evm/evm/Cargo.toml @@ -56,3 +56,5 @@ indicatif.workspace = true serde_json.workspace = true serde.workspace = true uuid.workspace = true +rayon.workspace = true +tokio.workspace = true diff --git a/crates/evm/evm/src/executors/corpus.rs b/crates/evm/evm/src/executors/corpus.rs index eb5431e3ba57a..b579db628361d 100644 --- a/crates/evm/evm/src/executors/corpus.rs +++ b/crates/evm/evm/src/executors/corpus.rs @@ -1,8 +1,44 @@ +//! Corpus management for parallel fuzzing with coverage-guided mutation. +//! +//! This module implements a corpus-based fuzzing system that stores, mutates, and shares +//! transaction sequences across multiple fuzzing workers. Each corpus entry represents a +//! sequence of transactions that has produced interesting coverage, and can be mutated to +//! discover new execution paths. +//! +//! ## File System Structure +//! +//! The corpus is organized on disk as follows: +//! +//! ```text +//! / +//! ├── worker0/ # Master (worker 0) directory +//! │ ├── corpus/ # Master's corpus entries +//! │ │ ├── -.json # Corpus entry (if small) +//! │ │ ├── -.json.gz # Corpus entry (if large, compressed) +//! │ └── sync/ # Directory where other workers export new findings +//! │ └── -.json # New entries from other workers +//! └── workerN/ # Worker N's directory +//! ├── corpus/ # Worker N's local corpus +//! │ └── ... +//! └── sync/ # Worker 2's sync directory +//! └── ... +//! ``` +//! +//! ## Workflow +//! +//! - Each worker maintains its own local corpus with entries stored as JSON files +//! - Workers export new interesting entries to the master's sync directory via hard links +//! - The master (worker0) imports new entries from its sync directory and exports them to all the +//! other workers +//! - Workers sync with the master to receive new corpus entries from other workers +//! - This all happens periodically, there is no clear order in which workers export or import +//! entries since it doesn't matter as long as the corpus eventually syncs across all workers + use crate::executors::{Executor, RawCallResult, invariant::execute_tx}; use alloy_dyn_abi::JsonAbiExt; use alloy_json_abi::Function; use alloy_primitives::Bytes; -use eyre::eyre; +use eyre::{Result, eyre}; use foundry_config::FuzzCorpusConfig; use foundry_evm_fuzz::{ BasicTxDetails, @@ -18,16 +54,26 @@ use proptest::{ use serde::Serialize; use std::{ fmt, - path::PathBuf, + path::{Path, PathBuf}, + sync::{ + Arc, + atomic::{AtomicUsize, Ordering}, + }, time::{SystemTime, UNIX_EPOCH}, }; use uuid::Uuid; -const METADATA_SUFFIX: &str = "metadata.json"; -const JSON_EXTENSION: &str = ".json"; +const WORKER: &str = "worker"; +const CORPUS_DIR: &str = "corpus"; +const SYNC_DIR: &str = "sync"; + const FAVORABILITY_THRESHOLD: f64 = 0.3; const COVERAGE_MAP_SIZE: usize = 65536; +/// Threshold for compressing corpus entries. +/// 4KiB is usually the minimum file size on popular file systems. +const GZIP_THRESHOLD: usize = 4 * 1024; + /// Possible mutation strategies to apply on a call sequence. #[derive(Debug, Clone)] enum MutationType { @@ -46,7 +92,7 @@ enum MutationType { } /// Holds Corpus information. -#[derive(Serialize)] +#[derive(Clone, Serialize)] struct CorpusEntry { // Unique corpus identifier. uuid: Uuid, @@ -60,32 +106,96 @@ struct CorpusEntry { // Whether this corpus is favored, i.e. producing new finds more often than // `FAVORABILITY_THRESHOLD`. is_favored: bool, + /// Timestamp of when this entry was written to disk in seconds. + #[serde(skip_serializing)] + timestamp: u64, } impl CorpusEntry { - /// New corpus from given call sequence and corpus path to read uuid. - pub fn new(tx_seq: Vec, path: PathBuf) -> eyre::Result { - let uuid = if let Some(stem) = path.file_stem().and_then(|s| s.to_str()) { - Uuid::try_from(stem.strip_suffix(JSON_EXTENSION).unwrap_or(stem).to_string())? - } else { - Uuid::new_v4() + /// Creates a corpus entry with a new UUID. + pub fn new(tx_seq: Vec) -> Self { + Self::new_with(tx_seq, Uuid::new_v4()) + } + + /// Creates a corpus entry with a path. + /// The UUID is parsed from the file name, otherwise a new UUID is generated. + pub fn new_existing(tx_seq: Vec, path: PathBuf) -> Result { + let Some(name) = path.file_name().and_then(|s| s.to_str()) else { + eyre::bail!("invalid corpus file path: {path:?}"); }; - Ok(Self { uuid, total_mutations: 0, new_finds_produced: 0, tx_seq, is_favored: false }) + let uuid = parse_corpus_filename(name)?.0; + Ok(Self::new_with(tx_seq, uuid)) } - /// New corpus with given call sequence and new uuid. - pub fn from_tx_seq(tx_seq: &[BasicTxDetails]) -> Self { + /// Creates a corpus entry with the given UUID. + pub fn new_with(tx_seq: Vec, uuid: Uuid) -> Self { Self { - uuid: Uuid::new_v4(), + uuid, total_mutations: 0, new_finds_produced: 0, - tx_seq: tx_seq.into(), + tx_seq, is_favored: false, + timestamp: SystemTime::now() + .duration_since(UNIX_EPOCH) + .expect("time went backwards") + .as_secs(), + } + } + + fn write_to_disk_in(&self, dir: &Path, can_gzip: bool) -> foundry_common::fs::Result<()> { + let file_name = self.file_name(can_gzip); + let path = dir.join(file_name); + if self.should_gzip(can_gzip) { + foundry_common::fs::write_json_gzip_file(&path, &self.tx_seq) + } else { + foundry_common::fs::write_json_file(&path, &self.tx_seq) + } + } + + fn file_name(&self, can_gzip: bool) -> String { + let ext = if self.should_gzip(can_gzip) { ".json.gz" } else { ".json" }; + format!("{}-{}{ext}", self.uuid, self.timestamp) + } + + fn should_gzip(&self, can_gzip: bool) -> bool { + if !can_gzip { + return false; + } + let size: usize = self.tx_seq.iter().map(|tx| tx.estimate_serialized_size()).sum(); + size > GZIP_THRESHOLD + } +} + +#[derive(Default)] +pub(crate) struct GlobalCorpusMetrics { + // Number of edges seen during the invariant run. + cumulative_edges_seen: AtomicUsize, + // Number of features (new hitcount bin of previously hit edge) seen during the invariant run. + cumulative_features_seen: AtomicUsize, + // Number of corpus entries. + corpus_count: AtomicUsize, + // Number of corpus entries that are favored. + favored_items: AtomicUsize, +} + +impl fmt::Display for GlobalCorpusMetrics { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + self.load().fmt(f) + } +} + +impl GlobalCorpusMetrics { + pub(crate) fn load(&self) -> CorpusMetrics { + CorpusMetrics { + cumulative_edges_seen: self.cumulative_edges_seen.load(Ordering::Relaxed), + cumulative_features_seen: self.cumulative_features_seen.load(Ordering::Relaxed), + corpus_count: self.corpus_count.load(Ordering::Relaxed), + favored_items: self.favored_items.load(Ordering::Relaxed), } } } -#[derive(Serialize, Default)] +#[derive(Serialize, Default, Clone)] pub(crate) struct CorpusMetrics { // Number of edges seen during the invariant run. cumulative_edges_seen: usize, @@ -128,35 +238,48 @@ impl CorpusMetrics { } } -/// Fuzz corpus manager, used in coverage guided fuzzing mode by both stateless and stateful tests. -pub(crate) struct CorpusManager { - // Fuzzed calls generator. - tx_generator: BoxedStrategy, - // Call sequence mutation strategy type generator. - mutation_generator: BoxedStrategy, - // Corpus configuration. - config: FuzzCorpusConfig, - // In-memory corpus, populated from persisted files and current runs. - // Mutation is performed on these. +/// Per-worker corpus manager. +pub struct WorkerCorpus { + /// Worker Id + id: usize, + /// In-memory corpus entries populated from the persisted files and + /// runs administered by this worker. in_memory_corpus: Vec, - // Identifier of current mutated entry. - current_mutated: Option, - // Number of failed replays from persisted corpus. - failed_replays: usize, - // History of binned hitcount of edges seen during fuzzing. + /// History of binned hitcount of edges seen during fuzzing history_map: Vec, - // Corpus metrics. + /// Number of failed replays from initial corpus + pub(crate) failed_replays: usize, + /// Worker Metrics pub(crate) metrics: CorpusMetrics, + /// Fuzzed calls generator. + tx_generator: BoxedStrategy, + /// Call sequence mutation strategy type generator used by stateful fuzzing. + mutation_generator: BoxedStrategy, + /// Identifier of current mutated entry for this worker. + current_mutated: Option, + /// Config + config: Arc, + /// Indices of new entries added to [`WorkerCorpus::in_memory_corpus`] since last sync. + new_entry_indices: Vec, + /// Last sync timestamp in seconds. + last_sync_timestamp: u64, + /// Worker Dir + /// corpus_dir/worker1/ + worker_dir: Option, + /// Metrics at last sync - used to calculate deltas while syncing with global metrics + last_sync_metrics: CorpusMetrics, } -impl CorpusManager { +impl WorkerCorpus { pub fn new( + id: usize, config: FuzzCorpusConfig, tx_generator: BoxedStrategy, - executor: &Executor, + // Only required by master worker (id = 0) to replay existing corpus. + executor: Option<&Executor>, fuzzed_function: Option<&Function>, fuzzed_contracts: Option<&FuzzRunIdentifiedContracts>, - ) -> eyre::Result { + ) -> Result { let mutation_generator = prop_oneof![ Just(MutationType::Splice), Just(MutationType::Repeat), @@ -166,65 +289,39 @@ impl CorpusManager { Just(MutationType::Abi), ] .boxed(); - let mut history_map = vec![0u8; COVERAGE_MAP_SIZE]; - let mut metrics = CorpusMetrics::default(); - let mut in_memory_corpus = vec![]; - let mut failed_replays = 0; - - // Early return if corpus dir / coverage guided fuzzing not configured. - let Some(corpus_dir) = &config.corpus_dir else { - return Ok(Self { - tx_generator, - mutation_generator, - config, - in_memory_corpus, - current_mutated: None, - failed_replays, - history_map, - metrics, - }); - }; - - // Ensure corpus dir for current test is created. - if !corpus_dir.is_dir() { - foundry_common::fs::create_dir_all(corpus_dir)?; - } - let can_replay_tx = |tx: &BasicTxDetails| -> bool { - fuzzed_contracts.is_some_and(|contracts| contracts.targets.lock().can_replay(tx)) - || fuzzed_function.is_some_and(|function| { - tx.call_details - .calldata - .get(..4) - .is_some_and(|selector| function.selector() == selector) - }) - }; + let worker_dir = config.corpus_dir.as_ref().map(|corpus_dir| { + let worker_dir = corpus_dir.join(format!("{WORKER}{id}")); + let worker_corpus = worker_dir.join(CORPUS_DIR); + let sync_dir = worker_dir.join(SYNC_DIR); - 'corpus_replay: for entry in std::fs::read_dir(corpus_dir)? { - let path = entry?.path(); - if path.is_file() - && let Some(name) = path.file_name().and_then(|s| s.to_str()) - && name.contains(METADATA_SUFFIX) - { - // Ignore metadata files - continue; - } + // Create the necessary directories for the worker. + let _ = foundry_common::fs::create_dir_all(&worker_corpus); + let _ = foundry_common::fs::create_dir_all(&sync_dir); - let read_corpus_result = match path.extension().and_then(|ext| ext.to_str()) { - Some("gz") => foundry_common::fs::read_json_gzip_file::>(&path), - _ => foundry_common::fs::read_json_file::>(&path), - }; + worker_dir + }); - let Ok(tx_seq) = read_corpus_result else { - trace!(target: "corpus", "failed to load corpus from {}", path.display()); - continue; - }; + let mut in_memory_corpus = vec![]; + let mut history_map = vec![0u8; COVERAGE_MAP_SIZE]; + let mut metrics = CorpusMetrics::default(); + let mut failed_replays = 0; - if !tx_seq.is_empty() { + if id == 0 + && let Some(corpus_dir) = &config.corpus_dir + { + // Master worker loads the initial corpus, if it exists. + // Then, [distribute]s it to workers. + let executor = executor.expect("Executor required for master worker"); + 'corpus_replay: for entry in read_corpus_dir(corpus_dir) { + let tx_seq = entry.read_tx_seq()?; + if tx_seq.is_empty() { + continue; + } // Warm up history map from loaded sequences. let mut executor = executor.clone(); for tx in &tx_seq { - if can_replay_tx(tx) { + if Self::can_replay_tx(tx, fuzzed_function, fuzzed_contracts) { let mut call_result = execute_tx(&mut executor, tx)?; let (new_coverage, is_edge) = call_result.merge_edge_coverage(&mut history_map); @@ -249,43 +346,49 @@ impl CorpusManager { metrics.corpus_count += 1; - trace!( + debug!( target: "corpus", "load sequence with len {} from corpus file {}", tx_seq.len(), - path.display() + entry.path.display() ); // Populate in memory corpus with the sequence from corpus file. - in_memory_corpus.push(CorpusEntry::new(tx_seq, path)?); + in_memory_corpus.push(CorpusEntry::new_with(tx_seq, entry.uuid)); } } Ok(Self { - tx_generator, - mutation_generator, - config, + id, in_memory_corpus, - current_mutated: None, - failed_replays, history_map, + failed_replays, metrics, + tx_generator, + mutation_generator, + current_mutated: None, + config: config.into(), + new_entry_indices: Default::default(), + last_sync_timestamp: 0, + worker_dir, + last_sync_metrics: Default::default(), }) } /// Updates stats for the given call sequence, if new coverage produced. /// Persists the call sequence (if corpus directory is configured and new coverage) and updates /// in-memory corpus. + #[instrument(skip_all)] pub fn process_inputs(&mut self, inputs: &[BasicTxDetails], new_coverage: bool) { - // Early return if corpus dir / coverage guided fuzzing is not configured. - let Some(corpus_dir) = &self.config.corpus_dir else { + let Some(worker_corpus) = &self.worker_dir else { return; }; + let worker_corpus = worker_corpus.join(CORPUS_DIR); // Update stats of current mutated primary corpus. if let Some(uuid) = &self.current_mutated { if let Some(corpus) = - self.in_memory_corpus.iter_mut().find(|corpus| corpus.uuid.eq(uuid)) + self.in_memory_corpus.iter_mut().find(|corpus| corpus.uuid == *uuid) { corpus.total_mutations += 1; if new_coverage { @@ -311,46 +414,54 @@ impl CorpusManager { return; } - let corpus = CorpusEntry::from_tx_seq(inputs); - let corpus_uuid = corpus.uuid; - - // Persist to disk if corpus dir is configured. - let write_result = if self.config.corpus_gzip { - foundry_common::fs::write_json_gzip_file( - corpus_dir.join(format!("{corpus_uuid}{JSON_EXTENSION}.gz")).as_path(), - &corpus.tx_seq, - ) - } else { - foundry_common::fs::write_json_file( - corpus_dir.join(format!("{corpus_uuid}{JSON_EXTENSION}")).as_path(), - &corpus.tx_seq, - ) - }; + assert!(!inputs.is_empty()); + let corpus = CorpusEntry::new(inputs.to_vec()); + // Persist to disk. + let write_result = corpus.write_to_disk_in(&worker_corpus, self.config.corpus_gzip); if let Err(err) = write_result { - debug!(target: "corpus", %err, "Failed to record call sequence {:?}", &corpus.tx_seq); + debug!(target: "corpus", %err, "failed to record call sequence {:?}", corpus.tx_seq); } else { trace!( target: "corpus", - "persisted {} inputs for new coverage in {corpus_uuid} corpus", - &corpus.tx_seq.len() + "persisted {} inputs for new coverage for {} corpus", + corpus.tx_seq.len(), + corpus.uuid, ); } + // Track in-memory corpus changes to update MasterWorker on sync. + let new_index = self.in_memory_corpus.len(); + self.new_entry_indices.push(new_index); + // This includes reverting txs in the corpus and `can_continue` removes // them. We want this as it is new coverage and may help reach the other branch. self.metrics.corpus_count += 1; self.in_memory_corpus.push(corpus); } + /// Collects coverage from call result and updates metrics. + pub fn merge_edge_coverage(&mut self, call_result: &mut RawCallResult) -> bool { + if !self.config.collect_edge_coverage() { + return false; + } + + let (new_coverage, is_edge) = call_result.merge_edge_coverage(&mut self.history_map); + if new_coverage { + self.metrics.update_seen(is_edge); + } + new_coverage + } + /// Generates new call sequence from in memory corpus. Evicts oldest corpus mutated more than /// configured max mutations value. Used by invariant test campaigns. + #[instrument(skip_all)] pub fn new_inputs( &mut self, test_runner: &mut TestRunner, fuzz_state: &EvmFuzzState, targeted_contracts: &FuzzRunIdentifiedContracts, - ) -> eyre::Result> { + ) -> Result> { let mut new_seq = vec![]; // Early return with first_input only if corpus dir / coverage guided fuzzing not @@ -368,6 +479,7 @@ impl CorpusManager { .new_tree(test_runner) .map_err(|err| eyre!("Could not generate mutation type {err}"))? .current(); + let rng = test_runner.rng(); let corpus_len = self.in_memory_corpus.len(); let primary = &self.in_memory_corpus[rng.random_range(0..corpus_len)]; @@ -411,7 +523,7 @@ impl CorpusManager { self.current_mutated = Some(primary.uuid); for (tx1, tx2) in primary.tx_seq.iter().zip(secondary.tx_seq.iter()) { - // chunks? + // TODO: chunks? let tx = if rng.random::() { tx1.clone() } else { tx2.clone() }; new_seq.push(tx); } @@ -451,8 +563,8 @@ impl CorpusManager { let idx = rng.random_range(0..new_seq.len()); let tx = new_seq.get_mut(idx).unwrap(); if let (_, Some(function)) = targets.fuzzed_artifacts(tx) { - // TODO add call_value to call details and mutate it as well as sender some - // of the time + // TODO: add call_value to call details and mutate it as well as sender some + // of the time. if !function.inputs.is_empty() { self.abi_mutate(tx, function, test_runner, fuzz_state)?; } @@ -470,27 +582,27 @@ impl CorpusManager { Ok(new_seq) } - /// Generates new input from in memory corpus. Evicts oldest corpus mutated more than - /// configured max mutations value. Used by fuzz test campaigns. + /// Generates a new input from the shared in memory corpus. Evicts oldest corpus mutated more + /// than configured max mutations value. Used by fuzz (stateless) test campaigns. + #[instrument(skip_all)] pub fn new_input( &mut self, test_runner: &mut TestRunner, fuzz_state: &EvmFuzzState, function: &Function, - ) -> eyre::Result { + ) -> Result { // Early return if not running with coverage guided fuzzing. if !self.config.is_coverage_guided() { return Ok(self.new_tx(test_runner)?.call_details.calldata); } - let tx = if !self.in_memory_corpus.is_empty() { - self.evict_oldest_corpus()?; + self.evict_oldest_corpus()?; + let tx = if !self.in_memory_corpus.is_empty() { let corpus = &self.in_memory_corpus [test_runner.rng().random_range(0..self.in_memory_corpus.len())]; self.current_mutated = Some(corpus.uuid); - let new_seq = corpus.tx_seq.clone(); - let mut tx = new_seq.first().unwrap().clone(); + let mut tx = corpus.tx_seq.first().unwrap().clone(); self.abi_mutate(&mut tx, function, test_runner, fuzz_state)?; tx } else { @@ -500,6 +612,15 @@ impl CorpusManager { Ok(tx.call_details.calldata) } + /// Generates single call from corpus strategy. + pub fn new_tx(&self, test_runner: &mut TestRunner) -> Result { + Ok(self + .tx_generator + .new_tree(test_runner) + .map_err(|_| eyre!("Could not generate case"))? + .current()) + } + /// Returns the next call to be used in call sequence. /// If coverage guided fuzzing is not configured or if previous input was discarded then this is /// a new tx from strategy. @@ -512,7 +633,7 @@ impl CorpusManager { sequence: &[BasicTxDetails], discarded: bool, depth: usize, - ) -> eyre::Result { + ) -> Result { // Early return with new input if corpus dir / coverage guided fuzzing not configured or if // call was discarded. if self.config.corpus_dir.is_none() || discarded { @@ -525,64 +646,34 @@ impl CorpusManager { return self.new_tx(test_runner); } - // Continue with the next call initial sequence + // Continue with the next call initial sequence. Ok(sequence[depth].clone()) } - /// Generates single call from corpus strategy. - pub fn new_tx(&mut self, test_runner: &mut TestRunner) -> eyre::Result { - Ok(self - .tx_generator - .new_tree(test_runner) - .map_err(|_| eyre!("Could not generate case"))? - .current()) - } - - /// Returns campaign failed replays. - pub fn failed_replays(self) -> usize { - self.failed_replays - } - - /// Collects coverage from call result and updates metrics. - pub fn merge_edge_coverage(&mut self, call_result: &mut RawCallResult) -> bool { - if !self.config.collect_edge_coverage() { - return false; - } - - let (new_coverage, is_edge) = call_result.merge_edge_coverage(&mut self.history_map); - if new_coverage { - self.metrics.update_seen(is_edge); - } - new_coverage - } - /// Flush the oldest corpus mutated more than configured max mutations unless they are /// favored. - fn evict_oldest_corpus(&mut self) -> eyre::Result<()> { + fn evict_oldest_corpus(&mut self) -> Result<()> { if self.in_memory_corpus.len() > self.config.corpus_min_size.max(1) && let Some(index) = self.in_memory_corpus.iter().position(|corpus| { corpus.total_mutations > self.config.corpus_min_mutations && !corpus.is_favored }) { - let corpus = self.in_memory_corpus.get(index).unwrap(); - - let uuid = corpus.uuid; - debug!(target: "corpus", "evict corpus {uuid}"); - - // Flush to disk the seed metadata at the time of eviction. - let eviction_time = SystemTime::now().duration_since(UNIX_EPOCH)?.as_secs(); - foundry_common::fs::write_json_file( - self.config - .corpus_dir - .clone() - .unwrap() - .join(format!("{uuid}-{eviction_time}-{METADATA_SUFFIX}")) - .as_path(), - &corpus, - )?; + let corpus = &self.in_memory_corpus[index]; + + trace!(target: "corpus", corpus=%serde_json::to_string(&corpus).unwrap(), "evict corpus"); // Remove corpus from memory. self.in_memory_corpus.remove(index); + + // Adjust the tracked indices. + self.new_entry_indices.retain_mut(|i| { + if *i > index { + *i -= 1; // Shift indices down. + true // Keep this index. + } else { + *i != index // Remove if it's the deleted index, keep otherwise. + } + }); } Ok(()) } @@ -595,7 +686,7 @@ impl CorpusManager { function: &Function, test_runner: &mut TestRunner, fuzz_state: &EvmFuzzState, - ) -> eyre::Result<()> { + ) -> Result<()> { // let rng = test_runner.rng(); let mut arg_mutation_rounds = test_runner.rng().random_range(0..=function.inputs.len()).max(1); @@ -630,6 +721,373 @@ impl CorpusManager { function.abi_encode_input(&prev_inputs).map_err(|e| eyre!(e.to_string()))?.into(); Ok(()) } + + // Sync Methods. + + /// Imports the new corpus entries from the `sync` directory. + /// These contain tx sequences which are replayed and used to update the history map. + fn load_sync_corpus(&self) -> Result)>> { + let Some(worker_dir) = &self.worker_dir else { + return Ok(vec![]); + }; + + let sync_dir = worker_dir.join(SYNC_DIR); + if !sync_dir.is_dir() { + return Ok(vec![]); + } + + let mut imports = vec![]; + for entry in read_corpus_dir(&sync_dir) { + if entry.timestamp <= self.last_sync_timestamp { + continue; + } + let tx_seq = entry.read_tx_seq()?; + if tx_seq.is_empty() { + warn!(target: "corpus", "skipping empty corpus entry: {}", entry.path.display()); + continue; + } + imports.push((entry, tx_seq)); + } + + if !imports.is_empty() { + debug!(target: "corpus", "imported {} new corpus entries", imports.len()); + } + + Ok(imports) + } + + /// Syncs and calibrates the in memory corpus and updates the history_map if new coverage is + /// found from the corpus findings of other workers. + #[instrument(skip_all)] + fn calibrate( + &mut self, + executor: &Executor, + fuzzed_function: Option<&Function>, + fuzzed_contracts: Option<&FuzzRunIdentifiedContracts>, + ) -> Result<()> { + let Some(worker_dir) = &self.worker_dir else { + return Ok(()); + }; + let corpus_dir = worker_dir.join(CORPUS_DIR); + + let mut executor = executor.clone(); + for (entry, tx_seq) in self.load_sync_corpus()? { + let mut new_coverage_on_sync = false; + for tx in &tx_seq { + if !Self::can_replay_tx(tx, fuzzed_function, fuzzed_contracts) { + continue; + } + + let mut call_result = execute_tx(&mut executor, tx)?; + + // Check if this provides new coverage. + let (new_coverage, is_edge) = + call_result.merge_edge_coverage(&mut self.history_map); + + if new_coverage { + self.metrics.update_seen(is_edge); + new_coverage_on_sync = true; + } + + // Commit only for stateful tests. + if fuzzed_contracts.is_some() { + executor.commit(&mut call_result); + } + + trace!( + target: "corpus", + %new_coverage, + ?tx, + "replayed tx for syncing", + ); + } + + let sync_path = &entry.path; + if new_coverage_on_sync { + // Move file from sync/ to corpus/ directory. + let corpus_path = corpus_dir.join(sync_path.components().next_back().unwrap()); + if let Err(err) = std::fs::rename(sync_path, &corpus_path) { + debug!(target: "corpus", %err, "failed to move synced corpus from {sync_path:?} to {corpus_path:?} dir"); + continue; + } + + debug!( + target: "corpus", + name=%entry.name(), + "moved synced corpus to corpus dir", + ); + + let corpus_entry = CorpusEntry::new_existing(tx_seq.to_vec(), entry.path.clone())?; + self.in_memory_corpus.push(corpus_entry); + } else { + // Remove the file as it did not generate new coverage. + if let Err(err) = std::fs::remove_file(&entry.path) { + debug!(target: "corpus", %err, "failed to remove synced corpus from {sync_path:?}"); + continue; + } + trace!(target: "corpus", "removed synced corpus from {sync_path:?}"); + } + } + + Ok(()) + } + + /// Exports the new corpus entries to the master worker's sync dir. + #[instrument(skip_all)] + fn export_to_master(&self) -> Result<()> { + // Master doesn't export (it only receives from others). + assert_ne!(self.id, 0, "non-master only"); + + // Early return if no new entries or corpus dir not configured. + if self.new_entry_indices.is_empty() || self.worker_dir.is_none() { + return Ok(()); + } + + let worker_dir = self.worker_dir.as_ref().unwrap(); + let Some(master_sync_dir) = self + .config + .corpus_dir + .as_ref() + .map(|dir| dir.join(format!("{WORKER}0")).join(SYNC_DIR)) + else { + return Ok(()); + }; + + let mut exported = 0; + let corpus_dir = worker_dir.join(CORPUS_DIR); + + for &index in &self.new_entry_indices { + let Some(corpus) = self.in_memory_corpus.get(index) else { continue }; + let file_name = corpus.file_name(self.config.corpus_gzip); + let file_path = corpus_dir.join(&file_name); + let sync_path = master_sync_dir.join(&file_name); + if let Err(err) = std::fs::hard_link(&file_path, &sync_path) { + debug!(target: "corpus", %err, "failed to export corpus {}", corpus.uuid); + continue; + } + exported += 1; + } + + debug!(target: "corpus", "exported {exported} new corpus entries"); + + Ok(()) + } + + /// Exports the global corpus to the `sync/` directories of all the non-master workers. + #[instrument(skip_all)] + fn export_to_workers(&mut self, num_workers: usize) -> Result<()> { + assert_eq!(self.id, 0, "master worker only"); + if self.worker_dir.is_none() { + return Ok(()); + } + + let worker_dir = self.worker_dir.as_ref().unwrap(); + let master_corpus_dir = worker_dir.join(CORPUS_DIR); + let filtered_master_corpus = read_corpus_dir(&master_corpus_dir) + .filter(|entry| entry.timestamp > self.last_sync_timestamp) + .collect::>(); + let mut any_distributed = false; + for target_worker in 1..num_workers { + let target_dir = self + .config + .corpus_dir + .as_ref() + .unwrap() + .join(format!("{WORKER}{target_worker}")) + .join(SYNC_DIR); + + if !target_dir.is_dir() { + foundry_common::fs::create_dir_all(&target_dir)?; + } + + for entry in &filtered_master_corpus { + let name = entry.name(); + let sync_path = target_dir.join(name); + if let Err(err) = std::fs::hard_link(&entry.path, &sync_path) { + debug!(target: "corpus", %err, from=?entry.path, to=?sync_path, "failed to distribute corpus"); + continue; + } + any_distributed = true; + trace!(target: "corpus", %name, ?target_dir, "distributed corpus"); + } + } + + debug!(target: "corpus", %any_distributed, "distributed master corpus to all workers"); + + Ok(()) + } + + // TODO(dani): currently only master syncs metrics? + /// Syncs local metrics with global corpus metrics by calculating and applying deltas. + pub(crate) fn sync_metrics(&mut self, global_corpus_metrics: &GlobalCorpusMetrics) { + // Calculate delta metrics since last sync. + let edges_delta = self + .metrics + .cumulative_edges_seen + .saturating_sub(self.last_sync_metrics.cumulative_edges_seen); + let features_delta = self + .metrics + .cumulative_features_seen + .saturating_sub(self.last_sync_metrics.cumulative_features_seen); + // For corpus count and favored items, calculate deltas. + let corpus_count_delta = + self.metrics.corpus_count as isize - self.last_sync_metrics.corpus_count as isize; + let favored_delta = + self.metrics.favored_items as isize - self.last_sync_metrics.favored_items as isize; + + // Add delta values to global metrics. + + if edges_delta > 0 { + global_corpus_metrics.cumulative_edges_seen.fetch_add(edges_delta, Ordering::Relaxed); + } + if features_delta > 0 { + global_corpus_metrics + .cumulative_features_seen + .fetch_add(features_delta, Ordering::Relaxed); + } + + if corpus_count_delta > 0 { + global_corpus_metrics + .corpus_count + .fetch_add(corpus_count_delta as usize, Ordering::Relaxed); + } else if corpus_count_delta < 0 { + global_corpus_metrics + .corpus_count + .fetch_sub((-corpus_count_delta) as usize, Ordering::Relaxed); + } + + if favored_delta > 0 { + global_corpus_metrics + .favored_items + .fetch_add(favored_delta as usize, Ordering::Relaxed); + } else if favored_delta < 0 { + global_corpus_metrics + .favored_items + .fetch_sub((-favored_delta) as usize, Ordering::Relaxed); + } + + // Store current metrics as last sync metrics for next delta calculation. + self.last_sync_metrics = self.metrics.clone(); + } + + /// Syncs the workers in_memory_corpus and history_map with the findings from other workers. + #[instrument(skip_all)] + pub fn sync( + &mut self, + num_workers: usize, + executor: &Executor, + fuzzed_function: Option<&Function>, + fuzzed_contracts: Option<&FuzzRunIdentifiedContracts>, + global_corpus_metrics: &GlobalCorpusMetrics, + ) -> Result<()> { + trace!(target: "corpus", "syncing"); + + self.sync_metrics(global_corpus_metrics); + + self.calibrate(executor, fuzzed_function, fuzzed_contracts)?; + if self.id == 0 { + self.export_to_workers(num_workers)?; + } else { + self.export_to_master()?; + } + + let last_sync = SystemTime::now().duration_since(UNIX_EPOCH)?.as_secs(); + self.last_sync_timestamp = last_sync; + + self.new_entry_indices.clear(); + + debug!(target: "corpus", last_sync, "synced"); + + Ok(()) + } + + /// Helper to check if a tx can be replayed. + fn can_replay_tx( + tx: &BasicTxDetails, + fuzzed_function: Option<&Function>, + fuzzed_contracts: Option<&FuzzRunIdentifiedContracts>, + ) -> bool { + fuzzed_contracts.is_some_and(|contracts| contracts.targets.lock().can_replay(tx)) + || fuzzed_function.is_some_and(|function| { + tx.call_details + .calldata + .get(..4) + .is_some_and(|selector| function.selector() == selector) + }) + } +} + +fn read_corpus_dir(path: &Path) -> impl Iterator { + let dir = match std::fs::read_dir(path) { + Ok(dir) => dir, + Err(err) => { + debug!(%err, ?path, "failed to read corpus directory"); + return vec![].into_iter(); + } + }; + dir.filter_map(|res| match res { + Ok(entry) => { + let path = entry.path(); + if !path.is_file() { + return None; + } + let name = if path.is_file() + && let Some(name) = path.file_name() + && let Some(name) = name.to_str() + { + name + } else { + return None; + }; + + if let Ok((uuid, timestamp)) = parse_corpus_filename(name) { + Some(CorpusDirEntry { path, uuid, timestamp }) + } else { + debug!(target: "corpus", ?path, "failed to parse corpus filename"); + None + } + } + Err(err) => { + debug!(%err, "failed to read corpus directory entry"); + None + } + }) + .collect::>() + .into_iter() +} + +struct CorpusDirEntry { + path: PathBuf, + uuid: Uuid, + timestamp: u64, +} + +impl CorpusDirEntry { + fn name(&self) -> &str { + self.path.file_name().unwrap().to_str().unwrap() + } + + fn read_tx_seq(&self) -> foundry_common::fs::Result> { + let path = &self.path; + if path.extension() == Some("gz".as_ref()) { + foundry_common::fs::read_json_gzip_file(path) + } else { + foundry_common::fs::read_json_file(path) + } + } +} + +/// Parses the corpus filename and returns the uuid and timestamp associated with it. +fn parse_corpus_filename(name: &str) -> Result<(Uuid, u64)> { + let name = name.trim_end_matches(".gz").trim_end_matches(".json"); + + let (uuid_str, timestamp_str) = + name.rsplit_once('-').ok_or_else(|| eyre!("invalid corpus filename format: {name}"))?; + + let uuid = Uuid::parse_str(uuid_str)?; + let timestamp = timestamp_str.parse()?; + + Ok((uuid, timestamp)) } #[cfg(test)] @@ -656,7 +1114,7 @@ mod tests { dir } - fn new_manager_with_single_corpus() -> (CorpusManager, Uuid) { + fn new_manager_with_single_corpus() -> (WorkerCorpus, Uuid) { let tx_gen = Just(basic_tx()).boxed(); let config = FuzzCorpusConfig { corpus_dir: Some(temp_corpus_dir()), @@ -667,18 +1125,28 @@ mod tests { }; let tx_seq = vec![basic_tx()]; - let corpus = CorpusEntry::from_tx_seq(&tx_seq); + let corpus = CorpusEntry::new(tx_seq); let seed_uuid = corpus.uuid; - let manager = CorpusManager { + // Create corpus root dir and worker subdirectory. + let corpus_root = config.corpus_dir.clone().unwrap(); + let worker_subdir = corpus_root.join("worker0"); + let _ = fs::create_dir_all(&worker_subdir); + + let manager = WorkerCorpus { + id: 0, tx_generator: tx_gen, mutation_generator: Just(MutationType::Repeat).boxed(), - config, + config: config.into(), in_memory_corpus: vec![corpus], current_mutated: Some(seed_uuid), failed_replays: 0, history_map: vec![0u8; COVERAGE_MAP_SIZE], metrics: CorpusMetrics::default(), + new_entry_indices: Default::default(), + last_sync_timestamp: 0, + worker_dir: Some(corpus_root), + last_sync_metrics: CorpusMetrics::default(), }; (manager, seed_uuid) @@ -689,13 +1157,13 @@ mod tests { let (mut manager, uuid) = new_manager_with_single_corpus(); let corpus = manager.in_memory_corpus.iter_mut().find(|c| c.uuid == uuid).unwrap(); corpus.total_mutations = 4; - corpus.new_finds_produced = 2; // ratio currently 0.5 if both increment → 3/5 = 0.6 > 0.3 + corpus.new_finds_produced = 2; // ratio currently 0.5 if both increment → 3/5 = 0.6 > 0.3. corpus.is_favored = false; - // ensure metrics start at 0 + // Ensure metrics start at 0. assert_eq!(manager.metrics.favored_items, 0); - // mark this as the currently mutated corpus and process a run with new coverage + // Mark this as the currently mutated corpus and process a run with new coverage. manager.current_mutated = Some(uuid); manager.process_inputs(&[basic_tx()], true); @@ -712,12 +1180,12 @@ mod tests { let (mut manager, uuid) = new_manager_with_single_corpus(); let corpus = manager.in_memory_corpus.iter_mut().find(|c| c.uuid == uuid).unwrap(); corpus.total_mutations = 9; - corpus.new_finds_produced = 3; // 3/9 = 0.333.. > 0.3; after +1: 3/10 = 0.3 => not favored - corpus.is_favored = true; // start as favored + corpus.new_finds_produced = 3; // 3/9 = 0.333.. > 0.3; after +1: 3/10 = 0.3 => not favored. + corpus.is_favored = true; // Start as favored. manager.metrics.favored_items = 1; - // Next run does NOT produce coverage → only total_mutations increments, ratio drops + // Next run does NOT produce coverage → only total_mutations increments, ratio drops. manager.current_mutated = Some(uuid); manager.process_inputs(&[basic_tx()], false); @@ -733,7 +1201,7 @@ mod tests { fn favored_is_false_on_ratio_equal_threshold() { let (mut manager, uuid) = new_manager_with_single_corpus(); let corpus = manager.in_memory_corpus.iter_mut().find(|c| c.uuid == uuid).unwrap(); - // After this call with new_coverage=true, totals become 10 and 3 → 0.3 + // After this call with new_coverage=true, totals become 10 and 3 → 0.3. corpus.total_mutations = 9; corpus.new_finds_produced = 2; corpus.is_favored = false; @@ -750,7 +1218,7 @@ mod tests { #[test] fn eviction_skips_favored_and_evicts_non_favored() { - // manager with two corpora + // Manager with two corpora. let tx_gen = Just(basic_tx()).boxed(); let config = FuzzCorpusConfig { corpus_dir: Some(temp_corpus_dir()), @@ -759,36 +1227,45 @@ mod tests { ..Default::default() }; - let mut favored = CorpusEntry::from_tx_seq(&[basic_tx()]); + let mut favored = CorpusEntry::new(vec![basic_tx()]); favored.total_mutations = 2; favored.is_favored = true; - let mut non_favored = CorpusEntry::from_tx_seq(&[basic_tx()]); + let mut non_favored = CorpusEntry::new(vec![basic_tx()]); non_favored.total_mutations = 2; non_favored.is_favored = false; let non_favored_uuid = non_favored.uuid; - let mut manager = CorpusManager { + let corpus_root = temp_corpus_dir(); + let worker_subdir = corpus_root.join("worker0"); + fs::create_dir_all(&worker_subdir).unwrap(); + + let mut manager = WorkerCorpus { + id: 0, tx_generator: tx_gen, mutation_generator: Just(MutationType::Repeat).boxed(), - config, + config: config.into(), in_memory_corpus: vec![favored, non_favored], current_mutated: None, failed_replays: 0, history_map: vec![0u8; COVERAGE_MAP_SIZE], metrics: CorpusMetrics::default(), + new_entry_indices: Default::default(), + last_sync_timestamp: 0, + worker_dir: Some(corpus_root), + last_sync_metrics: CorpusMetrics::default(), }; - // First eviction should remove the non-favored one + // First eviction should remove the non-favored one. manager.evict_oldest_corpus().unwrap(); assert_eq!(manager.in_memory_corpus.len(), 1); assert!(manager.in_memory_corpus.iter().all(|c| c.is_favored)); - // Attempt eviction again: only favored remains → should not remove + // Attempt eviction again: only favored remains → should not remove. manager.evict_oldest_corpus().unwrap(); assert_eq!(manager.in_memory_corpus.len(), 1, "favored corpus must not be evicted"); - // ensure the evicted one was the non-favored uuid + // Ensure the evicted one was the non-favored uuid. assert!(manager.in_memory_corpus.iter().all(|c| c.uuid != non_favored_uuid)); } } diff --git a/crates/evm/evm/src/executors/fuzz/mod.rs b/crates/evm/evm/src/executors/fuzz/mod.rs index 6da6a86b223b9..dcee02e5f8b29 100644 --- a/crates/evm/evm/src/executors/fuzz/mod.rs +++ b/crates/evm/evm/src/executors/fuzz/mod.rs @@ -1,9 +1,10 @@ use crate::executors::{ DURATION_BETWEEN_METRICS_REPORT, EarlyExit, Executor, FuzzTestTimer, RawCallResult, + corpus::{GlobalCorpusMetrics, WorkerCorpus}, }; use alloy_dyn_abi::JsonAbiExt; use alloy_json_abi::Function; -use alloy_primitives::{Address, Bytes, Log, U256, map::HashMap}; +use alloy_primitives::{Address, Bytes, Log, U256, keccak256, map::HashMap}; use eyre::Result; use foundry_common::sh_println; use foundry_config::FuzzConfig; @@ -22,41 +23,134 @@ use foundry_evm_traces::SparsedTraceArena; use indicatif::ProgressBar; use proptest::{ strategy::Strategy, - test_runner::{TestCaseError, TestRunner}, + test_runner::{RngAlgorithm, TestCaseError, TestRng, TestRunner}, }; +use rayon::iter::{IntoParallelIterator, ParallelIterator}; use serde_json::json; -use std::time::{Instant, SystemTime, UNIX_EPOCH}; +use std::{ + sync::{ + Arc, OnceLock, + atomic::{AtomicU32, Ordering}, + }, + time::{Instant, SystemTime, UNIX_EPOCH}, +}; mod types; -use crate::executors::corpus::CorpusManager; pub use types::{CaseOutcome, CounterExampleOutcome, FuzzOutcome}; -/// Contains data collected during fuzz test runs. +/// Corpus syncs across workers every `SYNC_INTERVAL` runs. +const SYNC_INTERVAL: u32 = 1000; + +/// Minimum number of runs per worker. +/// This is mainly to reduce the overall number of rayon jobs. +const MIN_RUNS_PER_WORKER: u32 = 64; + #[derive(Default)] -struct FuzzTestData { - // Stores the first fuzz case. - first_case: Option, - // Stored gas usage per fuzz case. +struct WorkerState { + /// Worker identifier + id: usize, + /// First fuzz case this worker encountered (with global run number) + first_case: Option<(u32, FuzzCase)>, + /// Gas usage for all cases this worker ran gas_by_case: Vec<(u64, u64)>, - // Stores the result and calldata of the last failed call, if any. + /// Counterexample if this worker found one counterexample: (Bytes, RawCallResult), - // Stores up to `max_traces_to_collect` traces. + /// Traces collected by this worker + /// + /// Stores up to `max_traces_to_collect` which is `config.gas_report_samples / num_workers` traces: Vec, - // Stores breakpoints for the last fuzz case. + /// Last breakpoints from this worker breakpoints: Option, - // Stores coverage information for all fuzz cases. + /// Coverage collected by this worker coverage: Option, - // Stores logs for all fuzz cases (when show_logs is true) or just the last run (when show_logs - // is false) + /// Logs from all cases this worker ran logs: Vec, - // Deprecated cheatcodes mapped to their replacements. + /// Deprecated cheatcodes seen by this worker deprecated_cheatcodes: HashMap<&'static str, Option<&'static str>>, - // Runs performed in fuzz test. + /// Number of runs this worker completed runs: u32, - // Current assume rejects of the fuzz run. - rejects: u32, - // Test failure. + /// Failure reason if this worker failed failure: Option, + /// Last run timestamp in milliseconds + /// + /// Used to identify which worker ran last and collect its traces and call breakpoints + last_run_timestamp: u128, + /// Failed corpus replays + failed_corpus_replays: usize, +} + +impl WorkerState { + fn new(worker_id: usize) -> Self { + Self { id: worker_id, ..Default::default() } + } +} + +/// Shared state for coordinating parallel fuzz workers +struct SharedFuzzState { + state: EvmFuzzState, + /// Total runs across workers + total_runs: Arc, + /// Found failure + /// + /// The worker that found the failure sets it's ID. + /// + /// This ID is then used to correctly extract the failure reason and counterexample. + failed_worker_id: OnceLock, + /// Total rejects across workers + total_rejects: Arc, + /// Fuzz timer + timer: FuzzTestTimer, + /// Global corpus metrics + global_corpus_metrics: GlobalCorpusMetrics, + + /// Global test suite early exit. + global_early_exit: EarlyExit, + /// Local fuzz early exit. + local_early_exit: EarlyExit, +} + +impl SharedFuzzState { + fn new(state: EvmFuzzState, timeout: Option, early_exit: EarlyExit) -> Self { + Self { + state, + total_runs: Arc::new(AtomicU32::new(0)), + failed_worker_id: OnceLock::new(), + total_rejects: Arc::new(AtomicU32::new(0)), + timer: FuzzTestTimer::new(timeout), + global_corpus_metrics: GlobalCorpusMetrics::default(), + global_early_exit: early_exit, + local_early_exit: EarlyExit::new(true), + } + } + + /// Increments the number of runs and returns the new value. + fn increment_runs(&self) -> u32 { + self.total_runs.fetch_add(1, Ordering::Relaxed) + 1 + } + + /// Increments and returns the new value of the number of rejected tests. + fn increment_rejects(&self) -> u32 { + self.total_rejects.fetch_add(1, Ordering::Relaxed) + 1 + } + + /// Returns `true` if the worker should continue running. + fn should_continue(&self) -> bool { + !(self.global_early_exit.should_stop() + || self.local_early_exit.should_stop() + || self.timer.is_timed_out()) + } + + /// Returns true if the worker was able to claim the failure, false if failure was set by + /// another worker + fn try_claim_failure(&self, worker_id: usize) -> bool { + let mut claimed = false; + let _ = self.failed_worker_id.get_or_init(|| { + claimed = true; + self.local_early_exit.record_failure(); + worker_id + }); + claimed + } } /// Wrapper around an [`Executor`] which provides fuzzing support using [`proptest`]. @@ -75,6 +169,8 @@ pub struct FuzzedExecutor { config: FuzzConfig, /// The persisted counterexample to be replayed, if any. persisted_failure: Option, + /// The number of parallel workers. + num_workers: usize, } impl FuzzedExecutor { @@ -86,7 +182,12 @@ impl FuzzedExecutor { config: FuzzConfig, persisted_failure: Option, ) -> Self { - Self { executor, runner, sender, config, persisted_failure } + let mut max_workers = Ord::max(1, config.runs / MIN_RUNS_PER_WORKER); + if config.runs == 0 { + max_workers = 0; + } + let num_workers = Ord::min(rayon::current_num_threads(), max_workers as usize); + Self { executor, runner, sender, config, persisted_failure, num_workers } } /// Fuzzes the provided function, assuming it is available at the contract at `address` @@ -104,14 +205,210 @@ impl FuzzedExecutor { rd: &RevertDecoder, progress: Option<&ProgressBar>, early_exit: &EarlyExit, + tokio_handle: &tokio::runtime::Handle, ) -> Result { - let state = &state; - // Stores the fuzz test execution data. - let mut test_data = FuzzTestData::default(); + let shared_state = SharedFuzzState::new(state, self.config.timeout, early_exit.clone()); + + debug!(n = self.num_workers, "spawning workers"); + let workers = (0..self.num_workers) + .into_par_iter() + .map(|worker_id| { + let _guard = tokio_handle.enter(); + let _guard = info_span!("fuzz_worker", id = worker_id).entered(); + let timer = Instant::now(); + let r = self.run_worker( + worker_id, + func, + fuzz_fixtures, + address, + rd, + &shared_state, + progress, + ); + debug!("finished in {:?}", timer.elapsed()); + r + }) + .collect::>>()?; + + Ok(self.aggregate_results(workers, func, &shared_state)) + } + + /// Granular and single-step function that runs only one fuzz and returns either a `CaseOutcome` + /// or a `CounterExampleOutcome` + fn single_fuzz( + &self, + address: Address, + calldata: Bytes, + coverage_metrics: &mut WorkerCorpus, + ) -> Result { + let mut call = self + .executor + .call_raw(self.sender, address, calldata.clone(), U256::ZERO) + .map_err(|e| TestCaseError::fail(e.to_string()))?; + let new_coverage = coverage_metrics.merge_edge_coverage(&mut call); + coverage_metrics.process_inputs( + &[BasicTxDetails { + warp: None, + roll: None, + sender: self.sender, + call_details: CallDetails { target: address, calldata: calldata.clone() }, + }], + new_coverage, + ); + + // Handle `vm.assume`. + if call.result.as_ref() == MAGIC_ASSUME { + return Err(TestCaseError::reject(FuzzError::AssumeReject)); + } + + let (breakpoints, deprecated_cheatcodes) = + call.cheatcodes.as_ref().map_or_else(Default::default, |cheats| { + (cheats.breakpoints.clone(), cheats.deprecated.clone()) + }); + + // Consider call success if test should not fail on reverts and reverter is not the + // cheatcode or test address. + let success = if !self.config.fail_on_revert + && call + .reverter + .is_some_and(|reverter| reverter != address && reverter != CHEATCODE_ADDRESS) + { + true + } else { + self.executor.is_raw_call_mut_success(address, &mut call, false) + }; + + if success { + Ok(FuzzOutcome::Case(CaseOutcome { + case: FuzzCase { calldata, gas: call.gas_used, stipend: call.stipend }, + traces: call.traces, + coverage: call.line_coverage, + breakpoints, + logs: call.logs, + deprecated_cheatcodes, + })) + } else { + Ok(FuzzOutcome::CounterExample(CounterExampleOutcome { + exit_reason: call.exit_reason, + counterexample: (calldata, call), + breakpoints, + })) + } + } + + /// Aggregates the results from all workers + fn aggregate_results( + &self, + mut workers: Vec, + func: &Function, + shared_state: &SharedFuzzState, + ) -> FuzzTestResult { + let mut result = FuzzTestResult::default(); + if workers.is_empty() { + result.success = true; + return result; + } + + // Find first case and last run worker. Set `failed_corpus_replays`. + let mut first_case_candidate = None; + let mut last_run_worker = None; + for (i, worker) in workers.iter().enumerate() { + if let Some((run, ref case)) = worker.first_case + && first_case_candidate.as_ref().is_none_or(|&(r, _)| run < r) + { + first_case_candidate = Some((run, case.clone())); + } + + if last_run_worker.is_none_or(|(t, _)| worker.last_run_timestamp > t) { + last_run_worker = Some((worker.last_run_timestamp, i)); + } + + // Only set replays from master which is responsible for replaying persisted corpus. + if worker.id == 0 { + result.failed_corpus_replays = worker.failed_corpus_replays; + } + } + result.first_case = first_case_candidate.map(|(_, case)| case).unwrap_or_default(); + let (_, last_run_worker_idx) = last_run_worker.expect("at least one worker"); + + if let Some(&failed_worker_id) = shared_state.failed_worker_id.get() { + result.success = false; + + let failed_worker_idx = workers.iter().position(|w| w.id == failed_worker_id).unwrap(); + let failed_worker = &mut workers[failed_worker_idx]; + + let (calldata, call) = std::mem::take(&mut failed_worker.counterexample); + result.labels = call.labels; + result.traces = call.traces.clone(); + result.breakpoints = call.cheatcodes.map(|c| c.breakpoints); + + match &failed_worker.failure { + Some(TestCaseError::Fail(reason)) => { + let reason = reason.to_string(); + result.reason = (!reason.is_empty()).then_some(reason); + let args = if let Some(data) = calldata.get(4..) { + func.abi_decode_input(data).unwrap_or_default() + } else { + vec![] + }; + result.counterexample = Some(CounterExample::Single( + BaseCounterExample::from_fuzz_call(calldata, args, call.traces), + )); + } + Some(TestCaseError::Reject(reason)) => { + let reason = reason.to_string(); + result.reason = (!reason.is_empty()).then_some(reason); + } + None => {} + } + } else { + let last_run_worker = &workers[last_run_worker_idx]; + result.success = true; + result.traces = last_run_worker.traces.last().cloned(); + result.breakpoints = last_run_worker.breakpoints.clone(); + } + + if !self.config.show_logs { + result.logs = workers[last_run_worker_idx].logs.clone(); + } + + for mut worker in workers { + result.gas_by_case.append(&mut worker.gas_by_case); + if self.config.show_logs { + result.logs.append(&mut worker.logs); + } + result.gas_report_traces.extend(worker.traces.into_iter().map(|t| t.arena)); + HitMaps::merge_opt(&mut result.line_coverage, worker.coverage); + result.deprecated_cheatcodes.extend(worker.deprecated_cheatcodes); + } + + if let Some(reason) = &result.reason + && let Some(reason) = SkipReason::decode_self(reason) + { + result.skipped = true; + result.reason = reason.0; + } + + result + } + + /// Runs a single fuzz worker + #[allow(clippy::too_many_arguments)] + fn run_worker( + &self, + worker_id: usize, + func: &Function, + fuzz_fixtures: &FuzzFixtures, + address: Address, + rd: &RevertDecoder, + shared_state: &SharedFuzzState, + progress: Option<&ProgressBar>, + ) -> Result { + // Prepare let dictionary_weight = self.config.dictionary.dictionary_weight.min(100); let strategy = proptest::prop_oneof![ 100 - dictionary_weight => fuzz_calldata(func.clone(), fuzz_fixtures), - dictionary_weight => fuzz_calldata_from_state(func.clone(), state), + dictionary_weight => fuzz_calldata_from_state(func.clone(), &shared_state.state), ] .prop_map(move |calldata| BasicTxDetails { warp: None, @@ -119,267 +416,224 @@ impl FuzzedExecutor { sender: Default::default(), call_details: CallDetails { target: Default::default(), calldata }, }); - // We want to collect at least one trace which will be displayed to user. - let max_traces_to_collect = std::cmp::max(1, self.config.gas_report_samples) as usize; - let mut corpus_manager = CorpusManager::new( + let mut corpus = WorkerCorpus::new( + worker_id, self.config.corpus.clone(), strategy.boxed(), - &self.executor, + // Master worker replays the persisted corpus using the executor + if worker_id == 0 { Some(&self.executor) } else { None }, Some(func), - None, + None, // fuzzed_contracts for invariant tests )?; - // Start timer for this fuzz test. - let timer = FuzzTestTimer::new(self.config.timeout); - let mut last_metrics_report = Instant::now(); - let max_runs = self.config.runs; - let continue_campaign = |runs: u32| { - if early_exit.should_stop() { - return false; - } + let mut worker = WorkerState::new(worker_id); + // We want to collect at least one trace which will be displayed to user. + let max_traces_to_collect = + std::cmp::max(1, self.config.gas_report_samples / self.num_workers as u32); - if timer.is_enabled() { !timer.is_timed_out() } else { runs < max_runs } + let worker_runs = self.runs_per_worker(worker_id); + debug!(worker_runs); + + let mut runner_config = self.runner.config().clone(); + runner_config.cases = worker_runs; + + let mut runner = if let Some(seed) = self.config.seed { + // For deterministic parallel fuzzing, derive a unique seed for each worker + let worker_seed = if worker_id == 0 { + // Master worker uses the provided seed as is. + seed + } else { + // Derive a worker-specific seed using keccak256(seed || worker_id) + let seed_data = + [&seed.to_be_bytes::<32>()[..], &worker_id.to_be_bytes()[..]].concat(); + U256::from_be_bytes(keccak256(seed_data).0) + }; + trace!(target: "forge::test", ?worker_seed, "deterministic seed for worker {worker_id}"); + let rng = TestRng::from_seed(RngAlgorithm::ChaCha, &worker_seed.to_be_bytes::<32>()); + TestRunner::new_with_rng(runner_config, rng) + } else { + TestRunner::new(runner_config) }; - 'stop: while continue_campaign(test_data.runs) { + let mut persisted_failure = self.persisted_failure.as_ref().filter(|_| worker_id == 0); + + // Offset to stagger corpus syncs across workers; so that workers don't sync at the same + // time. + let sync_offset = worker_id as u32 * 100; + let sync_threshold = SYNC_INTERVAL + sync_offset; + let mut runs_since_sync = sync_threshold; // Always sync at the start. + let mut last_metrics_report = Instant::now(); + // Continue while: + // 1. Global state allows (not timed out, not at global limit, no failure found) + // 2. Worker hasn't reached its specific run limit + 'stop: while shared_state.should_continue() && worker.runs < worker_runs { // If counterexample recorded, replay it first, without incrementing runs. - let input = if let Some(failure) = self.persisted_failure.take() + let input = if worker_id == 0 + && let Some(failure) = persisted_failure.take() && failure.calldata.get(..4).is_some_and(|selector| func.selector() == selector) { failure.calldata.clone() } else { - // If running with progress, then increment current run. - if let Some(progress) = progress { - progress.inc(1); - // Display metrics in progress bar. - if self.config.corpus.collect_edge_coverage() { - progress.set_message(format!("{}", &corpus_manager.metrics)); - } - } else if self.config.corpus.collect_edge_coverage() - && last_metrics_report.elapsed() > DURATION_BETWEEN_METRICS_REPORT - { - // Display metrics inline. - let metrics = json!({ - "timestamp": SystemTime::now() - .duration_since(UNIX_EPOCH)? - .as_secs(), - "test": func.name, - "metrics": &corpus_manager.metrics, - }); - let _ = sh_println!("{}", serde_json::to_string(&metrics)?); - last_metrics_report = Instant::now(); - }; - - test_data.runs += 1; - - match corpus_manager.new_input(&mut self.runner, state, func) { + runs_since_sync += 1; + if runs_since_sync >= sync_threshold { + let timer = Instant::now(); + corpus.sync( + self.num_workers, + &self.executor, + Some(func), + None, + &shared_state.global_corpus_metrics, + )?; + trace!("finished corpus sync in {:?}", timer.elapsed()); + runs_since_sync = 0; + } + + match corpus.new_input(&mut runner, &shared_state.state, func) { Ok(input) => input, Err(err) => { - test_data.failure = Some(TestCaseError::fail(format!( - "failed to generate fuzzed input: {err}" + worker.failure = Some(TestCaseError::fail(format!( + "failed to generate fuzzed input in worker {}: {err}", + worker.id ))); + shared_state.try_claim_failure(worker_id); break 'stop; } } }; - match self.single_fuzz(address, input, &mut corpus_manager) { + let mut inc_runs = || { + let total_runs = shared_state.increment_runs(); + debug_assert!( + shared_state.timer.is_enabled() || total_runs <= self.config.runs, + "worker runs were not distributed correctly" + ); + worker.runs += 1; + if let Some(progress) = progress { + progress.inc(1); + } + total_runs + }; + + worker.last_run_timestamp = SystemTime::now().duration_since(UNIX_EPOCH)?.as_millis(); + match self.single_fuzz(address, input, &mut corpus) { Ok(fuzz_outcome) => match fuzz_outcome { FuzzOutcome::Case(case) => { - test_data.gas_by_case.push((case.case.gas, case.case.stipend)); + let total_runs = inc_runs(); + + if worker_id == 0 && self.config.corpus.collect_edge_coverage() { + if let Some(progress) = progress { + corpus.sync_metrics(&shared_state.global_corpus_metrics); + progress + .set_message(format!("{}", shared_state.global_corpus_metrics)); + } else if last_metrics_report.elapsed() + > DURATION_BETWEEN_METRICS_REPORT + { + corpus.sync_metrics(&shared_state.global_corpus_metrics); + // Display metrics inline. + let metrics = json!({ + "timestamp": SystemTime::now() + .duration_since(UNIX_EPOCH)? + .as_secs(), + "test": func.name, + "metrics": shared_state.global_corpus_metrics.load(), + }); + let _ = sh_println!("{metrics}"); + last_metrics_report = Instant::now(); + } + } + + worker.gas_by_case.push((case.case.gas, case.case.stipend)); - if test_data.first_case.is_none() { - test_data.first_case.replace(case.case); + if worker.first_case.is_none() { + worker.first_case = Some((total_runs, case.case)); } if let Some(call_traces) = case.traces { - if test_data.traces.len() == max_traces_to_collect { - test_data.traces.pop(); + if worker.traces.len() == max_traces_to_collect as usize { + worker.traces.pop(); } - test_data.traces.push(call_traces); - test_data.breakpoints.replace(case.breakpoints); + worker.traces.push(call_traces); + worker.breakpoints = Some(case.breakpoints); } // Always store logs from the last run in test_data.logs for display at // verbosity >= 2. When show_logs is true, // accumulate all logs. When false, only keep the last run's logs. if self.config.show_logs { - test_data.logs.extend(case.logs); + worker.logs.extend(case.logs); } else { - test_data.logs = case.logs; + worker.logs = case.logs; } - HitMaps::merge_opt(&mut test_data.coverage, case.coverage); - test_data.deprecated_cheatcodes = case.deprecated_cheatcodes; + HitMaps::merge_opt(&mut worker.coverage, case.coverage); + worker.deprecated_cheatcodes = case.deprecated_cheatcodes; } FuzzOutcome::CounterExample(CounterExampleOutcome { exit_reason: status, counterexample: outcome, .. }) => { + inc_runs(); + let reason = rd.maybe_decode(&outcome.1.result, status); - test_data.logs.extend(outcome.1.logs.clone()); - test_data.counterexample = outcome; - test_data.failure = Some(TestCaseError::fail(reason.unwrap_or_default())); + worker.logs.extend(outcome.1.logs.clone()); + worker.counterexample = outcome; + worker.failure = Some(TestCaseError::fail(reason.unwrap_or_default())); + shared_state.try_claim_failure(worker_id); break 'stop; } }, - Err(err) => { - match err { - TestCaseError::Fail(_) => { - test_data.failure = Some(err); - break 'stop; - } - TestCaseError::Reject(_) => { - // Discard run and apply max rejects if configured. Saturate to handle - // the case of replayed failure, which doesn't count as a run. - test_data.runs = test_data.runs.saturating_sub(1); - test_data.rejects += 1; + Err(err) => match err { + TestCaseError::Fail(_) => { + worker.failure = Some(err); + shared_state.try_claim_failure(worker_id); + break 'stop; + } + TestCaseError::Reject(_) => { + let max = self.config.max_test_rejects; - // Update progress bar to reflect rejected runs. - if let Some(progress) = progress { - progress.set_message(format!("([{}] rejected)", test_data.rejects)); - progress.dec(1); - } + let total = shared_state.increment_rejects(); - if self.config.max_test_rejects > 0 - && test_data.rejects >= self.config.max_test_rejects - { - test_data.failure = Some(TestCaseError::reject( - FuzzError::TooManyRejects(self.config.max_test_rejects), - )); - break 'stop; - } + // Update progress bar to reflect rejected runs. + // TODO(dani): (pre-existing) conflicts with corpus metrics `set_message` + if !self.config.corpus.collect_edge_coverage() + && let Some(progress) = progress + { + progress.set_message(format!("([{total}] rejected)")); + } + + if max > 0 && total > max { + worker.failure = + Some(TestCaseError::reject(FuzzError::TooManyRejects(max))); + shared_state.try_claim_failure(worker_id); + break 'stop; } } - } + }, } } - let (calldata, call) = test_data.counterexample; - let mut traces = test_data.traces; - let (last_run_traces, last_run_breakpoints) = if test_data.failure.is_none() { - (traces.pop(), test_data.breakpoints) - } else { - (call.traces.clone(), call.cheatcodes.map(|c| c.breakpoints)) - }; - - // test_data.logs already contains the appropriate logs: - // - For failed tests: logs from the counterexample - // - For successful tests with show_logs=true: all logs from all runs - // - For successful tests with show_logs=false: logs from the last run only - let result_logs = test_data.logs; - - let mut result = FuzzTestResult { - first_case: test_data.first_case.unwrap_or_default(), - gas_by_case: test_data.gas_by_case, - success: test_data.failure.is_none(), - skipped: false, - reason: None, - counterexample: None, - logs: result_logs, - labels: call.labels, - traces: last_run_traces, - breakpoints: last_run_breakpoints, - gas_report_traces: traces.into_iter().map(|a| a.arena).collect(), - line_coverage: test_data.coverage, - deprecated_cheatcodes: test_data.deprecated_cheatcodes, - failed_corpus_replays: corpus_manager.failed_replays(), - }; - - match test_data.failure { - Some(TestCaseError::Fail(reason)) => { - let reason = reason.to_string(); - result.reason = (!reason.is_empty()).then_some(reason); - let args = if let Some(data) = calldata.get(4..) { - func.abi_decode_input(data).unwrap_or_default() - } else { - vec![] - }; - result.counterexample = Some(CounterExample::Single( - BaseCounterExample::from_fuzz_call(calldata, args, call.traces), - )); - } - Some(TestCaseError::Reject(reason)) => { - let reason = reason.to_string(); - result.reason = (!reason.is_empty()).then_some(reason); - } - None => {} + if worker_id == 0 { + worker.failed_corpus_replays = corpus.failed_replays; } - if let Some(reason) = &result.reason - && let Some(reason) = SkipReason::decode_self(reason) - { - result.skipped = true; - result.reason = reason.0; - } + // Logs stats + trace!("worker {worker_id} fuzz stats"); + shared_state.state.log_stats(); - state.log_stats(); - - Ok(result) + Ok(worker) } - /// Granular and single-step function that runs only one fuzz and returns either a `CaseOutcome` - /// or a `CounterExampleOutcome` - fn single_fuzz( - &mut self, - address: Address, - calldata: Bytes, - coverage_metrics: &mut CorpusManager, - ) -> Result { - let mut call = self - .executor - .call_raw(self.sender, address, calldata.clone(), U256::ZERO) - .map_err(|e| TestCaseError::fail(e.to_string()))?; - let new_coverage = coverage_metrics.merge_edge_coverage(&mut call); - coverage_metrics.process_inputs( - &[BasicTxDetails { - warp: None, - roll: None, - sender: self.sender, - call_details: CallDetails { target: address, calldata: calldata.clone() }, - }], - new_coverage, - ); - - // Handle `vm.assume`. - if call.result.as_ref() == MAGIC_ASSUME { - return Err(TestCaseError::reject(FuzzError::AssumeReject)); - } - - let (breakpoints, deprecated_cheatcodes) = - call.cheatcodes.as_ref().map_or_else(Default::default, |cheats| { - (cheats.breakpoints.clone(), cheats.deprecated.clone()) - }); - - // Consider call success if test should not fail on reverts and reverter is not the - // cheatcode or test address. - let success = if !self.config.fail_on_revert - && call - .reverter - .is_some_and(|reverter| reverter != address && reverter != CHEATCODE_ADDRESS) - { - true - } else { - self.executor.is_raw_call_mut_success(address, &mut call, false) - }; - - if success { - Ok(FuzzOutcome::Case(CaseOutcome { - case: FuzzCase { calldata, gas: call.gas_used, stipend: call.stipend }, - traces: call.traces, - coverage: call.line_coverage, - breakpoints, - logs: call.logs, - deprecated_cheatcodes, - })) - } else { - Ok(FuzzOutcome::CounterExample(CounterExampleOutcome { - exit_reason: call.exit_reason, - counterexample: (calldata, call), - breakpoints, - })) - } + /// Determines the number of runs per worker. + fn runs_per_worker(&self, worker_id: usize) -> u32 { + let worker_id = worker_id as u32; + let total_runs = self.config.runs; + let n = self.num_workers as u32; + let runs = total_runs / n; + let remainder = total_runs % n; + // Distribute the remainder evenly among the first `remainder` workers, + // assuming `worker_id` is in `0..n`. + if worker_id < remainder { runs + 1 } else { runs } } } diff --git a/crates/evm/evm/src/executors/invariant/mod.rs b/crates/evm/evm/src/executors/invariant/mod.rs index 4ea542ae94a39..52a5d9c42ca51 100644 --- a/crates/evm/evm/src/executors/invariant/mod.rs +++ b/crates/evm/evm/src/executors/invariant/mod.rs @@ -1,5 +1,8 @@ use crate::{ - executors::{Executor, RawCallResult}, + executors::{ + DURATION_BETWEEN_METRICS_REPORT, EarlyExit, EvmError, Executor, FuzzTestTimer, + RawCallResult, corpus::WorkerCorpus, + }, inspectors::Fuzzer, }; use alloy_primitives::{ @@ -8,7 +11,11 @@ use alloy_primitives::{ }; use alloy_sol_types::{SolCall, sol}; use eyre::{ContextCompat, Result, eyre}; -use foundry_common::contracts::{ContractsByAddress, ContractsByArtifact}; +use foundry_common::{ + TestFunctionExt, + contracts::{ContractsByAddress, ContractsByArtifact}, + sh_println, +}; use foundry_config::InvariantConfig; use foundry_evm_core::{ constants::{ @@ -30,6 +37,8 @@ use parking_lot::RwLock; use proptest::{strategy::Strategy, test_runner::TestRunner}; use result::{assert_after_invariant, assert_invariants, can_continue}; use revm::state::Account; +use serde::{Deserialize, Serialize}; +use serde_json::json; use std::{ collections::{HashMap as Map, btree_map::Entry}, sync::Arc, @@ -44,15 +53,9 @@ mod replay; pub use replay::{replay_error, replay_run}; mod result; -use foundry_common::{TestFunctionExt, sh_println}; pub use result::InvariantFuzzTestResult; -use serde::{Deserialize, Serialize}; -use serde_json::json; mod shrink; -use crate::executors::{ - DURATION_BETWEEN_METRICS_REPORT, EarlyExit, EvmError, FuzzTestTimer, corpus::CorpusManager, -}; pub use shrink::check_sequence; sol! { @@ -541,7 +544,7 @@ impl<'a> InvariantExecutor<'a> { gas_report_traces: result.gas_report_traces, line_coverage: result.line_coverage, metrics: result.metrics, - failed_corpus_replays: corpus_manager.failed_replays(), + failed_corpus_replays: corpus_manager.failed_replays, }) } @@ -553,7 +556,7 @@ impl<'a> InvariantExecutor<'a> { invariant_contract: &InvariantContract<'_>, fuzz_fixtures: &FuzzFixtures, fuzz_state: EvmFuzzState, - ) -> Result<(InvariantTest, CorpusManager)> { + ) -> Result<(InvariantTest, WorkerCorpus)> { // Finds out the chosen deployed contracts and/or senders. self.select_contract_artifacts(invariant_contract.address)?; let (targeted_senders, targeted_contracts) = @@ -620,13 +623,15 @@ impl<'a> InvariantExecutor<'a> { return Err(eyre!(error.revert_reason().unwrap_or_default())); } - let corpus_manager = CorpusManager::new( + let worker = WorkerCorpus::new( + 0, self.config.corpus.clone(), strategy.boxed(), - &self.executor, + Some(&self.executor), None, Some(&targeted_contracts), )?; + let invariant_test = InvariantTest::new( fuzz_state, targeted_contracts, @@ -635,7 +640,7 @@ impl<'a> InvariantExecutor<'a> { self.runner.clone(), ); - Ok((invariant_test, corpus_manager)) + Ok((invariant_test, worker)) } /// Fills the `InvariantExecutor` with the artifact identifier filters (in `path:name` string diff --git a/crates/evm/fuzz/src/lib.rs b/crates/evm/fuzz/src/lib.rs index ae1f011413606..1308be9b40241 100644 --- a/crates/evm/fuzz/src/lib.rs +++ b/crates/evm/fuzz/src/lib.rs @@ -36,25 +36,35 @@ pub use inspector::Fuzzer; /// Details of a transaction generated by fuzz strategy for fuzzing a target. #[derive(Clone, Debug, Serialize, Deserialize)] pub struct BasicTxDetails { - // Time (in seconds) to increase block timestamp before executing the tx. + /// Time (in seconds) to increase block timestamp before executing the tx. + #[serde(default, skip_serializing_if = "Option::is_none")] pub warp: Option, - // Number to increase block number before executing the tx. + /// Number to increase block number before executing the tx. + #[serde(default, skip_serializing_if = "Option::is_none")] pub roll: Option, - // Transaction sender address. + /// Transaction sender address. pub sender: Address, - // Transaction call details. + /// Transaction call details. + #[serde(flatten)] pub call_details: CallDetails, } /// Call details of a transaction generated to fuzz. #[derive(Clone, Debug, Serialize, Deserialize)] pub struct CallDetails { - // Address of target contract. + /// Address of target contract. pub target: Address, - // The data of the transaction. + /// The data of the transaction. pub calldata: Bytes, } +impl BasicTxDetails { + /// Returns an estimate of the serialized (JSON) size in bytes. + pub fn estimate_serialized_size(&self) -> usize { + size_of::() + self.call_details.calldata.len() * 2 + } +} + #[derive(Clone, Debug, Serialize, Deserialize)] #[expect(clippy::large_enum_variant)] pub enum CounterExample { @@ -277,7 +287,7 @@ pub struct FuzzTestResult { // Deprecated cheatcodes mapped to their replacements. pub deprecated_cheatcodes: HashMap<&'static str, Option<&'static str>>, - /// NUmber of failed replays from persisted corpus. + /// Number of failed replays from persisted corpus. pub failed_corpus_replays: usize, } diff --git a/crates/forge/src/result.rs b/crates/forge/src/result.rs index 7a0f42846f4c1..465516949c0f1 100644 --- a/crates/forge/src/result.rs +++ b/crates/forge/src/result.rs @@ -620,6 +620,7 @@ impl TestResult { failed_corpus_replays: 0, }; self.status = TestStatus::Failure; + debug!(?e, "failed to set up fuzz testing environment"); self.reason = Some(format!("failed to set up fuzz testing environment: {e}")); } diff --git a/crates/forge/src/runner.rs b/crates/forge/src/runner.rs index abf4805cf1343..acc5d65bc3d50 100644 --- a/crates/forge/src/runner.rs +++ b/crates/forge/src/runner.rs @@ -1017,6 +1017,7 @@ impl<'a> FunctionRunner<'a> { &self.cr.mcr.revert_decoder, progress.as_ref(), &self.tcfg.early_exit, + self.cr.tokio_handle, ) { Ok(x) => x, Err(e) => { diff --git a/crates/forge/tests/cli/test_cmd/fuzz.rs b/crates/forge/tests/cli/test_cmd/fuzz.rs index f9a48a0318c07..2cec1c30fd19d 100644 --- a/crates/forge/tests/cli/test_cmd/fuzz.rs +++ b/crates/forge/tests/cli/test_cmd/fuzz.rs @@ -88,7 +88,7 @@ forgetest_init!(test_fuzz_timeout, |prj, cmd| { import {Test} from "forge-std/Test.sol"; contract FuzzTimeoutTest is Test { - /// forge-config: default.fuzz.max-test-rejects = 50000 + /// forge-config: default.fuzz.max-test-rejects = 0 /// forge-config: default.fuzz.timeout = 1 function test_fuzz_bound(uint256 a) public pure { vm.assume(a == 0); @@ -97,7 +97,7 @@ contract FuzzTimeoutTest is Test { "#, ); - cmd.args(["test"]).assert_success().stdout_eq(str![[r#" + cmd.args(["test", "-j2"]).assert_success().stdout_eq(str![[r#" [COMPILING_FILES] with [SOLC_VERSION] [SOLC_VERSION] [ELAPSED] Compiler run successful! @@ -186,7 +186,7 @@ contract Counter { ); // Tests should fail as revert happens in cheatcode (assert) and test (require) contract. - cmd.assert_failure().stdout_eq(str![[r#" + cmd.args(["-j1"]).assert_failure().stdout_eq(str![[r#" [COMPILING_FILES] with [SOLC_VERSION] [SOLC_VERSION] [ELAPSED] Compiler run successful! @@ -248,7 +248,7 @@ contract CounterTest is Test { "#, ); // Tests should fail and record counterexample with value 200. - cmd.args(["test"]).assert_failure().stdout_eq(str![[r#" + cmd.args(["test", "-j1"]).assert_failure().stdout_eq(str![[r#" ... Failing tests: Encountered 1 failing test in test/Counter.t.sol:CounterTest @@ -324,7 +324,7 @@ contract CounterTest is Test { "#, ); // Test should fail with replayed counterexample 200 (0 runs). - cmd.forge_fuse().args(["test"]).assert_failure().stdout_eq(str![[r#" + cmd.forge_fuse().args(["test", "-j1"]).assert_failure().stdout_eq(str![[r#" ... Failing tests: Encountered 1 failing test in test/Counter.t.sol:CounterTest @@ -525,7 +525,7 @@ Suite result: FAILED. 0 passed; 1 failed; 0 skipped; [ELAPSED] assert.stdout_eq(expected.clone()); }; - cmd.arg("test"); + cmd.args(["test", "-j1"]); // Run several times, asserting that the failure persists and is the same. for _ in 0..3 { @@ -775,12 +775,12 @@ forgetest_init!(should_fuzz_literals, |prj, cmd| { ); // Helper to create expected output for a test failure - let expected_fail = |test_name: &str, type_sig: &str, value: &str, runs: u32| -> String { + let expected_fail = |test_name: &str, type_sig: &str, value: &str| -> String { format!( r#"No files changed, compilation skipped Ran 1 test for test/MagicFuzz.t.sol:MagicTest -[FAIL: panic: assertion failed (0x01); counterexample: calldata=[..] args=[{value}]] {test_name}({type_sig}) (runs: {runs}, [AVG_GAS]) +[FAIL: panic: assertion failed (0x01); counterexample: calldata=[..] args=[{value}]] {test_name}({type_sig}) (runs: [..], [AVG_GAS]) [..] Ran 1 test suite [ELAPSED]: 0 tests passed, 1 failed, 0 skipped (1 total tests) @@ -797,39 +797,37 @@ Encountered a total of 1 failing tests, 0 tests succeeded let mut test_literal = |seed: u32, test_name: &'static str, type_sig: &'static str, - expected_value: &'static str, - expected_runs: u32| { + expected_value: &'static str| { // the fuzzer is UNABLE to find a breaking input (fast) when NOT seeding from the AST prj.update_config(|config| { config.fuzz.runs = 100; config.fuzz.dictionary.max_fuzz_dictionary_literals = 0; config.fuzz.seed = Some(U256::from(seed)); }); - cmd.forge_fuse().args(["test", "--match-test", test_name]).assert_success(); + cmd.forge_fuse().args(["test", "--match-test", test_name, "-j1"]).assert_success(); // the fuzzer is ABLE to find a breaking input when seeding from the AST prj.update_config(|config| { config.fuzz.dictionary.max_fuzz_dictionary_literals = 10_000; }); - let expected_output = expected_fail(test_name, type_sig, expected_value, expected_runs); + let expected_output = expected_fail(test_name, type_sig, expected_value); cmd.forge_fuse() - .args(["test", "--match-test", test_name]) + .args(["test", "--match-test", test_name, "-j1"]) .assert_failure() .stdout_eq(expected_output); }; - test_literal(100, "testFuzz_Addr", "address", "0x6B175474E89094C44Da98b954EedeAC495271d0F", 28); - test_literal(200, "testFuzz_Number", "uint64", "1122334455 [1.122e9]", 5); - test_literal(300, "testFuzz_Integer", "int32", "-777", 0); + test_literal(100, "testFuzz_Addr", "address", "0x6B175474E89094C44Da98b954EedeAC495271d0F"); + test_literal(200, "testFuzz_Number", "uint64", "1122334455 [1.122e9]"); + test_literal(300, "testFuzz_Integer", "int32", "-777"); test_literal( 400, "testFuzz_Word", "bytes32", "0x6162636431323334000000000000000000000000000000000000000000000000", /* bytes32("abcd1234") */ - 7, ); - test_literal(500, "testFuzz_BytesFromHex", "bytes", "0xdeadbeef", 5); - test_literal(600, "testFuzz_String", "string", "\"xyzzy\"", 35); - test_literal(999, "testFuzz_BytesFromString", "bytes", "0x78797a7a79", 19); // abi.encodePacked("xyzzy") + test_literal(500, "testFuzz_BytesFromHex", "bytes", "0xdeadbeef"); + test_literal(600, "testFuzz_String", "string", "\"xyzzy\""); + test_literal(999, "testFuzz_BytesFromString", "bytes", "0x78797a7a79"); // abi.encodePacked("xyzzy") }); diff --git a/crates/forge/tests/cli/test_cmd/mod.rs b/crates/forge/tests/cli/test_cmd/mod.rs index 9666cf4d7e69d..656dcc5b7b3d3 100644 --- a/crates/forge/tests/cli/test_cmd/mod.rs +++ b/crates/forge/tests/cli/test_cmd/mod.rs @@ -875,14 +875,14 @@ contract CounterTest is Test { Compiler run successful! Ran 1 test for test/CounterFuzz.t.sol:CounterTest -[FAIL: panic: arithmetic underflow or overflow (0x11); counterexample: calldata=0xa76d58f5fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe args=[115792089237316195423570985008687907853269984665640564039457584007913129639934 [1.157e77]]] testAddOne(uint256) (runs: 27, [AVG_GAS]) +[FAIL: panic: arithmetic underflow or overflow (0x11); counterexample: calldata=[..] args=[..]] testAddOne(uint256) (runs: [..], [AVG_GAS]) Suite result: FAILED. 0 passed; 1 failed; 0 skipped; [ELAPSED] Ran 1 test suite [ELAPSED]: 0 tests passed, 1 failed, 0 skipped (1 total tests) Failing tests: Encountered 1 failing test in test/CounterFuzz.t.sol:CounterTest -[FAIL: panic: arithmetic underflow or overflow (0x11); counterexample: calldata=0xa76d58f5fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe args=[115792089237316195423570985008687907853269984665640564039457584007913129639934 [1.157e77]]] testAddOne(uint256) (runs: 27, [AVG_GAS]) +[FAIL: panic: arithmetic underflow or overflow (0x11); counterexample: calldata=[..] args=[..]] testAddOne(uint256) (runs: [..], [AVG_GAS]) Encountered a total of 1 failing tests, 0 tests succeeded @@ -4384,3 +4384,46 @@ Tip: Run `forge test --rerun` to retry only the 1 failed test "#]]); }); + +forgetest_init!(zero_runs, |prj, cmd| { + prj.wipe_contracts(); + prj.add_test( + "ZeroRuns.t.sol", + r#" +import {Test} from "forge-std/Test.sol"; + +contract Handler is Test { + function doSomething(uint256 param) public { + revert("unreachable"); + } +} + +contract ZeroRuns is Test { + Handler handler = new Handler(); + + /// forge-config: default.fuzz.runs = 0 + function test_fuzzZeroRuns(uint256 x) public { + revert("unreachable"); + } + + /// forge-config: default.invariant.runs = 0 + function invariant_zeroRuns() public {} + + /// forge-config: default.invariant.depth = 0 + function invariant_zeroDepth() public {} +} +"#, + ); + + cmd.args(["test"]).assert_success().stdout_eq(str![[r#" +... +Ran 3 tests for test/ZeroRuns.t.sol:ZeroRuns +[PASS] invariant_zeroDepth() (runs: 256, calls: 0, reverts: 0) +[PASS] invariant_zeroRuns() (runs: 0, calls: 0, reverts: 0) +[PASS] test_fuzzZeroRuns(uint256) (runs: 0, [AVG_GAS]) +Suite result: ok. 3 passed; 0 failed; 0 skipped; [ELAPSED] + +Ran 1 test suite [ELAPSED]: 3 tests passed, 0 failed, 0 skipped (3 total tests) + +"#]]); +});