diff --git a/crates/forge-runner/src/compiled_runnable.rs b/crates/forge-runner/src/compiled_runnable.rs index e8b1d4c16c..7921b38752 100644 --- a/crates/forge-runner/src/compiled_runnable.rs +++ b/crates/forge-runner/src/compiled_runnable.rs @@ -2,6 +2,7 @@ use crate::expected_result::ExpectedTestResult; use cairo_lang_sierra::{ids::GenericTypeId, program::Program}; use serde::Deserialize; use starknet_api::block::BlockNumber; +use std::num::NonZeroU32; use url::Url; #[derive(Debug, Clone)] @@ -36,6 +37,6 @@ pub struct ValidatedForkConfig { #[derive(Debug, Clone, PartialEq, Deserialize)] pub struct FuzzerConfig { - pub fuzzer_runs: u32, + pub fuzzer_runs: NonZeroU32, pub fuzzer_seed: u64, } diff --git a/crates/forge-runner/src/fuzzer.rs b/crates/forge-runner/src/fuzzer.rs index 65982536c7..bd85de9c08 100644 --- a/crates/forge-runner/src/fuzzer.rs +++ b/crates/forge-runner/src/fuzzer.rs @@ -1,5 +1,5 @@ use crate::fuzzer::arguments::CairoType; -use anyhow::Result; +use anyhow::{Ok, Result}; use rand::rngs::StdRng; use rand::Rng; @@ -7,55 +7,60 @@ mod arguments; mod random; pub use random::RandomFuzzer; +use std::num::NonZeroU32; + +#[derive(Debug, Clone)] +pub struct FuzzerArg { + cairo_type: CairoType, + run_with_min_value: u32, + run_with_max_value: u32, +} #[derive(Debug, Clone)] pub struct RunParams { /// Arguments - arguments: Vec, + arguments: Vec, /// Total number of runs - total_runs: u32, + total_runs: NonZeroU32, /// Number of already executed runs executed_runs: u32, - /// Run in which an argument has a min value - /// e.g. `run_with_min_value_argument[0] = 5` - /// means that the first argument will have the lowest possible value in 5th run - run_with_min_value_for_argument: Vec, - /// Run in which argument has a max value - /// e.g. `run_with_max_value_for_argument[0] = 5` - /// means that the first argument will have the highest possible value in 5th run - run_with_max_value_for_argument: Vec, } impl RunParams { - pub fn from(rng: &mut StdRng, total_runs: u32, arguments: &[&str]) -> Result { - assert!(total_runs >= 3); - + pub fn from(rng: &mut StdRng, total_runs: NonZeroU32, arguments: &[&str]) -> Result { let arguments = arguments .iter() - .map(|arg| CairoType::from_name(arg)) - .collect::>>()?; + .map(|arg| -> Result { + let argument = CairoType::from_name(arg)?; + if total_runs.get() >= 3 { + let run_with_min_value = rng.gen_range(1..=total_runs.get()); + let run_with_max_value = rng.gen_range(1..=total_runs.get()); - let run_with_min_value_for_argument: Vec = (0..arguments.len()) - .map(|_| rng.gen_range(1..=total_runs)) - .collect(); - let run_with_max_value_for_argument: Vec = run_with_min_value_for_argument - .iter() - .map(|&run_with_min| { - let run_with_max = rng.gen_range(1..=total_runs); - if run_with_max == run_with_min { - run_with_min % total_runs + 1 + let run_with_max_value = if run_with_max_value == run_with_min_value { + run_with_min_value % total_runs.get() + 1 + } else { + run_with_max_value + }; + + Ok(FuzzerArg { + cairo_type: argument, + run_with_max_value, + run_with_min_value, + }) } else { - run_with_max + Ok(FuzzerArg { + cairo_type: argument, + run_with_max_value: u32::MAX, + run_with_min_value: u32::MAX, + }) } }) - .collect(); + .collect::>>()?; Ok(Self { arguments, total_runs, executed_runs: 0, - run_with_min_value_for_argument, - run_with_max_value_for_argument, }) } } diff --git a/crates/forge-runner/src/fuzzer/arguments.rs b/crates/forge-runner/src/fuzzer/arguments.rs index 6e292adbd6..08fbd259e4 100644 --- a/crates/forge-runner/src/fuzzer/arguments.rs +++ b/crates/forge-runner/src/fuzzer/arguments.rs @@ -6,7 +6,7 @@ use num_traits::{One, Zero}; use rand::prelude::StdRng; use std::ops::{Add, Shl, Shr, Sub}; -#[derive(Debug, Clone)] +#[derive(Debug, Copy, Clone)] pub enum CairoType { U8, U16, @@ -17,35 +17,24 @@ pub enum CairoType { Felt252, } -pub trait Argument { - fn low(&self) -> BigUint; - fn high(&self) -> BigUint; - fn gen(&self, rng: &mut StdRng) -> Vec; - fn min(&self) -> Vec; - fn max(&self) -> Vec; -} - -impl Argument for CairoType { - fn low(&self) -> BigUint { +impl CairoType { + pub fn low() -> BigUint { BigUint::zero() } - fn high(&self) -> BigUint { + pub fn high(self) -> BigUint { match self { CairoType::U8 => BigUint::from(u8::MAX).add(BigUint::one()), CairoType::U16 => BigUint::from(u16::MAX).add(BigUint::one()), CairoType::U32 => BigUint::from(u32::MAX).add(BigUint::one()), CairoType::U64 => BigUint::from(u64::MAX).add(BigUint::one()), CairoType::U128 => BigUint::from(u128::MAX).add(BigUint::one()), - CairoType::U256 => { - let max = BigUint::from(1_u32); - max.shl(256) - } + CairoType::U256 => BigUint::from(1_u32).shl(256), CairoType::Felt252 => Felt252::prime(), } } - fn gen(&self, rng: &mut StdRng) -> Vec { + pub fn gen(self, rng: &mut StdRng) -> Vec { match self { CairoType::U8 | CairoType::U16 @@ -54,29 +43,29 @@ impl Argument for CairoType { | CairoType::U128 | CairoType::Felt252 => { vec![Felt252::from( - rng.gen_biguint_range(&self.low(), &self.high()), + rng.gen_biguint_range(&Self::low(), &self.high()), )] } CairoType::U256 => { - let val = rng.gen_biguint_range(&self.low(), &self.high()); + let val = rng.gen_biguint_range(&Self::low(), &self.high()); u256_to_felt252(val) } } } - fn min(&self) -> Vec { + pub fn min(self) -> Vec { match self { CairoType::U8 | CairoType::U16 | CairoType::U32 | CairoType::U64 | CairoType::U128 - | CairoType::Felt252 => vec![Felt252::from(self.low())], - CairoType::U256 => vec![Felt252::from(self.low()), Felt252::from(self.low())], + | CairoType::Felt252 => vec![Felt252::from(Self::low())], + CairoType::U256 => vec![Felt252::from(Self::low()), Felt252::from(Self::low())], } } - fn max(&self) -> Vec { + pub fn max(self) -> Vec { match self { CairoType::U8 | CairoType::U16 diff --git a/crates/forge-runner/src/fuzzer/random.rs b/crates/forge-runner/src/fuzzer/random.rs index 8498cf7ac2..a9e08d8fd2 100644 --- a/crates/forge-runner/src/fuzzer/random.rs +++ b/crates/forge-runner/src/fuzzer/random.rs @@ -1,9 +1,9 @@ -use crate::fuzzer::arguments::Argument; use crate::fuzzer::RunParams; use anyhow::Result; use cairo_felt::Felt252; use rand::prelude::StdRng; use rand::SeedableRng; +use std::num::NonZeroU32; #[derive(Debug, Clone)] pub struct RandomFuzzer { @@ -12,7 +12,7 @@ pub struct RandomFuzzer { } impl RandomFuzzer { - pub fn create(seed: u64, total_runs: u32, arguments: &[&str]) -> Result { + pub fn create(seed: u64, total_runs: NonZeroU32, arguments: &[&str]) -> Result { let mut rng = StdRng::seed_from_u64(seed); let run_params = RunParams::from(&mut rng, total_runs, arguments)?; @@ -20,56 +20,60 @@ impl RandomFuzzer { } pub fn next_args(&mut self) -> Vec { - assert!(self.run_params.executed_runs < self.run_params.total_runs); + assert!(self.run_params.executed_runs < self.run_params.total_runs.get()); self.next_run(); - let mut args = vec![]; - - for (index, argument) in self.run_params.arguments.iter().enumerate() { - if self.is_run_with_min_value_for_arg(index) { - args.extend(argument.min()); - } else if self.is_run_with_max_value_for_arg(index) { - args.extend(argument.max()); - } else { - args.extend(argument.gen(&mut self.rng)); - } - } - - args + self.run_params + .arguments + .iter() + .flat_map(|argument| { + let current_run = self.run_params.executed_runs; + + if argument.run_with_min_value == current_run { + argument.cairo_type.min() + } else if argument.run_with_max_value == current_run { + argument.cairo_type.max() + } else { + argument.cairo_type.gen(&mut self.rng) + } + }) + .collect() } fn next_run(&mut self) { self.run_params.executed_runs += 1; } - - fn is_run_with_min_value_for_arg(&self, arg_number: usize) -> bool { - let current_run = self.run_params.executed_runs; - self.run_params.run_with_min_value_for_argument[arg_number] == current_run - } - - fn is_run_with_max_value_for_arg(&self, arg_number: usize) -> bool { - let current_run = self.run_params.executed_runs; - self.run_params.run_with_max_value_for_argument[arg_number] == current_run - } } #[cfg(test)] mod tests { use super::*; - use crate::fuzzer::arguments::CairoType; + use crate::fuzzer::{arguments::CairoType, FuzzerArg}; use num_bigint::BigUint; use num_traits::Zero; use rand::{thread_rng, RngCore}; + impl FuzzerArg { + pub fn new( + cairo_type: CairoType, + run_with_min_value: u32, + run_with_max_value: u32, + ) -> Self { + Self { + cairo_type, + run_with_min_value, + run_with_max_value, + } + } + } + impl Default for RunParams { fn default() -> Self { Self { arguments: vec![], - total_runs: 256, + total_runs: NonZeroU32::new(256).unwrap(), executed_runs: 0, - run_with_min_value_for_argument: vec![], - run_with_max_value_for_argument: vec![], } } } @@ -90,10 +94,12 @@ mod tests { #[test] fn fuzzer_generates_different_values() { let run_params = RunParams { - run_with_max_value_for_argument: vec![3, 3, 3], - run_with_min_value_for_argument: vec![4, 4, 4], - arguments: vec![CairoType::Felt252, CairoType::U256, CairoType::U32], - total_runs: 10, + arguments: vec![ + FuzzerArg::new(CairoType::Felt252, 4, 3), + FuzzerArg::new(CairoType::U256, 4, 3), + FuzzerArg::new(CairoType::U32, 4, 3), + ], + total_runs: NonZeroU32::new(10).unwrap(), ..Default::default() }; let mut fuzzer = RandomFuzzer { @@ -113,27 +119,28 @@ mod tests { #[test] fn run_with_max_value() { let run_params = RunParams { - run_with_max_value_for_argument: vec![1], - run_with_min_value_for_argument: vec![2], - total_runs: 10, + arguments: vec![FuzzerArg::new(CairoType::Felt252, 2, 1)], + total_runs: NonZeroU32::new(10).unwrap(), ..Default::default() }; let mut fuzzer = RandomFuzzer { rng: StdRng::seed_from_u64(1234), run_params, }; + let current_run = fuzzer.run_params.executed_runs; + assert!(fuzzer.run_params.arguments[0].run_with_max_value != current_run); - assert!(!fuzzer.is_run_with_max_value_for_arg(0)); fuzzer.next_args(); - assert!(fuzzer.is_run_with_max_value_for_arg(0)); + + let current_run = fuzzer.run_params.executed_runs; + assert!(fuzzer.run_params.arguments[0].run_with_max_value == current_run); } #[test] fn run_with_min_value() { let run_params = RunParams { - run_with_max_value_for_argument: vec![2], - run_with_min_value_for_argument: vec![1], - total_runs: 10, + total_runs: NonZeroU32::new(10).unwrap(), + arguments: vec![FuzzerArg::new(CairoType::Felt252, 1, 2)], ..Default::default() }; let mut fuzzer = RandomFuzzer { @@ -141,18 +148,32 @@ mod tests { run_params, }; - assert!(!fuzzer.is_run_with_min_value_for_arg(0)); + let current_run = fuzzer.run_params.executed_runs; + assert!(fuzzer.run_params.arguments[0].run_with_min_value != current_run); + fuzzer.next_args(); - assert!(fuzzer.is_run_with_min_value_for_arg(0)); + + let current_run = fuzzer.run_params.executed_runs; + assert!(fuzzer.run_params.arguments[0].run_with_min_value == current_run); } #[test] fn using_seed_consistent_result() { let seed = thread_rng().next_u64(); - let mut fuzzer = RandomFuzzer::create(seed, 3, &["felt252", "felt252", "felt252"]).unwrap(); + let mut fuzzer = RandomFuzzer::create( + seed, + NonZeroU32::new(3).unwrap(), + &["felt252", "felt252", "felt252"], + ) + .unwrap(); let values = fuzzer.next_args(); - let mut fuzzer = RandomFuzzer::create(seed, 3, &["felt252", "felt252", "felt252"]).unwrap(); + let mut fuzzer = RandomFuzzer::create( + seed, + NonZeroU32::new(3).unwrap(), + &["felt252", "felt252", "felt252"], + ) + .unwrap(); let values_from_seed = fuzzer.next_args(); assert_eq!(values, values_from_seed); @@ -161,7 +182,7 @@ mod tests { #[test] fn min_and_max_used_at_least_once_for_each_arg() { let seed = thread_rng().next_u64(); - let runs_number = 10; + let runs_number = NonZeroU32::new(10).unwrap(); let arguments = vec!["felt252", "felt252", "felt252"]; let args_number = arguments.len(); @@ -170,7 +191,7 @@ mod tests { let mut min_used = vec![false; args_number]; let mut max_used = vec![false; args_number]; - for _ in 1..=runs_number { + for _ in 1..=runs_number.get() { let values = fuzzer.next_args(); for (i, value) in values.iter().enumerate() { assert!( @@ -190,7 +211,11 @@ mod tests { #[test] fn create_fuzzer_from_invalid_arguments() { - let result = RandomFuzzer::create(1234, 512, &["felt252", "invalid", "args"]); + let result = RandomFuzzer::create( + 1234, + NonZeroU32::new(512).unwrap(), + &["felt252", "invalid", "args"], + ); let err = result.unwrap_err(); assert_eq!( @@ -198,4 +223,14 @@ mod tests { "Tried to use incorrect type for fuzzing. Type = invalid is not supported" ); } + #[test] + fn fuzzer_less_than_3_runs() { + for runs in 1..2 { + let result = RandomFuzzer::create(1234, NonZeroU32::new(runs).unwrap(), &["felt252"]); + let mut fuzzer = result.unwrap(); + + // just check if it panics + fuzzer.next_args(); + } + } } diff --git a/crates/forge-runner/src/lib.rs b/crates/forge-runner/src/lib.rs index 2442b68f3a..464c9c6fa0 100644 --- a/crates/forge-runner/src/lib.rs +++ b/crates/forge-runner/src/lib.rs @@ -20,6 +20,7 @@ use profiler_api::run_profiler; use smol_str::SmolStr; use std::collections::HashMap; +use std::num::NonZeroU32; use std::sync::Arc; use test_case_summary::{AnyTestCaseSummary, Fuzzing}; use tokio::sync::mpsc::{channel, Sender}; @@ -88,7 +89,7 @@ impl ExecutionDataToSave { pub struct RunnerConfig { pub workspace_root: Utf8PathBuf, pub exit_first: bool, - pub fuzzer_runs: u32, + pub fuzzer_runs: NonZeroU32, pub fuzzer_seed: u64, pub detailed_resources: bool, pub execution_data_to_save: ExecutionDataToSave, @@ -101,7 +102,7 @@ impl RunnerConfig { pub fn new( workspace_root: Utf8PathBuf, exit_first: bool, - fuzzer_runs: u32, + fuzzer_runs: NonZeroU32, fuzzer_seed: u64, detailed_resources: bool, save_trace_data: bool, @@ -324,7 +325,7 @@ fn run_with_fuzzing( let mut tasks = FuturesUnordered::new(); - for _ in 1..=fuzzer_runs { + for _ in 1..=fuzzer_runs.get() { let args = fuzzer.next_args(); tasks.push(run_fuzz_test( @@ -368,7 +369,7 @@ fn run_with_fuzzing( // Because we execute tests parallel, it's possible to // get Passed after Skipped. To treat fuzzing a test as Passed // we have to ensure that all fuzzing subtests Passed - if runs != fuzzer_runs { + if runs != fuzzer_runs.get() { return Ok(TestCaseSummary::Skipped {}); }; }; diff --git a/crates/forge/src/main.rs b/crates/forge/src/main.rs index 0025055a50..d8dc46f9e0 100644 --- a/crates/forge/src/main.rs +++ b/crates/forge/src/main.rs @@ -1,4 +1,4 @@ -use anyhow::{anyhow, bail, Context, Result}; +use anyhow::{anyhow, Context, Result}; use camino::Utf8Path; use clap::{Parser, Subcommand, ValueEnum}; use configuration::load_package_config; @@ -23,6 +23,7 @@ use forge::block_number_map::BlockNumberMap; use semver::{Comparator, Op, Version, VersionReq}; use shared::print::print_as_warning; use std::env; +use std::num::NonZeroU32; use std::sync::Arc; use std::thread::available_parallelism; use tokio::runtime::Builder; @@ -30,7 +31,9 @@ use universal_sierra_compiler_api::UniversalSierraCompilerCommand; mod init; -const FUZZER_RUNS_DEFAULT: u32 = 256; +fn default_fuzzer_runs() -> NonZeroU32 { + NonZeroU32::new(256).unwrap() +} #[derive(Parser, Debug)] #[command(version)] @@ -80,8 +83,8 @@ struct TestArgs { packages_filter: PackagesFilter, /// Number of fuzzer runs - #[arg(short = 'r', long, value_parser = validate_fuzzer_runs_value)] - fuzzer_runs: Option, + #[arg(short = 'r', long)] + fuzzer_runs: Option, /// Seed for the fuzzer #[arg(short = 's', long)] fuzzer_seed: Option, @@ -118,16 +121,6 @@ struct TestArgs { max_n_steps: Option, } -fn validate_fuzzer_runs_value(val: &str) -> Result { - let parsed_val: u32 = val - .parse() - .map_err(|_| anyhow!("Failed to parse {val} as u32"))?; - if parsed_val < 3 { - bail!("Number of fuzzer runs must be greater than or equal to 3") - } - Ok(parsed_val) -} - fn extract_failed_tests( tests_summaries: Vec, ) -> impl Iterator { @@ -148,7 +141,7 @@ fn extract_failed_tests( fn combine_configs( workspace_root: &Utf8Path, exit_first: bool, - fuzzer_runs: Option, + fuzzer_runs: Option, fuzzer_seed: Option, detailed_resources: bool, save_trace_data: bool, @@ -161,7 +154,7 @@ fn combine_configs( exit_first || forge_config.exit_first, fuzzer_runs .or(forge_config.fuzzer_runs) - .unwrap_or(FUZZER_RUNS_DEFAULT), + .unwrap_or(default_fuzzer_runs()), fuzzer_seed .or(forge_config.fuzzer_seed) .unwrap_or_else(|| thread_rng().next_u64()), @@ -384,7 +377,7 @@ mod tests { RunnerConfig::new( workspace_root, false, - FUZZER_RUNS_DEFAULT, + default_fuzzer_runs(), config.fuzzer_seed, false, false, @@ -399,7 +392,7 @@ mod tests { let config_from_scarb = ForgeConfig { exit_first: true, fork: vec![], - fuzzer_runs: Some(1234), + fuzzer_runs: Some(NonZeroU32::new(1234).unwrap()), fuzzer_seed: Some(500), detailed_resources: true, save_trace_data: true, @@ -424,7 +417,7 @@ mod tests { RunnerConfig::new( workspace_root, true, - 1234, + NonZeroU32::new(1234).unwrap(), 500, true, true, @@ -441,7 +434,7 @@ mod tests { let config_from_scarb = ForgeConfig { exit_first: false, fork: vec![], - fuzzer_runs: Some(1234), + fuzzer_runs: Some(NonZeroU32::new(1234).unwrap()), fuzzer_seed: Some(1000), detailed_resources: false, save_trace_data: false, @@ -451,7 +444,7 @@ mod tests { let config = combine_configs( &workspace_root, true, - Some(100), + Some(NonZeroU32::new(100).unwrap()), Some(32), true, true, @@ -465,7 +458,7 @@ mod tests { RunnerConfig::new( workspace_root, true, - 100, + NonZeroU32::new(100).unwrap(), 32, true, true, diff --git a/crates/forge/src/scarb/config.rs b/crates/forge/src/scarb/config.rs index 9febe6f5c0..379966010a 100644 --- a/crates/forge/src/scarb/config.rs +++ b/crates/forge/src/scarb/config.rs @@ -2,7 +2,10 @@ use crate::compiled_raw::RawForkParams; use anyhow::{bail, Result}; use itertools::Itertools; use serde::Deserialize; -use std::collections::{HashMap, HashSet}; +use std::{ + collections::{HashMap, HashSet}, + num::NonZeroU32, +}; #[allow(clippy::module_name_repetitions)] #[allow(clippy::struct_excessive_bools)] @@ -11,7 +14,7 @@ pub struct ForgeConfig { /// Should runner exit after first failed test pub exit_first: bool, /// How many runs should fuzzer execute - pub fuzzer_runs: Option, + pub fuzzer_runs: Option, /// Seed to be used by fuzzer pub fuzzer_seed: Option, /// Display more detailed info about used resources @@ -57,7 +60,7 @@ pub(crate) struct RawForgeConfig { /// Should runner exit after first failed test pub exit_first: bool, /// How many runs should fuzzer execute - pub fuzzer_runs: Option, + pub fuzzer_runs: Option, /// Seed to be used by fuzzer pub fuzzer_seed: Option, #[serde(default)] diff --git a/crates/forge/test_utils/src/running_tests.rs b/crates/forge/test_utils/src/running_tests.rs index 26906f853f..eb88053720 100644 --- a/crates/forge/test_utils/src/running_tests.rs +++ b/crates/forge/test_utils/src/running_tests.rs @@ -7,6 +7,7 @@ use forge_runner::contracts_data::ContractsData; use forge_runner::test_crate_summary::TestCrateSummary; use forge_runner::{RunnerConfig, RunnerParams}; use shared::command::CommandExt; +use std::num::NonZeroU32; use std::path::PathBuf; use std::process::Command; use std::process::Stdio; @@ -33,7 +34,7 @@ pub fn run_test_case(test: &TestCase) -> Vec { Arc::new(RunnerConfig::new( Utf8PathBuf::from_path_buf(PathBuf::from(tempdir().unwrap().path())).unwrap(), false, - 256, + NonZeroU32::new(256).unwrap(), 12345, false, false, diff --git a/crates/forge/tests/e2e/fuzzing.rs b/crates/forge/tests/e2e/fuzzing.rs index ce611d1beb..460063d84d 100644 --- a/crates/forge/tests/e2e/fuzzing.rs +++ b/crates/forge/tests/e2e/fuzzing.rs @@ -146,7 +146,7 @@ fn fuzzing_incorrect_runs() { assert_stderr_contains( output, indoc! {r" - error: invalid value '0' for '--fuzzer-runs ': Number of fuzzer runs must be greater than or equal to 3 + error: invalid value '0' for '--fuzzer-runs ': number would be zero for non-zero type For more information, try '--help'. "}, diff --git a/crates/forge/tests/integration/setup_fork.rs b/crates/forge/tests/integration/setup_fork.rs index 20daee461c..33f495ef92 100644 --- a/crates/forge/tests/integration/setup_fork.rs +++ b/crates/forge/tests/integration/setup_fork.rs @@ -123,7 +123,7 @@ fn fork_aliased_decorator() { Arc::new(RunnerConfig::new( Utf8PathBuf::from_path_buf(PathBuf::from(tempdir().unwrap().path())).unwrap(), false, - 256, + 256.try_into().unwrap(), 12345, false, false,