feat(cheatcodes): vm.skip(bool) for skipping tests (#5205)

* feat(abi): add skip(bool) * feat: skip impl * feat: make skip only work at test level * feat: rewrite test runner to use status enum instead of bool * feat: simple tests * feat: works with fuzz tests * feat: works for invariant * chore: remove println * chore: clippy * chore: clippy * chore: prioritize skip decoding over abi decoding * chore: handle skips on invariant & fuzz tests more gracefully * feat: add skipped to test results * chore: clippy * fix: fixtures
foundry-rs · Jun 28, 2023 · ac5d367 · ac5d367
1 parent 3ae4c4b
commit ac5d367
Show file tree

Hide file tree

Showing 20 changed files with 254 additions and 41 deletions.
diff --git a/abi/abi/HEVM.sol b/abi/abi/HEVM.sol
@@ -71,6 +71,7 @@ expectRevert(bytes)
 expectRevert(bytes4)
 record()
 accesses(address)(bytes32[],bytes32[])
+skip(bool)
 
 recordLogs()
 getRecordedLogs()(Log[])

diff --git a/abi/src/bindings/hevm.rs b/abi/src/bindings/hevm.rs
diff --git a/cli/src/cmd/forge/test/mod.rs b/cli/src/cmd/forge/test/mod.rs
@@ -13,7 +13,7 @@ use forge::{
     decode::decode_console_logs,
     executor::inspector::CheatsConfig,
     gas_report::GasReport,
-    result::{SuiteResult, TestKind, TestResult},
+    result::{SuiteResult, TestKind, TestResult, TestStatus},
     trace::{
         identifier::{EtherscanIdentifier, LocalTraceIdentifier, SignaturesIdentifier},
         CallTraceDecoderBuilder, TraceKind,
@@ -348,12 +348,16 @@ impl TestOutcome {
 
     /// Iterator over all succeeding tests and their names
     pub fn successes(&self) -> impl Iterator<Item = (&String, &TestResult)> {
-        self.tests().filter(|(_, t)| t.success)
+        self.tests().filter(|(_, t)| t.status == TestStatus::Success)
     }
 
     /// Iterator over all failing tests and their names
     pub fn failures(&self) -> impl Iterator<Item = (&String, &TestResult)> {
-        self.tests().filter(|(_, t)| !t.success)
+        self.tests().filter(|(_, t)| t.status == TestStatus::Failure)
+    }
+
+    pub fn skips(&self) -> impl Iterator<Item = (&String, &TestResult)> {
+        self.tests().filter(|(_, t)| t.status == TestStatus::Skipped)
     }
 
     /// Iterator over all tests and their names
@@ -418,18 +422,21 @@ impl TestOutcome {
         let failed = self.failures().count();
         let result = if failed == 0 { Paint::green("ok") } else { Paint::red("FAILED") };
         format!(
-            "Test result: {}. {} passed; {} failed; finished in {:.2?}",
+            "Test result: {}. {} passed; {} failed; {} skipped; finished in {:.2?}",
             result,
             self.successes().count(),
             failed,
+            self.skips().count(),
             self.duration()
         )
     }
 }
 
 fn short_test_result(name: &str, result: &TestResult) {
-    let status = if result.success {
+    let status = if result.status == TestStatus::Success {
         Paint::green("[PASS]".to_string())
+    } else if result.status == TestStatus::Skipped {
+        Paint::yellow("[SKIP]".to_string())
     } else {
         let reason = result
             .reason
@@ -553,7 +560,7 @@ fn test(
                 short_test_result(name, result);
 
                 // If the test failed, we want to stop processing the rest of the tests
-                if fail_fast && !result.success {
+                if fail_fast && result.status == TestStatus::Failure {
                     break 'outer
                 }
 
@@ -596,10 +603,12 @@ fn test(
                             // tests At verbosity level 5, we display
                             // all traces for all tests
                             TraceKind::Setup => {
-                                (verbosity >= 5) || (verbosity == 4 && !result.success)
+                                (verbosity >= 5) ||
+                                    (verbosity == 4 && result.status == TestStatus::Failure)
                             }
                             TraceKind::Execution => {
-                                verbosity > 3 || (verbosity == 3 && !result.success)
+                                verbosity > 3 ||
+                                    (verbosity == 3 && result.status == TestStatus::Failure)
                             }
                             _ => false,
                         };

diff --git a/cli/tests/fixtures/can_check_snapshot.stdout b/cli/tests/fixtures/can_check_snapshot.stdout
@@ -4,4 +4,4 @@ Compiler run successful!
 
 Running 1 test for src/ATest.t.sol:ATest
 [PASS] testExample() (gas: 168)
-Test result: ok. 1 passed; 0 failed; finished in 4.42ms
+Test result: ok. 1 passed; 0 failed; 0 skipped; finished in 4.42ms
diff --git a/cli/tests/fixtures/can_run_test_in_custom_test_folder.stdout b/cli/tests/fixtures/can_run_test_in_custom_test_folder.stdout
@@ -4,4 +4,4 @@ Compiler run successful!
 
 Running 1 test for src/nested/forge-tests/MyTest.t.sol:MyTest
 [PASS] testTrue() (gas: 168)
-Test result: ok. 1 passed; 0 failed; finished in 2.93ms
+Test result: ok. 1 passed; 0 failed; 0 skipped; finished in 2.93ms
diff --git a/cli/tests/fixtures/can_test_repeatedly.stdout b/cli/tests/fixtures/can_test_repeatedly.stdout
@@ -3,4 +3,4 @@ No files changed, compilation skipped
 Running 2 tests for test/Counter.t.sol:CounterTest
 [PASS] testIncrement() (gas: 28334)
 [PASS] testSetNumber(uint256) (runs: 256, μ: 26521, ~: 28387)
-Test result: ok. 2 passed; 0 failed; finished in 9.42ms
+Test result: ok. 2 passed; 0 failed; 0 skipped; finished in 9.42ms
diff --git a/cli/tests/fixtures/can_use_libs_in_multi_fork.stdout b/cli/tests/fixtures/can_use_libs_in_multi_fork.stdout
@@ -4,4 +4,4 @@ Compiler run successful!
 
 Running 1 test for test/Contract.t.sol:ContractTest
 [PASS] test() (gas: 70373)
-Test result: ok. 1 passed; 0 failed; finished in 3.21s
+Test result: ok. 1 passed; 0 failed; 0 skipped; finished in 3.21s
diff --git a/cli/tests/fixtures/runs_tests_exactly_once_with_changed_versions.0.8.10.stdout b/cli/tests/fixtures/runs_tests_exactly_once_with_changed_versions.0.8.10.stdout
@@ -4,4 +4,4 @@ Compiler run successful!
 
 Running 1 test for src/Contract.t.sol:ContractTest
 [PASS] testExample() (gas: 190)
-Test result: ok. 1 passed; 0 failed; finished in 1.89ms
+Test result: ok. 1 passed; 0 failed; 0 skipped; finished in 1.89ms
diff --git a/cli/tests/fixtures/runs_tests_exactly_once_with_changed_versions.0.8.13.stdout b/cli/tests/fixtures/runs_tests_exactly_once_with_changed_versions.0.8.13.stdout
@@ -4,4 +4,4 @@ Compiler run successful!
 
 Running 1 test for src/Contract.t.sol:ContractTest
 [PASS] testExample() (gas: 190)
-Test result: ok. 1 passed; 0 failed; finished in 1.89ms
+Test result: ok. 1 passed; 0 failed; 0 skipped; finished in 1.89ms
diff --git a/evm/src/decode.rs b/evm/src/decode.rs
@@ -2,6 +2,7 @@
 use crate::{
     abi::ConsoleEvents::{self, *},
     error::ERROR_PREFIX,
+    executor::inspector::cheatcodes::util::MAGIC_SKIP_BYTES,
 };
 use ethers::{
     abi::{decode, AbiDecode, Contract as Abi, ParamType, RawLog, Token},
@@ -161,6 +162,10 @@ pub fn decode_revert(
             eyre::bail!("Unknown error selector")
         }
         _ => {
+            // See if the revert is caused by a skip() call.
+            if err == MAGIC_SKIP_BYTES {
+                return Ok("SKIPPED".to_string())
+            }
             // try to decode a custom error if provided an abi
             if let Some(abi) = maybe_abi {
                 for abi_error in abi.errors() {
@@ -178,7 +183,6 @@ pub fn decode_revert(
                     }
                 }
             }
-
             // optimistically try to decode as string, unknown selector or `CheatcodeError`
             String::decode(err)
                 .ok()

diff --git a/evm/src/executor/inspector/cheatcodes/mod.rs b/evm/src/executor/inspector/cheatcodes/mod.rs
@@ -1,7 +1,7 @@
 use self::{
     env::Broadcast,
     expect::{handle_expect_emit, handle_expect_revert, ExpectedCallType},
-    util::{check_if_fixed_gas_limit, process_create, BroadcastableTransactions},
+    util::{check_if_fixed_gas_limit, process_create, BroadcastableTransactions, MAGIC_SKIP_BYTES},
 };
 use crate::{
     abi::HEVMCalls,
@@ -115,6 +115,9 @@ pub struct Cheatcodes {
     /// Rememebered private keys
     pub script_wallets: Vec<LocalWallet>,
 
+    /// Whether the skip cheatcode was activated
+    pub skip: bool,
+
     /// Prank information
     pub prank: Option<Prank>,
 
@@ -744,6 +747,14 @@ where
             return (status, remaining_gas, retdata)
         }
 
+        if data.journaled_state.depth() == 0 && self.skip {
+            return (
+                InstructionResult::Revert,
+                remaining_gas,
+                Error::custom_bytes(MAGIC_SKIP_BYTES).encode_error().0,
+            )
+        }
+
         // Clean up pranks
         if let Some(prank) = &self.prank {
             if data.journaled_state.depth() == prank.depth {

diff --git a/evm/src/executor/inspector/cheatcodes/util.rs b/evm/src/executor/inspector/cheatcodes/util.rs
@@ -1,4 +1,4 @@
-use super::{ensure, fmt_err, Cheatcodes, Result};
+use super::{ensure, fmt_err, Cheatcodes, Error, Result};
 use crate::{
     abi::HEVMCalls,
     executor::backend::{
@@ -40,6 +40,8 @@ pub const DEFAULT_CREATE2_DEPLOYER: H160 = H160([
     78, 89, 180, 72, 71, 179, 121, 87, 133, 136, 146, 12, 167, 143, 191, 38, 192, 180, 149, 108,
 ]);
 
+pub const MAGIC_SKIP_BYTES: &[u8] = b"FOUNDRY::SKIP";
+
 /// Helps collecting transactions from different forks.
 #[derive(Debug, Clone, Default)]
 pub struct BroadcastableTransaction {
@@ -197,6 +199,21 @@ pub fn parse(s: &str, ty: &ParamType) -> Result {
         .map_err(|e| fmt_err!("Failed to parse `{s}` as type `{ty}`: {e}"))
 }
 
+pub fn skip(state: &mut Cheatcodes, depth: u64, skip: bool) -> Result {
+    if !skip {
+        return Ok(b"".into())
+    }
+
+    // Skip should not work if called deeper than at test level.
+    // As we're not returning the magic skip bytes, this will cause a test failure.
+    if depth > 1 {
+        return Err(Error::custom("The skip cheatcode can only be used at test level"))
+    }
+
+    state.skip = true;
+    Err(Error::custom_bytes(MAGIC_SKIP_BYTES))
+}
+
 #[instrument(level = "error", name = "util", target = "evm::cheatcodes", skip_all)]
 pub fn apply<DB: Database>(
     state: &mut Cheatcodes,
@@ -253,6 +270,7 @@ pub fn apply<DB: Database>(
         HEVMCalls::ParseInt(inner) => parse(&inner.0, &ParamType::Int(256)),
         HEVMCalls::ParseBytes32(inner) => parse(&inner.0, &ParamType::FixedBytes(32)),
         HEVMCalls::ParseBool(inner) => parse(&inner.0, &ParamType::Bool),
+        HEVMCalls::Skip(inner) => skip(state, data.journaled_state.depth(), inner.0),
         _ => return None,
     })
 }

diff --git a/evm/src/executor/mod.rs b/evm/src/executor/mod.rs
@@ -320,7 +320,6 @@ impl Executor {
             value,
         );
         let call_result = self.call_raw_with_env(env)?;
-
         convert_call_result(abi, &func, call_result)
     }
 
@@ -642,6 +641,9 @@ pub enum EvmError {
     /// Error which occurred during ABI encoding/decoding
     #[error(transparent)]
     AbiError(#[from] ethers::contract::AbiError),
+    /// Error caused which occurred due to calling the skip() cheatcode.
+    #[error("Skipped")]
+    SkipError,
     /// Any other error.
     #[error(transparent)]
     Eyre(#[from] eyre::Error),
@@ -669,6 +671,7 @@ pub struct DeployResult {
 /// The result of a call.
 #[derive(Debug)]
 pub struct CallResult<D: Detokenize> {
+    pub skipped: bool,
     /// Whether the call reverted or not
     pub reverted: bool,
     /// The decoded result of the call
@@ -798,7 +801,6 @@ fn convert_executed_result(
             (halt_to_instruction_result(reason), 0_u64, gas_used, None)
         }
     };
-
     let stipend = calc_stipend(&env.tx.data, env.cfg.spec_id);
 
     let result = match out {
@@ -897,11 +899,15 @@ fn convert_call_result<D: Detokenize>(
                 script_wallets,
                 env,
                 breakpoints,
+                skipped: false,
             })
         }
         _ => {
             let reason = decode::decode_revert(result.as_ref(), abi, Some(status))
                 .unwrap_or_else(|_| format!("{status:?}"));
+            if reason == "SKIPPED" {
+                return Err(EvmError::SkipError)
+            }
             Err(EvmError::Execution(Box::new(ExecutionErr {
                 reverted,
                 reason,