feat(forge): support multiple forks (#1715)

* refactor: move backed to separate module

* refactor: move fork db to evm crate

* feat: design multifork

* feat: more multi handler work

* fix: use new paths

* describe cheatcodes

* chore: tune cheatcodes

* refactor: move in memory db to evm

* docs: add some docs

* refactor: redesign multifork backend

* feat: api improvements

* chore: bump revm

* docs: more backend docs

* feat: implement multifork creation

* style: simplify locking

* feat: add spawning

* feat: refactor backend types

* feat: complete fork api

* refactor: simplify TestFilter

* refactor: extract helper types

* refactor: restructure runner

* chore(clippy): make clippy happy

* refactor: extract types

* refactor: simplify create2 deployer fn

* cleanup

* fix: failing tests

* test: ensure solc finished successfully

* refactor: introduce more types

* feat: add a bunch of revm trait impls

* clean up types

* refactor: remove generic Inspector impl

* feat: introduce Backendtrait

* chore: remove old types

* refactor: rename Backend type

* refactor: rename DatbaseExt trait

* feat: integrate new Backend type

* revertuse refcell again

* refactor: change to dedicated fuzz backend

* refactor: refactor conversion

* chore: some cleanup

* refactor: extract Fuzzbackend wrapper

* feat: implement cheat codes

* feat: implement fork cheat codes

* refactor: make it compile again

* refactor: add shutdown routine to Multifork

* refactor: improve backend

* make everything compile again

* add auto impl

* add config

* refactor: update outdated code

* chore: cleanup some code

* chore: make it compile again

* test: update failing tests

* chore(clippy): make clippy happy

* test: add simple fork cheatcode tests

* refactor: use execute function

* refactor: move snapshot cheatcode impl to separate mod

* feat: store subroutine with snapshot

* feat: add subroutine to revert call

* feat: add Cheats config type

* work on snapshots

* docs: write additional revert docs

* feat: check for failures

* fix: make compile again

* refactor: rename

* refactor: unify backend code

* feat: resolve rpc aliases

* feat: periodically flush rpc cache

* chore: derive default

* chore: use revm head

* chore: bump revm

* build: use revm 1.6.0

* fix: use new revm data structures

* feat: add roll fork cheat codes

* feat: add rpc helper functions

* docs: document rpc endpoints table

* test: add rpc endpoint tests

* Delete run.rs

* work on roll fork

* refactor: use local fork ids as ints

* test: update fork test

* extend trait

* fix: migrate new revm api

* patch revm git

* use revm naming

* fix: rpc urls api

* fix: return encoded errors

* chore: rustfmt

* chore: rustfmt

* feat: update env when selecting fork

* fix: fix a ton of bugs

* fix: failing tests

* chore: rm unused cheat

* chore: rm unused types

* feat: add more util cheat codes

* style: simplify create select

* docs: update docs

* test: more fork tests

* add active fork test

* docs: update cheatcode docs

* fix: capture env in snapshot

* test: add snapshot tests

Co-authored-by: Georgios Konstantopoulos <me@gakonst.com>
Co-authored-by: Oliver Nordbjerg <hi@notbjerg.me>

Cargo.toml

@@ -61,4 +61,4 @@ debug = 0
 #ethers-providers = { path = "../ethers-rs/ethers-providers" }
 #ethers-signers = { path = "../ethers-rs/ethers-signers" }
 #ethers-etherscan = { path = "../ethers-rs/ethers-etherscan" }
-#ethers-solc = { path = "../ethers-rs/ethers-solc" }
+#ethers-solc = { path = "../ethers-rs/ethers-solc" }

cli/src/cmd/cast/run.rs

@@ -1,4 +1,4 @@
-use crate::{cmd::Cmd, utils, utils::consume_config_rpc_url};
+use crate::{cmd::Cmd, utils::consume_config_rpc_url};
 use cast::trace::{identifier::SignaturesIdentifier, CallTraceDecoder};
 use clap::Parser;
 use ethers::{
@@ -10,7 +10,7 @@ use ethers::{
 use forge::{
     debug::DebugArena,
     executor::{
-        builder::Backend, inspector::CheatsConfig, opts::EvmOpts, DeployResult, ExecutorBuilder,
+        inspector::CheatsConfig, opts::EvmOpts, Backend, DeployResult, ExecutorBuilder,
         RawCallResult,
     },
     trace::{identifier::EtherscanIdentifier, CallTraceArena, CallTraceDecoderBuilder, TraceKind},
@@ -75,8 +75,7 @@ impl RunArgs {
 
             // Set up the execution environment
             let env = evm_opts.evm_env().await;
-            let db =
-                Backend::new(utils::get_fork(&evm_opts, &config.rpc_storage_caching), &env).await;
+            let db = Backend::spawn(evm_opts.get_fork(env.clone()));
 
             let builder = ExecutorBuilder::default()
                 .with_config(env)
@@ -112,11 +111,7 @@ impl RunArgs {
 
             // Execute our transaction
             let mut result = {
-                executor.set_tracing(true).set_gas_limit(tx.gas);
-
-                if self.debug {
-                    executor.set_debugger(true);
-                }
+                executor.set_tracing(true).set_gas_limit(tx.gas).set_debugger(self.debug);
 
                 if let Some(to) = tx.to {
                     let RawCallResult { reverted, gas, traces, debug: run_debug, .. } =

cli/src/cmd/forge/coverage.rs

@@ -17,7 +17,7 @@ use forge::{
     coverage::{
         CoverageMap, CoverageReporter, DebugReporter, LcovReporter, SummaryReporter, Visitor,
     },
-    executor::opts::EvmOpts,
+    executor::{inspector::CheatsConfig, opts::EvmOpts},
     result::SuiteResult,
     trace::identifier::LocalTraceIdentifier,
     MultiContractRunnerBuilder,
@@ -236,16 +236,20 @@ impl CoverageArgs {
         let fuzzer = proptest::test_runner::TestRunner::new(cfg);
         let root = project.paths.root;
 
+        let env = evm_opts.evm_env_blocking();
+
         // Build the contract runner
         let evm_spec = utils::evm_spec(&config.evm_version);
         let mut runner = MultiContractRunnerBuilder::default()
             .fuzzer(fuzzer)
             .initial_balance(evm_opts.initial_balance)
             .evm_spec(evm_spec)
             .sender(evm_opts.sender)
-            .with_fork(utils::get_fork(&evm_opts, &config.rpc_storage_caching))
+            .with_fork(evm_opts.get_fork(env.clone()))
+            .with_cheats_config(CheatsConfig::new(&config, &evm_opts))
             .set_coverage(true)
-            .build(root.clone(), output, evm_opts)?;
+            .build(root.clone(), output, env, evm_opts)?;
+
         let (tx, rx) = channel::<(String, SuiteResult)>();
 
         // Set up identifier

cli/src/cmd/forge/script/executor.rs

@@ -1,16 +1,15 @@
+use super::*;
 use crate::{
-    cmd::{
-        forge::script::{sequence::TransactionWithMetadata, *},
-        needs_setup,
-    },
+    cmd::{forge::script::sequence::TransactionWithMetadata, needs_setup},
     utils,
 };
+use cast::executor::inspector::CheatsConfig;
 use ethers::{
     solc::artifacts::CompactContractBytecode,
     types::{transaction::eip2718::TypedTransaction, Address, U256},
 };
 use forge::{
-    executor::{builder::Backend, inspector::CheatsConfig, ExecutorBuilder},
+    executor::{Backend, ExecutorBuilder},
     trace::CallTraceDecoder,
 };
 use std::collections::VecDeque;
@@ -146,24 +145,16 @@ impl ScriptArgs {
     }
 
     /// Creates the Runner that drives script execution
-    async fn prepare_runner(
-        &self,
-        script_config: &ScriptConfig,
-        sender: Address,
-    ) -> ScriptRunner<Backend> {
+    async fn prepare_runner(&self, script_config: &ScriptConfig, sender: Address) -> ScriptRunner {
         let env = script_config.evm_opts.evm_env().await;
 
         // the db backend that serves all the data
-        let db = Backend::new(
-            utils::get_fork(&script_config.evm_opts, &script_config.config.rpc_storage_caching),
-            &env,
-        )
-        .await;
+        let db = Backend::spawn(script_config.evm_opts.get_fork(env.clone()));
 
         let executor = ExecutorBuilder::default()
             .with_cheatcodes(CheatsConfig::new(&script_config.config, &script_config.evm_opts))
             .with_config(env)
-            .with_spec(crate::utils::evm_spec(&script_config.config.evm_version))
+            .with_spec(utils::evm_spec(&script_config.config.evm_version))
             .with_gas_limit(script_config.evm_opts.gas_limit())
             .set_tracing(script_config.evm_opts.verbosity >= 3 || self.debug)
             .set_debugger(self.debug)

cli/src/cmd/forge/script/runner.rs

@@ -1,20 +1,20 @@
 use super::*;
 use ethers::types::{Address, Bytes, NameOrAddress, U256};
 use forge::{
-    executor::{CallResult, DatabaseRef, DeployResult, EvmError, Executor, RawCallResult},
+    executor::{CallResult, DeployResult, EvmError, Executor, RawCallResult},
     trace::{CallTraceArena, TraceKind},
     CALLER,
 };
 
 /// Drives script execution
-pub struct ScriptRunner<DB: DatabaseRef> {
-    pub executor: Executor<DB>,
+pub struct ScriptRunner {
+    pub executor: Executor,
     pub initial_balance: U256,
     pub sender: Address,
 }
 
-impl<DB: DatabaseRef> ScriptRunner<DB> {
-    pub fn new(executor: Executor<DB>, initial_balance: U256, sender: Address) -> Self {
+impl ScriptRunner {
+    pub fn new(executor: Executor, initial_balance: U256, sender: Address) -> Self {
         Self { executor, initial_balance, sender }
     }
 

cli/src/cmd/forge/test.rs

@@ -490,16 +490,19 @@ pub fn custom_run(args: TestArgs, include_fuzz_tests: bool) -> eyre::Result<Test
         evm_opts.verbosity = 3;
     }
 
+    let env = evm_opts.evm_env_blocking();
+
     // Prepare the test builder
     let evm_spec = utils::evm_spec(&config.evm_version);
+
     let mut runner = MultiContractRunnerBuilder::default()
         .fuzzer(fuzzer)
         .initial_balance(evm_opts.initial_balance)
         .evm_spec(evm_spec)
         .sender(evm_opts.sender)
-        .with_fork(utils::get_fork(&evm_opts, &config.rpc_storage_caching))
+        .with_fork(evm_opts.get_fork(env.clone()))
         .with_cheats_config(CheatsConfig::new(&config, &evm_opts))
-        .build(project.paths.root, output, evm_opts)?;
+        .build(project.paths.root, output, env, evm_opts)?;
 
     if args.debug.is_some() {
         filter.test_pattern = args.debug;

cli/src/utils.rs

@@ -6,12 +6,12 @@ use ethers::{
     types::U256,
     utils::format_units,
 };
-use forge::executor::{opts::EvmOpts, Fork, SpecId};
-use foundry_config::{cache::StorageCachingConfig, Config};
+use forge::executor::SpecId;
+use foundry_config::Config;
 use std::{
     future::Future,
     ops::Mul,
-    path::{Path, PathBuf},
+    path::Path,
     process::{Command, Output},
     str::FromStr,
     sync::Arc,
@@ -171,61 +171,6 @@ pub fn block_on<F: Future>(future: F) -> F::Output {
     rt.block_on(future)
 }
 
-/// Helper function that returns the [Fork] to use, if any.
-///
-/// storage caching for the [Fork] will be enabled if
-///   - `fork_url` is present
-///   - `fork_block_number` is present
-///   - [StorageCachingConfig] allows the `fork_url` +  chain id pair
-///   - storage is allowed (`no_storage_caching = false`)
-///
-/// If all these criteria are met, then storage caching is enabled and storage info will be written
-/// to [Config::foundry_cache_dir()]/<str(chainid)>/<block>/storage.json
-///
-/// for `mainnet` and `--fork-block-number 14435000` on mac the corresponding storage cache will be
-/// at `~/.foundry/cache/mainnet/14435000/storage.json`
-pub fn get_fork(evm_opts: &EvmOpts, config: &StorageCachingConfig) -> Option<Fork> {
-    /// Returns the path where the cache file should be stored
-    ///
-    /// or `None` if caching should not be enabled
-    ///
-    /// See also [ Config::foundry_block_cache_file()]
-    fn get_block_storage_path(
-        evm_opts: &EvmOpts,
-        config: &StorageCachingConfig,
-        chain_id: u64,
-    ) -> Option<PathBuf> {
-        if evm_opts.no_storage_caching {
-            // storage caching explicitly opted out of
-            return None
-        }
-        let url = evm_opts.fork_url.as_ref()?;
-        // cache only if block explicitly pinned
-        let block = evm_opts.fork_block_number?;
-
-        if config.enable_for_endpoint(url) && config.enable_for_chain_id(chain_id) {
-            return Config::foundry_block_cache_file(chain_id, block)
-        }
-
-        None
-    }
-
-    if let Some(ref url) = evm_opts.fork_url {
-        let chain_id = evm_opts.get_chain_id();
-        let cache_storage = get_block_storage_path(evm_opts, config, chain_id);
-        let fork = Fork {
-            url: url.clone(),
-            pin_block: evm_opts.fork_block_number,
-            cache_path: cache_storage,
-            chain_id,
-            initial_backoff: evm_opts.fork_retry_backoff.unwrap_or(50),
-        };
-        return Some(fork)
-    }
-
-    None
-}
-
 /// Conditionally print a message
 ///
 /// This macro accepts a predicate and the message to print if the predicate is tru