feat: track and cache context of each compiler invocation (#140)

ref foundry-rs/foundry#7379
ref foundry-rs/foundry#4981
ref foundry-rs/foundry#2704

The way Solidity assigns `source_id`s is simply by order in which
sources are passed in compiler input. Thus, if we have two sources
`A.sol` and `B.sol`, then on the first (non-cached) compiler run,
`A.sol` will get assigned ID 1 and `B.sol` will have ID 2.

Then, if we change `B.sol` slightly and recompile, it will be the only
source in the input, so it will have ID 1.

The same ID collisions are more often appearing on multi-version and
multi-compiler builds. After many cached runs such discrepancies result
in debugger basically displaying random sources.

This PR adds a way to link an artifact to the build info for input that
produced it, thus allowing us to track correct `source_id -> source`
mapping for cached artifacts.

I've added `BuildContext` which is the foundry context we are tracking
for each compiler invocation. It is getting inlined into `BuildInfo`,
and read along with cached artifacts. `BuildContext`s are indexed by the
same IDs `BuildInfo`s are, and each `ArtifactId` now has a reference to
`build_id` which produced it.

Build info now produced on every compiler run, however, it does not
include complete input and output unless `project.build_info` is true,
to avoid overhead while keeping our internal logic working correctly.

deny.toml

@@ -55,6 +55,7 @@ allow = [
  # https://github.com/briansmith/webpki/issues/148
  "LicenseRef-webpki",
  "BSL-1.0",
+ "Unicode-3.0",
 ]
 
 # Allow 1 or more licenses on a per-crate basis, so that particular licenses

src/artifact_output/mod.rs

@@ -6,6 +6,7 @@ use crate::{
  BytecodeObject, CompactBytecode, CompactContractBytecodeCow, CompactDeployedBytecode,
  FileToContractsMap, SourceFile,
  },
+ cache::CachedArtifact,
  compile::output::{contracts::VersionedContracts, sources::VersionedSourceFiles},
  error::Result,
  sourcemap::{SourceMap, SyntaxError},
@@ -40,6 +41,8 @@ pub struct ArtifactId {
  pub source: PathBuf,
  /// `solc` version that produced this artifact
  pub version: Version,
+ /// `solc` build id
+ pub build_id: String,
 }
 
 impl ArtifactId {
@@ -68,7 +71,7 @@ impl ArtifactId {
 
  /// Returns a `<source path>:<name>` slug that uniquely identifies an artifact
  pub fn identifier(&self) -> String {
- format!("{}:{}", self.source.to_string_lossy(), self.name)
+ format!("{}:{}", self.source.display(), self.name)
  }
 
  /// Returns a `<filename><version>:<name>` slug that identifies an artifact
@@ -93,6 +96,7 @@ pub struct ArtifactFile<T> {
  pub file: PathBuf,
  /// `solc` version that produced this artifact
  pub version: Version,
+ pub build_id: String,
 }
 
 impl<T: Serialize> ArtifactFile<T> {
@@ -274,6 +278,7 @@ impl<T> Artifacts<T> {
  name,
  source: source.clone(),
  version: artifact.version.clone(),
+ build_id: artifact.build_id.clone(),
  }
  .with_slashed_paths(),
  &artifact.artifact,
@@ -299,6 +304,7 @@ impl<T> Artifacts<T> {
  name,
  source: source.clone(),
  version: artifact.version,
+ build_id: artifact.build_id.clone(),
  }
  .with_slashed_paths(),
  artifact.artifact,
@@ -821,9 +827,9 @@ pub trait ArtifactOutput {
  // we reuse the path, this will make sure that even if there are conflicting
  // files (files for witch `T::output_file()` would return the same path) we use
  // consistent output paths
- if let Some(existing_artifact) = ctx.existing_artifact(file, name, version).cloned() {
+ if let Some(existing_artifact) = ctx.existing_artifact(file, name, version) {
  trace!("use existing artifact file {:?}", existing_artifact,);
- existing_artifact
+ existing_artifact.to_path_buf()
  } else {
  let path = if versioned {
  Self::output_file_versioned(file, name, version)
@@ -863,7 +869,7 @@ pub trait ArtifactOutput {
  .existing_artifacts
  .values()
  .flat_map(|artifacts| artifacts.values().flat_map(|artifacts| artifacts.values()))
- .map(|p| p.to_slash_lossy().to_lowercase())
+ .map(|a| a.path.to_slash_lossy().to_lowercase())
  .collect::<HashSet<_>>();
 
  let mut files = contracts.keys().collect::<Vec<_>>();
@@ -911,6 +917,7 @@ pub trait ArtifactOutput {
  artifact,
  file: artifact_path,
  version: contract.version.clone(),
+ build_id: contract.build_id.clone(),
  };
 
  artifacts
@@ -969,6 +976,7 @@ pub trait ArtifactOutput {
  artifact,
  file: artifact_path,
  version: source.version.clone(),
+ build_id: source.build_id.clone(),
  });
  }
  }
@@ -1021,7 +1029,8 @@ pub struct OutputContext<'a> {
  /// └── inner
  /// └── a.sol
  /// ```
- pub existing_artifacts: BTreeMap<&'a Path, &'a BTreeMap<String, BTreeMap<Version, PathBuf>>>,
+ pub existing_artifacts:
+ BTreeMap<&'a Path, &'a BTreeMap<String, BTreeMap<Version, CachedArtifact>>>,
 }
 
 // === impl OutputContext
@@ -1047,9 +1056,12 @@ impl<'a> OutputContext<'a> {
  file: impl AsRef<Path>,
  contract: &str,
  version: &Version,
- ) -> Option<&PathBuf> {
+ ) -> Option<&Path> {
  self.existing_artifacts.get(file.as_ref()).and_then(|contracts| {
- contracts.get(contract).and_then(|versions| versions.get(version))
+ contracts
+ .get(contract)
+ .and_then(|versions| versions.get(version))
+ .map(|a| a.path.as_path())
  })
  }
 }

src/artifacts/mod.rs

@@ -1981,7 +1981,14 @@ impl SourceFiles {
 #[cfg(test)]
 mod tests {
  use super::*;
- use crate::AggregatedCompilerOutput;
+ use crate::{
+ buildinfo::RawBuildInfo,
+ compilers::{
+ solc::{SolcCompiler, SolcVersionedInput},
+ CompilerInput,
+ },
+ AggregatedCompilerOutput,
+ };
  use alloy_primitives::Address;
 
  #[test]
@@ -2014,8 +2021,16 @@ mod tests {
  sources: Default::default(),
  };
 
- let mut aggregated = AggregatedCompilerOutput::default();
- aggregated.extend("0.8.12".parse().unwrap(), out_converted);
+ let v: Version = "0.8.12".parse().unwrap();
+ let input = SolcVersionedInput::build(
+ Default::default(),
+ Default::default(),
+ SolcLanguage::Solidity,
+ v.clone(),
+ );
+ let build_info = RawBuildInfo::new(&input, &out_converted, true).unwrap();
+ let mut aggregated = AggregatedCompilerOutput::<SolcCompiler>::default();
+ aggregated.extend(v, build_info, out_converted);
  assert!(!aggregated.is_unchanged());
  }
 

src/buildinfo.rs

@@ -1,11 +1,18 @@
 //! Represents an entire build
 
-use crate::{utils, SolcError};
+use crate::{
+ compilers::{CompilationError, CompilerInput, CompilerOutput, Language},
+ error::Result,
+ utils,
+};
 use alloy_primitives::hex;
 use md5::Digest;
 use semver::Version;
-use serde::{de::DeserializeOwned, ser::SerializeStruct, Deserialize, Serialize, Serializer};
-use std::{cell::RefCell, path::Path, rc::Rc};
+use serde::{de::DeserializeOwned, Deserialize, Serialize};
+use std::{
+ collections::{BTreeMap, HashSet},
+ path::{Path, PathBuf},
+};
 
 pub const ETHERS_FORMAT_VERSION: &str = "ethers-rs-sol-build-info-1";
 
@@ -24,93 +31,129 @@ pub struct BuildInfo<I, O> {
 
 impl<I: DeserializeOwned, O: DeserializeOwned> BuildInfo<I, O> {
  /// Deserializes the `BuildInfo` object from the given file
- pub fn read(path: impl AsRef<Path>) -> Result<Self, SolcError> {
+ pub fn read(path: impl AsRef<Path>) -> Result<Self> {
  utils::read_json_file(path)
  }
 }
 
+/// Additional context we cache for each compiler run.
+#[derive(Debug, Clone, Deserialize, Serialize, PartialEq, Eq)]
+pub struct BuildContext<L> {
+ /// Mapping from internal compiler source id to path of the source file.
+ pub source_id_to_path: BTreeMap<u32, PathBuf>,
+ /// Language of the compiler.
+ pub language: L,
+}
+
+impl<L: Language> BuildContext<L> {
+ pub fn new<I, E>(input: &I, output: &CompilerOutput<E>) -> Result<Self>
+ where
+ I: CompilerInput<Language = L>,
+ {
+ let mut source_id_to_path = BTreeMap::new();
+
+ let input_sources = input.sources().map(|(path, _)| path).collect::<HashSet<_>>();
+ for (path, source) in output.sources.iter() {
+ if input_sources.contains(path.as_path()) {
+ source_id_to_path.insert(source.id, path.to_path_buf());
+ }
+ }
+
+ Ok(Self { source_id_to_path, language: input.language() })
+ }
+
+ pub fn join_all(&mut self, root: impl AsRef<Path>) {
+ self.source_id_to_path.values_mut().for_each(|path| {
+ *path = root.as_ref().join(path.as_path());
+ });
+ }
+
+ pub fn with_joined_paths(mut self, root: impl AsRef<Path>) -> Self {
+ self.join_all(root);
+ self
+ }
+}
+
 /// Represents `BuildInfo` object
 #[derive(Debug, Clone, Eq, PartialEq, Serialize, Deserialize)]
-pub struct RawBuildInfo {
+pub struct RawBuildInfo<L> {
  /// The hash that identifies the BuildInfo
  pub id: String,
+ #[serde(flatten)]
+ pub build_context: BuildContext<L>,
  /// serialized `BuildInfo` json
- pub build_info: String,
+ #[serde(flatten)]
+ pub build_info: BTreeMap<String, serde_json::Value>,
 }
 
 // === impl RawBuildInfo ===
 
-impl RawBuildInfo {
+impl<L: Language> RawBuildInfo<L> {
  /// Serializes a `BuildInfo` object
- pub fn new<I: Serialize, O: Serialize>(
+ pub fn new<I: CompilerInput<Language = L>, E: CompilationError>(
  input: &I,
- output: &O,
- version: &Version,
- ) -> serde_json::Result<RawBuildInfo> {
+ output: &CompilerOutput<E>,
+ full_build_info: bool,
+ ) -> Result<RawBuildInfo<L>> {
+ let version = input.version().clone();
+ let build_context = BuildContext::new(input, output)?;
+
  let mut hasher = md5::Md5::new();
- let w = BuildInfoWriter { buf: Rc::new(RefCell::new(Vec::with_capacity(128))) };
- let mut buf = w.clone();
- let mut serializer = serde_json::Serializer::pretty(&mut buf);
- let mut s = serializer.serialize_struct("BuildInfo", 6)?;
- s.serialize_field("_format", &ETHERS_FORMAT_VERSION)?;
+
+ hasher.update(ETHERS_FORMAT_VERSION);
+
  let solc_short = format!("{}.{}.{}", version.major, version.minor, version.patch);
- s.serialize_field("solcVersion", &solc_short)?;
- s.serialize_field("solcLongVersion", &version)?;
- s.serialize_field("input", input)?;
+ hasher.update(&solc_short);
+ hasher.update(version.to_string());
+
+ let input = serde_json::to_value(input)?;
+ hasher.update(&serde_json::to_string(&input)?);
 
  // create the hash for `{_format,solcVersion,solcLongVersion,input}`
  // N.B. this is not exactly the same as hashing the json representation of these values but
  // the must efficient one
- hasher.update(&*w.buf.borrow());
  let result = hasher.finalize();
  let id = hex::encode(result);
 
- s.serialize_field("id", &id)?;
- s.serialize_field("output", output)?;
- s.end()?;
-
- drop(buf);
+ let mut build_info = BTreeMap::new();
 
- let build_info = unsafe {
- // serde_json does not emit non UTF8
- String::from_utf8_unchecked(w.buf.take())
- };
+ if full_build_info {
+ build_info.insert("_format".to_string(), serde_json::to_value(ETHERS_FORMAT_VERSION)?);
+ build_info.insert("solcVersion".to_string(), serde_json::to_value(&solc_short)?);
+ build_info.insert("solcLongVersion".to_string(), serde_json::to_value(&version)?);
+ build_info.insert("input".to_string(), input);
+ build_info.insert("output".to_string(), serde_json::to_value(output)?);
+ }
 
- Ok(RawBuildInfo { id, build_info })
- }
-}
-
-#[derive(Clone)]
-struct BuildInfoWriter {
- buf: Rc<RefCell<Vec<u8>>>,
-}
-
-impl std::io::Write for BuildInfoWriter {
- fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {
- self.buf.borrow_mut().write(buf)
- }
-
- fn flush(&mut self) -> std::io::Result<()> {
- self.buf.borrow_mut().flush()
+ Ok(RawBuildInfo { id, build_info, build_context })
  }
 }
 
 #[cfg(test)]
 mod tests {
  use super::*;
- use crate::{artifacts::Error, compilers::CompilerOutput, SolcInput, Source};
+ use crate::{
+ artifacts::Error,
+ compilers::{
+ solc::{SolcLanguage, SolcVersionedInput},
+ CompilerOutput,
+ },
+ Source,
+ };
  use std::{collections::BTreeMap, path::PathBuf};
 
  #[test]
  fn build_info_serde() {
- let inputs = SolcInput::resolve_and_build(
+ let v: Version = "0.8.4+commit.c7e474f2".parse().unwrap();
+ let input = SolcVersionedInput::build(
  BTreeMap::from([(PathBuf::from("input.sol"), Source::new(""))]),
  Default::default(),
+ SolcLanguage::Solidity,
+ v,
  );
  let output = CompilerOutput::<Error>::default();
- let v: Version = "0.8.4+commit.c7e474f2".parse().unwrap();
- let raw_info = RawBuildInfo::new(&inputs[0], &output, &v).unwrap();
- let _info: BuildInfo<SolcInput, CompilerOutput<Error>> =
- serde_json::from_str(&raw_info.build_info).unwrap();
+ let raw_info = RawBuildInfo::new(&input, &output, true).unwrap();
+ let _info: BuildInfo<SolcVersionedInput, CompilerOutput<Error>> =
+ serde_json::from_str(&serde_json::to_string(&raw_info).unwrap()).unwrap();
  }
 }