From 728593b81a2f0b99a468c7591fdddd9912022887 Mon Sep 17 00:00:00 2001 From: Alex Ostrovski Date: Fri, 25 Oct 2024 11:51:19 +0300 Subject: [PATCH 01/19] Propagate errors in contract verifier --- core/lib/contract_verifier/src/error.rs | 2 +- core/lib/contract_verifier/src/lib.rs | 50 ++++++++++++++----------- 2 files changed, 29 insertions(+), 23 deletions(-) diff --git a/core/lib/contract_verifier/src/error.rs b/core/lib/contract_verifier/src/error.rs index c66756d1f121..b1d91c047136 100644 --- a/core/lib/contract_verifier/src/error.rs +++ b/core/lib/contract_verifier/src/error.rs @@ -1,4 +1,4 @@ -#[derive(Debug, Clone, thiserror::Error)] +#[derive(Debug, thiserror::Error)] pub enum ContractVerifierError { #[error("Internal error")] InternalError, diff --git a/core/lib/contract_verifier/src/lib.rs b/core/lib/contract_verifier/src/lib.rs index c8d9b89d834c..3f5fba6ebcde 100644 --- a/core/lib/contract_verifier/src/lib.rs +++ b/core/lib/contract_verifier/src/lib.rs @@ -4,7 +4,6 @@ use std::{ time::{Duration, Instant}, }; -use anyhow::Context as _; use chrono::Utc; use ethabi::{Contract, Token}; use lazy_static::lazy_static; @@ -72,8 +71,12 @@ impl ContractVerifier { // Bytecode should be present because it is checked when accepting request. let (deployed_bytecode, creation_tx_calldata) = storage .contract_verification_dal() - .get_contract_info_for_verification(request.req.contract_address).await - .unwrap() + .get_contract_info_for_verification(request.req.contract_address) + .await + .map_err(|err| { + tracing::error!("{err}"); + ContractVerifierError::InternalError + })? .ok_or_else(|| { tracing::warn!("Contract is missing in DB for already accepted verification request. Contract address: {:#?}", request.req.contract_address); ContractVerifierError::InternalError @@ -182,11 +185,9 @@ impl ContractVerifier { let contracts = output["contracts"] .get(file_name.as_str()) - .cloned() .ok_or(ContractVerifierError::MissingSource(file_name))?; let contract = contracts .get(&contract_name) - .cloned() .ok_or(ContractVerifierError::MissingContract(contract_name))?; let bytecode_str = contract["evm"]["bytecode"]["object"].as_str().ok_or( ContractVerifierError::AbstractContract(request.req.contract_name), @@ -446,15 +447,14 @@ impl ContractVerifier { storage: &mut Connection<'_, Core>, request_id: usize, verification_result: Result, - ) { + ) -> anyhow::Result<()> { match verification_result { Ok(info) => { storage .contract_verification_dal() .save_verification_info(info) - .await - .unwrap(); - tracing::info!("Successfully processed request with id = {}", request_id); + .await?; + tracing::info!("Successfully processed request with id = {request_id}"); } Err(error) => { let error_message = error.to_string(); @@ -467,11 +467,11 @@ impl ContractVerifier { storage .contract_verification_dal() .save_verification_error(request_id, error_message, compilation_errors, None) - .await - .unwrap(); - tracing::info!("Request with id = {} was failed", request_id); + .await?; + tracing::info!("Request with id = {request_id} was failed"); } } + Ok(()) } } @@ -485,25 +485,29 @@ impl JobProcessor for ContractVerifier { const BACKOFF_MULTIPLIER: u64 = 1; async fn get_next_job(&self) -> anyhow::Result> { - let mut connection = self.connection_pool.connection().await.unwrap(); - - // Time overhead for all operations except for compilation. + /// Time overhead for all operations except for compilation. const TIME_OVERHEAD: Duration = Duration::from_secs(10); + let mut connection = self + .connection_pool + .connection_tagged("contract_verifier") + .await?; // Considering that jobs that reach compilation timeout will be executed in // `compilation_timeout` + `non_compilation_time_overhead` (which is significantly less than `compilation_timeout`), // we re-pick up jobs that are being executed for a bit more than `compilation_timeout`. let job = connection .contract_verification_dal() .get_next_queued_verification_request(self.config.compilation_timeout() + TIME_OVERHEAD) - .await - .context("get_next_queued_verification_request()")?; - + .await?; Ok(job.map(|job| (job.id, job))) } async fn save_failure(&self, job_id: usize, _started_at: Instant, error: String) { - let mut connection = self.connection_pool.connection().await.unwrap(); + let mut connection = self + .connection_pool + .connection_tagged("contract_verifier") + .await + .unwrap(); connection .contract_verification_dal() @@ -529,11 +533,13 @@ impl JobProcessor for ContractVerifier { tokio::task::spawn(async move { tracing::info!("Started to process request with id = {}", job.id); - let mut connection = connection_pool.connection().await.unwrap(); - + // FIXME: long-living connection + let mut connection = connection_pool + .connection_tagged("contract_verifier") + .await?; let job_id = job.id; let verification_result = Self::verify(&mut connection, job, config).await; - Self::process_result(&mut connection, job_id, verification_result).await; + Self::process_result(&mut connection, job_id, verification_result).await?; API_CONTRACT_VERIFIER_METRICS .request_processing_time From c8f22ada0bf62c73ba1a069316160a26273e0037 Mon Sep 17 00:00:00 2001 From: Alex Ostrovski Date: Fri, 25 Oct 2024 12:22:24 +0300 Subject: [PATCH 02/19] Rework internal errors --- core/lib/contract_verifier/src/error.rs | 10 ++- core/lib/contract_verifier/src/lib.rs | 82 +++++++++++-------- .../lib/contract_verifier/src/zksolc_utils.rs | 31 ++++--- .../contract_verifier/src/zkvyper_utils.rs | 26 +++--- 4 files changed, 82 insertions(+), 67 deletions(-) diff --git a/core/lib/contract_verifier/src/error.rs b/core/lib/contract_verifier/src/error.rs index b1d91c047136..e55d2499c93b 100644 --- a/core/lib/contract_verifier/src/error.rs +++ b/core/lib/contract_verifier/src/error.rs @@ -1,7 +1,9 @@ +use zksync_dal::DalError; + #[derive(Debug, thiserror::Error)] pub enum ContractVerifierError { #[error("Internal error")] - InternalError, + Internal(#[from] anyhow::Error), #[error("Deployed bytecode is not equal to generated one from given source")] BytecodeMismatch, #[error("Constructor arguments are not correct")] @@ -23,3 +25,9 @@ pub enum ContractVerifierError { #[error("Failed to deserialize standard JSON input")] FailedToDeserializeInput, } + +impl From for ContractVerifierError { + fn from(err: DalError) -> Self { + Self::Internal(err.generalize()) + } +} diff --git a/core/lib/contract_verifier/src/lib.rs b/core/lib/contract_verifier/src/lib.rs index 3f5fba6ebcde..1f7bcaf4a8db 100644 --- a/core/lib/contract_verifier/src/lib.rs +++ b/core/lib/contract_verifier/src/lib.rs @@ -4,13 +4,14 @@ use std::{ time::{Duration, Instant}, }; +use anyhow::Context as _; use chrono::Utc; use ethabi::{Contract, Token}; use lazy_static::lazy_static; use regex::Regex; use tokio::time; use zksync_config::ContractVerifierConfig; -use zksync_dal::{Connection, ConnectionPool, Core, CoreDal}; +use zksync_dal::{ConnectionPool, Core, CoreDal}; use zksync_queued_job_processor::{async_trait, JobProcessor}; use zksync_types::{ contract_verification_api::{ @@ -47,7 +48,7 @@ enum ConstructorArgs { Ignore, } -#[derive(Debug)] +#[derive(Debug, Clone)] pub struct ContractVerifier { config: ContractVerifierConfig, connection_pool: ConnectionPool, @@ -62,25 +63,28 @@ impl ContractVerifier { } async fn verify( - storage: &mut Connection<'_, Core>, + &self, mut request: VerificationRequest, - config: ContractVerifierConfig, ) -> Result { - let artifacts = Self::compile(request.clone(), config).await?; + let artifacts = Self::compile(request.clone(), &self.config).await?; // Bytecode should be present because it is checked when accepting request. + let mut storage = self + .connection_pool + .connection_tagged("contract_verifier") + .await?; let (deployed_bytecode, creation_tx_calldata) = storage .contract_verification_dal() .get_contract_info_for_verification(request.req.contract_address) - .await - .map_err(|err| { - tracing::error!("{err}"); - ContractVerifierError::InternalError - })? - .ok_or_else(|| { - tracing::warn!("Contract is missing in DB for already accepted verification request. Contract address: {:#?}", request.req.contract_address); - ContractVerifierError::InternalError + .await? + .with_context(|| { + format!( + "Contract is missing in DB for already accepted verification request. Contract address: {:#?}", + request.req.contract_address + ) })?; + drop(storage); + let constructor_args = Self::decode_constructor_arguments_from_calldata( creation_tx_calldata, request.req.contract_address, @@ -116,7 +120,7 @@ impl ContractVerifier { async fn compile_zksolc( request: VerificationRequest, - config: ContractVerifierConfig, + config: &ContractVerifierConfig, ) -> Result { // Users may provide either just contract name or // source file name and contract name joined with ":". @@ -195,20 +199,22 @@ impl ContractVerifier { let bytecode = hex::decode(bytecode_str).unwrap(); let abi = contract["abi"].clone(); if !abi.is_array() { - tracing::error!( + let err = anyhow::anyhow!( "zksolc returned unexpected value for ABI: {}", serde_json::to_string_pretty(&abi).unwrap() ); - return Err(ContractVerifierError::InternalError); + return Err(err.into()); } Ok(CompilationArtifacts { bytecode, abi }) } ZkSolcOutput::YulSingleFile(output) => { let re = Regex::new(r"Contract `.*` bytecode: 0x([\da-f]+)").unwrap(); - let cap = re.captures(&output).unwrap(); - let bytecode_str = cap.get(1).unwrap().as_str(); - let bytecode = hex::decode(bytecode_str).unwrap(); + let cap = re + .captures(&output) + .context("Yul output doesn't match regex")?; + let bytecode_str = cap.get(1).context("no matches in Yul output")?.as_str(); + let bytecode = hex::decode(bytecode_str).context("invalid Yul output bytecode")?; Ok(CompilationArtifacts { bytecode, abi: serde_json::Value::Array(Vec::new()), @@ -219,7 +225,7 @@ impl ContractVerifier { async fn compile_zkvyper( request: VerificationRequest, - config: ContractVerifierConfig, + config: &ContractVerifierConfig, ) -> Result { // Users may provide either just contract name or // source file name and contract name joined with ":". @@ -264,17 +270,17 @@ impl ContractVerifier { let file_name = format!("{contract_name}.vy"); let object = output .as_object() - .cloned() - .ok_or(ContractVerifierError::InternalError)?; + .context("Vyper output is not an object")?; for (path, artifact) in object { let path = Path::new(&path); if path.file_name().unwrap().to_str().unwrap() == file_name { let bytecode_str = artifact["bytecode"] .as_str() - .ok_or(ContractVerifierError::InternalError)?; + .context("bytecode is not a string")?; let bytecode_without_prefix = bytecode_str.strip_prefix("0x").unwrap_or(bytecode_str); - let bytecode = hex::decode(bytecode_without_prefix).unwrap(); + let bytecode = + hex::decode(bytecode_without_prefix).context("failed decoding bytecode")?; return Ok(CompilationArtifacts { abi: artifact["abi"].clone(), bytecode, @@ -287,7 +293,7 @@ impl ContractVerifier { pub async fn compile( request: VerificationRequest, - config: ContractVerifierConfig, + config: &ContractVerifierConfig, ) -> Result { match request.req.source_code_data.compiler_type() { CompilerType::Solc => Self::compile_zksolc(request, config).await, @@ -444,10 +450,14 @@ impl ContractVerifier { } async fn process_result( - storage: &mut Connection<'_, Core>, + &self, request_id: usize, verification_result: Result, ) -> anyhow::Result<()> { + let mut storage = self + .connection_pool + .connection_tagged("contract_verifier") + .await?; match verification_result { Ok(info) => { storage @@ -457,7 +467,14 @@ impl ContractVerifier { tracing::info!("Successfully processed request with id = {request_id}"); } Err(error) => { - let error_message = error.to_string(); + let error_message = match &error { + ContractVerifierError::Internal(err) => { + // Do not expose the error externally, but log it. + tracing::warn!(request_id, "internal error processing request: {err}"); + "internal error".to_owned() + } + _ => error.to_string(), + }; let compilation_errors = match error { ContractVerifierError::CompilationError(compilation_errors) => { compilation_errors @@ -528,18 +545,13 @@ impl JobProcessor for ContractVerifier { job: VerificationRequest, started_at: Instant, ) -> tokio::task::JoinHandle> { - let connection_pool = self.connection_pool.clone(); - let config = self.config.clone(); + let this = self.clone(); tokio::task::spawn(async move { tracing::info!("Started to process request with id = {}", job.id); - // FIXME: long-living connection - let mut connection = connection_pool - .connection_tagged("contract_verifier") - .await?; let job_id = job.id; - let verification_result = Self::verify(&mut connection, job, config).await; - Self::process_result(&mut connection, job_id, verification_result).await?; + let verification_result = this.verify(job).await; + this.process_result(job_id, verification_result).await?; API_CONTRACT_VERIFIER_METRICS .request_processing_time diff --git a/core/lib/contract_verifier/src/zksolc_utils.rs b/core/lib/contract_verifier/src/zksolc_utils.rs index 08004632bcec..a961bb2d4f79 100644 --- a/core/lib/contract_verifier/src/zksolc_utils.rs +++ b/core/lib/contract_verifier/src/zksolc_utils.rs @@ -1,5 +1,6 @@ use std::{collections::HashMap, io::Write, path::PathBuf, process::Stdio}; +use anyhow::Context as _; use semver::Version; use serde::{Deserialize, Serialize}; @@ -16,6 +17,7 @@ pub enum ZkSolcInput { #[derive(Debug)] pub enum ZkSolcOutput { + // FIXME: incorrect abstraction boundary StandardJson(serde_json::Value), YulSingleFile(String), } @@ -136,26 +138,24 @@ impl ZkSolc { .arg("--standard-json") .stdin(Stdio::piped()) .spawn() - .map_err(|_err| ContractVerifierError::InternalError)?; + .context("failed spawning zksolc")?; let stdin = child.stdin.as_mut().unwrap(); - let content = serde_json::to_vec(&input).unwrap(); + let content = serde_json::to_vec(&input) + .context("cannot encode standard JSON input for zksolc")?; stdin .write_all(&content) .await - .map_err(|_err| ContractVerifierError::InternalError)?; + .context("failed writing standard JSON to zksolc stdin")?; stdin .flush() .await - .map_err(|_err| ContractVerifierError::InternalError)?; + .context("failed flushing standard JSON to zksolc")?; - let output = child - .wait_with_output() - .await - .map_err(|_err| ContractVerifierError::InternalError)?; + let output = child.wait_with_output().await.context("zksolc failed")?; if output.status.success() { Ok(ZkSolcOutput::StandardJson( serde_json::from_slice(&output.stdout) - .expect("Compiler output must be valid JSON"), + .context("zksolc output is not valid JSON")?, )) } else { Err(ContractVerifierError::CompilerError( @@ -170,9 +170,9 @@ impl ZkSolc { .suffix(".yul") .rand_bytes(0) .tempfile() - .map_err(|_err| ContractVerifierError::InternalError)?; + .context("cannot create temporary Yul file")?; file.write_all(source_code.as_bytes()) - .map_err(|_err| ContractVerifierError::InternalError)?; + .context("failed writing Yul file")?; let child = command .arg(file.path().to_str().unwrap()) .arg("--optimization") @@ -180,14 +180,11 @@ impl ZkSolc { .arg("--yul") .arg("--bin") .spawn() - .map_err(|_err| ContractVerifierError::InternalError)?; - let output = child - .wait_with_output() - .await - .map_err(|_err| ContractVerifierError::InternalError)?; + .context("failed spawning zksolc")?; + let output = child.wait_with_output().await.context("zksolc failed")?; if output.status.success() { Ok(ZkSolcOutput::YulSingleFile( - String::from_utf8(output.stdout).expect("Couldn't parse string"), + String::from_utf8(output.stdout).context("zksolc output is not UTF-8")?, )) } else { Err(ContractVerifierError::CompilerError( diff --git a/core/lib/contract_verifier/src/zkvyper_utils.rs b/core/lib/contract_verifier/src/zkvyper_utils.rs index c597f78d4588..e8eecb6ef421 100644 --- a/core/lib/contract_verifier/src/zkvyper_utils.rs +++ b/core/lib/contract_verifier/src/zkvyper_utils.rs @@ -1,5 +1,7 @@ use std::{collections::HashMap, fs::File, io::Write, path::PathBuf, process::Stdio}; +use anyhow::Context as _; + use crate::error::ContractVerifierError; #[derive(Debug)] @@ -37,32 +39,28 @@ impl ZkVyper { .stdout(Stdio::piped()) .stderr(Stdio::piped()); - let temp_dir = tempfile::tempdir().map_err(|_err| ContractVerifierError::InternalError)?; + let temp_dir = tempfile::tempdir().context("failed creating temporary dir")?; for (mut name, content) in input.sources { if !name.ends_with(".vy") { name += ".vy"; } - let path = temp_dir.path().join(name); + let path = temp_dir.path().join(&name); if let Some(prefix) = path.parent() { std::fs::create_dir_all(prefix) - .map_err(|_err| ContractVerifierError::InternalError)?; + .with_context(|| format!("failed creating parent dir for `{name}`"))?; } - let mut file = - File::create(&path).map_err(|_err| ContractVerifierError::InternalError)?; + let mut file = File::create(&path) + .with_context(|| format!("failed creating file for `{name}`"))?; file.write_all(content.as_bytes()) - .map_err(|_err| ContractVerifierError::InternalError)?; + .with_context(|| format!("failed writing to `{name}`"))?; command.arg(path.into_os_string()); } - let child = command - .spawn() - .map_err(|_err| ContractVerifierError::InternalError)?; - let output = child - .wait_with_output() - .await - .map_err(|_err| ContractVerifierError::InternalError)?; + let child = command.spawn().context("cannot spawn zkvyper")?; + let output = child.wait_with_output().await.context("zkvyper failed")?; if output.status.success() { - Ok(serde_json::from_slice(&output.stdout).expect("Compiler output must be valid JSON")) + Ok(serde_json::from_slice(&output.stdout) + .context("zkvyper output is not valid JSON")?) } else { Err(ContractVerifierError::CompilerError( "zkvyper".to_string(), From 6048b605e576786b6b83eb7f2b6aa68aeeb7ae9f Mon Sep 17 00:00:00 2001 From: Alex Ostrovski Date: Fri, 25 Oct 2024 15:55:40 +0300 Subject: [PATCH 03/19] Refactor compiler outputs --- core/lib/contract_verifier/src/lib.rs | 170 +++++------------- .../lib/contract_verifier/src/zksolc_utils.rs | 96 ++++++++-- .../contract_verifier/src/zkvyper_utils.rs | 46 ++++- 3 files changed, 168 insertions(+), 144 deletions(-) diff --git a/core/lib/contract_verifier/src/lib.rs b/core/lib/contract_verifier/src/lib.rs index 1f7bcaf4a8db..cf42f6b5ba94 100644 --- a/core/lib/contract_verifier/src/lib.rs +++ b/core/lib/contract_verifier/src/lib.rs @@ -8,7 +8,6 @@ use anyhow::Context as _; use chrono::Utc; use ethabi::{Contract, Token}; use lazy_static::lazy_static; -use regex::Regex; use tokio::time; use zksync_config::ContractVerifierConfig; use zksync_dal::{ConnectionPool, Core, CoreDal}; @@ -25,7 +24,7 @@ use zksync_utils::env::Workspace; use crate::{ error::ContractVerifierError, metrics::API_CONTRACT_VERIFIER_METRICS, - zksolc_utils::{Optimizer, Settings, Source, StandardJson, ZkSolc, ZkSolcInput, ZkSolcOutput}, + zksolc_utils::{Optimizer, Settings, Source, StandardJson, ZkSolc, ZkSolcInput}, zkvyper_utils::{ZkVyper, ZkVyperInput}, }; @@ -122,18 +121,7 @@ impl ContractVerifier { request: VerificationRequest, config: &ContractVerifierConfig, ) -> Result { - // Users may provide either just contract name or - // source file name and contract name joined with ":". - let (file_name, contract_name) = - if let Some((file_name, contract_name)) = request.req.contract_name.rsplit_once(':') { - (file_name.to_string(), contract_name.to_string()) - } else { - ( - format!("{}.sol", request.req.contract_name), - request.req.contract_name.clone(), - ) - }; - let input = Self::build_zksolc_input(request.clone(), file_name.clone())?; + let input = Self::build_zksolc_input(request.clone())?; let zksolc_path = Path::new(&home_path()) .join("etc") @@ -165,76 +153,15 @@ impl ContractVerifier { request.req.compiler_versions.zk_compiler_version(), ); - let output = time::timeout(config.compilation_timeout(), zksolc.async_compile(input)) + time::timeout(config.compilation_timeout(), zksolc.async_compile(input)) .await - .map_err(|_| ContractVerifierError::CompilationTimeout)??; - - match output { - ZkSolcOutput::StandardJson(output) => { - if let Some(errors) = output.get("errors") { - let errors = errors.as_array().unwrap().clone(); - if errors - .iter() - .any(|err| err["severity"].as_str().unwrap() == "error") - { - let error_messages = errors - .into_iter() - .map(|err| err["formattedMessage"].clone()) - .collect(); - return Err(ContractVerifierError::CompilationError( - serde_json::Value::Array(error_messages), - )); - } - } - - let contracts = output["contracts"] - .get(file_name.as_str()) - .ok_or(ContractVerifierError::MissingSource(file_name))?; - let contract = contracts - .get(&contract_name) - .ok_or(ContractVerifierError::MissingContract(contract_name))?; - let bytecode_str = contract["evm"]["bytecode"]["object"].as_str().ok_or( - ContractVerifierError::AbstractContract(request.req.contract_name), - )?; - let bytecode = hex::decode(bytecode_str).unwrap(); - let abi = contract["abi"].clone(); - if !abi.is_array() { - let err = anyhow::anyhow!( - "zksolc returned unexpected value for ABI: {}", - serde_json::to_string_pretty(&abi).unwrap() - ); - return Err(err.into()); - } - - Ok(CompilationArtifacts { bytecode, abi }) - } - ZkSolcOutput::YulSingleFile(output) => { - let re = Regex::new(r"Contract `.*` bytecode: 0x([\da-f]+)").unwrap(); - let cap = re - .captures(&output) - .context("Yul output doesn't match regex")?; - let bytecode_str = cap.get(1).context("no matches in Yul output")?.as_str(); - let bytecode = hex::decode(bytecode_str).context("invalid Yul output bytecode")?; - Ok(CompilationArtifacts { - bytecode, - abi: serde_json::Value::Array(Vec::new()), - }) - } - } + .map_err(|_| ContractVerifierError::CompilationTimeout)? } async fn compile_zkvyper( request: VerificationRequest, config: &ContractVerifierConfig, ) -> Result { - // Users may provide either just contract name or - // source file name and contract name joined with ":". - let contract_name = - if let Some((_file_name, contract_name)) = request.req.contract_name.rsplit_once(':') { - contract_name.to_string() - } else { - request.req.contract_name.clone() - }; let input = Self::build_zkvyper_input(request.clone())?; let zkvyper_path = Path::new(&home_path()) @@ -262,33 +189,9 @@ impl ContractVerifier { } let zkvyper = ZkVyper::new(zkvyper_path, vyper_path); - - let output = time::timeout(config.compilation_timeout(), zkvyper.async_compile(input)) + time::timeout(config.compilation_timeout(), zkvyper.async_compile(input)) .await - .map_err(|_| ContractVerifierError::CompilationTimeout)??; - - let file_name = format!("{contract_name}.vy"); - let object = output - .as_object() - .context("Vyper output is not an object")?; - for (path, artifact) in object { - let path = Path::new(&path); - if path.file_name().unwrap().to_str().unwrap() == file_name { - let bytecode_str = artifact["bytecode"] - .as_str() - .context("bytecode is not a string")?; - let bytecode_without_prefix = - bytecode_str.strip_prefix("0x").unwrap_or(bytecode_str); - let bytecode = - hex::decode(bytecode_without_prefix).context("failed decoding bytecode")?; - return Ok(CompilationArtifacts { - abi: artifact["abi"].clone(), - bytecode, - }); - } - } - - Err(ContractVerifierError::MissingContract(contract_name)) + .map_err(|_| ContractVerifierError::CompilationTimeout)? } pub async fn compile( @@ -303,24 +206,31 @@ impl ContractVerifier { fn build_zksolc_input( request: VerificationRequest, - file_name: String, ) -> Result { - let default_output_selection = serde_json::json!( - { - "*": { - "*": [ "abi" ], - "": [ "abi" ] - } + // Users may provide either just contract name or + // source file name and contract name joined with ":". + let (file_name, contract_name) = + if let Some((file_name, contract_name)) = request.req.contract_name.rsplit_once(':') { + (file_name.to_string(), contract_name.to_string()) + } else { + ( + format!("{}.sol", request.req.contract_name), + request.req.contract_name.clone(), + ) + }; + let default_output_selection = serde_json::json!({ + "*": { + "*": [ "abi" ], + "": [ "abi" ] } - ); + }); match request.req.source_code_data { SourceCodeData::SolSingleFile(source_code) => { let source = Source { content: source_code, }; - let sources: HashMap = - vec![(file_name, source)].into_iter().collect(); + let sources = HashMap::from([(file_name.clone(), source)]); let optimizer = Optimizer { enabled: request.req.optimization_used, mode: request.req.optimizer_mode.and_then(|s| s.chars().next()), @@ -338,11 +248,15 @@ impl ContractVerifier { ), }; - Ok(ZkSolcInput::StandardJson(StandardJson { - language: "Solidity".to_string(), - sources, - settings, - })) + Ok(ZkSolcInput::StandardJson { + input: StandardJson { + language: "Solidity".to_string(), + sources, + settings, + }, + contract_name, + file_name, + }) } SourceCodeData::StandardJsonInput(map) => { let mut compiler_input: StandardJson = @@ -350,24 +264,38 @@ impl ContractVerifier { .map_err(|_| ContractVerifierError::FailedToDeserializeInput)?; // Set default output selection even if it is different in request. compiler_input.settings.output_selection = Some(default_output_selection); - Ok(ZkSolcInput::StandardJson(compiler_input)) + Ok(ZkSolcInput::StandardJson { + input: compiler_input, + contract_name, + file_name, + }) } SourceCodeData::YulSingleFile(source_code) => Ok(ZkSolcInput::YulSingleFile { source_code, is_system: request.req.is_system, }), - _ => panic!("Unexpected SourceCode variant"), + other => unreachable!("Unexpected `SourceCodeData` variant: {other:?}"), } } fn build_zkvyper_input( request: VerificationRequest, ) -> Result { + // Users may provide either just contract name or + // source file name and contract name joined with ":". + let contract_name = + if let Some((_, contract_name)) = request.req.contract_name.rsplit_once(':') { + contract_name.to_owned() + } else { + request.req.contract_name.clone() + }; + let sources = match request.req.source_code_data { SourceCodeData::VyperMultiFile(s) => s, - _ => panic!("Unexpected SourceCode variant"), + other => unreachable!("unexpected `SourceCodeData` variant: {other:?}"), }; Ok(ZkVyperInput { + contract_name, sources, optimizer_mode: request.req.optimizer_mode, }) diff --git a/core/lib/contract_verifier/src/zksolc_utils.rs b/core/lib/contract_verifier/src/zksolc_utils.rs index a961bb2d4f79..8c6066a11f38 100644 --- a/core/lib/contract_verifier/src/zksolc_utils.rs +++ b/core/lib/contract_verifier/src/zksolc_utils.rs @@ -1,27 +1,26 @@ use std::{collections::HashMap, io::Write, path::PathBuf, process::Stdio}; use anyhow::Context as _; +use regex::Regex; use semver::Version; use serde::{Deserialize, Serialize}; +use zksync_types::contract_verification_api::CompilationArtifacts; use crate::error::ContractVerifierError; #[derive(Debug)] pub enum ZkSolcInput { - StandardJson(StandardJson), + StandardJson { + input: StandardJson, + contract_name: String, + file_name: String, + }, YulSingleFile { source_code: String, is_system: bool, }, } -#[derive(Debug)] -pub enum ZkSolcOutput { - // FIXME: incorrect abstraction boundary - StandardJson(serde_json::Value), - YulSingleFile(String), -} - #[derive(Debug, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct StandardJson { @@ -99,13 +98,13 @@ impl ZkSolc { pub async fn async_compile( &self, input: ZkSolcInput, - ) -> Result { + ) -> Result { use tokio::io::AsyncWriteExt; let mut command = tokio::process::Command::new(&self.zksolc_path); command.stdout(Stdio::piped()).stderr(Stdio::piped()); match &input { - ZkSolcInput::StandardJson(input) => { + ZkSolcInput::StandardJson { input, .. } => { if !self.is_post_1_5_0() { if input.settings.is_system { command.arg("--system-mode"); @@ -133,7 +132,11 @@ impl ZkSolc { } } match input { - ZkSolcInput::StandardJson(input) => { + ZkSolcInput::StandardJson { + input, + contract_name, + file_name, + } => { let mut child = command .arg("--standard-json") .stdin(Stdio::piped()) @@ -153,10 +156,9 @@ impl ZkSolc { let output = child.wait_with_output().await.context("zksolc failed")?; if output.status.success() { - Ok(ZkSolcOutput::StandardJson( - serde_json::from_slice(&output.stdout) - .context("zksolc output is not valid JSON")?, - )) + let output = serde_json::from_slice(&output.stdout) + .context("zksolc output is not valid JSON")?; + Self::parse_standard_json_output(&output, contract_name, file_name) } else { Err(ContractVerifierError::CompilerError( "zksolc".to_string(), @@ -183,9 +185,9 @@ impl ZkSolc { .context("failed spawning zksolc")?; let output = child.wait_with_output().await.context("zksolc failed")?; if output.status.success() { - Ok(ZkSolcOutput::YulSingleFile( - String::from_utf8(output.stdout).context("zksolc output is not UTF-8")?, - )) + let output = + String::from_utf8(output.stdout).context("zksolc output is not UTF-8")?; + Self::parse_single_file_yul_output(&output) } else { Err(ContractVerifierError::CompilerError( "zksolc".to_string(), @@ -196,6 +198,64 @@ impl ZkSolc { } } + fn parse_standard_json_output( + output: &serde_json::Value, + contract_name: String, + file_name: String, + ) -> Result { + if let Some(errors) = output.get("errors") { + let errors = errors.as_array().unwrap().clone(); + if errors + .iter() + .any(|err| err["severity"].as_str().unwrap() == "error") + { + let error_messages = errors + .into_iter() + .map(|err| err["formattedMessage"].clone()) + .collect(); + return Err(ContractVerifierError::CompilationError( + serde_json::Value::Array(error_messages), + )); + } + } + + let contracts = output["contracts"] + .get(&file_name) + .ok_or(ContractVerifierError::MissingSource(file_name))?; + let Some(contract) = contracts.get(&contract_name) else { + return Err(ContractVerifierError::MissingContract(contract_name)); + }; + let bytecode_str = contract["evm"]["bytecode"]["object"] + .as_str() + .ok_or(ContractVerifierError::AbstractContract(contract_name))?; + let bytecode = hex::decode(bytecode_str).unwrap(); + let abi = contract["abi"].clone(); + if !abi.is_array() { + let err = anyhow::anyhow!( + "zksolc returned unexpected value for ABI: {}", + serde_json::to_string_pretty(&abi).unwrap() + ); + return Err(err.into()); + } + + Ok(CompilationArtifacts { bytecode, abi }) + } + + fn parse_single_file_yul_output( + output: &str, + ) -> Result { + let re = Regex::new(r"Contract `.*` bytecode: 0x([\da-f]+)").unwrap(); + let cap = re + .captures(output) + .context("Yul output doesn't match regex")?; + let bytecode_str = cap.get(1).context("no matches in Yul output")?.as_str(); + let bytecode = hex::decode(bytecode_str).context("invalid Yul output bytecode")?; + Ok(CompilationArtifacts { + bytecode, + abi: serde_json::Value::Array(Vec::new()), + }) + } + pub fn is_post_1_5_0(&self) -> bool { // Special case if &self.zksolc_version == "vm-1.5.0-a167aa3" { diff --git a/core/lib/contract_verifier/src/zkvyper_utils.rs b/core/lib/contract_verifier/src/zkvyper_utils.rs index e8eecb6ef421..1ec5f15495f0 100644 --- a/core/lib/contract_verifier/src/zkvyper_utils.rs +++ b/core/lib/contract_verifier/src/zkvyper_utils.rs @@ -1,11 +1,19 @@ -use std::{collections::HashMap, fs::File, io::Write, path::PathBuf, process::Stdio}; +use std::{ + collections::HashMap, + fs::File, + io::Write, + path::{Path, PathBuf}, + process::Stdio, +}; use anyhow::Context as _; +use zksync_types::contract_verification_api::CompilationArtifacts; use crate::error::ContractVerifierError; #[derive(Debug)] -pub struct ZkVyperInput { +pub(crate) struct ZkVyperInput { + pub contract_name: String, pub sources: HashMap, pub optimizer_mode: Option, } @@ -26,7 +34,7 @@ impl ZkVyper { pub async fn async_compile( &self, input: ZkVyperInput, - ) -> Result { + ) -> Result { let mut command = tokio::process::Command::new(&self.zkvyper_path); if let Some(o) = input.optimizer_mode.as_ref() { command.arg("-O").arg(o); @@ -59,8 +67,9 @@ impl ZkVyper { let child = command.spawn().context("cannot spawn zkvyper")?; let output = child.wait_with_output().await.context("zkvyper failed")?; if output.status.success() { - Ok(serde_json::from_slice(&output.stdout) - .context("zkvyper output is not valid JSON")?) + let output = serde_json::from_slice(&output.stdout) + .context("zkvyper output is not valid JSON")?; + Self::parse_output(&output, input.contract_name) } else { Err(ContractVerifierError::CompilerError( "zkvyper".to_string(), @@ -68,4 +77,31 @@ impl ZkVyper { )) } } + + fn parse_output( + output: &serde_json::Value, + contract_name: String, + ) -> Result { + let file_name = format!("{contract_name}.vy"); + let object = output + .as_object() + .context("Vyper output is not an object")?; + for (path, artifact) in object { + let path = Path::new(&path); + if path.file_name().unwrap().to_str().unwrap() == file_name { + let bytecode_str = artifact["bytecode"] + .as_str() + .context("bytecode is not a string")?; + let bytecode_without_prefix = + bytecode_str.strip_prefix("0x").unwrap_or(bytecode_str); + let bytecode = + hex::decode(bytecode_without_prefix).context("failed decoding bytecode")?; + return Ok(CompilationArtifacts { + abi: artifact["abi"].clone(), + bytecode, + }); + } + } + Err(ContractVerifierError::MissingContract(contract_name)) + } } From 27aa72476e07d45fc71242630595ce582210bf8c Mon Sep 17 00:00:00 2001 From: Alex Ostrovski Date: Mon, 28 Oct 2024 13:17:06 +0200 Subject: [PATCH 04/19] Remove `lazy_static` dependency --- Cargo.lock | 1 - Cargo.toml | 1 - core/lib/contract_verifier/Cargo.toml | 1 - core/lib/contract_verifier/src/lib.rs | 24 +++++++++++------------- 4 files changed, 11 insertions(+), 16 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 597da3c1b31b..99ca1253ed0a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -9957,7 +9957,6 @@ dependencies = [ "chrono", "ethabi", "hex", - "lazy_static", "regex", "semver", "serde", diff --git a/Cargo.toml b/Cargo.toml index 6d51e5060aa8..e57a7e7024f4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -135,7 +135,6 @@ hyper = "1.3" insta = "1.29.0" itertools = "0.10" jsonrpsee = { version = "0.23", default-features = false } -lazy_static = "1.4" leb128 = "0.2.5" lru = { version = "0.12.1", default-features = false } mini-moka = "0.10.0" diff --git a/core/lib/contract_verifier/Cargo.toml b/core/lib/contract_verifier/Cargo.toml index 580982c9a700..b53581a7aa9f 100644 --- a/core/lib/contract_verifier/Cargo.toml +++ b/core/lib/contract_verifier/Cargo.toml @@ -27,7 +27,6 @@ ethabi.workspace = true vise.workspace = true hex.workspace = true serde = { workspace = true, features = ["derive"] } -lazy_static.workspace = true tempfile.workspace = true regex.workspace = true tracing.workspace = true diff --git a/core/lib/contract_verifier/src/lib.rs b/core/lib/contract_verifier/src/lib.rs index cf42f6b5ba94..b6cbb58bf649 100644 --- a/core/lib/contract_verifier/src/lib.rs +++ b/core/lib/contract_verifier/src/lib.rs @@ -7,7 +7,6 @@ use std::{ use anyhow::Context as _; use chrono::Utc; use ethabi::{Contract, Token}; -use lazy_static::lazy_static; use tokio::time; use zksync_config::ContractVerifierConfig; use zksync_dal::{ConnectionPool, Core, CoreDal}; @@ -33,10 +32,6 @@ mod metrics; mod zksolc_utils; mod zkvyper_utils; -lazy_static! { - static ref DEPLOYER_CONTRACT: Contract = zksync_contracts::deployer_contract(); -} - fn home_path() -> PathBuf { Workspace::locate().core() } @@ -50,6 +45,7 @@ enum ConstructorArgs { #[derive(Debug, Clone)] pub struct ContractVerifier { config: ContractVerifierConfig, + contract_deployer: Contract, connection_pool: ConnectionPool, } @@ -57,6 +53,7 @@ impl ContractVerifier { pub fn new(config: ContractVerifierConfig, connection_pool: ConnectionPool) -> Self { Self { config, + contract_deployer: zksync_contracts::deployer_contract(), connection_pool, } } @@ -84,7 +81,7 @@ impl ContractVerifier { })?; drop(storage); - let constructor_args = Self::decode_constructor_arguments_from_calldata( + let constructor_args = self.decode_constructor_arguments_from_calldata( creation_tx_calldata, request.req.contract_address, ); @@ -302,20 +299,21 @@ impl ContractVerifier { } fn decode_constructor_arguments_from_calldata( + &self, calldata: DeployContractCalldata, contract_address_to_verify: Address, ) -> ConstructorArgs { match calldata { DeployContractCalldata::Deploy(calldata) => { - let create = DEPLOYER_CONTRACT.function("create").unwrap(); - let create2 = DEPLOYER_CONTRACT.function("create2").unwrap(); - - let create_acc = DEPLOYER_CONTRACT.function("createAccount").unwrap(); - let create2_acc = DEPLOYER_CONTRACT.function("create2Account").unwrap(); - - let force_deploy = DEPLOYER_CONTRACT + let contract_deployer = &self.contract_deployer; + let create = contract_deployer.function("create").unwrap(); + let create2 = contract_deployer.function("create2").unwrap(); + let create_acc = contract_deployer.function("createAccount").unwrap(); + let create2_acc = contract_deployer.function("create2Account").unwrap(); + let force_deploy = contract_deployer .function("forceDeployOnAddresses") .unwrap(); + // It's assumed that `create` and `create2` methods have the same parameters // and the same for `createAccount` and `create2Account`. match &calldata[0..4] { From 6b4fa1ecdd48f8936fe61bd9c98fdca9b5317d3f Mon Sep 17 00:00:00 2001 From: Alex Ostrovski Date: Mon, 28 Oct 2024 13:35:06 +0200 Subject: [PATCH 05/19] Refactor compiler paths --- core/lib/contract_verifier/src/lib.rs | 65 ++---------- core/lib/contract_verifier/src/paths.rs | 98 +++++++++++++++++++ .../lib/contract_verifier/src/zksolc_utils.rs | 37 +++---- .../contract_verifier/src/zkvyper_utils.rs | 27 ++--- 4 files changed, 132 insertions(+), 95 deletions(-) create mode 100644 core/lib/contract_verifier/src/paths.rs diff --git a/core/lib/contract_verifier/src/lib.rs b/core/lib/contract_verifier/src/lib.rs index b6cbb58bf649..47afc91cb9b6 100644 --- a/core/lib/contract_verifier/src/lib.rs +++ b/core/lib/contract_verifier/src/lib.rs @@ -1,6 +1,5 @@ use std::{ collections::HashMap, - path::{Path, PathBuf}, time::{Duration, Instant}, }; @@ -18,24 +17,21 @@ use zksync_types::{ }, Address, }; -use zksync_utils::env::Workspace; use crate::{ error::ContractVerifierError, metrics::API_CONTRACT_VERIFIER_METRICS, + paths::CompilerPaths, zksolc_utils::{Optimizer, Settings, Source, StandardJson, ZkSolc, ZkSolcInput}, zkvyper_utils::{ZkVyper, ZkVyperInput}, }; pub mod error; mod metrics; +mod paths; mod zksolc_utils; mod zkvyper_utils; -fn home_path() -> PathBuf { - Workspace::locate().core() -} - #[derive(Debug)] enum ConstructorArgs { Check(Vec), @@ -118,35 +114,10 @@ impl ContractVerifier { request: VerificationRequest, config: &ContractVerifierConfig, ) -> Result { + let compiler_paths = CompilerPaths::for_solc(&request.req.compiler_versions).await?; let input = Self::build_zksolc_input(request.clone())?; - - let zksolc_path = Path::new(&home_path()) - .join("etc") - .join("zksolc-bin") - .join(request.req.compiler_versions.zk_compiler_version()) - .join("zksolc"); - if !zksolc_path.exists() { - return Err(ContractVerifierError::UnknownCompilerVersion( - "zksolc".to_string(), - request.req.compiler_versions.zk_compiler_version(), - )); - } - - let solc_path = Path::new(&home_path()) - .join("etc") - .join("solc-bin") - .join(request.req.compiler_versions.compiler_version()) - .join("solc"); - if !solc_path.exists() { - return Err(ContractVerifierError::UnknownCompilerVersion( - "solc".to_string(), - request.req.compiler_versions.compiler_version(), - )); - } - let zksolc = ZkSolc::new( - zksolc_path, - solc_path, + compiler_paths, request.req.compiler_versions.zk_compiler_version(), ); @@ -159,33 +130,9 @@ impl ContractVerifier { request: VerificationRequest, config: &ContractVerifierConfig, ) -> Result { + let compiler_paths = CompilerPaths::for_vyper(&request.req.compiler_versions).await?; let input = Self::build_zkvyper_input(request.clone())?; - - let zkvyper_path = Path::new(&home_path()) - .join("etc") - .join("zkvyper-bin") - .join(request.req.compiler_versions.zk_compiler_version()) - .join("zkvyper"); - if !zkvyper_path.exists() { - return Err(ContractVerifierError::UnknownCompilerVersion( - "zkvyper".to_string(), - request.req.compiler_versions.zk_compiler_version(), - )); - } - - let vyper_path = Path::new(&home_path()) - .join("etc") - .join("vyper-bin") - .join(request.req.compiler_versions.compiler_version()) - .join("vyper"); - if !vyper_path.exists() { - return Err(ContractVerifierError::UnknownCompilerVersion( - "vyper".to_string(), - request.req.compiler_versions.compiler_version(), - )); - } - - let zkvyper = ZkVyper::new(zkvyper_path, vyper_path); + let zkvyper = ZkVyper::new(compiler_paths); time::timeout(config.compilation_timeout(), zkvyper.async_compile(input)) .await .map_err(|_| ContractVerifierError::CompilationTimeout)? diff --git a/core/lib/contract_verifier/src/paths.rs b/core/lib/contract_verifier/src/paths.rs new file mode 100644 index 000000000000..9775d1f48d10 --- /dev/null +++ b/core/lib/contract_verifier/src/paths.rs @@ -0,0 +1,98 @@ +use std::path::{Path, PathBuf}; + +use anyhow::Context as _; +use tokio::fs; +use zksync_types::contract_verification_api::CompilerVersions; +use zksync_utils::env::Workspace; + +use crate::error::ContractVerifierError; + +fn home_path() -> PathBuf { + Workspace::locate().core() +} + +#[derive(Debug)] +pub(crate) struct CompilerPaths { + /// Path to the base (non-zk) compiler. + pub base: PathBuf, + /// Path to the zk compiler. + pub zk: PathBuf, +} + +impl CompilerPaths { + pub async fn for_solc(versions: &CompilerVersions) -> Result { + let zksolc_version = versions.zk_compiler_version(); + let zksolc_path = Path::new(&home_path()) + .join("etc") + .join("zksolc-bin") + .join(&zksolc_version) + .join("zksolc"); + if !fs::try_exists(&zksolc_path) + .await + .context("failed accessing zksolc")? + { + return Err(ContractVerifierError::UnknownCompilerVersion( + "zksolc".to_owned(), + zksolc_version, + )); + } + + let solc_version = versions.compiler_version(); + let solc_path = Path::new(&home_path()) + .join("etc") + .join("solc-bin") + .join(&solc_version) + .join("solc"); + if !fs::try_exists(&solc_path) + .await + .context("failed accessing solc")? + { + return Err(ContractVerifierError::UnknownCompilerVersion( + "solc".to_owned(), + solc_version, + )); + } + Ok(Self { + base: solc_path, + zk: zksolc_path, + }) + } + + pub async fn for_vyper(versions: &CompilerVersions) -> Result { + let zkvyper_version = versions.zk_compiler_version(); + let zkvyper_path = Path::new(&home_path()) + .join("etc") + .join("zkvyper-bin") + .join(&zkvyper_version) + .join("zkvyper"); + if !fs::try_exists(&zkvyper_path) + .await + .context("failed accessing zkvyper")? + { + return Err(ContractVerifierError::UnknownCompilerVersion( + "zkvyper".to_owned(), + zkvyper_version, + )); + } + + let vyper_version = versions.compiler_version(); + let vyper_path = Path::new(&home_path()) + .join("etc") + .join("vyper-bin") + .join(&vyper_version) + .join("vyper"); + if !fs::try_exists(&vyper_path) + .await + .context("failed accessing vyper")? + { + return Err(ContractVerifierError::UnknownCompilerVersion( + "vyper".to_owned(), + vyper_version, + )); + } + Ok(Self { + base: vyper_path, + zk: zkvyper_path, + }) + } +} diff --git a/core/lib/contract_verifier/src/zksolc_utils.rs b/core/lib/contract_verifier/src/zksolc_utils.rs index 8c6066a11f38..c62e3c3b32ed 100644 --- a/core/lib/contract_verifier/src/zksolc_utils.rs +++ b/core/lib/contract_verifier/src/zksolc_utils.rs @@ -1,4 +1,4 @@ -use std::{collections::HashMap, io::Write, path::PathBuf, process::Stdio}; +use std::{collections::HashMap, io::Write, process::Stdio}; use anyhow::Context as _; use regex::Regex; @@ -6,7 +6,7 @@ use semver::Version; use serde::{Deserialize, Serialize}; use zksync_types::contract_verification_api::CompilationArtifacts; -use crate::error::ContractVerifierError; +use crate::{error::ContractVerifierError, paths::CompilerPaths}; #[derive(Debug)] pub enum ZkSolcInput { @@ -76,21 +76,16 @@ impl Default for Optimizer { } } -pub struct ZkSolc { - zksolc_path: PathBuf, - solc_path: PathBuf, +#[derive(Debug)] +pub(crate) struct ZkSolc { + paths: CompilerPaths, zksolc_version: String, } impl ZkSolc { - pub fn new( - zksolc_path: impl Into, - solc_path: impl Into, - zksolc_version: String, - ) -> Self { + pub fn new(paths: CompilerPaths, zksolc_version: String) -> Self { ZkSolc { - zksolc_path: zksolc_path.into(), - solc_path: solc_path.into(), + paths, zksolc_version, } } @@ -100,7 +95,7 @@ impl ZkSolc { input: ZkSolcInput, ) -> Result { use tokio::io::AsyncWriteExt; - let mut command = tokio::process::Command::new(&self.zksolc_path); + let mut command = tokio::process::Command::new(&self.paths.zk); command.stdout(Stdio::piped()).stderr(Stdio::piped()); match &input { @@ -114,20 +109,20 @@ impl ZkSolc { } } - command.arg("--solc").arg(self.solc_path.to_str().unwrap()); + command.arg("--solc").arg(&self.paths.base); } ZkSolcInput::YulSingleFile { is_system, .. } => { if self.is_post_1_5_0() { if *is_system { command.arg("--enable-eravm-extensions"); } else { - command.arg("--solc").arg(self.solc_path.to_str().unwrap()); + command.arg("--solc").arg(&self.paths.base); } } else { if *is_system { command.arg("--system-mode"); } - command.arg("--solc").arg(self.solc_path.to_str().unwrap()); + command.arg("--solc").arg(&self.paths.base); } } } @@ -275,12 +270,18 @@ impl ZkSolc { #[cfg(test)] mod tests { - use crate::zksolc_utils::ZkSolc; + use std::path::PathBuf; + + use crate::{paths::CompilerPaths, zksolc_utils::ZkSolc}; #[test] fn check_is_post_1_5_0() { // Special case. - let mut zksolc = ZkSolc::new(".", ".", "vm-1.5.0-a167aa3".to_string()); + let compiler_paths = CompilerPaths { + base: PathBuf::default(), + zk: PathBuf::default(), + }; + let mut zksolc = ZkSolc::new(compiler_paths, "vm-1.5.0-a167aa3".to_string()); assert!(!zksolc.is_post_1_5_0(), "vm-1.5.0-a167aa3"); zksolc.zksolc_version = "v1.5.0".to_string(); diff --git a/core/lib/contract_verifier/src/zkvyper_utils.rs b/core/lib/contract_verifier/src/zkvyper_utils.rs index 1ec5f15495f0..1f62c249294d 100644 --- a/core/lib/contract_verifier/src/zkvyper_utils.rs +++ b/core/lib/contract_verifier/src/zkvyper_utils.rs @@ -1,15 +1,9 @@ -use std::{ - collections::HashMap, - fs::File, - io::Write, - path::{Path, PathBuf}, - process::Stdio, -}; +use std::{collections::HashMap, fs::File, io::Write, path::Path, process::Stdio}; use anyhow::Context as _; use zksync_types::contract_verification_api::CompilationArtifacts; -use crate::error::ContractVerifierError; +use crate::{error::ContractVerifierError, paths::CompilerPaths}; #[derive(Debug)] pub(crate) struct ZkVyperInput { @@ -18,30 +12,27 @@ pub(crate) struct ZkVyperInput { pub optimizer_mode: Option, } -pub struct ZkVyper { - zkvyper_path: PathBuf, - vyper_path: PathBuf, +#[derive(Debug)] +pub(crate) struct ZkVyper { + paths: CompilerPaths, } impl ZkVyper { - pub fn new(zkvyper_path: impl Into, vyper_path: impl Into) -> Self { - ZkVyper { - zkvyper_path: zkvyper_path.into(), - vyper_path: vyper_path.into(), - } + pub fn new(paths: CompilerPaths) -> Self { + Self { paths } } pub async fn async_compile( &self, input: ZkVyperInput, ) -> Result { - let mut command = tokio::process::Command::new(&self.zkvyper_path); + let mut command = tokio::process::Command::new(&self.paths.zk); if let Some(o) = input.optimizer_mode.as_ref() { command.arg("-O").arg(o); } command .arg("--vyper") - .arg(self.vyper_path.to_str().unwrap()) + .arg(&self.paths.base) .arg("-f") .arg("combined_json") .stdout(Stdio::piped()) From fe483ba070bab546ae4d8ca2503227531b56faa6 Mon Sep 17 00:00:00 2001 From: Alex Ostrovski Date: Mon, 28 Oct 2024 15:36:46 +0200 Subject: [PATCH 06/19] Sketch contract verifier tests --- Cargo.lock | 192 +++++++++++++++++ .../config/src/configs/contract_verifier.rs | 4 +- core/lib/contract_verifier/Cargo.toml | 6 + core/lib/contract_verifier/src/lib.rs | 5 +- core/lib/contract_verifier/src/tests.rs | 202 ++++++++++++++++++ 5 files changed, 406 insertions(+), 3 deletions(-) create mode 100644 core/lib/contract_verifier/src/tests.rs diff --git a/Cargo.lock b/Cargo.lock index 99ca1253ed0a..f96a27604918 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -186,6 +186,15 @@ version = "1.0.89" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "86fdf8605db99b54d3cd748a44c6d04df638eb5dafb219b135d0149bd0db01f6" +[[package]] +name = "arbitrary" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d5a26814d8dcb93b0e5a0ff3c6d80a8843bafb21b39e8e18a6f05471870e110" +dependencies = [ + "derive_arbitrary", +] + [[package]] name = "arr_macro" version = "0.1.3" @@ -1563,6 +1572,19 @@ dependencies = [ "compile-fmt", ] +[[package]] +name = "const-hex" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0121754e84117e65f9d90648ee6aa4882a6e63110307ab73967a4c5e7e69e586" +dependencies = [ + "cfg-if", + "cpufeatures", + "hex", + "proptest", + "serde", +] + [[package]] name = "const-oid" version = "0.9.6" @@ -2034,6 +2056,17 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "derive_arbitrary" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67e77553c4162a157adbf834ebae5b415acbecbeafc7a74b0e886657506a7611" +dependencies = [ + "proc-macro2 1.0.86", + "quote 1.0.37", + "syn 2.0.77", +] + [[package]] name = "derive_more" version = "0.99.18" @@ -2095,6 +2128,15 @@ dependencies = [ "subtle", ] +[[package]] +name = "dirs" +version = "5.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44c45a9d03d6676652bcb5e724c7e988de1acad23a711b5217ab9cbecbec2225" +dependencies = [ + "dirs-sys", +] + [[package]] name = "dirs-next" version = "2.0.0" @@ -2105,6 +2147,18 @@ dependencies = [ "dirs-sys-next", ] +[[package]] +name = "dirs-sys" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "520f05a5cbd335fae5a99ff7a6ab8627577660ee5cfd6a94a6a929b52ff0321c" +dependencies = [ + "libc", + "option-ext", + "redox_users", + "windows-sys 0.48.0", +] + [[package]] name = "dirs-sys-next" version = "0.1.2" @@ -2116,6 +2170,17 @@ dependencies = [ "winapi", ] +[[package]] +name = "displaydoc" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" +dependencies = [ + "proc-macro2 1.0.86", + "quote 1.0.37", + "syn 2.0.77", +] + [[package]] name = "dotenvy" version = "0.15.7" @@ -2632,6 +2697,16 @@ dependencies = [ "zksync_bellman", ] +[[package]] +name = "fs4" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec6fcfb3c0c1d71612528825042261419d5dade9678c39a781e05b63677d9b32" +dependencies = [ + "rustix", + "windows-sys 0.52.0", +] + [[package]] name = "fs_extra" version = "1.3.0" @@ -4276,6 +4351,12 @@ dependencies = [ "scopeguard", ] +[[package]] +name = "lockfree-object-pool" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9374ef4228402d4b7e403e5838cb880d9ee663314b0a900d5a6aabf0c213552e" + [[package]] name = "log" version = "0.4.22" @@ -4951,6 +5032,12 @@ dependencies = [ "tokio-stream", ] +[[package]] +name = "option-ext" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d" + [[package]] name = "ordered-float" version = "2.10.1" @@ -5466,6 +5553,22 @@ dependencies = [ "syn 2.0.77", ] +[[package]] +name = "proptest" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4c2511913b88df1637da85cc8d96ec8e43a3f8bb8ccb71ee1ac240d6f3df58d" +dependencies = [ + "bitflags 2.6.0", + "lazy_static", + "num-traits", + "rand 0.8.5", + "rand_chacha", + "rand_xorshift", + "regex-syntax 0.8.4", + "unarray", +] + [[package]] name = "prost" version = "0.12.6" @@ -5729,6 +5832,15 @@ dependencies = [ "getrandom", ] +[[package]] +name = "rand_xorshift" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d25bf25ec5ae4a3f1b92f929810509a2f53d7dca2f50b794ff57e3face536c8f" +dependencies = [ + "rand_core 0.6.4", +] + [[package]] name = "rand_xoshiro" version = "0.6.0" @@ -5927,6 +6039,7 @@ dependencies = [ "system-configuration 0.6.1", "tokio", "tokio-native-tls", + "tokio-socks", "tokio-util", "tower-service", "url", @@ -6980,6 +7093,12 @@ dependencies = [ "rand_core 0.6.4", ] +[[package]] +name = "simd-adler32" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d66dc143e6b11c1eddc06d5c423cfc97062865baf299914ab64caa38182078fe" + [[package]] name = "simdutf8" version = "0.1.5" @@ -7741,6 +7860,27 @@ dependencies = [ "zeroize", ] +[[package]] +name = "svm-rs" +version = "0.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "040017ebc08d781c457a3bfe9c5c2a99f902f8133eb91ef82b7876b053962ece" +dependencies = [ + "anyhow", + "const-hex", + "dirs", + "fs4", + "reqwest 0.12.7", + "semver", + "serde", + "serde_json", + "sha2 0.10.8", + "tempfile", + "thiserror", + "url", + "zip", +] + [[package]] name = "syn" version = "0.15.44" @@ -8186,6 +8326,18 @@ dependencies = [ "tokio", ] +[[package]] +name = "tokio-socks" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d4770b8024672c1101b3f6733eab95b18007dbe0847a8afe341fcf79e06043f" +dependencies = [ + "either", + "futures-util", + "thiserror", + "tokio", +] + [[package]] name = "tokio-stream" version = "0.1.16" @@ -8525,6 +8677,12 @@ dependencies = [ "libc", ] +[[package]] +name = "unarray" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eaea85b334db583fe3274d12b4cd1880032beab409c0d774be044d4480ab9a94" + [[package]] name = "unicase" version = "2.7.0" @@ -9309,6 +9467,23 @@ dependencies = [ "syn 2.0.77", ] +[[package]] +name = "zip" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc5e4288ea4057ae23afc69a4472434a87a2495cafce6632fd1c4ec9f5cf3494" +dependencies = [ + "arbitrary", + "crc32fast", + "crossbeam-utils", + "displaydoc", + "flate2", + "indexmap 2.5.0", + "memchr", + "thiserror", + "zopfli", +] + [[package]] name = "zk_evm" version = "0.131.0-rc.2" @@ -9958,9 +10133,11 @@ dependencies = [ "ethabi", "hex", "regex", + "reqwest 0.12.7", "semver", "serde", "serde_json", + "svm-rs", "tempfile", "thiserror", "tokio", @@ -9972,6 +10149,7 @@ dependencies = [ "zksync_queued_job_processor", "zksync_types", "zksync_utils", + "zksync_vm_interface", ] [[package]] @@ -11455,6 +11633,20 @@ dependencies = [ "zksync_types", ] +[[package]] +name = "zopfli" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5019f391bac5cf252e93bbcc53d039ffd62c7bfb7c150414d61369afe57e946" +dependencies = [ + "bumpalo", + "crc32fast", + "lockfree-object-pool", + "log", + "once_cell", + "simd-adler32", +] + [[package]] name = "zstd" version = "0.13.2" diff --git a/core/lib/config/src/configs/contract_verifier.rs b/core/lib/config/src/configs/contract_verifier.rs index 0016e1255de1..87b3418892f7 100644 --- a/core/lib/config/src/configs/contract_verifier.rs +++ b/core/lib/config/src/configs/contract_verifier.rs @@ -13,9 +13,9 @@ pub struct ContractVerifierConfig { pub polling_interval: Option, /// Port to which the Prometheus exporter server is listening. pub prometheus_port: u16, - pub threads_per_server: Option, + pub threads_per_server: Option, // FIXME: unused? pub port: u16, - pub url: String, + pub url: String, // FIXME: unused? } impl ContractVerifierConfig { diff --git a/core/lib/contract_verifier/Cargo.toml b/core/lib/contract_verifier/Cargo.toml index b53581a7aa9f..192593b1ebbb 100644 --- a/core/lib/contract_verifier/Cargo.toml +++ b/core/lib/contract_verifier/Cargo.toml @@ -31,3 +31,9 @@ tempfile.workspace = true regex.workspace = true tracing.workspace = true semver.workspace = true + +[dev-dependencies] +zksync_vm_interface.workspace = true + +reqwest.workspace = true +svm-rs = { version = "0.5.8", default-features = false, features = ["solc"] } # FIXME: add to deps diff --git a/core/lib/contract_verifier/src/lib.rs b/core/lib/contract_verifier/src/lib.rs index 47afc91cb9b6..d01f194edaa9 100644 --- a/core/lib/contract_verifier/src/lib.rs +++ b/core/lib/contract_verifier/src/lib.rs @@ -29,6 +29,8 @@ use crate::{ pub mod error; mod metrics; mod paths; +#[cfg(test)] +mod tests; mod zksolc_utils; mod zkvyper_utils; @@ -40,7 +42,7 @@ enum ConstructorArgs { #[derive(Debug, Clone)] pub struct ContractVerifier { - config: ContractVerifierConfig, + config: ContractVerifierConfig, // FIXME: replace with used fields contract_deployer: Contract, connection_pool: ConnectionPool, } @@ -252,6 +254,7 @@ impl ContractVerifier { ) -> ConstructorArgs { match calldata { DeployContractCalldata::Deploy(calldata) => { + // `unwrap`s below are safe as long as the contract deployer contract is correct. let contract_deployer = &self.contract_deployer; let create = contract_deployer.function("create").unwrap(); let create2 = contract_deployer.function("create2").unwrap(); diff --git a/core/lib/contract_verifier/src/tests.rs b/core/lib/contract_verifier/src/tests.rs new file mode 100644 index 000000000000..a0f2ddeeb888 --- /dev/null +++ b/core/lib/contract_verifier/src/tests.rs @@ -0,0 +1,202 @@ +//! Tests for the contract verifier. + +use std::{os::unix::fs::PermissionsExt, path::Path}; + +use tokio::{fs, sync::watch}; +use zksync_dal::Connection; +use zksync_types::{ + contract_verification_api::{CompilerVersions, VerificationIncomingRequest}, + get_code_key, get_known_code_key, + tx::IncludedTxLocation, + L1BatchNumber, L2BlockNumber, StorageLog, CONTRACT_DEPLOYER_ADDRESS, H256, +}; +use zksync_utils::{ + address_to_h256, + bytecode::{hash_bytecode, validate_bytecode}, +}; +use zksync_vm_interface::VmEvent; + +use super::*; + +const SOLC_VERSION: &str = "0.8.27"; +const ZKSOLC_VERSION: &str = "1.5.4"; + +async fn mock_deployment(storage: &mut Connection<'_, Core>, address: Address, bytecode: Vec) { + let bytecode_hash = hash_bytecode(&bytecode); + let logs = [ + StorageLog::new_write_log(get_code_key(&address), bytecode_hash), + StorageLog::new_write_log(get_known_code_key(&bytecode_hash), H256::from_low_u64_be(1)), + ]; + storage + .storage_logs_dal() + .append_storage_logs(L2BlockNumber(0), &logs) + .await + .unwrap(); + storage + .factory_deps_dal() + .insert_factory_deps( + L2BlockNumber(0), + &HashMap::from([(bytecode_hash, bytecode)]), + ) + .await + .unwrap(); + + let deployer_address = Address::repeat_byte(0xff); + let location = IncludedTxLocation { + tx_hash: H256::zero(), + tx_index_in_l2_block: 0, + tx_initiator_address: deployer_address, + }; + let deploy_event = VmEvent { + location: (L1BatchNumber(0), 0), + address: CONTRACT_DEPLOYER_ADDRESS, + indexed_topics: vec![ + VmEvent::DEPLOY_EVENT_SIGNATURE, + address_to_h256(&deployer_address), + bytecode_hash, + address_to_h256(&address), + ], + value: vec![], + }; + storage + .events_dal() + .save_events(L2BlockNumber(0), &[(location, vec![&deploy_event])]) + .await + .unwrap(); +} + +async fn install_zksolc(path: &Path) { + if fs::try_exists(path).await.unwrap() { + return; + } + + // We may race from several test processes here; this is OK-ish for tests. + let version = ZKSOLC_VERSION; + let compiler_prefix = match svm::platform() { + svm::Platform::LinuxAmd64 => "zksolc-linux-amd64-musl-", + svm::Platform::LinuxAarch64 => "zksolc-linux-arm64-musl-", + svm::Platform::MacOsAmd64 => "zksolc-macosx-amd64-", + svm::Platform::MacOsAarch64 => "zksolc-macosx-arm64-", + other => panic!("Unsupported platform: {other:?}"), + }; + let download_url = format!( + "https://github.com/matter-labs/zksolc-bin/releases/download/v{version}/{compiler_prefix}v{version}", + ); + let response = reqwest::Client::new() + .get(&download_url) + .send() + .await + .unwrap() + .error_for_status() + .unwrap(); + let response_bytes = response.bytes().await.unwrap(); + + fs::create_dir_all(path.parent().unwrap()).await.unwrap(); + fs::write(path, &response_bytes).await.unwrap(); + // Set the executable flag for the file. + fs::set_permissions(path, PermissionsExt::from_mode(0o755)) + .await + .unwrap(); +} + +async fn prepare_compilers() -> CompilerPaths { + let solc_version = SOLC_VERSION.parse().unwrap(); + let solc_path = svm::install(&solc_version) + .await + .expect("failed installing solc"); + let zksolc_path = svm::data_dir() + .join(format!("zksolc-{ZKSOLC_VERSION}")) + .join("zksolc"); + install_zksolc(&zksolc_path).await; + + CompilerPaths { + base: solc_path, + zk: zksolc_path, + } +} + +fn test_request(address: Address) -> VerificationIncomingRequest { + let contract_source = r#" + contract Counter { + uint256 value; + + function increment(uint256 x) external { + value += x; + } + } + "#; + VerificationIncomingRequest { + contract_address: address, + source_code_data: SourceCodeData::SolSingleFile(contract_source.into()), + contract_name: "Counter".to_owned(), + compiler_versions: CompilerVersions::Solc { + compiler_zksolc_version: ZKSOLC_VERSION.to_owned(), + compiler_solc_version: SOLC_VERSION.to_owned(), + }, + optimization_used: true, + optimizer_mode: None, + constructor_arguments: Default::default(), + is_system: false, + force_evmla: false, + } +} + +async fn compile_counter() -> CompilationArtifacts { + let compilers_path = prepare_compilers().await; + let req = test_request(Address::repeat_byte(1)); + let input = ContractVerifier::build_zksolc_input(VerificationRequest { id: 0, req }).unwrap(); + ZkSolc::new(compilers_path, ZKSOLC_VERSION.to_owned()) + .async_compile(input) + .await + .unwrap() +} + +#[tokio::test] +async fn compiler_works() { + let output = compile_counter().await; + validate_bytecode(&output.bytecode).unwrap(); + let items = output.abi.as_array().unwrap(); + assert_eq!(items.len(), 1); + let increment_function = items[0].as_object().unwrap(); + assert_eq!(increment_function["type"], "function"); + assert_eq!(increment_function["name"], "increment"); +} + +// FIXME: needs customizing compiler resolution to work +#[tokio::test] +async fn contract_verifier_basics() { + let output = compile_counter().await; + let pool = ConnectionPool::test_pool().await; + let mut storage = pool.connection().await.unwrap(); + let address = Address::repeat_byte(1); + mock_deployment(&mut storage, address, output.bytecode.clone()).await; + let req = test_request(address); + storage + .contract_verification_dal() + .add_contract_verification_request(req) + .await + .unwrap(); + + let config = ContractVerifierConfig { + compilation_timeout: 86_400, // sec + polling_interval: Some(50), // ms + + // Fields below are unused + prometheus_port: 0, + threads_per_server: None, + port: 0, + url: "".to_string(), + }; + let verifier = ContractVerifier::new(config, pool.clone()); + let (_stop_sender, stop_receiver) = watch::channel(false); + verifier.run(stop_receiver, Some(1)).await.unwrap(); + + let verification_info = storage + .contract_verification_dal() + .get_contract_verification_info(address) + .await + .unwrap() + .expect("no verification info"); + assert_eq!(verification_info.artifacts.bytecode, output.bytecode); + assert_eq!(verification_info.artifacts.abi, output.abi); +} From 9f2bf2fdf3dfb64881ecfbd7758854d10f7c0289 Mon Sep 17 00:00:00 2001 From: Alex Ostrovski Date: Mon, 28 Oct 2024 16:53:20 +0200 Subject: [PATCH 07/19] Make verifier tests work --- Cargo.lock | 1 + core/lib/contract_verifier/Cargo.toml | 1 + core/lib/contract_verifier/src/lib.rs | 109 ++++++++------ core/lib/contract_verifier/src/paths.rs | 68 +++++++-- core/lib/contract_verifier/src/tests.rs | 191 +++++++++++++++++------- core/node/test_utils/src/lib.rs | 2 +- 6 files changed, 253 insertions(+), 119 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f96a27604918..f2c7b5fc4b6b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -10146,6 +10146,7 @@ dependencies = [ "zksync_config", "zksync_contracts", "zksync_dal", + "zksync_node_test_utils", "zksync_queued_job_processor", "zksync_types", "zksync_utils", diff --git a/core/lib/contract_verifier/Cargo.toml b/core/lib/contract_verifier/Cargo.toml index 192593b1ebbb..4f6e1ed273e9 100644 --- a/core/lib/contract_verifier/Cargo.toml +++ b/core/lib/contract_verifier/Cargo.toml @@ -33,6 +33,7 @@ tracing.workspace = true semver.workspace = true [dev-dependencies] +zksync_node_test_utils.workspace = true zksync_vm_interface.workspace = true reqwest.workspace = true diff --git a/core/lib/contract_verifier/src/lib.rs b/core/lib/contract_verifier/src/lib.rs index d01f194edaa9..508e47bb12b4 100644 --- a/core/lib/contract_verifier/src/lib.rs +++ b/core/lib/contract_verifier/src/lib.rs @@ -1,5 +1,6 @@ use std::{ collections::HashMap, + sync::Arc, time::{Duration, Instant}, }; @@ -13,7 +14,7 @@ use zksync_queued_job_processor::{async_trait, JobProcessor}; use zksync_types::{ contract_verification_api::{ CompilationArtifacts, CompilerType, DeployContractCalldata, SourceCodeData, - VerificationInfo, VerificationRequest, + VerificationIncomingRequest, VerificationInfo, VerificationRequest, }, Address, }; @@ -21,7 +22,7 @@ use zksync_types::{ use crate::{ error::ContractVerifierError, metrics::API_CONTRACT_VERIFIER_METRICS, - paths::CompilerPaths, + paths::{CompilerResolver, EnvCompilerResolver}, zksolc_utils::{Optimizer, Settings, Source, StandardJson, ZkSolc, ZkSolcInput}, zkvyper_utils::{ZkVyper, ZkVyperInput}, }; @@ -45,6 +46,7 @@ pub struct ContractVerifier { config: ContractVerifierConfig, // FIXME: replace with used fields contract_deployer: Contract, connection_pool: ConnectionPool, + compiler_resolver: Arc, } impl ContractVerifier { @@ -53,6 +55,7 @@ impl ContractVerifier { config, contract_deployer: zksync_contracts::deployer_contract(), connection_pool, + compiler_resolver: Arc::::default(), } } @@ -60,17 +63,17 @@ impl ContractVerifier { &self, mut request: VerificationRequest, ) -> Result { - let artifacts = Self::compile(request.clone(), &self.config).await?; + let artifacts = self.compile(request.req.clone()).await?; // Bytecode should be present because it is checked when accepting request. let mut storage = self .connection_pool .connection_tagged("contract_verifier") .await?; - let (deployed_bytecode, creation_tx_calldata) = storage + let (deployed_bytecode, creation_tx_calldata) = dbg!(storage .contract_verification_dal() .get_contract_info_for_verification(request.req.contract_address) - .await? + .await)? .with_context(|| { format!( "Contract is missing in DB for already accepted verification request. Contract address: {:#?}", @@ -113,55 +116,66 @@ impl ContractVerifier { } async fn compile_zksolc( - request: VerificationRequest, - config: &ContractVerifierConfig, + &self, + req: VerificationIncomingRequest, ) -> Result { - let compiler_paths = CompilerPaths::for_solc(&request.req.compiler_versions).await?; - let input = Self::build_zksolc_input(request.clone())?; - let zksolc = ZkSolc::new( - compiler_paths, - request.req.compiler_versions.zk_compiler_version(), - ); - - time::timeout(config.compilation_timeout(), zksolc.async_compile(input)) - .await - .map_err(|_| ContractVerifierError::CompilationTimeout)? + let compiler_paths = self + .compiler_resolver + .resolve_solc(&req.compiler_versions) + .await?; + dbg!(&compiler_paths); + let zksolc_version = req.compiler_versions.zk_compiler_version(); + let input = Self::build_zksolc_input(req)?; + let zksolc = ZkSolc::new(compiler_paths, zksolc_version); + + time::timeout( + self.config.compilation_timeout(), + zksolc.async_compile(input), + ) + .await + .map_err(|_| ContractVerifierError::CompilationTimeout)? } async fn compile_zkvyper( - request: VerificationRequest, - config: &ContractVerifierConfig, + &self, + req: VerificationIncomingRequest, ) -> Result { - let compiler_paths = CompilerPaths::for_vyper(&request.req.compiler_versions).await?; - let input = Self::build_zkvyper_input(request.clone())?; + let compiler_paths = self + .compiler_resolver + .resolve_vyper(&req.compiler_versions) + .await?; + let input = Self::build_zkvyper_input(req)?; let zkvyper = ZkVyper::new(compiler_paths); - time::timeout(config.compilation_timeout(), zkvyper.async_compile(input)) - .await - .map_err(|_| ContractVerifierError::CompilationTimeout)? + time::timeout( + self.config.compilation_timeout(), + zkvyper.async_compile(input), + ) + .await + .map_err(|_| ContractVerifierError::CompilationTimeout)? } pub async fn compile( - request: VerificationRequest, - config: &ContractVerifierConfig, + &self, + req: VerificationIncomingRequest, ) -> Result { - match request.req.source_code_data.compiler_type() { - CompilerType::Solc => Self::compile_zksolc(request, config).await, - CompilerType::Vyper => Self::compile_zkvyper(request, config).await, + match req.source_code_data.compiler_type() { + CompilerType::Solc => dbg!(self.compile_zksolc(req).await), + CompilerType::Vyper => self.compile_zkvyper(req).await, } } fn build_zksolc_input( - request: VerificationRequest, + req: VerificationIncomingRequest, ) -> Result { // Users may provide either just contract name or // source file name and contract name joined with ":". let (file_name, contract_name) = - if let Some((file_name, contract_name)) = request.req.contract_name.rsplit_once(':') { + if let Some((file_name, contract_name)) = req.contract_name.rsplit_once(':') { (file_name.to_string(), contract_name.to_string()) } else { ( - format!("{}.sol", request.req.contract_name), - request.req.contract_name.clone(), + format!("{}.sol", req.contract_name), + req.contract_name.clone(), ) }; let default_output_selection = serde_json::json!({ @@ -171,22 +185,22 @@ impl ContractVerifier { } }); - match request.req.source_code_data { + match req.source_code_data { SourceCodeData::SolSingleFile(source_code) => { let source = Source { content: source_code, }; let sources = HashMap::from([(file_name.clone(), source)]); let optimizer = Optimizer { - enabled: request.req.optimization_used, - mode: request.req.optimizer_mode.and_then(|s| s.chars().next()), + enabled: req.optimization_used, + mode: req.optimizer_mode.and_then(|s| s.chars().next()), }; let optimizer_value = serde_json::to_value(optimizer).unwrap(); let settings = Settings { output_selection: Some(default_output_selection), - is_system: request.req.is_system, - force_evmla: request.req.force_evmla, + is_system: req.is_system, + force_evmla: req.force_evmla, other: serde_json::Value::Object( vec![("optimizer".to_string(), optimizer_value)] .into_iter() @@ -218,32 +232,31 @@ impl ContractVerifier { } SourceCodeData::YulSingleFile(source_code) => Ok(ZkSolcInput::YulSingleFile { source_code, - is_system: request.req.is_system, + is_system: req.is_system, }), other => unreachable!("Unexpected `SourceCodeData` variant: {other:?}"), } } fn build_zkvyper_input( - request: VerificationRequest, + req: VerificationIncomingRequest, ) -> Result { // Users may provide either just contract name or // source file name and contract name joined with ":". - let contract_name = - if let Some((_, contract_name)) = request.req.contract_name.rsplit_once(':') { - contract_name.to_owned() - } else { - request.req.contract_name.clone() - }; + let contract_name = if let Some((_, contract_name)) = req.contract_name.rsplit_once(':') { + contract_name.to_owned() + } else { + req.contract_name.clone() + }; - let sources = match request.req.source_code_data { + let sources = match req.source_code_data { SourceCodeData::VyperMultiFile(s) => s, other => unreachable!("unexpected `SourceCodeData` variant: {other:?}"), }; Ok(ZkVyperInput { contract_name, sources, - optimizer_mode: request.req.optimizer_mode, + optimizer_mode: req.optimizer_mode, }) } diff --git a/core/lib/contract_verifier/src/paths.rs b/core/lib/contract_verifier/src/paths.rs index 9775d1f48d10..4f5e487031b0 100644 --- a/core/lib/contract_verifier/src/paths.rs +++ b/core/lib/contract_verifier/src/paths.rs @@ -1,17 +1,14 @@ -use std::path::{Path, PathBuf}; +use std::{fmt, path::PathBuf}; use anyhow::Context as _; use tokio::fs; +use zksync_queued_job_processor::async_trait; use zksync_types::contract_verification_api::CompilerVersions; use zksync_utils::env::Workspace; use crate::error::ContractVerifierError; -fn home_path() -> PathBuf { - Workspace::locate().core() -} - -#[derive(Debug)] +#[derive(Debug, Clone)] pub(crate) struct CompilerPaths { /// Path to the base (non-zk) compiler. pub base: PathBuf, @@ -19,10 +16,45 @@ pub(crate) struct CompilerPaths { pub zk: PathBuf, } -impl CompilerPaths { - pub async fn for_solc(versions: &CompilerVersions) -> Result { +/// Encapsulates compiler paths resolution. +#[async_trait] +pub(crate) trait CompilerResolver: fmt::Debug + Send + Sync { + /// Resolves paths to `solc` / `zksolc`. + async fn resolve_solc( + &self, + versions: &CompilerVersions, + ) -> Result; + + /// Resolves paths to `vyper` / `zkvyper`. + async fn resolve_vyper( + &self, + versions: &CompilerVersions, + ) -> Result; +} + +/// Default [`CompilerResolver`] using pre-downloaded compilers. +#[derive(Debug)] +pub(crate) struct EnvCompilerResolver { + home_dir: PathBuf, +} + +impl Default for EnvCompilerResolver { + fn default() -> Self { + Self { + home_dir: Workspace::locate().core(), + } + } +} + +#[async_trait] +impl CompilerResolver for EnvCompilerResolver { + async fn resolve_solc( + &self, + versions: &CompilerVersions, + ) -> Result { let zksolc_version = versions.zk_compiler_version(); - let zksolc_path = Path::new(&home_path()) + let zksolc_path = self + .home_dir .join("etc") .join("zksolc-bin") .join(&zksolc_version) @@ -38,7 +70,8 @@ impl CompilerPaths { } let solc_version = versions.compiler_version(); - let solc_path = Path::new(&home_path()) + let solc_path = self + .home_dir .join("etc") .join("solc-bin") .join(&solc_version) @@ -52,15 +85,19 @@ impl CompilerPaths { solc_version, )); } - Ok(Self { + Ok(CompilerPaths { base: solc_path, zk: zksolc_path, }) } - pub async fn for_vyper(versions: &CompilerVersions) -> Result { + async fn resolve_vyper( + &self, + versions: &CompilerVersions, + ) -> Result { let zkvyper_version = versions.zk_compiler_version(); - let zkvyper_path = Path::new(&home_path()) + let zkvyper_path = self + .home_dir .join("etc") .join("zkvyper-bin") .join(&zkvyper_version) @@ -76,7 +113,8 @@ impl CompilerPaths { } let vyper_version = versions.compiler_version(); - let vyper_path = Path::new(&home_path()) + let vyper_path = self + .home_dir .join("etc") .join("vyper-bin") .join(&vyper_version) @@ -90,7 +128,7 @@ impl CompilerPaths { vyper_version, )); } - Ok(Self { + Ok(CompilerPaths { base: vyper_path, zk: zkvyper_path, }) diff --git a/core/lib/contract_verifier/src/tests.rs b/core/lib/contract_verifier/src/tests.rs index a0f2ddeeb888..0a190ab178f6 100644 --- a/core/lib/contract_verifier/src/tests.rs +++ b/core/lib/contract_verifier/src/tests.rs @@ -4,19 +4,23 @@ use std::{os::unix::fs::PermissionsExt, path::Path}; use tokio::{fs, sync::watch}; use zksync_dal::Connection; +use zksync_node_test_utils::{create_l1_batch, create_l2_block}; use zksync_types::{ contract_verification_api::{CompilerVersions, VerificationIncomingRequest}, get_code_key, get_known_code_key, + l2::L2Tx, tx::IncludedTxLocation, - L1BatchNumber, L2BlockNumber, StorageLog, CONTRACT_DEPLOYER_ADDRESS, H256, + Execute, L1BatchNumber, L2BlockNumber, ProtocolVersion, StorageLog, CONTRACT_DEPLOYER_ADDRESS, + H256, }; use zksync_utils::{ address_to_h256, bytecode::{hash_bytecode, validate_bytecode}, }; -use zksync_vm_interface::VmEvent; +use zksync_vm_interface::{TransactionExecutionMetrics, VmEvent}; use super::*; +use crate::paths::CompilerPaths; const SOLC_VERSION: &str = "0.8.27"; const ZKSOLC_VERSION: &str = "1.5.4"; @@ -36,14 +40,27 @@ async fn mock_deployment(storage: &mut Connection<'_, Core>, address: Address, b .factory_deps_dal() .insert_factory_deps( L2BlockNumber(0), - &HashMap::from([(bytecode_hash, bytecode)]), + &HashMap::from([(bytecode_hash, bytecode.clone())]), ) .await .unwrap(); + let mut deploy_tx = L2Tx { + execute: Execute::for_deploy(H256::zero(), bytecode, &[]), + common_data: Default::default(), + received_timestamp_ms: 0, + raw_bytes: Some(vec![0; 128].into()), + }; + deploy_tx.set_input(vec![0; 128], H256::repeat_byte(0x23)); + storage + .transactions_dal() + .insert_transaction_l2(&deploy_tx, TransactionExecutionMetrics::default()) + .await + .unwrap(); + let deployer_address = Address::repeat_byte(0xff); let location = IncludedTxLocation { - tx_hash: H256::zero(), + tx_hash: deploy_tx.hash(), tx_index_in_l2_block: 0, tx_initiator_address: deployer_address, }; @@ -65,53 +82,88 @@ async fn mock_deployment(storage: &mut Connection<'_, Core>, address: Address, b .unwrap(); } -async fn install_zksolc(path: &Path) { - if fs::try_exists(path).await.unwrap() { - return; +/// Test compiler resolver. +#[derive(Debug)] +struct TestCompilerResolver(CompilerPaths); + +impl TestCompilerResolver { + async fn install_zksolc(path: &Path) { + if fs::try_exists(path).await.unwrap() { + return; + } + + // We may race from several test processes here; this is OK-ish for tests. + let version = ZKSOLC_VERSION; + let compiler_prefix = match svm::platform() { + svm::Platform::LinuxAmd64 => "zksolc-linux-amd64-musl-", + svm::Platform::LinuxAarch64 => "zksolc-linux-arm64-musl-", + svm::Platform::MacOsAmd64 => "zksolc-macosx-amd64-", + svm::Platform::MacOsAarch64 => "zksolc-macosx-arm64-", + other => panic!("Unsupported platform: {other:?}"), + }; + let download_url = format!( + "https://github.com/matter-labs/zksolc-bin/releases/download/v{version}/{compiler_prefix}v{version}", + ); + let response = reqwest::Client::new() + .get(&download_url) + .send() + .await + .unwrap() + .error_for_status() + .unwrap(); + let response_bytes = response.bytes().await.unwrap(); + + fs::create_dir_all(path.parent().unwrap()).await.unwrap(); + fs::write(path, &response_bytes).await.unwrap(); + // Set the executable flag for the file. + fs::set_permissions(path, PermissionsExt::from_mode(0o755)) + .await + .unwrap(); } - // We may race from several test processes here; this is OK-ish for tests. - let version = ZKSOLC_VERSION; - let compiler_prefix = match svm::platform() { - svm::Platform::LinuxAmd64 => "zksolc-linux-amd64-musl-", - svm::Platform::LinuxAarch64 => "zksolc-linux-arm64-musl-", - svm::Platform::MacOsAmd64 => "zksolc-macosx-amd64-", - svm::Platform::MacOsAarch64 => "zksolc-macosx-arm64-", - other => panic!("Unsupported platform: {other:?}"), - }; - let download_url = format!( - "https://github.com/matter-labs/zksolc-bin/releases/download/v{version}/{compiler_prefix}v{version}", - ); - let response = reqwest::Client::new() - .get(&download_url) - .send() - .await - .unwrap() - .error_for_status() - .unwrap(); - let response_bytes = response.bytes().await.unwrap(); + async fn new() -> Self { + let solc_version = SOLC_VERSION.parse().unwrap(); + let solc_path = svm::install(&solc_version) + .await + .expect("failed installing solc"); + let zksolc_path = svm::data_dir() + .join(format!("zksolc-{ZKSOLC_VERSION}")) + .join("zksolc"); + Self::install_zksolc(&zksolc_path).await; - fs::create_dir_all(path.parent().unwrap()).await.unwrap(); - fs::write(path, &response_bytes).await.unwrap(); - // Set the executable flag for the file. - fs::set_permissions(path, PermissionsExt::from_mode(0o755)) - .await - .unwrap(); + Self(CompilerPaths { + base: solc_path, + zk: zksolc_path, + }) + } } -async fn prepare_compilers() -> CompilerPaths { - let solc_version = SOLC_VERSION.parse().unwrap(); - let solc_path = svm::install(&solc_version) - .await - .expect("failed installing solc"); - let zksolc_path = svm::data_dir() - .join(format!("zksolc-{ZKSOLC_VERSION}")) - .join("zksolc"); - install_zksolc(&zksolc_path).await; - - CompilerPaths { - base: solc_path, - zk: zksolc_path, +#[async_trait] +impl CompilerResolver for TestCompilerResolver { + async fn resolve_solc( + &self, + versions: &CompilerVersions, + ) -> Result { + if versions.compiler_version() != SOLC_VERSION { + return Err(ContractVerifierError::UnknownCompilerVersion( + "solc".to_owned(), + versions.compiler_version(), + )); + } + if versions.zk_compiler_version() != ZKSOLC_VERSION { + return Err(ContractVerifierError::UnknownCompilerVersion( + "zksolc".to_owned(), + versions.zk_compiler_version(), + )); + } + Ok(self.0.clone()) + } + + async fn resolve_vyper( + &self, + _versions: &CompilerVersions, + ) -> Result { + unreachable!("not tested") } } @@ -141,11 +193,10 @@ fn test_request(address: Address) -> VerificationIncomingRequest { } } -async fn compile_counter() -> CompilationArtifacts { - let compilers_path = prepare_compilers().await; +async fn compile_counter(compiler_paths: CompilerPaths) -> CompilationArtifacts { let req = test_request(Address::repeat_byte(1)); - let input = ContractVerifier::build_zksolc_input(VerificationRequest { id: 0, req }).unwrap(); - ZkSolc::new(compilers_path, ZKSOLC_VERSION.to_owned()) + let input = ContractVerifier::build_zksolc_input(req).unwrap(); + ZkSolc::new(compiler_paths, ZKSOLC_VERSION.to_owned()) .async_compile(input) .await .unwrap() @@ -153,7 +204,8 @@ async fn compile_counter() -> CompilationArtifacts { #[tokio::test] async fn compiler_works() { - let output = compile_counter().await; + let compiler_paths = TestCompilerResolver::new().await.0; + let output = compile_counter(compiler_paths).await; validate_bytecode(&output.bytecode).unwrap(); let items = output.abi.as_array().unwrap(); assert_eq!(items.len(), 1); @@ -162,16 +214,34 @@ async fn compiler_works() { assert_eq!(increment_function["name"], "increment"); } -// FIXME: needs customizing compiler resolution to work #[tokio::test] async fn contract_verifier_basics() { - let output = compile_counter().await; + let test_resolver = TestCompilerResolver::new().await; + let output = compile_counter(test_resolver.0.clone()).await; let pool = ConnectionPool::test_pool().await; let mut storage = pool.connection().await.unwrap(); + + // Storage must contain at least 1 block / batch for verifier-related queries to work correctly. + storage + .protocol_versions_dal() + .save_protocol_version_with_tx(&ProtocolVersion::default()) + .await + .unwrap(); + storage + .blocks_dal() + .insert_l2_block(&create_l2_block(0)) + .await + .unwrap(); + storage + .blocks_dal() + .insert_mock_l1_batch(&create_l1_batch(0)) + .await + .unwrap(); + let address = Address::repeat_byte(1); mock_deployment(&mut storage, address, output.bytecode.clone()).await; let req = test_request(address); - storage + let request_id = storage .contract_verification_dal() .add_contract_verification_request(req) .await @@ -187,10 +257,21 @@ async fn contract_verifier_basics() { port: 0, url: "".to_string(), }; - let verifier = ContractVerifier::new(config, pool.clone()); + let mut verifier = ContractVerifier::new(config, pool.clone()); + verifier.compiler_resolver = Arc::new(test_resolver); let (_stop_sender, stop_receiver) = watch::channel(false); verifier.run(stop_receiver, Some(1)).await.unwrap(); + let status = storage + .contract_verification_dal() + .get_verification_request_status(request_id) + .await + .unwrap() + .expect("no status"); + assert_eq!(status.error, None); + assert_eq!(status.compilation_errors, None); + assert_eq!(status.status, "successful"); + let verification_info = storage .contract_verification_dal() .get_contract_verification_info(address) diff --git a/core/node/test_utils/src/lib.rs b/core/node/test_utils/src/lib.rs index 86ce3aadd9a1..3f05375213de 100644 --- a/core/node/test_utils/src/lib.rs +++ b/core/node/test_utils/src/lib.rs @@ -9,7 +9,7 @@ use zksync_multivm::{ interface::{TransactionExecutionResult, TxExecutionStatus, VmExecutionMetrics}, utils::get_max_gas_per_pubdata_byte, }; -use zksync_node_genesis::GenesisParams; +use zksync_node_genesis::GenesisParams; // FIXME: not necessary use zksync_system_constants::{get_intrinsic_constants, ZKPORTER_IS_AVAILABLE}; use zksync_types::{ block::{L1BatchHeader, L2BlockHeader}, From 157e1d685e35d89320f396abe10549a88d1d8db2 Mon Sep 17 00:00:00 2001 From: Alex Ostrovski Date: Mon, 28 Oct 2024 17:05:01 +0200 Subject: [PATCH 08/19] Remove unnecessary deps from `test_utils` --- Cargo.lock | 3 +- core/node/consensus/src/storage/testonly.rs | 3 +- .../state_keeper/src/executor/tests/tester.rs | 5 +-- core/node/test_utils/Cargo.toml | 3 +- core/node/test_utils/src/lib.rs | 33 ++++++++++--------- 5 files changed, 25 insertions(+), 22 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f2c7b5fc4b6b..7e30b5721039 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -11028,11 +11028,10 @@ dependencies = [ "zksync_contracts", "zksync_dal", "zksync_merkle_tree", - "zksync_multivm", - "zksync_node_genesis", "zksync_system_constants", "zksync_types", "zksync_utils", + "zksync_vm_interface", ] [[package]] diff --git a/core/node/consensus/src/storage/testonly.rs b/core/node/consensus/src/storage/testonly.rs index 2aed011d23cf..0f29e2468267 100644 --- a/core/node/consensus/src/storage/testonly.rs +++ b/core/node/consensus/src/storage/testonly.rs @@ -73,7 +73,8 @@ impl ConnectionPool { L1BatchNumber(23), L2BlockNumber(87), vec![], - mock_genesis_params(protocol_version), + &BaseSystemContracts::load_from_disk(), + protocol_version, )) .await } diff --git a/core/node/state_keeper/src/executor/tests/tester.rs b/core/node/state_keeper/src/executor/tests/tester.rs index a02aeb47cafa..800bf398938d 100644 --- a/core/node/state_keeper/src/executor/tests/tester.rs +++ b/core/node/state_keeper/src/executor/tests/tester.rs @@ -19,7 +19,7 @@ use zksync_multivm::{ utils::StorageWritesDeduplicator, vm_latest::constants::INITIAL_STORAGE_WRITE_PUBDATA_BYTES, }; -use zksync_node_genesis::{create_genesis_l1_batch, GenesisParams}; +use zksync_node_genesis::create_genesis_l1_batch; use zksync_node_test_utils::{recover, Snapshot}; use zksync_state::{OwnedStorage, ReadStorageFactory, RocksdbStorageOptions}; use zksync_test_account::{Account, DeployContractsTx, TxType}; @@ -602,7 +602,8 @@ impl StorageSnapshot { L1BatchNumber(1), self.l2_block_number, snapshot_logs, - GenesisParams::mock(), + &BASE_SYSTEM_CONTRACTS, + ProtocolVersionId::latest(), ); let mut snapshot = recover(&mut storage, snapshot).await; snapshot.l2_block_hash = self.l2_block_hash; diff --git a/core/node/test_utils/Cargo.toml b/core/node/test_utils/Cargo.toml index af60008df570..6df100c51a7d 100644 --- a/core/node/test_utils/Cargo.toml +++ b/core/node/test_utils/Cargo.toml @@ -11,11 +11,10 @@ keywords.workspace = true categories.workspace = true [dependencies] -zksync_multivm.workspace = true zksync_types.workspace = true zksync_dal.workspace = true zksync_contracts.workspace = true zksync_merkle_tree.workspace = true zksync_system_constants.workspace = true +zksync_vm_interface.workspace = true zksync_utils.workspace = true -zksync_node_genesis.workspace = true diff --git a/core/node/test_utils/src/lib.rs b/core/node/test_utils/src/lib.rs index 3f05375213de..2b446fff12c5 100644 --- a/core/node/test_utils/src/lib.rs +++ b/core/node/test_utils/src/lib.rs @@ -2,14 +2,9 @@ use std::collections::HashMap; -use zksync_contracts::BaseSystemContractsHashes; +use zksync_contracts::{BaseSystemContracts, BaseSystemContractsHashes}; use zksync_dal::{Connection, Core, CoreDal}; use zksync_merkle_tree::{domain::ZkSyncTree, TreeInstruction}; -use zksync_multivm::{ - interface::{TransactionExecutionResult, TxExecutionStatus, VmExecutionMetrics}, - utils::get_max_gas_per_pubdata_byte, -}; -use zksync_node_genesis::GenesisParams; // FIXME: not necessary use zksync_system_constants::{get_intrinsic_constants, ZKPORTER_IS_AVAILABLE}; use zksync_types::{ block::{L1BatchHeader, L2BlockHeader}, @@ -27,6 +22,10 @@ use zksync_types::{ Address, K256PrivateKey, L1BatchNumber, L2BlockNumber, L2ChainId, Nonce, ProtocolVersion, ProtocolVersionId, StorageLog, H256, U256, }; +use zksync_vm_interface::{TransactionExecutionResult, TxExecutionStatus, VmExecutionMetrics}; + +/// Value for recent protocol versions. +const MAX_GAS_PER_PUBDATA_BYTE: u64 = 50_000; /// Creates an L2 block header with the specified number and deterministic contents. pub fn create_l2_block(number: u32) -> L2BlockHeader { @@ -39,7 +38,7 @@ pub fn create_l2_block(number: u32) -> L2BlockHeader { base_fee_per_gas: 100, batch_fee_input: BatchFeeInput::l1_pegged(100, 100), fee_account_address: Address::zero(), - gas_per_pubdata_limit: get_max_gas_per_pubdata_byte(ProtocolVersionId::latest().into()), + gas_per_pubdata_limit: MAX_GAS_PER_PUBDATA_BYTE, base_system_contracts_hashes: BaseSystemContractsHashes::default(), protocol_version: Some(ProtocolVersionId::latest()), virtual_blocks: 1, @@ -195,14 +194,14 @@ impl Snapshot { l1_batch: L1BatchNumber, l2_block: L2BlockNumber, storage_logs: Vec, - genesis_params: GenesisParams, + contracts: &BaseSystemContracts, + protocol_version: ProtocolVersionId, ) -> Self { - let contracts = genesis_params.base_system_contracts(); let l1_batch = L1BatchHeader::new( l1_batch, l1_batch.0.into(), contracts.hashes(), - genesis_params.minor_protocol_version(), + protocol_version, ); let l2_block = L2BlockHeader { number: l2_block, @@ -213,11 +212,9 @@ impl Snapshot { base_fee_per_gas: 100, batch_fee_input: BatchFeeInput::l1_pegged(100, 100), fee_account_address: Address::zero(), - gas_per_pubdata_limit: get_max_gas_per_pubdata_byte( - genesis_params.minor_protocol_version().into(), - ), + gas_per_pubdata_limit: MAX_GAS_PER_PUBDATA_BYTE, base_system_contracts_hashes: contracts.hashes(), - protocol_version: Some(genesis_params.minor_protocol_version()), + protocol_version: Some(protocol_version), virtual_blocks: 1, gas_limit: 0, logs_bloom: Default::default(), @@ -253,7 +250,13 @@ pub async fn prepare_recovery_snapshot( enumeration_index: i as u64 + 1, }) .collect(); - let snapshot = Snapshot::new(l1_batch, l2_block, storage_logs, GenesisParams::mock()); + let snapshot = Snapshot::new( + l1_batch, + l2_block, + storage_logs, + &BaseSystemContracts::load_from_disk(), + ProtocolVersionId::latest(), + ); recover(storage, snapshot).await } From 953a5805e192e533d8f08b336578fce20a93f35a Mon Sep 17 00:00:00 2001 From: Alex Ostrovski Date: Mon, 28 Oct 2024 17:09:19 +0200 Subject: [PATCH 09/19] Remove unnecessary deps from verifier lib --- Cargo.lock | 1 - core/bin/contract-verifier/src/main.rs | 2 +- core/lib/contract_verifier/Cargo.toml | 1 - core/lib/contract_verifier/src/lib.rs | 27 +++++++++---------------- core/lib/contract_verifier/src/tests.rs | 13 +----------- 5 files changed, 12 insertions(+), 32 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 7e30b5721039..296ffccfaf39 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -10143,7 +10143,6 @@ dependencies = [ "tokio", "tracing", "vise", - "zksync_config", "zksync_contracts", "zksync_dal", "zksync_node_test_utils", diff --git a/core/bin/contract-verifier/src/main.rs b/core/bin/contract-verifier/src/main.rs index a8162de13e9d..527e14c5ee2a 100644 --- a/core/bin/contract-verifier/src/main.rs +++ b/core/bin/contract-verifier/src/main.rs @@ -162,7 +162,7 @@ async fn main() -> anyhow::Result<()> { update_compiler_versions(&pool).await; - let contract_verifier = ContractVerifier::new(verifier_config, pool); + let contract_verifier = ContractVerifier::new(verifier_config.compilation_timeout(), pool); let tasks = vec![ // TODO PLA-335: Leftovers after the prover DB split. // The prover connection pool is not used by the contract verifier, but we need to pass it diff --git a/core/lib/contract_verifier/Cargo.toml b/core/lib/contract_verifier/Cargo.toml index 4f6e1ed273e9..6d8c19c64088 100644 --- a/core/lib/contract_verifier/Cargo.toml +++ b/core/lib/contract_verifier/Cargo.toml @@ -13,7 +13,6 @@ categories.workspace = true [dependencies] zksync_types.workspace = true zksync_dal.workspace = true -zksync_config.workspace = true zksync_contracts.workspace = true zksync_queued_job_processor.workspace = true zksync_utils.workspace = true diff --git a/core/lib/contract_verifier/src/lib.rs b/core/lib/contract_verifier/src/lib.rs index 508e47bb12b4..b9796aa738e4 100644 --- a/core/lib/contract_verifier/src/lib.rs +++ b/core/lib/contract_verifier/src/lib.rs @@ -8,7 +8,6 @@ use anyhow::Context as _; use chrono::Utc; use ethabi::{Contract, Token}; use tokio::time; -use zksync_config::ContractVerifierConfig; use zksync_dal::{ConnectionPool, Core, CoreDal}; use zksync_queued_job_processor::{async_trait, JobProcessor}; use zksync_types::{ @@ -43,16 +42,16 @@ enum ConstructorArgs { #[derive(Debug, Clone)] pub struct ContractVerifier { - config: ContractVerifierConfig, // FIXME: replace with used fields + compilation_timeout: Duration, contract_deployer: Contract, connection_pool: ConnectionPool, compiler_resolver: Arc, } impl ContractVerifier { - pub fn new(config: ContractVerifierConfig, connection_pool: ConnectionPool) -> Self { + pub fn new(compilation_timeout: Duration, connection_pool: ConnectionPool) -> Self { Self { - config, + compilation_timeout, contract_deployer: zksync_contracts::deployer_contract(), connection_pool, compiler_resolver: Arc::::default(), @@ -128,12 +127,9 @@ impl ContractVerifier { let input = Self::build_zksolc_input(req)?; let zksolc = ZkSolc::new(compiler_paths, zksolc_version); - time::timeout( - self.config.compilation_timeout(), - zksolc.async_compile(input), - ) - .await - .map_err(|_| ContractVerifierError::CompilationTimeout)? + time::timeout(self.compilation_timeout, zksolc.async_compile(input)) + .await + .map_err(|_| ContractVerifierError::CompilationTimeout)? } async fn compile_zkvyper( @@ -146,12 +142,9 @@ impl ContractVerifier { .await?; let input = Self::build_zkvyper_input(req)?; let zkvyper = ZkVyper::new(compiler_paths); - time::timeout( - self.config.compilation_timeout(), - zkvyper.async_compile(input), - ) - .await - .map_err(|_| ContractVerifierError::CompilationTimeout)? + time::timeout(self.compilation_timeout, zkvyper.async_compile(input)) + .await + .map_err(|_| ContractVerifierError::CompilationTimeout)? } pub async fn compile( @@ -403,7 +396,7 @@ impl JobProcessor for ContractVerifier { // we re-pick up jobs that are being executed for a bit more than `compilation_timeout`. let job = connection .contract_verification_dal() - .get_next_queued_verification_request(self.config.compilation_timeout() + TIME_OVERHEAD) + .get_next_queued_verification_request(self.compilation_timeout + TIME_OVERHEAD) .await?; Ok(job.map(|job| (job.id, job))) } diff --git a/core/lib/contract_verifier/src/tests.rs b/core/lib/contract_verifier/src/tests.rs index 0a190ab178f6..3332bbd0de6b 100644 --- a/core/lib/contract_verifier/src/tests.rs +++ b/core/lib/contract_verifier/src/tests.rs @@ -246,18 +246,7 @@ async fn contract_verifier_basics() { .add_contract_verification_request(req) .await .unwrap(); - - let config = ContractVerifierConfig { - compilation_timeout: 86_400, // sec - polling_interval: Some(50), // ms - - // Fields below are unused - prometheus_port: 0, - threads_per_server: None, - port: 0, - url: "".to_string(), - }; - let mut verifier = ContractVerifier::new(config, pool.clone()); + let mut verifier = ContractVerifier::new(Duration::from_secs(60), pool.clone()); verifier.compiler_resolver = Arc::new(test_resolver); let (_stop_sender, stop_receiver) = watch::channel(false); verifier.run(stop_receiver, Some(1)).await.unwrap(); From 91432c9902253a417936205c14b0cd909f772264 Mon Sep 17 00:00:00 2001 From: Alex Ostrovski Date: Mon, 28 Oct 2024 17:15:06 +0200 Subject: [PATCH 10/19] Remove unused fields in verifier config --- core/lib/config/src/configs/contract_verifier.rs | 7 ------- core/lib/config/src/testonly.rs | 3 --- core/lib/env_config/src/contract_verifier.rs | 7 ------- core/lib/protobuf_config/src/contract_verifier.rs | 10 ---------- .../src/proto/config/contract_verifier.proto | 7 ++++--- core/tests/ts-integration/src/env.ts | 2 +- etc/env/base/contract_verifier.toml | 3 --- etc/env/file_based/general.yaml | 3 --- 8 files changed, 5 insertions(+), 37 deletions(-) diff --git a/core/lib/config/src/configs/contract_verifier.rs b/core/lib/config/src/configs/contract_verifier.rs index 87b3418892f7..1dac0b17227e 100644 --- a/core/lib/config/src/configs/contract_verifier.rs +++ b/core/lib/config/src/configs/contract_verifier.rs @@ -9,13 +9,9 @@ use serde::Deserialize; pub struct ContractVerifierConfig { /// Max time of a single compilation (in s). pub compilation_timeout: u64, - /// Interval between polling db for verification requests (in ms). - pub polling_interval: Option, /// Port to which the Prometheus exporter server is listening. pub prometheus_port: u16, - pub threads_per_server: Option, // FIXME: unused? pub port: u16, - pub url: String, // FIXME: unused? } impl ContractVerifierConfig { @@ -23,9 +19,6 @@ impl ContractVerifierConfig { Duration::from_secs(self.compilation_timeout) } - pub fn polling_interval(&self) -> Duration { - Duration::from_millis(self.polling_interval.unwrap_or(1000)) - } pub fn bind_addr(&self) -> SocketAddr { SocketAddr::new(IpAddr::V4(Ipv4Addr::new(0, 0, 0, 0)), self.port) } diff --git a/core/lib/config/src/testonly.rs b/core/lib/config/src/testonly.rs index 21ff9e2351b6..2952e80ae6a3 100644 --- a/core/lib/config/src/testonly.rs +++ b/core/lib/config/src/testonly.rs @@ -241,11 +241,8 @@ impl Distribution for EncodeDist { fn sample(&self, rng: &mut R) -> configs::ContractVerifierConfig { configs::ContractVerifierConfig { compilation_timeout: self.sample(rng), - polling_interval: self.sample(rng), prometheus_port: self.sample(rng), - threads_per_server: self.sample(rng), port: self.sample(rng), - url: self.sample(rng), } } } diff --git a/core/lib/env_config/src/contract_verifier.rs b/core/lib/env_config/src/contract_verifier.rs index 3079a8daa9cf..484e06341586 100644 --- a/core/lib/env_config/src/contract_verifier.rs +++ b/core/lib/env_config/src/contract_verifier.rs @@ -18,11 +18,8 @@ mod tests { fn expected_config() -> ContractVerifierConfig { ContractVerifierConfig { compilation_timeout: 30, - polling_interval: Some(1000), prometheus_port: 3314, - threads_per_server: Some(128), port: 3070, - url: "127.0.0.1:3070".to_string(), } } @@ -31,12 +28,8 @@ mod tests { let mut lock = MUTEX.lock(); let config = r#" CONTRACT_VERIFIER_COMPILATION_TIMEOUT=30 - CONTRACT_VERIFIER_POLLING_INTERVAL=1000 CONTRACT_VERIFIER_PROMETHEUS_PORT=3314 CONTRACT_VERIFIER_PORT=3070 - CONTRACT_VERIFIER_URL=127.0.0.1:3070 - CONTRACT_VERIFIER_THREADS_PER_SERVER=128 - "#; lock.set_env(config); diff --git a/core/lib/protobuf_config/src/contract_verifier.rs b/core/lib/protobuf_config/src/contract_verifier.rs index e0b0517ea0f6..3fb7cfe0bdf7 100644 --- a/core/lib/protobuf_config/src/contract_verifier.rs +++ b/core/lib/protobuf_config/src/contract_verifier.rs @@ -10,29 +10,19 @@ impl ProtoRepr for proto::ContractVerifier { Ok(Self::Type { compilation_timeout: *required(&self.compilation_timeout) .context("compilation_timeout")?, - polling_interval: self.polling_interval, prometheus_port: required(&self.prometheus_port) .and_then(|x| Ok((*x).try_into()?)) .context("prometheus_port")?, - url: required(&self.url).cloned().context("url")?, port: required(&self.port) .and_then(|x| (*x).try_into().context("overflow")) .context("port")?, - threads_per_server: self - .threads_per_server - .map(|a| a.try_into()) - .transpose() - .context("threads_per_server")?, }) } fn build(this: &Self::Type) -> Self { Self { port: Some(this.port as u32), - url: Some(this.url.clone()), compilation_timeout: Some(this.compilation_timeout), - polling_interval: this.polling_interval, - threads_per_server: this.threads_per_server.map(|a| a as u32), prometheus_port: Some(this.prometheus_port.into()), } } diff --git a/core/lib/protobuf_config/src/proto/config/contract_verifier.proto b/core/lib/protobuf_config/src/proto/config/contract_verifier.proto index 31b1d3ed2ec4..8493274c6911 100644 --- a/core/lib/protobuf_config/src/proto/config/contract_verifier.proto +++ b/core/lib/protobuf_config/src/proto/config/contract_verifier.proto @@ -4,9 +4,10 @@ package zksync.config.contract_verifier; message ContractVerifier{ optional uint32 port = 1; // required; u16 - optional string url = 2; // required optional uint64 compilation_timeout = 3; - optional uint64 polling_interval = 4; - optional uint32 threads_per_server = 5; optional uint32 prometheus_port = 6; + + reserved 2; reserved "url"; + reserved 4; reserved "polling_interval"; + reserved 5; reserved "threads_per_server"; } diff --git a/core/tests/ts-integration/src/env.ts b/core/tests/ts-integration/src/env.ts index 1de917c2362c..a731c464d9f9 100644 --- a/core/tests/ts-integration/src/env.ts +++ b/core/tests/ts-integration/src/env.ts @@ -125,7 +125,7 @@ async function loadTestEnvironmentFromFile(fileConfig: FileConfig): Promise Date: Mon, 28 Oct 2024 21:31:51 +0200 Subject: [PATCH 11/19] Remove dependency on `multivm` --- Cargo.lock | 2 +- core/lib/prover_interface/Cargo.toml | 2 +- core/lib/prover_interface/src/inputs.rs | 2 +- prover/Cargo.lock | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 296ffccfaf39..81b8776b5aff 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -11167,9 +11167,9 @@ dependencies = [ "serde_with", "strum", "tokio", - "zksync_multivm", "zksync_object_store", "zksync_types", + "zksync_vm_interface", ] [[package]] diff --git a/core/lib/prover_interface/Cargo.toml b/core/lib/prover_interface/Cargo.toml index 889b80b4fbee..50671fb3acb4 100644 --- a/core/lib/prover_interface/Cargo.toml +++ b/core/lib/prover_interface/Cargo.toml @@ -11,7 +11,7 @@ keywords.workspace = true categories.workspace = true [dependencies] -zksync_multivm.workspace = true +zksync_vm_interface.workspace = true zksync_object_store.workspace = true zksync_types.workspace = true diff --git a/core/lib/prover_interface/src/inputs.rs b/core/lib/prover_interface/src/inputs.rs index cfc1d4a0d552..48a839dc9217 100644 --- a/core/lib/prover_interface/src/inputs.rs +++ b/core/lib/prover_interface/src/inputs.rs @@ -2,12 +2,12 @@ use std::{collections::HashMap, convert::TryInto, fmt::Debug}; use serde::{Deserialize, Serialize}; use serde_with::{serde_as, Bytes}; -use zksync_multivm::interface::{L1BatchEnv, SystemEnv}; use zksync_object_store::{_reexports::BoxedError, serialize_using_bincode, Bucket, StoredObject}; use zksync_types::{ basic_fri_types::Eip4844Blobs, block::L2BlockExecutionData, commitment::PubdataParams, witness_block_state::WitnessStorageState, L1BatchNumber, ProtocolVersionId, H256, U256, }; +use zksync_vm_interface::{L1BatchEnv, SystemEnv}; const HASH_LEN: usize = H256::len_bytes(); diff --git a/prover/Cargo.lock b/prover/Cargo.lock index d68ef368a4aa..7b44ff3ffc88 100644 --- a/prover/Cargo.lock +++ b/prover/Cargo.lock @@ -8496,9 +8496,9 @@ dependencies = [ "serde", "serde_with", "strum", - "zksync_multivm", "zksync_object_store", "zksync_types", + "zksync_vm_interface", ] [[package]] From 0d70b47cab13d025bea3be303818b0b797980a46 Mon Sep 17 00:00:00 2001 From: Alex Ostrovski Date: Tue, 29 Oct 2024 10:35:09 +0200 Subject: [PATCH 12/19] Fix verifier config in `zkstack` --- zkstack_cli/crates/zkstack/src/commands/explorer/init.rs | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/zkstack_cli/crates/zkstack/src/commands/explorer/init.rs b/zkstack_cli/crates/zkstack/src/commands/explorer/init.rs index 5c8e10ba2d81..096c45da5d8f 100644 --- a/zkstack_cli/crates/zkstack/src/commands/explorer/init.rs +++ b/zkstack_cli/crates/zkstack/src/commands/explorer/init.rs @@ -108,14 +108,15 @@ fn build_explorer_chain_config( // Get L2 RPC URL from general config let l2_rpc_url = general_config.get_l2_rpc_url()?; // Get Verification API URL from general config - let verification_api_url = general_config + let verification_api_port = general_config .contract_verifier .as_ref() - .map(|verifier| &verifier.url) - .context("verification_url")?; + .map(|verifier| verifier.port) + .context("verifier.port")?; + let verification_api_url = format!("http://127.0.0.1:{verification_api_port}"); // Build API URL let api_port = backend_config.ports.api_http_port; - let api_url = format!("http://127.0.0.1:{}", api_port); + let api_url = format!("http://127.0.0.1:{api_port}"); // Build explorer chain config Ok(ExplorerChainConfig { From 7d0c950795737ea9899ef32a14f2995df890038d Mon Sep 17 00:00:00 2001 From: Alex Ostrovski Date: Tue, 29 Oct 2024 10:35:28 +0200 Subject: [PATCH 13/19] Add tracing in verifier lib --- core/lib/contract_verifier/src/lib.rs | 176 +++++++++++++++++--------- 1 file changed, 118 insertions(+), 58 deletions(-) diff --git a/core/lib/contract_verifier/src/lib.rs b/core/lib/contract_verifier/src/lib.rs index b9796aa738e4..bb6889873859 100644 --- a/core/lib/contract_verifier/src/lib.rs +++ b/core/lib/contract_verifier/src/lib.rs @@ -1,5 +1,6 @@ use std::{ collections::HashMap, + fmt, sync::Arc, time::{Duration, Instant}, }; @@ -34,12 +35,20 @@ mod tests; mod zksolc_utils; mod zkvyper_utils; -#[derive(Debug)] enum ConstructorArgs { Check(Vec), Ignore, } +impl fmt::Debug for ConstructorArgs { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Check(args) => write!(formatter, "0x{}", hex::encode(args)), + Self::Ignore => formatter.write_str("(ignored)"), + } + } +} + #[derive(Debug, Clone)] pub struct ContractVerifier { compilation_timeout: Duration, @@ -58,6 +67,12 @@ impl ContractVerifier { } } + #[tracing::instrument( + level = "debug", + skip_all, + err, + fields(id = request.id, addr = ?request.req.contract_address) + )] async fn verify( &self, mut request: VerificationRequest, @@ -69,10 +84,10 @@ impl ContractVerifier { .connection_pool .connection_tagged("contract_verifier") .await?; - let (deployed_bytecode, creation_tx_calldata) = dbg!(storage + let (deployed_bytecode, creation_tx_calldata) = storage .contract_verification_dal() .get_contract_info_for_verification(request.req.contract_address) - .await)? + .await? .with_context(|| { format!( "Contract is missing in DB for already accepted verification request. Contract address: {:#?}", @@ -81,14 +96,12 @@ impl ContractVerifier { })?; drop(storage); - let constructor_args = self.decode_constructor_arguments_from_calldata( - creation_tx_calldata, - request.req.contract_address, - ); + let constructor_args = + self.decode_constructor_args(creation_tx_calldata, request.req.contract_address)?; if artifacts.bytecode != deployed_bytecode { tracing::info!( - "Bytecode mismatch req {}, deployed: 0x{}, compiled 0x{}", + "Bytecode mismatch req {}, deployed: 0x{}, compiled: 0x{}", request.id, hex::encode(deployed_bytecode), hex::encode(artifacts.bytecode) @@ -98,7 +111,13 @@ impl ContractVerifier { match constructor_args { ConstructorArgs::Check(args) => { - if request.req.constructor_arguments.0 != args { + let provided_constructor_args = &request.req.constructor_arguments.0; + if *provided_constructor_args != args { + tracing::trace!( + "Constructor args mismatch, deployed: 0x{}, provided in request: 0x{}", + hex::encode(&args), + hex::encode(provided_constructor_args) + ); return Err(ContractVerifierError::IncorrectConstructorArguments); } } @@ -107,10 +126,12 @@ impl ContractVerifier { } } + let verified_at = Utc::now(); + tracing::trace!(%verified_at, "verified request"); Ok(VerificationInfo { request, artifacts, - verified_at: Utc::now(), + verified_at, }) } @@ -122,7 +143,7 @@ impl ContractVerifier { .compiler_resolver .resolve_solc(&req.compiler_versions) .await?; - dbg!(&compiler_paths); + tracing::debug!(?compiler_paths, ?req.compiler_versions, "resolved compiler paths"); let zksolc_version = req.compiler_versions.zk_compiler_version(); let input = Self::build_zksolc_input(req)?; let zksolc = ZkSolc::new(compiler_paths, zksolc_version); @@ -147,12 +168,13 @@ impl ContractVerifier { .map_err(|_| ContractVerifierError::CompilationTimeout)? } - pub async fn compile( + #[tracing::instrument(level = "debug", skip_all)] + async fn compile( &self, req: VerificationIncomingRequest, ) -> Result { match req.source_code_data.compiler_type() { - CompilerType::Solc => dbg!(self.compile_zksolc(req).await), + CompilerType::Solc => self.compile_zksolc(req).await, CompilerType::Vyper => self.compile_zkvyper(req).await, } } @@ -253,84 +275,122 @@ impl ContractVerifier { }) } - fn decode_constructor_arguments_from_calldata( + /// All returned errors are internal. + #[tracing::instrument(level = "trace", skip_all, ret, err)] + fn decode_constructor_args( &self, calldata: DeployContractCalldata, contract_address_to_verify: Address, - ) -> ConstructorArgs { + ) -> anyhow::Result { match calldata { DeployContractCalldata::Deploy(calldata) => { - // `unwrap`s below are safe as long as the contract deployer contract is correct. + anyhow::ensure!( + calldata.len() >= 4, + "calldata doesn't include Solidity function selector" + ); + let contract_deployer = &self.contract_deployer; - let create = contract_deployer.function("create").unwrap(); - let create2 = contract_deployer.function("create2").unwrap(); - let create_acc = contract_deployer.function("createAccount").unwrap(); - let create2_acc = contract_deployer.function("create2Account").unwrap(); + let create = contract_deployer + .function("create") + .context("no `create` in contract deployer ABI")?; + let create2 = contract_deployer + .function("create2") + .context("no `create2` in contract deployer ABI")?; + let create_acc = contract_deployer + .function("createAccount") + .context("no `createAccount` in contract deployer ABI")?; + let create2_acc = contract_deployer + .function("create2Account") + .context("no `create2Account` in contract deployer ABI")?; let force_deploy = contract_deployer .function("forceDeployOnAddresses") - .unwrap(); + .context("no `forceDeployOnAddresses` in contract deployer ABI")?; + let (selector, token_data) = calldata.split_at(4); // It's assumed that `create` and `create2` methods have the same parameters // and the same for `createAccount` and `create2Account`. - match &calldata[0..4] { + Ok(match selector { selector if selector == create.short_signature() || selector == create2.short_signature() => { let tokens = create - .decode_input(&calldata[4..]) - .expect("Failed to decode input"); + .decode_input(token_data) + .context("failed to decode `create` / `create2` input")?; // Constructor arguments are in the third parameter. - ConstructorArgs::Check(tokens[2].clone().into_bytes().expect( - "The third parameter of `create/create2` should be of type `bytes`", - )) + ConstructorArgs::Check(tokens[2].clone().into_bytes().context( + "third parameter of `create/create2` should be of type `bytes`", + )?) } selector if selector == create_acc.short_signature() || selector == create2_acc.short_signature() => { let tokens = create - .decode_input(&calldata[4..]) - .expect("Failed to decode input"); + .decode_input(token_data) + .context("failed to decode `createAccount` / `create2Account` input")?; // Constructor arguments are in the third parameter. - ConstructorArgs::Check( - tokens[2].clone().into_bytes().expect( - "The third parameter of `createAccount/create2Account` should be of type `bytes`", - ), - ) + ConstructorArgs::Check(tokens[2].clone().into_bytes().context( + "third parameter of `createAccount/create2Account` should be of type `bytes`", + )?) } selector if selector == force_deploy.short_signature() => { - let tokens = force_deploy - .decode_input(&calldata[4..]) - .expect("Failed to decode input"); - let deployments = tokens[0].clone().into_array().unwrap(); - for deployment in deployments { - match deployment { - Token::Tuple(tokens) => { - let address = tokens[1].clone().into_address().unwrap(); - if address == contract_address_to_verify { - let call_constructor = - tokens[2].clone().into_bool().unwrap(); - return if call_constructor { - let input = tokens[4].clone().into_bytes().unwrap(); - ConstructorArgs::Check(input) - } else { - ConstructorArgs::Ignore - }; - } - } - _ => panic!("Expected `deployment` to be a tuple"), - } - } - panic!("Couldn't find force deployment for given address"); + Self::decode_force_deployment( + token_data, + force_deploy, + contract_address_to_verify, + ) + .context("failed decoding force deployment")? } _ => ConstructorArgs::Ignore, + }) + } + DeployContractCalldata::Ignore => Ok(ConstructorArgs::Ignore), + } + } + + fn decode_force_deployment( + token_data: &[u8], + force_deploy: ðabi::Function, + contract_address_to_verify: Address, + ) -> anyhow::Result { + let tokens = force_deploy + .decode_input(token_data) + .context("failed to decode `forceDeployOnAddresses` input")?; + let deployments = tokens[0] + .clone() + .into_array() + .context("first parameter of `forceDeployOnAddresses` is not an array")?; + for deployment in deployments { + match deployment { + Token::Tuple(tokens) => { + let address = tokens[1] + .clone() + .into_address() + .context("unexpected `address`")?; + if address == contract_address_to_verify { + let call_constructor = tokens[2] + .clone() + .into_bool() + .context("unexpected `call_constructor`")?; + return Ok(if call_constructor { + let input = tokens[4] + .clone() + .into_bytes() + .context("unexpected constructor input")?; + ConstructorArgs::Check(input) + } else { + ConstructorArgs::Ignore + }); + } } + _ => anyhow::bail!("expected `deployment` to be a tuple"), } - DeployContractCalldata::Ignore => ConstructorArgs::Ignore, } + anyhow::bail!("couldn't find force deployment for address {contract_address_to_verify:?}"); } + #[tracing::instrument(level = "debug", skip_all, err, fields(id = request_id))] async fn process_result( &self, request_id: usize, From 15e0fc223ed6a12ac5267fb3c9f72cf1ae7b1fe9 Mon Sep 17 00:00:00 2001 From: Alex Ostrovski Date: Tue, 29 Oct 2024 11:19:16 +0200 Subject: [PATCH 14/19] Move supported compiler versions --- core/bin/contract-verifier/src/main.rs | 104 +----------------- core/lib/contract_verifier/src/lib.rs | 72 +++++++++++- .../src/{paths.rs => resolver.rs} | 61 ++++++++++ core/lib/contract_verifier/src/tests.rs | 35 +++++- .../lib/contract_verifier/src/zksolc_utils.rs | 4 +- .../contract_verifier/src/zkvyper_utils.rs | 2 +- 6 files changed, 168 insertions(+), 110 deletions(-) rename core/lib/contract_verifier/src/{paths.rs => resolver.rs} (66%) diff --git a/core/bin/contract-verifier/src/main.rs b/core/bin/contract-verifier/src/main.rs index 527e14c5ee2a..6929f8bfe04d 100644 --- a/core/bin/contract-verifier/src/main.rs +++ b/core/bin/contract-verifier/src/main.rs @@ -7,105 +7,11 @@ use tokio::sync::watch; use zksync_config::configs::PrometheusConfig; use zksync_contract_verifier_lib::ContractVerifier; use zksync_core_leftovers::temp_config_store::{load_database_secrets, load_general_config}; -use zksync_dal::{ConnectionPool, Core, CoreDal}; +use zksync_dal::{ConnectionPool, Core}; use zksync_queued_job_processor::JobProcessor; -use zksync_utils::{env::Workspace, wait_for_tasks::ManagedTasks}; +use zksync_utils::wait_for_tasks::ManagedTasks; use zksync_vlog::prometheus::PrometheusExporterConfig; -async fn update_compiler_versions(connection_pool: &ConnectionPool) { - let mut storage = connection_pool.connection().await.unwrap(); - let mut transaction = storage.start_transaction().await.unwrap(); - - let zksync_home = Workspace::locate().core(); - - let zksolc_path = zksync_home.join("etc/zksolc-bin/"); - let zksolc_versions: Vec = std::fs::read_dir(zksolc_path) - .unwrap() - .filter_map(|file| { - let file = file.unwrap(); - let Ok(file_type) = file.file_type() else { - return None; - }; - if file_type.is_dir() { - file.file_name().into_string().ok() - } else { - None - } - }) - .collect(); - transaction - .contract_verification_dal() - .set_zksolc_versions(zksolc_versions) - .await - .unwrap(); - - let solc_path = zksync_home.join("etc/solc-bin/"); - let solc_versions: Vec = std::fs::read_dir(solc_path) - .unwrap() - .filter_map(|file| { - let file = file.unwrap(); - let Ok(file_type) = file.file_type() else { - return None; - }; - if file_type.is_dir() { - file.file_name().into_string().ok() - } else { - None - } - }) - .collect(); - transaction - .contract_verification_dal() - .set_solc_versions(solc_versions) - .await - .unwrap(); - - let zkvyper_path = zksync_home.join("etc/zkvyper-bin/"); - let zkvyper_versions: Vec = std::fs::read_dir(zkvyper_path) - .unwrap() - .filter_map(|file| { - let file = file.unwrap(); - let Ok(file_type) = file.file_type() else { - return None; - }; - if file_type.is_dir() { - file.file_name().into_string().ok() - } else { - None - } - }) - .collect(); - transaction - .contract_verification_dal() - .set_zkvyper_versions(zkvyper_versions) - .await - .unwrap(); - - let vyper_path = zksync_home.join("etc/vyper-bin/"); - let vyper_versions: Vec = std::fs::read_dir(vyper_path) - .unwrap() - .filter_map(|file| { - let file = file.unwrap(); - let Ok(file_type) = file.file_type() else { - return None; - }; - if file_type.is_dir() { - file.file_name().into_string().ok() - } else { - None - } - }) - .collect(); - - transaction - .contract_verification_dal() - .set_vyper_versions(vyper_versions) - .await - .unwrap(); - - transaction.commit().await.unwrap(); -} - #[derive(StructOpt)] #[structopt(name = "ZKsync contract code verifier", author = "Matter Labs")] struct Opt { @@ -160,9 +66,9 @@ async fn main() -> anyhow::Result<()> { .expect("Error setting Ctrl+C handler"); } - update_compiler_versions(&pool).await; - - let contract_verifier = ContractVerifier::new(verifier_config.compilation_timeout(), pool); + let contract_verifier = ContractVerifier::new(verifier_config.compilation_timeout(), pool) + .await + .context("failed initializing contract verifier")?; let tasks = vec![ // TODO PLA-335: Leftovers after the prover DB split. // The prover connection pool is not used by the contract verifier, but we need to pass it diff --git a/core/lib/contract_verifier/src/lib.rs b/core/lib/contract_verifier/src/lib.rs index bb6889873859..fc6b726ba125 100644 --- a/core/lib/contract_verifier/src/lib.rs +++ b/core/lib/contract_verifier/src/lib.rs @@ -22,14 +22,14 @@ use zksync_types::{ use crate::{ error::ContractVerifierError, metrics::API_CONTRACT_VERIFIER_METRICS, - paths::{CompilerResolver, EnvCompilerResolver}, + resolver::{CompilerResolver, EnvCompilerResolver}, zksolc_utils::{Optimizer, Settings, Source, StandardJson, ZkSolc, ZkSolcInput}, zkvyper_utils::{ZkVyper, ZkVyperInput}, }; pub mod error; mod metrics; -mod paths; +mod resolver; #[cfg(test)] mod tests; mod zksolc_utils; @@ -58,13 +58,75 @@ pub struct ContractVerifier { } impl ContractVerifier { - pub fn new(compilation_timeout: Duration, connection_pool: ConnectionPool) -> Self { - Self { + pub async fn new( + compilation_timeout: Duration, + connection_pool: ConnectionPool, + ) -> anyhow::Result { + Self::with_resolver( + compilation_timeout, + connection_pool, + Arc::::default(), + ) + .await + } + + async fn with_resolver( + compilation_timeout: Duration, + connection_pool: ConnectionPool, + compiler_resolver: Arc, + ) -> anyhow::Result { + let this = Self { compilation_timeout, contract_deployer: zksync_contracts::deployer_contract(), connection_pool, - compiler_resolver: Arc::::default(), + compiler_resolver, + }; + this.sync_compiler_versions().await?; + Ok(this) + } + + /// Synchronizes compiler versions. + #[tracing::instrument(level = "debug", skip_all)] + async fn sync_compiler_versions(&self) -> anyhow::Result<()> { + let supported_versions = self + .compiler_resolver + .supported_versions() + .await + .context("cannot get supported compilers")?; + if supported_versions.lacks_any_compiler() { + tracing::warn!( + ?supported_versions, + "contract verifier lacks support of at least one compiler entirely; it may be incorrectly set up" + ); } + tracing::info!( + ?supported_versions, + "persisting supported compiler versions" + ); + + let mut storage = self + .connection_pool + .connection_tagged("contract_verifier") + .await?; + let mut transaction = storage.start_transaction().await?; + transaction + .contract_verification_dal() + .set_zksolc_versions(supported_versions.zksolc) + .await?; + transaction + .contract_verification_dal() + .set_solc_versions(supported_versions.solc) + .await?; + transaction + .contract_verification_dal() + .set_zkvyper_versions(supported_versions.zkvyper) + .await?; + transaction + .contract_verification_dal() + .set_vyper_versions(supported_versions.vyper) + .await?; + transaction.commit().await?; + Ok(()) } #[tracing::instrument( diff --git a/core/lib/contract_verifier/src/paths.rs b/core/lib/contract_verifier/src/resolver.rs similarity index 66% rename from core/lib/contract_verifier/src/paths.rs rename to core/lib/contract_verifier/src/resolver.rs index 4f5e487031b0..dda542a5b4ed 100644 --- a/core/lib/contract_verifier/src/paths.rs +++ b/core/lib/contract_verifier/src/resolver.rs @@ -8,6 +8,23 @@ use zksync_utils::env::Workspace; use crate::error::ContractVerifierError; +#[derive(Debug)] +pub(crate) struct SupportedCompilerVersions { + pub solc: Vec, + pub zksolc: Vec, + pub vyper: Vec, + pub zkvyper: Vec, +} + +impl SupportedCompilerVersions { + pub fn lacks_any_compiler(&self) -> bool { + self.solc.is_empty() + || self.zksolc.is_empty() + || self.vyper.is_empty() + || self.zkvyper.is_empty() + } +} + #[derive(Debug, Clone)] pub(crate) struct CompilerPaths { /// Path to the base (non-zk) compiler. @@ -19,6 +36,9 @@ pub(crate) struct CompilerPaths { /// Encapsulates compiler paths resolution. #[async_trait] pub(crate) trait CompilerResolver: fmt::Debug + Send + Sync { + /// Returned errors are assumed to be fatal. + async fn supported_versions(&self) -> anyhow::Result; + /// Resolves paths to `solc` / `zksolc`. async fn resolve_solc( &self, @@ -46,8 +66,49 @@ impl Default for EnvCompilerResolver { } } +impl EnvCompilerResolver { + async fn read_dir(&self, dir: &str) -> anyhow::Result> { + let mut dir_entries = fs::read_dir(self.home_dir.join(dir)) + .await + .context("failed reading dir")?; + let mut versions = vec![]; + while let Some(entry) = dir_entries.next_entry().await? { + let Ok(file_type) = entry.file_type().await else { + continue; + }; + if file_type.is_dir() { + if let Ok(name) = entry.file_name().into_string() { + versions.push(name); + } + } + } + Ok(versions) + } +} + #[async_trait] impl CompilerResolver for EnvCompilerResolver { + async fn supported_versions(&self) -> anyhow::Result { + Ok(SupportedCompilerVersions { + solc: self + .read_dir("etc/solc-bin") + .await + .context("failed reading solc dir")?, + zksolc: self + .read_dir("etc/zksolc-bin") + .await + .context("failed reading zksolc dir")?, + vyper: self + .read_dir("etc/vyper-bin") + .await + .context("failed reading vyper dir")?, + zkvyper: self + .read_dir("etc/zkvyper-bin") + .await + .context("failed reading zkvyper dir")?, + }) + } + async fn resolve_solc( &self, versions: &CompilerVersions, diff --git a/core/lib/contract_verifier/src/tests.rs b/core/lib/contract_verifier/src/tests.rs index 3332bbd0de6b..b1cd25ec3a1b 100644 --- a/core/lib/contract_verifier/src/tests.rs +++ b/core/lib/contract_verifier/src/tests.rs @@ -20,7 +20,7 @@ use zksync_utils::{ use zksync_vm_interface::{TransactionExecutionMetrics, VmEvent}; use super::*; -use crate::paths::CompilerPaths; +use crate::resolver::{CompilerPaths, SupportedCompilerVersions}; const SOLC_VERSION: &str = "0.8.27"; const ZKSOLC_VERSION: &str = "1.5.4"; @@ -140,6 +140,15 @@ impl TestCompilerResolver { #[async_trait] impl CompilerResolver for TestCompilerResolver { + async fn supported_versions(&self) -> anyhow::Result { + Ok(SupportedCompilerVersions { + solc: vec![SOLC_VERSION.to_owned()], + zksolc: vec![ZKSOLC_VERSION.to_owned()], + vyper: vec![], + zkvyper: vec![], + }) + } + async fn resolve_solc( &self, versions: &CompilerVersions, @@ -246,8 +255,28 @@ async fn contract_verifier_basics() { .add_contract_verification_request(req) .await .unwrap(); - let mut verifier = ContractVerifier::new(Duration::from_secs(60), pool.clone()); - verifier.compiler_resolver = Arc::new(test_resolver); + let verifier = ContractVerifier::with_resolver( + Duration::from_secs(60), + pool.clone(), + Arc::new(test_resolver), + ) + .await + .unwrap(); + + // Check that the compiler versions are synced. + let solc_versions = storage + .contract_verification_dal() + .get_solc_versions() + .await + .unwrap(); + assert_eq!(solc_versions, [SOLC_VERSION]); + let zksolc_versions = storage + .contract_verification_dal() + .get_zksolc_versions() + .await + .unwrap(); + assert_eq!(zksolc_versions, [ZKSOLC_VERSION]); + let (_stop_sender, stop_receiver) = watch::channel(false); verifier.run(stop_receiver, Some(1)).await.unwrap(); diff --git a/core/lib/contract_verifier/src/zksolc_utils.rs b/core/lib/contract_verifier/src/zksolc_utils.rs index c62e3c3b32ed..7825e562a709 100644 --- a/core/lib/contract_verifier/src/zksolc_utils.rs +++ b/core/lib/contract_verifier/src/zksolc_utils.rs @@ -6,7 +6,7 @@ use semver::Version; use serde::{Deserialize, Serialize}; use zksync_types::contract_verification_api::CompilationArtifacts; -use crate::{error::ContractVerifierError, paths::CompilerPaths}; +use crate::{error::ContractVerifierError, resolver::CompilerPaths}; #[derive(Debug)] pub enum ZkSolcInput { @@ -272,7 +272,7 @@ impl ZkSolc { mod tests { use std::path::PathBuf; - use crate::{paths::CompilerPaths, zksolc_utils::ZkSolc}; + use crate::{resolver::CompilerPaths, zksolc_utils::ZkSolc}; #[test] fn check_is_post_1_5_0() { diff --git a/core/lib/contract_verifier/src/zkvyper_utils.rs b/core/lib/contract_verifier/src/zkvyper_utils.rs index 1f62c249294d..725858a3abe4 100644 --- a/core/lib/contract_verifier/src/zkvyper_utils.rs +++ b/core/lib/contract_verifier/src/zkvyper_utils.rs @@ -3,7 +3,7 @@ use std::{collections::HashMap, fs::File, io::Write, path::Path, process::Stdio} use anyhow::Context as _; use zksync_types::contract_verification_api::CompilationArtifacts; -use crate::{error::ContractVerifierError, paths::CompilerPaths}; +use crate::{error::ContractVerifierError, resolver::CompilerPaths}; #[derive(Debug)] pub(crate) struct ZkVyperInput { From b530449d4df83c617e3fe088b66fc62f48174641 Mon Sep 17 00:00:00 2001 From: Alex Ostrovski Date: Tue, 29 Oct 2024 12:12:40 +0200 Subject: [PATCH 15/19] Refactor compiler resolution --- Cargo.lock | 191 ------------------ core/lib/contract_verifier/Cargo.toml | 3 - core/lib/contract_verifier/src/lib.rs | 18 +- core/lib/contract_verifier/src/resolver.rs | 36 +++- .../lib/contract_verifier/src/zksolc_utils.rs | 163 ++++++++------- .../contract_verifier/src/zkvyper_utils.rs | 67 +++--- 6 files changed, 156 insertions(+), 322 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 81b8776b5aff..dcf27bdf8ac1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -186,15 +186,6 @@ version = "1.0.89" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "86fdf8605db99b54d3cd748a44c6d04df638eb5dafb219b135d0149bd0db01f6" -[[package]] -name = "arbitrary" -version = "1.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d5a26814d8dcb93b0e5a0ff3c6d80a8843bafb21b39e8e18a6f05471870e110" -dependencies = [ - "derive_arbitrary", -] - [[package]] name = "arr_macro" version = "0.1.3" @@ -1572,19 +1563,6 @@ dependencies = [ "compile-fmt", ] -[[package]] -name = "const-hex" -version = "1.13.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0121754e84117e65f9d90648ee6aa4882a6e63110307ab73967a4c5e7e69e586" -dependencies = [ - "cfg-if", - "cpufeatures", - "hex", - "proptest", - "serde", -] - [[package]] name = "const-oid" version = "0.9.6" @@ -2056,17 +2034,6 @@ dependencies = [ "syn 1.0.109", ] -[[package]] -name = "derive_arbitrary" -version = "1.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67e77553c4162a157adbf834ebae5b415acbecbeafc7a74b0e886657506a7611" -dependencies = [ - "proc-macro2 1.0.86", - "quote 1.0.37", - "syn 2.0.77", -] - [[package]] name = "derive_more" version = "0.99.18" @@ -2128,15 +2095,6 @@ dependencies = [ "subtle", ] -[[package]] -name = "dirs" -version = "5.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44c45a9d03d6676652bcb5e724c7e988de1acad23a711b5217ab9cbecbec2225" -dependencies = [ - "dirs-sys", -] - [[package]] name = "dirs-next" version = "2.0.0" @@ -2147,18 +2105,6 @@ dependencies = [ "dirs-sys-next", ] -[[package]] -name = "dirs-sys" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "520f05a5cbd335fae5a99ff7a6ab8627577660ee5cfd6a94a6a929b52ff0321c" -dependencies = [ - "libc", - "option-ext", - "redox_users", - "windows-sys 0.48.0", -] - [[package]] name = "dirs-sys-next" version = "0.1.2" @@ -2170,17 +2116,6 @@ dependencies = [ "winapi", ] -[[package]] -name = "displaydoc" -version = "0.2.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" -dependencies = [ - "proc-macro2 1.0.86", - "quote 1.0.37", - "syn 2.0.77", -] - [[package]] name = "dotenvy" version = "0.15.7" @@ -2697,16 +2632,6 @@ dependencies = [ "zksync_bellman", ] -[[package]] -name = "fs4" -version = "0.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec6fcfb3c0c1d71612528825042261419d5dade9678c39a781e05b63677d9b32" -dependencies = [ - "rustix", - "windows-sys 0.52.0", -] - [[package]] name = "fs_extra" version = "1.3.0" @@ -4351,12 +4276,6 @@ dependencies = [ "scopeguard", ] -[[package]] -name = "lockfree-object-pool" -version = "0.1.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9374ef4228402d4b7e403e5838cb880d9ee663314b0a900d5a6aabf0c213552e" - [[package]] name = "log" version = "0.4.22" @@ -5032,12 +4951,6 @@ dependencies = [ "tokio-stream", ] -[[package]] -name = "option-ext" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d" - [[package]] name = "ordered-float" version = "2.10.1" @@ -5553,22 +5466,6 @@ dependencies = [ "syn 2.0.77", ] -[[package]] -name = "proptest" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4c2511913b88df1637da85cc8d96ec8e43a3f8bb8ccb71ee1ac240d6f3df58d" -dependencies = [ - "bitflags 2.6.0", - "lazy_static", - "num-traits", - "rand 0.8.5", - "rand_chacha", - "rand_xorshift", - "regex-syntax 0.8.4", - "unarray", -] - [[package]] name = "prost" version = "0.12.6" @@ -5832,15 +5729,6 @@ dependencies = [ "getrandom", ] -[[package]] -name = "rand_xorshift" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d25bf25ec5ae4a3f1b92f929810509a2f53d7dca2f50b794ff57e3face536c8f" -dependencies = [ - "rand_core 0.6.4", -] - [[package]] name = "rand_xoshiro" version = "0.6.0" @@ -6039,7 +5927,6 @@ dependencies = [ "system-configuration 0.6.1", "tokio", "tokio-native-tls", - "tokio-socks", "tokio-util", "tower-service", "url", @@ -7093,12 +6980,6 @@ dependencies = [ "rand_core 0.6.4", ] -[[package]] -name = "simd-adler32" -version = "0.3.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d66dc143e6b11c1eddc06d5c423cfc97062865baf299914ab64caa38182078fe" - [[package]] name = "simdutf8" version = "0.1.5" @@ -7860,27 +7741,6 @@ dependencies = [ "zeroize", ] -[[package]] -name = "svm-rs" -version = "0.5.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "040017ebc08d781c457a3bfe9c5c2a99f902f8133eb91ef82b7876b053962ece" -dependencies = [ - "anyhow", - "const-hex", - "dirs", - "fs4", - "reqwest 0.12.7", - "semver", - "serde", - "serde_json", - "sha2 0.10.8", - "tempfile", - "thiserror", - "url", - "zip", -] - [[package]] name = "syn" version = "0.15.44" @@ -8326,18 +8186,6 @@ dependencies = [ "tokio", ] -[[package]] -name = "tokio-socks" -version = "0.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d4770b8024672c1101b3f6733eab95b18007dbe0847a8afe341fcf79e06043f" -dependencies = [ - "either", - "futures-util", - "thiserror", - "tokio", -] - [[package]] name = "tokio-stream" version = "0.1.16" @@ -8677,12 +8525,6 @@ dependencies = [ "libc", ] -[[package]] -name = "unarray" -version = "0.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eaea85b334db583fe3274d12b4cd1880032beab409c0d774be044d4480ab9a94" - [[package]] name = "unicase" version = "2.7.0" @@ -9467,23 +9309,6 @@ dependencies = [ "syn 2.0.77", ] -[[package]] -name = "zip" -version = "2.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc5e4288ea4057ae23afc69a4472434a87a2495cafce6632fd1c4ec9f5cf3494" -dependencies = [ - "arbitrary", - "crc32fast", - "crossbeam-utils", - "displaydoc", - "flate2", - "indexmap 2.5.0", - "memchr", - "thiserror", - "zopfli", -] - [[package]] name = "zk_evm" version = "0.131.0-rc.2" @@ -10133,11 +9958,9 @@ dependencies = [ "ethabi", "hex", "regex", - "reqwest 0.12.7", "semver", "serde", "serde_json", - "svm-rs", "tempfile", "thiserror", "tokio", @@ -11632,20 +11455,6 @@ dependencies = [ "zksync_types", ] -[[package]] -name = "zopfli" -version = "0.8.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5019f391bac5cf252e93bbcc53d039ffd62c7bfb7c150414d61369afe57e946" -dependencies = [ - "bumpalo", - "crc32fast", - "lockfree-object-pool", - "log", - "once_cell", - "simd-adler32", -] - [[package]] name = "zstd" version = "0.13.2" diff --git a/core/lib/contract_verifier/Cargo.toml b/core/lib/contract_verifier/Cargo.toml index 6d8c19c64088..3b2ab33294e3 100644 --- a/core/lib/contract_verifier/Cargo.toml +++ b/core/lib/contract_verifier/Cargo.toml @@ -34,6 +34,3 @@ semver.workspace = true [dev-dependencies] zksync_node_test_utils.workspace = true zksync_vm_interface.workspace = true - -reqwest.workspace = true -svm-rs = { version = "0.5.8", default-features = false, features = ["solc"] } # FIXME: add to deps diff --git a/core/lib/contract_verifier/src/lib.rs b/core/lib/contract_verifier/src/lib.rs index fc6b726ba125..8f7033f6d1fd 100644 --- a/core/lib/contract_verifier/src/lib.rs +++ b/core/lib/contract_verifier/src/lib.rs @@ -23,8 +23,8 @@ use crate::{ error::ContractVerifierError, metrics::API_CONTRACT_VERIFIER_METRICS, resolver::{CompilerResolver, EnvCompilerResolver}, - zksolc_utils::{Optimizer, Settings, Source, StandardJson, ZkSolc, ZkSolcInput}, - zkvyper_utils::{ZkVyper, ZkVyperInput}, + zksolc_utils::{Optimizer, Settings, Source, StandardJson, ZkSolcInput}, + zkvyper_utils::ZkVyperInput, }; pub mod error; @@ -201,16 +201,14 @@ impl ContractVerifier { &self, req: VerificationIncomingRequest, ) -> Result { - let compiler_paths = self + let zksolc = self .compiler_resolver .resolve_solc(&req.compiler_versions) .await?; - tracing::debug!(?compiler_paths, ?req.compiler_versions, "resolved compiler paths"); - let zksolc_version = req.compiler_versions.zk_compiler_version(); + tracing::debug!(?zksolc, ?req.compiler_versions, "resolved compiler"); let input = Self::build_zksolc_input(req)?; - let zksolc = ZkSolc::new(compiler_paths, zksolc_version); - time::timeout(self.compilation_timeout, zksolc.async_compile(input)) + time::timeout(self.compilation_timeout, zksolc.compile(input)) .await .map_err(|_| ContractVerifierError::CompilationTimeout)? } @@ -219,13 +217,13 @@ impl ContractVerifier { &self, req: VerificationIncomingRequest, ) -> Result { - let compiler_paths = self + let zkvyper = self .compiler_resolver .resolve_vyper(&req.compiler_versions) .await?; + tracing::debug!(?zkvyper, ?req.compiler_versions, "resolved compiler"); let input = Self::build_zkvyper_input(req)?; - let zkvyper = ZkVyper::new(compiler_paths); - time::timeout(self.compilation_timeout, zkvyper.async_compile(input)) + time::timeout(self.compilation_timeout, zkvyper.compile(input)) .await .map_err(|_| ContractVerifierError::CompilationTimeout)? } diff --git a/core/lib/contract_verifier/src/resolver.rs b/core/lib/contract_verifier/src/resolver.rs index dda542a5b4ed..7281bcf53d09 100644 --- a/core/lib/contract_verifier/src/resolver.rs +++ b/core/lib/contract_verifier/src/resolver.rs @@ -3,10 +3,14 @@ use std::{fmt, path::PathBuf}; use anyhow::Context as _; use tokio::fs; use zksync_queued_job_processor::async_trait; -use zksync_types::contract_verification_api::CompilerVersions; +use zksync_types::contract_verification_api::{CompilationArtifacts, CompilerVersions}; use zksync_utils::env::Workspace; -use crate::error::ContractVerifierError; +use crate::{ + error::ContractVerifierError, + zksolc_utils::{ZkSolc, ZkSolcInput}, + zkvyper_utils::{ZkVyper, ZkVyperInput}, +}; #[derive(Debug)] pub(crate) struct SupportedCompilerVersions { @@ -43,13 +47,21 @@ pub(crate) trait CompilerResolver: fmt::Debug + Send + Sync { async fn resolve_solc( &self, versions: &CompilerVersions, - ) -> Result; + ) -> Result>, ContractVerifierError>; /// Resolves paths to `vyper` / `zkvyper`. async fn resolve_vyper( &self, versions: &CompilerVersions, - ) -> Result; + ) -> Result>, ContractVerifierError>; +} + +#[async_trait] +pub(crate) trait Compiler: Send + fmt::Debug { + async fn compile( + self: Box, + input: In, + ) -> Result; } /// Default [`CompilerResolver`] using pre-downloaded compilers. @@ -112,7 +124,7 @@ impl CompilerResolver for EnvCompilerResolver { async fn resolve_solc( &self, versions: &CompilerVersions, - ) -> Result { + ) -> Result>, ContractVerifierError> { let zksolc_version = versions.zk_compiler_version(); let zksolc_path = self .home_dir @@ -146,16 +158,18 @@ impl CompilerResolver for EnvCompilerResolver { solc_version, )); } - Ok(CompilerPaths { + + let compiler_paths = CompilerPaths { base: solc_path, zk: zksolc_path, - }) + }; + Ok(Box::new(ZkSolc::new(compiler_paths, zksolc_version))) } async fn resolve_vyper( &self, versions: &CompilerVersions, - ) -> Result { + ) -> Result>, ContractVerifierError> { let zkvyper_version = versions.zk_compiler_version(); let zkvyper_path = self .home_dir @@ -189,9 +203,11 @@ impl CompilerResolver for EnvCompilerResolver { vyper_version, )); } - Ok(CompilerPaths { + + let compiler_paths = CompilerPaths { base: vyper_path, zk: zkvyper_path, - }) + }; + Ok(Box::new(ZkVyper::new(compiler_paths))) } } diff --git a/core/lib/contract_verifier/src/zksolc_utils.rs b/core/lib/contract_verifier/src/zksolc_utils.rs index 7825e562a709..0a3d84ab555b 100644 --- a/core/lib/contract_verifier/src/zksolc_utils.rs +++ b/core/lib/contract_verifier/src/zksolc_utils.rs @@ -4,9 +4,14 @@ use anyhow::Context as _; use regex::Regex; use semver::Version; use serde::{Deserialize, Serialize}; +use tokio::io::AsyncWriteExt; +use zksync_queued_job_processor::async_trait; use zksync_types::contract_verification_api::CompilationArtifacts; -use crate::{error::ContractVerifierError, resolver::CompilerPaths}; +use crate::{ + error::ContractVerifierError, + resolver::{Compiler, CompilerPaths}, +}; #[derive(Debug)] pub enum ZkSolcInput { @@ -90,11 +95,87 @@ impl ZkSolc { } } - pub async fn async_compile( - &self, + fn parse_standard_json_output( + output: &serde_json::Value, + contract_name: String, + file_name: String, + ) -> Result { + if let Some(errors) = output.get("errors") { + let errors = errors.as_array().unwrap().clone(); + if errors + .iter() + .any(|err| err["severity"].as_str().unwrap() == "error") + { + let error_messages = errors + .into_iter() + .map(|err| err["formattedMessage"].clone()) + .collect(); + return Err(ContractVerifierError::CompilationError( + serde_json::Value::Array(error_messages), + )); + } + } + + let contracts = output["contracts"] + .get(&file_name) + .ok_or(ContractVerifierError::MissingSource(file_name))?; + let Some(contract) = contracts.get(&contract_name) else { + return Err(ContractVerifierError::MissingContract(contract_name)); + }; + let bytecode_str = contract["evm"]["bytecode"]["object"] + .as_str() + .ok_or(ContractVerifierError::AbstractContract(contract_name))?; + let bytecode = hex::decode(bytecode_str).unwrap(); + let abi = contract["abi"].clone(); + if !abi.is_array() { + let err = anyhow::anyhow!( + "zksolc returned unexpected value for ABI: {}", + serde_json::to_string_pretty(&abi).unwrap() + ); + return Err(err.into()); + } + + Ok(CompilationArtifacts { bytecode, abi }) + } + + fn parse_single_file_yul_output( + output: &str, + ) -> Result { + let re = Regex::new(r"Contract `.*` bytecode: 0x([\da-f]+)").unwrap(); + let cap = re + .captures(output) + .context("Yul output doesn't match regex")?; + let bytecode_str = cap.get(1).context("no matches in Yul output")?.as_str(); + let bytecode = hex::decode(bytecode_str).context("invalid Yul output bytecode")?; + Ok(CompilationArtifacts { + bytecode, + abi: serde_json::Value::Array(Vec::new()), + }) + } + + fn is_post_1_5_0(&self) -> bool { + // Special case + if &self.zksolc_version == "vm-1.5.0-a167aa3" { + false + } else if let Some(version) = self.zksolc_version.strip_prefix("v") { + if let Ok(semver) = Version::parse(version) { + let target = Version::new(1, 5, 0); + semver >= target + } else { + true + } + } else { + true + } + } +} + +#[async_trait] +impl Compiler for ZkSolc { + async fn compile( + self: Box, input: ZkSolcInput, ) -> Result { - use tokio::io::AsyncWriteExt; let mut command = tokio::process::Command::new(&self.paths.zk); command.stdout(Stdio::piped()).stderr(Stdio::piped()); @@ -192,80 +273,6 @@ impl ZkSolc { } } } - - fn parse_standard_json_output( - output: &serde_json::Value, - contract_name: String, - file_name: String, - ) -> Result { - if let Some(errors) = output.get("errors") { - let errors = errors.as_array().unwrap().clone(); - if errors - .iter() - .any(|err| err["severity"].as_str().unwrap() == "error") - { - let error_messages = errors - .into_iter() - .map(|err| err["formattedMessage"].clone()) - .collect(); - return Err(ContractVerifierError::CompilationError( - serde_json::Value::Array(error_messages), - )); - } - } - - let contracts = output["contracts"] - .get(&file_name) - .ok_or(ContractVerifierError::MissingSource(file_name))?; - let Some(contract) = contracts.get(&contract_name) else { - return Err(ContractVerifierError::MissingContract(contract_name)); - }; - let bytecode_str = contract["evm"]["bytecode"]["object"] - .as_str() - .ok_or(ContractVerifierError::AbstractContract(contract_name))?; - let bytecode = hex::decode(bytecode_str).unwrap(); - let abi = contract["abi"].clone(); - if !abi.is_array() { - let err = anyhow::anyhow!( - "zksolc returned unexpected value for ABI: {}", - serde_json::to_string_pretty(&abi).unwrap() - ); - return Err(err.into()); - } - - Ok(CompilationArtifacts { bytecode, abi }) - } - - fn parse_single_file_yul_output( - output: &str, - ) -> Result { - let re = Regex::new(r"Contract `.*` bytecode: 0x([\da-f]+)").unwrap(); - let cap = re - .captures(output) - .context("Yul output doesn't match regex")?; - let bytecode_str = cap.get(1).context("no matches in Yul output")?.as_str(); - let bytecode = hex::decode(bytecode_str).context("invalid Yul output bytecode")?; - Ok(CompilationArtifacts { - bytecode, - abi: serde_json::Value::Array(Vec::new()), - }) - } - - pub fn is_post_1_5_0(&self) -> bool { - // Special case - if &self.zksolc_version == "vm-1.5.0-a167aa3" { - false - } else if let Some(version) = self.zksolc_version.strip_prefix("v") { - if let Ok(semver) = Version::parse(version) { - let target = Version::new(1, 5, 0); - semver >= target - } else { - true - } - } else { - true - } - } } #[cfg(test)] diff --git a/core/lib/contract_verifier/src/zkvyper_utils.rs b/core/lib/contract_verifier/src/zkvyper_utils.rs index 725858a3abe4..bc2cd7e996c1 100644 --- a/core/lib/contract_verifier/src/zkvyper_utils.rs +++ b/core/lib/contract_verifier/src/zkvyper_utils.rs @@ -1,9 +1,13 @@ use std::{collections::HashMap, fs::File, io::Write, path::Path, process::Stdio}; use anyhow::Context as _; +use zksync_queued_job_processor::async_trait; use zksync_types::contract_verification_api::CompilationArtifacts; -use crate::{error::ContractVerifierError, resolver::CompilerPaths}; +use crate::{ + error::ContractVerifierError, + resolver::{Compiler, CompilerPaths}, +}; #[derive(Debug)] pub(crate) struct ZkVyperInput { @@ -22,8 +26,38 @@ impl ZkVyper { Self { paths } } - pub async fn async_compile( - &self, + fn parse_output( + output: &serde_json::Value, + contract_name: String, + ) -> Result { + let file_name = format!("{contract_name}.vy"); + let object = output + .as_object() + .context("Vyper output is not an object")?; + for (path, artifact) in object { + let path = Path::new(&path); + if path.file_name().unwrap().to_str().unwrap() == file_name { + let bytecode_str = artifact["bytecode"] + .as_str() + .context("bytecode is not a string")?; + let bytecode_without_prefix = + bytecode_str.strip_prefix("0x").unwrap_or(bytecode_str); + let bytecode = + hex::decode(bytecode_without_prefix).context("failed decoding bytecode")?; + return Ok(CompilationArtifacts { + abi: artifact["abi"].clone(), + bytecode, + }); + } + } + Err(ContractVerifierError::MissingContract(contract_name)) + } +} + +#[async_trait] +impl Compiler for ZkVyper { + async fn compile( + self: Box, input: ZkVyperInput, ) -> Result { let mut command = tokio::process::Command::new(&self.paths.zk); @@ -68,31 +102,4 @@ impl ZkVyper { )) } } - - fn parse_output( - output: &serde_json::Value, - contract_name: String, - ) -> Result { - let file_name = format!("{contract_name}.vy"); - let object = output - .as_object() - .context("Vyper output is not an object")?; - for (path, artifact) in object { - let path = Path::new(&path); - if path.file_name().unwrap().to_str().unwrap() == file_name { - let bytecode_str = artifact["bytecode"] - .as_str() - .context("bytecode is not a string")?; - let bytecode_without_prefix = - bytecode_str.strip_prefix("0x").unwrap_or(bytecode_str); - let bytecode = - hex::decode(bytecode_without_prefix).context("failed decoding bytecode")?; - return Ok(CompilationArtifacts { - abi: artifact["abi"].clone(), - bytecode, - }); - } - } - Err(ContractVerifierError::MissingContract(contract_name)) - } } From 4dbf5c9b18edba28dd102adcdabcb6c338685e4c Mon Sep 17 00:00:00 2001 From: Alex Ostrovski Date: Tue, 29 Oct 2024 12:12:52 +0200 Subject: [PATCH 16/19] Update contract verifier tests --- core/lib/contract_verifier/src/tests.rs | 254 ++++++++++++++++-------- 1 file changed, 168 insertions(+), 86 deletions(-) diff --git a/core/lib/contract_verifier/src/tests.rs b/core/lib/contract_verifier/src/tests.rs index b1cd25ec3a1b..edf81a9d016c 100644 --- a/core/lib/contract_verifier/src/tests.rs +++ b/core/lib/contract_verifier/src/tests.rs @@ -1,8 +1,8 @@ //! Tests for the contract verifier. -use std::{os::unix::fs::PermissionsExt, path::Path}; +use std::env; -use tokio::{fs, sync::watch}; +use tokio::sync::watch; use zksync_dal::Connection; use zksync_node_test_utils::{create_l1_batch, create_l2_block}; use zksync_types::{ @@ -20,7 +20,7 @@ use zksync_utils::{ use zksync_vm_interface::{TransactionExecutionMetrics, VmEvent}; use super::*; -use crate::resolver::{CompilerPaths, SupportedCompilerVersions}; +use crate::resolver::{Compiler, SupportedCompilerVersions}; const SOLC_VERSION: &str = "0.8.27"; const ZKSOLC_VERSION: &str = "1.5.4"; @@ -82,64 +82,41 @@ async fn mock_deployment(storage: &mut Connection<'_, Core>, address: Address, b .unwrap(); } -/// Test compiler resolver. -#[derive(Debug)] -struct TestCompilerResolver(CompilerPaths); +#[derive(Clone)] +struct MockCompilerResolver { + zksolc: Arc< + dyn Fn(ZkSolcInput) -> Result + Send + Sync, + >, +} -impl TestCompilerResolver { - async fn install_zksolc(path: &Path) { - if fs::try_exists(path).await.unwrap() { - return; - } +impl fmt::Debug for MockCompilerResolver { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter + .debug_struct("MockCompilerResolver") + .finish_non_exhaustive() + } +} - // We may race from several test processes here; this is OK-ish for tests. - let version = ZKSOLC_VERSION; - let compiler_prefix = match svm::platform() { - svm::Platform::LinuxAmd64 => "zksolc-linux-amd64-musl-", - svm::Platform::LinuxAarch64 => "zksolc-linux-arm64-musl-", - svm::Platform::MacOsAmd64 => "zksolc-macosx-amd64-", - svm::Platform::MacOsAarch64 => "zksolc-macosx-arm64-", - other => panic!("Unsupported platform: {other:?}"), - }; - let download_url = format!( - "https://github.com/matter-labs/zksolc-bin/releases/download/v{version}/{compiler_prefix}v{version}", - ); - let response = reqwest::Client::new() - .get(&download_url) - .send() - .await - .unwrap() - .error_for_status() - .unwrap(); - let response_bytes = response.bytes().await.unwrap(); - - fs::create_dir_all(path.parent().unwrap()).await.unwrap(); - fs::write(path, &response_bytes).await.unwrap(); - // Set the executable flag for the file. - fs::set_permissions(path, PermissionsExt::from_mode(0o755)) - .await - .unwrap(); +impl MockCompilerResolver { + fn new(zksolc: impl Fn(ZkSolcInput) -> CompilationArtifacts + 'static + Send + Sync) -> Self { + Self { + zksolc: Arc::new(move |input| Ok(zksolc(input))), + } } +} - async fn new() -> Self { - let solc_version = SOLC_VERSION.parse().unwrap(); - let solc_path = svm::install(&solc_version) - .await - .expect("failed installing solc"); - let zksolc_path = svm::data_dir() - .join(format!("zksolc-{ZKSOLC_VERSION}")) - .join("zksolc"); - Self::install_zksolc(&zksolc_path).await; - - Self(CompilerPaths { - base: solc_path, - zk: zksolc_path, - }) +#[async_trait] +impl Compiler for MockCompilerResolver { + async fn compile( + self: Box, + input: ZkSolcInput, + ) -> Result { + (self.zksolc)(input) } } #[async_trait] -impl CompilerResolver for TestCompilerResolver { +impl CompilerResolver for MockCompilerResolver { async fn supported_versions(&self) -> anyhow::Result { Ok(SupportedCompilerVersions { solc: vec![SOLC_VERSION.to_owned()], @@ -152,7 +129,7 @@ impl CompilerResolver for TestCompilerResolver { async fn resolve_solc( &self, versions: &CompilerVersions, - ) -> Result { + ) -> Result>, ContractVerifierError> { if versions.compiler_version() != SOLC_VERSION { return Err(ContractVerifierError::UnknownCompilerVersion( "solc".to_owned(), @@ -165,13 +142,13 @@ impl CompilerResolver for TestCompilerResolver { versions.zk_compiler_version(), )); } - Ok(self.0.clone()) + Ok(Box::new(self.clone())) } async fn resolve_vyper( &self, _versions: &CompilerVersions, - ) -> Result { + ) -> Result>, ContractVerifierError> { unreachable!("not tested") } } @@ -202,34 +179,21 @@ fn test_request(address: Address) -> VerificationIncomingRequest { } } -async fn compile_counter(compiler_paths: CompilerPaths) -> CompilationArtifacts { - let req = test_request(Address::repeat_byte(1)); - let input = ContractVerifier::build_zksolc_input(req).unwrap(); - ZkSolc::new(compiler_paths, ZKSOLC_VERSION.to_owned()) - .async_compile(input) - .await - .unwrap() -} - -#[tokio::test] -async fn compiler_works() { - let compiler_paths = TestCompilerResolver::new().await.0; - let output = compile_counter(compiler_paths).await; - validate_bytecode(&output.bytecode).unwrap(); - let items = output.abi.as_array().unwrap(); - assert_eq!(items.len(), 1); - let increment_function = items[0].as_object().unwrap(); - assert_eq!(increment_function["type"], "function"); - assert_eq!(increment_function["name"], "increment"); +fn counter_contract_abi() -> serde_json::Value { + serde_json::json!([{ + "inputs": [{ + "internalType": "uint256", + "name": "x", + "type": "uint256", + }], + "name": "increment", + "outputs": [], + "stateMutability": "nonpayable", + "type": "function", + }]) } -#[tokio::test] -async fn contract_verifier_basics() { - let test_resolver = TestCompilerResolver::new().await; - let output = compile_counter(test_resolver.0.clone()).await; - let pool = ConnectionPool::test_pool().await; - let mut storage = pool.connection().await.unwrap(); - +async fn prepare_storage(storage: &mut Connection<'_, Core>) { // Storage must contain at least 1 block / batch for verifier-related queries to work correctly. storage .protocol_versions_dal() @@ -246,19 +210,42 @@ async fn contract_verifier_basics() { .insert_mock_l1_batch(&create_l1_batch(0)) .await .unwrap(); +} + +#[tokio::test] +async fn contract_verifier_basics() { + let pool = ConnectionPool::test_pool().await; + let mut storage = pool.connection().await.unwrap(); + let expected_bytecode = vec![0_u8; 32]; + prepare_storage(&mut storage).await; let address = Address::repeat_byte(1); - mock_deployment(&mut storage, address, output.bytecode.clone()).await; + mock_deployment(&mut storage, address, expected_bytecode.clone()).await; let req = test_request(address); let request_id = storage .contract_verification_dal() .add_contract_verification_request(req) .await .unwrap(); + + let mock_resolver = MockCompilerResolver::new(|input| { + let ZkSolcInput::StandardJson { input, .. } = &input else { + panic!("unexpected input"); + }; + assert_eq!(input.language, "Solidity"); + assert_eq!(input.sources.len(), 1); + let source = input.sources.values().next().unwrap(); + assert!(source.content.contains("contract Counter"), "{source:?}"); + + CompilationArtifacts { + bytecode: vec![0; 32], + abi: counter_contract_abi(), + } + }); let verifier = ContractVerifier::with_resolver( Duration::from_secs(60), pool.clone(), - Arc::new(test_resolver), + Arc::new(mock_resolver), ) .await .unwrap(); @@ -280,6 +267,15 @@ async fn contract_verifier_basics() { let (_stop_sender, stop_receiver) = watch::channel(false); verifier.run(stop_receiver, Some(1)).await.unwrap(); + assert_request_success(&mut storage, request_id, address, &expected_bytecode).await; +} + +async fn assert_request_success( + storage: &mut Connection<'_, Core>, + request_id: usize, + address: Address, + expected_bytecode: &[u8], +) { let status = storage .contract_verification_dal() .get_verification_request_status(request_id) @@ -296,6 +292,92 @@ async fn contract_verifier_basics() { .await .unwrap() .expect("no verification info"); - assert_eq!(verification_info.artifacts.bytecode, output.bytecode); - assert_eq!(verification_info.artifacts.abi, output.abi); + assert_eq!(verification_info.artifacts.bytecode, *expected_bytecode); + assert_eq!(verification_info.artifacts.abi, counter_contract_abi()); +} + +async fn checked_env_resolver() -> Option<(EnvCompilerResolver, SupportedCompilerVersions)> { + let compiler_resolver = EnvCompilerResolver::default(); + let supported_compilers = compiler_resolver.supported_versions().await.ok()?; + if supported_compilers.zksolc.is_empty() || supported_compilers.solc.is_empty() { + return None; + } + Some((compiler_resolver, supported_compilers)) +} + +fn assert_no_compilers_expected() { + assert_ne!( + env::var("RUN_CONTRACT_VERIFICATION_TEST").ok().as_deref(), + Some("true"), + "Expected pre-installed compilers since `RUN_CONTRACT_VERIFICATION_TEST=true`, but they are not installed. \ + Use `zkstack contract-verifier init` to install compilers" + ); + println!("No compilers found, skipping the test"); +} + +#[tokio::test] +async fn using_real_compiler() { + let Some((compiler_resolver, supported_compilers)) = checked_env_resolver().await else { + assert_no_compilers_expected(); + return; + }; + + let versions = CompilerVersions::Solc { + compiler_zksolc_version: supported_compilers.zksolc[0].clone(), + compiler_solc_version: supported_compilers.solc[0].clone(), + }; + let compiler = compiler_resolver.resolve_solc(&versions).await.unwrap(); + let req = VerificationIncomingRequest { + compiler_versions: versions, + ..test_request(Address::repeat_byte(1)) + }; + let input = ContractVerifier::build_zksolc_input(req).unwrap(); + let output = compiler.compile(input).await.unwrap(); + + validate_bytecode(&output.bytecode).unwrap(); + assert_eq!(output.abi, counter_contract_abi()); +} + +#[tokio::test] +async fn using_real_compiler_in_verifier() { + let Some((compiler_resolver, supported_compilers)) = checked_env_resolver().await else { + assert_no_compilers_expected(); + return; + }; + + let versions = CompilerVersions::Solc { + compiler_zksolc_version: supported_compilers.zksolc[0].clone(), + compiler_solc_version: supported_compilers.solc[0].clone(), + }; + let address = Address::repeat_byte(1); + let compiler = compiler_resolver.resolve_solc(&versions).await.unwrap(); + let req = VerificationIncomingRequest { + compiler_versions: versions, + ..test_request(Address::repeat_byte(1)) + }; + let input = ContractVerifier::build_zksolc_input(req.clone()).unwrap(); + let output = compiler.compile(input).await.unwrap(); + + let pool = ConnectionPool::test_pool().await; + let mut storage = pool.connection().await.unwrap(); + prepare_storage(&mut storage).await; + mock_deployment(&mut storage, address, output.bytecode.clone()).await; + let request_id = storage + .contract_verification_dal() + .add_contract_verification_request(req) + .await + .unwrap(); + + let verifier = ContractVerifier::with_resolver( + Duration::from_secs(60), + pool.clone(), + Arc::new(compiler_resolver), + ) + .await + .unwrap(); + + let (_stop_sender, stop_receiver) = watch::channel(false); + verifier.run(stop_receiver, Some(1)).await.unwrap(); + + assert_request_success(&mut storage, request_id, address, &output.bytecode).await; } From fdde54922d24142f72a27a875ffbc3f181c5ba65 Mon Sep 17 00:00:00 2001 From: Alex Ostrovski Date: Tue, 29 Oct 2024 14:21:52 +0200 Subject: [PATCH 17/19] Add some contract verifier docs --- core/lib/contract_verifier/src/lib.rs | 3 +++ core/lib/contract_verifier/src/metrics.rs | 1 + core/lib/contract_verifier/src/resolver.rs | 13 ++++++++++--- 3 files changed, 14 insertions(+), 3 deletions(-) diff --git a/core/lib/contract_verifier/src/lib.rs b/core/lib/contract_verifier/src/lib.rs index 8f7033f6d1fd..f5face9f8a56 100644 --- a/core/lib/contract_verifier/src/lib.rs +++ b/core/lib/contract_verifier/src/lib.rs @@ -1,3 +1,5 @@ +//! Contract verifier able to verify contracts created with `zksolc` or `zkvyper` toolchains. + use std::{ collections::HashMap, fmt, @@ -58,6 +60,7 @@ pub struct ContractVerifier { } impl ContractVerifier { + /// Creates a new verifier instance. pub async fn new( compilation_timeout: Duration, connection_pool: ConnectionPool, diff --git a/core/lib/contract_verifier/src/metrics.rs b/core/lib/contract_verifier/src/metrics.rs index fd98f51cd560..1c6796cd7f38 100644 --- a/core/lib/contract_verifier/src/metrics.rs +++ b/core/lib/contract_verifier/src/metrics.rs @@ -5,6 +5,7 @@ use vise::{Buckets, Histogram, Metrics}; #[derive(Debug, Metrics)] #[metrics(prefix = "api_contract_verifier")] pub(crate) struct ApiContractVerifierMetrics { + /// Latency of processing a single request. #[metrics(buckets = Buckets::LATENCIES)] pub request_processing_time: Histogram, } diff --git a/core/lib/contract_verifier/src/resolver.rs b/core/lib/contract_verifier/src/resolver.rs index 7281bcf53d09..5e7729006698 100644 --- a/core/lib/contract_verifier/src/resolver.rs +++ b/core/lib/contract_verifier/src/resolver.rs @@ -12,6 +12,7 @@ use crate::{ zkvyper_utils::{ZkVyper, ZkVyperInput}, }; +/// Compiler versions supported by a [`CompilerResolver`]. #[derive(Debug)] pub(crate) struct SupportedCompilerVersions { pub solc: Vec, @@ -40,31 +41,37 @@ pub(crate) struct CompilerPaths { /// Encapsulates compiler paths resolution. #[async_trait] pub(crate) trait CompilerResolver: fmt::Debug + Send + Sync { + /// Returns compiler versions supported by this resolver. + /// + /// # Errors + /// /// Returned errors are assumed to be fatal. async fn supported_versions(&self) -> anyhow::Result; - /// Resolves paths to `solc` / `zksolc`. + /// Resolves a `zksolc` compiler. async fn resolve_solc( &self, versions: &CompilerVersions, ) -> Result>, ContractVerifierError>; - /// Resolves paths to `vyper` / `zkvyper`. + /// Resolves a `zkvyper` compiler. async fn resolve_vyper( &self, versions: &CompilerVersions, ) -> Result>, ContractVerifierError>; } +/// Encapsulates a one-off compilation process. #[async_trait] pub(crate) trait Compiler: Send + fmt::Debug { + /// Performs compilation. async fn compile( self: Box, input: In, ) -> Result; } -/// Default [`CompilerResolver`] using pre-downloaded compilers. +/// Default [`CompilerResolver`] using pre-downloaded compilers in the `/etc` subdirectories (relative to the workspace). #[derive(Debug)] pub(crate) struct EnvCompilerResolver { home_dir: PathBuf, From e9990b0d915c5690263b73762507f2a2a1e55836 Mon Sep 17 00:00:00 2001 From: Alex Ostrovski Date: Tue, 29 Oct 2024 14:44:34 +0200 Subject: [PATCH 18/19] Add more contract verifier tests --- .../src/{tests.rs => tests/mod.rs} | 118 ++++++++------- core/lib/contract_verifier/src/tests/real.rs | 141 ++++++++++++++++++ 2 files changed, 204 insertions(+), 55 deletions(-) rename core/lib/contract_verifier/src/{tests.rs => tests/mod.rs} (81%) create mode 100644 core/lib/contract_verifier/src/tests/real.rs diff --git a/core/lib/contract_verifier/src/tests.rs b/core/lib/contract_verifier/src/tests/mod.rs similarity index 81% rename from core/lib/contract_verifier/src/tests.rs rename to core/lib/contract_verifier/src/tests/mod.rs index edf81a9d016c..0c1e9fe937d5 100644 --- a/core/lib/contract_verifier/src/tests.rs +++ b/core/lib/contract_verifier/src/tests/mod.rs @@ -1,7 +1,5 @@ //! Tests for the contract verifier. -use std::env; - use tokio::sync::watch; use zksync_dal::Connection; use zksync_node_test_utils::{create_l1_batch, create_l2_block}; @@ -13,15 +11,14 @@ use zksync_types::{ Execute, L1BatchNumber, L2BlockNumber, ProtocolVersion, StorageLog, CONTRACT_DEPLOYER_ADDRESS, H256, }; -use zksync_utils::{ - address_to_h256, - bytecode::{hash_bytecode, validate_bytecode}, -}; +use zksync_utils::{address_to_h256, bytecode::hash_bytecode}; use zksync_vm_interface::{TransactionExecutionMetrics, VmEvent}; use super::*; use crate::resolver::{Compiler, SupportedCompilerVersions}; +mod real; + const SOLC_VERSION: &str = "0.8.27"; const ZKSOLC_VERSION: &str = "1.5.4"; @@ -305,73 +302,75 @@ async fn checked_env_resolver() -> Option<(EnvCompilerResolver, SupportedCompile Some((compiler_resolver, supported_compilers)) } -fn assert_no_compilers_expected() { - assert_ne!( - env::var("RUN_CONTRACT_VERIFICATION_TEST").ok().as_deref(), - Some("true"), - "Expected pre-installed compilers since `RUN_CONTRACT_VERIFICATION_TEST=true`, but they are not installed. \ - Use `zkstack contract-verifier init` to install compilers" - ); - println!("No compilers found, skipping the test"); -} - #[tokio::test] -async fn using_real_compiler() { - let Some((compiler_resolver, supported_compilers)) = checked_env_resolver().await else { - assert_no_compilers_expected(); - return; - }; +async fn bytecode_mismatch_error() { + let pool = ConnectionPool::test_pool().await; + let mut storage = pool.connection().await.unwrap(); + prepare_storage(&mut storage).await; - let versions = CompilerVersions::Solc { - compiler_zksolc_version: supported_compilers.zksolc[0].clone(), - compiler_solc_version: supported_compilers.solc[0].clone(), - }; - let compiler = compiler_resolver.resolve_solc(&versions).await.unwrap(); - let req = VerificationIncomingRequest { - compiler_versions: versions, - ..test_request(Address::repeat_byte(1)) - }; - let input = ContractVerifier::build_zksolc_input(req).unwrap(); - let output = compiler.compile(input).await.unwrap(); + let address = Address::repeat_byte(1); + mock_deployment(&mut storage, address, vec![0xff; 32]).await; + let req = test_request(address); + let request_id = storage + .contract_verification_dal() + .add_contract_verification_request(req) + .await + .unwrap(); + + let mock_resolver = MockCompilerResolver::new(|_| CompilationArtifacts { + bytecode: vec![0; 32], + abi: counter_contract_abi(), + }); + let verifier = ContractVerifier::with_resolver( + Duration::from_secs(60), + pool.clone(), + Arc::new(mock_resolver), + ) + .await + .unwrap(); - validate_bytecode(&output.bytecode).unwrap(); - assert_eq!(output.abi, counter_contract_abi()); + let (_stop_sender, stop_receiver) = watch::channel(false); + verifier.run(stop_receiver, Some(1)).await.unwrap(); + + let status = storage + .contract_verification_dal() + .get_verification_request_status(request_id) + .await + .unwrap() + .expect("no status"); + assert_eq!(status.status, "failed"); + assert!(status.compilation_errors.is_none(), "{status:?}"); + let error = status.error.unwrap(); + assert!(error.contains("bytecode"), "{error}"); } #[tokio::test] -async fn using_real_compiler_in_verifier() { - let Some((compiler_resolver, supported_compilers)) = checked_env_resolver().await else { - assert_no_compilers_expected(); - return; - }; +async fn no_compiler_version() { + let pool = ConnectionPool::test_pool().await; + let mut storage = pool.connection().await.unwrap(); + prepare_storage(&mut storage).await; - let versions = CompilerVersions::Solc { - compiler_zksolc_version: supported_compilers.zksolc[0].clone(), - compiler_solc_version: supported_compilers.solc[0].clone(), - }; let address = Address::repeat_byte(1); - let compiler = compiler_resolver.resolve_solc(&versions).await.unwrap(); + mock_deployment(&mut storage, address, vec![0xff; 32]).await; let req = VerificationIncomingRequest { - compiler_versions: versions, - ..test_request(Address::repeat_byte(1)) + compiler_versions: CompilerVersions::Solc { + compiler_zksolc_version: ZKSOLC_VERSION.to_owned(), + compiler_solc_version: "1.0.0".to_owned(), // a man can dream + }, + ..test_request(address) }; - let input = ContractVerifier::build_zksolc_input(req.clone()).unwrap(); - let output = compiler.compile(input).await.unwrap(); - - let pool = ConnectionPool::test_pool().await; - let mut storage = pool.connection().await.unwrap(); - prepare_storage(&mut storage).await; - mock_deployment(&mut storage, address, output.bytecode.clone()).await; let request_id = storage .contract_verification_dal() .add_contract_verification_request(req) .await .unwrap(); + let mock_resolver = + MockCompilerResolver::new(|_| unreachable!("should reject unknown solc version")); let verifier = ContractVerifier::with_resolver( Duration::from_secs(60), pool.clone(), - Arc::new(compiler_resolver), + Arc::new(mock_resolver), ) .await .unwrap(); @@ -379,5 +378,14 @@ async fn using_real_compiler_in_verifier() { let (_stop_sender, stop_receiver) = watch::channel(false); verifier.run(stop_receiver, Some(1)).await.unwrap(); - assert_request_success(&mut storage, request_id, address, &output.bytecode).await; + let status = storage + .contract_verification_dal() + .get_verification_request_status(request_id) + .await + .unwrap() + .expect("no status"); + assert_eq!(status.status, "failed"); + assert!(status.compilation_errors.is_none(), "{status:?}"); + let error = status.error.unwrap(); + assert!(error.contains("solc version"), "{error}"); } diff --git a/core/lib/contract_verifier/src/tests/real.rs b/core/lib/contract_verifier/src/tests/real.rs new file mode 100644 index 000000000000..b7acc753fd6d --- /dev/null +++ b/core/lib/contract_verifier/src/tests/real.rs @@ -0,0 +1,141 @@ +//! Tests using real compiler toolchains. Should be prepared by calling `zkstack contract-verifier init` +//! with at least one `solc` and `zksolc` version. If there are no compilers, the tests will be ignored +//! unless the `RUN_CONTRACT_VERIFICATION_TEST` env var is set to `true`, in which case the tests will fail. + +use std::{env, sync::Arc, time::Duration}; + +use zksync_utils::bytecode::validate_bytecode; + +use super::*; + +fn assert_no_compilers_expected() { + assert_ne!( + env::var("RUN_CONTRACT_VERIFICATION_TEST").ok().as_deref(), + Some("true"), + "Expected pre-installed compilers since `RUN_CONTRACT_VERIFICATION_TEST=true`, but they are not installed. \ + Use `zkstack contract-verifier init` to install compilers" + ); + println!("No compilers found, skipping the test"); +} + +#[tokio::test] +async fn using_real_compiler() { + let Some((compiler_resolver, supported_compilers)) = checked_env_resolver().await else { + assert_no_compilers_expected(); + return; + }; + + let versions = CompilerVersions::Solc { + compiler_zksolc_version: supported_compilers.zksolc[0].clone(), + compiler_solc_version: supported_compilers.solc[0].clone(), + }; + let compiler = compiler_resolver.resolve_solc(&versions).await.unwrap(); + let req = VerificationIncomingRequest { + compiler_versions: versions, + ..test_request(Address::repeat_byte(1)) + }; + let input = ContractVerifier::build_zksolc_input(req).unwrap(); + let output = compiler.compile(input).await.unwrap(); + + validate_bytecode(&output.bytecode).unwrap(); + assert_eq!(output.abi, counter_contract_abi()); +} + +#[tokio::test] +async fn using_real_compiler_in_verifier() { + let Some((compiler_resolver, supported_compilers)) = checked_env_resolver().await else { + assert_no_compilers_expected(); + return; + }; + + let versions = CompilerVersions::Solc { + compiler_zksolc_version: supported_compilers.zksolc[0].clone(), + compiler_solc_version: supported_compilers.solc[0].clone(), + }; + let address = Address::repeat_byte(1); + let compiler = compiler_resolver.resolve_solc(&versions).await.unwrap(); + let req = VerificationIncomingRequest { + compiler_versions: versions, + ..test_request(Address::repeat_byte(1)) + }; + let input = ContractVerifier::build_zksolc_input(req.clone()).unwrap(); + let output = compiler.compile(input).await.unwrap(); + + let pool = ConnectionPool::test_pool().await; + let mut storage = pool.connection().await.unwrap(); + prepare_storage(&mut storage).await; + mock_deployment(&mut storage, address, output.bytecode.clone()).await; + let request_id = storage + .contract_verification_dal() + .add_contract_verification_request(req) + .await + .unwrap(); + + let verifier = ContractVerifier::with_resolver( + Duration::from_secs(60), + pool.clone(), + Arc::new(compiler_resolver), + ) + .await + .unwrap(); + + let (_stop_sender, stop_receiver) = watch::channel(false); + verifier.run(stop_receiver, Some(1)).await.unwrap(); + + assert_request_success(&mut storage, request_id, address, &output.bytecode).await; +} + +#[tokio::test] +async fn compilation_errors() { + let Some((compiler_resolver, supported_compilers)) = checked_env_resolver().await else { + assert_no_compilers_expected(); + return; + }; + + let versions = CompilerVersions::Solc { + compiler_zksolc_version: supported_compilers.zksolc[0].clone(), + compiler_solc_version: supported_compilers.solc[0].clone(), + }; + let address = Address::repeat_byte(1); + let req = VerificationIncomingRequest { + compiler_versions: versions, + source_code_data: SourceCodeData::SolSingleFile("contract ???".to_owned()), + ..test_request(Address::repeat_byte(1)) + }; + + let pool = ConnectionPool::test_pool().await; + let mut storage = pool.connection().await.unwrap(); + prepare_storage(&mut storage).await; + mock_deployment(&mut storage, address, vec![0; 32]).await; + + let request_id = storage + .contract_verification_dal() + .add_contract_verification_request(req) + .await + .unwrap(); + + let verifier = ContractVerifier::with_resolver( + Duration::from_secs(60), + pool.clone(), + Arc::new(compiler_resolver), + ) + .await + .unwrap(); + + let (_stop_sender, stop_receiver) = watch::channel(false); + verifier.run(stop_receiver, Some(1)).await.unwrap(); + + let status = storage + .contract_verification_dal() + .get_verification_request_status(request_id) + .await + .unwrap() + .expect("no status"); + assert_eq!(status.status, "failed"); + let compilation_errors = status.compilation_errors.unwrap(); + assert!(!compilation_errors.is_empty()); + let has_parser_error = compilation_errors + .iter() + .any(|err| err.contains("ParserError") && err.contains("Counter.sol")); + assert!(has_parser_error, "{compilation_errors:?}"); +} From ddc5c8e38ac869113386e63686a6e4b75221171d Mon Sep 17 00:00:00 2001 From: Alex Ostrovski Date: Tue, 29 Oct 2024 14:53:14 +0200 Subject: [PATCH 19/19] Run contract verifier tests in CI --- .github/workflows/ci-core-reusable.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/ci-core-reusable.yml b/.github/workflows/ci-core-reusable.yml index fb43133868b0..094a5335826a 100644 --- a/.github/workflows/ci-core-reusable.yml +++ b/.github/workflows/ci-core-reusable.yml @@ -34,6 +34,7 @@ jobs: echo "SCCACHE_GCS_SERVICE_ACCOUNT=gha-ci-runners@matterlabs-infra.iam.gserviceaccount.com" >> .env echo "SCCACHE_GCS_RW_MODE=READ_WRITE" >> .env echo "RUSTC_WRAPPER=sccache" >> .env + echo RUN_CONTRACT_VERIFICATION_TEST=true >> .env # TODO: Remove when we after upgrade of hardhat-plugins - name: pre-download compilers @@ -73,6 +74,9 @@ jobs: - name: Contracts unit tests run: ci_run yarn l1-contracts test + - name: Download compilers for contract verifier tests + run: ci_run zkstack contract-verifier init --zksolc-version=v1.5.3 --zkvyper-version=v1.5.4 --solc-version=0.8.26 --vyper-version=v0.3.10 --era-vm-solc-version=0.8.26-1.0.1 --only --chain era + - name: Rust unit tests run: | ci_run zkstack dev test rust