From ac32fa0db2216fff9932383ad97413466eae95ef Mon Sep 17 00:00:00 2001 From: Mengran Lan Date: Tue, 27 Aug 2024 17:45:58 +0800 Subject: [PATCH 1/3] feat(prover): use params map to initialize provers --- prover/Cargo.lock | 40 ++++++++++----------- prover/Cargo.toml | 4 +-- prover/src/zk_circuits_handler.rs | 1 + prover/src/zk_circuits_handler/common.rs | 19 ++++++++++ prover/src/zk_circuits_handler/darwin.rs | 30 ++++++++++++---- prover/src/zk_circuits_handler/darwin_v2.rs | 30 ++++++++++++---- 6 files changed, 88 insertions(+), 36 deletions(-) create mode 100644 prover/src/zk_circuits_handler/common.rs diff --git a/prover/Cargo.lock b/prover/Cargo.lock index 10dfca9ae..cff651cc3 100644 --- a/prover/Cargo.lock +++ b/prover/Cargo.lock @@ -31,7 +31,7 @@ dependencies = [ [[package]] name = "aggregator" version = "0.12.0" -source = "git+https://github.com/scroll-tech/zkevm-circuits.git?tag=v0.12.0#6a1f65a1f99429f3725ef4d6788f5643bb61aa6f" +source = "git+https://github.com/scroll-tech/zkevm-circuits.git?tag=v0.12.2#6f7b46a3b1ccf9dc448735e8455e1ac6f9e30643" dependencies = [ "ark-std 0.3.0", "bitstream-io", @@ -65,7 +65,7 @@ dependencies = [ [[package]] name = "aggregator" version = "0.13.0" -source = "git+https://github.com/scroll-tech/zkevm-circuits.git?tag=v0.13.0#cf71a0e324d42f7895363f4f3e8575d79bb7ea97" +source = "git+https://github.com/scroll-tech/zkevm-circuits.git?tag=v0.13.1#4009e5593f13ba73f64f556011ee5ef47bc4ebf3" dependencies = [ "ark-std 0.3.0", "bitstream-io", @@ -635,7 +635,7 @@ checksum = "79296716171880943b8470b5f8d03aa55eb2e645a4874bdbb28adb49162e012c" [[package]] name = "bus-mapping" version = "0.12.0" -source = "git+https://github.com/scroll-tech/zkevm-circuits.git?tag=v0.12.0#6a1f65a1f99429f3725ef4d6788f5643bb61aa6f" +source = "git+https://github.com/scroll-tech/zkevm-circuits.git?tag=v0.12.2#6f7b46a3b1ccf9dc448735e8455e1ac6f9e30643" dependencies = [ "eth-types 0.12.0", "ethers-core 2.0.7 (git+https://github.com/scroll-tech/ethers-rs.git?branch=v2.0.7)", @@ -660,7 +660,7 @@ dependencies = [ [[package]] name = "bus-mapping" version = "0.13.0" -source = "git+https://github.com/scroll-tech/zkevm-circuits.git?tag=v0.13.0#cf71a0e324d42f7895363f4f3e8575d79bb7ea97" +source = "git+https://github.com/scroll-tech/zkevm-circuits.git?tag=v0.13.1#4009e5593f13ba73f64f556011ee5ef47bc4ebf3" dependencies = [ "eth-types 0.13.0", "ethers-core 2.0.7 (git+https://github.com/scroll-tech/ethers-rs.git?branch=v2.0.7)", @@ -1310,7 +1310,7 @@ dependencies = [ [[package]] name = "eth-types" version = "0.12.0" -source = "git+https://github.com/scroll-tech/zkevm-circuits.git?tag=v0.12.0#6a1f65a1f99429f3725ef4d6788f5643bb61aa6f" +source = "git+https://github.com/scroll-tech/zkevm-circuits.git?tag=v0.12.2#6f7b46a3b1ccf9dc448735e8455e1ac6f9e30643" dependencies = [ "base64 0.13.1", "ethers-core 2.0.7 (git+https://github.com/scroll-tech/ethers-rs.git?branch=v2.0.7)", @@ -1338,7 +1338,7 @@ dependencies = [ [[package]] name = "eth-types" version = "0.13.0" -source = "git+https://github.com/scroll-tech/zkevm-circuits.git?tag=v0.13.0#cf71a0e324d42f7895363f4f3e8575d79bb7ea97" +source = "git+https://github.com/scroll-tech/zkevm-circuits.git?tag=v0.13.1#4009e5593f13ba73f64f556011ee5ef47bc4ebf3" dependencies = [ "base64 0.13.1", "ethers-core 2.0.7 (git+https://github.com/scroll-tech/ethers-rs.git?branch=v2.0.7)", @@ -1558,7 +1558,7 @@ dependencies = [ [[package]] name = "external-tracer" version = "0.12.0" -source = "git+https://github.com/scroll-tech/zkevm-circuits.git?tag=v0.12.0#6a1f65a1f99429f3725ef4d6788f5643bb61aa6f" +source = "git+https://github.com/scroll-tech/zkevm-circuits.git?tag=v0.12.2#6f7b46a3b1ccf9dc448735e8455e1ac6f9e30643" dependencies = [ "eth-types 0.12.0", "geth-utils 0.12.0", @@ -1571,7 +1571,7 @@ dependencies = [ [[package]] name = "external-tracer" version = "0.13.0" -source = "git+https://github.com/scroll-tech/zkevm-circuits.git?tag=v0.13.0#cf71a0e324d42f7895363f4f3e8575d79bb7ea97" +source = "git+https://github.com/scroll-tech/zkevm-circuits.git?tag=v0.13.1#4009e5593f13ba73f64f556011ee5ef47bc4ebf3" dependencies = [ "eth-types 0.13.0", "geth-utils 0.13.0", @@ -1788,7 +1788,7 @@ dependencies = [ [[package]] name = "gadgets" version = "0.12.0" -source = "git+https://github.com/scroll-tech/zkevm-circuits.git?tag=v0.12.0#6a1f65a1f99429f3725ef4d6788f5643bb61aa6f" +source = "git+https://github.com/scroll-tech/zkevm-circuits.git?tag=v0.12.2#6f7b46a3b1ccf9dc448735e8455e1ac6f9e30643" dependencies = [ "eth-types 0.12.0", "halo2_proofs", @@ -1800,7 +1800,7 @@ dependencies = [ [[package]] name = "gadgets" version = "0.13.0" -source = "git+https://github.com/scroll-tech/zkevm-circuits.git?tag=v0.13.0#cf71a0e324d42f7895363f4f3e8575d79bb7ea97" +source = "git+https://github.com/scroll-tech/zkevm-circuits.git?tag=v0.13.1#4009e5593f13ba73f64f556011ee5ef47bc4ebf3" dependencies = [ "eth-types 0.13.0", "halo2_proofs", @@ -1823,7 +1823,7 @@ dependencies = [ [[package]] name = "geth-utils" version = "0.12.0" -source = "git+https://github.com/scroll-tech/zkevm-circuits.git?tag=v0.12.0#6a1f65a1f99429f3725ef4d6788f5643bb61aa6f" +source = "git+https://github.com/scroll-tech/zkevm-circuits.git?tag=v0.12.2#6f7b46a3b1ccf9dc448735e8455e1ac6f9e30643" dependencies = [ "env_logger 0.10.2", "gobuild", @@ -1833,7 +1833,7 @@ dependencies = [ [[package]] name = "geth-utils" version = "0.13.0" -source = "git+https://github.com/scroll-tech/zkevm-circuits.git?tag=v0.13.0#cf71a0e324d42f7895363f4f3e8575d79bb7ea97" +source = "git+https://github.com/scroll-tech/zkevm-circuits.git?tag=v0.13.1#4009e5593f13ba73f64f556011ee5ef47bc4ebf3" dependencies = [ "env_logger 0.10.2", "gobuild", @@ -2673,7 +2673,7 @@ dependencies = [ [[package]] name = "mock" version = "0.12.0" -source = "git+https://github.com/scroll-tech/zkevm-circuits.git?tag=v0.12.0#6a1f65a1f99429f3725ef4d6788f5643bb61aa6f" +source = "git+https://github.com/scroll-tech/zkevm-circuits.git?tag=v0.12.2#6f7b46a3b1ccf9dc448735e8455e1ac6f9e30643" dependencies = [ "eth-types 0.12.0", "ethers-core 2.0.7 (git+https://github.com/scroll-tech/ethers-rs.git?branch=v2.0.7)", @@ -2688,7 +2688,7 @@ dependencies = [ [[package]] name = "mock" version = "0.13.0" -source = "git+https://github.com/scroll-tech/zkevm-circuits.git?tag=v0.13.0#cf71a0e324d42f7895363f4f3e8575d79bb7ea97" +source = "git+https://github.com/scroll-tech/zkevm-circuits.git?tag=v0.13.1#4009e5593f13ba73f64f556011ee5ef47bc4ebf3" dependencies = [ "eth-types 0.13.0", "ethers-core 2.0.7 (git+https://github.com/scroll-tech/ethers-rs.git?branch=v2.0.7)", @@ -2703,7 +2703,7 @@ dependencies = [ [[package]] name = "mpt-zktrie" version = "0.12.0" -source = "git+https://github.com/scroll-tech/zkevm-circuits.git?tag=v0.12.0#6a1f65a1f99429f3725ef4d6788f5643bb61aa6f" +source = "git+https://github.com/scroll-tech/zkevm-circuits.git?tag=v0.12.2#6f7b46a3b1ccf9dc448735e8455e1ac6f9e30643" dependencies = [ "eth-types 0.12.0", "halo2curves", @@ -2717,7 +2717,7 @@ dependencies = [ [[package]] name = "mpt-zktrie" version = "0.13.0" -source = "git+https://github.com/scroll-tech/zkevm-circuits.git?tag=v0.13.0#cf71a0e324d42f7895363f4f3e8575d79bb7ea97" +source = "git+https://github.com/scroll-tech/zkevm-circuits.git?tag=v0.13.1#4009e5593f13ba73f64f556011ee5ef47bc4ebf3" dependencies = [ "eth-types 0.13.0", "halo2curves", @@ -3309,7 +3309,7 @@ dependencies = [ [[package]] name = "prover" version = "0.12.0" -source = "git+https://github.com/scroll-tech/zkevm-circuits.git?tag=v0.12.0#6a1f65a1f99429f3725ef4d6788f5643bb61aa6f" +source = "git+https://github.com/scroll-tech/zkevm-circuits.git?tag=v0.12.2#6f7b46a3b1ccf9dc448735e8455e1ac6f9e30643" dependencies = [ "aggregator 0.12.0", "anyhow", @@ -3343,7 +3343,7 @@ dependencies = [ [[package]] name = "prover" version = "0.13.0" -source = "git+https://github.com/scroll-tech/zkevm-circuits.git?tag=v0.13.0#cf71a0e324d42f7895363f4f3e8575d79bb7ea97" +source = "git+https://github.com/scroll-tech/zkevm-circuits.git?tag=v0.13.1#4009e5593f13ba73f64f556011ee5ef47bc4ebf3" dependencies = [ "aggregator 0.13.0", "anyhow", @@ -5338,7 +5338,7 @@ dependencies = [ [[package]] name = "zkevm-circuits" version = "0.12.0" -source = "git+https://github.com/scroll-tech/zkevm-circuits.git?tag=v0.12.0#6a1f65a1f99429f3725ef4d6788f5643bb61aa6f" +source = "git+https://github.com/scroll-tech/zkevm-circuits.git?tag=v0.12.2#6f7b46a3b1ccf9dc448735e8455e1ac6f9e30643" dependencies = [ "array-init", "bus-mapping 0.12.0", @@ -5380,7 +5380,7 @@ dependencies = [ [[package]] name = "zkevm-circuits" version = "0.13.0" -source = "git+https://github.com/scroll-tech/zkevm-circuits.git?tag=v0.13.0#cf71a0e324d42f7895363f4f3e8575d79bb7ea97" +source = "git+https://github.com/scroll-tech/zkevm-circuits.git?tag=v0.13.1#4009e5593f13ba73f64f556011ee5ef47bc4ebf3" dependencies = [ "array-init", "bus-mapping 0.13.0", diff --git a/prover/Cargo.toml b/prover/Cargo.toml index 96b60de9b..36db6a39b 100644 --- a/prover/Cargo.toml +++ b/prover/Cargo.toml @@ -29,8 +29,8 @@ ethers-core = { git = "https://github.com/scroll-tech/ethers-rs.git", branch = " ethers-providers = { git = "https://github.com/scroll-tech/ethers-rs.git", branch = "v2.0.7" } halo2_proofs = { git = "https://github.com/scroll-tech/halo2.git", branch = "v1.1" } snark-verifier-sdk = { git = "https://github.com/scroll-tech/snark-verifier", branch = "develop", default-features = false, features = ["loader_halo2", "loader_evm", "halo2-pse"] } -prover_darwin = { git = "https://github.com/scroll-tech/zkevm-circuits.git", tag = "v0.12.0", package = "prover", default-features = false, features = ["parallel_syn", "scroll"] } -prover_darwin_v2 = { git = "https://github.com/scroll-tech/zkevm-circuits.git", tag = "v0.13.0", package = "prover", default-features = false, features = ["parallel_syn", "scroll"] } +prover_darwin = { git = "https://github.com/scroll-tech/zkevm-circuits.git", tag = "v0.12.2", package = "prover", default-features = false, features = ["parallel_syn", "scroll"] } +prover_darwin_v2 = { git = "https://github.com/scroll-tech/zkevm-circuits.git", tag = "v0.13.1", package = "prover", default-features = false, features = ["parallel_syn", "scroll"] } base64 = "0.13.1" reqwest = { version = "0.12.4", features = ["gzip"] } reqwest-middleware = "0.3" diff --git a/prover/src/zk_circuits_handler.rs b/prover/src/zk_circuits_handler.rs index 06622d9a7..d1a8eb38c 100644 --- a/prover/src/zk_circuits_handler.rs +++ b/prover/src/zk_circuits_handler.rs @@ -1,3 +1,4 @@ +mod common; mod darwin; mod darwin_v2; diff --git a/prover/src/zk_circuits_handler/common.rs b/prover/src/zk_circuits_handler/common.rs new file mode 100644 index 000000000..8bd192e8c --- /dev/null +++ b/prover/src/zk_circuits_handler/common.rs @@ -0,0 +1,19 @@ +use std::{collections::BTreeMap, rc::Rc}; + +use once_cell::sync::OnceCell; + +use halo2_proofs::{halo2curves::bn256::Bn256, poly::kzg::commitment::ParamsKZG}; + +static mut PARAMS_MAP: OnceCell>>> = OnceCell::new(); + +pub fn get_params_map<'a, F>(load_params_func: F) -> &'a BTreeMap> +where + F: FnOnce() -> BTreeMap>, +{ + unsafe { + PARAMS_MAP.get_or_init(|| { + let params_map = load_params_func(); + Rc::new(params_map) + }) + } +} diff --git a/prover/src/zk_circuits_handler/darwin.rs b/prover/src/zk_circuits_handler/darwin.rs index e18dc2474..8f92a8efe 100644 --- a/prover/src/zk_circuits_handler/darwin.rs +++ b/prover/src/zk_circuits_handler/darwin.rs @@ -11,9 +11,9 @@ use crate::types::{CommonHash, Task}; use std::{cell::RefCell, cmp::Ordering, env, rc::Rc}; use prover_darwin::{ - aggregator::Prover as BatchProver, check_chunk_hashes, zkevm::Prover as ChunkProver, - BatchProof, BatchProvingTask, BlockTrace, BundleProof, BundleProvingTask, ChunkInfo, - ChunkProof, ChunkProvingTask, + aggregator::Prover as BatchProver, check_chunk_hashes, common::Prover as CommonProver, + zkevm::Prover as ChunkProver, BatchProof, BatchProvingTask, BlockTrace, BundleProof, + BundleProvingTask, ChunkInfo, ChunkProof, ChunkProvingTask, }; // Only used for debugging. @@ -39,8 +39,8 @@ fn get_block_number(block_trace: &BlockTrace) -> Option { #[derive(Default)] pub struct DarwinHandler { - chunk_prover: Option>, - batch_prover: Option>, + chunk_prover: Option>>, + batch_prover: Option>>, geth_client: Option>>, } @@ -54,13 +54,29 @@ impl DarwinHandler { ) -> Result { match prover_type { ProverType::Chunk => Ok(Self { - chunk_prover: Some(RefCell::new(ChunkProver::from_dirs(params_dir, assets_dir))), + chunk_prover: { + let degrees = prover_darwin::config::ZKEVM_DEGREES.clone(); + let params_map = super::common::get_params_map(|| { + CommonProver::load_params_map(params_dir, °rees) + }); + Some(RefCell::new(ChunkProver::from_params_and_assets( + params_map, assets_dir, + ))) + }, batch_prover: None, geth_client, }), ProverType::Batch => Ok(Self { - batch_prover: Some(RefCell::new(BatchProver::from_dirs(params_dir, assets_dir))), + batch_prover: { + let degrees = prover_darwin::config::AGG_DEGREES.clone(); + let params_map = super::common::get_params_map(|| { + CommonProver::load_params_map(params_dir, °rees) + }); + Some(RefCell::new(BatchProver::from_params_and_assets( + params_map, assets_dir, + ))) + }, chunk_prover: None, geth_client, }), diff --git a/prover/src/zk_circuits_handler/darwin_v2.rs b/prover/src/zk_circuits_handler/darwin_v2.rs index 9c7ce6910..6ce2af95e 100644 --- a/prover/src/zk_circuits_handler/darwin_v2.rs +++ b/prover/src/zk_circuits_handler/darwin_v2.rs @@ -11,9 +11,9 @@ use crate::types::{CommonHash, Task}; use std::{cell::RefCell, cmp::Ordering, env, rc::Rc}; use prover_darwin_v2::{ - aggregator::Prover as BatchProver, check_chunk_hashes, zkevm::Prover as ChunkProver, - BatchProof, BatchProvingTask, BlockTrace, BundleProof, BundleProvingTask, ChunkInfo, - ChunkProof, ChunkProvingTask, + aggregator::Prover as BatchProver, check_chunk_hashes, common::Prover as CommonProver, + zkevm::Prover as ChunkProver, BatchProof, BatchProvingTask, BlockTrace, BundleProof, + BundleProvingTask, ChunkInfo, ChunkProof, ChunkProvingTask, }; // Only used for debugging. @@ -39,8 +39,8 @@ fn get_block_number(block_trace: &BlockTrace) -> Option { #[derive(Default)] pub struct DarwinV2Handler { - chunk_prover: Option>, - batch_prover: Option>, + chunk_prover: Option>>, + batch_prover: Option>>, geth_client: Option>>, } @@ -54,13 +54,29 @@ impl DarwinV2Handler { ) -> Result { match prover_type { ProverType::Chunk => Ok(Self { - chunk_prover: Some(RefCell::new(ChunkProver::from_dirs(params_dir, assets_dir))), + chunk_prover: { + let degrees = prover_darwin_v2::config::ZKEVM_DEGREES.clone(); + let params_map = super::common::get_params_map(|| { + CommonProver::load_params_map(params_dir, °rees) + }); + Some(RefCell::new(ChunkProver::from_params_and_assets( + params_map, assets_dir, + ))) + }, batch_prover: None, geth_client, }), ProverType::Batch => Ok(Self { - batch_prover: Some(RefCell::new(BatchProver::from_dirs(params_dir, assets_dir))), + batch_prover: { + let degrees = prover_darwin_v2::config::AGG_DEGREES.clone(); + let params_map = super::common::get_params_map(|| { + CommonProver::load_params_map(params_dir, °rees) + }); + Some(RefCell::new(BatchProver::from_params_and_assets( + params_map, assets_dir, + ))) + }, chunk_prover: None, geth_client, }), From 8671048546f5a32a81a43a7806acff5726392fdb Mon Sep 17 00:00:00 2001 From: amoylan2 Date: Tue, 27 Aug 2024 10:53:59 +0000 Subject: [PATCH 2/3] =?UTF-8?q?chore:=20auto=20version=20bump=E2=80=89[bot?= =?UTF-8?q?]?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- common/version/version.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/version/version.go b/common/version/version.go index 53c479ec7..90b47dcbf 100644 --- a/common/version/version.go +++ b/common/version/version.go @@ -5,7 +5,7 @@ import ( "runtime/debug" ) -var tag = "v4.4.51" +var tag = "v4.4.52" var commit = func() string { if info, ok := debug.ReadBuildInfo(); ok { From a0e612c1d15204b85449ad0ec607d5b502cd9bf8 Mon Sep 17 00:00:00 2001 From: Mengran Lan Date: Tue, 27 Aug 2024 19:00:24 +0800 Subject: [PATCH 3/3] add log when initializing params_map --- prover/src/main.rs | 1 + prover/src/zk_circuits_handler/darwin.rs | 13 +++++++++++++ prover/src/zk_circuits_handler/darwin_v2.rs | 13 +++++++++++++ 3 files changed, 27 insertions(+) diff --git a/prover/src/main.rs b/prover/src/main.rs index d1bf82830..75553187a 100644 --- a/prover/src/main.rs +++ b/prover/src/main.rs @@ -1,4 +1,5 @@ #![feature(lazy_cell)] +#![feature(core_intrinsics)] mod config; mod coordinator_client; diff --git a/prover/src/zk_circuits_handler/darwin.rs b/prover/src/zk_circuits_handler/darwin.rs index 8f92a8efe..da5b5f88d 100644 --- a/prover/src/zk_circuits_handler/darwin.rs +++ b/prover/src/zk_circuits_handler/darwin.rs @@ -52,11 +52,18 @@ impl DarwinHandler { assets_dir: &str, geth_client: Option>>, ) -> Result { + let class_name = std::intrinsics::type_name::(); match prover_type { ProverType::Chunk => Ok(Self { chunk_prover: { let degrees = prover_darwin::config::ZKEVM_DEGREES.clone(); let params_map = super::common::get_params_map(|| { + log::info!( + "calling get_params_map from {}, prover_type: {:?}, degrees: {:?}", + class_name, + prover_type, + degrees + ); CommonProver::load_params_map(params_dir, °rees) }); Some(RefCell::new(ChunkProver::from_params_and_assets( @@ -71,6 +78,12 @@ impl DarwinHandler { batch_prover: { let degrees = prover_darwin::config::AGG_DEGREES.clone(); let params_map = super::common::get_params_map(|| { + log::info!( + "calling get_params_map from {}, prover_type: {:?}, degrees: {:?}", + class_name, + prover_type, + degrees + ); CommonProver::load_params_map(params_dir, °rees) }); Some(RefCell::new(BatchProver::from_params_and_assets( diff --git a/prover/src/zk_circuits_handler/darwin_v2.rs b/prover/src/zk_circuits_handler/darwin_v2.rs index 6ce2af95e..616d57b20 100644 --- a/prover/src/zk_circuits_handler/darwin_v2.rs +++ b/prover/src/zk_circuits_handler/darwin_v2.rs @@ -52,11 +52,18 @@ impl DarwinV2Handler { assets_dir: &str, geth_client: Option>>, ) -> Result { + let class_name = std::intrinsics::type_name::(); match prover_type { ProverType::Chunk => Ok(Self { chunk_prover: { let degrees = prover_darwin_v2::config::ZKEVM_DEGREES.clone(); let params_map = super::common::get_params_map(|| { + log::info!( + "calling get_params_map from {}, prover_type: {:?}, degrees: {:?}", + class_name, + prover_type, + degrees + ); CommonProver::load_params_map(params_dir, °rees) }); Some(RefCell::new(ChunkProver::from_params_and_assets( @@ -71,6 +78,12 @@ impl DarwinV2Handler { batch_prover: { let degrees = prover_darwin_v2::config::AGG_DEGREES.clone(); let params_map = super::common::get_params_map(|| { + log::info!( + "calling get_params_map from {}, prover_type: {:?}, degrees: {:?}", + class_name, + prover_type, + degrees + ); CommonProver::load_params_map(params_dir, °rees) }); Some(RefCell::new(BatchProver::from_params_and_assets(