diff --git a/Cargo.lock b/Cargo.lock index b2bd58bd3595..7669c99d77bb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1947,6 +1947,12 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "diff" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56254986775e3233ffa9c4d7d3faaf6d36a2c09d30b20687e9f88bc8bafc16c8" + [[package]] name = "digest" version = "0.9.0" @@ -4726,6 +4732,16 @@ version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" +[[package]] +name = "pretty_assertions" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af7cee1a6c8a5b9208b3cb1061f10c0cb689087b3d8ce85fb9d2dd7a29b6ba66" +dependencies = [ + "diff", + "yansi", +] + [[package]] name = "prettyplease" version = "0.2.15" @@ -7870,6 +7886,12 @@ dependencies = [ "linked-hash-map", ] +[[package]] +name = "yansi" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09041cd90cf85f7f8b2df60c646f853b7f535ce68f85244eb6731cf89fa498ec" + [[package]] name = "zerocopy" version = "0.7.31" @@ -8291,6 +8313,7 @@ name = "zksync_config" version = "0.1.0" dependencies = [ "anyhow", + "rand 0.8.5", "serde", "zksync_basic_types", ] @@ -8824,6 +8847,22 @@ dependencies = [ "syn 2.0.38", ] +[[package]] +name = "zksync_protobuf_config" +version = "0.1.0" +dependencies = [ + "anyhow", + "pretty_assertions", + "prost", + "rand 0.8.5", + "serde_json", + "zksync_basic_types", + "zksync_config", + "zksync_protobuf", + "zksync_protobuf_build", + "zksync_types", +] + [[package]] name = "zksync_queued_job_processor" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index f7ddf706e6ad..e377cee289c3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -33,6 +33,7 @@ members = [ "core/lib/state", "core/lib/storage", "core/lib/types", + "core/lib/protobuf_config", "core/lib/utils", "core/lib/vlog", "core/lib/multivm", diff --git a/core/lib/basic_types/src/basic_fri_types.rs b/core/lib/basic_types/src/basic_fri_types.rs index 47444c6675f9..545acf8214cd 100644 --- a/core/lib/basic_types/src/basic_fri_types.rs +++ b/core/lib/basic_types/src/basic_fri_types.rs @@ -2,7 +2,7 @@ use serde::{Deserialize, Serialize}; -#[derive(Debug, Deserialize, Serialize, Clone, Eq, Hash, PartialEq)] +#[derive(Debug, Deserialize, Serialize, Clone, Eq, Hash, PartialEq, PartialOrd, Ord)] pub struct CircuitIdRoundTuple { pub circuit_id: u8, pub aggregation_round: u8, diff --git a/core/lib/config/Cargo.toml b/core/lib/config/Cargo.toml index 2532e474ca29..ecbc781df655 100644 --- a/core/lib/config/Cargo.toml +++ b/core/lib/config/Cargo.toml @@ -13,4 +13,5 @@ categories = ["cryptography"] zksync_basic_types = { path = "../../lib/basic_types" } anyhow = "1.0" +rand = "0.8" serde = { version = "1.0", features = ["derive"] } diff --git a/core/lib/config/src/configs/database.rs b/core/lib/config/src/configs/database.rs index 578cd6be46a6..acca253d4024 100644 --- a/core/lib/config/src/configs/database.rs +++ b/core/lib/config/src/configs/database.rs @@ -123,7 +123,7 @@ impl DBConfig { /// Collection of different database URLs and general PostgreSQL options. /// All the entries are optional, since some components may only require a subset of them, /// and any component may have overrides. -#[derive(Debug, Clone)] +#[derive(Debug, Clone, PartialEq)] pub struct PostgresConfig { /// URL for the main (sequencer) database. pub master_url: Option, diff --git a/core/lib/config/src/configs/proof_data_handler.rs b/core/lib/config/src/configs/proof_data_handler.rs index b773efbd7df2..e81e55b8a530 100644 --- a/core/lib/config/src/configs/proof_data_handler.rs +++ b/core/lib/config/src/configs/proof_data_handler.rs @@ -15,6 +15,7 @@ pub struct ProofDataHandlerConfig { pub protocol_version_loading_mode: ProtocolVersionLoadingMode, pub fri_protocol_version_id: u16, } + impl ProofDataHandlerConfig { pub fn proof_generation_timeout(&self) -> Duration { Duration::from_secs(self.proof_generation_timeout_in_secs as u64) diff --git a/core/lib/config/src/lib.rs b/core/lib/config/src/lib.rs index d139596b80bc..cde1582a1a28 100644 --- a/core/lib/config/src/lib.rs +++ b/core/lib/config/src/lib.rs @@ -6,3 +6,4 @@ pub use crate::configs::{ }; pub mod configs; +pub mod testonly; diff --git a/core/lib/config/src/testonly.rs b/core/lib/config/src/testonly.rs new file mode 100644 index 000000000000..64a38ff51ae2 --- /dev/null +++ b/core/lib/config/src/testonly.rs @@ -0,0 +1,708 @@ +use std::collections::HashSet; + +use rand::{distributions::Alphanumeric, Rng}; +use zksync_basic_types::{ + basic_fri_types::CircuitIdRoundTuple, network::Network, Address, L2ChainId, H256, +}; + +use crate::configs; + +/// Generator of random configs. +pub struct Gen<'a, R: Rng> { + /// Underlying RNG. + pub rng: &'a mut R, + /// Generate configs with only required fields. + pub required_only: bool, + /// Generate decimal fractions for f64 + /// to avoid rounding errors of decimal encodings. + pub decimal_fractions: bool, +} + +impl<'a, R: Rng> Gen<'a, R> { + pub fn gen(&mut self) -> C { + C::sample(self) + } +} + +pub trait RandomConfig { + fn sample(g: &mut Gen) -> Self; +} + +impl RandomConfig for String { + fn sample(g: &mut Gen) -> Self { + let n = g.rng.gen_range(5..10); + g.rng + .sample_iter(&Alphanumeric) + .take(n) + .map(char::from) + .collect() + } +} + +impl RandomConfig for Option { + fn sample(g: &mut Gen) -> Self { + if g.required_only { + return None; + } + Some(g.gen()) + } +} + +impl RandomConfig for Vec { + fn sample(g: &mut Gen) -> Self { + if g.required_only { + return vec![]; + } + (0..g.rng.gen_range(5..10)).map(|_| g.gen()).collect() + } +} + +impl RandomConfig for HashSet { + fn sample(g: &mut Gen) -> Self { + if g.required_only { + return HashSet::new(); + } + (0..g.rng.gen_range(5..10)).map(|_| g.gen()).collect() + } +} + +impl RandomConfig for bool { + fn sample(g: &mut Gen) -> Self { + g.rng.gen() + } +} + +impl RandomConfig for u8 { + fn sample(g: &mut Gen) -> Self { + g.rng.gen() + } +} + +impl RandomConfig for u16 { + fn sample(g: &mut Gen) -> Self { + g.rng.gen() + } +} + +impl RandomConfig for u32 { + fn sample(g: &mut Gen) -> Self { + g.rng.gen() + } +} + +impl RandomConfig for u64 { + fn sample(g: &mut Gen) -> Self { + g.rng.gen() + } +} + +impl RandomConfig for f64 { + fn sample(g: &mut Gen) -> Self { + if g.decimal_fractions { + const PRECISION: usize = 1000000; + return g.rng.gen_range(0..PRECISION) as f64 / PRECISION as f64; + } + g.rng.gen() + } +} + +impl RandomConfig for usize { + fn sample(g: &mut Gen) -> Self { + g.rng.gen() + } +} + +impl RandomConfig for std::num::NonZeroU32 { + fn sample(g: &mut Gen) -> Self { + g.rng.gen() + } +} + +impl RandomConfig for Address { + fn sample(g: &mut Gen) -> Self { + g.rng.gen() + } +} + +impl RandomConfig for H256 { + fn sample(g: &mut Gen) -> Self { + g.rng.gen() + } +} + +impl RandomConfig for Network { + fn sample(g: &mut Gen) -> Self { + match g.rng.gen_range(0..8) { + 0 => Self::Mainnet, + 1 => Self::Rinkeby, + 2 => Self::Ropsten, + 3 => Self::Goerli, + 4 => Self::Sepolia, + 5 => Self::Localhost, + 6 => Self::Unknown, + _ => Self::Test, + } + } +} + +impl RandomConfig for configs::chain::FeeModelVersion { + fn sample(g: &mut Gen) -> Self { + match g.rng.gen_range(0..2) { + 0 => Self::V1, + _ => Self::V2, + } + } +} + +impl RandomConfig for configs::AlertsConfig { + fn sample(g: &mut Gen) -> Self { + Self { + sporadic_crypto_errors_substrs: g.gen(), + } + } +} + +impl RandomConfig for configs::ApiConfig { + fn sample(g: &mut Gen) -> Self { + Self { + web3_json_rpc: g.gen(), + contract_verification: g.gen(), + prometheus: g.gen(), + healthcheck: g.gen(), + merkle_tree: g.gen(), + } + } +} + +impl RandomConfig for configs::api::Web3JsonRpcConfig { + fn sample(g: &mut Gen) -> Self { + Self { + http_port: g.gen(), + http_url: g.gen(), + ws_port: g.gen(), + ws_url: g.gen(), + req_entities_limit: g.gen(), + filters_limit: g.gen(), + subscriptions_limit: g.gen(), + pubsub_polling_interval: g.gen(), + max_nonce_ahead: g.gen(), + gas_price_scale_factor: g.gen(), + request_timeout: g.gen(), + account_pks: g.gen(), + estimate_gas_scale_factor: g.gen(), + estimate_gas_acceptable_overestimation: g.gen(), + l1_to_l2_transactions_compatibility_mode: g.gen(), + max_tx_size: g.gen(), + vm_execution_cache_misses_limit: g.gen(), + vm_concurrency_limit: g.gen(), + factory_deps_cache_size_mb: g.gen(), + initial_writes_cache_size_mb: g.gen(), + latest_values_cache_size_mb: g.gen(), + fee_history_limit: g.gen(), + max_batch_request_size: g.gen(), + max_response_body_size_mb: g.gen(), + websocket_requests_per_minute_limit: g.gen(), + tree_api_url: g.gen(), + } + } +} + +impl RandomConfig for configs::api::HealthCheckConfig { + fn sample(g: &mut Gen) -> Self { + Self { port: g.gen() } + } +} + +impl RandomConfig for configs::api::ContractVerificationApiConfig { + fn sample(g: &mut Gen) -> Self { + Self { + port: g.gen(), + url: g.gen(), + } + } +} + +impl RandomConfig for configs::api::MerkleTreeApiConfig { + fn sample(g: &mut Gen) -> Self { + Self { port: g.gen() } + } +} + +impl RandomConfig for configs::PrometheusConfig { + fn sample(g: &mut Gen) -> Self { + Self { + listener_port: g.gen(), + pushgateway_url: g.gen(), + push_interval_ms: g.gen(), + } + } +} + +impl RandomConfig for configs::chain::NetworkConfig { + fn sample(g: &mut Gen) -> Self { + Self { + network: g.gen(), + zksync_network: g.gen(), + zksync_network_id: L2ChainId::max(), + } + } +} + +impl RandomConfig for configs::chain::StateKeeperConfig { + fn sample(g: &mut Gen) -> Self { + Self { + transaction_slots: g.gen(), + block_commit_deadline_ms: g.gen(), + miniblock_commit_deadline_ms: g.gen(), + miniblock_seal_queue_capacity: g.gen(), + max_single_tx_gas: g.gen(), + max_allowed_l2_tx_gas_limit: g.gen(), + reject_tx_at_geometry_percentage: g.gen(), + reject_tx_at_eth_params_percentage: g.gen(), + reject_tx_at_gas_percentage: g.gen(), + close_block_at_geometry_percentage: g.gen(), + close_block_at_eth_params_percentage: g.gen(), + close_block_at_gas_percentage: g.gen(), + fee_account_addr: g.gen(), + minimal_l2_gas_price: g.gen(), + compute_overhead_part: g.gen(), + pubdata_overhead_part: g.gen(), + batch_overhead_l1_gas: g.gen(), + max_gas_per_batch: g.gen(), + max_pubdata_per_batch: g.gen(), + fee_model_version: g.gen(), + validation_computational_gas_limit: g.gen(), + save_call_traces: g.gen(), + virtual_blocks_interval: g.gen(), + virtual_blocks_per_miniblock: g.gen(), + upload_witness_inputs_to_gcs: g.gen(), + enum_index_migration_chunk_size: g.gen(), + } + } +} + +impl RandomConfig for configs::chain::OperationsManagerConfig { + fn sample(g: &mut Gen) -> Self { + Self { + delay_interval: g.gen(), + } + } +} + +impl RandomConfig for configs::chain::CircuitBreakerConfig { + fn sample(g: &mut Gen) -> Self { + Self { + sync_interval_ms: g.gen(), + http_req_max_retry_number: g.gen(), + http_req_retry_interval_sec: g.gen(), + replication_lag_limit_sec: g.gen(), + } + } +} + +impl RandomConfig for configs::chain::MempoolConfig { + fn sample(g: &mut Gen) -> Self { + Self { + sync_interval_ms: g.gen(), + sync_batch_size: g.gen(), + capacity: g.gen(), + stuck_tx_timeout: g.gen(), + remove_stuck_txs: g.gen(), + delay_interval: g.gen(), + } + } +} + +impl RandomConfig for configs::ContractVerifierConfig { + fn sample(g: &mut Gen) -> Self { + Self { + compilation_timeout: g.gen(), + polling_interval: g.gen(), + prometheus_port: g.gen(), + } + } +} + +impl RandomConfig for configs::contracts::ProverAtGenesis { + fn sample(g: &mut Gen) -> Self { + match g.rng.gen_range(0..2) { + 0 => Self::Fri, + _ => Self::Old, + } + } +} + +impl RandomConfig for configs::ContractsConfig { + fn sample(g: &mut Gen) -> Self { + Self { + governance_addr: g.gen(), + mailbox_facet_addr: g.gen(), + executor_facet_addr: g.gen(), + admin_facet_addr: g.gen(), + getters_facet_addr: g.gen(), + verifier_addr: g.gen(), + diamond_init_addr: g.gen(), + diamond_upgrade_init_addr: g.gen(), + diamond_proxy_addr: g.gen(), + validator_timelock_addr: g.gen(), + genesis_tx_hash: g.gen(), + l1_erc20_bridge_proxy_addr: g.gen(), + l1_erc20_bridge_impl_addr: g.gen(), + l2_erc20_bridge_addr: g.gen(), + l1_weth_bridge_proxy_addr: g.gen(), + l2_weth_bridge_addr: g.gen(), + l1_allow_list_addr: g.gen(), + l2_testnet_paymaster_addr: g.gen(), + recursion_scheduler_level_vk_hash: g.gen(), + recursion_node_level_vk_hash: g.gen(), + recursion_leaf_level_vk_hash: g.gen(), + recursion_circuits_set_vks_hash: g.gen(), + l1_multicall3_addr: g.gen(), + fri_recursion_scheduler_level_vk_hash: g.gen(), + fri_recursion_node_level_vk_hash: g.gen(), + fri_recursion_leaf_level_vk_hash: g.gen(), + prover_at_genesis: g.gen(), + snark_wrapper_vk_hash: g.gen(), + } + } +} + +impl RandomConfig for configs::database::MerkleTreeMode { + fn sample(g: &mut Gen) -> Self { + match g.rng.gen_range(0..2) { + 0 => Self::Full, + _ => Self::Lightweight, + } + } +} + +impl RandomConfig for configs::database::MerkleTreeConfig { + fn sample(g: &mut Gen) -> Self { + Self { + path: g.gen(), + mode: g.gen(), + multi_get_chunk_size: g.gen(), + block_cache_size_mb: g.gen(), + memtable_capacity_mb: g.gen(), + stalled_writes_timeout_sec: g.gen(), + max_l1_batches_per_iter: g.gen(), + } + } +} + +impl RandomConfig for configs::database::DBConfig { + fn sample(g: &mut Gen) -> Self { + Self { + state_keeper_db_path: g.gen(), + merkle_tree: g.gen(), + } + } +} + +impl RandomConfig for configs::database::PostgresConfig { + fn sample(g: &mut Gen) -> Self { + Self { + master_url: g.gen(), + replica_url: g.gen(), + prover_url: g.gen(), + max_connections: g.gen(), + statement_timeout_sec: g.gen(), + } + } +} + +impl RandomConfig for configs::ETHClientConfig { + fn sample(g: &mut Gen) -> Self { + Self { + chain_id: g.gen(), + web3_url: g.gen(), + } + } +} + +impl RandomConfig for configs::ETHSenderConfig { + fn sample(g: &mut Gen) -> Self { + Self { + sender: g.gen(), + gas_adjuster: g.gen(), + } + } +} + +impl RandomConfig for configs::eth_sender::ProofSendingMode { + fn sample(g: &mut Gen) -> Self { + match g.rng.gen_range(0..3) { + 0 => Self::OnlyRealProofs, + 1 => Self::OnlySampledProofs, + _ => Self::SkipEveryProof, + } + } +} + +impl RandomConfig for configs::eth_sender::ProofLoadingMode { + fn sample(g: &mut Gen) -> Self { + match g.rng.gen_range(0..2) { + 0 => Self::OldProofFromDb, + _ => Self::FriProofFromGcs, + } + } +} + +impl RandomConfig for configs::eth_sender::SenderConfig { + fn sample(g: &mut Gen) -> Self { + Self { + aggregated_proof_sizes: g.gen(), + wait_confirmations: g.gen(), + tx_poll_period: g.gen(), + aggregate_tx_poll_period: g.gen(), + max_txs_in_flight: g.gen(), + proof_sending_mode: g.gen(), + max_aggregated_tx_gas: g.gen(), + max_eth_tx_data_size: g.gen(), + max_aggregated_blocks_to_commit: g.gen(), + max_aggregated_blocks_to_execute: g.gen(), + aggregated_block_commit_deadline: g.gen(), + aggregated_block_prove_deadline: g.gen(), + aggregated_block_execute_deadline: g.gen(), + timestamp_criteria_max_allowed_lag: g.gen(), + l1_batch_min_age_before_execute_seconds: g.gen(), + max_acceptable_priority_fee_in_gwei: g.gen(), + proof_loading_mode: g.gen(), + } + } +} + +impl RandomConfig for configs::eth_sender::GasAdjusterConfig { + fn sample(g: &mut Gen) -> Self { + Self { + default_priority_fee_per_gas: g.gen(), + max_base_fee_samples: g.gen(), + pricing_formula_parameter_a: g.gen(), + pricing_formula_parameter_b: g.gen(), + internal_l1_pricing_multiplier: g.gen(), + internal_enforced_l1_gas_price: g.gen(), + poll_period: g.gen(), + max_l1_gas_price: g.gen(), + } + } +} + +impl RandomConfig for configs::ETHWatchConfig { + fn sample(g: &mut Gen) -> Self { + Self { + confirmations_for_eth_event: g.gen(), + eth_node_poll_interval: g.gen(), + } + } +} + +impl RandomConfig for configs::FriProofCompressorConfig { + fn sample(g: &mut Gen) -> Self { + Self { + compression_mode: g.gen(), + prometheus_listener_port: g.gen(), + prometheus_pushgateway_url: g.gen(), + prometheus_push_interval_ms: g.gen(), + generation_timeout_in_secs: g.gen(), + max_attempts: g.gen(), + universal_setup_path: g.gen(), + universal_setup_download_url: g.gen(), + verify_wrapper_proof: g.gen(), + } + } +} + +impl RandomConfig for configs::fri_prover::SetupLoadMode { + fn sample(g: &mut Gen) -> Self { + match g.rng.gen_range(0..2) { + 0 => Self::FromDisk, + _ => Self::FromMemory, + } + } +} + +impl RandomConfig for configs::FriProverConfig { + fn sample(g: &mut Gen) -> Self { + Self { + setup_data_path: g.gen(), + prometheus_port: g.gen(), + max_attempts: g.gen(), + generation_timeout_in_secs: g.gen(), + base_layer_circuit_ids_to_be_verified: g.gen(), + recursive_layer_circuit_ids_to_be_verified: g.gen(), + setup_load_mode: g.gen(), + specialized_group_id: g.gen(), + witness_vector_generator_thread_count: g.gen(), + queue_capacity: g.gen(), + witness_vector_receiver_port: g.gen(), + zone_read_url: g.gen(), + shall_save_to_public_bucket: g.gen(), + } + } +} + +impl RandomConfig for configs::FriProverGatewayConfig { + fn sample(g: &mut Gen) -> Self { + Self { + api_url: g.gen(), + api_poll_duration_secs: g.gen(), + prometheus_listener_port: g.gen(), + prometheus_pushgateway_url: g.gen(), + prometheus_push_interval_ms: g.gen(), + } + } +} + +impl RandomConfig for CircuitIdRoundTuple { + fn sample(g: &mut Gen) -> Self { + Self { + circuit_id: g.gen(), + aggregation_round: g.gen(), + } + } +} + +impl RandomConfig for configs::fri_prover_group::FriProverGroupConfig { + fn sample(g: &mut Gen) -> Self { + Self { + group_0: g.gen(), + group_1: g.gen(), + group_2: g.gen(), + group_3: g.gen(), + group_4: g.gen(), + group_5: g.gen(), + group_6: g.gen(), + group_7: g.gen(), + group_8: g.gen(), + group_9: g.gen(), + group_10: g.gen(), + group_11: g.gen(), + group_12: g.gen(), + } + } +} + +impl RandomConfig for configs::FriWitnessGeneratorConfig { + fn sample(g: &mut Gen) -> Self { + Self { + generation_timeout_in_secs: g.gen(), + max_attempts: g.gen(), + blocks_proving_percentage: g.gen(), + dump_arguments_for_blocks: g.gen(), + last_l1_batch_to_process: g.gen(), + force_process_block: g.gen(), + shall_save_to_public_bucket: g.gen(), + } + } +} + +impl RandomConfig for configs::FriWitnessVectorGeneratorConfig { + fn sample(g: &mut Gen) -> Self { + Self { + max_prover_reservation_duration_in_secs: g.gen(), + prover_instance_wait_timeout_in_secs: g.gen(), + prover_instance_poll_time_in_milli_secs: g.gen(), + prometheus_listener_port: g.gen(), + prometheus_pushgateway_url: g.gen(), + prometheus_push_interval_ms: g.gen(), + specialized_group_id: g.gen(), + } + } +} + +impl RandomConfig for configs::house_keeper::HouseKeeperConfig { + fn sample(g: &mut Gen) -> Self { + Self { + l1_batch_metrics_reporting_interval_ms: g.gen(), + gpu_prover_queue_reporting_interval_ms: g.gen(), + prover_job_retrying_interval_ms: g.gen(), + prover_stats_reporting_interval_ms: g.gen(), + witness_job_moving_interval_ms: g.gen(), + witness_generator_stats_reporting_interval_ms: g.gen(), + fri_witness_job_moving_interval_ms: g.gen(), + fri_prover_job_retrying_interval_ms: g.gen(), + fri_witness_generator_job_retrying_interval_ms: g.gen(), + prover_db_pool_size: g.gen(), + fri_prover_stats_reporting_interval_ms: g.gen(), + fri_proof_compressor_job_retrying_interval_ms: g.gen(), + fri_proof_compressor_stats_reporting_interval_ms: g.gen(), + } + } +} + +impl RandomConfig for configs::object_store::ObjectStoreMode { + fn sample(g: &mut Gen) -> Self { + match g.rng.gen_range(0..4) { + 0 => Self::GCS, + 1 => Self::GCSWithCredentialFile, + 2 => Self::FileBacked, + _ => Self::GCSAnonymousReadOnly, + } + } +} + +impl RandomConfig for configs::ObjectStoreConfig { + fn sample(g: &mut Gen) -> Self { + Self { + bucket_base_url: g.gen(), + mode: g.gen(), + file_backed_base_path: g.gen(), + gcs_credential_file_path: g.gen(), + max_retries: g.gen(), + } + } +} + +impl RandomConfig for configs::proof_data_handler::ProtocolVersionLoadingMode { + fn sample(g: &mut Gen) -> Self { + match g.rng.gen_range(0..2) { + 0 => Self::FromDb, + _ => Self::FromEnvVar, + } + } +} + +impl RandomConfig for configs::ProofDataHandlerConfig { + fn sample(g: &mut Gen) -> Self { + Self { + http_port: g.gen(), + proof_generation_timeout_in_secs: g.gen(), + protocol_version_loading_mode: g.gen(), + fri_protocol_version_id: g.gen(), + } + } +} + +impl RandomConfig for configs::SnapshotsCreatorConfig { + fn sample(g: &mut Gen) -> Self { + Self { + storage_logs_chunk_size: g.gen(), + concurrent_queries_count: g.gen(), + } + } +} + +impl RandomConfig for configs::witness_generator::BasicWitnessGeneratorDataSource { + fn sample(g: &mut Gen) -> Self { + match g.rng.gen_range(0..2) { + 0 => Self::FromPostgres, + 1 => Self::FromPostgresShadowBlob, + _ => Self::FromBlob, + } + } +} + +impl RandomConfig for configs::WitnessGeneratorConfig { + fn sample(g: &mut Gen) -> Self { + Self { + generation_timeout_in_secs: g.gen(), + initial_setup_key_path: g.gen(), + key_download_url: g.gen(), + max_attempts: g.gen(), + blocks_proving_percentage: g.gen(), + dump_arguments_for_blocks: g.gen(), + last_l1_batch_to_process: g.gen(), + data_source: g.gen(), + } + } +} diff --git a/core/lib/protobuf_config/Cargo.toml b/core/lib/protobuf_config/Cargo.toml new file mode 100644 index 000000000000..a91ca2188422 --- /dev/null +++ b/core/lib/protobuf_config/Cargo.toml @@ -0,0 +1,30 @@ +[package] +name = "zksync_protobuf_config" +version = "0.1.0" +edition = "2021" +authors = ["The Matter Labs Team "] +homepage = "https://zksync.io/" +repository = "https://github.com/matter-labs/zksync-era" +license = "MIT OR Apache-2.0" +keywords = ["blockchain", "zksync"] +categories = ["cryptography"] + +links = "zksync_protobuf_config_proto" + +[dependencies] +serde_json = "1.0" +zksync_basic_types = { path = "../basic_types" } +zksync_config = { path = "../config" } +zksync_protobuf = { version = "0.1.0", git = "https://github.com/matter-labs/era-consensus.git", rev = "5b3d383d7a65b0fbe2a771fecf4313f5083be9ae" } +zksync_types = { path = "../types" } + +anyhow = "1.0" +prost = "0.12.1" + +[dev-dependencies] +rand = "0.8" +pretty_assertions = "1.4.0" + +[build-dependencies] +zksync_protobuf_build = { version = "0.1.0", git = "https://github.com/matter-labs/era-consensus.git", rev = "5b3d383d7a65b0fbe2a771fecf4313f5083be9ae" } + diff --git a/core/lib/protobuf_config/build.rs b/core/lib/protobuf_config/build.rs new file mode 100644 index 000000000000..66afd8fea6da --- /dev/null +++ b/core/lib/protobuf_config/build.rs @@ -0,0 +1,12 @@ +//! Generates rust code from protobufs. +fn main() { + zksync_protobuf_build::Config { + input_root: "src/proto".into(), + proto_root: "zksync/config".into(), + dependencies: vec![], + protobuf_crate: "::zksync_protobuf".parse().unwrap(), + is_public: true, + } + .generate() + .unwrap(); +} diff --git a/core/lib/protobuf_config/src/alerts.rs b/core/lib/protobuf_config/src/alerts.rs new file mode 100644 index 000000000000..9c314301934f --- /dev/null +++ b/core/lib/protobuf_config/src/alerts.rs @@ -0,0 +1,18 @@ +use zksync_config::configs::AlertsConfig; + +use crate::{proto, repr::ProtoRepr}; + +impl ProtoRepr for proto::Alerts { + type Type = AlertsConfig; + fn read(&self) -> anyhow::Result { + Ok(Self::Type { + sporadic_crypto_errors_substrs: self.sporadic_crypto_errors_substrs.clone(), + }) + } + + fn build(this: &Self::Type) -> Self { + Self { + sporadic_crypto_errors_substrs: this.sporadic_crypto_errors_substrs.clone(), + } + } +} diff --git a/core/lib/protobuf_config/src/api.rs b/core/lib/protobuf_config/src/api.rs new file mode 100644 index 000000000000..26173de29ff0 --- /dev/null +++ b/core/lib/protobuf_config/src/api.rs @@ -0,0 +1,222 @@ +use anyhow::Context as _; +use zksync_config::configs::{api, ApiConfig}; +use zksync_protobuf::required; + +use crate::{ + parse_h256, proto, + repr::{read_required_repr, ProtoRepr}, +}; + +impl ProtoRepr for proto::Api { + type Type = ApiConfig; + fn read(&self) -> anyhow::Result { + Ok(Self::Type { + web3_json_rpc: read_required_repr(&self.web3_json_rpc).context("web3_json_rpc")?, + contract_verification: read_required_repr(&self.contract_verification) + .context("contract_verification")?, + prometheus: read_required_repr(&self.prometheus).context("prometheus")?, + healthcheck: read_required_repr(&self.healthcheck).context("healthcheck")?, + merkle_tree: read_required_repr(&self.merkle_tree).context("merkle_tree")?, + }) + } + + fn build(this: &Self::Type) -> Self { + Self { + web3_json_rpc: Some(ProtoRepr::build(&this.web3_json_rpc)), + contract_verification: Some(ProtoRepr::build(&this.contract_verification)), + prometheus: Some(ProtoRepr::build(&this.prometheus)), + healthcheck: Some(ProtoRepr::build(&this.healthcheck)), + merkle_tree: Some(ProtoRepr::build(&this.merkle_tree)), + } + } +} + +impl ProtoRepr for proto::Web3JsonRpc { + type Type = api::Web3JsonRpcConfig; + fn read(&self) -> anyhow::Result { + Ok(Self::Type { + http_port: required(&self.http_port) + .and_then(|p| Ok((*p).try_into()?)) + .context("http_port")?, + http_url: required(&self.http_url).context("http_url")?.clone(), + ws_port: required(&self.ws_port) + .and_then(|p| Ok((*p).try_into()?)) + .context("ws_port")?, + ws_url: required(&self.ws_url).context("ws_url")?.clone(), + req_entities_limit: self.req_entities_limit, + filters_limit: self.filters_limit, + subscriptions_limit: self.subscriptions_limit, + pubsub_polling_interval: self.pubsub_polling_interval, + max_nonce_ahead: *required(&self.max_nonce_ahead).context("max_nonce_ahead")?, + gas_price_scale_factor: *required(&self.gas_price_scale_factor) + .context("gas_price_scale_factor")?, + request_timeout: self.request_timeout, + account_pks: self + .account_pks + .as_ref() + .map(|keys| { + keys.keys + .iter() + .enumerate() + .map(|(i, k)| parse_h256(k).context(i)) + .collect::>() + .context("keys") + }) + .transpose() + .context("account_pks")?, + estimate_gas_scale_factor: *required(&self.estimate_gas_scale_factor) + .context("estimate_gas_scale_factor")?, + estimate_gas_acceptable_overestimation: *required( + &self.estimate_gas_acceptable_overestimation, + ) + .context("acceptable_overestimation")?, + l1_to_l2_transactions_compatibility_mode: *required( + &self.l1_to_l2_transactions_compatibility_mode, + ) + .context("l1_to_l2_transactions_compatibility_mode")?, + max_tx_size: required(&self.max_tx_size) + .and_then(|x| Ok((*x).try_into()?)) + .context("max_tx_size")?, + vm_execution_cache_misses_limit: self + .vm_execution_cache_misses_limit + .map(|x| x.try_into()) + .transpose() + .context("vm_execution_cache_misses_limit")?, + vm_concurrency_limit: self + .vm_concurrency_limit + .map(|x| x.try_into()) + .transpose() + .context("vm_concurrency_limit")?, + factory_deps_cache_size_mb: self + .factory_deps_cache_size_mb + .map(|x| x.try_into()) + .transpose() + .context("factory_deps_cache_size_mb")?, + initial_writes_cache_size_mb: self + .initial_writes_cache_size_mb + .map(|x| x.try_into()) + .transpose() + .context("initial_writes_cache_size_mb")?, + latest_values_cache_size_mb: self + .latest_values_cache_size_mb + .map(|x| x.try_into()) + .transpose() + .context("latests_values_cache_size_mb")?, + fee_history_limit: self.fee_history_limit, + max_batch_request_size: self + .max_batch_request_size + .map(|x| x.try_into()) + .transpose() + .context("max_batch_requres_size")?, + max_response_body_size_mb: self + .max_response_body_size_mb + .map(|x| x.try_into()) + .transpose() + .context("max_response_body_size_mb")?, + websocket_requests_per_minute_limit: self + .websocket_requests_per_minute_limit + .map(|x| x.try_into()) + .transpose() + .context("websocket_requests_per_minute_limit")?, + tree_api_url: self.tree_api_url.clone(), + }) + } + fn build(this: &Self::Type) -> Self { + Self { + http_port: Some(this.http_port.into()), + http_url: Some(this.http_url.clone()), + ws_port: Some(this.ws_port.into()), + ws_url: Some(this.ws_url.clone()), + req_entities_limit: this.req_entities_limit, + filters_limit: this.filters_limit, + subscriptions_limit: this.subscriptions_limit, + pubsub_polling_interval: this.pubsub_polling_interval, + max_nonce_ahead: Some(this.max_nonce_ahead), + gas_price_scale_factor: Some(this.gas_price_scale_factor), + request_timeout: this.request_timeout, + account_pks: this.account_pks.as_ref().map(|keys| proto::PrivateKeys { + keys: keys.iter().map(|k| k.as_bytes().into()).collect(), + }), + estimate_gas_scale_factor: Some(this.estimate_gas_scale_factor), + estimate_gas_acceptable_overestimation: Some( + this.estimate_gas_acceptable_overestimation, + ), + l1_to_l2_transactions_compatibility_mode: Some( + this.l1_to_l2_transactions_compatibility_mode, + ), + max_tx_size: Some(this.max_tx_size.try_into().unwrap()), + vm_execution_cache_misses_limit: this + .vm_execution_cache_misses_limit + .map(|x| x.try_into().unwrap()), + vm_concurrency_limit: this.vm_concurrency_limit.map(|x| x.try_into().unwrap()), + factory_deps_cache_size_mb: this + .factory_deps_cache_size_mb + .map(|x| x.try_into().unwrap()), + initial_writes_cache_size_mb: this + .initial_writes_cache_size_mb + .map(|x| x.try_into().unwrap()), + latest_values_cache_size_mb: this + .latest_values_cache_size_mb + .map(|x| x.try_into().unwrap()), + fee_history_limit: this.fee_history_limit, + max_batch_request_size: this.max_batch_request_size.map(|x| x.try_into().unwrap()), + max_response_body_size_mb: this + .max_response_body_size_mb + .map(|x| x.try_into().unwrap()), + websocket_requests_per_minute_limit: this + .websocket_requests_per_minute_limit + .map(|x| x.into()), + tree_api_url: this.tree_api_url.clone(), + } + } +} + +impl ProtoRepr for proto::ContractVerificationApi { + type Type = api::ContractVerificationApiConfig; + fn read(&self) -> anyhow::Result { + Ok(Self::Type { + port: required(&self.port) + .and_then(|p| Ok((*p).try_into()?)) + .context("port")?, + url: required(&self.url).context("url")?.clone(), + }) + } + fn build(this: &Self::Type) -> Self { + Self { + port: Some(this.port.into()), + url: Some(this.url.clone()), + } + } +} + +impl ProtoRepr for proto::HealthCheck { + type Type = api::HealthCheckConfig; + fn read(&self) -> anyhow::Result { + Ok(Self::Type { + port: required(&self.port) + .and_then(|p| Ok((*p).try_into()?)) + .context("port")?, + }) + } + fn build(this: &Self::Type) -> Self { + Self { + port: Some(this.port.into()), + } + } +} + +impl ProtoRepr for proto::MerkleTreeApi { + type Type = api::MerkleTreeApiConfig; + fn read(&self) -> anyhow::Result { + Ok(Self::Type { + port: required(&self.port) + .and_then(|p| Ok((*p).try_into()?)) + .context("port")?, + }) + } + fn build(this: &Self::Type) -> Self { + Self { + port: Some(this.port.into()), + } + } +} diff --git a/core/lib/protobuf_config/src/chain.rs b/core/lib/protobuf_config/src/chain.rs new file mode 100644 index 000000000000..fc775c763681 --- /dev/null +++ b/core/lib/protobuf_config/src/chain.rs @@ -0,0 +1,248 @@ +use anyhow::Context as _; +use zksync_basic_types::network::Network; +use zksync_config::configs; +use zksync_protobuf::required; + +use crate::{parse_h160, proto, repr::ProtoRepr}; + +impl proto::Network { + fn new(n: &Network) -> Self { + match n { + Network::Mainnet => Self::Mainnet, + Network::Rinkeby => Self::Rinkeby, + Network::Ropsten => Self::Ropsten, + Network::Goerli => Self::Goerli, + Network::Sepolia => Self::Sepolia, + Network::Localhost => Self::Localhost, + Network::Unknown => Self::Unknown, + Network::Test => Self::Test, + } + } + + fn parse(&self) -> Network { + match self { + Self::Mainnet => Network::Mainnet, + Self::Rinkeby => Network::Rinkeby, + Self::Ropsten => Network::Ropsten, + Self::Goerli => Network::Goerli, + Self::Sepolia => Network::Sepolia, + Self::Localhost => Network::Localhost, + Self::Unknown => Network::Unknown, + Self::Test => Network::Test, + } + } +} + +impl proto::FeeModelVersion { + fn new(n: &configs::chain::FeeModelVersion) -> Self { + use configs::chain::FeeModelVersion as From; + match n { + From::V1 => Self::V1, + From::V2 => Self::V2, + } + } + + fn parse(&self) -> configs::chain::FeeModelVersion { + use configs::chain::FeeModelVersion as To; + match self { + Self::V1 => To::V1, + Self::V2 => To::V2, + } + } +} + +impl ProtoRepr for proto::EthNetwork { + type Type = configs::chain::NetworkConfig; + fn read(&self) -> anyhow::Result { + Ok(Self::Type { + network: required(&self.network) + .and_then(|x| Ok(proto::Network::try_from(*x)?)) + .context("network")? + .parse(), + zksync_network: required(&self.zksync_network) + .context("zksync_network")? + .clone(), + zksync_network_id: required(&self.zksync_network_id) + .and_then(|x| (*x).try_into().map_err(anyhow::Error::msg)) + .context("zksync_network_id")?, + }) + } + + fn build(this: &Self::Type) -> Self { + Self { + network: Some(proto::Network::new(&this.network).into()), + zksync_network: Some(this.zksync_network.clone()), + zksync_network_id: Some(this.zksync_network_id.as_u64()), + } + } +} + +impl ProtoRepr for proto::StateKeeper { + type Type = configs::chain::StateKeeperConfig; + fn read(&self) -> anyhow::Result { + Ok(Self::Type { + transaction_slots: required(&self.transaction_slots) + .and_then(|x| Ok((*x).try_into()?)) + .context("transaction_slots")?, + block_commit_deadline_ms: *required(&self.block_commit_deadline_ms) + .context("block_commit_deadline_ms")?, + miniblock_commit_deadline_ms: *required(&self.miniblock_commit_deadline_ms) + .context("miniblock_commit_deadline_ms")?, + miniblock_seal_queue_capacity: required(&self.miniblock_seal_queue_capacity) + .and_then(|x| Ok((*x).try_into()?)) + .context("miniblock_seal_queue_capacity")?, + max_single_tx_gas: *required(&self.max_single_tx_gas).context("max_single_tx_gas")?, + max_allowed_l2_tx_gas_limit: *required(&self.max_allowed_l2_tx_gas_limit) + .context("max_allowed_l2_tx_gas_limit")?, + reject_tx_at_geometry_percentage: *required(&self.reject_tx_at_geometry_percentage) + .context("reject_tx_at_geometry_percentage")?, + reject_tx_at_eth_params_percentage: *required(&self.reject_tx_at_eth_params_percentage) + .context("reject_tx_at_eth_params_percentage")?, + reject_tx_at_gas_percentage: *required(&self.reject_tx_at_gas_percentage) + .context("reject_tx_at_gas_percentage")?, + close_block_at_geometry_percentage: *required(&self.close_block_at_geometry_percentage) + .context("close_block_at_geometry_percentage")?, + close_block_at_eth_params_percentage: *required( + &self.close_block_at_eth_params_percentage, + ) + .context("close_block_at_eth_params_percentage")?, + close_block_at_gas_percentage: *required(&self.close_block_at_gas_percentage) + .context("close_block_at_gas_percentage")?, + fee_account_addr: required(&self.fee_account_addr) + .and_then(|a| parse_h160(a)) + .context("fee_account_addr")?, + minimal_l2_gas_price: *required(&self.minimal_l2_gas_price) + .context("minimal_l2_gas_price")?, + compute_overhead_part: *required(&self.compute_overhead_part) + .context("compute_overhead_part")?, + pubdata_overhead_part: *required(&self.pubdata_overhead_part) + .context("pubdata_overhead_part")?, + batch_overhead_l1_gas: *required(&self.batch_overhead_l1_gas) + .context("batch_overhead_l1_gas")?, + max_gas_per_batch: *required(&self.max_gas_per_batch).context("max_gas_per_batch")?, + max_pubdata_per_batch: *required(&self.max_pubdata_per_batch) + .context("max_pubdata_per_batch")?, + fee_model_version: required(&self.fee_model_version) + .and_then(|x| Ok(proto::FeeModelVersion::try_from(*x)?)) + .context("fee_model_version")? + .parse(), + validation_computational_gas_limit: *required(&self.validation_computational_gas_limit) + .context("validation_computational_gas_limit")?, + save_call_traces: *required(&self.save_call_traces).context("save_call_traces")?, + virtual_blocks_interval: *required(&self.virtual_blocks_interval) + .context("virtual_blocks_interval")?, + virtual_blocks_per_miniblock: *required(&self.virtual_blocks_per_miniblock) + .context("virtual_blocks_per_miniblock")?, + upload_witness_inputs_to_gcs: *required(&self.upload_witness_inputs_to_gcs) + .context("upload_witness_inputs_to_gcs")?, + enum_index_migration_chunk_size: self + .enum_index_migration_chunk_size + .map(|x| x.try_into()) + .transpose() + .context("enum_index_migration_chunk_size")?, + }) + } + + fn build(this: &Self::Type) -> Self { + Self { + transaction_slots: Some(this.transaction_slots.try_into().unwrap()), + block_commit_deadline_ms: Some(this.block_commit_deadline_ms), + miniblock_commit_deadline_ms: Some(this.miniblock_commit_deadline_ms), + miniblock_seal_queue_capacity: Some( + this.miniblock_seal_queue_capacity.try_into().unwrap(), + ), + max_single_tx_gas: Some(this.max_single_tx_gas), + max_allowed_l2_tx_gas_limit: Some(this.max_allowed_l2_tx_gas_limit), + reject_tx_at_geometry_percentage: Some(this.reject_tx_at_geometry_percentage), + reject_tx_at_eth_params_percentage: Some(this.reject_tx_at_eth_params_percentage), + reject_tx_at_gas_percentage: Some(this.reject_tx_at_gas_percentage), + close_block_at_geometry_percentage: Some(this.close_block_at_geometry_percentage), + close_block_at_eth_params_percentage: Some(this.close_block_at_eth_params_percentage), + close_block_at_gas_percentage: Some(this.close_block_at_gas_percentage), + fee_account_addr: Some(this.fee_account_addr.as_bytes().into()), + minimal_l2_gas_price: Some(this.minimal_l2_gas_price), + compute_overhead_part: Some(this.compute_overhead_part), + pubdata_overhead_part: Some(this.pubdata_overhead_part), + batch_overhead_l1_gas: Some(this.batch_overhead_l1_gas), + max_gas_per_batch: Some(this.max_gas_per_batch), + max_pubdata_per_batch: Some(this.max_pubdata_per_batch), + fee_model_version: Some(proto::FeeModelVersion::new(&this.fee_model_version).into()), + validation_computational_gas_limit: Some(this.validation_computational_gas_limit), + save_call_traces: Some(this.save_call_traces), + virtual_blocks_interval: Some(this.virtual_blocks_interval), + virtual_blocks_per_miniblock: Some(this.virtual_blocks_per_miniblock), + upload_witness_inputs_to_gcs: Some(this.upload_witness_inputs_to_gcs), + enum_index_migration_chunk_size: this + .enum_index_migration_chunk_size + .as_ref() + .map(|x| (*x).try_into().unwrap()), + } + } +} + +impl ProtoRepr for proto::OperationsManager { + type Type = configs::chain::OperationsManagerConfig; + fn read(&self) -> anyhow::Result { + Ok(Self::Type { + delay_interval: *required(&self.delay_interval).context("delay_interval")?, + }) + } + + fn build(this: &Self::Type) -> Self { + Self { + delay_interval: Some(this.delay_interval), + } + } +} + +impl ProtoRepr for proto::Mempool { + type Type = configs::chain::MempoolConfig; + fn read(&self) -> anyhow::Result { + Ok(Self::Type { + sync_interval_ms: *required(&self.sync_interval_ms).context("sync_interval_ms")?, + sync_batch_size: required(&self.sync_batch_size) + .and_then(|x| Ok((*x).try_into()?)) + .context("sync_batch_size")?, + capacity: *required(&self.capacity).context("capacity")?, + stuck_tx_timeout: *required(&self.stuck_tx_timeout).context("stuck_tx_timeout")?, + remove_stuck_txs: *required(&self.remove_stuck_txs).context("remove_stuck_txs")?, + delay_interval: *required(&self.delay_interval).context("delay_interval")?, + }) + } + + fn build(this: &Self::Type) -> Self { + Self { + sync_interval_ms: Some(this.sync_interval_ms), + sync_batch_size: Some(this.sync_batch_size.try_into().unwrap()), + capacity: Some(this.capacity), + stuck_tx_timeout: Some(this.stuck_tx_timeout), + remove_stuck_txs: Some(this.remove_stuck_txs), + delay_interval: Some(this.delay_interval), + } + } +} + +impl ProtoRepr for proto::CircuitBreaker { + type Type = configs::chain::CircuitBreakerConfig; + fn read(&self) -> anyhow::Result { + Ok(Self::Type { + sync_interval_ms: *required(&self.sync_interval_ms).context("sync_interval_ms")?, + http_req_max_retry_number: required(&self.http_req_max_retry_number) + .and_then(|x| Ok((*x).try_into()?)) + .context("http_req_max_retry_number")?, + http_req_retry_interval_sec: required(&self.http_req_retry_interval_sec) + .and_then(|x| Ok((*x).try_into()?)) + .context("http_req_retry_interval_sec")?, + replication_lag_limit_sec: self.replication_lag_limit_sec, + }) + } + + fn build(this: &Self::Type) -> Self { + Self { + sync_interval_ms: Some(this.sync_interval_ms), + http_req_max_retry_number: Some(this.http_req_max_retry_number.try_into().unwrap()), + http_req_retry_interval_sec: Some(this.http_req_retry_interval_sec.into()), + replication_lag_limit_sec: this.replication_lag_limit_sec, + } + } +} diff --git a/core/lib/protobuf_config/src/contract_verifier.rs b/core/lib/protobuf_config/src/contract_verifier.rs new file mode 100644 index 000000000000..caa109fdca36 --- /dev/null +++ b/core/lib/protobuf_config/src/contract_verifier.rs @@ -0,0 +1,27 @@ +use anyhow::Context as _; +use zksync_config::configs; +use zksync_protobuf::required; + +use crate::{proto, repr::ProtoRepr}; + +impl ProtoRepr for proto::ContractVerifier { + type Type = configs::ContractVerifierConfig; + fn read(&self) -> anyhow::Result { + Ok(Self::Type { + compilation_timeout: *required(&self.compilation_timeout) + .context("compilation_timeout")?, + polling_interval: self.polling_interval, + prometheus_port: required(&self.prometheus_port) + .and_then(|x| Ok((*x).try_into()?)) + .context("prometheus_port")?, + }) + } + + fn build(this: &Self::Type) -> Self { + Self { + compilation_timeout: Some(this.compilation_timeout), + polling_interval: this.polling_interval, + prometheus_port: Some(this.prometheus_port.into()), + } + } +} diff --git a/core/lib/protobuf_config/src/contracts.rs b/core/lib/protobuf_config/src/contracts.rs new file mode 100644 index 000000000000..6521bd0bf810 --- /dev/null +++ b/core/lib/protobuf_config/src/contracts.rs @@ -0,0 +1,179 @@ +use anyhow::Context as _; +use zksync_config::configs; +use zksync_protobuf::required; + +use crate::{parse_h160, parse_h256, proto, repr::ProtoRepr}; + +impl proto::ProverAtGenesis { + fn new(x: &configs::contracts::ProverAtGenesis) -> Self { + use configs::contracts::ProverAtGenesis as From; + match x { + From::Fri => Self::Fri, + From::Old => Self::Old, + } + } + + fn parse(&self) -> configs::contracts::ProverAtGenesis { + use configs::contracts::ProverAtGenesis as To; + match self { + Self::Fri => To::Fri, + Self::Old => To::Old, + } + } +} + +impl ProtoRepr for proto::Contracts { + type Type = configs::ContractsConfig; + fn read(&self) -> anyhow::Result { + Ok(Self::Type { + governance_addr: required(&self.governance_addr) + .and_then(|x| parse_h160(x)) + .context("governance_addr")?, + mailbox_facet_addr: required(&self.mailbox_facet_addr) + .and_then(|x| parse_h160(x)) + .context("mailbox_facet_addr")?, + executor_facet_addr: required(&self.executor_facet_addr) + .and_then(|x| parse_h160(x)) + .context("executor_facet_addr")?, + admin_facet_addr: required(&self.admin_facet_addr) + .and_then(|x| parse_h160(x)) + .context("admin_facet_addr")?, + getters_facet_addr: required(&self.getters_facet_addr) + .and_then(|x| parse_h160(x)) + .context("getters_facet_addr")?, + verifier_addr: required(&self.verifier_addr) + .and_then(|x| parse_h160(x)) + .context("verifier_addr")?, + diamond_init_addr: required(&self.diamond_init_addr) + .and_then(|x| parse_h160(x)) + .context("diamond_init_addr")?, + diamond_upgrade_init_addr: required(&self.diamond_upgrade_init_addr) + .and_then(|x| parse_h160(x)) + .context("diamond_upgrade_init_addr")?, + diamond_proxy_addr: required(&self.diamond_proxy_addr) + .and_then(|x| parse_h160(x)) + .context("diamond_proxy_addr")?, + validator_timelock_addr: required(&self.validator_timelock_addr) + .and_then(|x| parse_h160(x)) + .context("validator_timelock_addr")?, + genesis_tx_hash: required(&self.genesis_tx_hash) + .and_then(|x| parse_h256(x)) + .context("genesis_tx_hash")?, + l1_erc20_bridge_proxy_addr: required(&self.l1_erc20_bridge_proxy_addr) + .and_then(|x| parse_h160(x)) + .context("l1_erc20_bridge_proxy_addr")?, + l1_erc20_bridge_impl_addr: required(&self.l1_erc20_bridge_impl_addr) + .and_then(|x| parse_h160(x)) + .context("l1_erc20_bridge_impl_addr")?, + l2_erc20_bridge_addr: required(&self.l2_erc20_bridge_addr) + .and_then(|x| parse_h160(x)) + .context("l2_erc20_bridge_addr")?, + l1_weth_bridge_proxy_addr: self + .l1_weth_bridge_proxy_addr + .as_ref() + .map(|x| parse_h160(x)) + .transpose() + .context("l1_weth_bridge_proxy_addr")?, + l2_weth_bridge_addr: self + .l2_weth_bridge_addr + .as_ref() + .map(|x| parse_h160(x)) + .transpose() + .context("l2_weth_bridge_addr")?, + l1_allow_list_addr: required(&self.l1_allow_list_addr) + .and_then(|x| parse_h160(x)) + .context("l1_allow_list_addr")?, + l2_testnet_paymaster_addr: self + .l2_testnet_paymaster_addr + .as_ref() + .map(|x| parse_h160(x)) + .transpose() + .context("l2_testnet_paymaster_addr")?, + recursion_scheduler_level_vk_hash: required(&self.recursion_scheduler_level_vk_hash) + .and_then(|x| parse_h256(x)) + .context("recursion_scheduler_level_vk_hash")?, + recursion_node_level_vk_hash: required(&self.recursion_node_level_vk_hash) + .and_then(|x| parse_h256(x)) + .context("recursion_node_level_vk_hash")?, + recursion_leaf_level_vk_hash: required(&self.recursion_leaf_level_vk_hash) + .and_then(|x| parse_h256(x)) + .context("recursion_leaf_level_vk_hash")?, + recursion_circuits_set_vks_hash: required(&self.recursion_circuits_set_vks_hash) + .and_then(|x| parse_h256(x)) + .context("recursion_circuits_set_vks_hash")?, + l1_multicall3_addr: required(&self.l1_multicall3_addr) + .and_then(|x| parse_h160(x)) + .context("l1_multicall3_addr")?, + fri_recursion_scheduler_level_vk_hash: required( + &self.fri_recursion_scheduler_level_vk_hash, + ) + .and_then(|x| parse_h256(x)) + .context("fri_recursion_scheduler_level_vk_hash")?, + fri_recursion_node_level_vk_hash: required(&self.fri_recursion_node_level_vk_hash) + .and_then(|x| parse_h256(x)) + .context("fri_recursion_node_level_vk_hash")?, + fri_recursion_leaf_level_vk_hash: required(&self.fri_recursion_leaf_level_vk_hash) + .and_then(|x| parse_h256(x)) + .context("fri_recursion_leaf_level_vk_hash")?, + prover_at_genesis: required(&self.prover_at_genesis) + .and_then(|x| Ok(proto::ProverAtGenesis::try_from(*x)?)) + .context("prover_at_genesis")? + .parse(), + snark_wrapper_vk_hash: required(&self.snark_wrapper_vk_hash) + .and_then(|x| parse_h256(x)) + .context("snark_wrapper_vk_hash")?, + }) + } + + fn build(this: &Self::Type) -> Self { + Self { + governance_addr: Some(this.governance_addr.as_bytes().into()), + mailbox_facet_addr: Some(this.mailbox_facet_addr.as_bytes().into()), + executor_facet_addr: Some(this.executor_facet_addr.as_bytes().into()), + admin_facet_addr: Some(this.admin_facet_addr.as_bytes().into()), + getters_facet_addr: Some(this.getters_facet_addr.as_bytes().into()), + verifier_addr: Some(this.verifier_addr.as_bytes().into()), + diamond_init_addr: Some(this.diamond_init_addr.as_bytes().into()), + diamond_upgrade_init_addr: Some(this.diamond_upgrade_init_addr.as_bytes().into()), + diamond_proxy_addr: Some(this.diamond_proxy_addr.as_bytes().into()), + validator_timelock_addr: Some(this.validator_timelock_addr.as_bytes().into()), + genesis_tx_hash: Some(this.genesis_tx_hash.as_bytes().into()), + l1_erc20_bridge_proxy_addr: Some(this.l1_erc20_bridge_proxy_addr.as_bytes().into()), + l1_erc20_bridge_impl_addr: Some(this.l1_erc20_bridge_impl_addr.as_bytes().into()), + l2_erc20_bridge_addr: Some(this.l2_erc20_bridge_addr.as_bytes().into()), + l1_weth_bridge_proxy_addr: this + .l1_weth_bridge_proxy_addr + .as_ref() + .map(|x| x.as_bytes().into()), + l2_weth_bridge_addr: this + .l2_weth_bridge_addr + .as_ref() + .map(|x| x.as_bytes().into()), + l1_allow_list_addr: Some(this.l1_allow_list_addr.as_bytes().into()), + l2_testnet_paymaster_addr: this + .l2_testnet_paymaster_addr + .as_ref() + .map(|x| x.as_bytes().into()), + recursion_scheduler_level_vk_hash: Some( + this.recursion_scheduler_level_vk_hash.as_bytes().into(), + ), + recursion_node_level_vk_hash: Some(this.recursion_node_level_vk_hash.as_bytes().into()), + recursion_leaf_level_vk_hash: Some(this.recursion_leaf_level_vk_hash.as_bytes().into()), + recursion_circuits_set_vks_hash: Some( + this.recursion_circuits_set_vks_hash.as_bytes().into(), + ), + l1_multicall3_addr: Some(this.l1_multicall3_addr.as_bytes().into()), + fri_recursion_scheduler_level_vk_hash: Some( + this.fri_recursion_scheduler_level_vk_hash.as_bytes().into(), + ), + fri_recursion_node_level_vk_hash: Some( + this.fri_recursion_node_level_vk_hash.as_bytes().into(), + ), + fri_recursion_leaf_level_vk_hash: Some( + this.fri_recursion_leaf_level_vk_hash.as_bytes().into(), + ), + prover_at_genesis: Some(proto::ProverAtGenesis::new(&this.prover_at_genesis).into()), + snark_wrapper_vk_hash: Some(this.snark_wrapper_vk_hash.as_bytes().into()), + } + } +} diff --git a/core/lib/protobuf_config/src/database.rs b/core/lib/protobuf_config/src/database.rs new file mode 100644 index 000000000000..90e0b2bf42e1 --- /dev/null +++ b/core/lib/protobuf_config/src/database.rs @@ -0,0 +1,107 @@ +use anyhow::Context as _; +use zksync_config::configs; +use zksync_protobuf::required; + +use crate::{ + proto, + repr::{read_required_repr, ProtoRepr}, +}; + +impl proto::MerkleTreeMode { + fn new(x: &configs::database::MerkleTreeMode) -> Self { + use configs::database::MerkleTreeMode as From; + match x { + From::Full => Self::Full, + From::Lightweight => Self::Lightweight, + } + } + + fn parse(&self) -> configs::database::MerkleTreeMode { + use configs::database::MerkleTreeMode as To; + match self { + Self::Full => To::Full, + Self::Lightweight => To::Lightweight, + } + } +} + +impl ProtoRepr for proto::MerkleTree { + type Type = configs::database::MerkleTreeConfig; + fn read(&self) -> anyhow::Result { + Ok(Self::Type { + path: required(&self.path).context("path")?.clone(), + mode: required(&self.mode) + .and_then(|x| Ok(proto::MerkleTreeMode::try_from(*x)?)) + .context("mode")? + .parse(), + multi_get_chunk_size: required(&self.multi_get_chunk_size) + .and_then(|x| Ok((*x).try_into()?)) + .context("multi_get_chunk_size")?, + block_cache_size_mb: required(&self.block_cache_size_mb) + .and_then(|x| Ok((*x).try_into()?)) + .context("block_cache_size_mb")?, + memtable_capacity_mb: required(&self.memtable_capacity_mb) + .and_then(|x| Ok((*x).try_into()?)) + .context("memtable_capacity_mb")?, + stalled_writes_timeout_sec: *required(&self.stalled_writes_timeout_sec) + .context("stalled_writes_timeout_sec")?, + max_l1_batches_per_iter: required(&self.max_l1_batches_per_iter) + .and_then(|x| Ok((*x).try_into()?)) + .context("max_l1_batches_per_iter")?, + }) + } + + fn build(this: &Self::Type) -> Self { + Self { + path: Some(this.path.clone()), + mode: Some(proto::MerkleTreeMode::new(&this.mode).into()), + multi_get_chunk_size: Some(this.multi_get_chunk_size.try_into().unwrap()), + block_cache_size_mb: Some(this.block_cache_size_mb.try_into().unwrap()), + memtable_capacity_mb: Some(this.memtable_capacity_mb.try_into().unwrap()), + stalled_writes_timeout_sec: Some(this.stalled_writes_timeout_sec), + max_l1_batches_per_iter: Some(this.max_l1_batches_per_iter.try_into().unwrap()), + } + } +} + +impl ProtoRepr for proto::Db { + type Type = configs::database::DBConfig; + fn read(&self) -> anyhow::Result { + Ok(Self::Type { + state_keeper_db_path: required(&self.state_keeper_db_path) + .context("state_keeper_db_path")? + .clone(), + merkle_tree: read_required_repr(&self.merkle_tree).context("merkle_tree")?, + }) + } + + fn build(this: &Self::Type) -> Self { + Self { + state_keeper_db_path: Some(this.state_keeper_db_path.clone()), + merkle_tree: Some(ProtoRepr::build(&this.merkle_tree)), + } + } +} + +impl ProtoRepr for proto::Postgres { + type Type = configs::database::PostgresConfig; + fn read(&self) -> anyhow::Result { + Ok(Self::Type { + master_url: self.master_url.clone(), + replica_url: self.replica_url.clone(), + prover_url: self.prover_url.clone(), + max_connections: self.max_connections, + statement_timeout_sec: self.statement_timeout_sec, + }) + } + + fn build(this: &Self::Type) -> Self { + Self { + master_url: this.master_url.clone(), + replica_url: this.replica_url.clone(), + prover_url: this.prover_url.clone(), + max_connections: this.max_connections, + statement_timeout_sec: this.statement_timeout_sec, + } + } +} diff --git a/core/lib/protobuf_config/src/eth_client.rs b/core/lib/protobuf_config/src/eth_client.rs new file mode 100644 index 000000000000..4965b2e95a67 --- /dev/null +++ b/core/lib/protobuf_config/src/eth_client.rs @@ -0,0 +1,22 @@ +use anyhow::Context as _; +use zksync_config::configs; +use zksync_protobuf::required; + +use crate::{proto, repr::ProtoRepr}; + +impl ProtoRepr for proto::EthClient { + type Type = configs::ETHClientConfig; + fn read(&self) -> anyhow::Result { + Ok(Self::Type { + chain_id: *required(&self.chain_id).context("chain_id")?, + web3_url: required(&self.web3_url).context("web3_url")?.clone(), + }) + } + + fn build(this: &Self::Type) -> Self { + Self { + chain_id: Some(this.chain_id), + web3_url: Some(this.web3_url.clone()), + } + } +} diff --git a/core/lib/protobuf_config/src/eth_sender.rs b/core/lib/protobuf_config/src/eth_sender.rs new file mode 100644 index 000000000000..6d6f430b2c71 --- /dev/null +++ b/core/lib/protobuf_config/src/eth_sender.rs @@ -0,0 +1,177 @@ +use anyhow::Context as _; +use zksync_config::configs; +use zksync_protobuf::required; + +use crate::{ + proto, + repr::{read_required_repr, ProtoRepr}, +}; + +impl proto::ProofSendingMode { + fn new(x: &configs::eth_sender::ProofSendingMode) -> Self { + use configs::eth_sender::ProofSendingMode as From; + match x { + From::OnlyRealProofs => Self::OnlyRealProofs, + From::OnlySampledProofs => Self::OnlySampledProofs, + From::SkipEveryProof => Self::SkipEveryProof, + } + } + + fn parse(&self) -> configs::eth_sender::ProofSendingMode { + use configs::eth_sender::ProofSendingMode as To; + match self { + Self::OnlyRealProofs => To::OnlyRealProofs, + Self::OnlySampledProofs => To::OnlySampledProofs, + Self::SkipEveryProof => To::SkipEveryProof, + } + } +} + +impl proto::ProofLoadingMode { + fn new(x: &configs::eth_sender::ProofLoadingMode) -> Self { + use configs::eth_sender::ProofLoadingMode as From; + match x { + From::OldProofFromDb => Self::OldProofFromDb, + From::FriProofFromGcs => Self::FriProofFromGcs, + } + } + + fn parse(&self) -> configs::eth_sender::ProofLoadingMode { + use configs::eth_sender::ProofLoadingMode as To; + match self { + Self::OldProofFromDb => To::OldProofFromDb, + Self::FriProofFromGcs => To::FriProofFromGcs, + } + } +} + +impl ProtoRepr for proto::EthSender { + type Type = configs::eth_sender::ETHSenderConfig; + fn read(&self) -> anyhow::Result { + Ok(Self::Type { + sender: read_required_repr(&self.sender).context("sender")?, + gas_adjuster: read_required_repr(&self.gas_adjuster).context("gas_adjuster")?, + }) + } + + fn build(this: &Self::Type) -> Self { + Self { + sender: Some(ProtoRepr::build(&this.sender)), + gas_adjuster: Some(ProtoRepr::build(&this.gas_adjuster)), + } + } +} + +impl ProtoRepr for proto::Sender { + type Type = configs::eth_sender::SenderConfig; + fn read(&self) -> anyhow::Result { + Ok(Self::Type { + aggregated_proof_sizes: self + .aggregated_proof_sizes + .iter() + .enumerate() + .map(|(i, x)| (*x).try_into().context(i)) + .collect::>() + .context("aggregated_proof_sizes")?, + wait_confirmations: self.wait_confirmations, + tx_poll_period: *required(&self.tx_poll_period).context("tx_poll_period")?, + aggregate_tx_poll_period: *required(&self.aggregate_tx_poll_period) + .context("aggregate_tx_poll_period")?, + max_txs_in_flight: *required(&self.max_txs_in_flight).context("max_txs_in_flight")?, + proof_sending_mode: required(&self.proof_sending_mode) + .and_then(|x| Ok(proto::ProofSendingMode::try_from(*x)?)) + .context("proof_sending_mode")? + .parse(), + max_aggregated_tx_gas: *required(&self.max_aggregated_tx_gas) + .context("max_aggregated_tx_gas")?, + max_eth_tx_data_size: required(&self.max_eth_tx_data_size) + .and_then(|x| Ok((*x).try_into()?)) + .context("max_eth_tx_data_size")?, + max_aggregated_blocks_to_commit: *required(&self.max_aggregated_blocks_to_commit) + .context("max_aggregated_blocks_to_commit")?, + max_aggregated_blocks_to_execute: *required(&self.max_aggregated_blocks_to_execute) + .context("max_aggregated_blocks_to_execute")?, + aggregated_block_commit_deadline: *required(&self.aggregated_block_commit_deadline) + .context("aggregated_block_commit_deadline")?, + aggregated_block_prove_deadline: *required(&self.aggregated_block_prove_deadline) + .context("aggregated_block_prove_deadline")?, + aggregated_block_execute_deadline: *required(&self.aggregated_block_execute_deadline) + .context("aggregated_block_execute_deadline")?, + timestamp_criteria_max_allowed_lag: required(&self.timestamp_criteria_max_allowed_lag) + .and_then(|x| Ok((*x).try_into()?)) + .context("timestamp_criteria_max_allowed_lag")?, + l1_batch_min_age_before_execute_seconds: self.l1_batch_min_age_before_execute_seconds, + max_acceptable_priority_fee_in_gwei: *required( + &self.max_acceptable_priority_fee_in_gwei, + ) + .context("max_acceptable_priority_fee_in_gwei")?, + proof_loading_mode: required(&self.proof_loading_mode) + .and_then(|x| Ok(proto::ProofLoadingMode::try_from(*x)?)) + .context("proof_loading_mode")? + .parse(), + }) + } + + fn build(this: &Self::Type) -> Self { + Self { + aggregated_proof_sizes: this + .aggregated_proof_sizes + .iter() + .map(|x| (*x).try_into().unwrap()) + .collect(), + wait_confirmations: this.wait_confirmations, + tx_poll_period: Some(this.tx_poll_period), + aggregate_tx_poll_period: Some(this.aggregate_tx_poll_period), + max_txs_in_flight: Some(this.max_txs_in_flight), + proof_sending_mode: Some(proto::ProofSendingMode::new(&this.proof_sending_mode).into()), + max_aggregated_tx_gas: Some(this.max_aggregated_tx_gas), + max_eth_tx_data_size: Some(this.max_eth_tx_data_size.try_into().unwrap()), + max_aggregated_blocks_to_commit: Some(this.max_aggregated_blocks_to_commit), + max_aggregated_blocks_to_execute: Some(this.max_aggregated_blocks_to_execute), + aggregated_block_commit_deadline: Some(this.aggregated_block_commit_deadline), + aggregated_block_prove_deadline: Some(this.aggregated_block_prove_deadline), + aggregated_block_execute_deadline: Some(this.aggregated_block_execute_deadline), + timestamp_criteria_max_allowed_lag: Some( + this.timestamp_criteria_max_allowed_lag.try_into().unwrap(), + ), + l1_batch_min_age_before_execute_seconds: this.l1_batch_min_age_before_execute_seconds, + max_acceptable_priority_fee_in_gwei: Some(this.max_acceptable_priority_fee_in_gwei), + proof_loading_mode: Some(proto::ProofLoadingMode::new(&this.proof_loading_mode).into()), + } + } +} + +impl ProtoRepr for proto::GasAdjuster { + type Type = configs::eth_sender::GasAdjusterConfig; + fn read(&self) -> anyhow::Result { + Ok(Self::Type { + default_priority_fee_per_gas: *required(&self.default_priority_fee_per_gas) + .context("default_priority_fee_per_gas")?, + max_base_fee_samples: required(&self.max_base_fee_samples) + .and_then(|x| Ok((*x).try_into()?)) + .context("max_base_fee_samples")?, + pricing_formula_parameter_a: *required(&self.pricing_formula_parameter_a) + .context("pricing_formula_parameter_a")?, + pricing_formula_parameter_b: *required(&self.pricing_formula_parameter_b) + .context("pricing_formula_parameter_b")?, + internal_l1_pricing_multiplier: *required(&self.internal_l1_pricing_multiplier) + .context("internal_l1_pricing_multiplier")?, + internal_enforced_l1_gas_price: self.internal_enforced_l1_gas_price, + poll_period: *required(&self.poll_period).context("poll_period")?, + max_l1_gas_price: self.max_l1_gas_price, + }) + } + + fn build(this: &Self::Type) -> Self { + Self { + default_priority_fee_per_gas: Some(this.default_priority_fee_per_gas), + max_base_fee_samples: Some(this.max_base_fee_samples.try_into().unwrap()), + pricing_formula_parameter_a: Some(this.pricing_formula_parameter_a), + pricing_formula_parameter_b: Some(this.pricing_formula_parameter_b), + internal_l1_pricing_multiplier: Some(this.internal_l1_pricing_multiplier), + internal_enforced_l1_gas_price: this.internal_enforced_l1_gas_price, + poll_period: Some(this.poll_period), + max_l1_gas_price: this.max_l1_gas_price, + } + } +} diff --git a/core/lib/protobuf_config/src/eth_watch.rs b/core/lib/protobuf_config/src/eth_watch.rs new file mode 100644 index 000000000000..42c1d20f5220 --- /dev/null +++ b/core/lib/protobuf_config/src/eth_watch.rs @@ -0,0 +1,23 @@ +use anyhow::Context as _; +use zksync_config::configs; +use zksync_protobuf::required; + +use crate::{proto, repr::ProtoRepr}; + +impl ProtoRepr for proto::EthWatch { + type Type = configs::ETHWatchConfig; + fn read(&self) -> anyhow::Result { + Ok(Self::Type { + confirmations_for_eth_event: self.confirmations_for_eth_event, + eth_node_poll_interval: *required(&self.eth_node_poll_interval) + .context("eth_node_poll_interval")?, + }) + } + + fn build(this: &Self::Type) -> Self { + Self { + confirmations_for_eth_event: this.confirmations_for_eth_event, + eth_node_poll_interval: Some(this.eth_node_poll_interval), + } + } +} diff --git a/core/lib/protobuf_config/src/fri_proof_compressor.rs b/core/lib/protobuf_config/src/fri_proof_compressor.rs new file mode 100644 index 000000000000..5f1753c9dfa2 --- /dev/null +++ b/core/lib/protobuf_config/src/fri_proof_compressor.rs @@ -0,0 +1,49 @@ +use anyhow::Context as _; +use zksync_config::configs; +use zksync_protobuf::required; + +use crate::{proto, repr::ProtoRepr}; + +impl ProtoRepr for proto::FriProofCompressor { + type Type = configs::FriProofCompressorConfig; + fn read(&self) -> anyhow::Result { + Ok(Self::Type { + compression_mode: required(&self.compression_mode) + .and_then(|x| Ok((*x).try_into()?)) + .context("compression_mode")?, + prometheus_listener_port: required(&self.prometheus_listener_port) + .and_then(|x| Ok((*x).try_into()?)) + .context("prometheus_listener_port")?, + prometheus_pushgateway_url: required(&self.prometheus_pushgateway_url) + .context("prometheus_pushgateway_url")? + .clone(), + prometheus_push_interval_ms: self.prometheus_push_interval_ms, + generation_timeout_in_secs: required(&self.generation_timeout_in_secs) + .and_then(|x| Ok((*x).try_into()?)) + .context("generation_timeout_in_secs")?, + max_attempts: *required(&self.max_attempts).context("max_attempts")?, + universal_setup_path: required(&self.universal_setup_path) + .context("universal_setup_path")? + .clone(), + universal_setup_download_url: required(&self.universal_setup_download_url) + .context("universal_setup_download_url")? + .clone(), + verify_wrapper_proof: *required(&self.verify_wrapper_proof) + .context("verify_wrapper_proof")?, + }) + } + + fn build(this: &Self::Type) -> Self { + Self { + compression_mode: Some(this.compression_mode.into()), + prometheus_listener_port: Some(this.prometheus_listener_port.into()), + prometheus_pushgateway_url: Some(this.prometheus_pushgateway_url.clone()), + prometheus_push_interval_ms: this.prometheus_push_interval_ms, + generation_timeout_in_secs: Some(this.generation_timeout_in_secs.into()), + max_attempts: Some(this.max_attempts), + universal_setup_path: Some(this.universal_setup_path.clone()), + universal_setup_download_url: Some(this.universal_setup_download_url.clone()), + verify_wrapper_proof: Some(this.verify_wrapper_proof), + } + } +} diff --git a/core/lib/protobuf_config/src/fri_prover.rs b/core/lib/protobuf_config/src/fri_prover.rs new file mode 100644 index 000000000000..bcd2d6a9d99b --- /dev/null +++ b/core/lib/protobuf_config/src/fri_prover.rs @@ -0,0 +1,98 @@ +use anyhow::Context as _; +use zksync_config::configs; +use zksync_protobuf::required; + +use crate::{proto, repr::ProtoRepr}; + +impl proto::SetupLoadMode { + fn new(x: &configs::fri_prover::SetupLoadMode) -> Self { + use configs::fri_prover::SetupLoadMode as From; + match x { + From::FromDisk => Self::FromDisk, + From::FromMemory => Self::FromMemory, + } + } + + fn parse(&self) -> configs::fri_prover::SetupLoadMode { + use configs::fri_prover::SetupLoadMode as To; + match self { + Self::FromDisk => To::FromDisk, + Self::FromMemory => To::FromMemory, + } + } +} + +impl ProtoRepr for proto::FriProver { + type Type = configs::FriProverConfig; + fn read(&self) -> anyhow::Result { + Ok(Self::Type { + setup_data_path: required(&self.setup_data_path) + .context("setup_data_path")? + .clone(), + prometheus_port: required(&self.prometheus_port) + .and_then(|x| Ok((*x).try_into()?)) + .context("prometheus_port")?, + max_attempts: *required(&self.max_attempts).context("max_attempts")?, + generation_timeout_in_secs: required(&self.generation_timeout_in_secs) + .and_then(|x| Ok((*x).try_into()?)) + .context("generation_timeout_in_secs")?, + base_layer_circuit_ids_to_be_verified: required( + &self.base_layer_circuit_ids_to_be_verified, + ) + .context("base_layer_circuit_ids_to_be_verified")? + .clone(), + recursive_layer_circuit_ids_to_be_verified: required( + &self.recursive_layer_circuit_ids_to_be_verified, + ) + .context("recursive_layer_circuit_ids_to_be_verified")? + .clone(), + setup_load_mode: required(&self.setup_load_mode) + .and_then(|x| Ok(proto::SetupLoadMode::try_from(*x)?)) + .context("setup_load_mode")? + .parse(), + specialized_group_id: required(&self.specialized_group_id) + .and_then(|x| Ok((*x).try_into()?)) + .context("specialized_group_id")?, + witness_vector_generator_thread_count: self + .witness_vector_generator_thread_count + .map(|x| x.try_into()) + .transpose() + .context("witness_vector_generator_thread_count")?, + queue_capacity: required(&self.queue_capacity) + .and_then(|x| Ok((*x).try_into()?)) + .context("queue_capacity")?, + witness_vector_receiver_port: required(&self.witness_vector_receiver_port) + .and_then(|x| Ok((*x).try_into()?)) + .context("witness_vector_receiver_port")?, + zone_read_url: required(&self.zone_read_url) + .context("zone_read_url")? + .clone(), + shall_save_to_public_bucket: *required(&self.shall_save_to_public_bucket) + .context("shall_save_to_public_bucket")?, + }) + } + + fn build(this: &Self::Type) -> Self { + Self { + setup_data_path: Some(this.setup_data_path.clone()), + prometheus_port: Some(this.prometheus_port.into()), + max_attempts: Some(this.max_attempts), + generation_timeout_in_secs: Some(this.generation_timeout_in_secs.into()), + base_layer_circuit_ids_to_be_verified: Some( + this.base_layer_circuit_ids_to_be_verified.clone(), + ), + recursive_layer_circuit_ids_to_be_verified: Some( + this.recursive_layer_circuit_ids_to_be_verified.clone(), + ), + setup_load_mode: Some(proto::SetupLoadMode::new(&this.setup_load_mode).into()), + specialized_group_id: Some(this.specialized_group_id.into()), + witness_vector_generator_thread_count: this + .witness_vector_generator_thread_count + .map(|x| x.try_into().unwrap()), + queue_capacity: Some(this.queue_capacity.try_into().unwrap()), + witness_vector_receiver_port: Some(this.witness_vector_receiver_port.into()), + zone_read_url: Some(this.zone_read_url.clone()), + shall_save_to_public_bucket: Some(this.shall_save_to_public_bucket), + } + } +} diff --git a/core/lib/protobuf_config/src/fri_prover_gateway.rs b/core/lib/protobuf_config/src/fri_prover_gateway.rs new file mode 100644 index 000000000000..5d791635762d --- /dev/null +++ b/core/lib/protobuf_config/src/fri_prover_gateway.rs @@ -0,0 +1,34 @@ +use anyhow::Context as _; +use zksync_config::configs; +use zksync_protobuf::required; + +use crate::{proto, repr::ProtoRepr}; + +impl ProtoRepr for proto::FriProverGateway { + type Type = configs::FriProverGatewayConfig; + fn read(&self) -> anyhow::Result { + Ok(Self::Type { + api_url: required(&self.api_url).context("api_url")?.clone(), + api_poll_duration_secs: required(&self.api_poll_duration_secs) + .and_then(|x| Ok((*x).try_into()?)) + .context("api_poll_duration_secs")?, + prometheus_listener_port: required(&self.prometheus_listener_port) + .and_then(|x| Ok((*x).try_into()?)) + .context("prometheus_listener_port")?, + prometheus_pushgateway_url: required(&self.prometheus_pushgateway_url) + .context("prometheus_pushgateway_url")? + .clone(), + prometheus_push_interval_ms: self.prometheus_push_interval_ms, + }) + } + + fn build(this: &Self::Type) -> Self { + Self { + api_url: Some(this.api_url.clone()), + api_poll_duration_secs: Some(this.api_poll_duration_secs.into()), + prometheus_listener_port: Some(this.prometheus_listener_port.into()), + prometheus_pushgateway_url: Some(this.prometheus_pushgateway_url.clone()), + prometheus_push_interval_ms: this.prometheus_push_interval_ms, + } + } +} diff --git a/core/lib/protobuf_config/src/fri_prover_group.rs b/core/lib/protobuf_config/src/fri_prover_group.rs new file mode 100644 index 000000000000..9c8454657612 --- /dev/null +++ b/core/lib/protobuf_config/src/fri_prover_group.rs @@ -0,0 +1,81 @@ +use std::collections::HashSet; + +use anyhow::Context as _; +use zksync_basic_types::basic_fri_types::CircuitIdRoundTuple; +use zksync_config::configs; +use zksync_protobuf::required; + +use crate::{proto, repr::ProtoRepr}; + +impl ProtoRepr for proto::CircuitIdRoundTuple { + type Type = CircuitIdRoundTuple; + fn read(&self) -> anyhow::Result { + Ok(Self::Type { + circuit_id: required(&self.circuit_id) + .and_then(|x| Ok((*x).try_into()?)) + .context("circuit_id")?, + aggregation_round: required(&self.aggregation_round) + .and_then(|x| Ok((*x).try_into()?)) + .context("aggregation_round")?, + }) + } + + fn build(this: &Self::Type) -> Self { + Self { + circuit_id: Some(this.circuit_id.into()), + aggregation_round: Some(this.aggregation_round.into()), + } + } +} + +fn read_vec(v: &[proto::CircuitIdRoundTuple]) -> anyhow::Result> { + v.iter() + .enumerate() + .map(|(i, x)| x.read().context(i)) + .collect() +} + +fn build_vec(v: &HashSet) -> Vec { + let mut v: Vec<_> = v.iter().cloned().collect(); + v.sort(); + v.iter().map(ProtoRepr::build).collect() +} + +impl ProtoRepr for proto::FriProverGroup { + type Type = configs::fri_prover_group::FriProverGroupConfig; + fn read(&self) -> anyhow::Result { + Ok(Self::Type { + group_0: read_vec(&self.group_0).context("group_0")?, + group_1: read_vec(&self.group_1).context("group_1")?, + group_2: read_vec(&self.group_2).context("group_2")?, + group_3: read_vec(&self.group_3).context("group_3")?, + group_4: read_vec(&self.group_4).context("group_4")?, + group_5: read_vec(&self.group_5).context("group_5")?, + group_6: read_vec(&self.group_6).context("group_6")?, + group_7: read_vec(&self.group_7).context("group_7")?, + group_8: read_vec(&self.group_8).context("group_8")?, + group_9: read_vec(&self.group_9).context("group_9")?, + group_10: read_vec(&self.group_10).context("group_10")?, + group_11: read_vec(&self.group_11).context("group_11")?, + group_12: read_vec(&self.group_12).context("group_12")?, + }) + } + + fn build(this: &Self::Type) -> Self { + Self { + group_0: build_vec(&this.group_0), + group_1: build_vec(&this.group_1), + group_2: build_vec(&this.group_2), + group_3: build_vec(&this.group_3), + group_4: build_vec(&this.group_4), + group_5: build_vec(&this.group_5), + group_6: build_vec(&this.group_6), + group_7: build_vec(&this.group_7), + group_8: build_vec(&this.group_8), + group_9: build_vec(&this.group_9), + group_10: build_vec(&this.group_10), + group_11: build_vec(&this.group_11), + group_12: build_vec(&this.group_12), + } + } +} diff --git a/core/lib/protobuf_config/src/fri_witness_generator.rs b/core/lib/protobuf_config/src/fri_witness_generator.rs new file mode 100644 index 000000000000..fef20dcc3892 --- /dev/null +++ b/core/lib/protobuf_config/src/fri_witness_generator.rs @@ -0,0 +1,39 @@ +use anyhow::Context as _; +use zksync_config::configs; +use zksync_protobuf::required; + +use crate::{proto, repr::ProtoRepr}; + +impl ProtoRepr for proto::FriWitnessGenerator { + type Type = configs::FriWitnessGeneratorConfig; + fn read(&self) -> anyhow::Result { + Ok(Self::Type { + generation_timeout_in_secs: required(&self.generation_timeout_in_secs) + .and_then(|x| Ok((*x).try_into()?)) + .context("generation_timeout_in_secs")?, + max_attempts: *required(&self.max_attempts).context("max_attempts")?, + blocks_proving_percentage: self + .blocks_proving_percentage + .map(|x| x.try_into()) + .transpose() + .context("blocks_proving_percentage")?, + dump_arguments_for_blocks: self.dump_arguments_for_blocks.clone(), + last_l1_batch_to_process: self.last_l1_batch_to_process, + force_process_block: self.force_process_block, + shall_save_to_public_bucket: *required(&self.shall_save_to_public_bucket) + .context("shall_save_to_public_bucket")?, + }) + } + + fn build(this: &Self::Type) -> Self { + Self { + generation_timeout_in_secs: Some(this.generation_timeout_in_secs.into()), + max_attempts: Some(this.max_attempts), + blocks_proving_percentage: this.blocks_proving_percentage.map(|x| x.into()), + dump_arguments_for_blocks: this.dump_arguments_for_blocks.clone(), + last_l1_batch_to_process: this.last_l1_batch_to_process, + force_process_block: this.force_process_block, + shall_save_to_public_bucket: Some(this.shall_save_to_public_bucket), + } + } +} diff --git a/core/lib/protobuf_config/src/fri_witness_vector_generator.rs b/core/lib/protobuf_config/src/fri_witness_vector_generator.rs new file mode 100644 index 000000000000..9c2b432a80a5 --- /dev/null +++ b/core/lib/protobuf_config/src/fri_witness_vector_generator.rs @@ -0,0 +1,56 @@ +use anyhow::Context as _; +use zksync_config::configs; +use zksync_protobuf::required; + +use crate::{proto, repr::ProtoRepr}; + +impl ProtoRepr for proto::FriWitnessVectorGenerator { + type Type = configs::FriWitnessVectorGeneratorConfig; + fn read(&self) -> anyhow::Result { + Ok(Self::Type { + max_prover_reservation_duration_in_secs: required( + &self.max_prover_reservation_duration_in_secs, + ) + .and_then(|x| Ok((*x).try_into()?)) + .context("max_prover_reservation_duration_in_secs")?, + prover_instance_wait_timeout_in_secs: required( + &self.prover_instance_wait_timeout_in_secs, + ) + .and_then(|x| Ok((*x).try_into()?)) + .context("prover_instance_wait_timeout_in_secs")?, + prover_instance_poll_time_in_milli_secs: required( + &self.prover_instance_poll_time_in_milli_secs, + ) + .and_then(|x| Ok((*x).try_into()?)) + .context("prover_instance_poll_time_in_milli_secs")?, + prometheus_listener_port: required(&self.prometheus_listener_port) + .and_then(|x| Ok((*x).try_into()?)) + .context("prometheus_listener_port")?, + prometheus_pushgateway_url: required(&self.prometheus_pushgateway_url) + .context("prometheus_pushgateway_url")? + .clone(), + prometheus_push_interval_ms: self.prometheus_push_interval_ms, + specialized_group_id: required(&self.specialized_group_id) + .and_then(|x| Ok((*x).try_into()?)) + .context("specialized_group_id")?, + }) + } + + fn build(this: &Self::Type) -> Self { + Self { + max_prover_reservation_duration_in_secs: Some( + this.max_prover_reservation_duration_in_secs.into(), + ), + prover_instance_wait_timeout_in_secs: Some( + this.prover_instance_wait_timeout_in_secs.into(), + ), + prover_instance_poll_time_in_milli_secs: Some( + this.prover_instance_poll_time_in_milli_secs.into(), + ), + prometheus_listener_port: Some(this.prometheus_listener_port.into()), + prometheus_pushgateway_url: Some(this.prometheus_pushgateway_url.clone()), + prometheus_push_interval_ms: this.prometheus_push_interval_ms, + specialized_group_id: Some(this.specialized_group_id.into()), + } + } +} diff --git a/core/lib/protobuf_config/src/house_keeper.rs b/core/lib/protobuf_config/src/house_keeper.rs new file mode 100644 index 000000000000..00843db3a7cd --- /dev/null +++ b/core/lib/protobuf_config/src/house_keeper.rs @@ -0,0 +1,87 @@ +use anyhow::Context as _; +use zksync_config::configs; +use zksync_protobuf::required; + +use crate::{proto, repr::ProtoRepr}; + +impl ProtoRepr for proto::HouseKeeper { + type Type = configs::house_keeper::HouseKeeperConfig; + fn read(&self) -> anyhow::Result { + Ok(Self::Type { + l1_batch_metrics_reporting_interval_ms: *required( + &self.l1_batch_metrics_reporting_interval_ms, + ) + .context("l1_batch_metrics_reporting_interval_ms")?, + gpu_prover_queue_reporting_interval_ms: *required( + &self.gpu_prover_queue_reporting_interval_ms, + ) + .context("gpu_prover_queue_reporting_interval_ms")?, + prover_job_retrying_interval_ms: *required(&self.prover_job_retrying_interval_ms) + .context("prover_job_retrying_interval_ms")?, + prover_stats_reporting_interval_ms: *required(&self.prover_stats_reporting_interval_ms) + .context("prover_stats_reporting_interval_ms")?, + witness_job_moving_interval_ms: *required(&self.witness_job_moving_interval_ms) + .context("witness_job_moving_interval_ms")?, + witness_generator_stats_reporting_interval_ms: *required( + &self.witness_generator_stats_reporting_interval_ms, + ) + .context("witness_generator_stats_reporting_interval_ms")?, + fri_witness_job_moving_interval_ms: *required(&self.fri_witness_job_moving_interval_ms) + .context("fri_witness_job_moving_interval_ms")?, + fri_prover_job_retrying_interval_ms: *required( + &self.fri_prover_job_retrying_interval_ms, + ) + .context("fri_prover_job_retrying_interval_ms")?, + fri_witness_generator_job_retrying_interval_ms: *required( + &self.fri_witness_generator_job_retrying_interval_ms, + ) + .context("fri_witness_generator_job_retrying_interval_ms")?, + prover_db_pool_size: *required(&self.prover_db_pool_size) + .context("prover_db_pool_size")?, + fri_prover_stats_reporting_interval_ms: *required( + &self.fri_prover_stats_reporting_interval_ms, + ) + .context("fri_prover_stats_reporting_interval_ms")?, + fri_proof_compressor_job_retrying_interval_ms: *required( + &self.fri_proof_compressor_job_retrying_interval_ms, + ) + .context("fri_proof_compressor_job_retrying_interval_ms")?, + fri_proof_compressor_stats_reporting_interval_ms: *required( + &self.fri_proof_compressor_stats_reporting_interval_ms, + ) + .context("fri_proof_compressor_stats_reporting_interval_ms")?, + }) + } + + fn build(this: &Self::Type) -> Self { + Self { + l1_batch_metrics_reporting_interval_ms: Some( + this.l1_batch_metrics_reporting_interval_ms, + ), + gpu_prover_queue_reporting_interval_ms: Some( + this.gpu_prover_queue_reporting_interval_ms, + ), + prover_job_retrying_interval_ms: Some(this.prover_job_retrying_interval_ms), + prover_stats_reporting_interval_ms: Some(this.prover_stats_reporting_interval_ms), + witness_job_moving_interval_ms: Some(this.witness_job_moving_interval_ms), + witness_generator_stats_reporting_interval_ms: Some( + this.witness_generator_stats_reporting_interval_ms, + ), + fri_witness_job_moving_interval_ms: Some(this.fri_witness_job_moving_interval_ms), + fri_prover_job_retrying_interval_ms: Some(this.fri_prover_job_retrying_interval_ms), + fri_witness_generator_job_retrying_interval_ms: Some( + this.fri_witness_generator_job_retrying_interval_ms, + ), + prover_db_pool_size: Some(this.prover_db_pool_size), + fri_prover_stats_reporting_interval_ms: Some( + this.fri_prover_stats_reporting_interval_ms, + ), + fri_proof_compressor_job_retrying_interval_ms: Some( + this.fri_proof_compressor_job_retrying_interval_ms, + ), + fri_proof_compressor_stats_reporting_interval_ms: Some( + this.fri_proof_compressor_stats_reporting_interval_ms, + ), + } + } +} diff --git a/core/lib/protobuf_config/src/lib.rs b/core/lib/protobuf_config/src/lib.rs new file mode 100644 index 000000000000..164fbfa7c55f --- /dev/null +++ b/core/lib/protobuf_config/src/lib.rs @@ -0,0 +1,43 @@ +//! Defined protobuf mapping for the config files. +//! It allows to encode the configs using: +//! * protobuf binary format +//! * protobuf text format +//! * protobuf json format + +mod alerts; +mod api; +mod chain; +mod contract_verifier; +mod contracts; +mod database; +mod eth_client; +mod eth_sender; +mod eth_watch; +mod fri_proof_compressor; +mod fri_prover; +mod fri_prover_gateway; +mod fri_prover_group; +mod fri_witness_generator; +mod fri_witness_vector_generator; +mod house_keeper; +mod object_store; +mod proof_data_handler; +mod snapshots_creator; +mod witness_generator; + +pub mod proto; +mod repr; +#[cfg(test)] +mod tests; +mod utils; + +use anyhow::Context as _; +use zksync_types::{H160, H256}; + +fn parse_h256(bytes: &[u8]) -> anyhow::Result { + Ok(<[u8; 32]>::try_from(bytes).context("invalid size")?.into()) +} + +fn parse_h160(bytes: &[u8]) -> anyhow::Result { + Ok(<[u8; 20]>::try_from(bytes).context("invalid size")?.into()) +} diff --git a/core/lib/protobuf_config/src/object_store.rs b/core/lib/protobuf_config/src/object_store.rs new file mode 100644 index 000000000000..b845007caa67 --- /dev/null +++ b/core/lib/protobuf_config/src/object_store.rs @@ -0,0 +1,60 @@ +use anyhow::Context as _; +use zksync_config::configs; +use zksync_protobuf::required; + +use crate::{proto, repr::ProtoRepr}; + +impl proto::ObjectStoreMode { + fn new(x: &configs::object_store::ObjectStoreMode) -> Self { + type From = configs::object_store::ObjectStoreMode; + match x { + From::GCS => Self::Gcs, + From::GCSWithCredentialFile => Self::GcsWithCredentialFile, + From::FileBacked => Self::FileBacked, + From::GCSAnonymousReadOnly => Self::GcsAnonymousReadOnly, + } + } + fn parse(&self) -> configs::object_store::ObjectStoreMode { + type To = configs::object_store::ObjectStoreMode; + match self { + Self::Gcs => To::GCS, + Self::GcsWithCredentialFile => To::GCSWithCredentialFile, + Self::FileBacked => To::FileBacked, + Self::GcsAnonymousReadOnly => To::GCSAnonymousReadOnly, + } + } +} + +impl ProtoRepr for proto::ObjectStore { + type Type = configs::ObjectStoreConfig; + fn read(&self) -> anyhow::Result { + Ok(Self::Type { + bucket_base_url: required(&self.bucket_base_url) + .context("bucket_base_url")? + .clone(), + mode: required(&self.mode) + .and_then(|x| Ok(proto::ObjectStoreMode::try_from(*x)?)) + .context("mode")? + .parse(), + file_backed_base_path: required(&self.file_backed_base_path) + .context("file_backed_base_path")? + .clone(), + gcs_credential_file_path: required(&self.gcs_credential_file_path) + .context("gcs_credential_file_path")? + .clone(), + max_retries: required(&self.max_retries) + .and_then(|x| Ok((*x).try_into()?)) + .context("max_retries")?, + }) + } + + fn build(this: &Self::Type) -> Self { + Self { + bucket_base_url: Some(this.bucket_base_url.clone()), + mode: Some(proto::ObjectStoreMode::new(&this.mode).into()), + file_backed_base_path: Some(this.file_backed_base_path.clone()), + gcs_credential_file_path: Some(this.gcs_credential_file_path.clone()), + max_retries: Some(this.max_retries.into()), + } + } +} diff --git a/core/lib/protobuf_config/src/proof_data_handler.rs b/core/lib/protobuf_config/src/proof_data_handler.rs new file mode 100644 index 000000000000..38712eccf4ff --- /dev/null +++ b/core/lib/protobuf_config/src/proof_data_handler.rs @@ -0,0 +1,54 @@ +use anyhow::Context as _; +use zksync_config::configs; +use zksync_protobuf::required; + +use crate::{proto, repr::ProtoRepr}; + +impl proto::ProtocolVersionLoadingMode { + fn new(x: &configs::proof_data_handler::ProtocolVersionLoadingMode) -> Self { + type From = configs::proof_data_handler::ProtocolVersionLoadingMode; + match x { + From::FromDb => Self::FromDb, + From::FromEnvVar => Self::FromEnvVar, + } + } + fn parse(&self) -> configs::proof_data_handler::ProtocolVersionLoadingMode { + type To = configs::proof_data_handler::ProtocolVersionLoadingMode; + match self { + Self::FromDb => To::FromDb, + Self::FromEnvVar => To::FromEnvVar, + } + } +} + +impl ProtoRepr for proto::ProofDataHandler { + type Type = configs::ProofDataHandlerConfig; + fn read(&self) -> anyhow::Result { + Ok(Self::Type { + http_port: required(&self.http_port) + .and_then(|x| Ok((*x).try_into()?)) + .context("http_port")?, + proof_generation_timeout_in_secs: required(&self.proof_generation_timeout_in_secs) + .and_then(|x| Ok((*x).try_into()?)) + .context("proof_generation_timeout_in_secs")?, + protocol_version_loading_mode: required(&self.protocol_version_loading_mode) + .and_then(|x| Ok(proto::ProtocolVersionLoadingMode::try_from(*x)?)) + .context("protocol_version_loading_mode")? + .parse(), + fri_protocol_version_id: required(&self.fri_protocol_version_id) + .and_then(|x| Ok((*x).try_into()?)) + .context("fri_protocol_version_id")?, + }) + } + + fn build(this: &Self::Type) -> Self { + Self { + http_port: Some(this.http_port.into()), + proof_generation_timeout_in_secs: Some(this.proof_generation_timeout_in_secs.into()), + protocol_version_loading_mode: Some( + proto::ProtocolVersionLoadingMode::new(&this.protocol_version_loading_mode).into(), + ), + fri_protocol_version_id: Some(this.fri_protocol_version_id.into()), + } + } +} diff --git a/core/lib/protobuf_config/src/proto/alerts.proto b/core/lib/protobuf_config/src/proto/alerts.proto new file mode 100644 index 000000000000..6d9905b79f78 --- /dev/null +++ b/core/lib/protobuf_config/src/proto/alerts.proto @@ -0,0 +1,7 @@ +syntax = "proto3"; + +package zksync.config; + +message Alerts { + repeated string sporadic_crypto_errors_substrs = 1; +} diff --git a/core/lib/protobuf_config/src/proto/api.proto b/core/lib/protobuf_config/src/proto/api.proto new file mode 100644 index 000000000000..ce13e45d3656 --- /dev/null +++ b/core/lib/protobuf_config/src/proto/api.proto @@ -0,0 +1,59 @@ +syntax = "proto3"; + +package zksync.config; + +import "zksync/config/utils.proto"; + +message PrivateKeys { + repeated bytes keys = 1; // H256 +} + +message Web3JsonRpc { + optional uint32 http_port = 1; // required; u16 + optional string http_url = 2; // required + optional uint32 ws_port = 3; // required; u16 + optional string ws_url = 4; // required + optional uint32 req_entities_limit = 5; // optional + optional uint32 filters_limit = 6; // optional + optional uint32 subscriptions_limit = 7; // optional + optional uint64 pubsub_polling_interval = 8; // optional + optional uint32 max_nonce_ahead = 9; // required + optional double gas_price_scale_factor = 10; // required + optional uint64 request_timeout = 11; // seconds + optional PrivateKeys account_pks = 12; // optional + optional double estimate_gas_scale_factor = 13; // required + optional uint32 estimate_gas_acceptable_overestimation = 14; // required + optional bool l1_to_l2_transactions_compatibility_mode = 15; // required + optional uint64 max_tx_size = 16; // required; B + optional uint64 vm_execution_cache_misses_limit = 17; // optional + optional uint64 vm_concurrency_limit = 18; // optional + optional uint64 factory_deps_cache_size_mb = 19; // optional; MB + optional uint64 initial_writes_cache_size_mb = 20; // optional; MB + optional uint64 latest_values_cache_size_mb = 21; // optional; MB + optional uint64 fee_history_limit = 22; // optional + optional uint64 max_batch_request_size = 23; // optional + optional uint64 max_response_body_size_mb = 24; // optional; MB + optional uint32 websocket_requests_per_minute_limit = 25; // optional + optional string tree_api_url = 26; // optional +} + +message ContractVerificationApi { + optional uint32 port = 1; // required; u16 + optional string url = 2; // required +} + +message HealthCheck { + optional uint32 port = 1; // required; u16 +} + +message MerkleTreeApi { + optional uint32 port = 1; // required; u16 +} + +message Api { + optional Web3JsonRpc web3_json_rpc = 1; // required + optional ContractVerificationApi contract_verification = 2; // required + optional Prometheus prometheus = 3; // required + optional HealthCheck healthcheck = 4; // required + optional MerkleTreeApi merkle_tree = 5; // required +} diff --git a/core/lib/protobuf_config/src/proto/chain.proto b/core/lib/protobuf_config/src/proto/chain.proto new file mode 100644 index 000000000000..8e08d47271b5 --- /dev/null +++ b/core/lib/protobuf_config/src/proto/chain.proto @@ -0,0 +1,76 @@ +syntax = "proto3"; + +package zksync.config; + +enum Network { + UNKNOWN = 0; + MAINNET = 1; + RINKEBY = 2; + ROPSTEN = 3; + GOERLI = 4; + SEPOLIA = 5; + LOCALHOST = 6; + TEST = 7; +} + +enum FeeModelVersion { + V1 = 0; + V2 = 1; +} + +message EthNetwork { + optional Network network = 1; // required + optional string zksync_network = 2; // required + optional uint64 zksync_network_id = 3; // required; L2ChainId +} + +message StateKeeper { + optional uint64 transaction_slots = 1; // required + optional uint64 block_commit_deadline_ms = 2; // required; ms + optional uint64 miniblock_commit_deadline_ms = 3; // required; ms + optional uint64 miniblock_seal_queue_capacity = 4; // required + optional uint32 max_single_tx_gas = 5; // required; gwei? + optional uint32 max_allowed_l2_tx_gas_limit = 6; // required; wei? + optional double reject_tx_at_geometry_percentage = 7; // required; % + optional double reject_tx_at_eth_params_percentage = 8; // required; % + optional double reject_tx_at_gas_percentage = 9; // required; % + optional double close_block_at_geometry_percentage = 10; // required; % + optional double close_block_at_eth_params_percentage = 11; // required; % + optional double close_block_at_gas_percentage = 12; // required; % + optional bytes fee_account_addr = 13; // required; H160 + optional uint64 minimal_l2_gas_price = 14; // required; wei? + optional double compute_overhead_part = 15; // required; [0,1] + optional double pubdata_overhead_part = 16; // required; [0,1] + optional uint64 batch_overhead_l1_gas = 17; // required; wei? + optional uint64 max_gas_per_batch = 18; // required; wei? + optional uint64 max_pubdata_per_batch = 19; // required; bytes? + optional FeeModelVersion fee_model_version = 20; // required + optional uint32 validation_computational_gas_limit = 21; // required; wei? + optional bool save_call_traces = 22; // required + optional uint32 virtual_blocks_interval = 23; // required + optional uint32 virtual_blocks_per_miniblock = 24; // required + optional bool upload_witness_inputs_to_gcs = 25; // required + optional uint64 enum_index_migration_chunk_size = 26; // optional +} + +message OperationsManager { + optional uint64 delay_interval = 1; // required; ms +} + +message Mempool { + optional uint64 sync_interval_ms = 1; // required; ms + optional uint64 sync_batch_size = 2; // required; ? + optional uint64 capacity = 3; // required; ? + optional uint64 stuck_tx_timeout = 4; // required; s + optional bool remove_stuck_txs = 5; // required + optional uint64 delay_interval = 6; // required; ms +} + +message CircuitBreaker { + optional uint64 sync_interval_ms = 1; // required; ms + optional uint64 http_req_max_retry_number = 2; // required + optional uint32 http_req_retry_interval_sec = 3; // required; s + optional uint32 replication_lag_limit_sec = 4; // optional; s +} + + diff --git a/core/lib/protobuf_config/src/proto/contract_verifier.proto b/core/lib/protobuf_config/src/proto/contract_verifier.proto new file mode 100644 index 000000000000..f76ba2b12274 --- /dev/null +++ b/core/lib/protobuf_config/src/proto/contract_verifier.proto @@ -0,0 +1,9 @@ +syntax = "proto3"; + +package zksync.config; + +message ContractVerifier { + optional uint64 compilation_timeout = 1; // required; s + optional uint64 polling_interval = 2; // optional; ms + optional uint32 prometheus_port = 3; // required; u16 +} diff --git a/core/lib/protobuf_config/src/proto/contracts.proto b/core/lib/protobuf_config/src/proto/contracts.proto new file mode 100644 index 000000000000..1acda022cd90 --- /dev/null +++ b/core/lib/protobuf_config/src/proto/contracts.proto @@ -0,0 +1,39 @@ +syntax = "proto3"; + +package zksync.config; + +enum ProverAtGenesis { + FRI = 0; + OLD = 1; +} + +message Contracts { + optional bytes governance_addr = 1; // required; H160 + optional bytes mailbox_facet_addr = 2; // required; H160 + optional bytes executor_facet_addr = 3; // required; H160 + optional bytes admin_facet_addr = 4; // required; H160 + optional bytes getters_facet_addr = 5; // required; H160 + optional bytes verifier_addr = 6; // required; H160 + optional bytes diamond_init_addr = 7; // required; H160 + optional bytes diamond_upgrade_init_addr = 8; // required; H160 + optional bytes diamond_proxy_addr = 9; // required; H160 + optional bytes validator_timelock_addr = 10; // required; H160 + optional bytes genesis_tx_hash = 11; // required; H256 + optional bytes l1_erc20_bridge_proxy_addr = 12; // required; H160 + optional bytes l1_erc20_bridge_impl_addr = 13; // required; H160 + optional bytes l2_erc20_bridge_addr = 14; // required; H160 + optional bytes l1_weth_bridge_proxy_addr = 15; // optional; H160 + optional bytes l2_weth_bridge_addr = 16; // optional; H160 + optional bytes l1_allow_list_addr = 17; // required; H160 + optional bytes l2_testnet_paymaster_addr = 18; // optional; H160 + optional bytes recursion_scheduler_level_vk_hash = 19; // required; H256 + optional bytes recursion_node_level_vk_hash = 20; // required; H256 + optional bytes recursion_leaf_level_vk_hash = 21; // required; H256 + optional bytes recursion_circuits_set_vks_hash = 22; // required; H256 + optional bytes l1_multicall3_addr = 23; // required; H160 + optional bytes fri_recursion_scheduler_level_vk_hash = 24; // required; H256 + optional bytes fri_recursion_node_level_vk_hash = 25; // required; H256 + optional bytes fri_recursion_leaf_level_vk_hash = 26; // required; H256 + optional ProverAtGenesis prover_at_genesis = 27; // required + optional bytes snark_wrapper_vk_hash = 28; // required; H256 +} diff --git a/core/lib/protobuf_config/src/proto/database.proto b/core/lib/protobuf_config/src/proto/database.proto new file mode 100644 index 000000000000..64e1e7fd300e --- /dev/null +++ b/core/lib/protobuf_config/src/proto/database.proto @@ -0,0 +1,31 @@ +syntax = "proto3"; + +package zksync.config; + +enum MerkleTreeMode { + FULL = 0; + LIGHTWEIGHT = 1; +} + +message MerkleTree { + optional string path = 1; // optional; fs path + optional MerkleTreeMode mode = 2; // optional + optional uint64 multi_get_chunk_size = 3; // optional; ? + optional uint64 block_cache_size_mb = 4; // optional; MB + optional uint64 memtable_capacity_mb = 5; // optional; MB + optional uint64 stalled_writes_timeout_sec = 6; // optional; s + optional uint64 max_l1_batches_per_iter = 7; // optional +} + +message DB { + optional string state_keeper_db_path = 1; // optional; fs path + optional MerkleTree merkle_tree = 2; // optional +} + +message Postgres { + optional string master_url = 1; // optional + optional string replica_url = 2; // optional + optional string prover_url = 3; // optional + optional uint32 max_connections = 4; // optional + optional uint64 statement_timeout_sec = 5; // optional; s +} diff --git a/core/lib/protobuf_config/src/proto/eth_client.proto b/core/lib/protobuf_config/src/proto/eth_client.proto new file mode 100644 index 000000000000..50723dc2dd15 --- /dev/null +++ b/core/lib/protobuf_config/src/proto/eth_client.proto @@ -0,0 +1,8 @@ +syntax = "proto3"; + +package zksync.config; + +message ETHClient { + optional uint64 chain_id = 1; // required; TODO: shouldn't it be Network? + optional string web3_url = 2; // required +} diff --git a/core/lib/protobuf_config/src/proto/eth_sender.proto b/core/lib/protobuf_config/src/proto/eth_sender.proto new file mode 100644 index 000000000000..b5e866d04126 --- /dev/null +++ b/core/lib/protobuf_config/src/proto/eth_sender.proto @@ -0,0 +1,51 @@ +syntax = "proto3"; + +package zksync.config; + +message ETHSender { + optional Sender sender = 1; // required + optional GasAdjuster gas_adjuster = 2; // required +} + +enum ProofSendingMode { + ONLY_REAL_PROOFS = 0; + ONLY_SAMPLED_PROOFS = 1; + SKIP_EVERY_PROOF = 2; +} + +enum ProofLoadingMode { + OLD_PROOF_FROM_DB = 0; + FRI_PROOF_FROM_GCS = 1; +} + +message Sender { + repeated uint64 aggregated_proof_sizes = 1; // ? + optional uint64 wait_confirmations = 2; // optional + optional uint64 tx_poll_period = 3; // required; s + optional uint64 aggregate_tx_poll_period = 4; // required; s + optional uint64 max_txs_in_flight = 5; // required + optional ProofSendingMode proof_sending_mode = 6; // required + optional uint32 max_aggregated_tx_gas = 7; // required; wei? + optional uint64 max_eth_tx_data_size = 8; // required; ? + optional uint32 max_aggregated_blocks_to_commit = 9; // required + optional uint32 max_aggregated_blocks_to_execute = 10; // required + optional uint64 aggregated_block_commit_deadline = 11; // required; ? + optional uint64 aggregated_block_prove_deadline = 12; // required; ? + optional uint64 aggregated_block_execute_deadline = 13; // required; ? + optional uint64 timestamp_criteria_max_allowed_lag = 14; // required; ? + optional uint64 l1_batch_min_age_before_execute_seconds = 15; // optional; s + optional uint64 max_acceptable_priority_fee_in_gwei = 16; // required; gwei + optional ProofLoadingMode proof_loading_mode = 17; // required + // operator_private_key? +} + +message GasAdjuster { + optional uint64 default_priority_fee_per_gas = 1; // required; wei? + optional uint64 max_base_fee_samples = 2; // required; wei? + optional double pricing_formula_parameter_a = 3; // required + optional double pricing_formula_parameter_b = 4; // required + optional double internal_l1_pricing_multiplier = 5; // required + optional uint64 internal_enforced_l1_gas_price = 6; // optional; wei? + optional uint64 poll_period = 7; // required; s + optional uint64 max_l1_gas_price = 8; // optional; wei? +} diff --git a/core/lib/protobuf_config/src/proto/eth_watch.proto b/core/lib/protobuf_config/src/proto/eth_watch.proto new file mode 100644 index 000000000000..fe78fd33238c --- /dev/null +++ b/core/lib/protobuf_config/src/proto/eth_watch.proto @@ -0,0 +1,8 @@ +syntax = "proto3"; + +package zksync.config; + +message ETHWatch { + optional uint64 confirmations_for_eth_event = 1; // optional + optional uint64 eth_node_poll_interval = 2; // required; ms +} diff --git a/core/lib/protobuf_config/src/proto/fri_proof_compressor.proto b/core/lib/protobuf_config/src/proto/fri_proof_compressor.proto new file mode 100644 index 000000000000..b8880936c5ef --- /dev/null +++ b/core/lib/protobuf_config/src/proto/fri_proof_compressor.proto @@ -0,0 +1,15 @@ +syntax = "proto3"; + +package zksync.config; + +message FriProofCompressor { + optional uint32 compression_mode = 1; // required; u8 + optional uint32 prometheus_listener_port = 2; // required; u16 + optional string prometheus_pushgateway_url = 3; // required + optional uint64 prometheus_push_interval_ms = 4; // optional; ms + optional uint32 generation_timeout_in_secs = 5; // required; s + optional uint32 max_attempts = 6; // required + optional string universal_setup_path = 7; // required; fs path + optional string universal_setup_download_url = 8; // required + optional bool verify_wrapper_proof = 9; // required +} diff --git a/core/lib/protobuf_config/src/proto/fri_prover.proto b/core/lib/protobuf_config/src/proto/fri_prover.proto new file mode 100644 index 000000000000..5547bd9b8da9 --- /dev/null +++ b/core/lib/protobuf_config/src/proto/fri_prover.proto @@ -0,0 +1,24 @@ +syntax = "proto3"; + +package zksync.config; + +enum SetupLoadMode { + FROM_DISK = 0; + FROM_MEMORY = 1; +} + +message FriProver { + optional string setup_data_path = 1; // required; fs path? + optional uint32 prometheus_port = 2; // required; u16 + optional uint32 max_attempts = 3; // required + optional uint32 generation_timeout_in_secs = 4; // required; s + optional bytes base_layer_circuit_ids_to_be_verified = 5; // required + optional bytes recursive_layer_circuit_ids_to_be_verified = 6; // required + optional SetupLoadMode setup_load_mode = 7; // required + optional uint32 specialized_group_id = 8; // required; u8 + optional uint64 witness_vector_generator_thread_count = 9; // optional + optional uint64 queue_capacity = 10; // required + optional uint32 witness_vector_receiver_port = 11; // required; u16 + optional string zone_read_url = 12; // required + optional bool shall_save_to_public_bucket = 13; // required +} diff --git a/core/lib/protobuf_config/src/proto/fri_prover_gateway.proto b/core/lib/protobuf_config/src/proto/fri_prover_gateway.proto new file mode 100644 index 000000000000..716b761607c2 --- /dev/null +++ b/core/lib/protobuf_config/src/proto/fri_prover_gateway.proto @@ -0,0 +1,11 @@ +syntax = "proto3"; + +package zksync.config; + +message FriProverGateway { + optional string api_url = 1; // required + optional uint32 api_poll_duration_secs = 2; // required; s + optional uint32 prometheus_listener_port = 3; // required; u16 + optional string prometheus_pushgateway_url = 4; // required + optional uint64 prometheus_push_interval_ms = 5; // optional; ms +} diff --git a/core/lib/protobuf_config/src/proto/fri_prover_group.proto b/core/lib/protobuf_config/src/proto/fri_prover_group.proto new file mode 100644 index 000000000000..8a4dd9511b41 --- /dev/null +++ b/core/lib/protobuf_config/src/proto/fri_prover_group.proto @@ -0,0 +1,24 @@ +syntax = "proto3"; + +package zksync.config; + +message CircuitIdRoundTuple { + optional uint32 circuit_id = 1; // required; u8 + optional uint32 aggregation_round = 2; // required; u8 +} + +message FriProverGroup { + repeated CircuitIdRoundTuple group_0 = 1; + repeated CircuitIdRoundTuple group_1 = 2; + repeated CircuitIdRoundTuple group_2 = 3; + repeated CircuitIdRoundTuple group_3 = 4; + repeated CircuitIdRoundTuple group_4 = 5; + repeated CircuitIdRoundTuple group_5 = 6; + repeated CircuitIdRoundTuple group_6 = 7; + repeated CircuitIdRoundTuple group_7 = 8; + repeated CircuitIdRoundTuple group_8 = 9; + repeated CircuitIdRoundTuple group_9 = 10; + repeated CircuitIdRoundTuple group_10 = 11; + repeated CircuitIdRoundTuple group_11 = 12; + repeated CircuitIdRoundTuple group_12 = 13; +} diff --git a/core/lib/protobuf_config/src/proto/fri_witness_generator.proto b/core/lib/protobuf_config/src/proto/fri_witness_generator.proto new file mode 100644 index 000000000000..dda7f1e8fef1 --- /dev/null +++ b/core/lib/protobuf_config/src/proto/fri_witness_generator.proto @@ -0,0 +1,13 @@ +syntax = "proto3"; + +package zksync.config; + +message FriWitnessGenerator { + optional uint32 generation_timeout_in_secs = 1; // required; s + optional uint32 max_attempts = 2; // required + optional uint32 blocks_proving_percentage = 3; // optional; 0-100 + repeated uint32 dump_arguments_for_blocks = 4; + optional uint32 last_l1_batch_to_process = 5; // optional + optional uint32 force_process_block = 6; // optional + optional bool shall_save_to_public_bucket = 7; // required +} diff --git a/core/lib/protobuf_config/src/proto/fri_witness_vector_generator.proto b/core/lib/protobuf_config/src/proto/fri_witness_vector_generator.proto new file mode 100644 index 000000000000..5ec4c0eca291 --- /dev/null +++ b/core/lib/protobuf_config/src/proto/fri_witness_vector_generator.proto @@ -0,0 +1,13 @@ +syntax = "proto3"; + +package zksync.config; + +message FriWitnessVectorGenerator { + optional uint32 max_prover_reservation_duration_in_secs = 1; // required; s + optional uint32 prover_instance_wait_timeout_in_secs = 2; // required; s + optional uint32 prover_instance_poll_time_in_milli_secs = 3; // required; ms + optional uint32 prometheus_listener_port = 4; // required; u16 + optional string prometheus_pushgateway_url = 5; // required + optional uint64 prometheus_push_interval_ms = 6; // optional; ms + optional uint32 specialized_group_id = 7; // required; u8 +} diff --git a/core/lib/protobuf_config/src/proto/house_keeper.proto b/core/lib/protobuf_config/src/proto/house_keeper.proto new file mode 100644 index 000000000000..d27fb6da71e2 --- /dev/null +++ b/core/lib/protobuf_config/src/proto/house_keeper.proto @@ -0,0 +1,19 @@ +syntax = "proto3"; + +package zksync.config; + +message HouseKeeper { + optional uint64 l1_batch_metrics_reporting_interval_ms = 1; // required; ms + optional uint64 gpu_prover_queue_reporting_interval_ms = 2; // required; ms + optional uint64 prover_job_retrying_interval_ms = 3; // required; ms + optional uint64 prover_stats_reporting_interval_ms = 4; // required ms + optional uint64 witness_job_moving_interval_ms = 5; // required; ms + optional uint64 witness_generator_stats_reporting_interval_ms = 6; // required; ms + optional uint64 fri_witness_job_moving_interval_ms = 7; // required; ms + optional uint64 fri_prover_job_retrying_interval_ms = 8; // required; ms + optional uint64 fri_witness_generator_job_retrying_interval_ms = 9; // required; ms + optional uint32 prover_db_pool_size = 10; // required + optional uint64 fri_prover_stats_reporting_interval_ms = 11; // required; ms + optional uint64 fri_proof_compressor_job_retrying_interval_ms = 12; // required; ms + optional uint64 fri_proof_compressor_stats_reporting_interval_ms = 13; // required; ms +} diff --git a/core/lib/protobuf_config/src/proto/mod.rs b/core/lib/protobuf_config/src/proto/mod.rs new file mode 100644 index 000000000000..660bf4c5b4cc --- /dev/null +++ b/core/lib/protobuf_config/src/proto/mod.rs @@ -0,0 +1,2 @@ +#![allow(warnings)] +include!(concat!(env!("OUT_DIR"), "/src/proto/gen.rs")); diff --git a/core/lib/protobuf_config/src/proto/object_store.proto b/core/lib/protobuf_config/src/proto/object_store.proto new file mode 100644 index 000000000000..941799c07c20 --- /dev/null +++ b/core/lib/protobuf_config/src/proto/object_store.proto @@ -0,0 +1,18 @@ +syntax = "proto3"; + +package zksync.config; + +enum ObjectStoreMode { + GCS = 0; + GCS_WITH_CREDENTIAL_FILE = 1; + FILE_BACKED = 2; + GCS_ANONYMOUS_READ_ONLY = 3; +} + +message ObjectStore { + optional string bucket_base_url = 1; // required; url + optional ObjectStoreMode mode = 2; // required + optional string file_backed_base_path = 3; // required; fs path + optional string gcs_credential_file_path = 4; // required; fs path + optional uint32 max_retries = 5; // required +} diff --git a/core/lib/protobuf_config/src/proto/proof_data_handler.proto b/core/lib/protobuf_config/src/proto/proof_data_handler.proto new file mode 100644 index 000000000000..a76e15ed5ce9 --- /dev/null +++ b/core/lib/protobuf_config/src/proto/proof_data_handler.proto @@ -0,0 +1,15 @@ +syntax = "proto3"; + +package zksync.config; + +enum ProtocolVersionLoadingMode { + FROM_DB = 0; + FROM_ENV_VAR = 1; +} + +message ProofDataHandler { + optional uint32 http_port = 1; // required; u16 + optional uint32 proof_generation_timeout_in_secs = 2; // required; s + optional ProtocolVersionLoadingMode protocol_version_loading_mode = 3; // required + optional uint32 fri_protocol_version_id = 4; // required; u16 +} diff --git a/core/lib/protobuf_config/src/proto/snapshots_creator.proto b/core/lib/protobuf_config/src/proto/snapshots_creator.proto new file mode 100644 index 000000000000..27e4c66435d7 --- /dev/null +++ b/core/lib/protobuf_config/src/proto/snapshots_creator.proto @@ -0,0 +1,8 @@ +syntax = "proto3"; + +package zksync.config; + +message SnapshotsCreator { + optional uint64 storage_logs_chunk_size = 1; // optional + optional uint32 concurrent_queries_count = 2; // optional +} diff --git a/core/lib/protobuf_config/src/proto/utils.proto b/core/lib/protobuf_config/src/proto/utils.proto new file mode 100644 index 000000000000..40c6c10c756e --- /dev/null +++ b/core/lib/protobuf_config/src/proto/utils.proto @@ -0,0 +1,9 @@ +syntax = "proto3"; + +package zksync.config; + +message Prometheus { + optional uint32 listener_port = 1; // required + optional string pushgateway_url = 2; // required + optional uint64 push_interval_ms = 3; +} diff --git a/core/lib/protobuf_config/src/proto/witness_generator.proto b/core/lib/protobuf_config/src/proto/witness_generator.proto new file mode 100644 index 000000000000..5836c5892e73 --- /dev/null +++ b/core/lib/protobuf_config/src/proto/witness_generator.proto @@ -0,0 +1,20 @@ +syntax = "proto3"; + +package zksync.config; + +enum BasicWitnessGeneratorDataSource { + FROM_POSTGRES = 0; + FROM_POSTGRES_SHADOW_BLOB = 1; + FROM_BLOB = 2; +} + +message WitnessGenerator { + optional uint32 generation_timeout_in_secs = 1; // required; s + optional string initial_setup_key_path = 2; // required; fs path + optional string key_download_url = 3; // required; url + optional uint32 max_attempts = 4; // required + optional uint32 blocks_proving_percentage = 5; // optional; 0-100 (percentage) + repeated uint32 dump_arguments_for_blocks = 6; + optional uint32 last_l1_batch_to_process = 7; // optional + optional BasicWitnessGeneratorDataSource data_source = 8; // required +} diff --git a/core/lib/protobuf_config/src/repr.rs b/core/lib/protobuf_config/src/repr.rs new file mode 100644 index 000000000000..8b71ee63c847 --- /dev/null +++ b/core/lib/protobuf_config/src/repr.rs @@ -0,0 +1,15 @@ +use anyhow::Context as _; + +/// Trait reverse to `zksync_protobuf::ProtoFmt` for cases where +/// you would like to specify a custom proto encoding for an externally defined type. +pub(crate) trait ProtoRepr: + zksync_protobuf::build::prost_reflect::ReflectMessage + Default +{ + type Type; + fn read(&self) -> anyhow::Result; + fn build(this: &Self::Type) -> Self; +} + +pub(crate) fn read_required_repr(field: &Option

) -> anyhow::Result { + field.as_ref().context("missing field")?.read() +} diff --git a/core/lib/protobuf_config/src/snapshots_creator.rs b/core/lib/protobuf_config/src/snapshots_creator.rs new file mode 100644 index 000000000000..1ea5aada1203 --- /dev/null +++ b/core/lib/protobuf_config/src/snapshots_creator.rs @@ -0,0 +1,24 @@ +use anyhow::Context as _; +use zksync_config::configs; +use zksync_protobuf::required; + +use crate::{proto, repr::ProtoRepr}; + +impl ProtoRepr for proto::SnapshotsCreator { + type Type = configs::SnapshotsCreatorConfig; + fn read(&self) -> anyhow::Result { + Ok(Self::Type { + storage_logs_chunk_size: *required(&self.storage_logs_chunk_size) + .context("storage_logs_chunk_size")?, + concurrent_queries_count: *required(&self.concurrent_queries_count) + .context("concurrent_queries_count")?, + }) + } + + fn build(this: &Self::Type) -> Self { + Self { + storage_logs_chunk_size: Some(this.storage_logs_chunk_size), + concurrent_queries_count: Some(this.concurrent_queries_count), + } + } +} diff --git a/core/lib/protobuf_config/src/tests.rs b/core/lib/protobuf_config/src/tests.rs new file mode 100644 index 000000000000..de4803a10085 --- /dev/null +++ b/core/lib/protobuf_config/src/tests.rs @@ -0,0 +1,92 @@ +use pretty_assertions::assert_eq; +use rand::Rng; +use zksync_config::testonly; + +use crate::{proto, repr::ProtoRepr}; + +fn encode(msg: &P::Type) -> Vec { + let msg = P::build(msg); + zksync_protobuf::canonical_raw(&msg.encode_to_vec(), &msg.descriptor()).unwrap() +} + +fn decode(bytes: &[u8]) -> anyhow::Result { + P::read(&P::decode(bytes)?) +} + +fn encode_json(msg: &P::Type) -> String { + let mut s = serde_json::Serializer::pretty(vec![]); + zksync_protobuf::serde::serialize_proto(&P::build(msg), &mut s).unwrap(); + String::from_utf8(s.into_inner()).unwrap() +} + +fn decode_json(json: &str) -> anyhow::Result { + let mut d = serde_json::Deserializer::from_str(json); + P::read(&zksync_protobuf::serde::deserialize_proto(&mut d)?) +} + +#[track_caller] +fn encode_decode(rng: &mut impl Rng) +where + P::Type: PartialEq + std::fmt::Debug + testonly::RandomConfig, +{ + for required_only in [false, true] { + let want: P::Type = testonly::Gen { + rng, + required_only, + decimal_fractions: false, + } + .gen(); + let got = decode::

(&encode::

(&want)).unwrap(); + assert_eq!(&want, &got, "binary encoding"); + + let want: P::Type = testonly::Gen { + rng, + required_only, + decimal_fractions: true, + } + .gen(); + let got = decode_json::

(&encode_json::

(&want)).unwrap(); + assert_eq!(&want, &got, "json encoding"); + } +} + +/// Tests config <-> proto (boilerplate) conversions. +#[test] +fn test_encoding() { + let rng = &mut rand::thread_rng(); + encode_decode::(rng); + encode_decode::(rng); + encode_decode::(rng); + encode_decode::(rng); + encode_decode::(rng); + encode_decode::(rng); + encode_decode::(rng); + encode_decode::(rng); + encode_decode::(rng); + encode_decode::(rng); + encode_decode::(rng); + encode_decode::(rng); + encode_decode::(rng); + encode_decode::(rng); + encode_decode::(rng); + encode_decode::(rng); + encode_decode::(rng); + encode_decode::(rng); + encode_decode::(rng); + encode_decode::(rng); + encode_decode::(rng); + encode_decode::(rng); + encode_decode::(rng); + encode_decode::(rng); + encode_decode::(rng); + encode_decode::(rng); + encode_decode::(rng); + encode_decode::(rng); + encode_decode::(rng); + encode_decode::(rng); + encode_decode::(rng); + encode_decode::(rng); + encode_decode::(rng); + encode_decode::(rng); + encode_decode::(rng); +} diff --git a/core/lib/protobuf_config/src/utils.rs b/core/lib/protobuf_config/src/utils.rs new file mode 100644 index 000000000000..d4089df01dfb --- /dev/null +++ b/core/lib/protobuf_config/src/utils.rs @@ -0,0 +1,28 @@ +use anyhow::Context as _; +use zksync_config::configs::PrometheusConfig; +use zksync_protobuf::required; + +use crate::{proto, repr::ProtoRepr}; + +impl ProtoRepr for proto::Prometheus { + type Type = PrometheusConfig; + fn read(&self) -> anyhow::Result { + Ok(PrometheusConfig { + listener_port: required(&self.listener_port) + .and_then(|p| Ok((*p).try_into()?)) + .context("listener_port")?, + pushgateway_url: required(&self.pushgateway_url) + .context("pushgateway_url")? + .clone(), + push_interval_ms: self.push_interval_ms, + }) + } + + fn build(this: &Self::Type) -> Self { + Self { + listener_port: Some(this.listener_port.into()), + pushgateway_url: Some(this.pushgateway_url.clone()), + push_interval_ms: this.push_interval_ms, + } + } +} diff --git a/core/lib/protobuf_config/src/witness_generator.rs b/core/lib/protobuf_config/src/witness_generator.rs new file mode 100644 index 000000000000..d4513ebffb53 --- /dev/null +++ b/core/lib/protobuf_config/src/witness_generator.rs @@ -0,0 +1,68 @@ +use anyhow::Context as _; +use zksync_config::configs; +use zksync_protobuf::required; + +use crate::{proto, repr::ProtoRepr}; + +impl proto::BasicWitnessGeneratorDataSource { + fn new(x: &configs::witness_generator::BasicWitnessGeneratorDataSource) -> Self { + type From = configs::witness_generator::BasicWitnessGeneratorDataSource; + match x { + From::FromPostgres => Self::FromPostgres, + From::FromPostgresShadowBlob => Self::FromPostgresShadowBlob, + From::FromBlob => Self::FromBlob, + } + } + fn parse(&self) -> configs::witness_generator::BasicWitnessGeneratorDataSource { + type To = configs::witness_generator::BasicWitnessGeneratorDataSource; + match self { + Self::FromPostgres => To::FromPostgres, + Self::FromPostgresShadowBlob => To::FromPostgresShadowBlob, + Self::FromBlob => To::FromBlob, + } + } +} + +impl ProtoRepr for proto::WitnessGenerator { + type Type = configs::WitnessGeneratorConfig; + fn read(&self) -> anyhow::Result { + Ok(Self::Type { + generation_timeout_in_secs: required(&self.generation_timeout_in_secs) + .and_then(|x| Ok((*x).try_into()?)) + .context("generation_timeout_in_secs")?, + initial_setup_key_path: required(&self.initial_setup_key_path) + .context("initial_setup_key_path")? + .clone(), + key_download_url: required(&self.key_download_url) + .context("key_download_url")? + .clone(), + max_attempts: *required(&self.max_attempts).context("max_attempts")?, + blocks_proving_percentage: self + .blocks_proving_percentage + .map(|x| x.try_into()) + .transpose() + .context("blocks_proving_percentage")?, + dump_arguments_for_blocks: self.dump_arguments_for_blocks.clone(), + last_l1_batch_to_process: self.last_l1_batch_to_process, + data_source: required(&self.data_source) + .and_then(|x| Ok(proto::BasicWitnessGeneratorDataSource::try_from(*x)?)) + .context("data_source")? + .parse(), + }) + } + + fn build(this: &Self::Type) -> Self { + Self { + generation_timeout_in_secs: Some(this.generation_timeout_in_secs.into()), + initial_setup_key_path: Some(this.initial_setup_key_path.clone()), + key_download_url: Some(this.key_download_url.clone()), + max_attempts: Some(this.max_attempts), + blocks_proving_percentage: this.blocks_proving_percentage.map(|x| x.into()), + dump_arguments_for_blocks: this.dump_arguments_for_blocks.clone(), + last_l1_batch_to_process: this.last_l1_batch_to_process, + data_source: Some( + proto::BasicWitnessGeneratorDataSource::new(&this.data_source).into(), + ), + } + } +} diff --git a/prover/Cargo.lock b/prover/Cargo.lock index b972377545c9..4e5b5bc1763b 100644 --- a/prover/Cargo.lock +++ b/prover/Cargo.lock @@ -6991,6 +6991,7 @@ name = "zksync_config" version = "0.1.0" dependencies = [ "anyhow", + "rand 0.8.5", "serde", "zksync_basic_types", ] diff --git a/prover/Cargo.toml b/prover/Cargo.toml index e450fce09c08..fd0d764ea558 100644 --- a/prover/Cargo.toml +++ b/prover/Cargo.toml @@ -15,6 +15,7 @@ members = [ resolver = "2" + # for `perf` profiling [profile.perf] inherits = "release"