feat(core): Split config definitions and deserialization (#414)

Instead of a thousand words:

<img width="392" alt="image"
src="https://github.com/matter-labs/zksync-era/assets/12111581/b0230f94-29bb-4ef5-a779-7017fa4977f2">

This is the first PR of N preparing zkSync codebase for zkStack.

---

In this episode:

- `<ConfigName>::from_env()` methods were extracted from `zksync_config`
to a new crate `zksync_env_config`.
- Any usage of `from_env()` has been removed from `core/lib` directory.
In other words, configs are now deserialized only in binaries.
- Some crates were redesigned somewhat, for example,
`ObjectStoreFactory` now lacks `*_from_env` methods and can only be
created via config.
- Some configs obtained `for_tests()` constructors that allow creating
configs without accessing env variables. Values there mostly match
localhost defaults. Some tests have been rewritten to enforce values in
the configs (thus they are now more resilient than they were before).

The focus for this PR has been to remove any `from_env()` usage, so
there are some unsolved issues, e.g.
- We still access some env variables via `std::env::var`.
- We have introduced a temporary ugly structure `TempConfigStore` that
_reminds_ the former `ZkSyncConfig` but doesn't have the same level of
danger.

These (and other) things are to be resolved soon.

Cargo.toml

@@ -21,6 +21,7 @@ members = [
     "core/lib/circuit_breaker",
     "core/lib/commitment_utils",
     "core/lib/dal",
+    "core/lib/env_config",
     "core/lib/eth_client",
     "core/lib/eth_signer",
     "core/lib/mempool",

core/bin/block_reverter/Cargo.toml

@@ -12,6 +12,7 @@ publish = false # We don't want to publish our binaries.
 
 [dependencies]
 zksync_config = { path = "../../lib/config" }
+zksync_env_config = { path = "../../lib/env_config" }
 zksync_dal = { path = "../../lib/dal" }
 zksync_types = { path = "../../lib/types" }
 zksync_core = { path = "../../lib/zksync_core" }

core/bin/block_reverter/src/main.rs

@@ -4,6 +4,7 @@ use tokio::io::{self, AsyncReadExt};
 
 use zksync_config::{ContractsConfig, DBConfig, ETHClientConfig, ETHSenderConfig};
 use zksync_dal::{connection::DbVariant, ConnectionPool};
+use zksync_env_config::FromEnv;
 use zksync_types::{L1BatchNumber, U256};
 
 use zksync_core::block_reverter::{

core/bin/contract-verifier/Cargo.toml

@@ -13,6 +13,7 @@ publish = false # We don't want to publish our binaries.
 [dependencies]
 zksync_types = { path = "../../lib/types" }
 zksync_dal = { path = "../../lib/dal" }
+zksync_env_config = { path = "../../lib/env_config" }
 zksync_config = { path = "../../lib/config" }
 zksync_contracts = { path = "../../lib/contracts" }
 zksync_queued_job_processor = { path = "../../lib/queued_job_processor" }

core/bin/contract-verifier/src/main.rs

@@ -4,6 +4,7 @@ use anyhow::Context as _;
 use prometheus_exporter::PrometheusExporterConfig;
 use zksync_config::{configs::PrometheusConfig, ApiConfig, ContractVerifierConfig};
 use zksync_dal::ConnectionPool;
+use zksync_env_config::FromEnv;
 use zksync_queued_job_processor::JobProcessor;
 use zksync_utils::wait_for_tasks::wait_for_tasks;
 

core/bin/contract-verifier/src/verifier.rs

@@ -12,6 +12,7 @@ use tokio::time;
 
 use zksync_config::ContractVerifierConfig;
 use zksync_dal::{ConnectionPool, StorageProcessor};
+use zksync_env_config::FromEnv;
 use zksync_queued_job_processor::{async_trait, JobProcessor};
 use zksync_types::{
     contract_verification_api::{

core/bin/external_node/src/config/mod.rs

@@ -3,8 +3,7 @@ use serde::Deserialize;
 use std::{env, time::Duration};
 use url::Url;
 
-use zksync_basic_types::{Address, L1ChainId, L2ChainId, MiniblockNumber, H256};
-use zksync_contracts::BaseSystemContractsHashes;
+use zksync_basic_types::{Address, L1ChainId, L2ChainId, MiniblockNumber};
 use zksync_core::api_server::{
     tx_sender::TxSenderConfig, web3::state::InternalApiConfig, web3::Namespace,
 };
@@ -32,9 +31,6 @@ pub struct RemoteENConfig {
     pub l2_chain_id: L2ChainId,
     pub l1_chain_id: L1ChainId,
 
-    pub default_aa_hash: H256,
-    pub bootloader_hash: H256,
-
     pub fair_l2_gas_price: u64,
 }
 
@@ -76,7 +72,6 @@ impl RemoteENConfig {
             .await
             .context("Failed to fetch last miniblock header")?
             .expect("Block is known to exist");
-        let base_system_contract_hashes = block_header.base_system_contracts_hashes;
 
         Ok(Self {
             diamond_proxy_addr,
@@ -87,18 +82,9 @@ impl RemoteENConfig {
             l2_weth_bridge_addr: bridges.l2_weth_bridge,
             l2_chain_id,
             l1_chain_id,
-            default_aa_hash: base_system_contract_hashes.default_aa,
-            bootloader_hash: base_system_contract_hashes.bootloader,
             fair_l2_gas_price: block_header.l2_fair_gas_price,
         })
     }
-
-    pub fn base_system_contracts_hashes(&self) -> BaseSystemContractsHashes {
-        BaseSystemContractsHashes {
-            default_aa: self.default_aa_hash,
-            bootloader: self.bootloader_hash,
-        }
-    }
 }
 
 /// This part of the external node config is completely optional to provide.
@@ -509,8 +495,6 @@ impl From<ExternalNodeConfig> for TxSenderConfig {
             max_nonce_ahead: config.optional.max_nonce_ahead,
             fair_l2_gas_price: config.remote.fair_l2_gas_price,
             vm_execution_cache_misses_limit: config.optional.vm_execution_cache_misses_limit,
-            default_aa: config.remote.default_aa_hash,
-            bootloader: config.remote.bootloader_hash,
             // We set these values to the maximum since we don't know the actual values
             // and they will be enforced by the main node anyway.
             max_allowed_l2_tx_gas_limit: u32::MAX,

core/bin/merkle_tree_consistency_checker/Cargo.toml

@@ -12,6 +12,7 @@ publish = false # We don't want to publish our binaries.
 
 [dependencies]
 zksync_config = { path = "../../lib/config" }
+zksync_env_config = { path = "../../lib/env_config" }
 zksync_merkle_tree = { path = "../../lib/merkle_tree" }
 zksync_types = { path = "../../lib/types" }
 zksync_storage = { path = "../../lib/storage" }

core/bin/merkle_tree_consistency_checker/src/main.rs

@@ -4,6 +4,7 @@ use clap::Parser;
 use std::{path::Path, time::Instant};
 
 use zksync_config::DBConfig;
+use zksync_env_config::FromEnv;
 use zksync_merkle_tree::domain::ZkSyncTree;
 use zksync_storage::RocksDB;
 use zksync_types::L1BatchNumber;

core/bin/rocksdb_util/Cargo.toml

@@ -12,6 +12,7 @@ publish = false # We don't want to publish our binaries.
 
 [dependencies]
 zksync_config = { path = "../../lib/config" }
+zksync_env_config = { path = "../../lib/env_config" }
 zksync_storage = { path = "../../lib/storage" }
 
 anyhow = "1.0"

core/bin/rocksdb_util/src/main.rs

@@ -2,6 +2,7 @@ use anyhow::Context as _;
 use clap::{Parser, Subcommand};
 
 use zksync_config::DBConfig;
+use zksync_env_config::FromEnv;
 use zksync_storage::rocksdb::{
     backup::{BackupEngine, BackupEngineOptions, RestoreOptions},
     Env, Error, Options, DB,

core/bin/zksync_server/Cargo.toml

@@ -12,6 +12,7 @@ publish = false # We don't want to publish our binaries.
 
 [dependencies]
 zksync_config = { path = "../../lib/config" }
+zksync_env_config = { path = "../../lib/env_config" }
 zksync_storage = { path = "../../lib/storage" }
 zksync_utils = { path = "../../lib/utils" }
 zksync_types = { path = "../../lib/types" }

core/bin/zksync_server/src/main.rs

@@ -2,13 +2,28 @@ use anyhow::Context as _;
 use clap::Parser;
 
 use std::{str::FromStr, time::Duration};
-use zksync_config::configs::chain::NetworkConfig;
 
-use zksync_config::{ContractsConfig, ETHClientConfig, ETHSenderConfig};
+use zksync_config::{
+    configs::{
+        api::{HealthCheckConfig, MerkleTreeApiConfig, Web3JsonRpcConfig},
+        chain::{
+            CircuitBreakerConfig, MempoolConfig, NetworkConfig, OperationsManagerConfig,
+            StateKeeperConfig,
+        },
+        house_keeper::HouseKeeperConfig,
+        FriProofCompressorConfig, FriProverConfig, FriWitnessGeneratorConfig, PrometheusConfig,
+        ProofDataHandlerConfig, ProverGroupConfig, WitnessGeneratorConfig,
+    },
+    ApiConfig, ContractsConfig, DBConfig, ETHClientConfig, ETHSenderConfig, ETHWatchConfig,
+    FetcherConfig, GasAdjusterConfig, ObjectStoreConfig, ProverConfigs,
+};
+
+use zksync_core::temp_config_store::TempConfigStore;
 use zksync_core::{
     genesis_init, initialize_components, is_genesis_needed, setup_sigint_handler, Component,
     Components,
 };
+use zksync_env_config::FromEnv;
 use zksync_storage::RocksDB;
 use zksync_utils::wait_for_tasks::wait_for_tasks;
 
@@ -102,9 +117,42 @@ async fn main() -> anyhow::Result<()> {
         [Component::WitnessGenerator(Some(_), _)]
     );
 
+    // TODO (QIT-22): Only deserialize configs on demand.
+    // Right now, we are trying to deserialize all the configs that may be needed by `zksync_core`.
+    // "May" is the key word here, since some configs are only used by certain component configuration,
+    // hence we are using `Option`s.
+    let configs = TempConfigStore {
+        health_check_config: HealthCheckConfig::from_env().ok(),
+        merkle_tree_api_config: MerkleTreeApiConfig::from_env().ok(),
+        web3_json_rpc_config: Web3JsonRpcConfig::from_env().ok(),
+        circuit_breaker_config: CircuitBreakerConfig::from_env().ok(),
+        mempool_config: MempoolConfig::from_env().ok(),
+        network_config: NetworkConfig::from_env().ok(),
+        operations_manager_config: OperationsManagerConfig::from_env().ok(),
+        state_keeper_config: StateKeeperConfig::from_env().ok(),
+        house_keeper_config: HouseKeeperConfig::from_env().ok(),
+        fri_proof_compressor_config: FriProofCompressorConfig::from_env().ok(),
+        fri_prover_config: FriProverConfig::from_env().ok(),
+        fri_witness_generator_config: FriWitnessGeneratorConfig::from_env().ok(),
+        prometheus_config: PrometheusConfig::from_env().ok(),
+        proof_data_handler_config: ProofDataHandlerConfig::from_env().ok(),
+        prover_group_config: ProverGroupConfig::from_env().ok(),
+        witness_generator_config: WitnessGeneratorConfig::from_env().ok(),
+        api_config: ApiConfig::from_env().ok(),
+        contracts_config: ContractsConfig::from_env().ok(),
+        db_config: DBConfig::from_env().ok(),
+        eth_client_config: ETHClientConfig::from_env().ok(),
+        eth_sender_config: ETHSenderConfig::from_env().ok(),
+        eth_watch_config: ETHWatchConfig::from_env().ok(),
+        fetcher_config: FetcherConfig::from_env().ok(),
+        gas_adjuster_config: GasAdjusterConfig::from_env().ok(),
+        prover_configs: ProverConfigs::from_env().ok(),
+        object_store_config: ObjectStoreConfig::from_env().ok(),
+    };
+
     // Run core actors.
     let (core_task_handles, stop_sender, cb_receiver, health_check_handle) =
-        initialize_components(components, is_only_oneshot_witness_generator_task)
+        initialize_components(&configs, components, is_only_oneshot_witness_generator_task)
             .await
             .context("Unable to start Core actors")?;
 

core/lib/config/Cargo.toml

@@ -11,15 +11,8 @@ categories = ["cryptography"]
 
 [dependencies]
 zksync_basic_types = { path = "../../lib/basic_types" }
-zksync_utils = { path = "../../lib/utils" }
 zksync_contracts = { path = "../../lib/contracts" }
 
 anyhow = "1.0"
-url = "2.1"
-num = "0.3.1"
 serde = { version = "1.0", features = ["derive"] }
-serde_json = "1.0"
 envy = "0.4"
-once_cell = "1.13.0"
-bigdecimal = "0.2.2"
-hex = "0.4"

core/lib/config/src/configs/alerts.rs

@@ -1,47 +1,8 @@
-// Built-in uses
-// External uses
 use serde::Deserialize;
 
-use super::envy_load;
-
 #[derive(Debug, Deserialize, Clone, PartialEq)]
 pub struct AlertsConfig {
     /// List of panics' messages from external crypto code,
     /// that are sporadic and needed to be handled separately
     pub sporadic_crypto_errors_substrs: Vec<String>,
 }
-
-impl AlertsConfig {
-    pub fn from_env() -> anyhow::Result<Self> {
-        envy_load("sporadic_crypto_errors_substrs", "ALERTS_")
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-    use crate::configs::test_utils::EnvMutex;
-
-    static MUTEX: EnvMutex = EnvMutex::new();
-
-    fn expected_config() -> AlertsConfig {
-        AlertsConfig {
-            sporadic_crypto_errors_substrs: vec![
-                "EventDestroyErr".to_string(),
-                "Can't free memory of DeviceBuf".to_string(),
-                "value: PoisonError".to_string(),
-            ],
-        }
-    }
-
-    #[test]
-    fn test_from_env() {
-        let mut lock = MUTEX.lock();
-        let config = r#"
-            ALERTS_SPORADIC_CRYPTO_ERRORS_SUBSTRS="EventDestroyErr,Can't free memory of DeviceBuf,value: PoisonError"
-        "#;
-        lock.set_env(config);
-
-        assert_eq!(AlertsConfig::from_env().unwrap(), expected_config());
-    }
-}