feat(zk_inception): Add ecosystem ports scanner (#2849)

## What ❔
Added `EcosystemPortsScanner` tool to scan ecosystem directory config
and find all values for ports.
Returns `EcosystemPorts` object with ports found and custom description
of a service.
`EcosystemPorts` also contains methods to: check whether the port is
assigned, allocate a new port in range, etc.

## Why ❔
With multiple chains and number of services (like explorer, etc.)
growing - it is useful to have a tool (`EcosystemPorts`) that can be
used to allocate new ports, thus, lowering the chance of port conflicts
within created configuration.

## Checklist
- [ ] PR title corresponds to the body of PR (we generate changelog
entries from PRs).
- [ ] Tests for the changes have been added / updated.
- [ ] Documentation comments have been added / updated.
- [ ] Code has been formatted via `zk fmt` and `zk lint`.

---------

Co-authored-by: Matías Ignacio González <maigonzalez@fi.uba.ar>

.github/workflows/ci-core-reusable.yml

@@ -248,7 +248,6 @@ jobs:
           --server-db-name=zksync_server_localhost_validium \
           --prover-db-url=postgres://postgres:notsecurepassword@localhost:5432 \
           --prover-db-name=zksync_prover_localhost_validium \
-          --port-offset 2000 \
           --chain validium
 
       - name: Create and initialize chain with Custom Token
@@ -272,7 +271,6 @@ jobs:
           --server-db-name=zksync_server_localhost_custom_token \
           --prover-db-url=postgres://postgres:notsecurepassword@localhost:5432 \
           --prover-db-name=zksync_prover_localhost_custom_token \
-          --port-offset 3000 \
           --chain custom_token
 
       - name: Create and register chain with transactions signed "offline"
@@ -331,7 +329,6 @@ jobs:
           --server-db-name=zksync_server_localhost_consensus \
           --prover-db-url=postgres://postgres:notsecurepassword@localhost:5432 \
           --prover-db-name=zksync_prover_localhost_consensus \
-          --port-offset 4000 \
           --chain consensus
 
       - name: Build test dependencies

etc/env/file_based/general.yaml

@@ -80,7 +80,7 @@ operations_manager:
 contract_verifier:
   compilation_timeout: 240
   polling_interval: 1000
-  prometheus_port: 3314
+  prometheus_port: 3318
   port: 3070
   url: http://127.0.0.1:3070
   threads_per_server: 128

zk_toolbox/crates/config/src/general.rs

@@ -4,14 +4,14 @@ use anyhow::Context;
 use common::yaml::merge_yaml;
 use url::Url;
 use xshell::Shell;
+use zksync_config::configs::object_store::ObjectStoreMode;
 pub use zksync_config::configs::GeneralConfig;
-use zksync_config::configs::{consensus::Host, object_store::ObjectStoreMode};
 use zksync_protobuf_config::{decode_yaml_repr, encode_yaml_repr};
 
 use crate::{
     consts::GENERAL_FILE,
     traits::{ConfigWithL2RpcUrl, FileConfigWithDefaultName, ReadConfig, SaveConfig},
-    ChainConfig, DEFAULT_CONSENSUS_PORT,
+    ChainConfig,
 };
 
 pub struct RocksDbs {
@@ -113,68 +113,6 @@ pub fn set_file_artifacts(config: &mut GeneralConfig, file_artifacts: FileArtifa
     set_artifact_path!(config.core_object_store, file_artifacts.core_object_store);
 }
 
-pub fn ports_config(config: &GeneralConfig) -> Option<PortsConfig> {
-    let api = config.api_config.as_ref()?;
-    let contract_verifier = config.contract_verifier.as_ref()?;
-    let consensus_port = if let Some(consensus_config) = config.clone().consensus_config {
-        consensus_config.server_addr.port()
-    } else {
-        DEFAULT_CONSENSUS_PORT
-    };
-
-    Some(PortsConfig {
-        web3_json_rpc_http_port: api.web3_json_rpc.http_port,
-        web3_json_rpc_ws_port: api.web3_json_rpc.ws_port,
-        healthcheck_port: api.healthcheck.port,
-        merkle_tree_port: api.merkle_tree.port,
-        prometheus_listener_port: api.prometheus.listener_port,
-        contract_verifier_port: contract_verifier.port,
-        consensus_port,
-    })
-}
-
-pub fn update_ports(config: &mut GeneralConfig, ports_config: &PortsConfig) -> anyhow::Result<()> {
-    let api = config
-        .api_config
-        .as_mut()
-        .context("Api config is not presented")?;
-    let contract_verifier = config
-        .contract_verifier
-        .as_mut()
-        .context("Contract Verifier config is not presented")?;
-    let prometheus = config
-        .prometheus_config
-        .as_mut()
-        .context("Prometheus config is not presented")?;
-    if let Some(consensus) = config.consensus_config.as_mut() {
-        consensus.server_addr.set_port(ports_config.consensus_port);
-        update_port_in_host(&mut consensus.public_addr, ports_config.consensus_port)?;
-    }
-
-    api.web3_json_rpc.http_port = ports_config.web3_json_rpc_http_port;
-    update_port_in_url(
-        &mut api.web3_json_rpc.http_url,
-        ports_config.web3_json_rpc_http_port,
-    )?;
-    api.web3_json_rpc.ws_port = ports_config.web3_json_rpc_ws_port;
-    update_port_in_url(
-        &mut api.web3_json_rpc.ws_url,
-        ports_config.web3_json_rpc_ws_port,
-    )?;
-    contract_verifier.port = ports_config.contract_verifier_port;
-    update_port_in_url(
-        &mut contract_verifier.url,
-        ports_config.contract_verifier_port,
-    )?;
-    api.healthcheck.port = ports_config.healthcheck_port;
-    api.merkle_tree.port = ports_config.merkle_tree_port;
-    api.prometheus.listener_port = ports_config.prometheus_listener_port;
-
-    prometheus.listener_port = ports_config.prometheus_listener_port;
-
-    Ok(())
-}
-
 pub fn override_config(shell: &Shell, path: PathBuf, chain: &ChainConfig) -> anyhow::Result<()> {
     let chain_config_path = chain.path_to_general_config();
     let override_config = serde_yaml::from_str(&shell.read_file(path)?)?;
@@ -184,60 +122,10 @@ pub fn override_config(shell: &Shell, path: PathBuf, chain: &ChainConfig) -> any
     Ok(())
 }
 
-fn update_port_in_url(http_url: &mut String, port: u16) -> anyhow::Result<()> {
-    let mut http_url_url = Url::parse(http_url)?;
-    if let Err(()) = http_url_url.set_port(Some(port)) {
-        anyhow::bail!("Wrong url, setting port is impossible");
-    }
-    *http_url = http_url_url.to_string();
-    Ok(())
-}
-
-fn update_port_in_host(host: &mut Host, port: u16) -> anyhow::Result<()> {
-    let url = Url::parse(&format!("http://{}", host.0))?;
-    let host_str = url.host_str().context("Failed to get host")?;
-    host.0 = format!("{host_str}:{port}");
-    Ok(())
-}
-
 impl FileConfigWithDefaultName for GeneralConfig {
     const FILE_NAME: &'static str = GENERAL_FILE;
 }
 
-pub struct PortsConfig {
-    pub web3_json_rpc_http_port: u16,
-    pub web3_json_rpc_ws_port: u16,
-    pub healthcheck_port: u16,
-    pub merkle_tree_port: u16,
-    pub prometheus_listener_port: u16,
-    pub contract_verifier_port: u16,
-    pub consensus_port: u16,
-}
-
-impl PortsConfig {
-    pub fn apply_offset(&mut self, offset: u16) {
-        self.web3_json_rpc_http_port += offset;
-        self.web3_json_rpc_ws_port += offset;
-        self.healthcheck_port += offset;
-        self.merkle_tree_port += offset;
-        self.prometheus_listener_port += offset;
-        self.contract_verifier_port += offset;
-        self.consensus_port += offset;
-    }
-
-    pub fn next_empty_ports_config(&self) -> PortsConfig {
-        Self {
-            web3_json_rpc_http_port: self.web3_json_rpc_http_port + 100,
-            web3_json_rpc_ws_port: self.web3_json_rpc_ws_port + 100,
-            healthcheck_port: self.healthcheck_port + 100,
-            merkle_tree_port: self.merkle_tree_port + 100,
-            prometheus_listener_port: self.prometheus_listener_port + 100,
-            contract_verifier_port: self.contract_verifier_port + 100,
-            consensus_port: self.consensus_port + 100,
-        }
-    }
-}
-
 impl SaveConfig for GeneralConfig {
     fn save(&self, shell: &Shell, path: impl AsRef<Path>) -> anyhow::Result<()> {
         let bytes =

zk_toolbox/crates/zk_inception/src/commands/chain/args/init.rs

@@ -1,5 +1,3 @@
-use std::str::FromStr;
-
 use clap::Parser;
 use common::{forge::ForgeScriptArgs, Prompt};
 use config::ChainConfig;
@@ -13,35 +11,10 @@ use crate::{
     defaults::LOCAL_RPC_URL,
     messages::{
         MSG_DEPLOY_PAYMASTER_PROMPT, MSG_GENESIS_ARGS_HELP, MSG_L1_RPC_URL_HELP,
-        MSG_L1_RPC_URL_INVALID_ERR, MSG_L1_RPC_URL_PROMPT, MSG_PORT_OFFSET_HELP,
+        MSG_L1_RPC_URL_INVALID_ERR, MSG_L1_RPC_URL_PROMPT, MSG_NO_PORT_REALLOCATION_HELP,
     },
 };
 
-#[derive(Clone, Debug, Serialize, Deserialize)]
-pub struct PortOffset(u16);
-
-impl PortOffset {
-    pub fn from_chain_id(chain_id: u16) -> Self {
-        Self((chain_id - 1) * 100)
-    }
-}
-
-impl FromStr for PortOffset {
-    type Err = String;
-
-    fn from_str(s: &str) -> Result<Self, Self::Err> {
-        s.parse::<u16>()
-            .map(PortOffset)
-            .map_err(|_| "Invalid port offset".to_string())
-    }
-}
-
-impl From<PortOffset> for u16 {
-    fn from(port_offset: PortOffset) -> Self {
-        port_offset.0
-    }
-}
-
 #[derive(Debug, Clone, Serialize, Deserialize, Parser)]
 pub struct InitArgs {
     /// All ethereum environment related arguments
@@ -55,8 +28,8 @@ pub struct InitArgs {
     pub deploy_paymaster: Option<bool>,
     #[clap(long, help = MSG_L1_RPC_URL_HELP)]
     pub l1_rpc_url: Option<String>,
-    #[clap(long, help = MSG_PORT_OFFSET_HELP)]
-    pub port_offset: Option<PortOffset>,
+    #[clap(long, help = MSG_NO_PORT_REALLOCATION_HELP, default_value = "false", default_missing_value = "true", num_args = 0..=1)]
+    pub no_port_reallocation: bool,
 }
 
 impl InitArgs {
@@ -86,10 +59,7 @@ impl InitArgs {
             genesis_args: self.genesis_args.fill_values_with_prompt(config),
             deploy_paymaster,
             l1_rpc_url,
-            port_offset: self
-                .port_offset
-                .unwrap_or(PortOffset::from_chain_id(config.id as u16))
-                .into(),
+            no_port_reallocation: self.no_port_reallocation,
         }
     }
 }
@@ -100,5 +70,5 @@ pub struct InitArgsFinal {
     pub genesis_args: GenesisArgsFinal,
     pub deploy_paymaster: bool,
     pub l1_rpc_url: String,
-    pub port_offset: u16,
+    pub no_port_reallocation: bool,
 }

zk_toolbox/crates/zk_inception/src/commands/chain/init.rs

@@ -1,8 +1,8 @@
-use anyhow::{bail, Context};
+use anyhow::Context;
 use common::{config::global_config, git, logger, spinner::Spinner};
 use config::{
-    copy_configs, ports_config, set_l1_rpc_url, traits::SaveConfigWithBasePath,
-    update_from_chain_config, update_ports, ChainConfig, EcosystemConfig, GeneralConfig,
+    copy_configs, set_l1_rpc_url, traits::SaveConfigWithBasePath, update_from_chain_config,
+    ChainConfig, EcosystemConfig, DEFAULT_CONSENSUS_PORT,
 };
 use types::BaseToken;
 use xshell::Shell;
@@ -20,14 +20,17 @@ use crate::{
         },
         portal::update_portal_config,
     },
+    defaults::PORT_RANGE_END,
     messages::{
         msg_initializing_chain, MSG_ACCEPTING_ADMIN_SPINNER, MSG_CHAIN_INITIALIZED,
         MSG_CHAIN_NOT_FOUND_ERR, MSG_DEPLOYING_PAYMASTER, MSG_GENESIS_DATABASE_ERR,
-        MSG_PORTAL_FAILED_TO_CREATE_CONFIG_ERR, MSG_PORTS_CONFIG_ERR,
-        MSG_REGISTERING_CHAIN_SPINNER, MSG_SELECTED_CONFIG,
+        MSG_PORTAL_FAILED_TO_CREATE_CONFIG_ERR, MSG_REGISTERING_CHAIN_SPINNER, MSG_SELECTED_CONFIG,
         MSG_UPDATING_TOKEN_MULTIPLIER_SETTER_SPINNER, MSG_WALLET_TOKEN_MULTIPLIER_SETTER_NOT_FOUND,
     },
-    utils::consensus::{generate_consensus_keys, get_consensus_config, get_consensus_secrets},
+    utils::{
+        consensus::{generate_consensus_keys, get_consensus_config, get_consensus_secrets},
+        ports::EcosystemPortsScanner,
+    },
 };
 
 pub(crate) async fn run(args: InitArgs, shell: &Shell) -> anyhow::Result<()> {
@@ -54,15 +57,31 @@ pub async fn init(
     ecosystem_config: &EcosystemConfig,
     chain_config: &ChainConfig,
 ) -> anyhow::Result<()> {
+    let mut ecosystem_ports = EcosystemPortsScanner::scan(shell)?;
     copy_configs(shell, &ecosystem_config.link_to_code, &chain_config.configs)?;
 
+    if !init_args.no_port_reallocation {
+        ecosystem_ports.allocate_ports_in_yaml(
+            shell,
+            &chain_config.path_to_general_config(),
+            chain_config.id,
+        )?;
+    }
     let mut general_config = chain_config.get_general_config()?;
-    apply_port_offset(init_args.port_offset, &mut general_config)?;
-    let ports = ports_config(&general_config).context(MSG_PORTS_CONFIG_ERR)?;
+
+    // TODO: This is a temporary solution. We should allocate consensus port using `EcosystemPorts::allocate_ports_in_yaml`
+    let offset = ((chain_config.id - 1) * 100) as u16;
+    let consensus_port_range = DEFAULT_CONSENSUS_PORT + offset..PORT_RANGE_END;
+    let consensus_port =
+        ecosystem_ports.allocate_port(consensus_port_range, "Consensus".to_string())?;
 
     let consensus_keys = generate_consensus_keys();
-    let consensus_config =
-        get_consensus_config(chain_config, ports, Some(consensus_keys.clone()), None)?;
+    let consensus_config = get_consensus_config(
+        chain_config,
+        consensus_port,
+        Some(consensus_keys.clone()),
+        None,
+    )?;
     general_config.consensus_config = Some(consensus_config);
     general_config.save_with_base_path(shell, &chain_config.configs)?;
 
@@ -176,15 +195,3 @@ pub async fn init(
 
     Ok(())
 }
-
-fn apply_port_offset(port_offset: u16, general_config: &mut GeneralConfig) -> anyhow::Result<()> {
-    let Some(mut ports_config) = ports_config(general_config) else {
-        bail!("Missing ports config");
-    };
-
-    ports_config.apply_offset(port_offset);
-
-    update_ports(general_config, &ports_config)?;
-
-    Ok(())
-}

zk_toolbox/crates/zk_inception/src/commands/ecosystem/args/init.rs

@@ -12,7 +12,8 @@ use crate::{
     messages::{
         MSG_DEPLOY_ECOSYSTEM_PROMPT, MSG_DEPLOY_ERC20_PROMPT, MSG_DEPLOY_PAYMASTER_PROMPT,
         MSG_DEV_ARG_HELP, MSG_GENESIS_ARGS_HELP, MSG_L1_RPC_URL_HELP, MSG_L1_RPC_URL_INVALID_ERR,
-        MSG_L1_RPC_URL_PROMPT, MSG_OBSERVABILITY_HELP, MSG_OBSERVABILITY_PROMPT,
+        MSG_L1_RPC_URL_PROMPT, MSG_NO_PORT_REALLOCATION_HELP, MSG_OBSERVABILITY_HELP,
+        MSG_OBSERVABILITY_PROMPT,
     },
 };
 
@@ -92,6 +93,8 @@ pub struct EcosystemInitArgs {
     pub dev: bool,
     #[clap(long, short = 'o', help = MSG_OBSERVABILITY_HELP, default_missing_value = "true", num_args = 0..=1)]
     pub observability: Option<bool>,
+    #[clap(long, help = MSG_NO_PORT_REALLOCATION_HELP, default_value = "false", default_missing_value = "true", num_args = 0..=1)]
+    pub no_port_reallocation: bool,
 }
 
 impl EcosystemInitArgs {
@@ -129,6 +132,7 @@ impl EcosystemInitArgs {
             forge_args: self.forge_args.clone(),
             dev: self.dev,
             observability,
+            no_port_reallocation: self.no_port_reallocation,
         }
     }
 }
@@ -141,4 +145,5 @@ pub struct EcosystemInitArgsFinal {
     pub forge_args: ForgeScriptArgs,
     pub dev: bool,
     pub observability: bool,
+    pub no_port_reallocation: bool,
 }

zk_toolbox/crates/zk_inception/src/commands/ecosystem/init.rs

@@ -31,7 +31,7 @@ use super::{
 use crate::{
     accept_ownership::{accept_admin, accept_owner},
     commands::{
-        chain::{self, args::init::PortOffset},
+        chain::{self},
         ecosystem::create_configs::{
             create_erc20_deployment_config, create_initial_deployments_config,
         },
@@ -112,7 +112,7 @@ pub async fn run(args: EcosystemInitArgs, shell: &Shell) -> anyhow::Result<()> {
             genesis_args: genesis_args.clone().fill_values_with_prompt(&chain_config),
             deploy_paymaster: final_ecosystem_args.deploy_paymaster,
             l1_rpc_url: final_ecosystem_args.ecosystem.l1_rpc_url.clone(),
-            port_offset: PortOffset::from_chain_id(chain_config.id as u16).into(),
+            no_port_reallocation: final_ecosystem_args.no_port_reallocation,
         };
 
         chain::init::init(