From 3f18964d2286a0770e0462c227830d0b283b7ab4 Mon Sep 17 00:00:00 2001 From: Alexander Melnikov Date: Wed, 28 Aug 2024 02:03:08 -0600 Subject: [PATCH 01/12] zk_inception explorer support --- zk_toolbox/crates/common/src/docker.rs | 9 +- zk_toolbox/crates/config/src/apps.rs | 67 ++++ zk_toolbox/crates/config/src/consts.rs | 19 +- .../crates/config/src/docker_compose.rs | 43 +++ zk_toolbox/crates/config/src/explorer.rs | 111 +++++++ .../crates/config/src/explorer_compose.rs | 285 ++++++++++++++++++ zk_toolbox/crates/config/src/lib.rs | 5 + zk_toolbox/crates/config/src/portal.rs | 56 ++-- .../zk_inception/src/commands/args/mod.rs | 2 - .../zk_inception/src/commands/args/portal.rs | 12 - .../zk_inception/src/commands/chain/init.rs | 4 +- .../zk_inception/src/commands/containers.rs | 2 +- .../src/commands/ecosystem/create.rs | 6 +- .../src/commands/ecosystem/create_configs.rs | 12 +- .../zk_inception/src/commands/explorer.rs | 239 +++++++++++++++ .../crates/zk_inception/src/commands/mod.rs | 1 + .../zk_inception/src/commands/portal.rs | 113 +++---- .../crates/zk_inception/src/defaults.rs | 15 + zk_toolbox/crates/zk_inception/src/main.rs | 14 +- .../crates/zk_inception/src/messages.rs | 21 +- .../crates/zk_inception/src/utils/mod.rs | 1 + .../crates/zk_inception/src/utils/ports.rs | 162 ++++++++++ 22 files changed, 1095 insertions(+), 104 deletions(-) create mode 100644 zk_toolbox/crates/config/src/apps.rs create mode 100644 zk_toolbox/crates/config/src/docker_compose.rs create mode 100644 zk_toolbox/crates/config/src/explorer.rs create mode 100644 zk_toolbox/crates/config/src/explorer_compose.rs delete mode 100644 zk_toolbox/crates/zk_inception/src/commands/args/portal.rs create mode 100644 zk_toolbox/crates/zk_inception/src/commands/explorer.rs create mode 100644 zk_toolbox/crates/zk_inception/src/utils/ports.rs diff --git a/zk_toolbox/crates/common/src/docker.rs b/zk_toolbox/crates/common/src/docker.rs index 0ca31383f9cc..338b4885aeab 100644 --- a/zk_toolbox/crates/common/src/docker.rs +++ b/zk_toolbox/crates/common/src/docker.rs @@ -4,8 +4,13 @@ use xshell::{cmd, Shell}; use crate::cmd::Cmd; -pub fn up(shell: &Shell, docker_compose_file: &str) -> anyhow::Result<()> { - Ok(Cmd::new(cmd!(shell, "docker compose -f {docker_compose_file} up -d")).run()?) +pub fn up(shell: &Shell, docker_compose_file: &str, detach: bool) -> anyhow::Result<()> { + let args = if detach { vec!["-d"] } else { vec![] }; + Ok(Cmd::new(cmd!( + shell, + "docker compose -f {docker_compose_file} up {args...}" + )) + .run()?) } pub fn down(shell: &Shell, docker_compose_file: &str) -> anyhow::Result<()> { diff --git a/zk_toolbox/crates/config/src/apps.rs b/zk_toolbox/crates/config/src/apps.rs new file mode 100644 index 000000000000..76203cbf8aa1 --- /dev/null +++ b/zk_toolbox/crates/config/src/apps.rs @@ -0,0 +1,67 @@ +use std::path::{Path, PathBuf}; + +use serde::{Deserialize, Serialize}; +use xshell::Shell; + +use crate::{ + consts::{APPS_CONFIG_FILE, LOCAL_CONFIGS_PATH}, + traits::{FileConfigWithDefaultName, ReadConfig, SaveConfig, ZkToolboxConfig}, +}; + +pub const DEFAULT_EXPLORER_PORT: u16 = 3010; +pub const DEFAULT_PORTAL_PORT: u16 = 3030; + +/// Ecosystem level configuration for the apps (portal and explorer). +#[derive(Debug, Serialize, Deserialize, Clone)] +pub struct AppsEcosystemConfig { + pub portal: AppEcosystemConfig, + pub explorer: AppEcosystemConfig, +} + +#[derive(Debug, Serialize, Deserialize, Clone)] +pub struct AppEcosystemConfig { + pub http_port: u16, + pub http_url: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub chains_enabled: Option>, +} + +impl ZkToolboxConfig for AppsEcosystemConfig {} +impl FileConfigWithDefaultName for AppsEcosystemConfig { + const FILE_NAME: &'static str = APPS_CONFIG_FILE; +} + +impl AppsEcosystemConfig { + pub fn default() -> Self { + AppsEcosystemConfig { + portal: AppEcosystemConfig { + http_port: DEFAULT_PORTAL_PORT, + http_url: format!("http://127.0.0.1:{}", DEFAULT_PORTAL_PORT), + chains_enabled: None, + }, + explorer: AppEcosystemConfig { + http_port: DEFAULT_EXPLORER_PORT, + http_url: format!("http://127.0.0.1:{}", DEFAULT_EXPLORER_PORT), + chains_enabled: None, + }, + } + } + + pub fn get_config_path(ecosystem_base_path: &Path) -> PathBuf { + ecosystem_base_path + .join(LOCAL_CONFIGS_PATH) + .join(APPS_CONFIG_FILE) + } + + pub fn read_or_create_default(shell: &Shell) -> anyhow::Result { + let config_path = Self::get_config_path(&shell.current_dir()); + match Self::read(shell, &config_path) { + Ok(config) => Ok(config), + Err(_) => { + let config = Self::default(); + config.save(shell, &config_path)?; + Ok(config) + } + } + } +} diff --git a/zk_toolbox/crates/config/src/consts.rs b/zk_toolbox/crates/config/src/consts.rs index 4de534b816d5..70fd4e659d09 100644 --- a/zk_toolbox/crates/config/src/consts.rs +++ b/zk_toolbox/crates/config/src/consts.rs @@ -30,11 +30,26 @@ pub const ERA_OBSERVABILITY_COMPOSE_FILE: &str = "era-observability/docker-compo pub const ERA_OBSERBAVILITY_DIR: &str = "era-observability"; /// Era observability repo link pub const ERA_OBSERBAVILITY_GIT_REPO: &str = "https://github.com/matter-labs/era-observability"; +pub(crate) const LOCAL_APPS_PATH: &str = "apps/"; +pub(crate) const LOCAL_CHAINS_PATH: &str = "chains/"; pub(crate) const LOCAL_CONFIGS_PATH: &str = "configs/"; +pub(crate) const LOCAL_GENERATED_PATH: &str = ".generated/"; pub(crate) const LOCAL_DB_PATH: &str = "db/"; -/// Name of portal config file -pub const PORTAL_CONFIG_FILE: &str = "portal.config.js"; +/// Name of apps config file +pub const APPS_CONFIG_FILE: &str = "apps.yaml"; +/// Name of portal runtime config file (auto-generated) +pub const PORTAL_RUNTIME_CONFIG_FILE: &str = "portal.config.js"; +/// Name of chain-level portal config JSON file +pub const PORTAL_CHAIN_CONFIG_FILE: &str = "portal.config.json"; +/// Name of explorer runtime config file (auto-generated) +pub const EXPLORER_RUNTIME_CONFIG_FILE: &str = "explorer.config.js"; +/// Name of chain-level explorer config JSON file +pub const EXPLORER_CHAIN_CONFIG_FILE: &str = "explorer.config.json"; +/// Name of explorer docker compose file (auto-generated) +pub const EXPLORER_DOCKER_COMPOSE_FILE: &str = "explorer-docker-compose.yml"; +/// Name of chain-level explorer backend docker compose file +pub const EXPLORER_BACKEND_DOCKER_COMPOSE_FILE: &str = "explorer-backend-docker-compose.yml"; /// Path to ecosystem contacts pub(crate) const ECOSYSTEM_PATH: &str = "etc/env/ecosystems"; diff --git a/zk_toolbox/crates/config/src/docker_compose.rs b/zk_toolbox/crates/config/src/docker_compose.rs new file mode 100644 index 000000000000..6aab16ba1a16 --- /dev/null +++ b/zk_toolbox/crates/config/src/docker_compose.rs @@ -0,0 +1,43 @@ +use std::collections::HashMap; + +use serde::{Deserialize, Serialize}; + +use crate::traits::ZkToolboxConfig; + +#[derive(Debug, Serialize, Deserialize, Clone)] +pub struct DockerComposeConfig { + pub services: HashMap, +} + +#[derive(Debug, Serialize, Deserialize, Clone)] +pub struct DockerComposeService { + pub image: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub platform: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub ports: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + pub environment: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + pub volumes: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + pub depends_on: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + pub restart: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub extra_hosts: Option>, +} + +impl ZkToolboxConfig for DockerComposeConfig {} + +impl DockerComposeConfig { + pub fn new() -> Self { + DockerComposeConfig { + services: HashMap::new(), + } + } + + pub fn add_service(&mut self, name: &str, service: DockerComposeService) { + self.services.insert(name.to_string(), service); + } +} diff --git a/zk_toolbox/crates/config/src/explorer.rs b/zk_toolbox/crates/config/src/explorer.rs new file mode 100644 index 000000000000..6a55f791a9c7 --- /dev/null +++ b/zk_toolbox/crates/config/src/explorer.rs @@ -0,0 +1,111 @@ +use std::path::{Path, PathBuf}; + +use serde::{Deserialize, Serialize}; +use xshell::Shell; + +use crate::{ + consts::{ + EXPLORER_CHAIN_CONFIG_FILE, EXPLORER_RUNTIME_CONFIG_FILE, LOCAL_APPS_PATH, + LOCAL_CHAINS_PATH, LOCAL_CONFIGS_PATH, LOCAL_GENERATED_PATH, + }, + traits::{ReadConfig, SaveConfig, ZkToolboxConfig}, +}; + +/// Explorer configuration file. This file is auto-generated during the "explorer" command +/// and is used to inject the runtime configuration into the explorer app. +#[derive(Serialize, Deserialize, Debug, Clone)] +#[serde(rename_all = "camelCase")] +pub struct ExplorerRuntimeConfig { + pub app_environment: String, + pub environment_config: EnvironmentConfig, +} + +#[derive(Serialize, Deserialize, Debug, Clone)] +pub struct EnvironmentConfig { + pub networks: Vec, +} + +/// Explorer chain configuration file. This file is created on the chain level +/// and is used to configure the explorer for a specific chain. It serves as a building block for +/// the explorer runtime config. +#[derive(Serialize, Deserialize, Debug, Clone)] +#[serde(rename_all = "camelCase")] +pub struct ExplorerChainConfig { + pub name: String, // L2 network chain name (the one used during the chain initialization) + pub l2_network_name: String, // How the network is displayed in the app dropdown + pub l2_chain_id: u64, + pub rpc_url: String, // L2 RPC URL + pub api_url: String, // L2 API URL + pub base_token_address: String, // L2 base token address (currently always 0x800A) + pub hostnames: Vec, // Custom domain to use when switched to this chain in the app + pub icon: String, // Icon to show in the explorer dropdown + pub maintenance: bool, // Maintenance warning + pub published: bool, // If false, the chain will not be shown in the explorer dropdown + #[serde(skip_serializing_if = "Option::is_none")] + pub bridge_url: Option, // Link to the portal bridge + #[serde(skip_serializing_if = "Option::is_none")] + pub l1_explorer_url: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub verification_api_url: Option, // L2 verification API URL +} + +impl ExplorerRuntimeConfig { + pub fn new(explorer_chain_configs: Vec) -> Self { + Self { + app_environment: "default".to_string(), + environment_config: EnvironmentConfig { + networks: explorer_chain_configs, + }, + } + } + + pub fn get_config_path(ecosystem_base_path: &Path) -> PathBuf { + ecosystem_base_path + .join(LOCAL_CONFIGS_PATH) + .join(LOCAL_GENERATED_PATH) + .join(EXPLORER_RUNTIME_CONFIG_FILE) + } +} + +impl SaveConfig for ExplorerRuntimeConfig { + fn save(&self, shell: &Shell, path: impl AsRef) -> anyhow::Result<()> { + // The block-explorer-app is served as a pre-built static app in a Docker image. + // It uses a JavaScript file (config.js) that injects the configuration at runtime + // by overwriting the '##runtimeConfig' property of the window object. + // Therefore, we generate a JavaScript file instead of a JSON file. + // This file will be mounted to the Docker image when it runs. + let json = serde_json::to_string_pretty(&self)?; + let config_js_content = format!("window['##runtimeConfig'] = {};", json); + Ok(shell.write_file(path, config_js_content.as_bytes())?) + } +} + +impl ReadConfig for ExplorerRuntimeConfig { + fn read(shell: &Shell, path: impl AsRef) -> anyhow::Result { + let config_js_content = shell.read_file(path)?; + // Extract the JSON part from the JavaScript file + let json_start = config_js_content + .find('{') + .ok_or_else(|| anyhow::anyhow!("Invalid config file format"))?; + let json_end = config_js_content + .rfind('}') + .ok_or_else(|| anyhow::anyhow!("Invalid config file format"))?; + let json_str = &config_js_content[json_start..=json_end]; + // Parse the JSON into ExplorerRuntimeConfig + let config: ExplorerRuntimeConfig = serde_json::from_str(json_str)?; + Ok(config) + } +} + +impl ExplorerChainConfig { + pub fn get_config_path(ecosystem_base_path: &Path, chain_name: &str) -> PathBuf { + ecosystem_base_path + .join(LOCAL_CHAINS_PATH) + .join(chain_name) + .join(LOCAL_CONFIGS_PATH) + .join(LOCAL_APPS_PATH) + .join(EXPLORER_CHAIN_CONFIG_FILE) + } +} + +impl ZkToolboxConfig for ExplorerChainConfig {} diff --git a/zk_toolbox/crates/config/src/explorer_compose.rs b/zk_toolbox/crates/config/src/explorer_compose.rs new file mode 100644 index 000000000000..eb2743b62f90 --- /dev/null +++ b/zk_toolbox/crates/config/src/explorer_compose.rs @@ -0,0 +1,285 @@ +use std::{ + collections::HashMap, + path::{Path, PathBuf}, +}; + +use anyhow::Context; +use serde::{Deserialize, Serialize}; +use url::Url; + +use crate::{ + consts::{ + EXPLORER_BACKEND_DOCKER_COMPOSE_FILE, EXPLORER_DOCKER_COMPOSE_FILE, LOCAL_APPS_PATH, + LOCAL_CHAINS_PATH, LOCAL_CONFIGS_PATH, LOCAL_GENERATED_PATH, + }, + docker_compose::{DockerComposeConfig, DockerComposeService}, + traits::ZkToolboxConfig, +}; + +/// Explorer docker compose file. This file is auto-generated during "explorer" command +/// and is passed to Docker Compose to launch the explorer app and backend services. +#[derive(Debug, Serialize, Deserialize, Clone)] +pub struct ExplorerComposeConfig { + #[serde(flatten)] + pub docker_compose: DockerComposeConfig, +} + +/// Chain-level explorer backend docker compose file. This file is created when the explorer is +/// initialized for the chain. It contains the configuration for the chain explorer backend +/// services and serves as a building block for the main explorer docker compose file. +#[derive(Debug, Serialize, Deserialize, Clone)] +pub struct ExplorerBackendComposeConfig { + #[serde(flatten)] + pub docker_compose: DockerComposeConfig, +} + +impl ZkToolboxConfig for ExplorerComposeConfig {} +impl ZkToolboxConfig for ExplorerBackendComposeConfig {} + +#[derive(Debug, Clone)] +pub struct ExplorerAppServiceConfig { + pub port: u16, + pub config_path: PathBuf, +} + +#[derive(Debug, Clone)] +pub struct ExplorerBackendServiceConfig { + pub db_url: String, + pub l2_rpc_url: String, + pub service_ports: ExplorerBackendServicePorts, +} + +#[derive(Debug, Clone)] +pub struct ExplorerBackendServicePorts { + pub api_port: u16, + pub data_fetcher_port: u16, + pub worker_port: u16, +} + +impl ExplorerComposeConfig { + pub fn new( + app_config: ExplorerAppServiceConfig, + backend_configs: Vec, + ) -> anyhow::Result { + let mut services = HashMap::new(); + let mut app_depends_on = Vec::new(); + + // Add services from backend configs + for backend_config in backend_configs.iter() { + for (service_name, service) in &backend_config.docker_compose.services { + if service.image.contains("block-explorer-api") { + app_depends_on.push(service_name.clone()); + } + services.insert(service_name.clone(), service.clone()); + } + } + + services.insert( + "explorer-app".to_string(), + Self::create_app_service(app_config, Some(app_depends_on)), + ); + + let config = Self { + docker_compose: DockerComposeConfig { services }, + }; + Ok(config) + } + + fn create_app_service( + app_config: ExplorerAppServiceConfig, + depends_on: Option>, + ) -> DockerComposeService { + DockerComposeService { + image: "matterlabs/block-explorer-app".to_string(), + platform: Some("linux/amd64".to_string()), + ports: Some(vec![format!("{}:3010", app_config.port)]), + volumes: Some(vec![format!( + "{}:/usr/src/app/packages/app/dist/config.js", + app_config.config_path.display() + )]), + depends_on, + restart: None, + environment: None, + extra_hosts: None, + } + } + + pub fn get_config_path(ecosystem_base_path: &Path) -> PathBuf { + ecosystem_base_path + .join(LOCAL_CONFIGS_PATH) + .join(LOCAL_GENERATED_PATH) + .join(EXPLORER_DOCKER_COMPOSE_FILE) + } +} + +impl ExplorerBackendComposeConfig { + pub fn new(chain_name: &str, config: ExplorerBackendServiceConfig) -> anyhow::Result { + let config = config.adjust_for_docker()?; + + // Parse database URL + let db_url = Url::parse(&config.db_url).context("Failed to parse database URL")?; + let db_host = db_url + .host_str() + .context("Failed to parse database URL: no host")? + .to_string(); + let db_user = db_url.username().to_string(); + let db_password = db_url + .password() + .context("Failed to parse database URL: no password")? + .to_string(); + let db_name = db_url.path().trim_start_matches('/').to_string(); + + let mut services: HashMap = HashMap::new(); + services.insert( + format!("explorer-api-{}", chain_name), + Self::create_api_service(chain_name, config.service_ports.api_port, &config.db_url), + ); + services.insert( + format!("explorer-data-fetcher-{}", chain_name), + Self::create_data_fetcher_service( + config.service_ports.data_fetcher_port, + &config.l2_rpc_url, + ), + ); + services.insert( + format!("explorer-worker-{}", chain_name), + Self::create_worker_service( + chain_name, + config.service_ports.worker_port, + config.service_ports.data_fetcher_port, + &config.l2_rpc_url, + &db_host, + &db_user, + &db_password, + &db_name, + ), + ); + + Ok(Self { + docker_compose: DockerComposeConfig { services }, + }) + } + + fn create_api_service(chain_name: &str, port: u16, db_url: &str) -> DockerComposeService { + DockerComposeService { + image: "matterlabs/block-explorer-api".to_string(), + platform: Some("linux/amd64".to_string()), + ports: Some(vec![format!("{}:{}", port, port)]), + volumes: None, + depends_on: Some(vec![format!("explorer-worker-{}", chain_name)]), + restart: None, + environment: Some(HashMap::from([ + ("PORT".to_string(), port.to_string()), + ("LOG_LEVEL".to_string(), "verbose".to_string()), + ("NODE_ENV".to_string(), "development".to_string()), + ("DATABASE_URL".to_string(), db_url.to_string()), + ])), + extra_hosts: Some(vec!["host.docker.internal:host-gateway".to_string()]), + } + } + + fn create_data_fetcher_service(port: u16, l2_rpc_url: &str) -> DockerComposeService { + DockerComposeService { + image: "matterlabs/block-explorer-data-fetcher".to_string(), + platform: Some("linux/amd64".to_string()), + ports: Some(vec![format!("{}:{}", port, port)]), + volumes: None, + depends_on: None, + restart: None, + environment: Some(HashMap::from([ + ("PORT".to_string(), port.to_string()), + ("LOG_LEVEL".to_string(), "verbose".to_string()), + ("NODE_ENV".to_string(), "development".to_string()), + ("BLOCKCHAIN_RPC_URL".to_string(), l2_rpc_url.to_string()), + ])), + extra_hosts: Some(vec!["host.docker.internal:host-gateway".to_string()]), + } + } + + fn create_worker_service( + chain_name: &str, + port: u16, + data_fetcher_port: u16, + l2_rpc_url: &str, + db_host: &str, + db_user: &str, + db_password: &str, + db_name: &str, + ) -> DockerComposeService { + let data_fetcher_url = format!( + "http://explorer-data-fetcher-{}:{}", + chain_name, data_fetcher_port + ); + DockerComposeService { + image: "matterlabs/block-explorer-worker".to_string(), + platform: Some("linux/amd64".to_string()), + ports: None, + volumes: None, + depends_on: None, + restart: None, + environment: Some(HashMap::from([ + ("PORT".to_string(), port.to_string()), + ("LOG_LEVEL".to_string(), "verbose".to_string()), + ("NODE_ENV".to_string(), "development".to_string()), + ("DATABASE_HOST".to_string(), db_host.to_string()), + ("DATABASE_USER".to_string(), db_user.to_string()), + ("DATABASE_PASSWORD".to_string(), db_password.to_string()), + ("DATABASE_NAME".to_string(), db_name.to_string()), + ("BLOCKCHAIN_RPC_URL".to_string(), l2_rpc_url.to_string()), + ("DATA_FETCHER_URL".to_string(), data_fetcher_url), + ( + "BATCHES_PROCESSING_POLLING_INTERVAL".to_string(), + "1000".to_string(), + ), + ])), + extra_hosts: Some(vec!["host.docker.internal:host-gateway".to_string()]), + } + } + + pub fn get_config_path(ecosystem_base_path: &Path, chain_name: &str) -> PathBuf { + ecosystem_base_path + .join(LOCAL_CHAINS_PATH) + .join(chain_name) + .join(LOCAL_CONFIGS_PATH) + .join(LOCAL_APPS_PATH) + .join(EXPLORER_BACKEND_DOCKER_COMPOSE_FILE) + } +} + +impl ExplorerBackendServiceConfig { + fn is_localhost(host: &str) -> bool { + host == "localhost" || host == "127.0.0.1" || host == "[::1]" + } + + pub fn adjust_for_docker(&self) -> anyhow::Result { + // Parse database URL + let mut db_url = Url::parse(&self.db_url).context("Failed to parse database URL")?; + let db_host = db_url + .host_str() + .context("Failed to parse database URL: no host")? + .to_string(); + + // Replace localhost with host.docker.internal for database URL + if Self::is_localhost(&db_host) { + db_url.set_host(Some("host.docker.internal"))?; + } + + // Parse L2 RPC URL + let mut l2_rpc_url = Url::parse(&self.l2_rpc_url).context("Failed to parse L2 RPC URL")?; + let l2_host = l2_rpc_url + .host_str() + .context("Failed to parse L2 RPC URL: no host")? + .to_string(); + + // Replace localhost with host.docker.internal for L2 RPC URL + if Self::is_localhost(&l2_host) { + l2_rpc_url.set_host(Some("host.docker.internal"))?; + } + + Ok(Self { + db_url: db_url.to_string(), + l2_rpc_url: l2_rpc_url.to_string(), + service_ports: self.service_ports.clone(), + }) + } +} diff --git a/zk_toolbox/crates/config/src/lib.rs b/zk_toolbox/crates/config/src/lib.rs index 4e00962229bc..3c7443f24490 100644 --- a/zk_toolbox/crates/config/src/lib.rs +++ b/zk_toolbox/crates/config/src/lib.rs @@ -1,3 +1,4 @@ +pub use apps::*; pub use chain::*; pub use consts::*; pub use contracts::*; @@ -11,6 +12,7 @@ pub use wallet_creation::*; pub use wallets::*; pub use zksync_protobuf_config::{decode_yaml_repr, encode_yaml_repr}; +mod apps; mod chain; mod consts; mod contracts; @@ -23,6 +25,9 @@ mod secrets; mod wallet_creation; mod wallets; +pub mod docker_compose; +pub mod explorer; +pub mod explorer_compose; pub mod external_node; pub mod forge_interface; pub mod portal; diff --git a/zk_toolbox/crates/config/src/portal.rs b/zk_toolbox/crates/config/src/portal.rs index 4b68d5744cd9..78d9acfc43de 100644 --- a/zk_toolbox/crates/config/src/portal.rs +++ b/zk_toolbox/crates/config/src/portal.rs @@ -5,28 +5,27 @@ use types::TokenInfo; use xshell::Shell; use crate::{ - consts::{LOCAL_CONFIGS_PATH, PORTAL_CONFIG_FILE}, - traits::{FileConfigWithDefaultName, ReadConfig, SaveConfig}, + consts::{ + LOCAL_APPS_PATH, LOCAL_CHAINS_PATH, LOCAL_CONFIGS_PATH, LOCAL_GENERATED_PATH, + PORTAL_CHAIN_CONFIG_FILE, PORTAL_RUNTIME_CONFIG_FILE, + }, + traits::{ReadConfig, SaveConfig, ZkToolboxConfig}, }; +/// Portal configuration file. This file is auto-generated during the "portal" command +/// and is used to inject the runtime configuration into the portal app. #[derive(Serialize, Deserialize, Debug, Clone)] #[serde(rename_all = "camelCase")] pub struct PortalRuntimeConfig { pub node_type: String, - pub hyperchains_config: HyperchainsConfig, + pub hyperchains_config: Vec, } +/// Portal chain configuration file. This file is created on the chain level +/// and is used to configure the portal for a specific chain. It serves as a building block for +/// the portal runtime config. #[derive(Serialize, Deserialize, Debug, Clone)] -pub struct HyperchainsConfig(pub Vec); - -impl HyperchainsConfig { - pub fn is_empty(&self) -> bool { - self.0.is_empty() - } -} - -#[derive(Serialize, Deserialize, Debug, Clone)] -pub struct HyperchainConfig { +pub struct PortalChainConfig { pub network: NetworkConfig, pub tokens: Vec, } @@ -35,8 +34,8 @@ pub struct HyperchainConfig { #[serde(rename_all = "camelCase")] pub struct NetworkConfig { pub id: u64, // L2 Network ID - pub key: String, // L2 Network key - pub name: String, // L2 Network name + pub key: String, // L2 Network key (chain name used during the initialization) + pub name: String, // L2 Network name (displayed in the app dropdown) pub rpc_url: String, // L2 RPC URL #[serde(skip_serializing_if = "Option::is_none")] pub block_explorer_url: Option, // L2 Block Explorer URL @@ -82,17 +81,21 @@ pub struct TokenConfig { } impl PortalRuntimeConfig { + pub fn new(portal_chain_configs: Vec) -> Self { + Self { + node_type: "hyperchain".to_string(), + hyperchains_config: portal_chain_configs, + } + } + pub fn get_config_path(ecosystem_base_path: &Path) -> PathBuf { ecosystem_base_path .join(LOCAL_CONFIGS_PATH) - .join(PORTAL_CONFIG_FILE) + .join(LOCAL_GENERATED_PATH) + .join(PORTAL_RUNTIME_CONFIG_FILE) } } -impl FileConfigWithDefaultName for PortalRuntimeConfig { - const FILE_NAME: &'static str = PORTAL_CONFIG_FILE; -} - impl SaveConfig for PortalRuntimeConfig { fn save(&self, shell: &Shell, path: impl AsRef) -> anyhow::Result<()> { // The dapp-portal is served as a pre-built static app in a Docker image. @@ -122,3 +125,16 @@ impl ReadConfig for PortalRuntimeConfig { Ok(config) } } + +impl PortalChainConfig { + pub fn get_config_path(ecosystem_base_path: &Path, chain_name: &str) -> PathBuf { + ecosystem_base_path + .join(LOCAL_CHAINS_PATH) + .join(chain_name) + .join(LOCAL_CONFIGS_PATH) + .join(LOCAL_APPS_PATH) + .join(PORTAL_CHAIN_CONFIG_FILE) + } +} + +impl ZkToolboxConfig for PortalChainConfig {} diff --git a/zk_toolbox/crates/zk_inception/src/commands/args/mod.rs b/zk_toolbox/crates/zk_inception/src/commands/args/mod.rs index a27b653edf52..d18b05c910e5 100644 --- a/zk_toolbox/crates/zk_inception/src/commands/args/mod.rs +++ b/zk_toolbox/crates/zk_inception/src/commands/args/mod.rs @@ -1,9 +1,7 @@ pub use containers::*; -pub use portal::*; pub use run_server::*; pub use update::*; mod containers; -mod portal; mod run_server; mod update; diff --git a/zk_toolbox/crates/zk_inception/src/commands/args/portal.rs b/zk_toolbox/crates/zk_inception/src/commands/args/portal.rs deleted file mode 100644 index e31058aad5d0..000000000000 --- a/zk_toolbox/crates/zk_inception/src/commands/args/portal.rs +++ /dev/null @@ -1,12 +0,0 @@ -use clap::Parser; -use serde::{Deserialize, Serialize}; - -#[derive(Debug, Serialize, Deserialize, Parser)] -pub struct PortalArgs { - #[clap( - long, - default_value = "3030", - help = "The port number for the portal app" - )] - pub port: u16, -} diff --git a/zk_toolbox/crates/zk_inception/src/commands/chain/init.rs b/zk_toolbox/crates/zk_inception/src/commands/chain/init.rs index 9d1c0d543ee0..2daf3ee61f74 100644 --- a/zk_toolbox/crates/zk_inception/src/commands/chain/init.rs +++ b/zk_toolbox/crates/zk_inception/src/commands/chain/init.rs @@ -27,7 +27,7 @@ use crate::{ genesis::genesis, set_token_multiplier_setter::set_token_multiplier_setter, }, - portal::create_and_save_portal_config, + portal::create_and_save_portal_chain_config, }, consts::AMOUNT_FOR_DISTRIBUTION_TO_WALLETS, messages::{ @@ -149,7 +149,7 @@ pub async fn init( .await .context(MSG_GENESIS_DATABASE_ERR)?; - create_and_save_portal_config(ecosystem_config, shell) + create_and_save_portal_chain_config(chain_config, shell) .await .context(MSG_PORTAL_FAILED_TO_CREATE_CONFIG_ERR)?; diff --git a/zk_toolbox/crates/zk_inception/src/commands/containers.rs b/zk_toolbox/crates/zk_inception/src/commands/containers.rs index 17c32c04bc2f..81d7970df839 100644 --- a/zk_toolbox/crates/zk_inception/src/commands/containers.rs +++ b/zk_toolbox/crates/zk_inception/src/commands/containers.rs @@ -40,7 +40,7 @@ pub fn initialize_docker(shell: &Shell, ecosystem: &EcosystemConfig) -> anyhow:: } fn start_container(shell: &Shell, compose_file: &str, retry_msg: &str) -> anyhow::Result<()> { - while let Err(err) = docker::up(shell, compose_file) { + while let Err(err) = docker::up(shell, compose_file, true) { logger::error(err.to_string()); if !common::PromptConfirm::new(retry_msg).default(true).ask() { return Err(err); diff --git a/zk_toolbox/crates/zk_inception/src/commands/ecosystem/create.rs b/zk_toolbox/crates/zk_inception/src/commands/ecosystem/create.rs index f9940c8a9798..356b5322980f 100644 --- a/zk_toolbox/crates/zk_inception/src/commands/ecosystem/create.rs +++ b/zk_toolbox/crates/zk_inception/src/commands/ecosystem/create.rs @@ -15,7 +15,10 @@ use crate::{ containers::{initialize_docker, start_containers}, ecosystem::{ args::create::EcosystemCreateArgs, - create_configs::{create_erc20_deployment_config, create_initial_deployments_config}, + create_configs::{ + create_apps_config, create_erc20_deployment_config, + create_initial_deployments_config, + }, }, }, messages::{ @@ -75,6 +78,7 @@ fn create(args: EcosystemCreateArgs, shell: &Shell) -> anyhow::Result<()> { create_initial_deployments_config(shell, &configs_path)?; create_erc20_deployment_config(shell, &configs_path)?; + create_apps_config(shell, &configs_path)?; let ecosystem_config = EcosystemConfig { name: ecosystem_name.clone(), diff --git a/zk_toolbox/crates/zk_inception/src/commands/ecosystem/create_configs.rs b/zk_toolbox/crates/zk_inception/src/commands/ecosystem/create_configs.rs index b4f42313e3d0..38358355ff97 100644 --- a/zk_toolbox/crates/zk_inception/src/commands/ecosystem/create_configs.rs +++ b/zk_toolbox/crates/zk_inception/src/commands/ecosystem/create_configs.rs @@ -2,7 +2,8 @@ use std::path::Path; use config::{ forge_interface::deploy_ecosystem::input::{Erc20DeploymentConfig, InitialDeploymentConfig}, - traits::SaveConfigWithCommentAndBasePath, + traits::{SaveConfigWithBasePath, SaveConfigWithCommentAndBasePath}, + AppsEcosystemConfig, }; use xshell::Shell; @@ -33,3 +34,12 @@ pub fn create_erc20_deployment_config( )?; Ok(config) } + +pub fn create_apps_config( + shell: &Shell, + ecosystem_configs_path: &Path, +) -> anyhow::Result { + let config = AppsEcosystemConfig::default(); + config.save_with_base_path(shell, ecosystem_configs_path)?; + Ok(config) +} diff --git a/zk_toolbox/crates/zk_inception/src/commands/explorer.rs b/zk_toolbox/crates/zk_inception/src/commands/explorer.rs new file mode 100644 index 000000000000..7c71974a7c2d --- /dev/null +++ b/zk_toolbox/crates/zk_inception/src/commands/explorer.rs @@ -0,0 +1,239 @@ +use std::{collections::HashSet, path::Path}; + +use anyhow::Context; +use common::{db, docker, logger, Prompt}; +use config::{ + explorer::*, + explorer_compose::*, + traits::{ReadConfig, SaveConfig}, + AppsEcosystemConfig, ChainConfig, EcosystemConfig, +}; +use slugify_rs::slugify; +use xshell::Shell; + +use crate::{ + consts::L2_BASE_TOKEN_ADDRESS, + defaults::{ + generate_explorer_db_name, DATABASE_EXPLORER_URL, DEFAULT_EXPLORER_API_PORT, + DEFAULT_EXPLORER_DATA_FETCHER_PORT, DEFAULT_EXPLORER_WORKER_PORT, + }, + messages::{ + msg_explorer_db_name_prompt, msg_explorer_db_url_prompt, + msg_explorer_initializing_database_for, msg_explorer_starting_on, + MSG_EXPLORER_FAILED_TO_DROP_DATABASE_ERR, + }, + utils::ports::EcosystemPortsScanner, +}; + +fn get_l2_rpc_url(chain_config: &ChainConfig) -> anyhow::Result { + // Get L2 RPC URL from general config + let general_config = chain_config.get_general_config()?; + let rpc_url = general_config + .api_config + .as_ref() + .map(|api_config| &api_config.web3_json_rpc.http_url) + .context("api_config")?; + Ok(rpc_url.to_string()) +} + +async fn create_explorer_chain_config( + chain_config: &ChainConfig, +) -> anyhow::Result { + let l2_rpc_url = get_l2_rpc_url(chain_config)?; + // Get Verification API URL from general config + let general_config = chain_config.get_general_config()?; + let verification_api_url = general_config + .contract_verifier + .as_ref() + .map(|verifier| &verifier.url) + .context("verification_url")?; + + // Build network config + Ok(ExplorerChainConfig { + name: chain_config.name.clone(), + l2_network_name: chain_config.name.clone(), + l2_chain_id: chain_config.chain_id.as_u64(), + rpc_url: l2_rpc_url.to_string(), + api_url: "http://127.0.0.1:3002".to_string(), // TODO: probably need chain-level apps.yaml as well + base_token_address: L2_BASE_TOKEN_ADDRESS.to_string(), + hostnames: Vec::new(), + icon: "/images/icons/zksync-arrows.svg".to_string(), + maintenance: false, + published: true, + bridge_url: None, + l1_explorer_url: None, + verification_api_url: Some(verification_api_url.to_string()), + }) +} + +pub async fn create_and_save_explorer_chain_config( + chain_config: &ChainConfig, + shell: &Shell, +) -> anyhow::Result { + let explorer_config = create_explorer_chain_config(chain_config).await?; + let config_path = + ExplorerChainConfig::get_config_path(&shell.current_dir(), &chain_config.name); + explorer_config.save(shell, config_path)?; + Ok(explorer_config) +} + +pub async fn run(shell: &Shell) -> anyhow::Result<()> { + let ecosystem_config: EcosystemConfig = EcosystemConfig::from_file(shell)?; + let ecosystem_path = shell.current_dir(); + // Get ecosystem level apps.yaml config + let apps_config = AppsEcosystemConfig::read_or_create_default(shell)?; + // What chains to run the explorer for + let chains_enabled = apps_config + .explorer + .chains_enabled + .unwrap_or_else(|| ecosystem_config.list_of_chains()); + // Keep track of allocated ports (initialized lazily) + let mut allocated_ports: HashSet = HashSet::new(); + + // For each chain - initialize if needed or read previously saved configs + let mut explorer_chain_configs = Vec::new(); + let mut backend_configs = Vec::new(); + for chain_name in chains_enabled.iter() { + let chain_config = ecosystem_config + .load_chain(Some(chain_name.clone())) + .ok_or_else(|| anyhow::anyhow!("Failed to load chain config for {}", chain_name))?; + + // Should initialize? Check if explorer backend docker compose file exists + let mut should_initialize = false; + let backend_compose_path = + ExplorerBackendComposeConfig::get_config_path(&ecosystem_path, chain_name); + let backend_compose_config = + match ExplorerBackendComposeConfig::read(shell, &backend_compose_path) { + Ok(config) => config, + Err(_) => { + should_initialize = true; + // Initialize explorer database + logger::info(msg_explorer_initializing_database_for(&chain_name)); + let db_config = fill_database_values_with_prompt(&chain_config); + initialize_explorer_database(&db_config).await?; + + // Allocate ports for backend services + let service_ports: ExplorerBackendServicePorts = + allocate_explorer_services_ports(&ecosystem_path, &mut allocated_ports)?; + + let l2_rpc_url = get_l2_rpc_url(&chain_config)?; + let backend_service_config = ExplorerBackendServiceConfig { + db_url: db_config.full_url().to_string(), + l2_rpc_url: l2_rpc_url.to_string(), + service_ports, + }; + + // Create and save docker compose for chain backend services + let backend_compose_config = + ExplorerBackendComposeConfig::new(chain_name, backend_service_config)?; + backend_compose_config.save(shell, &backend_compose_path)?; + backend_compose_config + } + }; + backend_configs.push(backend_compose_config); + + // Read or create explorer chain config (JSON) + let explorer_chain_config_path = + ExplorerChainConfig::get_config_path(&ecosystem_path, chain_name); + let explorer_chain_config = match should_initialize { + true => create_and_save_explorer_chain_config(&chain_config, shell).await?, + false => match ExplorerChainConfig::read(shell, &explorer_chain_config_path) { + Ok(config) => config, + Err(_) => create_and_save_explorer_chain_config(&chain_config, shell).await?, + }, + }; + explorer_chain_configs.push(explorer_chain_config); + } + + // Generate and save explorer runtime config (JS) + let explorer_runtime_config = ExplorerRuntimeConfig::new(explorer_chain_configs); + let explorer_runtime_config_path = ExplorerRuntimeConfig::get_config_path(&ecosystem_path); + explorer_runtime_config.save(shell, &explorer_runtime_config_path)?; + + // Generate and save explorer docker compose + let app_config = ExplorerAppServiceConfig { + port: apps_config.explorer.http_port, + config_path: explorer_runtime_config_path, + }; + let explorer_compose_config = ExplorerComposeConfig::new(app_config, backend_configs)?; + let explorer_compose_path = ExplorerComposeConfig::get_config_path(&ecosystem_path); + explorer_compose_config.save(&shell, &explorer_compose_path)?; + + // Launch explorer using docker compose + logger::info(format!( + "Using generated docker compose file at {}", + explorer_compose_path.display() + )); + logger::info(msg_explorer_starting_on( + "127.0.0.1", + apps_config.explorer.http_port, + )); + run_explorer(shell, &explorer_compose_path)?; + Ok(()) +} + +fn run_explorer(shell: &Shell, explorer_compose_config_path: &Path) -> anyhow::Result<()> { + if let Some(docker_compose_file) = explorer_compose_config_path.to_str() { + // TODO: Containers are keep running after the process is killed (Ctrl+C) + docker::up(shell, docker_compose_file, true) + .with_context(|| "Failed to run docker compose for Explorer")?; + } else { + anyhow::bail!("Invalid docker compose file"); + } + Ok(()) +} + +fn fill_database_values_with_prompt(config: &ChainConfig) -> db::DatabaseConfig { + let defaul_db_name: String = generate_explorer_db_name(config); + let chain_name = config.name.clone(); + let explorer_db_url = Prompt::new(&msg_explorer_db_url_prompt(&chain_name)) + .default(DATABASE_EXPLORER_URL.as_str()) + .ask(); + let explorer_db_name: String = Prompt::new(&msg_explorer_db_name_prompt(&chain_name)) + .default(&defaul_db_name) + .ask(); + let explorer_db_name = slugify!(&explorer_db_name, separator = "_"); + return db::DatabaseConfig::new(explorer_db_url, explorer_db_name); +} + +pub async fn initialize_explorer_database( + explorer_db_config: &db::DatabaseConfig, +) -> anyhow::Result<()> { + db::drop_db_if_exists(explorer_db_config) + .await + .context(MSG_EXPLORER_FAILED_TO_DROP_DATABASE_ERR)?; + db::init_db(explorer_db_config).await?; + Ok(()) +} + +// TODO: WIP, need to add fallback options and add more checks +// TODO: probably need to move this to a separate module and make it more generic +pub fn allocate_explorer_services_ports( + ecosystem_path: &Path, + allocated_ports: &mut HashSet, +) -> anyhow::Result { + if allocated_ports.is_empty() { + let ports = EcosystemPortsScanner::scan(ecosystem_path)?; + allocated_ports.extend(ports.get_assigned_ports()); + } + + let mut service_ports = ExplorerBackendServicePorts { + api_port: DEFAULT_EXPLORER_API_PORT, + data_fetcher_port: DEFAULT_EXPLORER_DATA_FETCHER_PORT, + worker_port: DEFAULT_EXPLORER_WORKER_PORT, + }; + + let offset = 100; + while allocated_ports.contains(&service_ports.api_port) + || allocated_ports.contains(&service_ports.data_fetcher_port) + || allocated_ports.contains(&service_ports.worker_port) + { + service_ports.api_port += offset; + service_ports.data_fetcher_port += offset; + service_ports.worker_port += offset; + } + allocated_ports.insert(service_ports.api_port); + allocated_ports.insert(service_ports.data_fetcher_port); + allocated_ports.insert(service_ports.worker_port); + Ok(service_ports) +} diff --git a/zk_toolbox/crates/zk_inception/src/commands/mod.rs b/zk_toolbox/crates/zk_inception/src/commands/mod.rs index 0ac363beb2da..523faea04786 100644 --- a/zk_toolbox/crates/zk_inception/src/commands/mod.rs +++ b/zk_toolbox/crates/zk_inception/src/commands/mod.rs @@ -3,6 +3,7 @@ pub mod chain; pub mod containers; pub mod contract_verifier; pub mod ecosystem; +pub mod explorer; pub mod external_node; pub mod portal; pub mod prover; diff --git a/zk_toolbox/crates/zk_inception/src/commands/portal.rs b/zk_toolbox/crates/zk_inception/src/commands/portal.rs index cc939f3fb3ea..445b48e9ac24 100644 --- a/zk_toolbox/crates/zk_inception/src/commands/portal.rs +++ b/zk_toolbox/crates/zk_inception/src/commands/portal.rs @@ -1,29 +1,30 @@ use std::{collections::HashMap, path::Path}; -use anyhow::{anyhow, Context}; +use anyhow::Context; use common::{docker, ethereum, logger}; use config::{ portal::*, traits::{ReadConfig, SaveConfig}, - ChainConfig, EcosystemConfig, + AppsEcosystemConfig, ChainConfig, EcosystemConfig, }; use ethers::types::Address; use types::{BaseToken, TokenInfo}; use xshell::Shell; use crate::{ - commands::args::PortalArgs, consts::{L2_BASE_TOKEN_ADDRESS, PORTAL_DOCKER_CONTAINER_PORT, PORTAL_DOCKER_IMAGE}, messages::{ - msg_portal_starting_on, MSG_PORTAL_CONFIG_IS_EMPTY_ERR, + msg_portal_starting_on, MSG_PORTAL_FAILED_TO_CREATE_ANY_CHAIN_CONFIG_ERR, MSG_PORTAL_FAILED_TO_CREATE_CONFIG_ERR, MSG_PORTAL_FAILED_TO_RUN_DOCKER_ERR, }, }; -async fn create_hyperchain_config(chain_config: &ChainConfig) -> anyhow::Result { +async fn create_portal_chain_config( + chain_config: &ChainConfig, +) -> anyhow::Result { // Get L2 RPC URL from general config let general_config = chain_config.get_general_config()?; - let rpc_url = general_config + let rpc_url: &String = general_config .api_config .as_ref() .map(|api_config| &api_config.web3_json_rpc.http_url) @@ -68,7 +69,7 @@ async fn create_hyperchain_config(chain_config: &ChainConfig) -> anyhow::Result< name: Some(base_token_info.name.to_string()), }]; // Build hyperchain config - Ok(HyperchainConfig { + Ok(PortalChainConfig { network: NetworkConfig { id: chain_config.chain_id.as_u64(), key: chain_config.name.clone(), @@ -83,69 +84,75 @@ async fn create_hyperchain_config(chain_config: &ChainConfig) -> anyhow::Result< }) } -async fn create_hyperchains_config( - chain_configs: &[ChainConfig], -) -> anyhow::Result { - let mut hyperchain_configs = Vec::new(); - for chain_config in chain_configs { - if let Ok(config) = create_hyperchain_config(chain_config).await { - hyperchain_configs.push(config) - } - } - Ok(HyperchainsConfig(hyperchain_configs)) +pub async fn create_and_save_portal_chain_config( + chain_config: &ChainConfig, + shell: &Shell, +) -> anyhow::Result { + let portal_config = create_portal_chain_config(chain_config).await?; + let config_path = PortalChainConfig::get_config_path(&shell.current_dir(), &chain_config.name); + portal_config.save(shell, config_path)?; + Ok(portal_config) } -pub async fn create_portal_config( +async fn create_portal_runtime_config( + shell: &Shell, ecosystem_config: &EcosystemConfig, + chain_names: Vec, ) -> anyhow::Result { - let chains: Vec = ecosystem_config.list_of_chains(); - let mut chain_configs = Vec::new(); - for chain in chains { - if let Some(chain_config) = ecosystem_config.load_chain(Some(chain.clone())) { - chain_configs.push(chain_config) + let ecosystem_base_path = &shell.current_dir(); + let mut hyperchains_config = Vec::new(); + + for chain_name in chain_names { + let config_path = PortalChainConfig::get_config_path(ecosystem_base_path, &chain_name); + + let portal_chain_config = match PortalChainConfig::read(shell, &config_path) { + Ok(config) => Some(config), + Err(_) => match ecosystem_config.load_chain(Some(chain_name.clone())) { + Some(chain_config) => match create_portal_chain_config(&chain_config).await { + Ok(config) => Some(config), + Err(_) => None, + }, + None => None, + }, + }; + if let Some(config) = portal_chain_config { + hyperchains_config.push(config); } } - let hyperchains_config = create_hyperchains_config(&chain_configs).await?; if hyperchains_config.is_empty() { - anyhow::bail!("Failed to create any valid hyperchain config") + anyhow::bail!(MSG_PORTAL_FAILED_TO_CREATE_ANY_CHAIN_CONFIG_ERR) } - let runtime_config = PortalRuntimeConfig { - node_type: "hyperchain".to_string(), - hyperchains_config, - }; + let runtime_config = PortalRuntimeConfig::new(hyperchains_config); Ok(runtime_config) } -pub async fn create_and_save_portal_config( - ecosystem_config: &EcosystemConfig, - shell: &Shell, -) -> anyhow::Result { - let portal_config = create_portal_config(ecosystem_config).await?; - let config_path = PortalRuntimeConfig::get_config_path(&shell.current_dir()); - portal_config.save(shell, config_path)?; - Ok(portal_config) -} - -pub async fn run(shell: &Shell, args: PortalArgs) -> anyhow::Result<()> { +pub async fn run(shell: &Shell) -> anyhow::Result<()> { let ecosystem_config: EcosystemConfig = EcosystemConfig::from_file(shell)?; + // Get ecosystem level apps.yaml config + let apps_config = AppsEcosystemConfig::read_or_create_default(shell)?; + // What chains to run the portal for? + let chains_enabled = apps_config + .portal + .chains_enabled + .unwrap_or_else(|| ecosystem_config.list_of_chains()); + + // Generate portal runtime config + let runtime_config = create_portal_runtime_config(shell, &ecosystem_config, chains_enabled) + .await + .context(MSG_PORTAL_FAILED_TO_CREATE_CONFIG_ERR)?; let config_path = PortalRuntimeConfig::get_config_path(&shell.current_dir()); + runtime_config.save(shell, &config_path)?; + logger::info(format!( - "Using portal config file at {}", + "Using generated portal config file at {}", config_path.display() )); - let portal_config = match PortalRuntimeConfig::read(shell, &config_path) { - Ok(config) => config, - Err(_) => create_and_save_portal_config(&ecosystem_config, shell) - .await - .context(MSG_PORTAL_FAILED_TO_CREATE_CONFIG_ERR)?, - }; - if portal_config.hyperchains_config.is_empty() { - return Err(anyhow!(MSG_PORTAL_CONFIG_IS_EMPTY_ERR)); - } - - logger::info(msg_portal_starting_on("127.0.0.1", args.port)); - run_portal(shell, &config_path, args.port)?; + logger::info(msg_portal_starting_on( + "127.0.0.1", + apps_config.portal.http_port, + )); + run_portal(shell, &config_path, apps_config.portal.http_port)?; Ok(()) } diff --git a/zk_toolbox/crates/zk_inception/src/defaults.rs b/zk_toolbox/crates/zk_inception/src/defaults.rs index 34b0eeae4195..66c48d867360 100644 --- a/zk_toolbox/crates/zk_inception/src/defaults.rs +++ b/zk_toolbox/crates/zk_inception/src/defaults.rs @@ -7,8 +7,15 @@ lazy_static! { Url::parse("postgres://postgres:notsecurepassword@localhost:5432").unwrap(); pub static ref DATABASE_PROVER_URL: Url = Url::parse("postgres://postgres:notsecurepassword@localhost:5432").unwrap(); + pub static ref DATABASE_EXPLORER_URL: Url = + Url::parse("postgres://postgres:notsecurepassword@localhost:5432").unwrap(); } +// Default ports for services +pub const DEFAULT_EXPLORER_WORKER_PORT: u16 = 3001; +pub const DEFAULT_EXPLORER_API_PORT: u16 = 3002; +pub const DEFAULT_EXPLORER_DATA_FETCHER_PORT: u16 = 3040; + pub const ROCKS_DB_STATE_KEEPER: &str = "state_keeper"; pub const ROCKS_DB_TREE: &str = "tree"; pub const ROCKS_DB_PROTECTIVE_READS: &str = "protective_reads"; @@ -40,6 +47,14 @@ pub fn generate_db_names(config: &ChainConfig) -> DBNames { } } +pub fn generate_explorer_db_name(config: &ChainConfig) -> String { + return format!( + "zksync_explorer_{}_{}", + config.l1_network.to_string().to_ascii_lowercase(), + config.name + ); +} + pub fn generate_external_node_db_name(config: &ChainConfig) -> String { format!( "external_node_{}_{}", diff --git a/zk_toolbox/crates/zk_inception/src/main.rs b/zk_toolbox/crates/zk_inception/src/main.rs index cb1b5388196a..4323029d4e90 100644 --- a/zk_toolbox/crates/zk_inception/src/main.rs +++ b/zk_toolbox/crates/zk_inception/src/main.rs @@ -13,11 +13,8 @@ use config::EcosystemConfig; use xshell::Shell; use crate::commands::{ - args::{PortalArgs, RunServerArgs}, - chain::ChainCommands, - ecosystem::EcosystemCommands, - external_node::ExternalNodeCommands, - prover::ProverCommands, + args::RunServerArgs, chain::ChainCommands, ecosystem::EcosystemCommands, + external_node::ExternalNodeCommands, prover::ProverCommands, }; pub mod accept_ownership; @@ -60,7 +57,9 @@ pub enum InceptionSubcommands { #[command(subcommand)] ContractVerifier(ContractVerifierCommands), /// Run dapp-portal - Portal(PortalArgs), + Portal, + /// Run block-explorer + Explorer, /// Update ZKsync #[command(alias = "u")] Update(UpdateArgs), @@ -123,7 +122,8 @@ async fn run_subcommand(inception_args: Inception, shell: &Shell) -> anyhow::Res InceptionSubcommands::ContractVerifier(args) => { commands::contract_verifier::run(shell, args).await? } - InceptionSubcommands::Portal(args) => commands::portal::run(shell, args).await?, + InceptionSubcommands::Explorer => commands::explorer::run(shell).await?, + InceptionSubcommands::Portal => commands::portal::run(shell).await?, InceptionSubcommands::Update(args) => commands::update::run(shell, args)?, InceptionSubcommands::Markdown => { clap_markdown::print_help_markdown::(); diff --git a/zk_toolbox/crates/zk_inception/src/messages.rs b/zk_toolbox/crates/zk_inception/src/messages.rs index 9975627025ac..e5aad36cdcfb 100644 --- a/zk_toolbox/crates/zk_inception/src/messages.rs +++ b/zk_toolbox/crates/zk_inception/src/messages.rs @@ -194,6 +194,14 @@ pub(super) fn msg_server_db_name_prompt(chain_name: &str) -> String { format!("Please provide server database name for chain {chain_name}") } +pub(super) fn msg_explorer_db_url_prompt(chain_name: &str) -> String { + format!("Please provide explorer database url for chain {chain_name}") +} + +pub(super) fn msg_explorer_db_name_prompt(chain_name: &str) -> String { + format!("Please provide explorer database name for chain {chain_name}") +} + /// Chain initialize bridges related messages pub(super) const MSG_DEPLOYING_L2_CONTRACT_SPINNER: &str = "Deploying l2 contracts"; @@ -226,7 +234,8 @@ pub(super) const MSG_FAILED_TO_RUN_SERVER_ERR: &str = "Failed to start server"; pub(super) const MSG_PREPARING_EN_CONFIGS: &str = "Preparing External Node config"; /// Portal related messages -pub(super) const MSG_PORTAL_CONFIG_IS_EMPTY_ERR: &str = "Hyperchains config is empty"; +pub(super) const MSG_PORTAL_FAILED_TO_CREATE_ANY_CHAIN_CONFIG_ERR: &str = + "Failed to create any valid hyperchain config"; pub(super) const MSG_PORTAL_FAILED_TO_CREATE_CONFIG_ERR: &str = "Failed to create portal config"; pub(super) const MSG_PORTAL_FAILED_TO_RUN_DOCKER_ERR: &str = "Failed to run portal docker container"; @@ -234,6 +243,16 @@ pub(super) fn msg_portal_starting_on(host: &str, port: u16) -> String { format!("Starting portal on http://{host}:{port}") } +/// Explorer related messages +pub(super) const MSG_EXPLORER_FAILED_TO_DROP_DATABASE_ERR: &str = + "Failed to drop explorer database"; +pub(super) fn msg_explorer_initializing_database_for(chain: &str) -> String { + format!("Initializing explorer database for {chain} chain") +} +pub(super) fn msg_explorer_starting_on(host: &str, port: u16) -> String { + format!("Starting explorer on http://{host}:{port}") +} + /// Forge utils related messages pub(super) const MSG_DEPLOYER_PK_NOT_SET_ERR: &str = "Deployer private key is not set"; diff --git a/zk_toolbox/crates/zk_inception/src/utils/mod.rs b/zk_toolbox/crates/zk_inception/src/utils/mod.rs index a84f0a336de5..e8bfeeebf053 100644 --- a/zk_toolbox/crates/zk_inception/src/utils/mod.rs +++ b/zk_toolbox/crates/zk_inception/src/utils/mod.rs @@ -1,2 +1,3 @@ pub mod forge; +pub mod ports; pub mod rocks_db; diff --git a/zk_toolbox/crates/zk_inception/src/utils/ports.rs b/zk_toolbox/crates/zk_inception/src/utils/ports.rs new file mode 100644 index 000000000000..19907d1f80a8 --- /dev/null +++ b/zk_toolbox/crates/zk_inception/src/utils/ports.rs @@ -0,0 +1,162 @@ +use std::{ + collections::{HashMap, HashSet}, + fs, + path::Path, +}; + +use anyhow::{Context, Result}; +use serde_yaml::Value; + +#[derive(Debug, Clone)] +pub struct PortInfo { + pub file_name: String, + pub key_path: String, +} + +impl PortInfo { + pub fn to_string(&self) -> String { + format!("[{}] {}", self.file_name, self.key_path) + } +} + +pub struct EcosystemPorts { + pub ports: HashMap>, +} + +impl EcosystemPorts { + pub fn new() -> Self { + Self { + ports: HashMap::new(), + } + } + + pub fn get_assigned_ports(&self) -> HashSet { + self.ports.keys().cloned().collect() + } + + pub fn get_port_info(&self, port: u16) -> Option<&Vec> { + self.ports.get(&port) + } + + pub fn is_port_assigned(&self, port: u16) -> bool { + self.ports.contains_key(&port) + } + + pub fn add_port_info(&mut self, port: u16, info: PortInfo) { + self.ports.entry(port).or_insert_with(Vec::new).push(info); + } + + pub fn print(&self) { + let mut port_vec: Vec<_> = self.ports.iter().collect(); + port_vec.sort_by_key(|&(key, _)| key); + for (port, port_infos) in port_vec { + for port_info in port_infos { + println!("{} > {}", port_info.to_string(), port); + } + } + } +} + +pub struct EcosystemPortsScanner {} + +impl EcosystemPortsScanner { + /// Scans the ecosystem directory for YAML files and extracts port information + /// Does not work with docker files + pub fn scan(ecosystem_dir: &Path) -> Result { + let skip_dirs: HashSet<&str> = vec!["db", "target", "volumes"].into_iter().collect(); + + let mut ecosystem_ports = EcosystemPorts::new(); + + let subdirs = ["chains", "configs"]; + for subdir in &subdirs { + let dir = ecosystem_dir.join(subdir); + if dir.is_dir() { + if let Err(e) = Self::scan_directory(&dir, &mut ecosystem_ports, &skip_dirs) { + eprintln!("Error scanning directory {:?}: {}", dir, e); + } + } + } + + Ok(ecosystem_ports) + } + + fn scan_directory( + dir: &Path, + ecosystem_ports: &mut EcosystemPorts, + skip_dirs: &HashSet<&str>, + ) -> Result<()> { + if let Some(dir_name) = dir.file_name().and_then(|n| n.to_str()) { + if skip_dirs.contains(dir_name) { + return Ok(()); + } + } + + for entry in fs::read_dir(dir).context("Failed to read directory")? { + let entry = entry.context("Failed to read directory entry")?; + let path = entry.path(); + if path.is_dir() { + if let Err(e) = Self::scan_directory(&path, ecosystem_ports, skip_dirs) { + eprintln!("Error scanning directory {:?}: {}", path, e); + } + } else if path.is_file() { + if let Some(extension) = path.extension() { + if extension == "yaml" || extension == "yml" { + if let Err(e) = Self::process_yaml_file(&path, ecosystem_ports) { + eprintln!("Error processing file {:?}: {}", path, e); + } + } + } + } + } + Ok(()) + } + + fn process_yaml_file(file_path: &Path, ecosystem_ports: &mut EcosystemPorts) -> Result<()> { + let contents = fs::read_to_string(file_path) + .with_context(|| format!("Failed to read file: {:?}", file_path))?; + let value: Value = serde_yaml::from_str(&contents) + .with_context(|| format!("Failed to parse YAML in file: {:?}", file_path))?; + Self::traverse_yaml(&value, "", file_path, ecosystem_ports); + Ok(()) + } + + fn traverse_yaml( + value: &Value, + path: &str, + file_path: &Path, + ecosystem_ports: &mut EcosystemPorts, + ) { + match value { + Value::Mapping(map) => { + for (key, val) in map { + let new_path = if path.is_empty() { + key.as_str().unwrap_or_default().to_string() + } else { + format!("{}:{}", path, key.as_str().unwrap_or_default()) + }; + + if key.as_str().map(|s| s.ends_with("port")).unwrap_or(false) { + if let Some(port) = val.as_u64().and_then(|p| u16::try_from(p).ok()) { + ecosystem_ports.add_port_info( + port, + PortInfo { + file_name: file_path.to_string_lossy().into_owned(), + key_path: new_path.clone(), + }, + ); + } + } + + Self::traverse_yaml(val, &new_path, file_path, ecosystem_ports); + } + } + Value::Sequence(seq) => { + for (index, val) in seq.iter().enumerate() { + let new_path = format!("{}:{}", path, index); + Self::traverse_yaml(val, &new_path, file_path, ecosystem_ports); + } + } + _ => {} + } + } +} From aac5fd9c23d33c83e812cade61e55d6defc86bd1 Mon Sep 17 00:00:00 2001 From: Alexander Melnikov Date: Fri, 30 Aug 2024 03:26:45 -0600 Subject: [PATCH 02/12] follow up --- zk_toolbox/crates/common/src/docker.rs | 7 +- zk_toolbox/crates/config/src/apps.rs | 52 +++++- .../crates/config/src/docker_compose.rs | 16 ++ .../crates/config/src/explorer_compose.rs | 108 +++-------- .../zk_inception/src/commands/chain/init.rs | 4 +- .../zk_inception/src/commands/explorer.rs | 172 ++++++++++-------- .../zk_inception/src/commands/portal.rs | 12 +- .../crates/zk_inception/src/messages.rs | 2 + .../crates/zk_inception/src/utils/ports.rs | 113 ++++++++---- 9 files changed, 288 insertions(+), 198 deletions(-) diff --git a/zk_toolbox/crates/common/src/docker.rs b/zk_toolbox/crates/common/src/docker.rs index 338b4885aeab..b42ce485c7c6 100644 --- a/zk_toolbox/crates/common/src/docker.rs +++ b/zk_toolbox/crates/common/src/docker.rs @@ -6,11 +6,12 @@ use crate::cmd::Cmd; pub fn up(shell: &Shell, docker_compose_file: &str, detach: bool) -> anyhow::Result<()> { let args = if detach { vec!["-d"] } else { vec![] }; - Ok(Cmd::new(cmd!( + let mut cmd = Cmd::new(cmd!( shell, "docker compose -f {docker_compose_file} up {args...}" - )) - .run()?) + )); + cmd = if !detach { cmd.with_force_run() } else { cmd }; + Ok(cmd.run()?) } pub fn down(shell: &Shell, docker_compose_file: &str) -> anyhow::Result<()> { diff --git a/zk_toolbox/crates/config/src/apps.rs b/zk_toolbox/crates/config/src/apps.rs index 76203cbf8aa1..09d2431ee15f 100644 --- a/zk_toolbox/crates/config/src/apps.rs +++ b/zk_toolbox/crates/config/src/apps.rs @@ -1,10 +1,11 @@ use std::path::{Path, PathBuf}; use serde::{Deserialize, Serialize}; +use url::Url; use xshell::Shell; use crate::{ - consts::{APPS_CONFIG_FILE, LOCAL_CONFIGS_PATH}, + consts::{APPS_CONFIG_FILE, LOCAL_CHAINS_PATH, LOCAL_CONFIGS_PATH}, traits::{FileConfigWithDefaultName, ReadConfig, SaveConfig, ZkToolboxConfig}, }; @@ -31,6 +32,28 @@ impl FileConfigWithDefaultName for AppsEcosystemConfig { const FILE_NAME: &'static str = APPS_CONFIG_FILE; } +/// Chain level configuration for the apps. +#[derive(Debug, Serialize, Deserialize, Clone)] +pub struct AppsChainConfig { + pub explorer: AppsChainExplorerConfig, +} + +#[derive(Debug, Serialize, Deserialize, Clone)] +pub struct AppsChainExplorerConfig { + pub database_url: Url, + pub services: ExplorerServicesConfig, +} + +#[derive(Debug, Serialize, Deserialize, Clone)] +pub struct ExplorerServicesConfig { + pub api_http_port: u16, + pub data_fetcher_http_port: u16, + pub worker_http_port: u16, + pub batches_processing_polling_interval: Option, +} + +impl ZkToolboxConfig for AppsChainConfig {} + impl AppsEcosystemConfig { pub fn default() -> Self { AppsEcosystemConfig { @@ -65,3 +88,30 @@ impl AppsEcosystemConfig { } } } + +impl AppsChainConfig { + pub fn new(explorer: AppsChainExplorerConfig) -> Self { + AppsChainConfig { explorer } + } + + pub fn get_config_path(ecosystem_base_path: &Path, chain_name: &str) -> PathBuf { + ecosystem_base_path + .join(LOCAL_CHAINS_PATH) + .join(chain_name) + .join(LOCAL_CONFIGS_PATH) + .join(APPS_CONFIG_FILE) + } +} + +impl ExplorerServicesConfig { + pub fn get_batches_processing_polling_interval(&self) -> u64 { + self.batches_processing_polling_interval + .unwrap_or_else(|| 1000) + } + + pub fn with_defaults(mut self) -> Self { + self.batches_processing_polling_interval = + Some(self.get_batches_processing_polling_interval()); + self + } +} diff --git a/zk_toolbox/crates/config/src/docker_compose.rs b/zk_toolbox/crates/config/src/docker_compose.rs index 6aab16ba1a16..a0a0914c1a7b 100644 --- a/zk_toolbox/crates/config/src/docker_compose.rs +++ b/zk_toolbox/crates/config/src/docker_compose.rs @@ -1,6 +1,7 @@ use std::collections::HashMap; use serde::{Deserialize, Serialize}; +use url::Url; use crate::traits::ZkToolboxConfig; @@ -40,4 +41,19 @@ impl DockerComposeConfig { pub fn add_service(&mut self, name: &str, service: DockerComposeService) { self.services.insert(name.to_string(), service); } + + pub fn adjust_host_for_docker(mut url: Url) -> anyhow::Result { + if let Some(host) = url.host_str() { + if Self::is_localhost(host) { + url.set_host(Some("host.docker.internal"))?; + } + } else { + anyhow::bail!("Failed to parse: no host"); + } + Ok(url) + } + + fn is_localhost(host: &str) -> bool { + host == "localhost" || host == "127.0.0.1" || host == "[::1]" + } } diff --git a/zk_toolbox/crates/config/src/explorer_compose.rs b/zk_toolbox/crates/config/src/explorer_compose.rs index eb2743b62f90..aca07b5133ac 100644 --- a/zk_toolbox/crates/config/src/explorer_compose.rs +++ b/zk_toolbox/crates/config/src/explorer_compose.rs @@ -8,10 +8,8 @@ use serde::{Deserialize, Serialize}; use url::Url; use crate::{ - consts::{ - EXPLORER_BACKEND_DOCKER_COMPOSE_FILE, EXPLORER_DOCKER_COMPOSE_FILE, LOCAL_APPS_PATH, - LOCAL_CHAINS_PATH, LOCAL_CONFIGS_PATH, LOCAL_GENERATED_PATH, - }, + apps::AppsChainExplorerConfig, + consts::{EXPLORER_DOCKER_COMPOSE_FILE, LOCAL_CONFIGS_PATH, LOCAL_GENERATED_PATH}, docker_compose::{DockerComposeConfig, DockerComposeService}, traits::ZkToolboxConfig, }; @@ -24,38 +22,23 @@ pub struct ExplorerComposeConfig { pub docker_compose: DockerComposeConfig, } -/// Chain-level explorer backend docker compose file. This file is created when the explorer is -/// initialized for the chain. It contains the configuration for the chain explorer backend -/// services and serves as a building block for the main explorer docker compose file. +impl ZkToolboxConfig for ExplorerComposeConfig {} + +/// Chain-level explorer backend docker compose config. It contains the configuration for +/// api, data fetcher, and worker services. This configs is generated during "explorer" command +/// and serves as a building block for the main explorer #[derive(Debug, Serialize, Deserialize, Clone)] pub struct ExplorerBackendComposeConfig { #[serde(flatten)] pub docker_compose: DockerComposeConfig, } -impl ZkToolboxConfig for ExplorerComposeConfig {} -impl ZkToolboxConfig for ExplorerBackendComposeConfig {} - #[derive(Debug, Clone)] pub struct ExplorerAppServiceConfig { pub port: u16, pub config_path: PathBuf, } -#[derive(Debug, Clone)] -pub struct ExplorerBackendServiceConfig { - pub db_url: String, - pub l2_rpc_url: String, - pub service_ports: ExplorerBackendServicePorts, -} - -#[derive(Debug, Clone)] -pub struct ExplorerBackendServicePorts { - pub api_port: u16, - pub data_fetcher_port: u16, - pub worker_port: u16, -} - impl ExplorerComposeConfig { pub fn new( app_config: ExplorerAppServiceConfig, @@ -113,11 +96,15 @@ impl ExplorerComposeConfig { } impl ExplorerBackendComposeConfig { - pub fn new(chain_name: &str, config: ExplorerBackendServiceConfig) -> anyhow::Result { - let config = config.adjust_for_docker()?; + pub fn new( + chain_name: &str, + l2_rpc_url: Url, + config: &AppsChainExplorerConfig, + ) -> anyhow::Result { + let db_url = DockerComposeConfig::adjust_host_for_docker(config.database_url.clone())?; + let l2_rpc_url = DockerComposeConfig::adjust_host_for_docker(l2_rpc_url)?; // Parse database URL - let db_url = Url::parse(&config.db_url).context("Failed to parse database URL")?; let db_host = db_url .host_str() .context("Failed to parse database URL: no host")? @@ -132,26 +119,31 @@ impl ExplorerBackendComposeConfig { let mut services: HashMap = HashMap::new(); services.insert( format!("explorer-api-{}", chain_name), - Self::create_api_service(chain_name, config.service_ports.api_port, &config.db_url), + Self::create_api_service( + chain_name, + config.services.api_http_port, + &db_url.to_string(), + ), ); services.insert( format!("explorer-data-fetcher-{}", chain_name), Self::create_data_fetcher_service( - config.service_ports.data_fetcher_port, - &config.l2_rpc_url, + config.services.data_fetcher_http_port, + &l2_rpc_url.to_string(), ), ); services.insert( format!("explorer-worker-{}", chain_name), Self::create_worker_service( chain_name, - config.service_ports.worker_port, - config.service_ports.data_fetcher_port, - &config.l2_rpc_url, + config.services.worker_http_port, + config.services.data_fetcher_http_port, + &l2_rpc_url.to_string(), &db_host, &db_user, &db_password, &db_name, + config.services.get_batches_processing_polling_interval(), ), ); @@ -205,6 +197,7 @@ impl ExplorerBackendComposeConfig { db_user: &str, db_password: &str, db_name: &str, + batches_processing_polling_interval: u64, ) -> DockerComposeService { let data_fetcher_url = format!( "http://explorer-data-fetcher-{}:{}", @@ -229,57 +222,10 @@ impl ExplorerBackendComposeConfig { ("DATA_FETCHER_URL".to_string(), data_fetcher_url), ( "BATCHES_PROCESSING_POLLING_INTERVAL".to_string(), - "1000".to_string(), + batches_processing_polling_interval.to_string(), ), ])), extra_hosts: Some(vec!["host.docker.internal:host-gateway".to_string()]), } } - - pub fn get_config_path(ecosystem_base_path: &Path, chain_name: &str) -> PathBuf { - ecosystem_base_path - .join(LOCAL_CHAINS_PATH) - .join(chain_name) - .join(LOCAL_CONFIGS_PATH) - .join(LOCAL_APPS_PATH) - .join(EXPLORER_BACKEND_DOCKER_COMPOSE_FILE) - } -} - -impl ExplorerBackendServiceConfig { - fn is_localhost(host: &str) -> bool { - host == "localhost" || host == "127.0.0.1" || host == "[::1]" - } - - pub fn adjust_for_docker(&self) -> anyhow::Result { - // Parse database URL - let mut db_url = Url::parse(&self.db_url).context("Failed to parse database URL")?; - let db_host = db_url - .host_str() - .context("Failed to parse database URL: no host")? - .to_string(); - - // Replace localhost with host.docker.internal for database URL - if Self::is_localhost(&db_host) { - db_url.set_host(Some("host.docker.internal"))?; - } - - // Parse L2 RPC URL - let mut l2_rpc_url = Url::parse(&self.l2_rpc_url).context("Failed to parse L2 RPC URL")?; - let l2_host = l2_rpc_url - .host_str() - .context("Failed to parse L2 RPC URL: no host")? - .to_string(); - - // Replace localhost with host.docker.internal for L2 RPC URL - if Self::is_localhost(&l2_host) { - l2_rpc_url.set_host(Some("host.docker.internal"))?; - } - - Ok(Self { - db_url: db_url.to_string(), - l2_rpc_url: l2_rpc_url.to_string(), - service_ports: self.service_ports.clone(), - }) - } } diff --git a/zk_toolbox/crates/zk_inception/src/commands/chain/init.rs b/zk_toolbox/crates/zk_inception/src/commands/chain/init.rs index 2daf3ee61f74..ce0ff706a505 100644 --- a/zk_toolbox/crates/zk_inception/src/commands/chain/init.rs +++ b/zk_toolbox/crates/zk_inception/src/commands/chain/init.rs @@ -27,7 +27,7 @@ use crate::{ genesis::genesis, set_token_multiplier_setter::set_token_multiplier_setter, }, - portal::create_and_save_portal_chain_config, + portal::create_portal_chain_config, }, consts::AMOUNT_FOR_DISTRIBUTION_TO_WALLETS, messages::{ @@ -149,7 +149,7 @@ pub async fn init( .await .context(MSG_GENESIS_DATABASE_ERR)?; - create_and_save_portal_chain_config(chain_config, shell) + create_portal_chain_config(chain_config, shell) .await .context(MSG_PORTAL_FAILED_TO_CREATE_CONFIG_ERR)?; diff --git a/zk_toolbox/crates/zk_inception/src/commands/explorer.rs b/zk_toolbox/crates/zk_inception/src/commands/explorer.rs index 7c71974a7c2d..995d87955151 100644 --- a/zk_toolbox/crates/zk_inception/src/commands/explorer.rs +++ b/zk_toolbox/crates/zk_inception/src/commands/explorer.rs @@ -1,4 +1,4 @@ -use std::{collections::HashSet, path::Path}; +use std::path::Path; use anyhow::Context; use common::{db, docker, logger, Prompt}; @@ -6,9 +6,11 @@ use config::{ explorer::*, explorer_compose::*, traits::{ReadConfig, SaveConfig}, - AppsEcosystemConfig, ChainConfig, EcosystemConfig, + AppsChainConfig, AppsChainExplorerConfig, AppsEcosystemConfig, ChainConfig, EcosystemConfig, + ExplorerServicesConfig, }; use slugify_rs::slugify; +use url::Url; use xshell::Shell; use crate::{ @@ -20,9 +22,9 @@ use crate::{ messages::{ msg_explorer_db_name_prompt, msg_explorer_db_url_prompt, msg_explorer_initializing_database_for, msg_explorer_starting_on, - MSG_EXPLORER_FAILED_TO_DROP_DATABASE_ERR, + MSG_EXPLORER_FAILED_TO_ALLOCATE_PORTS_ERR, MSG_EXPLORER_FAILED_TO_DROP_DATABASE_ERR, }, - utils::ports::EcosystemPortsScanner, + utils::ports::{EcosystemPorts, EcosystemPortsScanner}, }; fn get_l2_rpc_url(chain_config: &ChainConfig) -> anyhow::Result { @@ -36,8 +38,9 @@ fn get_l2_rpc_url(chain_config: &ChainConfig) -> anyhow::Result { Ok(rpc_url.to_string()) } -async fn create_explorer_chain_config( +async fn build_explorer_chain_config( chain_config: &ChainConfig, + apps_chain_explorer_config: &AppsChainExplorerConfig, ) -> anyhow::Result { let l2_rpc_url = get_l2_rpc_url(chain_config)?; // Get Verification API URL from general config @@ -47,14 +50,17 @@ async fn create_explorer_chain_config( .as_ref() .map(|verifier| &verifier.url) .context("verification_url")?; + // Build API URL + let api_port = apps_chain_explorer_config.services.api_http_port; + let api_url = format!("http://127.0.0.1:{}", api_port); - // Build network config + // Build explorer chain config Ok(ExplorerChainConfig { name: chain_config.name.clone(), l2_network_name: chain_config.name.clone(), l2_chain_id: chain_config.chain_id.as_u64(), rpc_url: l2_rpc_url.to_string(), - api_url: "http://127.0.0.1:3002".to_string(), // TODO: probably need chain-level apps.yaml as well + api_url: api_url.to_string(), base_token_address: L2_BASE_TOKEN_ADDRESS.to_string(), hostnames: Vec::new(), icon: "/images/icons/zksync-arrows.svg".to_string(), @@ -66,11 +72,12 @@ async fn create_explorer_chain_config( }) } -pub async fn create_and_save_explorer_chain_config( +pub async fn create_explorer_chain_config( chain_config: &ChainConfig, + apps_chain_config: &AppsChainExplorerConfig, shell: &Shell, ) -> anyhow::Result { - let explorer_config = create_explorer_chain_config(chain_config).await?; + let explorer_config = build_explorer_chain_config(chain_config, apps_chain_config).await?; let config_path = ExplorerChainConfig::get_config_path(&shell.current_dir(), &chain_config.name); explorer_config.save(shell, config_path)?; @@ -80,6 +87,9 @@ pub async fn create_and_save_explorer_chain_config( pub async fn run(shell: &Shell) -> anyhow::Result<()> { let ecosystem_config: EcosystemConfig = EcosystemConfig::from_file(shell)?; let ecosystem_path = shell.current_dir(); + // Keep track of allocated ports (initialized lazily) + let mut ecosystem_ports: Option = None; + // Get ecosystem level apps.yaml config let apps_config = AppsEcosystemConfig::read_or_create_default(shell)?; // What chains to run the explorer for @@ -87,10 +97,8 @@ pub async fn run(shell: &Shell) -> anyhow::Result<()> { .explorer .chains_enabled .unwrap_or_else(|| ecosystem_config.list_of_chains()); - // Keep track of allocated ports (initialized lazily) - let mut allocated_ports: HashSet = HashSet::new(); - // For each chain - initialize if needed or read previously saved configs + // For each chain - initialize if needed or read previously created configs let mut explorer_chain_configs = Vec::new(); let mut backend_configs = Vec::new(); for chain_name in chains_enabled.iter() { @@ -98,48 +106,57 @@ pub async fn run(shell: &Shell) -> anyhow::Result<()> { .load_chain(Some(chain_name.clone())) .ok_or_else(|| anyhow::anyhow!("Failed to load chain config for {}", chain_name))?; - // Should initialize? Check if explorer backend docker compose file exists + // Should initialize? Check if apps chain config exists let mut should_initialize = false; - let backend_compose_path = - ExplorerBackendComposeConfig::get_config_path(&ecosystem_path, chain_name); + let apps_chain_config_path = AppsChainConfig::get_config_path(&ecosystem_path, &chain_name); + let apps_chain_config = match AppsChainConfig::read(shell, &apps_chain_config_path) { + Ok(config) => config, + Err(_) => { + should_initialize = true; + // Initialize explorer database + logger::info(msg_explorer_initializing_database_for(&chain_name)); + let db_config = fill_database_values_with_prompt(&chain_config); + initialize_explorer_database(&db_config).await?; + + // Allocate ports for backend services + let services_config = + allocate_explorer_services_ports(&ecosystem_path, &mut ecosystem_ports) + .context(MSG_EXPLORER_FAILED_TO_ALLOCATE_PORTS_ERR)?; + + // Build and save apps chain config + let app_chain_config = AppsChainConfig { + explorer: AppsChainExplorerConfig { + database_url: db_config.full_url(), + services: services_config, + }, + }; + app_chain_config.save(shell, &apps_chain_config_path)?; + app_chain_config + } + }; + + let l2_rpc_url = get_l2_rpc_url(&chain_config)?; + let l2_rpc_url = Url::parse(&l2_rpc_url).context("Failed to parse L2 RPC URL")?; + + // Build backend compose config for the explorer chain services let backend_compose_config = - match ExplorerBackendComposeConfig::read(shell, &backend_compose_path) { - Ok(config) => config, - Err(_) => { - should_initialize = true; - // Initialize explorer database - logger::info(msg_explorer_initializing_database_for(&chain_name)); - let db_config = fill_database_values_with_prompt(&chain_config); - initialize_explorer_database(&db_config).await?; - - // Allocate ports for backend services - let service_ports: ExplorerBackendServicePorts = - allocate_explorer_services_ports(&ecosystem_path, &mut allocated_ports)?; - - let l2_rpc_url = get_l2_rpc_url(&chain_config)?; - let backend_service_config = ExplorerBackendServiceConfig { - db_url: db_config.full_url().to_string(), - l2_rpc_url: l2_rpc_url.to_string(), - service_ports, - }; - - // Create and save docker compose for chain backend services - let backend_compose_config = - ExplorerBackendComposeConfig::new(chain_name, backend_service_config)?; - backend_compose_config.save(shell, &backend_compose_path)?; - backend_compose_config - } - }; + ExplorerBackendComposeConfig::new(chain_name, l2_rpc_url, &apps_chain_config.explorer)?; backend_configs.push(backend_compose_config); // Read or create explorer chain config (JSON) let explorer_chain_config_path = ExplorerChainConfig::get_config_path(&ecosystem_path, chain_name); let explorer_chain_config = match should_initialize { - true => create_and_save_explorer_chain_config(&chain_config, shell).await?, + true => { + create_explorer_chain_config(&chain_config, &apps_chain_config.explorer, shell) + .await? + } false => match ExplorerChainConfig::read(shell, &explorer_chain_config_path) { Ok(config) => config, - Err(_) => create_and_save_explorer_chain_config(&chain_config, shell).await?, + Err(_) => { + create_explorer_chain_config(&chain_config, &apps_chain_config.explorer, shell) + .await? + } }, }; explorer_chain_configs.push(explorer_chain_config); @@ -174,9 +191,7 @@ pub async fn run(shell: &Shell) -> anyhow::Result<()> { fn run_explorer(shell: &Shell, explorer_compose_config_path: &Path) -> anyhow::Result<()> { if let Some(docker_compose_file) = explorer_compose_config_path.to_str() { - // TODO: Containers are keep running after the process is killed (Ctrl+C) - docker::up(shell, docker_compose_file, true) - .with_context(|| "Failed to run docker compose for Explorer")?; + docker::up(shell, docker_compose_file, false)?; } else { anyhow::bail!("Invalid docker compose file"); } @@ -206,34 +221,45 @@ pub async fn initialize_explorer_database( Ok(()) } -// TODO: WIP, need to add fallback options and add more checks -// TODO: probably need to move this to a separate module and make it more generic -pub fn allocate_explorer_services_ports( +fn allocate_explorer_services_ports( ecosystem_path: &Path, - allocated_ports: &mut HashSet, -) -> anyhow::Result { - if allocated_ports.is_empty() { - let ports = EcosystemPortsScanner::scan(ecosystem_path)?; - allocated_ports.extend(ports.get_assigned_ports()); - } + ecosystem_ports: &mut Option, +) -> anyhow::Result { + let default_ports = vec![ + DEFAULT_EXPLORER_API_PORT, + DEFAULT_EXPLORER_DATA_FETCHER_PORT, + DEFAULT_EXPLORER_WORKER_PORT, + ]; - let mut service_ports = ExplorerBackendServicePorts { - api_port: DEFAULT_EXPLORER_API_PORT, - data_fetcher_port: DEFAULT_EXPLORER_DATA_FETCHER_PORT, - worker_port: DEFAULT_EXPLORER_WORKER_PORT, + // Get or scan ecosystem folder configs to find assigned ports + let ports = match ecosystem_ports { + Some(ref mut res) => res, + None => ecosystem_ports.get_or_insert( + EcosystemPortsScanner::scan(&ecosystem_path) + .context("Failed to scan ecosystem ports")?, + ), }; - let offset = 100; - while allocated_ports.contains(&service_ports.api_port) - || allocated_ports.contains(&service_ports.data_fetcher_port) - || allocated_ports.contains(&service_ports.worker_port) - { - service_ports.api_port += offset; - service_ports.data_fetcher_port += offset; - service_ports.worker_port += offset; - } - allocated_ports.insert(service_ports.api_port); - allocated_ports.insert(service_ports.data_fetcher_port); - allocated_ports.insert(service_ports.worker_port); - Ok(service_ports) + // Try to allocate intuitive ports with an offset from the defaults + let allocated = match ports.allocate_ports(&default_ports, 3001..4000, 100) { + Ok(allocated) => allocated, + Err(_) => { + // Allocate one by one + let mut allocated = Vec::new(); + for _ in 0..3 { + let port = ports.allocate_port(3001..4000)?; + allocated.push(port); + } + allocated + } + }; + + // Build the explorer services config + let services_config = ExplorerServicesConfig { + api_http_port: allocated[0], + data_fetcher_http_port: allocated[1], + worker_http_port: allocated[2], + batches_processing_polling_interval: None, + }; + Ok(services_config.with_defaults()) } diff --git a/zk_toolbox/crates/zk_inception/src/commands/portal.rs b/zk_toolbox/crates/zk_inception/src/commands/portal.rs index 445b48e9ac24..54d90045d534 100644 --- a/zk_toolbox/crates/zk_inception/src/commands/portal.rs +++ b/zk_toolbox/crates/zk_inception/src/commands/portal.rs @@ -19,7 +19,7 @@ use crate::{ }, }; -async fn create_portal_chain_config( +async fn build_portal_chain_config( chain_config: &ChainConfig, ) -> anyhow::Result { // Get L2 RPC URL from general config @@ -84,17 +84,17 @@ async fn create_portal_chain_config( }) } -pub async fn create_and_save_portal_chain_config( +pub async fn create_portal_chain_config( chain_config: &ChainConfig, shell: &Shell, ) -> anyhow::Result { - let portal_config = create_portal_chain_config(chain_config).await?; + let portal_config = build_portal_chain_config(chain_config).await?; let config_path = PortalChainConfig::get_config_path(&shell.current_dir(), &chain_config.name); portal_config.save(shell, config_path)?; Ok(portal_config) } -async fn create_portal_runtime_config( +async fn build_portal_runtime_config( shell: &Shell, ecosystem_config: &EcosystemConfig, chain_names: Vec, @@ -108,7 +108,7 @@ async fn create_portal_runtime_config( let portal_chain_config = match PortalChainConfig::read(shell, &config_path) { Ok(config) => Some(config), Err(_) => match ecosystem_config.load_chain(Some(chain_name.clone())) { - Some(chain_config) => match create_portal_chain_config(&chain_config).await { + Some(chain_config) => match build_portal_chain_config(&chain_config).await { Ok(config) => Some(config), Err(_) => None, }, @@ -137,7 +137,7 @@ pub async fn run(shell: &Shell) -> anyhow::Result<()> { .unwrap_or_else(|| ecosystem_config.list_of_chains()); // Generate portal runtime config - let runtime_config = create_portal_runtime_config(shell, &ecosystem_config, chains_enabled) + let runtime_config = build_portal_runtime_config(shell, &ecosystem_config, chains_enabled) .await .context(MSG_PORTAL_FAILED_TO_CREATE_CONFIG_ERR)?; let config_path = PortalRuntimeConfig::get_config_path(&shell.current_dir()); diff --git a/zk_toolbox/crates/zk_inception/src/messages.rs b/zk_toolbox/crates/zk_inception/src/messages.rs index e5aad36cdcfb..aa9afd8940a6 100644 --- a/zk_toolbox/crates/zk_inception/src/messages.rs +++ b/zk_toolbox/crates/zk_inception/src/messages.rs @@ -244,6 +244,8 @@ pub(super) fn msg_portal_starting_on(host: &str, port: u16) -> String { } /// Explorer related messages +pub(super) const MSG_EXPLORER_FAILED_TO_ALLOCATE_PORTS_ERR: &str = + "Failed to allocate ports for explorer services"; pub(super) const MSG_EXPLORER_FAILED_TO_DROP_DATABASE_ERR: &str = "Failed to drop explorer database"; pub(super) fn msg_explorer_initializing_database_for(chain: &str) -> String { diff --git a/zk_toolbox/crates/zk_inception/src/utils/ports.rs b/zk_toolbox/crates/zk_inception/src/utils/ports.rs index 19907d1f80a8..9b6a72d7c5db 100644 --- a/zk_toolbox/crates/zk_inception/src/utils/ports.rs +++ b/zk_toolbox/crates/zk_inception/src/utils/ports.rs @@ -1,21 +1,32 @@ use std::{ collections::{HashMap, HashSet}, fs, + ops::Range, path::Path, }; -use anyhow::{Context, Result}; +use anyhow::Result; +use common::logger; use serde_yaml::Value; #[derive(Debug, Clone)] pub struct PortInfo { - pub file_name: String, - pub key_path: String, + pub description: String, } impl PortInfo { - pub fn to_string(&self) -> String { - format!("[{}] {}", self.file_name, self.key_path) + pub fn new(description: String) -> Self { + Self { description } + } + + pub fn from_config(file_path: &Path, key_path: String) -> Self { + Self { + description: format!("[{}] {}", file_path.display(), key_path), + } + } + + pub fn dynamically_allocated() -> Self { + Self::new("Dynamically allocated".to_string()) } } @@ -26,7 +37,13 @@ pub struct EcosystemPorts { impl EcosystemPorts { pub fn new() -> Self { Self { - ports: HashMap::new(), + ports: HashMap::from([ + (3000, vec![PortInfo::new("Observability".to_string())]), + ( + 3052, + vec![PortInfo::new("External node gateway".to_string())], + ), + ]), } } @@ -34,10 +51,6 @@ impl EcosystemPorts { self.ports.keys().cloned().collect() } - pub fn get_port_info(&self, port: u16) -> Option<&Vec> { - self.ports.get(&port) - } - pub fn is_port_assigned(&self, port: u16) -> bool { self.ports.contains_key(&port) } @@ -46,12 +59,57 @@ impl EcosystemPorts { self.ports.entry(port).or_insert_with(Vec::new).push(info); } + pub fn allocate_port(&mut self, range: Range) -> Result { + for port in range { + if !self.is_port_assigned(port) { + self.add_port_info(port, PortInfo::dynamically_allocated()); + return Ok(port); + } + } + anyhow::bail!("No available ports in the given range") + } + + /// Allocates a set of ports based on given base ports, incrementing by offset if needed. + /// Tries base ports first, then base + offset, base + 2*offset, etc., until finding + /// available ports or exceeding the range. Returns allocated ports or an error if + /// no suitable ports are found. + pub fn allocate_ports( + &mut self, + base_ports: &[u16], + range: Range, + offset: u16, + ) -> Result> { + let mut i = 0; + loop { + let candidate_ports: Vec = + base_ports.iter().map(|&port| port + i * offset).collect(); + + // Check if all candidate ports are within the range + if candidate_ports.iter().any(|&port| !range.contains(&port)) { + anyhow::bail!("No suitable ports found within the given range"); + } + + // Check if all candidate ports are available + if candidate_ports + .iter() + .all(|&port| !self.is_port_assigned(port)) + { + // Allocate all ports + for &port in &candidate_ports { + self.add_port_info(port, PortInfo::dynamically_allocated()); + } + return Ok(candidate_ports); + } + i += 1; + } + } + pub fn print(&self) { let mut port_vec: Vec<_> = self.ports.iter().collect(); port_vec.sort_by_key(|&(key, _)| key); for (port, port_infos) in port_vec { for port_info in port_infos { - println!("{} > {}", port_info.to_string(), port); + println!("{} > {}", port_info.description, port); } } } @@ -60,10 +118,11 @@ impl EcosystemPorts { pub struct EcosystemPortsScanner {} impl EcosystemPortsScanner { - /// Scans the ecosystem directory for YAML files and extracts port information - /// Does not work with docker files + /// Scans the ecosystem directory for YAML files and extracts port information. + /// Specifically, it looks for keys ending with "port" and collects their values. + /// Note: Port information from Docker Compose files will not be picked up by this method. pub fn scan(ecosystem_dir: &Path) -> Result { - let skip_dirs: HashSet<&str> = vec!["db", "target", "volumes"].into_iter().collect(); + let skip_dirs: HashSet<&str> = vec!["db", "volumes"].into_iter().collect(); let mut ecosystem_ports = EcosystemPorts::new(); @@ -72,7 +131,7 @@ impl EcosystemPortsScanner { let dir = ecosystem_dir.join(subdir); if dir.is_dir() { if let Err(e) = Self::scan_directory(&dir, &mut ecosystem_ports, &skip_dirs) { - eprintln!("Error scanning directory {:?}: {}", dir, e); + logger::warn(format!("Error scanning directory {:?}: {}", dir, e)); } } } @@ -91,19 +150,14 @@ impl EcosystemPortsScanner { } } - for entry in fs::read_dir(dir).context("Failed to read directory")? { - let entry = entry.context("Failed to read directory entry")?; - let path = entry.path(); + for entry in fs::read_dir(dir)? { + let path = entry?.path(); if path.is_dir() { - if let Err(e) = Self::scan_directory(&path, ecosystem_ports, skip_dirs) { - eprintln!("Error scanning directory {:?}: {}", path, e); - } + let _ = Self::scan_directory(&path, ecosystem_ports, skip_dirs); } else if path.is_file() { if let Some(extension) = path.extension() { if extension == "yaml" || extension == "yml" { - if let Err(e) = Self::process_yaml_file(&path, ecosystem_ports) { - eprintln!("Error processing file {:?}: {}", path, e); - } + let _ = Self::process_yaml_file(&path, ecosystem_ports); } } } @@ -112,10 +166,8 @@ impl EcosystemPortsScanner { } fn process_yaml_file(file_path: &Path, ecosystem_ports: &mut EcosystemPorts) -> Result<()> { - let contents = fs::read_to_string(file_path) - .with_context(|| format!("Failed to read file: {:?}", file_path))?; - let value: Value = serde_yaml::from_str(&contents) - .with_context(|| format!("Failed to parse YAML in file: {:?}", file_path))?; + let contents = fs::read_to_string(file_path)?; + let value: Value = serde_yaml::from_str(&contents)?; Self::traverse_yaml(&value, "", file_path, ecosystem_ports); Ok(()) } @@ -139,10 +191,7 @@ impl EcosystemPortsScanner { if let Some(port) = val.as_u64().and_then(|p| u16::try_from(p).ok()) { ecosystem_ports.add_port_info( port, - PortInfo { - file_name: file_path.to_string_lossy().into_owned(), - key_path: new_path.clone(), - }, + PortInfo::from_config(file_path, new_path.clone()), ); } } From 6097f68c6b1ac5f40d93a934e8913c2d71d9ed75 Mon Sep 17 00:00:00 2001 From: Alexander Melnikov Date: Fri, 30 Aug 2024 04:26:22 -0600 Subject: [PATCH 03/12] fmt --- zk_toolbox/crates/common/src/docker.rs | 12 +++++++ zk_toolbox/crates/config/src/apps.rs | 34 ++++++++++--------- zk_toolbox/crates/config/src/consts.rs | 2 -- .../crates/config/src/docker_compose.rs | 20 +++-------- .../crates/config/src/explorer_compose.rs | 15 ++++---- 5 files changed, 43 insertions(+), 40 deletions(-) diff --git a/zk_toolbox/crates/common/src/docker.rs b/zk_toolbox/crates/common/src/docker.rs index b42ce485c7c6..3eacd9e4d2d2 100644 --- a/zk_toolbox/crates/common/src/docker.rs +++ b/zk_toolbox/crates/common/src/docker.rs @@ -1,5 +1,6 @@ use std::collections::HashMap; +use url::Url; use xshell::{cmd, Shell}; use crate::cmd::Cmd; @@ -30,3 +31,14 @@ pub fn run( } Ok(Cmd::new(cmd!(shell, "docker run {args...} {docker_image}")).run()?) } + +pub fn adjust_localhost_for_docker(mut url: Url) -> anyhow::Result { + if let Some(host) = url.host_str() { + if host == "localhost" || host == "127.0.0.1" { + url.set_host(Some("host.docker.internal"))?; + } + } else { + anyhow::bail!("Failed to parse: no host"); + } + Ok(url) +} diff --git a/zk_toolbox/crates/config/src/apps.rs b/zk_toolbox/crates/config/src/apps.rs index 09d2431ee15f..bc6cc28f1416 100644 --- a/zk_toolbox/crates/config/src/apps.rs +++ b/zk_toolbox/crates/config/src/apps.rs @@ -55,21 +55,6 @@ pub struct ExplorerServicesConfig { impl ZkToolboxConfig for AppsChainConfig {} impl AppsEcosystemConfig { - pub fn default() -> Self { - AppsEcosystemConfig { - portal: AppEcosystemConfig { - http_port: DEFAULT_PORTAL_PORT, - http_url: format!("http://127.0.0.1:{}", DEFAULT_PORTAL_PORT), - chains_enabled: None, - }, - explorer: AppEcosystemConfig { - http_port: DEFAULT_EXPLORER_PORT, - http_url: format!("http://127.0.0.1:{}", DEFAULT_EXPLORER_PORT), - chains_enabled: None, - }, - } - } - pub fn get_config_path(ecosystem_base_path: &Path) -> PathBuf { ecosystem_base_path .join(LOCAL_CONFIGS_PATH) @@ -89,6 +74,23 @@ impl AppsEcosystemConfig { } } +impl Default for AppsEcosystemConfig { + fn default() -> Self { + AppsEcosystemConfig { + portal: AppEcosystemConfig { + http_port: DEFAULT_PORTAL_PORT, + http_url: format!("http://127.0.0.1:{}", DEFAULT_PORTAL_PORT), + chains_enabled: None, + }, + explorer: AppEcosystemConfig { + http_port: DEFAULT_EXPLORER_PORT, + http_url: format!("http://127.0.0.1:{}", DEFAULT_EXPLORER_PORT), + chains_enabled: None, + }, + } + } +} + impl AppsChainConfig { pub fn new(explorer: AppsChainExplorerConfig) -> Self { AppsChainConfig { explorer } @@ -106,7 +108,7 @@ impl AppsChainConfig { impl ExplorerServicesConfig { pub fn get_batches_processing_polling_interval(&self) -> u64 { self.batches_processing_polling_interval - .unwrap_or_else(|| 1000) + .unwrap_or(1000) } pub fn with_defaults(mut self) -> Self { diff --git a/zk_toolbox/crates/config/src/consts.rs b/zk_toolbox/crates/config/src/consts.rs index 70fd4e659d09..5754c94beb09 100644 --- a/zk_toolbox/crates/config/src/consts.rs +++ b/zk_toolbox/crates/config/src/consts.rs @@ -48,8 +48,6 @@ pub const EXPLORER_RUNTIME_CONFIG_FILE: &str = "explorer.config.js"; pub const EXPLORER_CHAIN_CONFIG_FILE: &str = "explorer.config.json"; /// Name of explorer docker compose file (auto-generated) pub const EXPLORER_DOCKER_COMPOSE_FILE: &str = "explorer-docker-compose.yml"; -/// Name of chain-level explorer backend docker compose file -pub const EXPLORER_BACKEND_DOCKER_COMPOSE_FILE: &str = "explorer-backend-docker-compose.yml"; /// Path to ecosystem contacts pub(crate) const ECOSYSTEM_PATH: &str = "etc/env/ecosystems"; diff --git a/zk_toolbox/crates/config/src/docker_compose.rs b/zk_toolbox/crates/config/src/docker_compose.rs index a0a0914c1a7b..92f62fa3010d 100644 --- a/zk_toolbox/crates/config/src/docker_compose.rs +++ b/zk_toolbox/crates/config/src/docker_compose.rs @@ -1,7 +1,6 @@ use std::collections::HashMap; use serde::{Deserialize, Serialize}; -use url::Url; use crate::traits::ZkToolboxConfig; @@ -41,19 +40,10 @@ impl DockerComposeConfig { pub fn add_service(&mut self, name: &str, service: DockerComposeService) { self.services.insert(name.to_string(), service); } +} - pub fn adjust_host_for_docker(mut url: Url) -> anyhow::Result { - if let Some(host) = url.host_str() { - if Self::is_localhost(host) { - url.set_host(Some("host.docker.internal"))?; - } - } else { - anyhow::bail!("Failed to parse: no host"); - } - Ok(url) - } - - fn is_localhost(host: &str) -> bool { - host == "localhost" || host == "127.0.0.1" || host == "[::1]" +impl Default for DockerComposeConfig { + fn default() -> Self { + Self::new() } -} +} \ No newline at end of file diff --git a/zk_toolbox/crates/config/src/explorer_compose.rs b/zk_toolbox/crates/config/src/explorer_compose.rs index aca07b5133ac..926511b6d4ec 100644 --- a/zk_toolbox/crates/config/src/explorer_compose.rs +++ b/zk_toolbox/crates/config/src/explorer_compose.rs @@ -4,6 +4,7 @@ use std::{ }; use anyhow::Context; +use common::docker::adjust_localhost_for_docker; use serde::{Deserialize, Serialize}; use url::Url; @@ -25,8 +26,8 @@ pub struct ExplorerComposeConfig { impl ZkToolboxConfig for ExplorerComposeConfig {} /// Chain-level explorer backend docker compose config. It contains the configuration for -/// api, data fetcher, and worker services. This configs is generated during "explorer" command -/// and serves as a building block for the main explorer +/// api, data fetcher, and worker services. This config is generated during "explorer" command +/// and serves as a building block for the main explorer docker compose file. #[derive(Debug, Serialize, Deserialize, Clone)] pub struct ExplorerBackendComposeConfig { #[serde(flatten)] @@ -101,8 +102,8 @@ impl ExplorerBackendComposeConfig { l2_rpc_url: Url, config: &AppsChainExplorerConfig, ) -> anyhow::Result { - let db_url = DockerComposeConfig::adjust_host_for_docker(config.database_url.clone())?; - let l2_rpc_url = DockerComposeConfig::adjust_host_for_docker(l2_rpc_url)?; + let db_url = adjust_localhost_for_docker(config.database_url.clone())?; + let l2_rpc_url = adjust_localhost_for_docker(l2_rpc_url)?; // Parse database URL let db_host = db_url @@ -122,14 +123,14 @@ impl ExplorerBackendComposeConfig { Self::create_api_service( chain_name, config.services.api_http_port, - &db_url.to_string(), + db_url.as_ref(), ), ); services.insert( format!("explorer-data-fetcher-{}", chain_name), Self::create_data_fetcher_service( config.services.data_fetcher_http_port, - &l2_rpc_url.to_string(), + l2_rpc_url.as_ref(), ), ); services.insert( @@ -138,7 +139,7 @@ impl ExplorerBackendComposeConfig { chain_name, config.services.worker_http_port, config.services.data_fetcher_http_port, - &l2_rpc_url.to_string(), + l2_rpc_url.as_ref(), &db_host, &db_user, &db_password, From 56a3be90ad6e1684b8a548a92666551e7ea4ca63 Mon Sep 17 00:00:00 2001 From: Alexander Melnikov Date: Fri, 30 Aug 2024 19:37:08 -0600 Subject: [PATCH 04/12] lint + comments --- zk_toolbox/crates/config/src/apps.rs | 3 +- zk_toolbox/crates/config/src/consts.rs | 10 ++ .../crates/config/src/docker_compose.rs | 11 +- .../crates/config/src/explorer_compose.rs | 119 ++++++++++-------- .../zk_inception/src/commands/explorer.rs | 10 +- zk_toolbox/crates/zk_inception/src/consts.rs | 1 + .../crates/zk_inception/src/defaults.rs | 4 +- .../crates/zk_inception/src/utils/ports.rs | 2 +- 8 files changed, 88 insertions(+), 72 deletions(-) diff --git a/zk_toolbox/crates/config/src/apps.rs b/zk_toolbox/crates/config/src/apps.rs index bc6cc28f1416..e40f7839ddda 100644 --- a/zk_toolbox/crates/config/src/apps.rs +++ b/zk_toolbox/crates/config/src/apps.rs @@ -7,6 +7,7 @@ use xshell::Shell; use crate::{ consts::{APPS_CONFIG_FILE, LOCAL_CHAINS_PATH, LOCAL_CONFIGS_PATH}, traits::{FileConfigWithDefaultName, ReadConfig, SaveConfig, ZkToolboxConfig}, + EXPLORER_BATCHES_PROCESSING_POLLING_INTERVAL, }; pub const DEFAULT_EXPLORER_PORT: u16 = 3010; @@ -108,7 +109,7 @@ impl AppsChainConfig { impl ExplorerServicesConfig { pub fn get_batches_processing_polling_interval(&self) -> u64 { self.batches_processing_polling_interval - .unwrap_or(1000) + .unwrap_or(EXPLORER_BATCHES_PROCESSING_POLLING_INTERVAL) } pub fn with_defaults(mut self) -> Self { diff --git a/zk_toolbox/crates/config/src/consts.rs b/zk_toolbox/crates/config/src/consts.rs index 01b44660961c..0913d8e4908d 100644 --- a/zk_toolbox/crates/config/src/consts.rs +++ b/zk_toolbox/crates/config/src/consts.rs @@ -50,6 +50,16 @@ pub const EXPLORER_CHAIN_CONFIG_FILE: &str = "explorer.config.json"; /// Name of explorer docker compose file (auto-generated) pub const EXPLORER_DOCKER_COMPOSE_FILE: &str = "explorer-docker-compose.yml"; +pub const EXPLORER_API_DOCKER_IMAGE: &str = "matterlabs/block-explorer-api"; +pub const EXPLORER_APP_DOCKER_IMAGE: &str = "matterlabs/block-explorer-app"; +pub const EXPLORER_DATA_FETCHER_DOCKER_IMAGE: &str = "matterlabs/block-explorer-data-fetcher"; +pub const EXPLORER_WORKER_DOCKER_IMAGE: &str = "matterlabs/block-explorer-worker"; + +/// Path to the JS runtime config for the block-explorer-app docker container to be mounted to +pub const EXPLORER_APP_DOCKER_CONFIG_PATH: &str = "/usr/src/app/packages/app/dist/config.js"; +/// Interval (in milliseconds) for polling new batches to process in explorer app +pub const EXPLORER_BATCHES_PROCESSING_POLLING_INTERVAL: u64 = 1000; + /// Path to ecosystem contacts pub(crate) const ECOSYSTEM_PATH: &str = "etc/env/ecosystems"; diff --git a/zk_toolbox/crates/config/src/docker_compose.rs b/zk_toolbox/crates/config/src/docker_compose.rs index 92f62fa3010d..650ef90f6194 100644 --- a/zk_toolbox/crates/config/src/docker_compose.rs +++ b/zk_toolbox/crates/config/src/docker_compose.rs @@ -4,7 +4,7 @@ use serde::{Deserialize, Serialize}; use crate::traits::ZkToolboxConfig; -#[derive(Debug, Serialize, Deserialize, Clone)] +#[derive(Debug, Default, Serialize, Deserialize, Clone)] pub struct DockerComposeConfig { pub services: HashMap, } @@ -32,9 +32,7 @@ impl ZkToolboxConfig for DockerComposeConfig {} impl DockerComposeConfig { pub fn new() -> Self { - DockerComposeConfig { - services: HashMap::new(), - } + Self::default() } pub fn add_service(&mut self, name: &str, service: DockerComposeService) { @@ -42,8 +40,3 @@ impl DockerComposeConfig { } } -impl Default for DockerComposeConfig { - fn default() -> Self { - Self::new() - } -} \ No newline at end of file diff --git a/zk_toolbox/crates/config/src/explorer_compose.rs b/zk_toolbox/crates/config/src/explorer_compose.rs index 926511b6d4ec..528f5aee3b0e 100644 --- a/zk_toolbox/crates/config/src/explorer_compose.rs +++ b/zk_toolbox/crates/config/src/explorer_compose.rs @@ -4,13 +4,17 @@ use std::{ }; use anyhow::Context; -use common::docker::adjust_localhost_for_docker; +use common::{db, docker::adjust_localhost_for_docker}; use serde::{Deserialize, Serialize}; use url::Url; use crate::{ apps::AppsChainExplorerConfig, - consts::{EXPLORER_DOCKER_COMPOSE_FILE, LOCAL_CONFIGS_PATH, LOCAL_GENERATED_PATH}, + consts::{ + EXPLORER_API_DOCKER_IMAGE, EXPLORER_APP_DOCKER_CONFIG_PATH, EXPLORER_APP_DOCKER_IMAGE, + EXPLORER_DATA_FETCHER_DOCKER_IMAGE, EXPLORER_DOCKER_COMPOSE_FILE, + EXPLORER_WORKER_DOCKER_IMAGE, LOCAL_CONFIGS_PATH, LOCAL_GENERATED_PATH, + }, docker_compose::{DockerComposeConfig, DockerComposeService}, traits::ZkToolboxConfig, }; @@ -34,6 +38,7 @@ pub struct ExplorerBackendComposeConfig { pub docker_compose: DockerComposeConfig, } +/// Strcuture to hold the parameters for the explorer app service. #[derive(Debug, Clone)] pub struct ExplorerAppServiceConfig { pub port: u16, @@ -41,6 +46,8 @@ pub struct ExplorerAppServiceConfig { } impl ExplorerComposeConfig { + const APP_NAME: &'static str = "explorer-app"; // app service name in the docker compose + pub fn new( app_config: ExplorerAppServiceConfig, backend_configs: Vec, @@ -51,7 +58,7 @@ impl ExplorerComposeConfig { // Add services from backend configs for backend_config in backend_configs.iter() { for (service_name, service) in &backend_config.docker_compose.services { - if service.image.contains("block-explorer-api") { + if service.image.starts_with(EXPLORER_API_DOCKER_IMAGE) { app_depends_on.push(service_name.clone()); } services.insert(service_name.clone(), service.clone()); @@ -59,7 +66,7 @@ impl ExplorerComposeConfig { } services.insert( - "explorer-app".to_string(), + Self::APP_NAME.to_string(), Self::create_app_service(app_config, Some(app_depends_on)), ); @@ -74,12 +81,13 @@ impl ExplorerComposeConfig { depends_on: Option>, ) -> DockerComposeService { DockerComposeService { - image: "matterlabs/block-explorer-app".to_string(), + image: EXPLORER_APP_DOCKER_IMAGE.to_string(), platform: Some("linux/amd64".to_string()), ports: Some(vec![format!("{}:3010", app_config.port)]), volumes: Some(vec![format!( - "{}:/usr/src/app/packages/app/dist/config.js", - app_config.config_path.display() + "{}:{}", + app_config.config_path.display(), + EXPLORER_APP_DOCKER_CONFIG_PATH.to_string(), )]), depends_on, restart: None, @@ -105,48 +113,29 @@ impl ExplorerBackendComposeConfig { let db_url = adjust_localhost_for_docker(config.database_url.clone())?; let l2_rpc_url = adjust_localhost_for_docker(l2_rpc_url)?; - // Parse database URL - let db_host = db_url - .host_str() - .context("Failed to parse database URL: no host")? - .to_string(); - let db_user = db_url.username().to_string(); - let db_password = db_url - .password() - .context("Failed to parse database URL: no password")? - .to_string(); - let db_name = db_url.path().trim_start_matches('/').to_string(); - let mut services: HashMap = HashMap::new(); services.insert( - format!("explorer-api-{}", chain_name), - Self::create_api_service( - chain_name, - config.services.api_http_port, - db_url.as_ref(), - ), + Self::api_name(chain_name), + Self::create_api_service(chain_name, config.services.api_http_port, db_url.as_ref()), ); services.insert( - format!("explorer-data-fetcher-{}", chain_name), + Self::data_fetcher_name(chain_name), Self::create_data_fetcher_service( config.services.data_fetcher_http_port, l2_rpc_url.as_ref(), ), ); - services.insert( - format!("explorer-worker-{}", chain_name), - Self::create_worker_service( - chain_name, - config.services.worker_http_port, - config.services.data_fetcher_http_port, - l2_rpc_url.as_ref(), - &db_host, - &db_user, - &db_password, - &db_name, - config.services.get_batches_processing_polling_interval(), - ), - ); + + let worker = Self::create_worker_service( + chain_name, + config.services.worker_http_port, + config.services.data_fetcher_http_port, + l2_rpc_url.as_ref(), + &db_url, + config.services.get_batches_processing_polling_interval(), + ) + .context("Failed to create worker service")?; + services.insert(Self::worker_name(chain_name), worker); Ok(Self { docker_compose: DockerComposeConfig { services }, @@ -155,11 +144,11 @@ impl ExplorerBackendComposeConfig { fn create_api_service(chain_name: &str, port: u16, db_url: &str) -> DockerComposeService { DockerComposeService { - image: "matterlabs/block-explorer-api".to_string(), + image: EXPLORER_API_DOCKER_IMAGE.to_string(), platform: Some("linux/amd64".to_string()), ports: Some(vec![format!("{}:{}", port, port)]), volumes: None, - depends_on: Some(vec![format!("explorer-worker-{}", chain_name)]), + depends_on: Some(vec![Self::worker_name(chain_name)]), restart: None, environment: Some(HashMap::from([ ("PORT".to_string(), port.to_string()), @@ -173,7 +162,7 @@ impl ExplorerBackendComposeConfig { fn create_data_fetcher_service(port: u16, l2_rpc_url: &str) -> DockerComposeService { DockerComposeService { - image: "matterlabs/block-explorer-data-fetcher".to_string(), + image: EXPLORER_DATA_FETCHER_DOCKER_IMAGE.to_string(), platform: Some("linux/amd64".to_string()), ports: Some(vec![format!("{}:{}", port, port)]), volumes: None, @@ -194,18 +183,27 @@ impl ExplorerBackendComposeConfig { port: u16, data_fetcher_port: u16, l2_rpc_url: &str, - db_host: &str, - db_user: &str, - db_password: &str, - db_name: &str, + db_url: &Url, batches_processing_polling_interval: u64, - ) -> DockerComposeService { + ) -> anyhow::Result { let data_fetcher_url = format!( - "http://explorer-data-fetcher-{}:{}", - chain_name, data_fetcher_port + "http://{}:{}", + Self::data_fetcher_name(chain_name), + data_fetcher_port ); - DockerComposeService { - image: "matterlabs/block-explorer-worker".to_string(), + + // Parse database URL + let db_config = db::DatabaseConfig::from_url(db_url)?; + let db_user = db_url.username().to_string(); + let db_password = db_url.password().unwrap_or(""); + let db_port = db_url.port().unwrap_or(5432); + let db_host = db_url + .host_str() + .context("Failed to parse database host")? + .to_string(); + + Ok(DockerComposeService { + image: EXPLORER_WORKER_DOCKER_IMAGE.to_string(), platform: Some("linux/amd64".to_string()), ports: None, volumes: None, @@ -216,9 +214,10 @@ impl ExplorerBackendComposeConfig { ("LOG_LEVEL".to_string(), "verbose".to_string()), ("NODE_ENV".to_string(), "development".to_string()), ("DATABASE_HOST".to_string(), db_host.to_string()), + ("DATABASE_PORT".to_string(), db_port.to_string()), ("DATABASE_USER".to_string(), db_user.to_string()), ("DATABASE_PASSWORD".to_string(), db_password.to_string()), - ("DATABASE_NAME".to_string(), db_name.to_string()), + ("DATABASE_NAME".to_string(), db_config.name.to_string()), ("BLOCKCHAIN_RPC_URL".to_string(), l2_rpc_url.to_string()), ("DATA_FETCHER_URL".to_string(), data_fetcher_url), ( @@ -227,6 +226,18 @@ impl ExplorerBackendComposeConfig { ), ])), extra_hosts: Some(vec!["host.docker.internal:host-gateway".to_string()]), - } + }) + } + + fn worker_name(chain_name: &str) -> String { + format!("explorer-worker-{}", chain_name) + } + + fn api_name(chain_name: &str) -> String { + format!("explorer-api-{}", chain_name) + } + + fn data_fetcher_name(chain_name: &str) -> String { + format!("explorer-data-fetcher-{}", chain_name) } } diff --git a/zk_toolbox/crates/zk_inception/src/commands/explorer.rs b/zk_toolbox/crates/zk_inception/src/commands/explorer.rs index 995d87955151..8bbed18aad05 100644 --- a/zk_toolbox/crates/zk_inception/src/commands/explorer.rs +++ b/zk_toolbox/crates/zk_inception/src/commands/explorer.rs @@ -108,13 +108,13 @@ pub async fn run(shell: &Shell) -> anyhow::Result<()> { // Should initialize? Check if apps chain config exists let mut should_initialize = false; - let apps_chain_config_path = AppsChainConfig::get_config_path(&ecosystem_path, &chain_name); + let apps_chain_config_path = AppsChainConfig::get_config_path(&ecosystem_path, chain_name); let apps_chain_config = match AppsChainConfig::read(shell, &apps_chain_config_path) { Ok(config) => config, Err(_) => { should_initialize = true; // Initialize explorer database - logger::info(msg_explorer_initializing_database_for(&chain_name)); + logger::info(msg_explorer_initializing_database_for(chain_name)); let db_config = fill_database_values_with_prompt(&chain_config); initialize_explorer_database(&db_config).await?; @@ -174,7 +174,7 @@ pub async fn run(shell: &Shell) -> anyhow::Result<()> { }; let explorer_compose_config = ExplorerComposeConfig::new(app_config, backend_configs)?; let explorer_compose_path = ExplorerComposeConfig::get_config_path(&ecosystem_path); - explorer_compose_config.save(&shell, &explorer_compose_path)?; + explorer_compose_config.save(shell, &explorer_compose_path)?; // Launch explorer using docker compose logger::info(format!( @@ -208,7 +208,7 @@ fn fill_database_values_with_prompt(config: &ChainConfig) -> db::DatabaseConfig .default(&defaul_db_name) .ask(); let explorer_db_name = slugify!(&explorer_db_name, separator = "_"); - return db::DatabaseConfig::new(explorer_db_url, explorer_db_name); + db::DatabaseConfig::new(explorer_db_url, explorer_db_name) } pub async fn initialize_explorer_database( @@ -235,7 +235,7 @@ fn allocate_explorer_services_ports( let ports = match ecosystem_ports { Some(ref mut res) => res, None => ecosystem_ports.get_or_insert( - EcosystemPortsScanner::scan(&ecosystem_path) + EcosystemPortsScanner::scan(ecosystem_path) .context("Failed to scan ecosystem ports")?, ), }; diff --git a/zk_toolbox/crates/zk_inception/src/consts.rs b/zk_toolbox/crates/zk_inception/src/consts.rs index 7463dc28570e..49f6c8a9ed3f 100644 --- a/zk_toolbox/crates/zk_inception/src/consts.rs +++ b/zk_toolbox/crates/zk_inception/src/consts.rs @@ -8,5 +8,6 @@ pub const DEFAULT_CREDENTIALS_FILE: &str = "~/.config/gcloud/application_default pub const DEFAULT_PROOF_STORE_DIR: &str = "artifacts"; pub const BELLMAN_CUDA_DIR: &str = "era-bellman-cuda"; pub const L2_BASE_TOKEN_ADDRESS: &str = "0x000000000000000000000000000000000000800A"; + pub const PORTAL_DOCKER_IMAGE: &str = "matterlabs/dapp-portal"; pub const PORTAL_DOCKER_CONTAINER_PORT: u16 = 3000; diff --git a/zk_toolbox/crates/zk_inception/src/defaults.rs b/zk_toolbox/crates/zk_inception/src/defaults.rs index 66c48d867360..8e8d16bca9ec 100644 --- a/zk_toolbox/crates/zk_inception/src/defaults.rs +++ b/zk_toolbox/crates/zk_inception/src/defaults.rs @@ -48,11 +48,11 @@ pub fn generate_db_names(config: &ChainConfig) -> DBNames { } pub fn generate_explorer_db_name(config: &ChainConfig) -> String { - return format!( + format!( "zksync_explorer_{}_{}", config.l1_network.to_string().to_ascii_lowercase(), config.name - ); + ) } pub fn generate_external_node_db_name(config: &ChainConfig) -> String { diff --git a/zk_toolbox/crates/zk_inception/src/utils/ports.rs b/zk_toolbox/crates/zk_inception/src/utils/ports.rs index 9b6a72d7c5db..029970c6ae2c 100644 --- a/zk_toolbox/crates/zk_inception/src/utils/ports.rs +++ b/zk_toolbox/crates/zk_inception/src/utils/ports.rs @@ -56,7 +56,7 @@ impl EcosystemPorts { } pub fn add_port_info(&mut self, port: u16, info: PortInfo) { - self.ports.entry(port).or_insert_with(Vec::new).push(info); + self.ports.entry(port).or_default().push(info); } pub fn allocate_port(&mut self, range: Range) -> Result { From 84fc49c40681e877a532e32d0e68c42751bd7eda Mon Sep 17 00:00:00 2001 From: Alexander Melnikov Date: Mon, 2 Sep 2024 19:39:57 -0600 Subject: [PATCH 05/12] addresses comments --- zk_toolbox/crates/config/src/apps.rs | 30 ++- zk_toolbox/crates/config/src/consts.rs | 5 + .../crates/config/src/docker_compose.rs | 3 +- .../crates/config/src/explorer_compose.rs | 6 +- zk_toolbox/crates/config/src/general.rs | 13 +- zk_toolbox/crates/config/src/traits.rs | 5 + .../zk_inception/src/commands/explorer.rs | 71 +++---- .../zk_inception/src/commands/portal.rs | 21 +- .../crates/zk_inception/src/utils/ports.rs | 180 +++++++++++++----- 9 files changed, 211 insertions(+), 123 deletions(-) diff --git a/zk_toolbox/crates/config/src/apps.rs b/zk_toolbox/crates/config/src/apps.rs index e40f7839ddda..5acff173732c 100644 --- a/zk_toolbox/crates/config/src/apps.rs +++ b/zk_toolbox/crates/config/src/apps.rs @@ -5,14 +5,14 @@ use url::Url; use xshell::Shell; use crate::{ - consts::{APPS_CONFIG_FILE, LOCAL_CHAINS_PATH, LOCAL_CONFIGS_PATH}, + consts::{ + APPS_CONFIG_FILE, DEFAULT_EXPLORER_PORT, DEFAULT_PORTAL_PORT, LOCAL_CHAINS_PATH, + LOCAL_CONFIGS_PATH, + }, traits::{FileConfigWithDefaultName, ReadConfig, SaveConfig, ZkToolboxConfig}, EXPLORER_BATCHES_PROCESSING_POLLING_INTERVAL, }; -pub const DEFAULT_EXPLORER_PORT: u16 = 3010; -pub const DEFAULT_PORTAL_PORT: u16 = 3030; - /// Ecosystem level configuration for the apps (portal and explorer). #[derive(Debug, Serialize, Deserialize, Clone)] pub struct AppsEcosystemConfig { @@ -24,8 +24,6 @@ pub struct AppsEcosystemConfig { pub struct AppEcosystemConfig { pub http_port: u16, pub http_url: String, - #[serde(skip_serializing_if = "Option::is_none")] - pub chains_enabled: Option>, } impl ZkToolboxConfig for AppsEcosystemConfig {} @@ -50,7 +48,7 @@ pub struct ExplorerServicesConfig { pub api_http_port: u16, pub data_fetcher_http_port: u16, pub worker_http_port: u16, - pub batches_processing_polling_interval: Option, + pub batches_processing_polling_interval: u64, } impl ZkToolboxConfig for AppsChainConfig {} @@ -81,12 +79,10 @@ impl Default for AppsEcosystemConfig { portal: AppEcosystemConfig { http_port: DEFAULT_PORTAL_PORT, http_url: format!("http://127.0.0.1:{}", DEFAULT_PORTAL_PORT), - chains_enabled: None, }, explorer: AppEcosystemConfig { http_port: DEFAULT_EXPLORER_PORT, http_url: format!("http://127.0.0.1:{}", DEFAULT_EXPLORER_PORT), - chains_enabled: None, }, } } @@ -107,14 +103,12 @@ impl AppsChainConfig { } impl ExplorerServicesConfig { - pub fn get_batches_processing_polling_interval(&self) -> u64 { - self.batches_processing_polling_interval - .unwrap_or(EXPLORER_BATCHES_PROCESSING_POLLING_INTERVAL) - } - - pub fn with_defaults(mut self) -> Self { - self.batches_processing_polling_interval = - Some(self.get_batches_processing_polling_interval()); - self + pub fn new(api_http_port: u16, data_fetcher_http_port: u16, worker_http_port: u16) -> Self { + ExplorerServicesConfig { + api_http_port, + data_fetcher_http_port, + worker_http_port, + batches_processing_polling_interval: EXPLORER_BATCHES_PROCESSING_POLLING_INTERVAL, + } } } diff --git a/zk_toolbox/crates/config/src/consts.rs b/zk_toolbox/crates/config/src/consts.rs index 0913d8e4908d..1867f5be88fb 100644 --- a/zk_toolbox/crates/config/src/consts.rs +++ b/zk_toolbox/crates/config/src/consts.rs @@ -50,6 +50,11 @@ pub const EXPLORER_CHAIN_CONFIG_FILE: &str = "explorer.config.json"; /// Name of explorer docker compose file (auto-generated) pub const EXPLORER_DOCKER_COMPOSE_FILE: &str = "explorer-docker-compose.yml"; +/// Default port for the explorer app +pub const DEFAULT_EXPLORER_PORT: u16 = 3010; +/// Default port for the portal app +pub const DEFAULT_PORTAL_PORT: u16 = 3030; + pub const EXPLORER_API_DOCKER_IMAGE: &str = "matterlabs/block-explorer-api"; pub const EXPLORER_APP_DOCKER_IMAGE: &str = "matterlabs/block-explorer-app"; pub const EXPLORER_DATA_FETCHER_DOCKER_IMAGE: &str = "matterlabs/block-explorer-data-fetcher"; diff --git a/zk_toolbox/crates/config/src/docker_compose.rs b/zk_toolbox/crates/config/src/docker_compose.rs index 650ef90f6194..01a78766cf91 100644 --- a/zk_toolbox/crates/config/src/docker_compose.rs +++ b/zk_toolbox/crates/config/src/docker_compose.rs @@ -26,6 +26,8 @@ pub struct DockerComposeService { pub restart: Option, #[serde(skip_serializing_if = "Option::is_none")] pub extra_hosts: Option>, + #[serde(flatten)] + pub other: serde_json::Value, } impl ZkToolboxConfig for DockerComposeConfig {} @@ -39,4 +41,3 @@ impl DockerComposeConfig { self.services.insert(name.to_string(), service); } } - diff --git a/zk_toolbox/crates/config/src/explorer_compose.rs b/zk_toolbox/crates/config/src/explorer_compose.rs index 528f5aee3b0e..70f63b25a49b 100644 --- a/zk_toolbox/crates/config/src/explorer_compose.rs +++ b/zk_toolbox/crates/config/src/explorer_compose.rs @@ -93,6 +93,7 @@ impl ExplorerComposeConfig { restart: None, environment: None, extra_hosts: None, + other: serde_json::Value::Null, } } @@ -132,7 +133,7 @@ impl ExplorerBackendComposeConfig { config.services.data_fetcher_http_port, l2_rpc_url.as_ref(), &db_url, - config.services.get_batches_processing_polling_interval(), + config.services.batches_processing_polling_interval, ) .context("Failed to create worker service")?; services.insert(Self::worker_name(chain_name), worker); @@ -157,6 +158,7 @@ impl ExplorerBackendComposeConfig { ("DATABASE_URL".to_string(), db_url.to_string()), ])), extra_hosts: Some(vec!["host.docker.internal:host-gateway".to_string()]), + other: serde_json::Value::Null, } } @@ -175,6 +177,7 @@ impl ExplorerBackendComposeConfig { ("BLOCKCHAIN_RPC_URL".to_string(), l2_rpc_url.to_string()), ])), extra_hosts: Some(vec!["host.docker.internal:host-gateway".to_string()]), + other: serde_json::Value::Null, } } @@ -226,6 +229,7 @@ impl ExplorerBackendComposeConfig { ), ])), extra_hosts: Some(vec!["host.docker.internal:host-gateway".to_string()]), + other: serde_json::Value::Null, }) } diff --git a/zk_toolbox/crates/config/src/general.rs b/zk_toolbox/crates/config/src/general.rs index 3426b21c6f6e..a9e96f0ce95d 100644 --- a/zk_toolbox/crates/config/src/general.rs +++ b/zk_toolbox/crates/config/src/general.rs @@ -9,7 +9,7 @@ use zksync_protobuf_config::{decode_yaml_repr, encode_yaml_repr}; use crate::{ consts::GENERAL_FILE, - traits::{FileConfigWithDefaultName, ReadConfig, SaveConfig}, + traits::{ConfigWithL2RpcUrl, FileConfigWithDefaultName, ReadConfig, SaveConfig}, }; pub struct RocksDbs { @@ -211,3 +211,14 @@ impl ReadConfig for GeneralConfig { decode_yaml_repr::(&path, false) } } + +impl ConfigWithL2RpcUrl for GeneralConfig { + fn get_l2_rpc_url(&self) -> anyhow::Result { + self.api_config + .as_ref() + .map(|api_config| &api_config.web3_json_rpc.http_url) + .context("API config is missing")? + .parse() + .context("Failed to parse L2 RPC URL") + } +} diff --git a/zk_toolbox/crates/config/src/traits.rs b/zk_toolbox/crates/config/src/traits.rs index 1f00b39b040a..bb0722762e31 100644 --- a/zk_toolbox/crates/config/src/traits.rs +++ b/zk_toolbox/crates/config/src/traits.rs @@ -5,6 +5,7 @@ use common::files::{ read_json_file, read_toml_file, read_yaml_file, save_json_file, save_toml_file, save_yaml_file, }; use serde::{de::DeserializeOwned, Serialize}; +use url::Url; use xshell::Shell; // Configs that we use only inside zk toolbox, we don't have protobuf implementation for them. @@ -156,3 +157,7 @@ fn save_with_comment( } Ok(()) } + +pub trait ConfigWithL2RpcUrl { + fn get_l2_rpc_url(&self) -> anyhow::Result; +} diff --git a/zk_toolbox/crates/zk_inception/src/commands/explorer.rs b/zk_toolbox/crates/zk_inception/src/commands/explorer.rs index 8bbed18aad05..a7b2f3f86161 100644 --- a/zk_toolbox/crates/zk_inception/src/commands/explorer.rs +++ b/zk_toolbox/crates/zk_inception/src/commands/explorer.rs @@ -1,16 +1,15 @@ use std::path::Path; use anyhow::Context; -use common::{db, docker, logger, Prompt}; +use common::{config::global_config, db, docker, logger, Prompt}; use config::{ explorer::*, explorer_compose::*, - traits::{ReadConfig, SaveConfig}, + traits::{ConfigWithL2RpcUrl, ReadConfig, SaveConfig}, AppsChainConfig, AppsChainExplorerConfig, AppsEcosystemConfig, ChainConfig, EcosystemConfig, ExplorerServicesConfig, }; use slugify_rs::slugify; -use url::Url; use xshell::Shell; use crate::{ @@ -27,24 +26,14 @@ use crate::{ utils::ports::{EcosystemPorts, EcosystemPortsScanner}, }; -fn get_l2_rpc_url(chain_config: &ChainConfig) -> anyhow::Result { - // Get L2 RPC URL from general config - let general_config = chain_config.get_general_config()?; - let rpc_url = general_config - .api_config - .as_ref() - .map(|api_config| &api_config.web3_json_rpc.http_url) - .context("api_config")?; - Ok(rpc_url.to_string()) -} - -async fn build_explorer_chain_config( +fn build_explorer_chain_config( chain_config: &ChainConfig, apps_chain_explorer_config: &AppsChainExplorerConfig, ) -> anyhow::Result { - let l2_rpc_url = get_l2_rpc_url(chain_config)?; - // Get Verification API URL from general config let general_config = chain_config.get_general_config()?; + // Get L2 RPC URL from general config + let l2_rpc_url = general_config.get_l2_rpc_url()?; + // Get Verification API URL from general config let verification_api_url = general_config .contract_verifier .as_ref() @@ -72,12 +61,12 @@ async fn build_explorer_chain_config( }) } -pub async fn create_explorer_chain_config( +pub fn create_explorer_chain_config( chain_config: &ChainConfig, apps_chain_config: &AppsChainExplorerConfig, shell: &Shell, ) -> anyhow::Result { - let explorer_config = build_explorer_chain_config(chain_config, apps_chain_config).await?; + let explorer_config = build_explorer_chain_config(chain_config, apps_chain_config)?; let config_path = ExplorerChainConfig::get_config_path(&shell.current_dir(), &chain_config.name); explorer_config.save(shell, config_path)?; @@ -85,18 +74,18 @@ pub async fn create_explorer_chain_config( } pub async fn run(shell: &Shell) -> anyhow::Result<()> { - let ecosystem_config: EcosystemConfig = EcosystemConfig::from_file(shell)?; + let ecosystem_config = EcosystemConfig::from_file(shell)?; let ecosystem_path = shell.current_dir(); // Keep track of allocated ports (initialized lazily) let mut ecosystem_ports: Option = None; // Get ecosystem level apps.yaml config let apps_config = AppsEcosystemConfig::read_or_create_default(shell)?; - // What chains to run the explorer for - let chains_enabled = apps_config - .explorer - .chains_enabled - .unwrap_or_else(|| ecosystem_config.list_of_chains()); + // What chains to run the explorer for? + let chains_enabled = match global_config().chain_name { + Some(ref chain_name) => vec![chain_name.clone()], + None => ecosystem_config.list_of_chains(), + }; // For each chain - initialize if needed or read previously created configs let mut explorer_chain_configs = Vec::new(); @@ -119,9 +108,8 @@ pub async fn run(shell: &Shell) -> anyhow::Result<()> { initialize_explorer_database(&db_config).await?; // Allocate ports for backend services - let services_config = - allocate_explorer_services_ports(&ecosystem_path, &mut ecosystem_ports) - .context(MSG_EXPLORER_FAILED_TO_ALLOCATE_PORTS_ERR)?; + let services_config = allocate_explorer_services_ports(shell, &mut ecosystem_ports) + .context(MSG_EXPLORER_FAILED_TO_ALLOCATE_PORTS_ERR)?; // Build and save apps chain config let app_chain_config = AppsChainConfig { @@ -135,10 +123,8 @@ pub async fn run(shell: &Shell) -> anyhow::Result<()> { } }; - let l2_rpc_url = get_l2_rpc_url(&chain_config)?; - let l2_rpc_url = Url::parse(&l2_rpc_url).context("Failed to parse L2 RPC URL")?; - // Build backend compose config for the explorer chain services + let l2_rpc_url = chain_config.get_general_config()?.get_l2_rpc_url()?; let backend_compose_config = ExplorerBackendComposeConfig::new(chain_name, l2_rpc_url, &apps_chain_config.explorer)?; backend_configs.push(backend_compose_config); @@ -148,14 +134,12 @@ pub async fn run(shell: &Shell) -> anyhow::Result<()> { ExplorerChainConfig::get_config_path(&ecosystem_path, chain_name); let explorer_chain_config = match should_initialize { true => { - create_explorer_chain_config(&chain_config, &apps_chain_config.explorer, shell) - .await? + create_explorer_chain_config(&chain_config, &apps_chain_config.explorer, shell)? } false => match ExplorerChainConfig::read(shell, &explorer_chain_config_path) { Ok(config) => config, Err(_) => { - create_explorer_chain_config(&chain_config, &apps_chain_config.explorer, shell) - .await? + create_explorer_chain_config(&chain_config, &apps_chain_config.explorer, shell)? } }, }; @@ -222,7 +206,7 @@ pub async fn initialize_explorer_database( } fn allocate_explorer_services_ports( - ecosystem_path: &Path, + shell: &Shell, ecosystem_ports: &mut Option, ) -> anyhow::Result { let default_ports = vec![ @@ -235,8 +219,7 @@ fn allocate_explorer_services_ports( let ports = match ecosystem_ports { Some(ref mut res) => res, None => ecosystem_ports.get_or_insert( - EcosystemPortsScanner::scan(ecosystem_path) - .context("Failed to scan ecosystem ports")?, + EcosystemPortsScanner::scan(shell).context("Failed to scan ecosystem ports")?, ), }; @@ -255,11 +238,9 @@ fn allocate_explorer_services_ports( }; // Build the explorer services config - let services_config = ExplorerServicesConfig { - api_http_port: allocated[0], - data_fetcher_http_port: allocated[1], - worker_http_port: allocated[2], - batches_processing_polling_interval: None, - }; - Ok(services_config.with_defaults()) + Ok(ExplorerServicesConfig::new( + allocated[0], + allocated[1], + allocated[2], + )) } diff --git a/zk_toolbox/crates/zk_inception/src/commands/portal.rs b/zk_toolbox/crates/zk_inception/src/commands/portal.rs index 54d90045d534..d7d42d0cec0f 100644 --- a/zk_toolbox/crates/zk_inception/src/commands/portal.rs +++ b/zk_toolbox/crates/zk_inception/src/commands/portal.rs @@ -1,10 +1,10 @@ use std::{collections::HashMap, path::Path}; use anyhow::Context; -use common::{docker, ethereum, logger}; +use common::{config::global_config, docker, ethereum, logger}; use config::{ portal::*, - traits::{ReadConfig, SaveConfig}, + traits::{ConfigWithL2RpcUrl, ReadConfig, SaveConfig}, AppsEcosystemConfig, ChainConfig, EcosystemConfig, }; use ethers::types::Address; @@ -23,12 +23,7 @@ async fn build_portal_chain_config( chain_config: &ChainConfig, ) -> anyhow::Result { // Get L2 RPC URL from general config - let general_config = chain_config.get_general_config()?; - let rpc_url: &String = general_config - .api_config - .as_ref() - .map(|api_config| &api_config.web3_json_rpc.http_url) - .context("api_config")?; + let l2_rpc_url = chain_config.get_general_config()?.get_l2_rpc_url()?; // Get L1 RPC URL from secrects config let secrets_config = chain_config.get_secrets_config()?; let l1_rpc_url = secrets_config @@ -74,7 +69,7 @@ async fn build_portal_chain_config( id: chain_config.chain_id.as_u64(), key: chain_config.name.clone(), name: chain_config.name.clone(), - rpc_url: rpc_url.to_string(), + rpc_url: l2_rpc_url.to_string(), l1_network, public_l1_network_id: None, block_explorer_url: None, @@ -131,10 +126,10 @@ pub async fn run(shell: &Shell) -> anyhow::Result<()> { // Get ecosystem level apps.yaml config let apps_config = AppsEcosystemConfig::read_or_create_default(shell)?; // What chains to run the portal for? - let chains_enabled = apps_config - .portal - .chains_enabled - .unwrap_or_else(|| ecosystem_config.list_of_chains()); + let chains_enabled = match global_config().chain_name { + Some(ref chain_name) => vec![chain_name.clone()], + None => ecosystem_config.list_of_chains(), + }; // Generate portal runtime config let runtime_config = build_portal_runtime_config(shell, &ecosystem_config, chains_enabled) diff --git a/zk_toolbox/crates/zk_inception/src/utils/ports.rs b/zk_toolbox/crates/zk_inception/src/utils/ports.rs index 029970c6ae2c..31f19db38510 100644 --- a/zk_toolbox/crates/zk_inception/src/utils/ports.rs +++ b/zk_toolbox/crates/zk_inception/src/utils/ports.rs @@ -1,48 +1,31 @@ use std::{ collections::{HashMap, HashSet}, - fs, + fmt, fs, ops::Range, path::Path, }; use anyhow::Result; use common::logger; +use config::EcosystemConfig; use serde_yaml::Value; +use xshell::Shell; -#[derive(Debug, Clone)] -pub struct PortInfo { - pub description: String, -} - -impl PortInfo { - pub fn new(description: String) -> Self { - Self { description } - } - - pub fn from_config(file_path: &Path, key_path: String) -> Self { - Self { - description: format!("[{}] {}", file_path.display(), key_path), - } - } - - pub fn dynamically_allocated() -> Self { - Self::new("Dynamically allocated".to_string()) - } -} +/// Ecosystem-level directories to include in the scan +const INCLUDE_DIRS: &[&str] = &["chains", "configs"]; +/// Skip directories with these names at all levels +const SKIP_DIRS: &[&str] = &["db", "volumes"]; pub struct EcosystemPorts { - pub ports: HashMap>, + pub ports: HashMap>, } impl EcosystemPorts { pub fn new() -> Self { Self { ports: HashMap::from([ - (3000, vec![PortInfo::new("Observability".to_string())]), - ( - 3052, - vec![PortInfo::new("External node gateway".to_string())], - ), + (3000, vec!["Observability".to_string()]), + (3052, vec!["External node gateway".to_string()]), ]), } } @@ -55,14 +38,14 @@ impl EcosystemPorts { self.ports.contains_key(&port) } - pub fn add_port_info(&mut self, port: u16, info: PortInfo) { + pub fn add_port_info(&mut self, port: u16, info: String) { self.ports.entry(port).or_default().push(info); } pub fn allocate_port(&mut self, range: Range) -> Result { for port in range { if !self.is_port_assigned(port) { - self.add_port_info(port, PortInfo::dynamically_allocated()); + self.add_port_info(port, "Dynamically allocated".to_string()); return Ok(port); } } @@ -86,7 +69,12 @@ impl EcosystemPorts { // Check if all candidate ports are within the range if candidate_ports.iter().any(|&port| !range.contains(&port)) { - anyhow::bail!("No suitable ports found within the given range"); + anyhow::bail!( + "No suitable ports found within the given range {:?}. Tried {} iterations with offset {}", + range, + i, + offset + ) } // Check if all candidate ports are available @@ -96,22 +84,25 @@ impl EcosystemPorts { { // Allocate all ports for &port in &candidate_ports { - self.add_port_info(port, PortInfo::dynamically_allocated()); + self.add_port_info(port, "Dynamically allocated".to_string()); } return Ok(candidate_ports); } i += 1; } } +} - pub fn print(&self) { +impl fmt::Display for EcosystemPorts { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { let mut port_vec: Vec<_> = self.ports.iter().collect(); port_vec.sort_by_key(|&(key, _)| key); for (port, port_infos) in port_vec { for port_info in port_infos { - println!("{} > {}", port_info.description, port); + writeln!(f, "{} > {}", port_info, port)?; } } + Ok(()) } } @@ -121,13 +112,13 @@ impl EcosystemPortsScanner { /// Scans the ecosystem directory for YAML files and extracts port information. /// Specifically, it looks for keys ending with "port" and collects their values. /// Note: Port information from Docker Compose files will not be picked up by this method. - pub fn scan(ecosystem_dir: &Path) -> Result { - let skip_dirs: HashSet<&str> = vec!["db", "volumes"].into_iter().collect(); + pub fn scan(shell: &Shell) -> Result { + let _ = EcosystemConfig::from_file(shell)?; // Find and validate the ecosystem directory + let ecosystem_dir = shell.current_dir(); + let skip_dirs: HashSet<&str> = SKIP_DIRS.iter().cloned().collect(); let mut ecosystem_ports = EcosystemPorts::new(); - - let subdirs = ["chains", "configs"]; - for subdir in &subdirs { + for subdir in INCLUDE_DIRS { let dir = ecosystem_dir.join(subdir); if dir.is_dir() { if let Err(e) = Self::scan_directory(&dir, &mut ecosystem_ports, &skip_dirs) { @@ -153,11 +144,15 @@ impl EcosystemPortsScanner { for entry in fs::read_dir(dir)? { let path = entry?.path(); if path.is_dir() { - let _ = Self::scan_directory(&path, ecosystem_ports, skip_dirs); + if let Err(e) = Self::scan_directory(&path, ecosystem_ports, skip_dirs) { + logger::warn(format!("Error scanning directory {:?}: {}", path, e)); + } } else if path.is_file() { if let Some(extension) = path.extension() { if extension == "yaml" || extension == "yml" { - let _ = Self::process_yaml_file(&path, ecosystem_ports); + if let Err(e) = Self::process_yaml_file(&path, ecosystem_ports) { + logger::warn(format!("Error processing YAML file {:?}: {}", path, e)); + } } } } @@ -189,10 +184,8 @@ impl EcosystemPortsScanner { if key.as_str().map(|s| s.ends_with("port")).unwrap_or(false) { if let Some(port) = val.as_u64().and_then(|p| u16::try_from(p).ok()) { - ecosystem_ports.add_port_info( - port, - PortInfo::from_config(file_path, new_path.clone()), - ); + let description = format!("[{}] {}", file_path.display(), new_path); + ecosystem_ports.add_port_info(port, description); } } @@ -209,3 +202,102 @@ impl EcosystemPortsScanner { } } } + +#[cfg(test)] +mod tests { + use std::path::PathBuf; + + use crate::utils::ports::{EcosystemPorts, EcosystemPortsScanner}; + + #[test] + fn test_allocate_ports() { + let mut ecosystem_ports = EcosystemPorts::new(); + + // Test case 1: Allocate ports within range + let result = ecosystem_ports.allocate_ports(&[8000, 8001, 8002], 8000..9000, 100); + assert!(result.is_ok()); + let allocated_ports = result.unwrap(); + assert_eq!(allocated_ports, vec![8000, 8001, 8002]); + + // Test case 2: Allocate ports with offset + let result = ecosystem_ports.allocate_ports(&[8000, 8001, 8002], 8000..9000, 100); + assert!(result.is_ok()); + let allocated_ports = result.unwrap(); + assert_eq!(allocated_ports, vec![8100, 8101, 8102]); + + // Test case 3: Fail to allocate ports - not available with offset + let result = ecosystem_ports.allocate_ports(&[8000, 8001, 8002], 8000..8200, 100); + assert!(result.is_err()); + + // Test case 4: Fail to allocate ports - base ports outside range + let result = ecosystem_ports.allocate_ports(&[9500, 9501, 9502], 8000..9000, 100); + assert!(result.is_err()); + + // Test case 5: Allocate consecutive ports + let result = ecosystem_ports.allocate_ports(&[8000, 8001, 8002], 8000..9000, 1); + assert!(result.is_ok()); + let allocated_ports = result.unwrap(); + assert_eq!(allocated_ports, vec![8003, 8004, 8005]); + + // Test case 6: Allocate ports at the edge of the range + let result = ecosystem_ports.allocate_ports(&[8998, 8999], 8000..9000, 1); + assert!(result.is_ok()); + let allocated_ports = result.unwrap(); + assert_eq!(allocated_ports, vec![8998, 8999]); + } + + #[test] + fn test_traverse_yaml() { + let yaml_content = r#" + api: + web3_json_rpc: + http_port: 3050 + ws_port: 3051 + api_namespaces: + - eth + gas_price_scale_factor: 1.5 + prometheus: + listener_port: 3412 + push_interval_ms: 100 + contract_verifier: + port: 3070 + prometheus: + listener_port: 3412 + "#; + + let value = serde_yaml::from_str(yaml_content).unwrap(); + let mut ecosystem_ports = EcosystemPorts::new(); + let file_path: PathBuf = PathBuf::from("test_config.yaml"); + + EcosystemPortsScanner::traverse_yaml(&value, "", &file_path, &mut ecosystem_ports); + + // Assigned ports: + assert!(ecosystem_ports.is_port_assigned(3050)); + assert!(ecosystem_ports.is_port_assigned(3051)); + assert!(ecosystem_ports.is_port_assigned(3070)); + assert!(ecosystem_ports.is_port_assigned(3412)); + + // Free ports: + assert!(!ecosystem_ports.is_port_assigned(3150)); + assert!(!ecosystem_ports.is_port_assigned(3151)); + + // Check description: + let port_3050_info = ecosystem_ports.ports.get(&3050).unwrap(); + assert_eq!(port_3050_info.len(), 1); + assert_eq!( + port_3050_info[0], + "[test_config.yaml] api:web3_json_rpc:http_port" + ); + + let port_3412_info = ecosystem_ports.ports.get(&3412).unwrap(); + assert_eq!(port_3412_info.len(), 2); + assert_eq!( + port_3412_info[0], + "[test_config.yaml] api:prometheus:listener_port" + ); + assert_eq!( + port_3412_info[1], + "[test_config.yaml] prometheus:listener_port" + ); + } +} From 20d71db3b062ab45d03b60c36dfbbee5815d851a Mon Sep 17 00:00:00 2001 From: Alexander Melnikov Date: Mon, 2 Sep 2024 22:57:54 -0600 Subject: [PATCH 06/12] separated into init, run, run-backend --- zk_toolbox/crates/config/src/consts.rs | 3 - .../crates/config/src/explorer_compose.rs | 96 ++----- .../zk_inception/src/commands/explorer.rs | 246 ------------------ .../src/commands/explorer/backend.rs | 50 ++++ .../src/commands/explorer/init.rs | 191 ++++++++++++++ .../zk_inception/src/commands/explorer/mod.rs | 25 ++ .../zk_inception/src/commands/explorer/run.rs | 85 ++++++ zk_toolbox/crates/zk_inception/src/consts.rs | 3 + zk_toolbox/crates/zk_inception/src/main.rs | 7 +- .../crates/zk_inception/src/messages.rs | 19 ++ 10 files changed, 392 insertions(+), 333 deletions(-) delete mode 100644 zk_toolbox/crates/zk_inception/src/commands/explorer.rs create mode 100644 zk_toolbox/crates/zk_inception/src/commands/explorer/backend.rs create mode 100644 zk_toolbox/crates/zk_inception/src/commands/explorer/init.rs create mode 100644 zk_toolbox/crates/zk_inception/src/commands/explorer/mod.rs create mode 100644 zk_toolbox/crates/zk_inception/src/commands/explorer/run.rs diff --git a/zk_toolbox/crates/config/src/consts.rs b/zk_toolbox/crates/config/src/consts.rs index 1867f5be88fb..e861c0be7048 100644 --- a/zk_toolbox/crates/config/src/consts.rs +++ b/zk_toolbox/crates/config/src/consts.rs @@ -56,12 +56,9 @@ pub const DEFAULT_EXPLORER_PORT: u16 = 3010; pub const DEFAULT_PORTAL_PORT: u16 = 3030; pub const EXPLORER_API_DOCKER_IMAGE: &str = "matterlabs/block-explorer-api"; -pub const EXPLORER_APP_DOCKER_IMAGE: &str = "matterlabs/block-explorer-app"; pub const EXPLORER_DATA_FETCHER_DOCKER_IMAGE: &str = "matterlabs/block-explorer-data-fetcher"; pub const EXPLORER_WORKER_DOCKER_IMAGE: &str = "matterlabs/block-explorer-worker"; -/// Path to the JS runtime config for the block-explorer-app docker container to be mounted to -pub const EXPLORER_APP_DOCKER_CONFIG_PATH: &str = "/usr/src/app/packages/app/dist/config.js"; /// Interval (in milliseconds) for polling new batches to process in explorer app pub const EXPLORER_BATCHES_PROCESSING_POLLING_INTERVAL: u64 = 1000; diff --git a/zk_toolbox/crates/config/src/explorer_compose.rs b/zk_toolbox/crates/config/src/explorer_compose.rs index 70f63b25a49b..416b5c80c864 100644 --- a/zk_toolbox/crates/config/src/explorer_compose.rs +++ b/zk_toolbox/crates/config/src/explorer_compose.rs @@ -11,99 +11,24 @@ use url::Url; use crate::{ apps::AppsChainExplorerConfig, consts::{ - EXPLORER_API_DOCKER_IMAGE, EXPLORER_APP_DOCKER_CONFIG_PATH, EXPLORER_APP_DOCKER_IMAGE, - EXPLORER_DATA_FETCHER_DOCKER_IMAGE, EXPLORER_DOCKER_COMPOSE_FILE, - EXPLORER_WORKER_DOCKER_IMAGE, LOCAL_CONFIGS_PATH, LOCAL_GENERATED_PATH, + EXPLORER_API_DOCKER_IMAGE, EXPLORER_DATA_FETCHER_DOCKER_IMAGE, + EXPLORER_DOCKER_COMPOSE_FILE, EXPLORER_WORKER_DOCKER_IMAGE, LOCAL_APPS_PATH, + LOCAL_CHAINS_PATH, LOCAL_CONFIGS_PATH, }, docker_compose::{DockerComposeConfig, DockerComposeService}, traits::ZkToolboxConfig, }; -/// Explorer docker compose file. This file is auto-generated during "explorer" command -/// and is passed to Docker Compose to launch the explorer app and backend services. -#[derive(Debug, Serialize, Deserialize, Clone)] -pub struct ExplorerComposeConfig { - #[serde(flatten)] - pub docker_compose: DockerComposeConfig, -} - -impl ZkToolboxConfig for ExplorerComposeConfig {} - /// Chain-level explorer backend docker compose config. It contains the configuration for -/// api, data fetcher, and worker services. This config is generated during "explorer" command -/// and serves as a building block for the main explorer docker compose file. +/// api, data fetcher, and worker services. +/// This config is auto-generated during "explorer run-backend" command. #[derive(Debug, Serialize, Deserialize, Clone)] pub struct ExplorerBackendComposeConfig { #[serde(flatten)] pub docker_compose: DockerComposeConfig, } -/// Strcuture to hold the parameters for the explorer app service. -#[derive(Debug, Clone)] -pub struct ExplorerAppServiceConfig { - pub port: u16, - pub config_path: PathBuf, -} - -impl ExplorerComposeConfig { - const APP_NAME: &'static str = "explorer-app"; // app service name in the docker compose - - pub fn new( - app_config: ExplorerAppServiceConfig, - backend_configs: Vec, - ) -> anyhow::Result { - let mut services = HashMap::new(); - let mut app_depends_on = Vec::new(); - - // Add services from backend configs - for backend_config in backend_configs.iter() { - for (service_name, service) in &backend_config.docker_compose.services { - if service.image.starts_with(EXPLORER_API_DOCKER_IMAGE) { - app_depends_on.push(service_name.clone()); - } - services.insert(service_name.clone(), service.clone()); - } - } - - services.insert( - Self::APP_NAME.to_string(), - Self::create_app_service(app_config, Some(app_depends_on)), - ); - - let config = Self { - docker_compose: DockerComposeConfig { services }, - }; - Ok(config) - } - - fn create_app_service( - app_config: ExplorerAppServiceConfig, - depends_on: Option>, - ) -> DockerComposeService { - DockerComposeService { - image: EXPLORER_APP_DOCKER_IMAGE.to_string(), - platform: Some("linux/amd64".to_string()), - ports: Some(vec![format!("{}:3010", app_config.port)]), - volumes: Some(vec![format!( - "{}:{}", - app_config.config_path.display(), - EXPLORER_APP_DOCKER_CONFIG_PATH.to_string(), - )]), - depends_on, - restart: None, - environment: None, - extra_hosts: None, - other: serde_json::Value::Null, - } - } - - pub fn get_config_path(ecosystem_base_path: &Path) -> PathBuf { - ecosystem_base_path - .join(LOCAL_CONFIGS_PATH) - .join(LOCAL_GENERATED_PATH) - .join(EXPLORER_DOCKER_COMPOSE_FILE) - } -} +impl ZkToolboxConfig for ExplorerBackendComposeConfig {} impl ExplorerBackendComposeConfig { pub fn new( @@ -244,4 +169,13 @@ impl ExplorerBackendComposeConfig { fn data_fetcher_name(chain_name: &str) -> String { format!("explorer-data-fetcher-{}", chain_name) } + + pub fn get_config_path(ecosystem_base_path: &Path, chain_name: &str) -> PathBuf { + ecosystem_base_path + .join(LOCAL_CHAINS_PATH) + .join(chain_name) + .join(LOCAL_CONFIGS_PATH) + .join(LOCAL_APPS_PATH) + .join(EXPLORER_DOCKER_COMPOSE_FILE) + } } diff --git a/zk_toolbox/crates/zk_inception/src/commands/explorer.rs b/zk_toolbox/crates/zk_inception/src/commands/explorer.rs deleted file mode 100644 index a7b2f3f86161..000000000000 --- a/zk_toolbox/crates/zk_inception/src/commands/explorer.rs +++ /dev/null @@ -1,246 +0,0 @@ -use std::path::Path; - -use anyhow::Context; -use common::{config::global_config, db, docker, logger, Prompt}; -use config::{ - explorer::*, - explorer_compose::*, - traits::{ConfigWithL2RpcUrl, ReadConfig, SaveConfig}, - AppsChainConfig, AppsChainExplorerConfig, AppsEcosystemConfig, ChainConfig, EcosystemConfig, - ExplorerServicesConfig, -}; -use slugify_rs::slugify; -use xshell::Shell; - -use crate::{ - consts::L2_BASE_TOKEN_ADDRESS, - defaults::{ - generate_explorer_db_name, DATABASE_EXPLORER_URL, DEFAULT_EXPLORER_API_PORT, - DEFAULT_EXPLORER_DATA_FETCHER_PORT, DEFAULT_EXPLORER_WORKER_PORT, - }, - messages::{ - msg_explorer_db_name_prompt, msg_explorer_db_url_prompt, - msg_explorer_initializing_database_for, msg_explorer_starting_on, - MSG_EXPLORER_FAILED_TO_ALLOCATE_PORTS_ERR, MSG_EXPLORER_FAILED_TO_DROP_DATABASE_ERR, - }, - utils::ports::{EcosystemPorts, EcosystemPortsScanner}, -}; - -fn build_explorer_chain_config( - chain_config: &ChainConfig, - apps_chain_explorer_config: &AppsChainExplorerConfig, -) -> anyhow::Result { - let general_config = chain_config.get_general_config()?; - // Get L2 RPC URL from general config - let l2_rpc_url = general_config.get_l2_rpc_url()?; - // Get Verification API URL from general config - let verification_api_url = general_config - .contract_verifier - .as_ref() - .map(|verifier| &verifier.url) - .context("verification_url")?; - // Build API URL - let api_port = apps_chain_explorer_config.services.api_http_port; - let api_url = format!("http://127.0.0.1:{}", api_port); - - // Build explorer chain config - Ok(ExplorerChainConfig { - name: chain_config.name.clone(), - l2_network_name: chain_config.name.clone(), - l2_chain_id: chain_config.chain_id.as_u64(), - rpc_url: l2_rpc_url.to_string(), - api_url: api_url.to_string(), - base_token_address: L2_BASE_TOKEN_ADDRESS.to_string(), - hostnames: Vec::new(), - icon: "/images/icons/zksync-arrows.svg".to_string(), - maintenance: false, - published: true, - bridge_url: None, - l1_explorer_url: None, - verification_api_url: Some(verification_api_url.to_string()), - }) -} - -pub fn create_explorer_chain_config( - chain_config: &ChainConfig, - apps_chain_config: &AppsChainExplorerConfig, - shell: &Shell, -) -> anyhow::Result { - let explorer_config = build_explorer_chain_config(chain_config, apps_chain_config)?; - let config_path = - ExplorerChainConfig::get_config_path(&shell.current_dir(), &chain_config.name); - explorer_config.save(shell, config_path)?; - Ok(explorer_config) -} - -pub async fn run(shell: &Shell) -> anyhow::Result<()> { - let ecosystem_config = EcosystemConfig::from_file(shell)?; - let ecosystem_path = shell.current_dir(); - // Keep track of allocated ports (initialized lazily) - let mut ecosystem_ports: Option = None; - - // Get ecosystem level apps.yaml config - let apps_config = AppsEcosystemConfig::read_or_create_default(shell)?; - // What chains to run the explorer for? - let chains_enabled = match global_config().chain_name { - Some(ref chain_name) => vec![chain_name.clone()], - None => ecosystem_config.list_of_chains(), - }; - - // For each chain - initialize if needed or read previously created configs - let mut explorer_chain_configs = Vec::new(); - let mut backend_configs = Vec::new(); - for chain_name in chains_enabled.iter() { - let chain_config = ecosystem_config - .load_chain(Some(chain_name.clone())) - .ok_or_else(|| anyhow::anyhow!("Failed to load chain config for {}", chain_name))?; - - // Should initialize? Check if apps chain config exists - let mut should_initialize = false; - let apps_chain_config_path = AppsChainConfig::get_config_path(&ecosystem_path, chain_name); - let apps_chain_config = match AppsChainConfig::read(shell, &apps_chain_config_path) { - Ok(config) => config, - Err(_) => { - should_initialize = true; - // Initialize explorer database - logger::info(msg_explorer_initializing_database_for(chain_name)); - let db_config = fill_database_values_with_prompt(&chain_config); - initialize_explorer_database(&db_config).await?; - - // Allocate ports for backend services - let services_config = allocate_explorer_services_ports(shell, &mut ecosystem_ports) - .context(MSG_EXPLORER_FAILED_TO_ALLOCATE_PORTS_ERR)?; - - // Build and save apps chain config - let app_chain_config = AppsChainConfig { - explorer: AppsChainExplorerConfig { - database_url: db_config.full_url(), - services: services_config, - }, - }; - app_chain_config.save(shell, &apps_chain_config_path)?; - app_chain_config - } - }; - - // Build backend compose config for the explorer chain services - let l2_rpc_url = chain_config.get_general_config()?.get_l2_rpc_url()?; - let backend_compose_config = - ExplorerBackendComposeConfig::new(chain_name, l2_rpc_url, &apps_chain_config.explorer)?; - backend_configs.push(backend_compose_config); - - // Read or create explorer chain config (JSON) - let explorer_chain_config_path = - ExplorerChainConfig::get_config_path(&ecosystem_path, chain_name); - let explorer_chain_config = match should_initialize { - true => { - create_explorer_chain_config(&chain_config, &apps_chain_config.explorer, shell)? - } - false => match ExplorerChainConfig::read(shell, &explorer_chain_config_path) { - Ok(config) => config, - Err(_) => { - create_explorer_chain_config(&chain_config, &apps_chain_config.explorer, shell)? - } - }, - }; - explorer_chain_configs.push(explorer_chain_config); - } - - // Generate and save explorer runtime config (JS) - let explorer_runtime_config = ExplorerRuntimeConfig::new(explorer_chain_configs); - let explorer_runtime_config_path = ExplorerRuntimeConfig::get_config_path(&ecosystem_path); - explorer_runtime_config.save(shell, &explorer_runtime_config_path)?; - - // Generate and save explorer docker compose - let app_config = ExplorerAppServiceConfig { - port: apps_config.explorer.http_port, - config_path: explorer_runtime_config_path, - }; - let explorer_compose_config = ExplorerComposeConfig::new(app_config, backend_configs)?; - let explorer_compose_path = ExplorerComposeConfig::get_config_path(&ecosystem_path); - explorer_compose_config.save(shell, &explorer_compose_path)?; - - // Launch explorer using docker compose - logger::info(format!( - "Using generated docker compose file at {}", - explorer_compose_path.display() - )); - logger::info(msg_explorer_starting_on( - "127.0.0.1", - apps_config.explorer.http_port, - )); - run_explorer(shell, &explorer_compose_path)?; - Ok(()) -} - -fn run_explorer(shell: &Shell, explorer_compose_config_path: &Path) -> anyhow::Result<()> { - if let Some(docker_compose_file) = explorer_compose_config_path.to_str() { - docker::up(shell, docker_compose_file, false)?; - } else { - anyhow::bail!("Invalid docker compose file"); - } - Ok(()) -} - -fn fill_database_values_with_prompt(config: &ChainConfig) -> db::DatabaseConfig { - let defaul_db_name: String = generate_explorer_db_name(config); - let chain_name = config.name.clone(); - let explorer_db_url = Prompt::new(&msg_explorer_db_url_prompt(&chain_name)) - .default(DATABASE_EXPLORER_URL.as_str()) - .ask(); - let explorer_db_name: String = Prompt::new(&msg_explorer_db_name_prompt(&chain_name)) - .default(&defaul_db_name) - .ask(); - let explorer_db_name = slugify!(&explorer_db_name, separator = "_"); - db::DatabaseConfig::new(explorer_db_url, explorer_db_name) -} - -pub async fn initialize_explorer_database( - explorer_db_config: &db::DatabaseConfig, -) -> anyhow::Result<()> { - db::drop_db_if_exists(explorer_db_config) - .await - .context(MSG_EXPLORER_FAILED_TO_DROP_DATABASE_ERR)?; - db::init_db(explorer_db_config).await?; - Ok(()) -} - -fn allocate_explorer_services_ports( - shell: &Shell, - ecosystem_ports: &mut Option, -) -> anyhow::Result { - let default_ports = vec![ - DEFAULT_EXPLORER_API_PORT, - DEFAULT_EXPLORER_DATA_FETCHER_PORT, - DEFAULT_EXPLORER_WORKER_PORT, - ]; - - // Get or scan ecosystem folder configs to find assigned ports - let ports = match ecosystem_ports { - Some(ref mut res) => res, - None => ecosystem_ports.get_or_insert( - EcosystemPortsScanner::scan(shell).context("Failed to scan ecosystem ports")?, - ), - }; - - // Try to allocate intuitive ports with an offset from the defaults - let allocated = match ports.allocate_ports(&default_ports, 3001..4000, 100) { - Ok(allocated) => allocated, - Err(_) => { - // Allocate one by one - let mut allocated = Vec::new(); - for _ in 0..3 { - let port = ports.allocate_port(3001..4000)?; - allocated.push(port); - } - allocated - } - }; - - // Build the explorer services config - Ok(ExplorerServicesConfig::new( - allocated[0], - allocated[1], - allocated[2], - )) -} diff --git a/zk_toolbox/crates/zk_inception/src/commands/explorer/backend.rs b/zk_toolbox/crates/zk_inception/src/commands/explorer/backend.rs new file mode 100644 index 000000000000..a7b9185d1557 --- /dev/null +++ b/zk_toolbox/crates/zk_inception/src/commands/explorer/backend.rs @@ -0,0 +1,50 @@ +use std::path::Path; + +use anyhow::Context; +use common::{config::global_config, docker}; +use config::{ + explorer_compose::*, + traits::{ConfigWithL2RpcUrl, ReadConfig, SaveConfig}, + AppsChainConfig, EcosystemConfig, +}; +use xshell::Shell; + +use crate::messages::{ + msg_explorer_chain_not_initialized, MSG_CHAIN_NOT_FOUND_ERR, + MSG_EXPLORER_FAILED_TO_RUN_DOCKER_SERVICES_ERR, +}; + +pub(crate) fn run(shell: &Shell) -> anyhow::Result<()> { + let ecosystem_config = EcosystemConfig::from_file(shell)?; + let chain_config = ecosystem_config + .load_chain(global_config().chain_name.clone()) + .context(MSG_CHAIN_NOT_FOUND_ERR)?; + let chain_name = chain_config.name.clone(); + // Read chain-level apps.yaml config + let ecosystem_path = shell.current_dir(); + let apps_chain_config_path = AppsChainConfig::get_config_path(&ecosystem_path, &chain_name); + if !apps_chain_config_path.exists() { + anyhow::bail!(msg_explorer_chain_not_initialized(&chain_name)); + } + let apps_chain_config = AppsChainConfig::read(shell, &apps_chain_config_path)?; + // Build docker compose config with the explorer chain backend services + let l2_rpc_url = chain_config.get_general_config()?.get_l2_rpc_url()?; + let backend_compose_config = + ExplorerBackendComposeConfig::new(&chain_name, l2_rpc_url, &apps_chain_config.explorer)?; + let backend_compose_config_path = + ExplorerBackendComposeConfig::get_config_path(&ecosystem_path, &chain_config.name); + backend_compose_config.save(shell, backend_compose_config_path.clone())?; + // Run docker compose + run_backend(shell, &backend_compose_config_path)?; + Ok(()) +} + +fn run_backend(shell: &Shell, explorer_compose_config_path: &Path) -> anyhow::Result<()> { + if let Some(docker_compose_file) = explorer_compose_config_path.to_str() { + docker::up(shell, docker_compose_file, false) + .context(MSG_EXPLORER_FAILED_TO_RUN_DOCKER_SERVICES_ERR)?; + } else { + anyhow::bail!("Invalid docker compose file"); + } + Ok(()) +} diff --git a/zk_toolbox/crates/zk_inception/src/commands/explorer/init.rs b/zk_toolbox/crates/zk_inception/src/commands/explorer/init.rs new file mode 100644 index 000000000000..7b341bf910d5 --- /dev/null +++ b/zk_toolbox/crates/zk_inception/src/commands/explorer/init.rs @@ -0,0 +1,191 @@ +use anyhow::Context; +use common::{config::global_config, db, logger, Prompt}; +use config::{ + explorer::*, + traits::{ConfigWithL2RpcUrl, ReadConfig, SaveConfig}, + AppsChainConfig, AppsChainExplorerConfig, ChainConfig, EcosystemConfig, ExplorerServicesConfig, +}; +use slugify_rs::slugify; +use url::Url; +use xshell::Shell; + +use crate::{ + consts::L2_BASE_TOKEN_ADDRESS, + defaults::{ + generate_explorer_db_name, DATABASE_EXPLORER_URL, DEFAULT_EXPLORER_API_PORT, + DEFAULT_EXPLORER_DATA_FETCHER_PORT, DEFAULT_EXPLORER_WORKER_PORT, + }, + messages::{ + msg_chain_load_err, msg_explorer_db_name_prompt, msg_explorer_db_url_prompt, + msg_explorer_initializing_database_for, MSG_EXPLORER_FAILED_TO_ALLOCATE_PORTS_ERR, + MSG_EXPLORER_FAILED_TO_DROP_DATABASE_ERR, MSG_EXPLORER_INITIALIZED, + }, + utils::ports::{EcosystemPorts, EcosystemPortsScanner}, +}; + +pub(crate) async fn run(shell: &Shell) -> anyhow::Result<()> { + let ecosystem_config = EcosystemConfig::from_file(shell)?; + // Keep track of allocated ports (initialized lazily) + let mut ecosystem_ports: Option = None; + // If specific_chain is provided, initialize only that chain; otherwise, initialize all chains + let chains_enabled = match global_config().chain_name { + Some(ref chain_name) => vec![chain_name.clone()], + None => ecosystem_config.list_of_chains(), + }; + // Initialize chains one by one + for chain_name in chains_enabled.iter() { + // Load chain config + let chain_config = ecosystem_config + .load_chain(Some(chain_name.clone())) + .context(msg_chain_load_err(chain_name))?; + // Initialize chain-level apps.yaml + let apps_chain_config = + initialize_apps_chain_config(shell, &chain_config, &mut ecosystem_ports)?; + // Initialize explorer database + initialize_explorer_database(&apps_chain_config.explorer.database_url).await?; + // Create chain-level explorer.json + create_explorer_chain_config(shell, &chain_config, &apps_chain_config.explorer)?; + } + logger::outro(MSG_EXPLORER_INITIALIZED); + Ok(()) +} + +fn initialize_apps_chain_config( + shell: &Shell, + chain_config: &ChainConfig, + ecosystem_ports: &mut Option, +) -> anyhow::Result { + let ecosystem_path = shell.current_dir(); + let apps_chain_config_path = + AppsChainConfig::get_config_path(&ecosystem_path, &chain_config.name); + // Check if apps chain config exists + if let Ok(apps_chain_config) = AppsChainConfig::read(shell, &apps_chain_config_path) { + return Ok(apps_chain_config); + } + // Prompt explorer database name + logger::info(msg_explorer_initializing_database_for(&chain_config.name)); + let db_config = fill_database_values_with_prompt(chain_config); + + // Allocate ports for backend services + let services_config = allocate_explorer_services_ports(shell, ecosystem_ports) + .context(MSG_EXPLORER_FAILED_TO_ALLOCATE_PORTS_ERR)?; + + // Build and save apps chain config + let app_chain_config = AppsChainConfig { + explorer: AppsChainExplorerConfig { + database_url: db_config.full_url(), + services: services_config, + }, + }; + app_chain_config.save(shell, &apps_chain_config_path)?; + Ok(app_chain_config) +} + +async fn initialize_explorer_database(db_url: &Url) -> anyhow::Result<()> { + let db_config = db::DatabaseConfig::from_url(db_url)?; + db::drop_db_if_exists(&db_config) + .await + .context(MSG_EXPLORER_FAILED_TO_DROP_DATABASE_ERR)?; + db::init_db(&db_config).await?; + Ok(()) +} + +fn fill_database_values_with_prompt(config: &ChainConfig) -> db::DatabaseConfig { + let defaul_db_name: String = generate_explorer_db_name(config); + let chain_name = config.name.clone(); + let explorer_db_url = Prompt::new(&msg_explorer_db_url_prompt(&chain_name)) + .default(DATABASE_EXPLORER_URL.as_str()) + .ask(); + let explorer_db_name: String = Prompt::new(&msg_explorer_db_name_prompt(&chain_name)) + .default(&defaul_db_name) + .ask(); + let explorer_db_name = slugify!(&explorer_db_name, separator = "_"); + db::DatabaseConfig::new(explorer_db_url, explorer_db_name) +} + +fn allocate_explorer_services_ports( + shell: &Shell, + ecosystem_ports: &mut Option, +) -> anyhow::Result { + let default_ports = vec![ + DEFAULT_EXPLORER_API_PORT, + DEFAULT_EXPLORER_DATA_FETCHER_PORT, + DEFAULT_EXPLORER_WORKER_PORT, + ]; + + // Get or scan ecosystem folder configs to find assigned ports + let ports = match ecosystem_ports { + Some(ref mut res) => res, + None => ecosystem_ports.get_or_insert( + EcosystemPortsScanner::scan(shell).context("Failed to scan ecosystem ports")?, + ), + }; + + // Try to allocate intuitive ports with an offset from the defaults + let allocated = match ports.allocate_ports(&default_ports, 3001..4000, 100) { + Ok(allocated) => allocated, + Err(_) => { + // Allocate one by one + let mut allocated = Vec::new(); + for _ in 0..3 { + let port = ports.allocate_port(3001..4000)?; + allocated.push(port); + } + allocated + } + }; + + // Build the explorer services config + Ok(ExplorerServicesConfig::new( + allocated[0], + allocated[1], + allocated[2], + )) +} + +fn create_explorer_chain_config( + shell: &Shell, + chain_config: &ChainConfig, + apps_chain_config: &AppsChainExplorerConfig, +) -> anyhow::Result { + let explorer_config = build_explorer_chain_config(chain_config, apps_chain_config)?; + let config_path = + ExplorerChainConfig::get_config_path(&shell.current_dir(), &chain_config.name); + explorer_config.save(shell, config_path)?; + Ok(explorer_config) +} + +fn build_explorer_chain_config( + chain_config: &ChainConfig, + apps_chain_explorer_config: &AppsChainExplorerConfig, +) -> anyhow::Result { + let general_config = chain_config.get_general_config()?; + // Get L2 RPC URL from general config + let l2_rpc_url = general_config.get_l2_rpc_url()?; + // Get Verification API URL from general config + let verification_api_url = general_config + .contract_verifier + .as_ref() + .map(|verifier| &verifier.url) + .context("verification_url")?; + // Build API URL + let api_port = apps_chain_explorer_config.services.api_http_port; + let api_url = format!("http://127.0.0.1:{}", api_port); + + // Build explorer chain config + Ok(ExplorerChainConfig { + name: chain_config.name.clone(), + l2_network_name: chain_config.name.clone(), + l2_chain_id: chain_config.chain_id.as_u64(), + rpc_url: l2_rpc_url.to_string(), + api_url: api_url.to_string(), + base_token_address: L2_BASE_TOKEN_ADDRESS.to_string(), + hostnames: Vec::new(), + icon: "/images/icons/zksync-arrows.svg".to_string(), + maintenance: false, + published: true, + bridge_url: None, + l1_explorer_url: None, + verification_api_url: Some(verification_api_url.to_string()), + }) +} diff --git a/zk_toolbox/crates/zk_inception/src/commands/explorer/mod.rs b/zk_toolbox/crates/zk_inception/src/commands/explorer/mod.rs new file mode 100644 index 000000000000..7333d1e0f883 --- /dev/null +++ b/zk_toolbox/crates/zk_inception/src/commands/explorer/mod.rs @@ -0,0 +1,25 @@ +use clap::Subcommand; +use xshell::Shell; + +mod backend; +mod init; +mod run; + +#[derive(Subcommand, Debug)] +pub enum ExplorerCommands { + /// Initialize explorer + Init, + /// Start explorer backend services (api, data_fetcher, worker) + #[command(alias = "backend")] + RunBackend, + /// Run explorer app + Run, +} + +pub(crate) async fn run(shell: &Shell, args: ExplorerCommands) -> anyhow::Result<()> { + match args { + ExplorerCommands::Init => init::run(shell).await, + ExplorerCommands::Run => run::run(shell), + ExplorerCommands::RunBackend => backend::run(shell), + } +} diff --git a/zk_toolbox/crates/zk_inception/src/commands/explorer/run.rs b/zk_toolbox/crates/zk_inception/src/commands/explorer/run.rs new file mode 100644 index 000000000000..f31599c850ce --- /dev/null +++ b/zk_toolbox/crates/zk_inception/src/commands/explorer/run.rs @@ -0,0 +1,85 @@ +use std::{collections::HashMap, path::Path}; + +use anyhow::Context; +use common::{config::global_config, docker, logger}; +use config::{ + explorer::*, + traits::{ReadConfig, SaveConfig}, + AppsEcosystemConfig, EcosystemConfig, +}; +use xshell::Shell; + +use crate::{ + consts::{EXPLORER_APP_DOCKER_CONFIG_PATH, EXPLORER_APP_DOCKER_IMAGE}, + messages::{ + msg_explorer_failed_to_configure_chain, msg_explorer_skipping_not_initialized_chain, + msg_explorer_starting_on, MSG_EXPLORER_FAILED_TO_FIND_ANY_VALID, + MSG_EXPLORER_FAILED_TO_RUN_DOCKER_ERR, + }, +}; + +pub(crate) fn run(shell: &Shell) -> anyhow::Result<()> { + let ecosystem_config = EcosystemConfig::from_file(shell)?; + let ecosystem_path = shell.current_dir(); + // Get ecosystem level apps.yaml config + let apps_config = AppsEcosystemConfig::read_or_create_default(shell)?; + // If specific_chain is provided, run only with that chain; otherwise, run with all chains + let chains_enabled = match global_config().chain_name { + Some(ref chain_name) => vec![chain_name.clone()], + None => ecosystem_config.list_of_chains(), + }; + + // Read chain configs one by one + let mut explorer_chain_configs = Vec::new(); + for chain_name in chains_enabled.iter() { + let explorer_chain_config_path = + ExplorerChainConfig::get_config_path(&ecosystem_path, chain_name); + if !explorer_chain_config_path.exists() { + logger::warn(msg_explorer_skipping_not_initialized_chain(chain_name)); + continue; + } + match ExplorerChainConfig::read(shell, &explorer_chain_config_path) { + Ok(config) => explorer_chain_configs.push(config), + Err(_) => logger::warn(msg_explorer_failed_to_configure_chain(chain_name)), + } + } + if explorer_chain_configs.is_empty() { + anyhow::bail!(MSG_EXPLORER_FAILED_TO_FIND_ANY_VALID); + } + + // Generate and save explorer runtime config (JS) + let runtime_config = ExplorerRuntimeConfig::new(explorer_chain_configs); + let config_path = ExplorerRuntimeConfig::get_config_path(&ecosystem_path); + runtime_config.save(shell, &config_path)?; + + logger::info(format!( + "Using generated explorer config file at {}", + config_path.display() + )); + logger::info(msg_explorer_starting_on( + "127.0.0.1", + apps_config.explorer.http_port, + )); + + run_explorer(shell, &config_path, apps_config.explorer.http_port)?; + Ok(()) +} + +fn run_explorer(shell: &Shell, config_file_path: &Path, port: u16) -> anyhow::Result<()> { + let port_mapping = format!("{}:{}", port, port); + let volume_mapping = format!( + "{}:{}", + config_file_path.display(), + EXPLORER_APP_DOCKER_CONFIG_PATH + ); + + let mut docker_args: HashMap = HashMap::new(); + docker_args.insert("--platform".to_string(), "linux/amd64".to_string()); + docker_args.insert("-p".to_string(), port_mapping); + docker_args.insert("-v".to_string(), volume_mapping); + docker_args.insert("-e".to_string(), format!("PORT={}", port)); + + docker::run(shell, EXPLORER_APP_DOCKER_IMAGE, docker_args) + .with_context(|| MSG_EXPLORER_FAILED_TO_RUN_DOCKER_ERR)?; + Ok(()) +} diff --git a/zk_toolbox/crates/zk_inception/src/consts.rs b/zk_toolbox/crates/zk_inception/src/consts.rs index 49f6c8a9ed3f..2d82f482d9a7 100644 --- a/zk_toolbox/crates/zk_inception/src/consts.rs +++ b/zk_toolbox/crates/zk_inception/src/consts.rs @@ -9,5 +9,8 @@ pub const DEFAULT_PROOF_STORE_DIR: &str = "artifacts"; pub const BELLMAN_CUDA_DIR: &str = "era-bellman-cuda"; pub const L2_BASE_TOKEN_ADDRESS: &str = "0x000000000000000000000000000000000000800A"; +/// Path to the JS runtime config for the block-explorer-app docker container to be mounted to +pub const EXPLORER_APP_DOCKER_CONFIG_PATH: &str = "/usr/src/app/packages/app/dist/config.js"; +pub const EXPLORER_APP_DOCKER_IMAGE: &str = "matterlabs/block-explorer-app"; pub const PORTAL_DOCKER_IMAGE: &str = "matterlabs/dapp-portal"; pub const PORTAL_DOCKER_CONTAINER_PORT: u16 = 3000; diff --git a/zk_toolbox/crates/zk_inception/src/main.rs b/zk_toolbox/crates/zk_inception/src/main.rs index 4323029d4e90..f6f7d83dede6 100644 --- a/zk_toolbox/crates/zk_inception/src/main.rs +++ b/zk_toolbox/crates/zk_inception/src/main.rs @@ -14,7 +14,7 @@ use xshell::Shell; use crate::commands::{ args::RunServerArgs, chain::ChainCommands, ecosystem::EcosystemCommands, - external_node::ExternalNodeCommands, prover::ProverCommands, + explorer::ExplorerCommands, external_node::ExternalNodeCommands, prover::ProverCommands, }; pub mod accept_ownership; @@ -59,7 +59,8 @@ pub enum InceptionSubcommands { /// Run dapp-portal Portal, /// Run block-explorer - Explorer, + #[command(subcommand)] + Explorer(ExplorerCommands), /// Update ZKsync #[command(alias = "u")] Update(UpdateArgs), @@ -122,7 +123,7 @@ async fn run_subcommand(inception_args: Inception, shell: &Shell) -> anyhow::Res InceptionSubcommands::ContractVerifier(args) => { commands::contract_verifier::run(shell, args).await? } - InceptionSubcommands::Explorer => commands::explorer::run(shell).await?, + InceptionSubcommands::Explorer(args) => commands::explorer::run(shell, args).await?, InceptionSubcommands::Portal => commands::portal::run(shell).await?, InceptionSubcommands::Update(args) => commands::update::run(shell, args)?, InceptionSubcommands::Markdown => { diff --git a/zk_toolbox/crates/zk_inception/src/messages.rs b/zk_toolbox/crates/zk_inception/src/messages.rs index 69c7de8407ed..af08f6e1b0f5 100644 --- a/zk_toolbox/crates/zk_inception/src/messages.rs +++ b/zk_toolbox/crates/zk_inception/src/messages.rs @@ -113,6 +113,9 @@ pub(super) fn msg_chain_doesnt_exist_err(chain_name: &str, chains: &Vec) chain_name, chains ) } +pub(super) fn msg_chain_load_err(chain_name: &str) -> String { + format!("Failed to load chain config for {chain_name}") +} /// Chain create related messages pub(super) const MSG_PROVER_MODE_HELP: &str = "Prover options"; @@ -249,12 +252,28 @@ pub(super) const MSG_EXPLORER_FAILED_TO_ALLOCATE_PORTS_ERR: &str = "Failed to allocate ports for explorer services"; pub(super) const MSG_EXPLORER_FAILED_TO_DROP_DATABASE_ERR: &str = "Failed to drop explorer database"; +pub(super) const MSG_EXPLORER_FAILED_TO_RUN_DOCKER_SERVICES_ERR: &str = + "Failed to run docker compose with explorer services"; +pub(super) const MSG_EXPLORER_FAILED_TO_RUN_DOCKER_ERR: &str = + "Failed to run explorer docker container"; +pub(super) const MSG_EXPLORER_INITIALIZED: &str = "Explorer has been initialized successfully"; +pub(super) const MSG_EXPLORER_FAILED_TO_FIND_ANY_VALID: &str = + "Failed to find any valid chains to run explorer for"; pub(super) fn msg_explorer_initializing_database_for(chain: &str) -> String { format!("Initializing explorer database for {chain} chain") } pub(super) fn msg_explorer_starting_on(host: &str, port: u16) -> String { format!("Starting explorer on http://{host}:{port}") } +pub(super) fn msg_explorer_chain_not_initialized(chain: &str) -> String { + format!("Chain is not initialized for explorer: run `zk_inception explorer init --chain {chain}` first") +} +pub(super) fn msg_explorer_skipping_not_initialized_chain(chain: &str) -> String { + format!("Chain {chain} is not initialized for explorer. Skipping..") +} +pub(super) fn msg_explorer_failed_to_configure_chain(chain: &str) -> String { + format!("Failed to configure explorer for chain {chain}") +} /// Forge utils related messages pub(super) const MSG_DEPLOYER_PK_NOT_SET_ERR: &str = "Deployer private key is not set"; From 20d1a5b1eea71962c98150bcd80175d4846c94ea Mon Sep 17 00:00:00 2001 From: Manuel Mauro Date: Tue, 3 Sep 2024 17:51:06 +0200 Subject: [PATCH 07/12] fix: correctly use chain config id instead of chain_id --- zk_toolbox/crates/zk_inception/src/commands/chain/args/init.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zk_toolbox/crates/zk_inception/src/commands/chain/args/init.rs b/zk_toolbox/crates/zk_inception/src/commands/chain/args/init.rs index 2253eeb314ef..434f134ba992 100644 --- a/zk_toolbox/crates/zk_inception/src/commands/chain/args/init.rs +++ b/zk_toolbox/crates/zk_inception/src/commands/chain/args/init.rs @@ -88,7 +88,7 @@ impl InitArgs { l1_rpc_url, port_offset: self .port_offset - .unwrap_or(PortOffset::from_chain_id(config.chain_id.as_u64() as u16)) + .unwrap_or(PortOffset::from_chain_id(config.id as u16)) .into(), } } From 47a3fb9505bcee09b51988ab18ac82d3e9252d98 Mon Sep 17 00:00:00 2001 From: Alexander Melnikov Date: Tue, 3 Sep 2024 18:49:38 -0600 Subject: [PATCH 08/12] shell instead of fs --- .../crates/config/src/explorer_compose.rs | 8 +- zk_toolbox/crates/config/src/general.rs | 2 +- .../src/commands/chain/args/init.rs | 2 +- .../src/commands/explorer/init.rs | 71 ++++---- .../zk_inception/src/commands/portal.rs | 11 +- zk_toolbox/crates/zk_inception/src/consts.rs | 6 +- .../crates/zk_inception/src/defaults.rs | 10 +- .../crates/zk_inception/src/messages.rs | 2 +- .../crates/zk_inception/src/utils/ports.rs | 154 +++++++++--------- 9 files changed, 134 insertions(+), 132 deletions(-) diff --git a/zk_toolbox/crates/config/src/explorer_compose.rs b/zk_toolbox/crates/config/src/explorer_compose.rs index 416b5c80c864..a94861a372a4 100644 --- a/zk_toolbox/crates/config/src/explorer_compose.rs +++ b/zk_toolbox/crates/config/src/explorer_compose.rs @@ -12,14 +12,14 @@ use crate::{ apps::AppsChainExplorerConfig, consts::{ EXPLORER_API_DOCKER_IMAGE, EXPLORER_DATA_FETCHER_DOCKER_IMAGE, - EXPLORER_DOCKER_COMPOSE_FILE, EXPLORER_WORKER_DOCKER_IMAGE, LOCAL_APPS_PATH, - LOCAL_CHAINS_PATH, LOCAL_CONFIGS_PATH, + EXPLORER_DOCKER_COMPOSE_FILE, EXPLORER_WORKER_DOCKER_IMAGE, LOCAL_CHAINS_PATH, + LOCAL_CONFIGS_PATH, LOCAL_GENERATED_PATH, }, docker_compose::{DockerComposeConfig, DockerComposeService}, traits::ZkToolboxConfig, }; -/// Chain-level explorer backend docker compose config. It contains the configuration for +/// Chain-level explorer backend docker compose file. It contains the configuration for /// api, data fetcher, and worker services. /// This config is auto-generated during "explorer run-backend" command. #[derive(Debug, Serialize, Deserialize, Clone)] @@ -175,7 +175,7 @@ impl ExplorerBackendComposeConfig { .join(LOCAL_CHAINS_PATH) .join(chain_name) .join(LOCAL_CONFIGS_PATH) - .join(LOCAL_APPS_PATH) + .join(LOCAL_GENERATED_PATH) .join(EXPLORER_DOCKER_COMPOSE_FILE) } } diff --git a/zk_toolbox/crates/config/src/general.rs b/zk_toolbox/crates/config/src/general.rs index a9e96f0ce95d..41c2e4c33cfd 100644 --- a/zk_toolbox/crates/config/src/general.rs +++ b/zk_toolbox/crates/config/src/general.rs @@ -127,7 +127,7 @@ pub fn update_ports(config: &mut GeneralConfig, ports_config: &PortsConfig) -> a let prometheus = config .prometheus_config .as_mut() - .context("Contract Verifier config is not presented")?; + .context("Prometheus config is not presented")?; api.web3_json_rpc.http_port = ports_config.web3_json_rpc_http_port; update_port_in_url( diff --git a/zk_toolbox/crates/zk_inception/src/commands/chain/args/init.rs b/zk_toolbox/crates/zk_inception/src/commands/chain/args/init.rs index 434f134ba992..c3639729ccdf 100644 --- a/zk_toolbox/crates/zk_inception/src/commands/chain/args/init.rs +++ b/zk_toolbox/crates/zk_inception/src/commands/chain/args/init.rs @@ -88,7 +88,7 @@ impl InitArgs { l1_rpc_url, port_offset: self .port_offset - .unwrap_or(PortOffset::from_chain_id(config.id as u16)) + .unwrap_or(PortOffset::from_chain_id((config.id - 1) as u16)) .into(), } } diff --git a/zk_toolbox/crates/zk_inception/src/commands/explorer/init.rs b/zk_toolbox/crates/zk_inception/src/commands/explorer/init.rs index 7b341bf910d5..c9d05adaf198 100644 --- a/zk_toolbox/crates/zk_inception/src/commands/explorer/init.rs +++ b/zk_toolbox/crates/zk_inception/src/commands/explorer/init.rs @@ -10,10 +10,10 @@ use url::Url; use xshell::Shell; use crate::{ - consts::L2_BASE_TOKEN_ADDRESS, + consts::{L2_BASE_TOKEN_ADDRESS, MAX_ALLOCATED_PORT, MIN_ALLOCATED_PORT}, defaults::{ - generate_explorer_db_name, DATABASE_EXPLORER_URL, DEFAULT_EXPLORER_API_PORT, - DEFAULT_EXPLORER_DATA_FETCHER_PORT, DEFAULT_EXPLORER_WORKER_PORT, + generate_explorer_db_name, DATABASE_EXPLORER_URL, EXPLORER_API_PORT, + EXPLORER_DATA_FETCHER_PORT, EXPLORER_WORKER_PORT, }, messages::{ msg_chain_load_err, msg_explorer_db_name_prompt, msg_explorer_db_url_prompt, @@ -25,9 +25,10 @@ use crate::{ pub(crate) async fn run(shell: &Shell) -> anyhow::Result<()> { let ecosystem_config = EcosystemConfig::from_file(shell)?; - // Keep track of allocated ports (initialized lazily) - let mut ecosystem_ports: Option = None; - // If specific_chain is provided, initialize only that chain; otherwise, initialize all chains + // Keep track of allocated ports: + let mut ecosystem_ports = + EcosystemPortsScanner::scan(shell).context("Failed to scan ecosystem ports")?; + // If specific chain is provided, initialize only that chain; otherwise, initialize all chains let chains_enabled = match global_config().chain_name { Some(ref chain_name) => vec![chain_name.clone()], None => ecosystem_config.list_of_chains(), @@ -53,7 +54,7 @@ pub(crate) async fn run(shell: &Shell) -> anyhow::Result<()> { fn initialize_apps_chain_config( shell: &Shell, chain_config: &ChainConfig, - ecosystem_ports: &mut Option, + ecosystem_ports: &mut EcosystemPorts, ) -> anyhow::Result { let ecosystem_path = shell.current_dir(); let apps_chain_config_path = @@ -67,7 +68,7 @@ fn initialize_apps_chain_config( let db_config = fill_database_values_with_prompt(chain_config); // Allocate ports for backend services - let services_config = allocate_explorer_services_ports(shell, ecosystem_ports) + let services_config = allocate_explorer_services_ports(ecosystem_ports) .context(MSG_EXPLORER_FAILED_TO_ALLOCATE_PORTS_ERR)?; // Build and save apps chain config @@ -104,43 +105,33 @@ fn fill_database_values_with_prompt(config: &ChainConfig) -> db::DatabaseConfig } fn allocate_explorer_services_ports( - shell: &Shell, - ecosystem_ports: &mut Option, + ports: &mut EcosystemPorts, ) -> anyhow::Result { let default_ports = vec![ - DEFAULT_EXPLORER_API_PORT, - DEFAULT_EXPLORER_DATA_FETCHER_PORT, - DEFAULT_EXPLORER_WORKER_PORT, + EXPLORER_API_PORT, + EXPLORER_DATA_FETCHER_PORT, + EXPLORER_WORKER_PORT, ]; - // Get or scan ecosystem folder configs to find assigned ports - let ports = match ecosystem_ports { - Some(ref mut res) => res, - None => ecosystem_ports.get_or_insert( - EcosystemPortsScanner::scan(shell).context("Failed to scan ecosystem ports")?, - ), - }; - // Try to allocate intuitive ports with an offset from the defaults - let allocated = match ports.allocate_ports(&default_ports, 3001..4000, 100) { - Ok(allocated) => allocated, - Err(_) => { - // Allocate one by one - let mut allocated = Vec::new(); - for _ in 0..3 { - let port = ports.allocate_port(3001..4000)?; - allocated.push(port); - } - allocated - } - }; - - // Build the explorer services config - Ok(ExplorerServicesConfig::new( - allocated[0], - allocated[1], - allocated[2], - )) + let range = MIN_ALLOCATED_PORT..MAX_ALLOCATED_PORT; + let offset = ports.find_offset(&default_ports, range.clone(), 100)?; + let services_config = ExplorerServicesConfig::new( + EXPLORER_API_PORT + offset, + EXPLORER_DATA_FETCHER_PORT + offset, + EXPLORER_WORKER_PORT + offset, + ); + // Assign ports and add port information: + ports.add_port_info(services_config.api_http_port, "Explorer API".to_string()); + ports.add_port_info( + services_config.data_fetcher_http_port, + "Explorer Data Fetcher".to_string(), + ); + ports.add_port_info( + services_config.worker_http_port, + "Explorer Worker".to_string(), + ); + Ok(services_config) } fn create_explorer_chain_config( diff --git a/zk_toolbox/crates/zk_inception/src/commands/portal.rs b/zk_toolbox/crates/zk_inception/src/commands/portal.rs index d7d42d0cec0f..fb12b73f5b16 100644 --- a/zk_toolbox/crates/zk_inception/src/commands/portal.rs +++ b/zk_toolbox/crates/zk_inception/src/commands/portal.rs @@ -12,7 +12,7 @@ use types::{BaseToken, TokenInfo}; use xshell::Shell; use crate::{ - consts::{L2_BASE_TOKEN_ADDRESS, PORTAL_DOCKER_CONTAINER_PORT, PORTAL_DOCKER_IMAGE}, + consts::{L2_BASE_TOKEN_ADDRESS, PORTAL_DOCKER_CONFIG_PATH, PORTAL_DOCKER_IMAGE}, messages::{ msg_portal_starting_on, MSG_PORTAL_FAILED_TO_CREATE_ANY_CHAIN_CONFIG_ERR, MSG_PORTAL_FAILED_TO_CREATE_CONFIG_ERR, MSG_PORTAL_FAILED_TO_RUN_DOCKER_ERR, @@ -152,13 +152,18 @@ pub async fn run(shell: &Shell) -> anyhow::Result<()> { } fn run_portal(shell: &Shell, config_file_path: &Path, port: u16) -> anyhow::Result<()> { - let port_mapping = format!("{}:{}", port, PORTAL_DOCKER_CONTAINER_PORT); - let volume_mapping = format!("{}:/usr/src/app/dist/config.js", config_file_path.display()); + let port_mapping = format!("{}:{}", port, port); + let volume_mapping = format!( + "{}:{}", + config_file_path.display(), + PORTAL_DOCKER_CONFIG_PATH + ); let mut docker_args: HashMap = HashMap::new(); docker_args.insert("--platform".to_string(), "linux/amd64".to_string()); docker_args.insert("-p".to_string(), port_mapping); docker_args.insert("-v".to_string(), volume_mapping); + docker_args.insert("-e".to_string(), format!("PORT={}", port)); docker::run(shell, PORTAL_DOCKER_IMAGE, docker_args) .with_context(|| MSG_PORTAL_FAILED_TO_RUN_DOCKER_ERR)?; diff --git a/zk_toolbox/crates/zk_inception/src/consts.rs b/zk_toolbox/crates/zk_inception/src/consts.rs index 2d82f482d9a7..e19be8752376 100644 --- a/zk_toolbox/crates/zk_inception/src/consts.rs +++ b/zk_toolbox/crates/zk_inception/src/consts.rs @@ -12,5 +12,9 @@ pub const L2_BASE_TOKEN_ADDRESS: &str = "0x0000000000000000000000000000000000008 /// Path to the JS runtime config for the block-explorer-app docker container to be mounted to pub const EXPLORER_APP_DOCKER_CONFIG_PATH: &str = "/usr/src/app/packages/app/dist/config.js"; pub const EXPLORER_APP_DOCKER_IMAGE: &str = "matterlabs/block-explorer-app"; +/// Path to the JS runtime config for the dapp-portal docker container to be mounted to +pub const PORTAL_DOCKER_CONFIG_PATH: &str = "/usr/src/app/dist/config.js"; pub const PORTAL_DOCKER_IMAGE: &str = "matterlabs/dapp-portal"; -pub const PORTAL_DOCKER_CONTAINER_PORT: u16 = 3000; + +pub const MIN_ALLOCATED_PORT: u16 = 3000; +pub const MAX_ALLOCATED_PORT: u16 = 65535; diff --git a/zk_toolbox/crates/zk_inception/src/defaults.rs b/zk_toolbox/crates/zk_inception/src/defaults.rs index 8e8d16bca9ec..1b6b9a66b2be 100644 --- a/zk_toolbox/crates/zk_inception/src/defaults.rs +++ b/zk_toolbox/crates/zk_inception/src/defaults.rs @@ -12,9 +12,13 @@ lazy_static! { } // Default ports for services -pub const DEFAULT_EXPLORER_WORKER_PORT: u16 = 3001; -pub const DEFAULT_EXPLORER_API_PORT: u16 = 3002; -pub const DEFAULT_EXPLORER_DATA_FETCHER_PORT: u16 = 3040; +pub const OBSERVABILITY_PORT: u16 = 3000; +pub const EXPLORER_WORKER_PORT: u16 = 3001; +pub const EXPLORER_API_PORT: u16 = 3002; +pub const EXPLORER_DATA_FETCHER_PORT: u16 = 3040; +pub const POSTGRES_DB_PORT: u16 = 5432; +pub const LOCAL_HTTP_RPC_PORT: u16 = 8545; +pub const LOCAL_WS_RPC_PORT: u16 = 8546; pub const ROCKS_DB_STATE_KEEPER: &str = "state_keeper"; pub const ROCKS_DB_TREE: &str = "tree"; diff --git a/zk_toolbox/crates/zk_inception/src/messages.rs b/zk_toolbox/crates/zk_inception/src/messages.rs index af08f6e1b0f5..ff0ac3bdcb7a 100644 --- a/zk_toolbox/crates/zk_inception/src/messages.rs +++ b/zk_toolbox/crates/zk_inception/src/messages.rs @@ -266,7 +266,7 @@ pub(super) fn msg_explorer_starting_on(host: &str, port: u16) -> String { format!("Starting explorer on http://{host}:{port}") } pub(super) fn msg_explorer_chain_not_initialized(chain: &str) -> String { - format!("Chain is not initialized for explorer: run `zk_inception explorer init --chain {chain}` first") + format!("Chain {chain} is not initialized for explorer: run `zk_inception explorer init --chain {chain}` first") } pub(super) fn msg_explorer_skipping_not_initialized_chain(chain: &str) -> String { format!("Chain {chain} is not initialized for explorer. Skipping..") diff --git a/zk_toolbox/crates/zk_inception/src/utils/ports.rs b/zk_toolbox/crates/zk_inception/src/utils/ports.rs index 31f19db38510..56c0509eb26a 100644 --- a/zk_toolbox/crates/zk_inception/src/utils/ports.rs +++ b/zk_toolbox/crates/zk_inception/src/utils/ports.rs @@ -1,33 +1,35 @@ use std::{ collections::{HashMap, HashSet}, - fmt, fs, + fmt, ops::Range, path::Path, }; -use anyhow::Result; -use common::logger; +use anyhow::{Context, Result}; use config::EcosystemConfig; use serde_yaml::Value; use xshell::Shell; -/// Ecosystem-level directories to include in the scan -const INCLUDE_DIRS: &[&str] = &["chains", "configs"]; -/// Skip directories with these names at all levels -const SKIP_DIRS: &[&str] = &["db", "volumes"]; +use crate::defaults::{ + LOCAL_HTTP_RPC_PORT, LOCAL_WS_RPC_PORT, OBSERVABILITY_PORT, POSTGRES_DB_PORT, +}; +#[derive(Default)] pub struct EcosystemPorts { pub ports: HashMap>, } impl EcosystemPorts { - pub fn new() -> Self { - Self { - ports: HashMap::from([ - (3000, vec!["Observability".to_string()]), - (3052, vec!["External node gateway".to_string()]), - ]), - } + /// Creates a new EcosystemPorts instance with default port configurations. + /// These ports are not configured in ecosystem yaml file configs, but are + /// commonly used or pre-assigned within the ecosystem. + pub fn with_default_ports() -> Self { + let mut ports = HashMap::new(); + ports.insert(OBSERVABILITY_PORT, vec!["Observability".to_string()]); + ports.insert(POSTGRES_DB_PORT, vec!["PostgreSQL".to_string()]); + ports.insert(LOCAL_HTTP_RPC_PORT, vec!["Local HTTP RPC".to_string()]); + ports.insert(LOCAL_WS_RPC_PORT, vec!["Local WebSockets RPC".to_string()]); + Self { ports } } pub fn get_assigned_ports(&self) -> HashSet { @@ -42,33 +44,36 @@ impl EcosystemPorts { self.ports.entry(port).or_default().push(info); } - pub fn allocate_port(&mut self, range: Range) -> Result { + pub fn allocate_port(&mut self, range: Range, info: String) -> Result { for port in range { if !self.is_port_assigned(port) { - self.add_port_info(port, "Dynamically allocated".to_string()); + self.add_port_info(port, info.to_string()); return Ok(port); } } anyhow::bail!("No available ports in the given range") } - /// Allocates a set of ports based on given base ports, incrementing by offset if needed. - /// Tries base ports first, then base + offset, base + 2*offset, etc., until finding - /// available ports or exceeding the range. Returns allocated ports or an error if - /// no suitable ports are found. - pub fn allocate_ports( + /// Finds the smallest multiple of the offset that, when added to the base ports, + /// results in a set of available ports within the specified range. + /// + /// The function iteratively checks port sets by adding multiples of the offset to the base ports: + /// base, base + offset, base + 2*offset, etc., until it finds a set where all ports are: + /// 1) Within the specified range + /// 2) Not already assigned + pub fn find_offset( &mut self, base_ports: &[u16], range: Range, offset: u16, - ) -> Result> { + ) -> Result { let mut i = 0; loop { let candidate_ports: Vec = base_ports.iter().map(|&port| port + i * offset).collect(); - // Check if all candidate ports are within the range - if candidate_ports.iter().any(|&port| !range.contains(&port)) { + // Check if any of the candidate ports ran beyond the upper range + if candidate_ports.iter().any(|&port| port >= range.end) { anyhow::bail!( "No suitable ports found within the given range {:?}. Tried {} iterations with offset {}", range, @@ -77,16 +82,12 @@ impl EcosystemPorts { ) } - // Check if all candidate ports are available + // Check if all candidate ports are within the range and not assigned if candidate_ports .iter() - .all(|&port| !self.is_port_assigned(port)) + .all(|&port| range.contains(&port) && !self.is_port_assigned(port)) { - // Allocate all ports - for &port in &candidate_ports { - self.add_port_info(port, "Dynamically allocated".to_string()); - } - return Ok(candidate_ports); + return Ok(offset * i); } i += 1; } @@ -113,55 +114,56 @@ impl EcosystemPortsScanner { /// Specifically, it looks for keys ending with "port" and collects their values. /// Note: Port information from Docker Compose files will not be picked up by this method. pub fn scan(shell: &Shell) -> Result { - let _ = EcosystemConfig::from_file(shell)?; // Find and validate the ecosystem directory - let ecosystem_dir = shell.current_dir(); + let ecosystem_config = EcosystemConfig::from_file(shell)?; + + // Create a list of directories to scan: + // - Ecosystem configs directory + // - Chain configs directories + let mut dirs = vec![ecosystem_config.config.clone()]; + for chain in ecosystem_config.list_of_chains() { + if let Some(chain_config) = ecosystem_config.load_chain(Some(chain)) { + dirs.push(chain_config.configs.clone()) + } + } - let skip_dirs: HashSet<&str> = SKIP_DIRS.iter().cloned().collect(); - let mut ecosystem_ports = EcosystemPorts::new(); - for subdir in INCLUDE_DIRS { - let dir = ecosystem_dir.join(subdir); + let mut ecosystem_ports = EcosystemPorts::with_default_ports(); + for dir in dirs { if dir.is_dir() { - if let Err(e) = Self::scan_directory(&dir, &mut ecosystem_ports, &skip_dirs) { - logger::warn(format!("Error scanning directory {:?}: {}", dir, e)); - } + Self::scan_yaml_files(shell, &dir, &mut ecosystem_ports) + .context(format!("Failed to scan directory {:?}", dir))?; } } Ok(ecosystem_ports) } - fn scan_directory( + /// Scans the given directory for YAML files in the immediate directory only (non-recursive). + /// Processes each YAML file found and updates the EcosystemPorts accordingly. + fn scan_yaml_files( + shell: &Shell, dir: &Path, ecosystem_ports: &mut EcosystemPorts, - skip_dirs: &HashSet<&str>, ) -> Result<()> { - if let Some(dir_name) = dir.file_name().and_then(|n| n.to_str()) { - if skip_dirs.contains(dir_name) { - return Ok(()); + for path in shell.read_dir(dir)? { + if !path.is_file() { + continue; } - } - - for entry in fs::read_dir(dir)? { - let path = entry?.path(); - if path.is_dir() { - if let Err(e) = Self::scan_directory(&path, ecosystem_ports, skip_dirs) { - logger::warn(format!("Error scanning directory {:?}: {}", path, e)); - } - } else if path.is_file() { - if let Some(extension) = path.extension() { - if extension == "yaml" || extension == "yml" { - if let Err(e) = Self::process_yaml_file(&path, ecosystem_ports) { - logger::warn(format!("Error processing YAML file {:?}: {}", path, e)); - } - } + if let Some(extension) = path.extension() { + if extension == "yaml" || extension == "yml" { + Self::process_yaml_file(shell, &path, ecosystem_ports) + .context(format!("Error processing YAML file {:?}", path))?; } } } Ok(()) } - fn process_yaml_file(file_path: &Path, ecosystem_ports: &mut EcosystemPorts) -> Result<()> { - let contents = fs::read_to_string(file_path)?; + fn process_yaml_file( + shell: &Shell, + file_path: &Path, + ecosystem_ports: &mut EcosystemPorts, + ) -> Result<()> { + let contents = shell.read_file(file_path)?; let value: Value = serde_yaml::from_str(&contents)?; Self::traverse_yaml(&value, "", file_path, ecosystem_ports); Ok(()) @@ -211,39 +213,35 @@ mod tests { #[test] fn test_allocate_ports() { - let mut ecosystem_ports = EcosystemPorts::new(); + let mut ecosystem_ports = EcosystemPorts::default(); // Test case 1: Allocate ports within range - let result = ecosystem_ports.allocate_ports(&[8000, 8001, 8002], 8000..9000, 100); + let result = ecosystem_ports.find_offset(&[8000, 8001, 8002], 8000..9000, 100); assert!(result.is_ok()); - let allocated_ports = result.unwrap(); - assert_eq!(allocated_ports, vec![8000, 8001, 8002]); + assert_eq!(result.unwrap(), 0); // Test case 2: Allocate ports with offset - let result = ecosystem_ports.allocate_ports(&[8000, 8001, 8002], 8000..9000, 100); + let result = ecosystem_ports.find_offset(&[8000, 8001, 8002], 8000..9000, 100); assert!(result.is_ok()); - let allocated_ports = result.unwrap(); - assert_eq!(allocated_ports, vec![8100, 8101, 8102]); + assert_eq!(result.unwrap(), 100); // Test case 3: Fail to allocate ports - not available with offset - let result = ecosystem_ports.allocate_ports(&[8000, 8001, 8002], 8000..8200, 100); + let result = ecosystem_ports.find_offset(&[8000, 8001, 8002], 8000..8200, 100); assert!(result.is_err()); // Test case 4: Fail to allocate ports - base ports outside range - let result = ecosystem_ports.allocate_ports(&[9500, 9501, 9502], 8000..9000, 100); + let result = ecosystem_ports.find_offset(&[9500, 9501, 9502], 8000..9000, 100); assert!(result.is_err()); // Test case 5: Allocate consecutive ports - let result = ecosystem_ports.allocate_ports(&[8000, 8001, 8002], 8000..9000, 1); + let result = ecosystem_ports.find_offset(&[8000, 8001, 8002], 8000..9000, 1); assert!(result.is_ok()); - let allocated_ports = result.unwrap(); - assert_eq!(allocated_ports, vec![8003, 8004, 8005]); + assert_eq!(result.unwrap(), 3); // Test case 6: Allocate ports at the edge of the range - let result = ecosystem_ports.allocate_ports(&[8998, 8999], 8000..9000, 1); + let result = ecosystem_ports.find_offset(&[8998, 8999], 8000..9000, 1); assert!(result.is_ok()); - let allocated_ports = result.unwrap(); - assert_eq!(allocated_ports, vec![8998, 8999]); + assert_eq!(result.unwrap(), 0); } #[test] @@ -266,7 +264,7 @@ mod tests { "#; let value = serde_yaml::from_str(yaml_content).unwrap(); - let mut ecosystem_ports = EcosystemPorts::new(); + let mut ecosystem_ports = EcosystemPorts::default(); let file_path: PathBuf = PathBuf::from("test_config.yaml"); EcosystemPortsScanner::traverse_yaml(&value, "", &file_path, &mut ecosystem_ports); From 5d8c7c82c00e17b1d1bcb06df170c5991aa8f920 Mon Sep 17 00:00:00 2001 From: Alexander Melnikov Date: Wed, 4 Sep 2024 08:12:08 -0600 Subject: [PATCH 09/12] fix: port offset by chain.id --- .../crates/zk_inception/src/commands/chain/args/init.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/zk_toolbox/crates/zk_inception/src/commands/chain/args/init.rs b/zk_toolbox/crates/zk_inception/src/commands/chain/args/init.rs index c3639729ccdf..9dd6c490bd78 100644 --- a/zk_toolbox/crates/zk_inception/src/commands/chain/args/init.rs +++ b/zk_toolbox/crates/zk_inception/src/commands/chain/args/init.rs @@ -22,7 +22,7 @@ pub struct PortOffset(u16); impl PortOffset { pub fn from_chain_id(chain_id: u16) -> Self { - Self(chain_id * 100) + Self((chain_id - 1) * 100) } } @@ -88,7 +88,7 @@ impl InitArgs { l1_rpc_url, port_offset: self .port_offset - .unwrap_or(PortOffset::from_chain_id((config.id - 1) as u16)) + .unwrap_or(PortOffset::from_chain_id(config.id as u16)) .into(), } } From c4959e49a7841c19a374262d510649bd6bfbc3ca Mon Sep 17 00:00:00 2001 From: Alexander Melnikov Date: Wed, 4 Sep 2024 11:08:22 -0600 Subject: [PATCH 10/12] moved ports logic to a separate PR --- zk_toolbox/crates/config/src/general.rs | 2 +- .../src/commands/chain/args/init.rs | 4 +- .../src/commands/explorer/init.rs | 46 +-- zk_toolbox/crates/zk_inception/src/consts.rs | 3 - .../crates/zk_inception/src/defaults.rs | 4 - .../crates/zk_inception/src/messages.rs | 2 - .../crates/zk_inception/src/utils/mod.rs | 1 - .../crates/zk_inception/src/utils/ports.rs | 301 ------------------ 8 files changed, 13 insertions(+), 350 deletions(-) delete mode 100644 zk_toolbox/crates/zk_inception/src/utils/ports.rs diff --git a/zk_toolbox/crates/config/src/general.rs b/zk_toolbox/crates/config/src/general.rs index 41c2e4c33cfd..a9e96f0ce95d 100644 --- a/zk_toolbox/crates/config/src/general.rs +++ b/zk_toolbox/crates/config/src/general.rs @@ -127,7 +127,7 @@ pub fn update_ports(config: &mut GeneralConfig, ports_config: &PortsConfig) -> a let prometheus = config .prometheus_config .as_mut() - .context("Prometheus config is not presented")?; + .context("Contract Verifier config is not presented")?; api.web3_json_rpc.http_port = ports_config.web3_json_rpc_http_port; update_port_in_url( diff --git a/zk_toolbox/crates/zk_inception/src/commands/chain/args/init.rs b/zk_toolbox/crates/zk_inception/src/commands/chain/args/init.rs index 9dd6c490bd78..2253eeb314ef 100644 --- a/zk_toolbox/crates/zk_inception/src/commands/chain/args/init.rs +++ b/zk_toolbox/crates/zk_inception/src/commands/chain/args/init.rs @@ -22,7 +22,7 @@ pub struct PortOffset(u16); impl PortOffset { pub fn from_chain_id(chain_id: u16) -> Self { - Self((chain_id - 1) * 100) + Self(chain_id * 100) } } @@ -88,7 +88,7 @@ impl InitArgs { l1_rpc_url, port_offset: self .port_offset - .unwrap_or(PortOffset::from_chain_id(config.id as u16)) + .unwrap_or(PortOffset::from_chain_id(config.chain_id.as_u64() as u16)) .into(), } } diff --git a/zk_toolbox/crates/zk_inception/src/commands/explorer/init.rs b/zk_toolbox/crates/zk_inception/src/commands/explorer/init.rs index c9d05adaf198..099c5232bf2b 100644 --- a/zk_toolbox/crates/zk_inception/src/commands/explorer/init.rs +++ b/zk_toolbox/crates/zk_inception/src/commands/explorer/init.rs @@ -10,24 +10,21 @@ use url::Url; use xshell::Shell; use crate::{ - consts::{L2_BASE_TOKEN_ADDRESS, MAX_ALLOCATED_PORT, MIN_ALLOCATED_PORT}, + commands::chain::args::init::PortOffset, + consts::L2_BASE_TOKEN_ADDRESS, defaults::{ generate_explorer_db_name, DATABASE_EXPLORER_URL, EXPLORER_API_PORT, EXPLORER_DATA_FETCHER_PORT, EXPLORER_WORKER_PORT, }, messages::{ msg_chain_load_err, msg_explorer_db_name_prompt, msg_explorer_db_url_prompt, - msg_explorer_initializing_database_for, MSG_EXPLORER_FAILED_TO_ALLOCATE_PORTS_ERR, - MSG_EXPLORER_FAILED_TO_DROP_DATABASE_ERR, MSG_EXPLORER_INITIALIZED, + msg_explorer_initializing_database_for, MSG_EXPLORER_FAILED_TO_DROP_DATABASE_ERR, + MSG_EXPLORER_INITIALIZED, }, - utils::ports::{EcosystemPorts, EcosystemPortsScanner}, }; pub(crate) async fn run(shell: &Shell) -> anyhow::Result<()> { let ecosystem_config = EcosystemConfig::from_file(shell)?; - // Keep track of allocated ports: - let mut ecosystem_ports = - EcosystemPortsScanner::scan(shell).context("Failed to scan ecosystem ports")?; // If specific chain is provided, initialize only that chain; otherwise, initialize all chains let chains_enabled = match global_config().chain_name { Some(ref chain_name) => vec![chain_name.clone()], @@ -40,8 +37,7 @@ pub(crate) async fn run(shell: &Shell) -> anyhow::Result<()> { .load_chain(Some(chain_name.clone())) .context(msg_chain_load_err(chain_name))?; // Initialize chain-level apps.yaml - let apps_chain_config = - initialize_apps_chain_config(shell, &chain_config, &mut ecosystem_ports)?; + let apps_chain_config = initialize_apps_chain_config(shell, &chain_config)?; // Initialize explorer database initialize_explorer_database(&apps_chain_config.explorer.database_url).await?; // Create chain-level explorer.json @@ -54,7 +50,6 @@ pub(crate) async fn run(shell: &Shell) -> anyhow::Result<()> { fn initialize_apps_chain_config( shell: &Shell, chain_config: &ChainConfig, - ecosystem_ports: &mut EcosystemPorts, ) -> anyhow::Result { let ecosystem_path = shell.current_dir(); let apps_chain_config_path = @@ -68,8 +63,7 @@ fn initialize_apps_chain_config( let db_config = fill_database_values_with_prompt(chain_config); // Allocate ports for backend services - let services_config = allocate_explorer_services_ports(ecosystem_ports) - .context(MSG_EXPLORER_FAILED_TO_ALLOCATE_PORTS_ERR)?; + let services_config = allocate_explorer_services_ports(chain_config); // Build and save apps chain config let app_chain_config = AppsChainConfig { @@ -104,34 +98,14 @@ fn fill_database_values_with_prompt(config: &ChainConfig) -> db::DatabaseConfig db::DatabaseConfig::new(explorer_db_url, explorer_db_name) } -fn allocate_explorer_services_ports( - ports: &mut EcosystemPorts, -) -> anyhow::Result { - let default_ports = vec![ - EXPLORER_API_PORT, - EXPLORER_DATA_FETCHER_PORT, - EXPLORER_WORKER_PORT, - ]; - +fn allocate_explorer_services_ports(chain_config: &ChainConfig) -> ExplorerServicesConfig { // Try to allocate intuitive ports with an offset from the defaults - let range = MIN_ALLOCATED_PORT..MAX_ALLOCATED_PORT; - let offset = ports.find_offset(&default_ports, range.clone(), 100)?; - let services_config = ExplorerServicesConfig::new( + let offset: u16 = PortOffset::from_chain_id(chain_config.id as u16).into(); + ExplorerServicesConfig::new( EXPLORER_API_PORT + offset, EXPLORER_DATA_FETCHER_PORT + offset, EXPLORER_WORKER_PORT + offset, - ); - // Assign ports and add port information: - ports.add_port_info(services_config.api_http_port, "Explorer API".to_string()); - ports.add_port_info( - services_config.data_fetcher_http_port, - "Explorer Data Fetcher".to_string(), - ); - ports.add_port_info( - services_config.worker_http_port, - "Explorer Worker".to_string(), - ); - Ok(services_config) + ) } fn create_explorer_chain_config( diff --git a/zk_toolbox/crates/zk_inception/src/consts.rs b/zk_toolbox/crates/zk_inception/src/consts.rs index e19be8752376..7db976c61033 100644 --- a/zk_toolbox/crates/zk_inception/src/consts.rs +++ b/zk_toolbox/crates/zk_inception/src/consts.rs @@ -15,6 +15,3 @@ pub const EXPLORER_APP_DOCKER_IMAGE: &str = "matterlabs/block-explorer-app"; /// Path to the JS runtime config for the dapp-portal docker container to be mounted to pub const PORTAL_DOCKER_CONFIG_PATH: &str = "/usr/src/app/dist/config.js"; pub const PORTAL_DOCKER_IMAGE: &str = "matterlabs/dapp-portal"; - -pub const MIN_ALLOCATED_PORT: u16 = 3000; -pub const MAX_ALLOCATED_PORT: u16 = 65535; diff --git a/zk_toolbox/crates/zk_inception/src/defaults.rs b/zk_toolbox/crates/zk_inception/src/defaults.rs index 1b6b9a66b2be..9f29b2c067ac 100644 --- a/zk_toolbox/crates/zk_inception/src/defaults.rs +++ b/zk_toolbox/crates/zk_inception/src/defaults.rs @@ -12,13 +12,9 @@ lazy_static! { } // Default ports for services -pub const OBSERVABILITY_PORT: u16 = 3000; pub const EXPLORER_WORKER_PORT: u16 = 3001; pub const EXPLORER_API_PORT: u16 = 3002; pub const EXPLORER_DATA_FETCHER_PORT: u16 = 3040; -pub const POSTGRES_DB_PORT: u16 = 5432; -pub const LOCAL_HTTP_RPC_PORT: u16 = 8545; -pub const LOCAL_WS_RPC_PORT: u16 = 8546; pub const ROCKS_DB_STATE_KEEPER: &str = "state_keeper"; pub const ROCKS_DB_TREE: &str = "tree"; diff --git a/zk_toolbox/crates/zk_inception/src/messages.rs b/zk_toolbox/crates/zk_inception/src/messages.rs index ff0ac3bdcb7a..098f668d4717 100644 --- a/zk_toolbox/crates/zk_inception/src/messages.rs +++ b/zk_toolbox/crates/zk_inception/src/messages.rs @@ -248,8 +248,6 @@ pub(super) fn msg_portal_starting_on(host: &str, port: u16) -> String { } /// Explorer related messages -pub(super) const MSG_EXPLORER_FAILED_TO_ALLOCATE_PORTS_ERR: &str = - "Failed to allocate ports for explorer services"; pub(super) const MSG_EXPLORER_FAILED_TO_DROP_DATABASE_ERR: &str = "Failed to drop explorer database"; pub(super) const MSG_EXPLORER_FAILED_TO_RUN_DOCKER_SERVICES_ERR: &str = diff --git a/zk_toolbox/crates/zk_inception/src/utils/mod.rs b/zk_toolbox/crates/zk_inception/src/utils/mod.rs index e8bfeeebf053..a84f0a336de5 100644 --- a/zk_toolbox/crates/zk_inception/src/utils/mod.rs +++ b/zk_toolbox/crates/zk_inception/src/utils/mod.rs @@ -1,3 +1,2 @@ pub mod forge; -pub mod ports; pub mod rocks_db; diff --git a/zk_toolbox/crates/zk_inception/src/utils/ports.rs b/zk_toolbox/crates/zk_inception/src/utils/ports.rs deleted file mode 100644 index 56c0509eb26a..000000000000 --- a/zk_toolbox/crates/zk_inception/src/utils/ports.rs +++ /dev/null @@ -1,301 +0,0 @@ -use std::{ - collections::{HashMap, HashSet}, - fmt, - ops::Range, - path::Path, -}; - -use anyhow::{Context, Result}; -use config::EcosystemConfig; -use serde_yaml::Value; -use xshell::Shell; - -use crate::defaults::{ - LOCAL_HTTP_RPC_PORT, LOCAL_WS_RPC_PORT, OBSERVABILITY_PORT, POSTGRES_DB_PORT, -}; - -#[derive(Default)] -pub struct EcosystemPorts { - pub ports: HashMap>, -} - -impl EcosystemPorts { - /// Creates a new EcosystemPorts instance with default port configurations. - /// These ports are not configured in ecosystem yaml file configs, but are - /// commonly used or pre-assigned within the ecosystem. - pub fn with_default_ports() -> Self { - let mut ports = HashMap::new(); - ports.insert(OBSERVABILITY_PORT, vec!["Observability".to_string()]); - ports.insert(POSTGRES_DB_PORT, vec!["PostgreSQL".to_string()]); - ports.insert(LOCAL_HTTP_RPC_PORT, vec!["Local HTTP RPC".to_string()]); - ports.insert(LOCAL_WS_RPC_PORT, vec!["Local WebSockets RPC".to_string()]); - Self { ports } - } - - pub fn get_assigned_ports(&self) -> HashSet { - self.ports.keys().cloned().collect() - } - - pub fn is_port_assigned(&self, port: u16) -> bool { - self.ports.contains_key(&port) - } - - pub fn add_port_info(&mut self, port: u16, info: String) { - self.ports.entry(port).or_default().push(info); - } - - pub fn allocate_port(&mut self, range: Range, info: String) -> Result { - for port in range { - if !self.is_port_assigned(port) { - self.add_port_info(port, info.to_string()); - return Ok(port); - } - } - anyhow::bail!("No available ports in the given range") - } - - /// Finds the smallest multiple of the offset that, when added to the base ports, - /// results in a set of available ports within the specified range. - /// - /// The function iteratively checks port sets by adding multiples of the offset to the base ports: - /// base, base + offset, base + 2*offset, etc., until it finds a set where all ports are: - /// 1) Within the specified range - /// 2) Not already assigned - pub fn find_offset( - &mut self, - base_ports: &[u16], - range: Range, - offset: u16, - ) -> Result { - let mut i = 0; - loop { - let candidate_ports: Vec = - base_ports.iter().map(|&port| port + i * offset).collect(); - - // Check if any of the candidate ports ran beyond the upper range - if candidate_ports.iter().any(|&port| port >= range.end) { - anyhow::bail!( - "No suitable ports found within the given range {:?}. Tried {} iterations with offset {}", - range, - i, - offset - ) - } - - // Check if all candidate ports are within the range and not assigned - if candidate_ports - .iter() - .all(|&port| range.contains(&port) && !self.is_port_assigned(port)) - { - return Ok(offset * i); - } - i += 1; - } - } -} - -impl fmt::Display for EcosystemPorts { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - let mut port_vec: Vec<_> = self.ports.iter().collect(); - port_vec.sort_by_key(|&(key, _)| key); - for (port, port_infos) in port_vec { - for port_info in port_infos { - writeln!(f, "{} > {}", port_info, port)?; - } - } - Ok(()) - } -} - -pub struct EcosystemPortsScanner {} - -impl EcosystemPortsScanner { - /// Scans the ecosystem directory for YAML files and extracts port information. - /// Specifically, it looks for keys ending with "port" and collects their values. - /// Note: Port information from Docker Compose files will not be picked up by this method. - pub fn scan(shell: &Shell) -> Result { - let ecosystem_config = EcosystemConfig::from_file(shell)?; - - // Create a list of directories to scan: - // - Ecosystem configs directory - // - Chain configs directories - let mut dirs = vec![ecosystem_config.config.clone()]; - for chain in ecosystem_config.list_of_chains() { - if let Some(chain_config) = ecosystem_config.load_chain(Some(chain)) { - dirs.push(chain_config.configs.clone()) - } - } - - let mut ecosystem_ports = EcosystemPorts::with_default_ports(); - for dir in dirs { - if dir.is_dir() { - Self::scan_yaml_files(shell, &dir, &mut ecosystem_ports) - .context(format!("Failed to scan directory {:?}", dir))?; - } - } - - Ok(ecosystem_ports) - } - - /// Scans the given directory for YAML files in the immediate directory only (non-recursive). - /// Processes each YAML file found and updates the EcosystemPorts accordingly. - fn scan_yaml_files( - shell: &Shell, - dir: &Path, - ecosystem_ports: &mut EcosystemPorts, - ) -> Result<()> { - for path in shell.read_dir(dir)? { - if !path.is_file() { - continue; - } - if let Some(extension) = path.extension() { - if extension == "yaml" || extension == "yml" { - Self::process_yaml_file(shell, &path, ecosystem_ports) - .context(format!("Error processing YAML file {:?}", path))?; - } - } - } - Ok(()) - } - - fn process_yaml_file( - shell: &Shell, - file_path: &Path, - ecosystem_ports: &mut EcosystemPorts, - ) -> Result<()> { - let contents = shell.read_file(file_path)?; - let value: Value = serde_yaml::from_str(&contents)?; - Self::traverse_yaml(&value, "", file_path, ecosystem_ports); - Ok(()) - } - - fn traverse_yaml( - value: &Value, - path: &str, - file_path: &Path, - ecosystem_ports: &mut EcosystemPorts, - ) { - match value { - Value::Mapping(map) => { - for (key, val) in map { - let new_path = if path.is_empty() { - key.as_str().unwrap_or_default().to_string() - } else { - format!("{}:{}", path, key.as_str().unwrap_or_default()) - }; - - if key.as_str().map(|s| s.ends_with("port")).unwrap_or(false) { - if let Some(port) = val.as_u64().and_then(|p| u16::try_from(p).ok()) { - let description = format!("[{}] {}", file_path.display(), new_path); - ecosystem_ports.add_port_info(port, description); - } - } - - Self::traverse_yaml(val, &new_path, file_path, ecosystem_ports); - } - } - Value::Sequence(seq) => { - for (index, val) in seq.iter().enumerate() { - let new_path = format!("{}:{}", path, index); - Self::traverse_yaml(val, &new_path, file_path, ecosystem_ports); - } - } - _ => {} - } - } -} - -#[cfg(test)] -mod tests { - use std::path::PathBuf; - - use crate::utils::ports::{EcosystemPorts, EcosystemPortsScanner}; - - #[test] - fn test_allocate_ports() { - let mut ecosystem_ports = EcosystemPorts::default(); - - // Test case 1: Allocate ports within range - let result = ecosystem_ports.find_offset(&[8000, 8001, 8002], 8000..9000, 100); - assert!(result.is_ok()); - assert_eq!(result.unwrap(), 0); - - // Test case 2: Allocate ports with offset - let result = ecosystem_ports.find_offset(&[8000, 8001, 8002], 8000..9000, 100); - assert!(result.is_ok()); - assert_eq!(result.unwrap(), 100); - - // Test case 3: Fail to allocate ports - not available with offset - let result = ecosystem_ports.find_offset(&[8000, 8001, 8002], 8000..8200, 100); - assert!(result.is_err()); - - // Test case 4: Fail to allocate ports - base ports outside range - let result = ecosystem_ports.find_offset(&[9500, 9501, 9502], 8000..9000, 100); - assert!(result.is_err()); - - // Test case 5: Allocate consecutive ports - let result = ecosystem_ports.find_offset(&[8000, 8001, 8002], 8000..9000, 1); - assert!(result.is_ok()); - assert_eq!(result.unwrap(), 3); - - // Test case 6: Allocate ports at the edge of the range - let result = ecosystem_ports.find_offset(&[8998, 8999], 8000..9000, 1); - assert!(result.is_ok()); - assert_eq!(result.unwrap(), 0); - } - - #[test] - fn test_traverse_yaml() { - let yaml_content = r#" - api: - web3_json_rpc: - http_port: 3050 - ws_port: 3051 - api_namespaces: - - eth - gas_price_scale_factor: 1.5 - prometheus: - listener_port: 3412 - push_interval_ms: 100 - contract_verifier: - port: 3070 - prometheus: - listener_port: 3412 - "#; - - let value = serde_yaml::from_str(yaml_content).unwrap(); - let mut ecosystem_ports = EcosystemPorts::default(); - let file_path: PathBuf = PathBuf::from("test_config.yaml"); - - EcosystemPortsScanner::traverse_yaml(&value, "", &file_path, &mut ecosystem_ports); - - // Assigned ports: - assert!(ecosystem_ports.is_port_assigned(3050)); - assert!(ecosystem_ports.is_port_assigned(3051)); - assert!(ecosystem_ports.is_port_assigned(3070)); - assert!(ecosystem_ports.is_port_assigned(3412)); - - // Free ports: - assert!(!ecosystem_ports.is_port_assigned(3150)); - assert!(!ecosystem_ports.is_port_assigned(3151)); - - // Check description: - let port_3050_info = ecosystem_ports.ports.get(&3050).unwrap(); - assert_eq!(port_3050_info.len(), 1); - assert_eq!( - port_3050_info[0], - "[test_config.yaml] api:web3_json_rpc:http_port" - ); - - let port_3412_info = ecosystem_ports.ports.get(&3412).unwrap(); - assert_eq!(port_3412_info.len(), 2); - assert_eq!( - port_3412_info[0], - "[test_config.yaml] api:prometheus:listener_port" - ); - assert_eq!( - port_3412_info[1], - "[test_config.yaml] prometheus:listener_port" - ); - } -} From 76a744d917599d11492e952b9f2a0e6d535247cd Mon Sep 17 00:00:00 2001 From: Alexander Melnikov Date: Wed, 4 Sep 2024 23:53:31 -0600 Subject: [PATCH 11/12] restructure configs --- zk_toolbox/crates/common/src/docker.rs | 15 +- zk_toolbox/crates/config/src/apps.rs | 57 +------- zk_toolbox/crates/config/src/consts.rs | 20 ++- .../crates/config/src/docker_compose.rs | 8 +- zk_toolbox/crates/config/src/explorer.rs | 134 +++++++++++------- .../crates/config/src/explorer_compose.rs | 111 ++++++++++----- zk_toolbox/crates/config/src/portal.rs | 129 +++++++++++------ .../zk_inception/src/commands/chain/init.rs | 4 +- .../src/commands/explorer/backend.rs | 23 +-- .../src/commands/explorer/init.rs | 85 +++++------ .../zk_inception/src/commands/explorer/run.rs | 93 ++++++------ .../zk_inception/src/commands/portal.rs | 125 +++++++++------- .../crates/zk_inception/src/defaults.rs | 5 - .../crates/zk_inception/src/messages.rs | 25 ++-- 14 files changed, 439 insertions(+), 395 deletions(-) diff --git a/zk_toolbox/crates/common/src/docker.rs b/zk_toolbox/crates/common/src/docker.rs index 3eacd9e4d2d2..a5731808814f 100644 --- a/zk_toolbox/crates/common/src/docker.rs +++ b/zk_toolbox/crates/common/src/docker.rs @@ -1,5 +1,3 @@ -use std::collections::HashMap; - use url::Url; use xshell::{cmd, Shell}; @@ -19,17 +17,8 @@ pub fn down(shell: &Shell, docker_compose_file: &str) -> anyhow::Result<()> { Ok(Cmd::new(cmd!(shell, "docker compose -f {docker_compose_file} down")).run()?) } -pub fn run( - shell: &Shell, - docker_image: &str, - docker_args: HashMap, -) -> anyhow::Result<()> { - let mut args = vec![]; - for (key, value) in docker_args.iter() { - args.push(key); - args.push(value); - } - Ok(Cmd::new(cmd!(shell, "docker run {args...} {docker_image}")).run()?) +pub fn run(shell: &Shell, docker_image: &str, docker_args: Vec) -> anyhow::Result<()> { + Ok(Cmd::new(cmd!(shell, "docker run {docker_args...} {docker_image}")).run()?) } pub fn adjust_localhost_for_docker(mut url: Url) -> anyhow::Result { diff --git a/zk_toolbox/crates/config/src/apps.rs b/zk_toolbox/crates/config/src/apps.rs index 5acff173732c..697b35b0851b 100644 --- a/zk_toolbox/crates/config/src/apps.rs +++ b/zk_toolbox/crates/config/src/apps.rs @@ -1,16 +1,11 @@ use std::path::{Path, PathBuf}; use serde::{Deserialize, Serialize}; -use url::Url; use xshell::Shell; use crate::{ - consts::{ - APPS_CONFIG_FILE, DEFAULT_EXPLORER_PORT, DEFAULT_PORTAL_PORT, LOCAL_CHAINS_PATH, - LOCAL_CONFIGS_PATH, - }, + consts::{APPS_CONFIG_FILE, DEFAULT_EXPLORER_PORT, DEFAULT_PORTAL_PORT, LOCAL_CONFIGS_PATH}, traits::{FileConfigWithDefaultName, ReadConfig, SaveConfig, ZkToolboxConfig}, - EXPLORER_BATCHES_PROCESSING_POLLING_INTERVAL, }; /// Ecosystem level configuration for the apps (portal and explorer). @@ -23,7 +18,6 @@ pub struct AppsEcosystemConfig { #[derive(Debug, Serialize, Deserialize, Clone)] pub struct AppEcosystemConfig { pub http_port: u16, - pub http_url: String, } impl ZkToolboxConfig for AppsEcosystemConfig {} @@ -31,28 +25,6 @@ impl FileConfigWithDefaultName for AppsEcosystemConfig { const FILE_NAME: &'static str = APPS_CONFIG_FILE; } -/// Chain level configuration for the apps. -#[derive(Debug, Serialize, Deserialize, Clone)] -pub struct AppsChainConfig { - pub explorer: AppsChainExplorerConfig, -} - -#[derive(Debug, Serialize, Deserialize, Clone)] -pub struct AppsChainExplorerConfig { - pub database_url: Url, - pub services: ExplorerServicesConfig, -} - -#[derive(Debug, Serialize, Deserialize, Clone)] -pub struct ExplorerServicesConfig { - pub api_http_port: u16, - pub data_fetcher_http_port: u16, - pub worker_http_port: u16, - pub batches_processing_polling_interval: u64, -} - -impl ZkToolboxConfig for AppsChainConfig {} - impl AppsEcosystemConfig { pub fn get_config_path(ecosystem_base_path: &Path) -> PathBuf { ecosystem_base_path @@ -78,37 +50,10 @@ impl Default for AppsEcosystemConfig { AppsEcosystemConfig { portal: AppEcosystemConfig { http_port: DEFAULT_PORTAL_PORT, - http_url: format!("http://127.0.0.1:{}", DEFAULT_PORTAL_PORT), }, explorer: AppEcosystemConfig { http_port: DEFAULT_EXPLORER_PORT, - http_url: format!("http://127.0.0.1:{}", DEFAULT_EXPLORER_PORT), }, } } } - -impl AppsChainConfig { - pub fn new(explorer: AppsChainExplorerConfig) -> Self { - AppsChainConfig { explorer } - } - - pub fn get_config_path(ecosystem_base_path: &Path, chain_name: &str) -> PathBuf { - ecosystem_base_path - .join(LOCAL_CHAINS_PATH) - .join(chain_name) - .join(LOCAL_CONFIGS_PATH) - .join(APPS_CONFIG_FILE) - } -} - -impl ExplorerServicesConfig { - pub fn new(api_http_port: u16, data_fetcher_http_port: u16, worker_http_port: u16) -> Self { - ExplorerServicesConfig { - api_http_port, - data_fetcher_http_port, - worker_http_port, - batches_processing_polling_interval: EXPLORER_BATCHES_PROCESSING_POLLING_INTERVAL, - } - } -} diff --git a/zk_toolbox/crates/config/src/consts.rs b/zk_toolbox/crates/config/src/consts.rs index e861c0be7048..1e1c0998f00e 100644 --- a/zk_toolbox/crates/config/src/consts.rs +++ b/zk_toolbox/crates/config/src/consts.rs @@ -40,20 +40,26 @@ pub(crate) const LOCAL_ARTIFACTS_PATH: &str = "artifacts/"; /// Name of apps config file pub const APPS_CONFIG_FILE: &str = "apps.yaml"; /// Name of portal runtime config file (auto-generated) -pub const PORTAL_RUNTIME_CONFIG_FILE: &str = "portal.config.js"; -/// Name of chain-level portal config JSON file -pub const PORTAL_CHAIN_CONFIG_FILE: &str = "portal.config.json"; +pub const PORTAL_JS_CONFIG_FILE: &str = "portal.config.js"; +/// Name of portal config JSON file +pub const PORTAL_CONFIG_FILE: &str = "portal.config.json"; /// Name of explorer runtime config file (auto-generated) -pub const EXPLORER_RUNTIME_CONFIG_FILE: &str = "explorer.config.js"; -/// Name of chain-level explorer config JSON file -pub const EXPLORER_CHAIN_CONFIG_FILE: &str = "explorer.config.json"; -/// Name of explorer docker compose file (auto-generated) +pub const EXPLORER_JS_CONFIG_FILE: &str = "explorer.config.js"; +/// Name of explorer config JSON file +pub const EXPLORER_CONFIG_FILE: &str = "explorer.config.json"; +/// Name of explorer docker compose file pub const EXPLORER_DOCKER_COMPOSE_FILE: &str = "explorer-docker-compose.yml"; /// Default port for the explorer app pub const DEFAULT_EXPLORER_PORT: u16 = 3010; /// Default port for the portal app pub const DEFAULT_PORTAL_PORT: u16 = 3030; +/// Default port for the explorer worker service +pub const DEFAULT_EXPLORER_WORKER_PORT: u16 = 3001; +/// Default port for the explorer API service +pub const DEFAULT_EXPLORER_API_PORT: u16 = 3002; +/// Default port for the explorer data fetcher service +pub const DEFAULT_EXPLORER_DATA_FETCHER_PORT: u16 = 3040; pub const EXPLORER_API_DOCKER_IMAGE: &str = "matterlabs/block-explorer-api"; pub const EXPLORER_DATA_FETCHER_DOCKER_IMAGE: &str = "matterlabs/block-explorer-data-fetcher"; diff --git a/zk_toolbox/crates/config/src/docker_compose.rs b/zk_toolbox/crates/config/src/docker_compose.rs index 01a78766cf91..05c6e73eaea5 100644 --- a/zk_toolbox/crates/config/src/docker_compose.rs +++ b/zk_toolbox/crates/config/src/docker_compose.rs @@ -7,6 +7,10 @@ use crate::traits::ZkToolboxConfig; #[derive(Debug, Default, Serialize, Deserialize, Clone)] pub struct DockerComposeConfig { pub services: HashMap, + #[serde(skip_serializing_if = "Option::is_none")] + pub name: Option, + #[serde(flatten)] + pub other: serde_json::Value, } #[derive(Debug, Serialize, Deserialize, Clone)] @@ -33,10 +37,6 @@ pub struct DockerComposeService { impl ZkToolboxConfig for DockerComposeConfig {} impl DockerComposeConfig { - pub fn new() -> Self { - Self::default() - } - pub fn add_service(&mut self, name: &str, service: DockerComposeService) { self.services.insert(name.to_string(), service); } diff --git a/zk_toolbox/crates/config/src/explorer.rs b/zk_toolbox/crates/config/src/explorer.rs index 6a55f791a9c7..ee7a59e5105c 100644 --- a/zk_toolbox/crates/config/src/explorer.rs +++ b/zk_toolbox/crates/config/src/explorer.rs @@ -5,19 +5,20 @@ use xshell::Shell; use crate::{ consts::{ - EXPLORER_CHAIN_CONFIG_FILE, EXPLORER_RUNTIME_CONFIG_FILE, LOCAL_APPS_PATH, - LOCAL_CHAINS_PATH, LOCAL_CONFIGS_PATH, LOCAL_GENERATED_PATH, + EXPLORER_CONFIG_FILE, EXPLORER_JS_CONFIG_FILE, LOCAL_APPS_PATH, LOCAL_CONFIGS_PATH, + LOCAL_GENERATED_PATH, }, traits::{ReadConfig, SaveConfig, ZkToolboxConfig}, }; -/// Explorer configuration file. This file is auto-generated during the "explorer" command -/// and is used to inject the runtime configuration into the explorer app. +/// Explorer JSON configuration file. This file contains configuration for the explorer app. #[derive(Serialize, Deserialize, Debug, Clone)] #[serde(rename_all = "camelCase")] -pub struct ExplorerRuntimeConfig { +pub struct ExplorerConfig { pub app_environment: String, pub environment_config: EnvironmentConfig, + #[serde(flatten)] + pub other: serde_json::Value, } #[derive(Serialize, Deserialize, Debug, Clone)] @@ -25,9 +26,6 @@ pub struct EnvironmentConfig { pub networks: Vec, } -/// Explorer chain configuration file. This file is created on the chain level -/// and is used to configure the explorer for a specific chain. It serves as a building block for -/// the explorer runtime config. #[derive(Serialize, Deserialize, Debug, Clone)] #[serde(rename_all = "camelCase")] pub struct ExplorerChainConfig { @@ -47,65 +45,103 @@ pub struct ExplorerChainConfig { pub l1_explorer_url: Option, #[serde(skip_serializing_if = "Option::is_none")] pub verification_api_url: Option, // L2 verification API URL + #[serde(flatten)] + pub other: serde_json::Value, } -impl ExplorerRuntimeConfig { - pub fn new(explorer_chain_configs: Vec) -> Self { - Self { - app_environment: "default".to_string(), - environment_config: EnvironmentConfig { - networks: explorer_chain_configs, - }, - } - } - +impl ExplorerConfig { + /// Returns the path to the explorer configuration file. pub fn get_config_path(ecosystem_base_path: &Path) -> PathBuf { ecosystem_base_path .join(LOCAL_CONFIGS_PATH) - .join(LOCAL_GENERATED_PATH) - .join(EXPLORER_RUNTIME_CONFIG_FILE) + .join(LOCAL_APPS_PATH) + .join(EXPLORER_CONFIG_FILE) } -} -impl SaveConfig for ExplorerRuntimeConfig { - fn save(&self, shell: &Shell, path: impl AsRef) -> anyhow::Result<()> { + /// Reads the existing config or creates a default one if it doesn't exist. + pub fn read_or_create_default(shell: &Shell) -> anyhow::Result { + let config_path = Self::get_config_path(&shell.current_dir()); + match Self::read(shell, &config_path) { + Ok(config) => Ok(config), + Err(_) => { + let config = Self::default(); + config.save(shell, &config_path)?; + Ok(config) + } + } + } + + /// Adds or updates a given chain configuration. + pub fn add_chain_config(&mut self, config: &ExplorerChainConfig) { + // Replace if config with the same network name already exists + if let Some(index) = self + .environment_config + .networks + .iter() + .position(|c| c.name == config.name) + { + self.environment_config.networks[index] = config.clone(); + return; + } + self.environment_config.networks.push(config.clone()); + } + + /// Retains only the chains whose names are present in the given vector. + pub fn filter(&mut self, chain_names: &[String]) { + self.environment_config + .networks + .retain(|config| chain_names.contains(&config.name)); + } + + /// Hides all chains except those specified in the given vector. + pub fn hide_except(&mut self, chain_names: &[String]) { + for network in &mut self.environment_config.networks { + network.published = chain_names.contains(&network.name); + } + } + + /// Checks if a chain with the given name exists in the configuration. + pub fn contains(&self, chain_name: &String) -> bool { + self.environment_config + .networks + .iter() + .any(|config| &config.name == chain_name) + } + + pub fn is_empty(&self) -> bool { + self.environment_config.networks.is_empty() + } + + pub fn save_as_js(&self, shell: &Shell) -> anyhow::Result { // The block-explorer-app is served as a pre-built static app in a Docker image. // It uses a JavaScript file (config.js) that injects the configuration at runtime // by overwriting the '##runtimeConfig' property of the window object. - // Therefore, we generate a JavaScript file instead of a JSON file. // This file will be mounted to the Docker image when it runs. + let path = Self::get_generated_js_config_path(&shell.current_dir()); let json = serde_json::to_string_pretty(&self)?; let config_js_content = format!("window['##runtimeConfig'] = {};", json); - Ok(shell.write_file(path, config_js_content.as_bytes())?) + shell.write_file(path.clone(), config_js_content.as_bytes())?; + Ok(path) } -} -impl ReadConfig for ExplorerRuntimeConfig { - fn read(shell: &Shell, path: impl AsRef) -> anyhow::Result { - let config_js_content = shell.read_file(path)?; - // Extract the JSON part from the JavaScript file - let json_start = config_js_content - .find('{') - .ok_or_else(|| anyhow::anyhow!("Invalid config file format"))?; - let json_end = config_js_content - .rfind('}') - .ok_or_else(|| anyhow::anyhow!("Invalid config file format"))?; - let json_str = &config_js_content[json_start..=json_end]; - // Parse the JSON into ExplorerRuntimeConfig - let config: ExplorerRuntimeConfig = serde_json::from_str(json_str)?; - Ok(config) + fn get_generated_js_config_path(ecosystem_base_path: &Path) -> PathBuf { + ecosystem_base_path + .join(LOCAL_CONFIGS_PATH) + .join(LOCAL_GENERATED_PATH) + .join(EXPLORER_JS_CONFIG_FILE) } } -impl ExplorerChainConfig { - pub fn get_config_path(ecosystem_base_path: &Path, chain_name: &str) -> PathBuf { - ecosystem_base_path - .join(LOCAL_CHAINS_PATH) - .join(chain_name) - .join(LOCAL_CONFIGS_PATH) - .join(LOCAL_APPS_PATH) - .join(EXPLORER_CHAIN_CONFIG_FILE) +impl Default for ExplorerConfig { + fn default() -> Self { + ExplorerConfig { + app_environment: "default".to_string(), + environment_config: EnvironmentConfig { + networks: Vec::new(), + }, + other: serde_json::Value::Null, + } } } -impl ZkToolboxConfig for ExplorerChainConfig {} +impl ZkToolboxConfig for ExplorerConfig {} diff --git a/zk_toolbox/crates/config/src/explorer_compose.rs b/zk_toolbox/crates/config/src/explorer_compose.rs index a94861a372a4..ca9abc1e3e23 100644 --- a/zk_toolbox/crates/config/src/explorer_compose.rs +++ b/zk_toolbox/crates/config/src/explorer_compose.rs @@ -9,19 +9,63 @@ use serde::{Deserialize, Serialize}; use url::Url; use crate::{ - apps::AppsChainExplorerConfig, consts::{ - EXPLORER_API_DOCKER_IMAGE, EXPLORER_DATA_FETCHER_DOCKER_IMAGE, - EXPLORER_DOCKER_COMPOSE_FILE, EXPLORER_WORKER_DOCKER_IMAGE, LOCAL_CHAINS_PATH, - LOCAL_CONFIGS_PATH, LOCAL_GENERATED_PATH, + DEFAULT_EXPLORER_API_PORT, DEFAULT_EXPLORER_DATA_FETCHER_PORT, + DEFAULT_EXPLORER_WORKER_PORT, EXPLORER_API_DOCKER_IMAGE, + EXPLORER_DATA_FETCHER_DOCKER_IMAGE, EXPLORER_DOCKER_COMPOSE_FILE, + EXPLORER_WORKER_DOCKER_IMAGE, LOCAL_CHAINS_PATH, LOCAL_CONFIGS_PATH, }, docker_compose::{DockerComposeConfig, DockerComposeService}, traits::ZkToolboxConfig, + EXPLORER_BATCHES_PROCESSING_POLLING_INTERVAL, }; -/// Chain-level explorer backend docker compose file. It contains the configuration for -/// api, data fetcher, and worker services. -/// This config is auto-generated during "explorer run-backend" command. +#[derive(Debug, Serialize, Deserialize, Clone)] +pub struct ExplorerBackendPorts { + pub api_http_port: u16, + pub data_fetcher_http_port: u16, + pub worker_http_port: u16, +} + +impl ExplorerBackendPorts { + pub fn with_offset(&self, offset: u16) -> Self { + ExplorerBackendPorts { + api_http_port: self.api_http_port + offset, + data_fetcher_http_port: self.data_fetcher_http_port + offset, + worker_http_port: self.worker_http_port + offset, + } + } +} + +impl Default for ExplorerBackendPorts { + fn default() -> Self { + ExplorerBackendPorts { + api_http_port: DEFAULT_EXPLORER_API_PORT, + data_fetcher_http_port: DEFAULT_EXPLORER_DATA_FETCHER_PORT, + worker_http_port: DEFAULT_EXPLORER_WORKER_PORT, + } + } +} + +#[derive(Debug, Serialize, Deserialize, Clone)] +pub struct ExplorerBackendConfig { + pub database_url: Url, + pub ports: ExplorerBackendPorts, + pub batches_processing_polling_interval: u64, +} + +impl ExplorerBackendConfig { + pub fn new(database_url: Url, ports: &ExplorerBackendPorts) -> Self { + ExplorerBackendConfig { + database_url, + ports: ports.clone(), + batches_processing_polling_interval: EXPLORER_BATCHES_PROCESSING_POLLING_INTERVAL, + } + } +} + +/// Chain-level explorer backend docker compose file. +/// It contains configuration for api, data fetcher, and worker services. #[derive(Debug, Serialize, Deserialize, Clone)] pub struct ExplorerBackendComposeConfig { #[serde(flatten)] @@ -31,50 +75,57 @@ pub struct ExplorerBackendComposeConfig { impl ZkToolboxConfig for ExplorerBackendComposeConfig {} impl ExplorerBackendComposeConfig { + const API_NAME: &'static str = "api"; + const DATA_FETCHER_NAME: &'static str = "data-fetcher"; + const WORKER_NAME: &'static str = "worker"; + pub fn new( chain_name: &str, l2_rpc_url: Url, - config: &AppsChainExplorerConfig, + config: &ExplorerBackendConfig, ) -> anyhow::Result { let db_url = adjust_localhost_for_docker(config.database_url.clone())?; let l2_rpc_url = adjust_localhost_for_docker(l2_rpc_url)?; let mut services: HashMap = HashMap::new(); services.insert( - Self::api_name(chain_name), - Self::create_api_service(chain_name, config.services.api_http_port, db_url.as_ref()), + Self::API_NAME.to_string(), + Self::create_api_service(config.ports.api_http_port, db_url.as_ref()), ); services.insert( - Self::data_fetcher_name(chain_name), + Self::DATA_FETCHER_NAME.to_string(), Self::create_data_fetcher_service( - config.services.data_fetcher_http_port, + config.ports.data_fetcher_http_port, l2_rpc_url.as_ref(), ), ); let worker = Self::create_worker_service( - chain_name, - config.services.worker_http_port, - config.services.data_fetcher_http_port, + config.ports.worker_http_port, + config.ports.data_fetcher_http_port, l2_rpc_url.as_ref(), &db_url, - config.services.batches_processing_polling_interval, + config.batches_processing_polling_interval, ) .context("Failed to create worker service")?; - services.insert(Self::worker_name(chain_name), worker); + services.insert(Self::WORKER_NAME.to_string(), worker); Ok(Self { - docker_compose: DockerComposeConfig { services }, + docker_compose: DockerComposeConfig { + name: Some(format!("{chain_name}-explorer")), + services, + other: serde_json::Value::Null, + }, }) } - fn create_api_service(chain_name: &str, port: u16, db_url: &str) -> DockerComposeService { + fn create_api_service(port: u16, db_url: &str) -> DockerComposeService { DockerComposeService { image: EXPLORER_API_DOCKER_IMAGE.to_string(), platform: Some("linux/amd64".to_string()), ports: Some(vec![format!("{}:{}", port, port)]), volumes: None, - depends_on: Some(vec![Self::worker_name(chain_name)]), + depends_on: Some(vec![Self::WORKER_NAME.to_string()]), restart: None, environment: Some(HashMap::from([ ("PORT".to_string(), port.to_string()), @@ -107,18 +158,13 @@ impl ExplorerBackendComposeConfig { } fn create_worker_service( - chain_name: &str, port: u16, data_fetcher_port: u16, l2_rpc_url: &str, db_url: &Url, batches_processing_polling_interval: u64, ) -> anyhow::Result { - let data_fetcher_url = format!( - "http://{}:{}", - Self::data_fetcher_name(chain_name), - data_fetcher_port - ); + let data_fetcher_url = format!("http://{}:{}", Self::DATA_FETCHER_NAME, data_fetcher_port); // Parse database URL let db_config = db::DatabaseConfig::from_url(db_url)?; @@ -158,24 +204,11 @@ impl ExplorerBackendComposeConfig { }) } - fn worker_name(chain_name: &str) -> String { - format!("explorer-worker-{}", chain_name) - } - - fn api_name(chain_name: &str) -> String { - format!("explorer-api-{}", chain_name) - } - - fn data_fetcher_name(chain_name: &str) -> String { - format!("explorer-data-fetcher-{}", chain_name) - } - pub fn get_config_path(ecosystem_base_path: &Path, chain_name: &str) -> PathBuf { ecosystem_base_path .join(LOCAL_CHAINS_PATH) .join(chain_name) .join(LOCAL_CONFIGS_PATH) - .join(LOCAL_GENERATED_PATH) .join(EXPLORER_DOCKER_COMPOSE_FILE) } } diff --git a/zk_toolbox/crates/config/src/portal.rs b/zk_toolbox/crates/config/src/portal.rs index 78d9acfc43de..c787c6cc7026 100644 --- a/zk_toolbox/crates/config/src/portal.rs +++ b/zk_toolbox/crates/config/src/portal.rs @@ -6,24 +6,22 @@ use xshell::Shell; use crate::{ consts::{ - LOCAL_APPS_PATH, LOCAL_CHAINS_PATH, LOCAL_CONFIGS_PATH, LOCAL_GENERATED_PATH, - PORTAL_CHAIN_CONFIG_FILE, PORTAL_RUNTIME_CONFIG_FILE, + LOCAL_APPS_PATH, LOCAL_CONFIGS_PATH, LOCAL_GENERATED_PATH, PORTAL_CONFIG_FILE, + PORTAL_JS_CONFIG_FILE, }, traits::{ReadConfig, SaveConfig, ZkToolboxConfig}, }; -/// Portal configuration file. This file is auto-generated during the "portal" command -/// and is used to inject the runtime configuration into the portal app. +/// Portal JSON configuration file. This file contains configuration for the portal app. #[derive(Serialize, Deserialize, Debug, Clone)] #[serde(rename_all = "camelCase")] -pub struct PortalRuntimeConfig { +pub struct PortalConfig { pub node_type: String, pub hyperchains_config: Vec, + #[serde(flatten)] + pub other: serde_json::Value, } -/// Portal chain configuration file. This file is created on the chain level -/// and is used to configure the portal for a specific chain. It serves as a building block for -/// the portal runtime config. #[derive(Serialize, Deserialize, Debug, Clone)] pub struct PortalChainConfig { pub network: NetworkConfig, @@ -38,6 +36,8 @@ pub struct NetworkConfig { pub name: String, // L2 Network name (displayed in the app dropdown) pub rpc_url: String, // L2 RPC URL #[serde(skip_serializing_if = "Option::is_none")] + pub hidden: Option, // If true, the chain will not be shown in the app dropdown + #[serde(skip_serializing_if = "Option::is_none")] pub block_explorer_url: Option, // L2 Block Explorer URL #[serde(skip_serializing_if = "Option::is_none")] pub block_explorer_api: Option, // L2 Block Explorer API @@ -45,6 +45,8 @@ pub struct NetworkConfig { pub public_l1_network_id: Option, // Ethereum Mainnet or Ethereum Sepolia Testnet ID #[serde(skip_serializing_if = "Option::is_none")] pub l1_network: Option, + #[serde(flatten)] + pub other: serde_json::Value, } #[derive(Serialize, Deserialize, Debug, Clone)] @@ -80,61 +82,94 @@ pub struct TokenConfig { pub name: Option, } -impl PortalRuntimeConfig { - pub fn new(portal_chain_configs: Vec) -> Self { - Self { - node_type: "hyperchain".to_string(), - hyperchains_config: portal_chain_configs, - } - } - +impl PortalConfig { + /// Returns the path to the portal configuration file. pub fn get_config_path(ecosystem_base_path: &Path) -> PathBuf { ecosystem_base_path .join(LOCAL_CONFIGS_PATH) - .join(LOCAL_GENERATED_PATH) - .join(PORTAL_RUNTIME_CONFIG_FILE) + .join(LOCAL_APPS_PATH) + .join(PORTAL_CONFIG_FILE) + } + + /// Reads the existing config or creates a default one if it doesn't exist. + pub fn read_or_create_default(shell: &Shell) -> anyhow::Result { + let config_path = Self::get_config_path(&shell.current_dir()); + match Self::read(shell, &config_path) { + Ok(config) => Ok(config), + Err(_) => { + let config = Self::default(); + config.save(shell, &config_path)?; + Ok(config) + } + } + } + + /// Adds or updates a given chain configuration. + pub fn add_chain_config(&mut self, config: &PortalChainConfig) { + // Replace if config with the same network key already exists + if let Some(index) = self + .hyperchains_config + .iter() + .position(|c| c.network.key == config.network.key) + { + self.hyperchains_config[index] = config.clone(); + return; + } + self.hyperchains_config.push(config.clone()); + } + + /// Retains only the chains whose names are present in the given vector. + pub fn filter(&mut self, chain_names: &[String]) { + self.hyperchains_config + .retain(|config| chain_names.contains(&config.network.key)); } -} -impl SaveConfig for PortalRuntimeConfig { - fn save(&self, shell: &Shell, path: impl AsRef) -> anyhow::Result<()> { + /// Hides all chains except those specified in the given vector. + pub fn hide_except(&mut self, chain_names: &[String]) { + for config in &mut self.hyperchains_config { + config.network.hidden = Some(!chain_names.contains(&config.network.key)); + } + } + + /// Checks if a chain with the given name exists in the configuration. + pub fn contains(&self, chain_name: &String) -> bool { + self.hyperchains_config + .iter() + .any(|config| &config.network.key == chain_name) + } + + pub fn is_empty(&self) -> bool { + self.hyperchains_config.is_empty() + } + + pub fn save_as_js(&self, shell: &Shell) -> anyhow::Result { // The dapp-portal is served as a pre-built static app in a Docker image. // It uses a JavaScript file (config.js) that injects the configuration at runtime // by overwriting the '##runtimeConfig' property of the window object. - // Therefore, we generate a JavaScript file instead of a JSON file. // This file will be mounted to the Docker image when it runs. + let path = Self::get_generated_js_config_path(&shell.current_dir()); let json = serde_json::to_string_pretty(&self)?; let config_js_content = format!("window['##runtimeConfig'] = {};", json); - Ok(shell.write_file(path, config_js_content.as_bytes())?) + shell.write_file(path.clone(), config_js_content.as_bytes())?; + Ok(path) } -} -impl ReadConfig for PortalRuntimeConfig { - fn read(shell: &Shell, path: impl AsRef) -> anyhow::Result { - let config_js_content = shell.read_file(path)?; - // Extract the JSON part from the JavaScript file - let json_start = config_js_content - .find('{') - .ok_or_else(|| anyhow::anyhow!("Invalid config file format"))?; - let json_end = config_js_content - .rfind('}') - .ok_or_else(|| anyhow::anyhow!("Invalid config file format"))?; - let json_str = &config_js_content[json_start..=json_end]; - // Parse the JSON into PortalRuntimeConfig - let config: PortalRuntimeConfig = serde_json::from_str(json_str)?; - Ok(config) + fn get_generated_js_config_path(ecosystem_base_path: &Path) -> PathBuf { + ecosystem_base_path + .join(LOCAL_CONFIGS_PATH) + .join(LOCAL_GENERATED_PATH) + .join(PORTAL_JS_CONFIG_FILE) } } -impl PortalChainConfig { - pub fn get_config_path(ecosystem_base_path: &Path, chain_name: &str) -> PathBuf { - ecosystem_base_path - .join(LOCAL_CHAINS_PATH) - .join(chain_name) - .join(LOCAL_CONFIGS_PATH) - .join(LOCAL_APPS_PATH) - .join(PORTAL_CHAIN_CONFIG_FILE) +impl Default for PortalConfig { + fn default() -> Self { + PortalConfig { + node_type: "hyperchain".to_string(), + hyperchains_config: Vec::new(), + other: serde_json::Value::Null, + } } } -impl ZkToolboxConfig for PortalChainConfig {} +impl ZkToolboxConfig for PortalConfig {} diff --git a/zk_toolbox/crates/zk_inception/src/commands/chain/init.rs b/zk_toolbox/crates/zk_inception/src/commands/chain/init.rs index 63dedb361a7d..793fbbf31aee 100644 --- a/zk_toolbox/crates/zk_inception/src/commands/chain/init.rs +++ b/zk_toolbox/crates/zk_inception/src/commands/chain/init.rs @@ -28,7 +28,7 @@ use crate::{ genesis::genesis, set_token_multiplier_setter::set_token_multiplier_setter, }, - portal::create_portal_chain_config, + portal::update_portal_config, }, consts::AMOUNT_FOR_DISTRIBUTION_TO_WALLETS, messages::{ @@ -154,7 +154,7 @@ pub async fn init( .await .context(MSG_GENESIS_DATABASE_ERR)?; - create_portal_chain_config(chain_config, shell) + update_portal_config(shell, chain_config) .await .context(MSG_PORTAL_FAILED_TO_CREATE_CONFIG_ERR)?; diff --git a/zk_toolbox/crates/zk_inception/src/commands/explorer/backend.rs b/zk_toolbox/crates/zk_inception/src/commands/explorer/backend.rs index a7b9185d1557..6fdd3faa9807 100644 --- a/zk_toolbox/crates/zk_inception/src/commands/explorer/backend.rs +++ b/zk_toolbox/crates/zk_inception/src/commands/explorer/backend.rs @@ -2,11 +2,7 @@ use std::path::Path; use anyhow::Context; use common::{config::global_config, docker}; -use config::{ - explorer_compose::*, - traits::{ConfigWithL2RpcUrl, ReadConfig, SaveConfig}, - AppsChainConfig, EcosystemConfig, -}; +use config::{explorer_compose::ExplorerBackendComposeConfig, EcosystemConfig}; use xshell::Shell; use crate::messages::{ @@ -20,22 +16,15 @@ pub(crate) fn run(shell: &Shell) -> anyhow::Result<()> { .load_chain(global_config().chain_name.clone()) .context(MSG_CHAIN_NOT_FOUND_ERR)?; let chain_name = chain_config.name.clone(); - // Read chain-level apps.yaml config + // Read chain-level explorer backend docker compose file let ecosystem_path = shell.current_dir(); - let apps_chain_config_path = AppsChainConfig::get_config_path(&ecosystem_path, &chain_name); - if !apps_chain_config_path.exists() { + let backend_config_path = + ExplorerBackendComposeConfig::get_config_path(&ecosystem_path, &chain_config.name); + if !backend_config_path.exists() { anyhow::bail!(msg_explorer_chain_not_initialized(&chain_name)); } - let apps_chain_config = AppsChainConfig::read(shell, &apps_chain_config_path)?; - // Build docker compose config with the explorer chain backend services - let l2_rpc_url = chain_config.get_general_config()?.get_l2_rpc_url()?; - let backend_compose_config = - ExplorerBackendComposeConfig::new(&chain_name, l2_rpc_url, &apps_chain_config.explorer)?; - let backend_compose_config_path = - ExplorerBackendComposeConfig::get_config_path(&ecosystem_path, &chain_config.name); - backend_compose_config.save(shell, backend_compose_config_path.clone())?; // Run docker compose - run_backend(shell, &backend_compose_config_path)?; + run_backend(shell, &backend_config_path)?; Ok(()) } diff --git a/zk_toolbox/crates/zk_inception/src/commands/explorer/init.rs b/zk_toolbox/crates/zk_inception/src/commands/explorer/init.rs index 099c5232bf2b..43700d91a0df 100644 --- a/zk_toolbox/crates/zk_inception/src/commands/explorer/init.rs +++ b/zk_toolbox/crates/zk_inception/src/commands/explorer/init.rs @@ -1,9 +1,10 @@ use anyhow::Context; use common::{config::global_config, db, logger, Prompt}; use config::{ - explorer::*, - traits::{ConfigWithL2RpcUrl, ReadConfig, SaveConfig}, - AppsChainConfig, AppsChainExplorerConfig, ChainConfig, EcosystemConfig, ExplorerServicesConfig, + explorer::{ExplorerChainConfig, ExplorerConfig}, + explorer_compose::{ExplorerBackendComposeConfig, ExplorerBackendConfig, ExplorerBackendPorts}, + traits::{ConfigWithL2RpcUrl, SaveConfig}, + ChainConfig, EcosystemConfig, }; use slugify_rs::slugify; use url::Url; @@ -12,10 +13,7 @@ use xshell::Shell; use crate::{ commands::chain::args::init::PortOffset, consts::L2_BASE_TOKEN_ADDRESS, - defaults::{ - generate_explorer_db_name, DATABASE_EXPLORER_URL, EXPLORER_API_PORT, - EXPLORER_DATA_FETCHER_PORT, EXPLORER_WORKER_PORT, - }, + defaults::{generate_explorer_db_name, DATABASE_EXPLORER_URL}, messages::{ msg_chain_load_err, msg_explorer_db_name_prompt, msg_explorer_db_url_prompt, msg_explorer_initializing_database_for, MSG_EXPLORER_FAILED_TO_DROP_DATABASE_ERR, @@ -31,49 +29,45 @@ pub(crate) async fn run(shell: &Shell) -> anyhow::Result<()> { None => ecosystem_config.list_of_chains(), }; // Initialize chains one by one + let mut explorer_config = ExplorerConfig::read_or_create_default(shell)?; for chain_name in chains_enabled.iter() { // Load chain config let chain_config = ecosystem_config .load_chain(Some(chain_name.clone())) .context(msg_chain_load_err(chain_name))?; - // Initialize chain-level apps.yaml - let apps_chain_config = initialize_apps_chain_config(shell, &chain_config)?; + // Build backend config - parameters required to create explorer backend services + let backend_config = build_backend_config(&chain_config); // Initialize explorer database - initialize_explorer_database(&apps_chain_config.explorer.database_url).await?; - // Create chain-level explorer.json - create_explorer_chain_config(shell, &chain_config, &apps_chain_config.explorer)?; + initialize_explorer_database(&backend_config.database_url).await?; + // Create explorer backend docker compose file + let l2_rpc_url = chain_config.get_general_config()?.get_l2_rpc_url()?; + let backend_compose_config = + ExplorerBackendComposeConfig::new(chain_name, l2_rpc_url, &backend_config)?; + let backend_compose_config_path = + ExplorerBackendComposeConfig::get_config_path(&shell.current_dir(), chain_name); + backend_compose_config.save(shell, &backend_compose_config_path)?; + // Add chain to explorer.json + let explorer_chain_config = build_explorer_chain_config(&chain_config, &backend_config)?; + explorer_config.add_chain_config(&explorer_chain_config); } + // Save explorer config + let config_path = ExplorerConfig::get_config_path(&shell.current_dir()); + explorer_config.save(shell, config_path)?; + logger::outro(MSG_EXPLORER_INITIALIZED); Ok(()) } -fn initialize_apps_chain_config( - shell: &Shell, - chain_config: &ChainConfig, -) -> anyhow::Result { - let ecosystem_path = shell.current_dir(); - let apps_chain_config_path = - AppsChainConfig::get_config_path(&ecosystem_path, &chain_config.name); - // Check if apps chain config exists - if let Ok(apps_chain_config) = AppsChainConfig::read(shell, &apps_chain_config_path) { - return Ok(apps_chain_config); - } +fn build_backend_config(chain_config: &ChainConfig) -> ExplorerBackendConfig { // Prompt explorer database name logger::info(msg_explorer_initializing_database_for(&chain_config.name)); let db_config = fill_database_values_with_prompt(chain_config); // Allocate ports for backend services - let services_config = allocate_explorer_services_ports(chain_config); + let backend_ports = allocate_explorer_services_ports(chain_config); - // Build and save apps chain config - let app_chain_config = AppsChainConfig { - explorer: AppsChainExplorerConfig { - database_url: db_config.full_url(), - services: services_config, - }, - }; - app_chain_config.save(shell, &apps_chain_config_path)?; - Ok(app_chain_config) + // Build explorer backend config + ExplorerBackendConfig::new(db_config.full_url(), &backend_ports) } async fn initialize_explorer_database(db_url: &Url) -> anyhow::Result<()> { @@ -98,31 +92,15 @@ fn fill_database_values_with_prompt(config: &ChainConfig) -> db::DatabaseConfig db::DatabaseConfig::new(explorer_db_url, explorer_db_name) } -fn allocate_explorer_services_ports(chain_config: &ChainConfig) -> ExplorerServicesConfig { +fn allocate_explorer_services_ports(chain_config: &ChainConfig) -> ExplorerBackendPorts { // Try to allocate intuitive ports with an offset from the defaults let offset: u16 = PortOffset::from_chain_id(chain_config.id as u16).into(); - ExplorerServicesConfig::new( - EXPLORER_API_PORT + offset, - EXPLORER_DATA_FETCHER_PORT + offset, - EXPLORER_WORKER_PORT + offset, - ) -} - -fn create_explorer_chain_config( - shell: &Shell, - chain_config: &ChainConfig, - apps_chain_config: &AppsChainExplorerConfig, -) -> anyhow::Result { - let explorer_config = build_explorer_chain_config(chain_config, apps_chain_config)?; - let config_path = - ExplorerChainConfig::get_config_path(&shell.current_dir(), &chain_config.name); - explorer_config.save(shell, config_path)?; - Ok(explorer_config) + ExplorerBackendPorts::default().with_offset(offset) } fn build_explorer_chain_config( chain_config: &ChainConfig, - apps_chain_explorer_config: &AppsChainExplorerConfig, + backend_config: &ExplorerBackendConfig, ) -> anyhow::Result { let general_config = chain_config.get_general_config()?; // Get L2 RPC URL from general config @@ -134,7 +112,7 @@ fn build_explorer_chain_config( .map(|verifier| &verifier.url) .context("verification_url")?; // Build API URL - let api_port = apps_chain_explorer_config.services.api_http_port; + let api_port = backend_config.ports.api_http_port; let api_url = format!("http://127.0.0.1:{}", api_port); // Build explorer chain config @@ -152,5 +130,6 @@ fn build_explorer_chain_config( bridge_url: None, l1_explorer_url: None, verification_api_url: Some(verification_api_url.to_string()), + other: serde_json::Value::Null, }) } diff --git a/zk_toolbox/crates/zk_inception/src/commands/explorer/run.rs b/zk_toolbox/crates/zk_inception/src/commands/explorer/run.rs index f31599c850ce..a6519f62edba 100644 --- a/zk_toolbox/crates/zk_inception/src/commands/explorer/run.rs +++ b/zk_toolbox/crates/zk_inception/src/commands/explorer/run.rs @@ -1,19 +1,15 @@ -use std::{collections::HashMap, path::Path}; +use std::path::Path; use anyhow::Context; use common::{config::global_config, docker, logger}; -use config::{ - explorer::*, - traits::{ReadConfig, SaveConfig}, - AppsEcosystemConfig, EcosystemConfig, -}; +use config::{explorer::*, traits::SaveConfig, AppsEcosystemConfig, EcosystemConfig}; use xshell::Shell; use crate::{ consts::{EXPLORER_APP_DOCKER_CONFIG_PATH, EXPLORER_APP_DOCKER_IMAGE}, messages::{ - msg_explorer_failed_to_configure_chain, msg_explorer_skipping_not_initialized_chain, - msg_explorer_starting_on, MSG_EXPLORER_FAILED_TO_FIND_ANY_VALID, + msg_explorer_running_with_config, msg_explorer_starting_on, + MSG_EXPLORER_FAILED_TO_CREATE_CONFIG_ERR, MSG_EXPLORER_FAILED_TO_FIND_ANY_CHAIN_ERR, MSG_EXPLORER_FAILED_TO_RUN_DOCKER_ERR, }, }; @@ -29,43 +25,46 @@ pub(crate) fn run(shell: &Shell) -> anyhow::Result<()> { None => ecosystem_config.list_of_chains(), }; - // Read chain configs one by one - let mut explorer_chain_configs = Vec::new(); - for chain_name in chains_enabled.iter() { - let explorer_chain_config_path = - ExplorerChainConfig::get_config_path(&ecosystem_path, chain_name); - if !explorer_chain_config_path.exists() { - logger::warn(msg_explorer_skipping_not_initialized_chain(chain_name)); - continue; - } - match ExplorerChainConfig::read(shell, &explorer_chain_config_path) { - Ok(config) => explorer_chain_configs.push(config), - Err(_) => logger::warn(msg_explorer_failed_to_configure_chain(chain_name)), - } - } - if explorer_chain_configs.is_empty() { - anyhow::bail!(MSG_EXPLORER_FAILED_TO_FIND_ANY_VALID); + // Read explorer config + let config_path = ExplorerConfig::get_config_path(&ecosystem_path); + let mut explorer_config = ExplorerConfig::read_or_create_default(shell) + .context(MSG_EXPLORER_FAILED_TO_CREATE_CONFIG_ERR)?; + + // Validate and update explorer config + explorer_config.filter(&ecosystem_config.list_of_chains()); + explorer_config.hide_except(&chains_enabled); + if explorer_config.is_empty() { + anyhow::bail!(MSG_EXPLORER_FAILED_TO_FIND_ANY_CHAIN_ERR); } - // Generate and save explorer runtime config (JS) - let runtime_config = ExplorerRuntimeConfig::new(explorer_chain_configs); - let config_path = ExplorerRuntimeConfig::get_config_path(&ecosystem_path); - runtime_config.save(shell, &config_path)?; + // Save explorer config + explorer_config.save(shell, &config_path)?; - logger::info(format!( - "Using generated explorer config file at {}", - config_path.display() - )); + let config_js_path = explorer_config + .save_as_js(shell) + .context(MSG_EXPLORER_FAILED_TO_CREATE_CONFIG_ERR)?; + + logger::info(msg_explorer_running_with_config(&config_path)); logger::info(msg_explorer_starting_on( "127.0.0.1", apps_config.explorer.http_port, )); - - run_explorer(shell, &config_path, apps_config.explorer.http_port)?; + let name = explorer_app_name(&ecosystem_config.name); + run_explorer( + shell, + &config_js_path, + &name, + apps_config.explorer.http_port, + )?; Ok(()) } -fn run_explorer(shell: &Shell, config_file_path: &Path, port: u16) -> anyhow::Result<()> { +fn run_explorer( + shell: &Shell, + config_file_path: &Path, + name: &str, + port: u16, +) -> anyhow::Result<()> { let port_mapping = format!("{}:{}", port, port); let volume_mapping = format!( "{}:{}", @@ -73,13 +72,27 @@ fn run_explorer(shell: &Shell, config_file_path: &Path, port: u16) -> anyhow::Re EXPLORER_APP_DOCKER_CONFIG_PATH ); - let mut docker_args: HashMap = HashMap::new(); - docker_args.insert("--platform".to_string(), "linux/amd64".to_string()); - docker_args.insert("-p".to_string(), port_mapping); - docker_args.insert("-v".to_string(), volume_mapping); - docker_args.insert("-e".to_string(), format!("PORT={}", port)); + let docker_args: Vec = vec![ + "--platform".to_string(), + "linux/amd64".to_string(), + "--name".to_string(), + name.to_string(), + "-p".to_string(), + port_mapping, + "-v".to_string(), + volume_mapping, + "-e".to_string(), + format!("PORT={}", port), + "--rm".to_string(), + ]; docker::run(shell, EXPLORER_APP_DOCKER_IMAGE, docker_args) .with_context(|| MSG_EXPLORER_FAILED_TO_RUN_DOCKER_ERR)?; Ok(()) } + +/// Generates a name for the explorer app Docker container. +/// Will be passed as `--name` argument to `docker run`. +fn explorer_app_name(ecosystem_name: &str) -> String { + format!("{}-explorer-app", ecosystem_name) +} diff --git a/zk_toolbox/crates/zk_inception/src/commands/portal.rs b/zk_toolbox/crates/zk_inception/src/commands/portal.rs index fb12b73f5b16..5bf211211779 100644 --- a/zk_toolbox/crates/zk_inception/src/commands/portal.rs +++ b/zk_toolbox/crates/zk_inception/src/commands/portal.rs @@ -1,10 +1,10 @@ -use std::{collections::HashMap, path::Path}; +use std::path::Path; use anyhow::Context; use common::{config::global_config, docker, ethereum, logger}; use config::{ portal::*, - traits::{ConfigWithL2RpcUrl, ReadConfig, SaveConfig}, + traits::{ConfigWithL2RpcUrl, SaveConfig}, AppsEcosystemConfig, ChainConfig, EcosystemConfig, }; use ethers::types::Address; @@ -14,8 +14,9 @@ use xshell::Shell; use crate::{ consts::{L2_BASE_TOKEN_ADDRESS, PORTAL_DOCKER_CONFIG_PATH, PORTAL_DOCKER_IMAGE}, messages::{ - msg_portal_starting_on, MSG_PORTAL_FAILED_TO_CREATE_ANY_CHAIN_CONFIG_ERR, - MSG_PORTAL_FAILED_TO_CREATE_CONFIG_ERR, MSG_PORTAL_FAILED_TO_RUN_DOCKER_ERR, + msg_portal_running_with_config, msg_portal_starting_on, + MSG_PORTAL_FAILED_TO_CREATE_CONFIG_ERR, MSG_PORTAL_FAILED_TO_FIND_ANY_CHAIN_ERR, + MSG_PORTAL_FAILED_TO_RUN_DOCKER_ERR, }, }; @@ -74,84 +75,88 @@ async fn build_portal_chain_config( public_l1_network_id: None, block_explorer_url: None, block_explorer_api: None, + hidden: None, + other: serde_json::Value::Null, }, tokens, }) } -pub async fn create_portal_chain_config( - chain_config: &ChainConfig, +pub async fn update_portal_config( shell: &Shell, -) -> anyhow::Result { - let portal_config = build_portal_chain_config(chain_config).await?; - let config_path = PortalChainConfig::get_config_path(&shell.current_dir(), &chain_config.name); + chain_config: &ChainConfig, +) -> anyhow::Result { + // Build and append portal chain config to the portal config + let portal_chain_config = build_portal_chain_config(chain_config).await?; + let mut portal_config = PortalConfig::read_or_create_default(shell)?; + portal_config.add_chain_config(&portal_chain_config); + // Save portal config + let config_path = PortalConfig::get_config_path(&shell.current_dir()); portal_config.save(shell, config_path)?; Ok(portal_config) } -async fn build_portal_runtime_config( - shell: &Shell, +/// Validates portal config - appends missing chains and removes unknown chains +async fn validate_portal_config( + portal_config: &mut PortalConfig, ecosystem_config: &EcosystemConfig, - chain_names: Vec, -) -> anyhow::Result { - let ecosystem_base_path = &shell.current_dir(); - let mut hyperchains_config = Vec::new(); - - for chain_name in chain_names { - let config_path = PortalChainConfig::get_config_path(ecosystem_base_path, &chain_name); - - let portal_chain_config = match PortalChainConfig::read(shell, &config_path) { - Ok(config) => Some(config), - Err(_) => match ecosystem_config.load_chain(Some(chain_name.clone())) { - Some(chain_config) => match build_portal_chain_config(&chain_config).await { - Ok(config) => Some(config), - Err(_) => None, - }, - None => None, - }, - }; - if let Some(config) = portal_chain_config { - hyperchains_config.push(config); +) -> anyhow::Result<()> { + let chain_names = ecosystem_config.list_of_chains(); + for chain_name in &chain_names { + if portal_config.contains(chain_name) { + continue; + } + // Append missing chain, chain might not be initialized, so ignoring errors + if let Some(chain_config) = ecosystem_config.load_chain(Some(chain_name.clone())) { + if let Ok(portal_chain_config) = build_portal_chain_config(&chain_config).await { + portal_config.add_chain_config(&portal_chain_config); + } } } - if hyperchains_config.is_empty() { - anyhow::bail!(MSG_PORTAL_FAILED_TO_CREATE_ANY_CHAIN_CONFIG_ERR) - } - let runtime_config = PortalRuntimeConfig::new(hyperchains_config); - Ok(runtime_config) + portal_config.filter(&chain_names); + Ok(()) } pub async fn run(shell: &Shell) -> anyhow::Result<()> { let ecosystem_config: EcosystemConfig = EcosystemConfig::from_file(shell)?; // Get ecosystem level apps.yaml config let apps_config = AppsEcosystemConfig::read_or_create_default(shell)?; - // What chains to run the portal for? + // Display all chains, unless --chain is passed let chains_enabled = match global_config().chain_name { Some(ref chain_name) => vec![chain_name.clone()], None => ecosystem_config.list_of_chains(), }; - // Generate portal runtime config - let runtime_config = build_portal_runtime_config(shell, &ecosystem_config, chains_enabled) - .await + // Read portal config + let config_path = PortalConfig::get_config_path(&shell.current_dir()); + let mut portal_config = PortalConfig::read_or_create_default(shell) .context(MSG_PORTAL_FAILED_TO_CREATE_CONFIG_ERR)?; - let config_path = PortalRuntimeConfig::get_config_path(&shell.current_dir()); - runtime_config.save(shell, &config_path)?; - logger::info(format!( - "Using generated portal config file at {}", - config_path.display() - )); + // Validate and update portal config + validate_portal_config(&mut portal_config, &ecosystem_config).await?; + portal_config.hide_except(&chains_enabled); + if portal_config.is_empty() { + anyhow::bail!(MSG_PORTAL_FAILED_TO_FIND_ANY_CHAIN_ERR); + } + + // Save portal config + portal_config.save(shell, &config_path)?; + let config_js_path = portal_config + .save_as_js(shell) + .context(MSG_PORTAL_FAILED_TO_CREATE_CONFIG_ERR)?; + + logger::info(msg_portal_running_with_config(&config_path)); logger::info(msg_portal_starting_on( "127.0.0.1", apps_config.portal.http_port, )); - run_portal(shell, &config_path, apps_config.portal.http_port)?; + let name = portal_app_name(&ecosystem_config.name); + run_portal(shell, &config_js_path, &name, apps_config.portal.http_port)?; Ok(()) } -fn run_portal(shell: &Shell, config_file_path: &Path, port: u16) -> anyhow::Result<()> { +fn run_portal(shell: &Shell, config_file_path: &Path, name: &str, port: u16) -> anyhow::Result<()> { let port_mapping = format!("{}:{}", port, port); let volume_mapping = format!( "{}:{}", @@ -159,13 +164,27 @@ fn run_portal(shell: &Shell, config_file_path: &Path, port: u16) -> anyhow::Resu PORTAL_DOCKER_CONFIG_PATH ); - let mut docker_args: HashMap = HashMap::new(); - docker_args.insert("--platform".to_string(), "linux/amd64".to_string()); - docker_args.insert("-p".to_string(), port_mapping); - docker_args.insert("-v".to_string(), volume_mapping); - docker_args.insert("-e".to_string(), format!("PORT={}", port)); + let docker_args: Vec = vec![ + "--platform".to_string(), + "linux/amd64".to_string(), + "--name".to_string(), + name.to_string(), + "-p".to_string(), + port_mapping, + "-v".to_string(), + volume_mapping, + "-e".to_string(), + format!("PORT={}", port), + "--rm".to_string(), + ]; docker::run(shell, PORTAL_DOCKER_IMAGE, docker_args) .with_context(|| MSG_PORTAL_FAILED_TO_RUN_DOCKER_ERR)?; Ok(()) } + +/// Generates a name for the portal app Docker container. +/// Will be passed as `--name` argument to `docker run`. +fn portal_app_name(ecosystem_name: &str) -> String { + format!("{}-portal-app", ecosystem_name) +} diff --git a/zk_toolbox/crates/zk_inception/src/defaults.rs b/zk_toolbox/crates/zk_inception/src/defaults.rs index 9f29b2c067ac..544e28377403 100644 --- a/zk_toolbox/crates/zk_inception/src/defaults.rs +++ b/zk_toolbox/crates/zk_inception/src/defaults.rs @@ -11,11 +11,6 @@ lazy_static! { Url::parse("postgres://postgres:notsecurepassword@localhost:5432").unwrap(); } -// Default ports for services -pub const EXPLORER_WORKER_PORT: u16 = 3001; -pub const EXPLORER_API_PORT: u16 = 3002; -pub const EXPLORER_DATA_FETCHER_PORT: u16 = 3040; - pub const ROCKS_DB_STATE_KEEPER: &str = "state_keeper"; pub const ROCKS_DB_TREE: &str = "tree"; pub const ROCKS_DB_PROTECTIVE_READS: &str = "protective_reads"; diff --git a/zk_toolbox/crates/zk_inception/src/messages.rs b/zk_toolbox/crates/zk_inception/src/messages.rs index 6d2e65b7073e..cca3e3b549b1 100644 --- a/zk_toolbox/crates/zk_inception/src/messages.rs +++ b/zk_toolbox/crates/zk_inception/src/messages.rs @@ -242,11 +242,14 @@ pub(super) const MSG_FAILED_TO_RUN_SERVER_ERR: &str = "Failed to start server"; pub(super) const MSG_PREPARING_EN_CONFIGS: &str = "Preparing External Node config"; /// Portal related messages -pub(super) const MSG_PORTAL_FAILED_TO_CREATE_ANY_CHAIN_CONFIG_ERR: &str = - "Failed to create any valid hyperchain config"; +pub(super) const MSG_PORTAL_FAILED_TO_FIND_ANY_CHAIN_ERR: &str = + "Failed to find any valid chain to run portal for"; pub(super) const MSG_PORTAL_FAILED_TO_CREATE_CONFIG_ERR: &str = "Failed to create portal config"; pub(super) const MSG_PORTAL_FAILED_TO_RUN_DOCKER_ERR: &str = "Failed to run portal docker container"; +pub(super) fn msg_portal_running_with_config(path: &Path) -> String { + format!("Running portal with configuration from: {}", path.display()) +} pub(super) fn msg_portal_starting_on(host: &str, port: u16) -> String { format!("Starting portal on http://{host}:{port}") } @@ -258,24 +261,26 @@ pub(super) const MSG_EXPLORER_FAILED_TO_RUN_DOCKER_SERVICES_ERR: &str = "Failed to run docker compose with explorer services"; pub(super) const MSG_EXPLORER_FAILED_TO_RUN_DOCKER_ERR: &str = "Failed to run explorer docker container"; +pub(super) const MSG_EXPLORER_FAILED_TO_CREATE_CONFIG_ERR: &str = + "Failed to create explorer config"; +pub(super) const MSG_EXPLORER_FAILED_TO_FIND_ANY_CHAIN_ERR: &str = + "Failed to find any valid chain to run explorer for. Did you run `zk_inception explorer init`?"; pub(super) const MSG_EXPLORER_INITIALIZED: &str = "Explorer has been initialized successfully"; -pub(super) const MSG_EXPLORER_FAILED_TO_FIND_ANY_VALID: &str = - "Failed to find any valid chains to run explorer for"; pub(super) fn msg_explorer_initializing_database_for(chain: &str) -> String { format!("Initializing explorer database for {chain} chain") } +pub(super) fn msg_explorer_running_with_config(path: &Path) -> String { + format!( + "Running explorer with configuration from: {}", + path.display() + ) +} pub(super) fn msg_explorer_starting_on(host: &str, port: u16) -> String { format!("Starting explorer on http://{host}:{port}") } pub(super) fn msg_explorer_chain_not_initialized(chain: &str) -> String { format!("Chain {chain} is not initialized for explorer: run `zk_inception explorer init --chain {chain}` first") } -pub(super) fn msg_explorer_skipping_not_initialized_chain(chain: &str) -> String { - format!("Chain {chain} is not initialized for explorer. Skipping..") -} -pub(super) fn msg_explorer_failed_to_configure_chain(chain: &str) -> String { - format!("Failed to configure explorer for chain {chain}") -} /// Forge utils related messages pub(super) const MSG_DEPLOYER_PK_NOT_SET_ERR: &str = "Deployer private key is not set"; From dfaa372a3985e22630d414240d394764838ebf46 Mon Sep 17 00:00:00 2001 From: Alexander Melnikov Date: Thu, 5 Sep 2024 13:48:00 -0600 Subject: [PATCH 12/12] readme --- zk_toolbox/README.md | 47 +++++++++++++++++++ .../zk_inception/src/commands/explorer/mod.rs | 6 ++- 2 files changed, 51 insertions(+), 2 deletions(-) diff --git a/zk_toolbox/README.md b/zk_toolbox/README.md index 469e36a65f64..a3b44fa98b32 100644 --- a/zk_toolbox/README.md +++ b/zk_toolbox/README.md @@ -247,6 +247,53 @@ Run the external node: zk_inception en run ``` +### Portal + +Once you have at least one chain initialized, you can run the [portal](https://github.com/matter-labs/dapp-portal) - a +web-app to bridge tokens between L1 and L2 and more: + +```bash +zk_inception portal +``` + +This command will start the dockerized portal app using configuration from `apps/portal.config.json` file inside your +ecosystem directory. You can edit this file to configure the portal app if needed. By default, portal starts on +`http://localhost:3030`, you can configure the port in `apps.yaml` file. + +### Explorer + +For better understanding of the blockchain data, you can use the +[explorer](https://github.com/matter-labs/block-explorer) - a web-app to view and inspect transactions, blocks, +contracts and more. + +First, each chain should be initialized: + +```bash +zk_inception explorer init +``` + +This command creates a database to store explorer data and generatesdocker compose file with explorer services +(`explorer-docker-compose.yml`). + +Next, for each chain you want to have an explorer, you need to start its backend services: + +```bash +zk_inception explorer backend --chain +``` + +This command uses previously created docker compose file to start the services (api, data fetcher, worker) required for +the explorer. + +Finally, you can run the explorer app: + +```bash +zk_inception explorer run +``` + +This command will start the dockerized explorer app using configuration from `apps/explorer.config.json` file inside +your ecosystem directory. You can edit this file to configure the app if needed. By default, explorer starts on +`http://localhost:3010`, you can configure the port in `apps.yaml` file. + ### Update To update your node: diff --git a/zk_toolbox/crates/zk_inception/src/commands/explorer/mod.rs b/zk_toolbox/crates/zk_inception/src/commands/explorer/mod.rs index 7333d1e0f883..4b66d49598c4 100644 --- a/zk_toolbox/crates/zk_inception/src/commands/explorer/mod.rs +++ b/zk_toolbox/crates/zk_inception/src/commands/explorer/mod.rs @@ -7,9 +7,11 @@ mod run; #[derive(Subcommand, Debug)] pub enum ExplorerCommands { - /// Initialize explorer + /// Initialize explorer (create database to store explorer data and generate docker + /// compose file with explorer services). Runs for all chains, unless --chain is passed Init, - /// Start explorer backend services (api, data_fetcher, worker) + /// Start explorer backend services (api, data_fetcher, worker) for a given chain. + /// Uses default chain, unless --chain is passed #[command(alias = "backend")] RunBackend, /// Run explorer app