feat(zk_toolbox): Add block explorer support to zk_toolbox

zk_toolbox/crates/common/src/docker.rs

@@ -1,11 +1,18 @@
 use std::collections::HashMap;
 
+use url::Url;
 use xshell::{cmd, Shell};
 
 use crate::cmd::Cmd;
 
-pub fn up(shell: &Shell, docker_compose_file: &str) -> anyhow::Result<()> {
-    Ok(Cmd::new(cmd!(shell, "docker compose -f {docker_compose_file} up -d")).run()?)
+pub fn up(shell: &Shell, docker_compose_file: &str, detach: bool) -> anyhow::Result<()> {
+    let args = if detach { vec!["-d"] } else { vec![] };
+    let mut cmd = Cmd::new(cmd!(
+        shell,
+        "docker compose -f {docker_compose_file} up {args...}"
+    ));
+    cmd = if !detach { cmd.with_force_run() } else { cmd };
+    Ok(cmd.run()?)
 }
 
 pub fn down(shell: &Shell, docker_compose_file: &str) -> anyhow::Result<()> {
@@ -24,3 +31,14 @@ pub fn run(
     }
     Ok(Cmd::new(cmd!(shell, "docker run {args...} {docker_image}")).run()?)
 }
+
+pub fn adjust_localhost_for_docker(mut url: Url) -> anyhow::Result<Url> {
+    if let Some(host) = url.host_str() {
+        if host == "localhost" || host == "127.0.0.1" {
+            url.set_host(Some("host.docker.internal"))?;
+        }
+    } else {
+        anyhow::bail!("Failed to parse: no host");
+    }
+    Ok(url)
+}

zk_toolbox/crates/config/src/apps.rs

@@ -0,0 +1,114 @@
+use std::path::{Path, PathBuf};
+
+use serde::{Deserialize, Serialize};
+use url::Url;
+use xshell::Shell;
+
+use crate::{
+    consts::{
+        APPS_CONFIG_FILE, DEFAULT_EXPLORER_PORT, DEFAULT_PORTAL_PORT, LOCAL_CHAINS_PATH,
+        LOCAL_CONFIGS_PATH,
+    },
+    traits::{FileConfigWithDefaultName, ReadConfig, SaveConfig, ZkToolboxConfig},
+    EXPLORER_BATCHES_PROCESSING_POLLING_INTERVAL,
+};
+
+/// Ecosystem level configuration for the apps (portal and explorer).
+#[derive(Debug, Serialize, Deserialize, Clone)]
+pub struct AppsEcosystemConfig {
+    pub portal: AppEcosystemConfig,
+    pub explorer: AppEcosystemConfig,
+}
+
+#[derive(Debug, Serialize, Deserialize, Clone)]
+pub struct AppEcosystemConfig {
+    pub http_port: u16,
+    pub http_url: String,
+}
+
+impl ZkToolboxConfig for AppsEcosystemConfig {}
+impl FileConfigWithDefaultName for AppsEcosystemConfig {
+    const FILE_NAME: &'static str = APPS_CONFIG_FILE;
+}
+
+/// Chain level configuration for the apps.
+#[derive(Debug, Serialize, Deserialize, Clone)]
+pub struct AppsChainConfig {
+    pub explorer: AppsChainExplorerConfig,
+}
+
+#[derive(Debug, Serialize, Deserialize, Clone)]
+pub struct AppsChainExplorerConfig {
+    pub database_url: Url,
+    pub services: ExplorerServicesConfig,
+}
+
+#[derive(Debug, Serialize, Deserialize, Clone)]
+pub struct ExplorerServicesConfig {
+    pub api_http_port: u16,
+    pub data_fetcher_http_port: u16,
+    pub worker_http_port: u16,
+    pub batches_processing_polling_interval: u64,
+}
+
+impl ZkToolboxConfig for AppsChainConfig {}
+
+impl AppsEcosystemConfig {
+    pub fn get_config_path(ecosystem_base_path: &Path) -> PathBuf {
+        ecosystem_base_path
+            .join(LOCAL_CONFIGS_PATH)
+            .join(APPS_CONFIG_FILE)
+    }
+
+    pub fn read_or_create_default(shell: &Shell) -> anyhow::Result<Self> {
+        let config_path = Self::get_config_path(&shell.current_dir());
+        match Self::read(shell, &config_path) {
+            Ok(config) => Ok(config),
+            Err(_) => {
+                let config = Self::default();
+                config.save(shell, &config_path)?;
+                Ok(config)
+            }
+        }
+    }
+}
+
+impl Default for AppsEcosystemConfig {
+    fn default() -> Self {
+        AppsEcosystemConfig {
+            portal: AppEcosystemConfig {
+                http_port: DEFAULT_PORTAL_PORT,
+                http_url: format!("http://127.0.0.1:{}", DEFAULT_PORTAL_PORT),
+            },
+            explorer: AppEcosystemConfig {
+                http_port: DEFAULT_EXPLORER_PORT,
+                http_url: format!("http://127.0.0.1:{}", DEFAULT_EXPLORER_PORT),
+            },
+        }
+    }
+}
+
+impl AppsChainConfig {
+    pub fn new(explorer: AppsChainExplorerConfig) -> Self {
+        AppsChainConfig { explorer }
+    }
+
+    pub fn get_config_path(ecosystem_base_path: &Path, chain_name: &str) -> PathBuf {
+        ecosystem_base_path
+            .join(LOCAL_CHAINS_PATH)
+            .join(chain_name)
+            .join(LOCAL_CONFIGS_PATH)
+            .join(APPS_CONFIG_FILE)
+    }
+}
+
+impl ExplorerServicesConfig {
+    pub fn new(api_http_port: u16, data_fetcher_http_port: u16, worker_http_port: u16) -> Self {
+        ExplorerServicesConfig {
+            api_http_port,
+            data_fetcher_http_port,
+            worker_http_port,
+            batches_processing_polling_interval: EXPLORER_BATCHES_PROCESSING_POLLING_INTERVAL,
+        }
+    }
+}

zk_toolbox/crates/config/src/consts.rs

@@ -30,12 +30,37 @@ pub const ERA_OBSERVABILITY_COMPOSE_FILE: &str = "era-observability/docker-compo
 pub const ERA_OBSERBAVILITY_DIR: &str = "era-observability";
 /// Era observability repo link
 pub const ERA_OBSERBAVILITY_GIT_REPO: &str = "https://github.com/matter-labs/era-observability";
+pub(crate) const LOCAL_APPS_PATH: &str = "apps/";
+pub(crate) const LOCAL_CHAINS_PATH: &str = "chains/";
 pub(crate) const LOCAL_CONFIGS_PATH: &str = "configs/";
+pub(crate) const LOCAL_GENERATED_PATH: &str = ".generated/";
 pub(crate) const LOCAL_DB_PATH: &str = "db/";
 pub(crate) const LOCAL_ARTIFACTS_PATH: &str = "artifacts/";
 
-/// Name of portal config file
-pub const PORTAL_CONFIG_FILE: &str = "portal.config.js";
+/// Name of apps config file
+pub const APPS_CONFIG_FILE: &str = "apps.yaml";
+/// Name of portal runtime config file (auto-generated)
+pub const PORTAL_RUNTIME_CONFIG_FILE: &str = "portal.config.js";
+/// Name of chain-level portal config JSON file
+pub const PORTAL_CHAIN_CONFIG_FILE: &str = "portal.config.json";
+/// Name of explorer runtime config file (auto-generated)
+pub const EXPLORER_RUNTIME_CONFIG_FILE: &str = "explorer.config.js";
+/// Name of chain-level explorer config JSON file
+pub const EXPLORER_CHAIN_CONFIG_FILE: &str = "explorer.config.json";
+/// Name of explorer docker compose file (auto-generated)
+pub const EXPLORER_DOCKER_COMPOSE_FILE: &str = "explorer-docker-compose.yml";
+
+/// Default port for the explorer app
+pub const DEFAULT_EXPLORER_PORT: u16 = 3010;
+/// Default port for the portal app
+pub const DEFAULT_PORTAL_PORT: u16 = 3030;
+
+pub const EXPLORER_API_DOCKER_IMAGE: &str = "matterlabs/block-explorer-api";
+pub const EXPLORER_DATA_FETCHER_DOCKER_IMAGE: &str = "matterlabs/block-explorer-data-fetcher";
+pub const EXPLORER_WORKER_DOCKER_IMAGE: &str = "matterlabs/block-explorer-worker";
+
+/// Interval (in milliseconds) for polling new batches to process in explorer app
+pub const EXPLORER_BATCHES_PROCESSING_POLLING_INTERVAL: u64 = 1000;
 
 /// Path to ecosystem contacts
 pub(crate) const ECOSYSTEM_PATH: &str = "etc/env/ecosystems";

zk_toolbox/crates/config/src/docker_compose.rs

@@ -0,0 +1,43 @@
+use std::collections::HashMap;
+
+use serde::{Deserialize, Serialize};
+
+use crate::traits::ZkToolboxConfig;
+
+#[derive(Debug, Default, Serialize, Deserialize, Clone)]
+pub struct DockerComposeConfig {
+    pub services: HashMap<String, DockerComposeService>,
+}
+
+#[derive(Debug, Serialize, Deserialize, Clone)]
+pub struct DockerComposeService {
+    pub image: String,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub platform: Option<String>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub ports: Option<Vec<String>>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub environment: Option<HashMap<String, String>>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub volumes: Option<Vec<String>>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub depends_on: Option<Vec<String>>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub restart: Option<String>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub extra_hosts: Option<Vec<String>>,
+    #[serde(flatten)]
+    pub other: serde_json::Value,
+}
+
+impl ZkToolboxConfig for DockerComposeConfig {}
+
+impl DockerComposeConfig {
+    pub fn new() -> Self {
+        Self::default()
+    }
+
+    pub fn add_service(&mut self, name: &str, service: DockerComposeService) {
+        self.services.insert(name.to_string(), service);
+    }
+}

zk_toolbox/crates/config/src/explorer.rs

@@ -0,0 +1,111 @@
+use std::path::{Path, PathBuf};
+
+use serde::{Deserialize, Serialize};
+use xshell::Shell;
+
+use crate::{
+    consts::{
+        EXPLORER_CHAIN_CONFIG_FILE, EXPLORER_RUNTIME_CONFIG_FILE, LOCAL_APPS_PATH,
+        LOCAL_CHAINS_PATH, LOCAL_CONFIGS_PATH, LOCAL_GENERATED_PATH,
+    },
+    traits::{ReadConfig, SaveConfig, ZkToolboxConfig},
+};
+
+/// Explorer configuration file. This file is auto-generated during the "explorer" command
+/// and is used to inject the runtime configuration into the explorer app.
+#[derive(Serialize, Deserialize, Debug, Clone)]
+#[serde(rename_all = "camelCase")]
+pub struct ExplorerRuntimeConfig {
+    pub app_environment: String,
+    pub environment_config: EnvironmentConfig,
+}
+
+#[derive(Serialize, Deserialize, Debug, Clone)]
+pub struct EnvironmentConfig {
+    pub networks: Vec<ExplorerChainConfig>,
+}
+
+/// Explorer chain configuration file. This file is created on the chain level
+/// and is used to configure the explorer for a specific chain. It serves as a building block for
+/// the explorer runtime config.
+#[derive(Serialize, Deserialize, Debug, Clone)]
+#[serde(rename_all = "camelCase")]
+pub struct ExplorerChainConfig {
+    pub name: String, // L2 network chain name (the one used during the chain initialization)
+    pub l2_network_name: String, // How the network is displayed in the app dropdown
+    pub l2_chain_id: u64,
+    pub rpc_url: String,            // L2 RPC URL
+    pub api_url: String,            // L2 API URL
+    pub base_token_address: String, // L2 base token address (currently always 0x800A)
+    pub hostnames: Vec<String>,     // Custom domain to use when switched to this chain in the app
+    pub icon: String,               // Icon to show in the explorer dropdown
+    pub maintenance: bool,          // Maintenance warning
+    pub published: bool, // If false, the chain will not be shown in the explorer dropdown
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub bridge_url: Option<String>, // Link to the portal bridge
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub l1_explorer_url: Option<String>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub verification_api_url: Option<String>, // L2 verification API URL
+}
+
+impl ExplorerRuntimeConfig {
+    pub fn new(explorer_chain_configs: Vec<ExplorerChainConfig>) -> Self {
+        Self {
+            app_environment: "default".to_string(),
+            environment_config: EnvironmentConfig {
+                networks: explorer_chain_configs,
+            },
+        }
+    }
+
+    pub fn get_config_path(ecosystem_base_path: &Path) -> PathBuf {
+        ecosystem_base_path
+            .join(LOCAL_CONFIGS_PATH)
+            .join(LOCAL_GENERATED_PATH)
+            .join(EXPLORER_RUNTIME_CONFIG_FILE)
+    }
+}
+
+impl SaveConfig for ExplorerRuntimeConfig {
+    fn save(&self, shell: &Shell, path: impl AsRef<Path>) -> anyhow::Result<()> {
+        // The block-explorer-app is served as a pre-built static app in a Docker image.
+        // It uses a JavaScript file (config.js) that injects the configuration at runtime
+        // by overwriting the '##runtimeConfig' property of the window object.
+        // Therefore, we generate a JavaScript file instead of a JSON file.
+        // This file will be mounted to the Docker image when it runs.
+        let json = serde_json::to_string_pretty(&self)?;
+        let config_js_content = format!("window['##runtimeConfig'] = {};", json);
+        Ok(shell.write_file(path, config_js_content.as_bytes())?)
+    }
+}
+
+impl ReadConfig for ExplorerRuntimeConfig {
+    fn read(shell: &Shell, path: impl AsRef<Path>) -> anyhow::Result<Self> {
+        let config_js_content = shell.read_file(path)?;
+        // Extract the JSON part from the JavaScript file
+        let json_start = config_js_content
+            .find('{')
+            .ok_or_else(|| anyhow::anyhow!("Invalid config file format"))?;
+        let json_end = config_js_content
+            .rfind('}')
+            .ok_or_else(|| anyhow::anyhow!("Invalid config file format"))?;
+        let json_str = &config_js_content[json_start..=json_end];
+        // Parse the JSON into ExplorerRuntimeConfig
+        let config: ExplorerRuntimeConfig = serde_json::from_str(json_str)?;
+        Ok(config)
+    }
+}
+
+impl ExplorerChainConfig {
+    pub fn get_config_path(ecosystem_base_path: &Path, chain_name: &str) -> PathBuf {
+        ecosystem_base_path
+            .join(LOCAL_CHAINS_PATH)
+            .join(chain_name)
+            .join(LOCAL_CONFIGS_PATH)
+            .join(LOCAL_APPS_PATH)
+            .join(EXPLORER_CHAIN_CONFIG_FILE)
+    }
+}
+
+impl ZkToolboxConfig for ExplorerChainConfig {}