From 038c397ce842601da5109c460b09dbf9d51cf2fc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Ignacio=20Gonz=C3=A1lez?= Date: Fri, 11 Oct 2024 08:08:36 -0300 Subject: [PATCH] feat(configs): Add port parameter to ConsensusConfig (#3051) Add port parameter to ConsensusConfig --- core/lib/config/src/configs/consensus.rs | 1 + core/lib/config/src/testonly.rs | 1 + core/lib/protobuf_config/src/consensus.rs | 2 + .../src/proto/core/consensus.proto | 3 ++ core/node/consensus/src/testonly.rs | 1 + etc/env/consensus_config.yaml | 1 + etc/env/en_consensus_config.yaml | 1 + etc/env/file_based/general.yaml | 7 ++++ zk_toolbox/crates/config/src/consts.rs | 2 - .../src/commands/chain/init/configs.rs | 25 ++++------- .../commands/external_node/prepare_configs.rs | 38 ++++++++--------- zk_toolbox/crates/zk_inception/src/consts.rs | 23 ---------- .../zk_inception/src/utils/consensus.rs | 42 ++----------------- .../crates/zk_inception/src/utils/ports.rs | 8 +++- 14 files changed, 52 insertions(+), 103 deletions(-) diff --git a/core/lib/config/src/configs/consensus.rs b/core/lib/config/src/configs/consensus.rs index 918d8f4adab9..7f5a0f56aa17 100644 --- a/core/lib/config/src/configs/consensus.rs +++ b/core/lib/config/src/configs/consensus.rs @@ -115,6 +115,7 @@ impl RpcConfig { /// Config (shared between main node and external node). #[derive(Clone, Debug, PartialEq)] pub struct ConsensusConfig { + pub port: Option, /// Local socket address to listen for the incoming connections. pub server_addr: std::net::SocketAddr, /// Public address of this node (should forward to `server_addr`) diff --git a/core/lib/config/src/testonly.rs b/core/lib/config/src/testonly.rs index a6ff30e04a90..960808aa6a69 100644 --- a/core/lib/config/src/testonly.rs +++ b/core/lib/config/src/testonly.rs @@ -802,6 +802,7 @@ impl Distribution for EncodeDist { fn sample(&self, rng: &mut R) -> configs::consensus::ConsensusConfig { use configs::consensus::{ConsensusConfig, Host, NodePublicKey}; ConsensusConfig { + port: self.sample(rng), server_addr: self.sample(rng), public_addr: Host(self.sample(rng)), max_payload_size: self.sample(rng), diff --git a/core/lib/protobuf_config/src/consensus.rs b/core/lib/protobuf_config/src/consensus.rs index 81cad437fe4b..2219b6a82ea8 100644 --- a/core/lib/protobuf_config/src/consensus.rs +++ b/core/lib/protobuf_config/src/consensus.rs @@ -148,6 +148,7 @@ impl ProtoRepr for proto::Config { }; Ok(Self::Type { + port: self.port.and_then(|x| x.try_into().ok()), server_addr: required(&self.server_addr) .and_then(|x| Ok(x.parse()?)) .context("server_addr")?, @@ -182,6 +183,7 @@ impl ProtoRepr for proto::Config { fn build(this: &Self::Type) -> Self { Self { + port: this.port.map(|x| x.into()), server_addr: Some(this.server_addr.to_string()), public_addr: Some(this.public_addr.0.clone()), max_payload_size: Some(this.max_payload_size.try_into().unwrap()), diff --git a/core/lib/protobuf_config/src/proto/core/consensus.proto b/core/lib/protobuf_config/src/proto/core/consensus.proto index 92527df739aa..9b0d69e7270c 100644 --- a/core/lib/protobuf_config/src/proto/core/consensus.proto +++ b/core/lib/protobuf_config/src/proto/core/consensus.proto @@ -70,6 +70,9 @@ message Config { reserved 3; reserved "validators"; + // Port to listen on, for incoming TCP connections. + optional uint32 port = 12; + // IP:port to listen on, for incoming TCP connections. // Use `0.0.0.0:` to listen on all network interfaces (i.e. on all IPs exposed by this VM). optional string server_addr = 1; // required; IpAddr diff --git a/core/node/consensus/src/testonly.rs b/core/node/consensus/src/testonly.rs index 4538337109a4..2ba961dacc31 100644 --- a/core/node/consensus/src/testonly.rs +++ b/core/node/consensus/src/testonly.rs @@ -154,6 +154,7 @@ fn make_config( genesis_spec: Option, ) -> config::ConsensusConfig { config::ConsensusConfig { + port: Some(cfg.server_addr.port()), server_addr: *cfg.server_addr, public_addr: config::Host(cfg.public_addr.0.clone()), max_payload_size: usize::MAX, diff --git a/etc/env/consensus_config.yaml b/etc/env/consensus_config.yaml index 304ea31fac9c..2564865eeb31 100644 --- a/etc/env/consensus_config.yaml +++ b/etc/env/consensus_config.yaml @@ -1,3 +1,4 @@ +port: 3054 server_addr: "127.0.0.1:3054" public_addr: "127.0.0.1:3054" max_payload_size: 2500000 diff --git a/etc/env/en_consensus_config.yaml b/etc/env/en_consensus_config.yaml index f759e72e891c..5c428866cb6c 100644 --- a/etc/env/en_consensus_config.yaml +++ b/etc/env/en_consensus_config.yaml @@ -1,3 +1,4 @@ +port: 3055 server_addr: '127.0.0.1:3055' public_addr: '127.0.0.1:3055' max_payload_size: 2500000 diff --git a/etc/env/file_based/general.yaml b/etc/env/file_based/general.yaml index a4ba8c0201a6..017d79dbe736 100644 --- a/etc/env/file_based/general.yaml +++ b/etc/env/file_based/general.yaml @@ -375,3 +375,10 @@ da_dispatcher: external_proof_integration_api: http_port: 3073 + +consensus: + port: 3054 + server_addr: "127.0.0.1:3054" + public_addr: "127.0.0.1:3054" + max_payload_size: 2500000 + gossip_dynamic_inbound_limit: 100 diff --git a/zk_toolbox/crates/config/src/consts.rs b/zk_toolbox/crates/config/src/consts.rs index 80b204cc6191..f462ce33b8f8 100644 --- a/zk_toolbox/crates/config/src/consts.rs +++ b/zk_toolbox/crates/config/src/consts.rs @@ -62,8 +62,6 @@ pub const DEFAULT_EXPLORER_WORKER_PORT: u16 = 3001; pub const DEFAULT_EXPLORER_API_PORT: u16 = 3002; /// Default port for the explorer data fetcher service pub const DEFAULT_EXPLORER_DATA_FETCHER_PORT: u16 = 3040; -/// Default port for consensus service -pub const DEFAULT_CONSENSUS_PORT: u16 = 3054; pub const EXPLORER_API_DOCKER_IMAGE: &str = "matterlabs/block-explorer-api"; pub const EXPLORER_DATA_FETCHER_DOCKER_IMAGE: &str = "matterlabs/block-explorer-data-fetcher"; diff --git a/zk_toolbox/crates/zk_inception/src/commands/chain/init/configs.rs b/zk_toolbox/crates/zk_inception/src/commands/chain/init/configs.rs index d0897473b832..37ee2e076ab9 100644 --- a/zk_toolbox/crates/zk_inception/src/commands/chain/init/configs.rs +++ b/zk_toolbox/crates/zk_inception/src/commands/chain/init/configs.rs @@ -2,7 +2,7 @@ use anyhow::Context; use common::logger; use config::{ copy_configs, set_l1_rpc_url, traits::SaveConfigWithBasePath, update_from_chain_config, - ChainConfig, ContractsConfig, EcosystemConfig, DEFAULT_CONSENSUS_PORT, + ChainConfig, ContractsConfig, EcosystemConfig, }; use ethers::types::Address; use xshell::Shell; @@ -15,13 +15,12 @@ use crate::{ }, portal::update_portal_config, }, - defaults::PORT_RANGE_END, messages::{ - MSG_CHAIN_CONFIGS_INITIALIZED, MSG_CHAIN_NOT_FOUND_ERR, + MSG_CHAIN_CONFIGS_INITIALIZED, MSG_CHAIN_NOT_FOUND_ERR, MSG_CONSENSUS_CONFIG_MISSING_ERR, MSG_PORTAL_FAILED_TO_CREATE_CONFIG_ERR, }, utils::{ - consensus::{generate_consensus_keys, get_consensus_config, get_consensus_secrets}, + consensus::{generate_consensus_keys, get_consensus_secrets, get_genesis_specs}, ports::EcosystemPortsScanner, }, }; @@ -57,22 +56,14 @@ pub async fn init_configs( )?; } - // Initialize general config let mut general_config = chain_config.get_general_config()?; - - // TODO: This is a temporary solution. We should allocate consensus port using `EcosystemPorts::allocate_ports_in_yaml` - let offset = ((chain_config.id - 1) * 100) as u16; - let consensus_port_range = DEFAULT_CONSENSUS_PORT + offset..PORT_RANGE_END; - let consensus_port = - ecosystem_ports.allocate_port(consensus_port_range, "Consensus".to_string())?; + let mut consensus_config = general_config + .consensus_config + .context(MSG_CONSENSUS_CONFIG_MISSING_ERR)?; let consensus_keys = generate_consensus_keys(); - let consensus_config = get_consensus_config( - chain_config, - consensus_port, - Some(consensus_keys.clone()), - None, - )?; + consensus_config.genesis_spec = Some(get_genesis_specs(chain_config, &consensus_keys)); + general_config.consensus_config = Some(consensus_config); general_config.save_with_base_path(shell, &chain_config.configs)?; diff --git a/zk_toolbox/crates/zk_inception/src/commands/external_node/prepare_configs.rs b/zk_toolbox/crates/zk_inception/src/commands/external_node/prepare_configs.rs index 5ab859d17f0a..d714a0f8e843 100644 --- a/zk_toolbox/crates/zk_inception/src/commands/external_node/prepare_configs.rs +++ b/zk_toolbox/crates/zk_inception/src/commands/external_node/prepare_configs.rs @@ -6,12 +6,12 @@ use config::{ external_node::ENConfig, set_rocks_db_config, traits::{FileConfigWithDefaultName, SaveConfigWithBasePath}, - ChainConfig, EcosystemConfig, GeneralConfig, SecretsConfig, DEFAULT_CONSENSUS_PORT, + ChainConfig, EcosystemConfig, GeneralConfig, SecretsConfig, }; use xshell::Shell; use zksync_basic_types::url::SensitiveUrl; use zksync_config::configs::{ - consensus::{ConsensusSecrets, NodeSecretKey, Secret}, + consensus::{ConsensusConfig, ConsensusSecrets, NodeSecretKey, Secret}, DatabaseSecrets, L1Secrets, }; use zksync_consensus_crypto::TextFmt; @@ -19,14 +19,13 @@ use zksync_consensus_roles as roles; use crate::{ commands::external_node::args::prepare_configs::{PrepareConfigArgs, PrepareConfigFinal}, - defaults::PORT_RANGE_END, messages::{ msg_preparing_en_config_is_done, MSG_CHAIN_NOT_INITIALIZED, MSG_CONSENSUS_CONFIG_MISSING_ERR, MSG_CONSENSUS_SECRETS_MISSING_ERR, MSG_CONSENSUS_SECRETS_NODE_KEY_MISSING_ERR, MSG_PREPARING_EN_CONFIGS, }, utils::{ - consensus::{get_consensus_config, node_public_key}, + consensus::node_public_key, ports::EcosystemPortsScanner, rocks_db::{recreate_rocksdb_dirs, RocksDBDirOption}, }, @@ -79,19 +78,12 @@ fn prepare_configs( bridge_addresses_refresh_interval_sec: None, }; let mut general_en = general.clone(); + general_en.consensus_config = None; let main_node_consensus_config = general .consensus_config .context(MSG_CONSENSUS_CONFIG_MISSING_ERR)?; - - // TODO: This is a temporary solution. We should allocate consensus port using `EcosystemPorts::allocate_ports_in_yaml` - ports.add_port_info( - main_node_consensus_config.server_addr.port(), - "Main node consensus".to_string(), - ); - let offset = ((config.id - 1) * 100) as u16; - let consensus_port_range = DEFAULT_CONSENSUS_PORT + offset..PORT_RANGE_END; - let consensus_port = ports.allocate_port(consensus_port_range, "Consensus".to_string())?; + let mut en_consensus_config = main_node_consensus_config.clone(); let mut gossip_static_outbound = BTreeMap::new(); let main_node_public_key = node_public_key( @@ -101,13 +93,8 @@ fn prepare_configs( .context(MSG_CONSENSUS_SECRETS_MISSING_ERR)?, )? .context(MSG_CONSENSUS_SECRETS_NODE_KEY_MISSING_ERR)?; - gossip_static_outbound.insert(main_node_public_key, main_node_consensus_config.public_addr); - - let en_consensus_config = - get_consensus_config(config, consensus_port, None, Some(gossip_static_outbound))?; - general_en.consensus_config = Some(en_consensus_config.clone()); - en_consensus_config.save_with_base_path(shell, en_configs_path)?; + en_consensus_config.gossip_static_outbound = gossip_static_outbound; // Set secrets config let node_key = roles::node::SecretKey::generate().encode(); @@ -128,16 +115,25 @@ fn prepare_configs( }), data_availability: None, }; - secrets.save_with_base_path(shell, en_configs_path)?; + let dirs = recreate_rocksdb_dirs(shell, &config.rocks_db_path, RocksDBDirOption::ExternalNode)?; set_rocks_db_config(&mut general_en, dirs)?; + general_en.save_with_base_path(shell, en_configs_path)?; en_config.save_with_base_path(shell, en_configs_path)?; + en_consensus_config.save_with_base_path(shell, en_configs_path)?; + secrets.save_with_base_path(shell, en_configs_path)?; + let offset = 0; // This is zero because general_en ports already have a chain offset ports.allocate_ports_in_yaml( shell, &GeneralConfig::get_path_with_base_path(en_configs_path), - 0, // This is zero because general_en ports already have a chain offset + offset, + )?; + ports.allocate_ports_in_yaml( + shell, + &ConsensusConfig::get_path_with_base_path(en_configs_path), + offset, )?; Ok(()) diff --git a/zk_toolbox/crates/zk_inception/src/consts.rs b/zk_toolbox/crates/zk_inception/src/consts.rs index 9f81847e3336..df27d2f02d2c 100644 --- a/zk_toolbox/crates/zk_inception/src/consts.rs +++ b/zk_toolbox/crates/zk_inception/src/consts.rs @@ -1,5 +1,3 @@ -use std::net::{IpAddr, Ipv4Addr}; - pub const AMOUNT_FOR_DISTRIBUTION_TO_WALLETS: u128 = 1000000000000000000000; pub const MINIMUM_BALANCE_FOR_WALLET: u128 = 5000000000000000000; @@ -12,27 +10,6 @@ pub const DEFAULT_UNSIGNED_TRANSACTIONS_DIR: &str = "transactions"; pub const BELLMAN_CUDA_DIR: &str = "era-bellman-cuda"; pub const L2_BASE_TOKEN_ADDRESS: &str = "0x000000000000000000000000000000000000800A"; -#[allow(non_upper_case_globals)] -const kB: usize = 1024; - -/// Max payload size for consensus in bytes -pub const MAX_PAYLOAD_SIZE: usize = 2_500_000; -/// Max batch size for consensus in bytes -/// Compute a default batch size, so operators are not caught out by the missing setting -/// while we're still working on batch syncing. The batch interval is ~1 minute, -/// so there will be ~60 blocks, and an Ethereum Merkle proof is ~1kB, but under high -/// traffic there can be thousands of huge transactions that quickly fill up blocks -/// and there could be more blocks in a batch then expected. We chose a generous -/// limit so as not to prevent any legitimate batch from being transmitted. -pub const MAX_BATCH_SIZE: usize = MAX_PAYLOAD_SIZE * 5000 + kB; -/// Gossip dynamic inbound limit for consensus -pub const GOSSIP_DYNAMIC_INBOUND_LIMIT: usize = 100; - -/// Public address for consensus -pub const CONSENSUS_PUBLIC_ADDRESS_HOST: IpAddr = IpAddr::V4(Ipv4Addr::UNSPECIFIED); -/// Server address for consensus -pub const CONSENSUS_SERVER_ADDRESS_HOST: IpAddr = IpAddr::V4(Ipv4Addr::LOCALHOST); - /// Path to the JS runtime config for the block-explorer-app docker container to be mounted to pub const EXPLORER_APP_DOCKER_CONFIG_PATH: &str = "/usr/src/app/packages/app/dist/config.js"; pub const EXPLORER_APP_DOCKER_IMAGE: &str = "matterlabs/block-explorer-app"; diff --git a/zk_toolbox/crates/zk_inception/src/utils/consensus.rs b/zk_toolbox/crates/zk_inception/src/utils/consensus.rs index 2979b4df0c19..946d28a33fbd 100644 --- a/zk_toolbox/crates/zk_inception/src/utils/consensus.rs +++ b/zk_toolbox/crates/zk_inception/src/utils/consensus.rs @@ -1,24 +1,14 @@ -use std::{ - collections::{BTreeMap, BTreeSet}, - net::SocketAddr, -}; - use anyhow::Context as _; use config::ChainConfig; use secrecy::{ExposeSecret, Secret}; use zksync_config::configs::consensus::{ - AttesterPublicKey, AttesterSecretKey, ConsensusConfig, ConsensusSecrets, GenesisSpec, Host, - NodePublicKey, NodeSecretKey, ProtocolVersion, ValidatorPublicKey, ValidatorSecretKey, - WeightedAttester, WeightedValidator, + AttesterPublicKey, AttesterSecretKey, ConsensusSecrets, GenesisSpec, NodePublicKey, + NodeSecretKey, ProtocolVersion, ValidatorPublicKey, ValidatorSecretKey, WeightedAttester, + WeightedValidator, }; use zksync_consensus_crypto::{Text, TextFmt}; use zksync_consensus_roles::{attester, node, validator}; -use crate::consts::{ - CONSENSUS_PUBLIC_ADDRESS_HOST, CONSENSUS_SERVER_ADDRESS_HOST, GOSSIP_DYNAMIC_INBOUND_LIMIT, - MAX_BATCH_SIZE, MAX_PAYLOAD_SIZE, -}; - pub(crate) fn parse_attester_committee( attesters: &[WeightedAttester], ) -> anyhow::Result { @@ -48,32 +38,6 @@ pub struct ConsensusPublicKeys { attester_key: attester::PublicKey, } -pub fn get_consensus_config( - chain_config: &ChainConfig, - consensus_port: u16, - consensus_keys: Option, - gossip_static_outbound: Option>, -) -> anyhow::Result { - let genesis_spec = - consensus_keys.map(|consensus_keys| get_genesis_specs(chain_config, &consensus_keys)); - - let public_addr = SocketAddr::new(CONSENSUS_PUBLIC_ADDRESS_HOST, consensus_port); - let server_addr = SocketAddr::new(CONSENSUS_SERVER_ADDRESS_HOST, consensus_port); - - Ok(ConsensusConfig { - server_addr, - public_addr: Host(public_addr.encode()), - genesis_spec, - max_payload_size: MAX_PAYLOAD_SIZE, - gossip_dynamic_inbound_limit: GOSSIP_DYNAMIC_INBOUND_LIMIT, - max_batch_size: MAX_BATCH_SIZE, - gossip_static_inbound: BTreeSet::new(), - gossip_static_outbound: gossip_static_outbound.unwrap_or_default(), - rpc: None, - debug_page_addr: None, - }) -} - pub fn generate_consensus_keys() -> ConsensusSecretKeys { ConsensusSecretKeys { validator_key: validator::SecretKey::generate(), diff --git a/zk_toolbox/crates/zk_inception/src/utils/ports.rs b/zk_toolbox/crates/zk_inception/src/utils/ports.rs index 018fb79f345d..04c8cef5ff59 100644 --- a/zk_toolbox/crates/zk_inception/src/utils/ports.rs +++ b/zk_toolbox/crates/zk_inception/src/utils/ports.rs @@ -1,4 +1,4 @@ -use std::{collections::HashMap, fmt, ops::Range, path::Path}; +use std::{collections::HashMap, fmt, net::SocketAddr, ops::Range, path::Path}; use anyhow::{bail, Context, Result}; use config::{ @@ -109,6 +109,12 @@ impl EcosystemPorts { } } } + } else if key.as_str().map(|s| s.ends_with("addr")).unwrap_or(false) { + let socket_addr = val.as_str().unwrap().parse::()?; + if let Some(new_port) = updated_ports.get(&socket_addr.port()) { + let new_socket_addr = SocketAddr::new(socket_addr.ip(), *new_port); + *val = Value::String(new_socket_addr.to_string()); + } } } // Continue traversing