diff --git a/Cargo.lock b/Cargo.lock index b3cd5f9a991..b8a800a3add 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1518,7 +1518,7 @@ dependencies = [ name = "clickhouse-admin-api" version = "0.1.0" dependencies = [ - "clickhouse-admin-types", + "clickhouse-admin-types-versions", "dropshot", "dropshot-api-manager-types", "omicron-common", @@ -1588,6 +1588,14 @@ dependencies = [ [[package]] name = "clickhouse-admin-types" version = "0.1.0" +dependencies = [ + "clickhouse-admin-types-versions", + "omicron-workspace-hack", +] + +[[package]] +name = "clickhouse-admin-types-versions" +version = "0.1.0" dependencies = [ "anyhow", "atomicwrites", diff --git a/Cargo.toml b/Cargo.toml index edb024dddd4..cfbe049e830 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -6,6 +6,7 @@ members = [ "clickhouse-admin", "clickhouse-admin/api", "clickhouse-admin/test-utils", + "clickhouse-admin/types/versions", "clients/bootstrap-agent-client", "clients/clickhouse-admin-keeper-client", "clients/clickhouse-admin-server-client", @@ -168,6 +169,7 @@ default-members = [ "clickhouse-admin", "clickhouse-admin/api", "clickhouse-admin/types", + "clickhouse-admin/types/versions", "clickhouse-admin/test-utils", "clients/bootstrap-agent-client", "clients/clickhouse-admin-keeper-client", @@ -409,6 +411,7 @@ clickhouse-admin-keeper-client = { path = "clients/clickhouse-admin-keeper-clien clickhouse-admin-server-client = { path = "clients/clickhouse-admin-server-client" } clickhouse-admin-single-client = { path = "clients/clickhouse-admin-single-client" } clickhouse-admin-types = { path = "clickhouse-admin/types" } +clickhouse-admin-types-versions = { path = "clickhouse-admin/types/versions" } clickhouse-admin-test-utils = { path = "clickhouse-admin/test-utils" } clickward = { git = "https://github.com/oxidecomputer/clickward", rev = "e3d9a1c35cf3cd04f9cb2e997b0ad88324d30737" } cockroach-admin-api = { path = "cockroach-admin/api" } diff --git a/clickhouse-admin/api/Cargo.toml b/clickhouse-admin/api/Cargo.toml index 51270d03e3b..3aa3f6fe787 100644 --- a/clickhouse-admin/api/Cargo.toml +++ b/clickhouse-admin/api/Cargo.toml @@ -8,7 +8,7 @@ license = "MPL-2.0" workspace = true [dependencies] -clickhouse-admin-types.workspace = true +clickhouse-admin-types-versions.workspace = true dropshot.workspace = true dropshot-api-manager-types.workspace = true omicron-common.workspace = true diff --git a/clickhouse-admin/api/src/lib.rs b/clickhouse-admin/api/src/lib.rs index 618b4eb9e3d..03ca1835cf7 100644 --- a/clickhouse-admin/api/src/lib.rs +++ b/clickhouse-admin/api/src/lib.rs @@ -2,12 +2,7 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at https://mozilla.org/MPL/2.0/. -use clickhouse_admin_types::{ - ClickhouseKeeperClusterMembership, DistributedDdlQueue, - GenerateConfigResult, KeeperConf, KeeperConfigurableSettings, Lgif, - MetricInfoPath, RaftConfig, ServerConfigurableSettings, SystemTimeSeries, - TimeSeriesSettingsQuery, -}; +use clickhouse_admin_types_versions::latest; use dropshot::{ HttpError, HttpResponseCreated, HttpResponseOk, HttpResponseUpdatedNoContent, Path, Query, RequestContext, TypedBody, @@ -73,8 +68,11 @@ pub trait ClickhouseAdminKeeperApi { }] async fn generate_config_and_enable_svc( rqctx: RequestContext, - body: TypedBody, - ) -> Result, HttpError>; + body: TypedBody, + ) -> Result< + HttpResponseCreated, + HttpError, + >; /// Retrieve the generation number of a configuration #[endpoint { @@ -94,7 +92,7 @@ pub trait ClickhouseAdminKeeperApi { }] async fn lgif( rqctx: RequestContext, - ) -> Result, HttpError>; + ) -> Result, HttpError>; /// Retrieve information from ClickHouse virtual node /keeper/config which /// contains last committed cluster configuration. @@ -104,7 +102,7 @@ pub trait ClickhouseAdminKeeperApi { }] async fn raft_config( rqctx: RequestContext, - ) -> Result, HttpError>; + ) -> Result, HttpError>; /// Retrieve configuration information from a keeper node. #[endpoint { @@ -113,7 +111,7 @@ pub trait ClickhouseAdminKeeperApi { }] async fn keeper_conf( rqctx: RequestContext, - ) -> Result, HttpError>; + ) -> Result, HttpError>; /// Retrieve cluster membership information from a keeper node. #[endpoint { @@ -122,7 +120,10 @@ pub trait ClickhouseAdminKeeperApi { }] async fn keeper_cluster_membership( rqctx: RequestContext, - ) -> Result, HttpError>; + ) -> Result< + HttpResponseOk, + HttpError, + >; } /// API interface for our clickhouse-admin-server server @@ -148,8 +149,11 @@ pub trait ClickhouseAdminServerApi { }] async fn generate_config_and_enable_svc( rqctx: RequestContext, - body: TypedBody, - ) -> Result, HttpError>; + body: TypedBody, + ) -> Result< + HttpResponseCreated, + HttpError, + >; /// Retrieve the generation number of a configuration #[endpoint { @@ -168,7 +172,10 @@ pub trait ClickhouseAdminServerApi { }] async fn distributed_ddl_queue( rqctx: RequestContext, - ) -> Result>, HttpError>; + ) -> Result< + HttpResponseOk>, + HttpError, + >; /// Retrieve time series from the system database. /// @@ -181,9 +188,9 @@ pub trait ClickhouseAdminServerApi { }] async fn system_timeseries_avg( rqctx: RequestContext, - path_params: Path, - query_params: Query, - ) -> Result>, HttpError>; + path_params: Path, + query_params: Query, + ) -> Result>, HttpError>; /// Idempotently initialize a replicated ClickHouse cluster database. #[endpoint { @@ -225,7 +232,7 @@ pub trait ClickhouseAdminSingleApi { }] async fn system_timeseries_avg( rqctx: RequestContext, - path_params: Path, - query_params: Query, - ) -> Result>, HttpError>; + path_params: Path, + query_params: Query, + ) -> Result>, HttpError>; } diff --git a/clickhouse-admin/src/clickhouse_cli.rs b/clickhouse-admin/src/clickhouse_cli.rs index 3e5d1f79170..d2aaa007a53 100644 --- a/clickhouse-admin/src/clickhouse_cli.rs +++ b/clickhouse-admin/src/clickhouse_cli.rs @@ -4,10 +4,12 @@ use anyhow::Result; use camino::Utf8PathBuf; -use clickhouse_admin_types::{ - ClickhouseKeeperClusterMembership, DistributedDdlQueue, KeeperConf, - KeeperId, Lgif, OXIMETER_CLUSTER, RaftConfig, SystemTimeSeries, - SystemTimeSeriesSettings, +use clickhouse_admin_types::OXIMETER_CLUSTER; +use clickhouse_admin_types::keeper::{ + ClickhouseKeeperClusterMembership, KeeperConf, KeeperId, Lgif, RaftConfig, +}; +use clickhouse_admin_types::server::{ + DistributedDdlQueue, SystemTimeSeries, SystemTimeSeriesSettings, }; use dropshot::HttpError; use illumos_utils::{ExecutionError, output_to_exec_error}; diff --git a/clickhouse-admin/src/clickward.rs b/clickhouse-admin/src/clickward.rs index b9355dc2c30..d9a5a7771f2 100644 --- a/clickhouse-admin/src/clickward.rs +++ b/clickhouse-admin/src/clickward.rs @@ -2,10 +2,9 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at https://mozilla.org/MPL/2.0/. -use clickhouse_admin_types::{ - KeeperConfig, KeeperConfigurableSettings, ReplicaConfig, - ServerConfigurableSettings, -}; +use clickhouse_admin_types::config::{KeeperConfig, ReplicaConfig}; +use clickhouse_admin_types::keeper::KeeperConfigurableSettings; +use clickhouse_admin_types::server::ServerConfigurableSettings; use dropshot::HttpError; use slog_error_chain::{InlineErrorChain, SlogInlineError}; diff --git a/clickhouse-admin/src/context.rs b/clickhouse-admin/src/context.rs index 2fbe30d1c19..9970c3af24b 100644 --- a/clickhouse-admin/src/context.rs +++ b/clickhouse-admin/src/context.rs @@ -6,11 +6,12 @@ use crate::{ClickhouseCli, Clickward}; use anyhow::{Context, Result, anyhow, bail}; use camino::Utf8PathBuf; +use clickhouse_admin_types::config::GenerateConfigResult; +use clickhouse_admin_types::keeper::KeeperConfigurableSettings; +use clickhouse_admin_types::server::ServerConfigurableSettings; use clickhouse_admin_types::{ CLICKHOUSE_KEEPER_CONFIG_DIR, CLICKHOUSE_KEEPER_CONFIG_FILE, CLICKHOUSE_SERVER_CONFIG_DIR, CLICKHOUSE_SERVER_CONFIG_FILE, - GenerateConfigResult, KeeperConfigurableSettings, - ServerConfigurableSettings, }; use dropshot::{ClientErrorStatusCode, HttpError}; use flume::{Receiver, Sender, TrySendError}; diff --git a/clickhouse-admin/src/http_entrypoints.rs b/clickhouse-admin/src/http_entrypoints.rs index ff8118d876d..b39b5fe4775 100644 --- a/clickhouse-admin/src/http_entrypoints.rs +++ b/clickhouse-admin/src/http_entrypoints.rs @@ -4,11 +4,14 @@ use crate::context::{KeeperServerContext, ServerContext}; use clickhouse_admin_api::*; -use clickhouse_admin_types::{ - ClickhouseKeeperClusterMembership, DistributedDdlQueue, - GenerateConfigResult, KeeperConf, KeeperConfigurableSettings, Lgif, - MetricInfoPath, RaftConfig, ServerConfigurableSettings, SystemTimeSeries, - SystemTimeSeriesSettings, TimeSeriesSettingsQuery, +use clickhouse_admin_types::config::GenerateConfigResult; +use clickhouse_admin_types::keeper::{ + ClickhouseKeeperClusterMembership, KeeperConf, KeeperConfigurableSettings, + Lgif, RaftConfig, +}; +use clickhouse_admin_types::server::{ + DistributedDdlQueue, MetricInfoPath, ServerConfigurableSettings, + SystemTimeSeries, SystemTimeSeriesSettings, TimeSeriesSettingsQuery, }; use dropshot::{ ApiDescription, ClientErrorStatusCode, HttpError, HttpResponseCreated, diff --git a/clickhouse-admin/tests/integration_test.rs b/clickhouse-admin/tests/integration_test.rs index d9d1f45f451..a9cc0dffef1 100644 --- a/clickhouse-admin/tests/integration_test.rs +++ b/clickhouse-admin/tests/integration_test.rs @@ -8,9 +8,10 @@ use clickhouse_admin_test_utils::{ default_clickhouse_cluster_test_deployment, default_clickhouse_log_ctx_and_path, }; -use clickhouse_admin_types::{ - ClickhouseHost, ClickhouseKeeperClusterMembership, KeeperId, - KeeperServerInfo, KeeperServerType, RaftConfig, +use clickhouse_admin_types::config::ClickhouseHost; +use clickhouse_admin_types::keeper::{ + ClickhouseKeeperClusterMembership, KeeperId, KeeperServerInfo, + KeeperServerType, RaftConfig, }; use omicron_clickhouse_admin::ClickhouseCli; use slog::{Drain, info, o}; @@ -83,7 +84,7 @@ async fn test_raft_config_parsing() -> anyhow::Result<()> { for i in 1..=num_keepers { let raft_port = get_keeper_raft_port(KeeperId(i)); keeper_servers.insert(KeeperServerInfo { - server_id: clickhouse_admin_types::KeeperId(i), + server_id: KeeperId(i), host: ClickhouseHost::Ipv6("::1".parse().unwrap()), raft_port, server_type: KeeperServerType::Participant, diff --git a/clickhouse-admin/types/Cargo.toml b/clickhouse-admin/types/Cargo.toml index f8aab17a28e..9da8e3861ec 100644 --- a/clickhouse-admin/types/Cargo.toml +++ b/clickhouse-admin/types/Cargo.toml @@ -8,22 +8,5 @@ license = "MPL-2.0" workspace = true [dependencies] -anyhow.workspace = true -atomicwrites.workspace = true -camino.workspace = true -camino-tempfile.workspace = true -chrono.workspace = true -derive_more.workspace = true -daft.workspace = true -itertools.workspace = true -omicron-common.workspace = true +clickhouse-admin-types-versions.workspace = true omicron-workspace-hack.workspace = true -schemars.workspace = true -serde.workspace = true -serde_json.workspace = true -slog.workspace = true -expectorate.workspace = true - -[dev-dependencies] -slog-async.workspace = true -slog-term.workspace = true diff --git a/clickhouse-admin/types/src/config.rs b/clickhouse-admin/types/src/config.rs index ba2f845c58e..c7037233972 100644 --- a/clickhouse-admin/types/src/config.rs +++ b/clickhouse-admin/types/src/config.rs @@ -2,686 +2,4 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at https://mozilla.org/MPL/2.0/. -use crate::{KeeperId, OXIMETER_CLUSTER, ServerId, path_schema}; -use anyhow::{Error, bail}; -use camino::Utf8PathBuf; -use omicron_common::address::{ - CLICKHOUSE_HTTP_PORT, CLICKHOUSE_INTERSERVER_PORT, - CLICKHOUSE_KEEPER_RAFT_PORT, CLICKHOUSE_KEEPER_TCP_PORT, - CLICKHOUSE_TCP_PORT, -}; -use omicron_common::api::external::Generation; -use schemars::JsonSchema; -use serde::{Deserialize, Serialize}; -use std::net::{Ipv4Addr, Ipv6Addr}; -use std::{fmt::Display, str::FromStr}; - -/// Result after generating a configuration file -#[derive(Debug, Clone, PartialEq, Eq, JsonSchema, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum GenerateConfigResult { - Replica(ReplicaConfig), - Keeper(KeeperConfig), -} - -/// Configuration for a ClickHouse replica server -#[derive(Debug, Clone, PartialEq, Eq, JsonSchema, Serialize, Deserialize)] -pub struct ReplicaConfig { - /// Logging settings - pub logger: LogConfig, - /// Parameter substitutions for replicated tables - pub macros: Macros, - /// Address the server is listening on - pub listen_host: Ipv6Addr, - /// Port for HTTP connections - pub http_port: u16, - /// Port for TCP connections - pub tcp_port: u16, - /// Port for interserver HTTP connections - pub interserver_http_port: u16, - /// Configuration of clusters used by the Distributed table engine and bythe cluster - /// table function - pub remote_servers: RemoteServers, - /// Contains settings that allow ClickHouse servers to interact with a Keeper cluster - pub keepers: KeeperConfigsForReplica, - /// Directory for all files generated by ClickHouse itself - #[schemars(schema_with = "path_schema")] - pub data_path: Utf8PathBuf, - /// A unique identifier for the configuration generation. - pub generation: Generation, -} - -impl ReplicaConfig { - /// A new ClickHouse replica server configuration with default ports and directories - pub fn new( - logger: LogConfig, - macros: Macros, - listen_host: Ipv6Addr, - remote_servers: Vec, - keepers: Vec, - path: Utf8PathBuf, - generation: Generation, - ) -> Self { - let data_path = path.join("data"); - let remote_servers = RemoteServers::new(remote_servers); - let keepers = KeeperConfigsForReplica::new(keepers); - - Self { - logger, - macros, - listen_host, - http_port: CLICKHOUSE_HTTP_PORT, - tcp_port: CLICKHOUSE_TCP_PORT, - interserver_http_port: CLICKHOUSE_INTERSERVER_PORT, - remote_servers, - keepers, - data_path, - generation, - } - } - - pub fn to_xml(&self) -> String { - let ReplicaConfig { - logger, - macros, - listen_host, - http_port, - tcp_port, - interserver_http_port, - remote_servers, - keepers, - data_path, - generation, - } = self; - let logger = logger.to_xml(); - let cluster = macros.cluster.clone(); - let id = macros.replica; - let macros = macros.to_xml(); - let keepers = keepers.to_xml(); - let remote_servers = remote_servers.to_xml(); - let user_files_path = data_path.clone().join("user_files"); - let temp_files_path = data_path.clone().join("tmp"); - let format_schema_path = data_path.clone().join("format_schemas"); - let backup_path = data_path.clone().join("backup"); - format!( - " - -{logger} - {data_path} - - - - random - - - - - - - - - ::/0 - - default - default - - - - - - - 3600 - 0 - 0 - 0 - 0 - 0 - - - - - - system - query_log
- Engine = MergeTree ORDER BY event_time TTL event_date + INTERVAL 7 DAY - 10000 - - - - system - metric_log
- - Engine = MergeTree ORDER BY event_time TTL event_date + INTERVAL 30 DAY - 7500 - 1000 - 1048576 - 8192 - 524288 - false - - - - system - asynchronous_metric_log
- - Engine = MergeTree ORDER BY event_time TTL event_date + INTERVAL 30 DAY - 7500 - 1000 - 1048576 - 8192 - 524288 - false - - - - system - part_log
- - - {temp_files_path} - {user_files_path} - default - {format_schema_path} - {cluster}_{id} - {listen_host} - {http_port} - {tcp_port} - {interserver_http_port} - {listen_host} - - - - - 604800 - - - 60 - - - 1000 - - - - {backup_path} - - - - - 1.0 - - - - 1.0 - -{macros} -{remote_servers} -{keepers} - - -" - ) - } -} - -#[derive(Debug, Clone, PartialEq, Eq, JsonSchema, Serialize, Deserialize)] -pub struct Macros { - pub shard: u64, - pub replica: ServerId, - pub cluster: String, -} - -impl Macros { - /// A new macros configuration block with default cluster - pub fn new(replica: ServerId) -> Self { - Self { shard: 1, replica, cluster: OXIMETER_CLUSTER.to_string() } - } - - pub fn to_xml(&self) -> String { - let Macros { shard, replica, cluster } = self; - format!( - " - - {shard} - {replica} - {cluster} - " - ) - } -} - -#[derive(Debug, Clone, PartialEq, Eq, JsonSchema, Serialize, Deserialize)] -pub struct RemoteServers { - pub cluster: String, - pub secret: String, - pub replicas: Vec, -} - -impl RemoteServers { - /// A new remote_servers configuration block with default cluster - pub fn new(replicas: Vec) -> Self { - Self { - cluster: OXIMETER_CLUSTER.to_string(), - // TODO(https://github.com/oxidecomputer/omicron/issues/3823): secret handling TBD - secret: "some-unique-value".to_string(), - replicas, - } - } - - pub fn to_xml(&self) -> String { - let RemoteServers { cluster, secret, replicas } = self; - - let mut s = format!( - " - - <{cluster}> - - {secret} - - true" - ); - - for r in replicas { - let ServerNodeConfig { host, port } = r; - let sanitised_host = match host { - ClickhouseHost::Ipv6(h) => h.to_string(), - ClickhouseHost::Ipv4(h) => h.to_string(), - ClickhouseHost::DomainName(h) => h.to_string(), - }; - - s.push_str(&format!( - " - - {sanitised_host} - {port} - " - )); - } - - s.push_str(&format!( - " - - - - " - )); - - s - } -} - -#[derive(Debug, Clone, PartialEq, Eq, JsonSchema, Serialize, Deserialize)] -pub struct KeeperConfigsForReplica { - pub nodes: Vec, -} - -impl KeeperConfigsForReplica { - pub fn new(nodes: Vec) -> Self { - Self { nodes } - } - - pub fn to_xml(&self) -> String { - let mut s = String::from(" "); - for node in &self.nodes { - let KeeperNodeConfig { host, port } = node; - - // ClickHouse servers have a small quirk, where when setting the - // keeper hosts as IPv6 addresses in the replica configuration file, - // they must be wrapped in square brackets. - // Otherwise, when running any query, a "Service not found" error - // appears. - // https://github.com/ClickHouse/ClickHouse/blob/a011990fd75628c63c7995c4f15475f1d4125d10/src/Coordination/KeeperStateManager.cpp#L149 - let sanitised_host = match host { - ClickhouseHost::Ipv6(h) => format!("[{h}]"), - ClickhouseHost::Ipv4(h) => h.to_string(), - ClickhouseHost::DomainName(h) => h.to_string(), - }; - - s.push_str(&format!( - " - - {sanitised_host} - {port} - ", - )); - } - s.push_str("\n "); - s - } -} - -#[derive( - Debug, - Clone, - PartialEq, - Eq, - Deserialize, - PartialOrd, - Ord, - Serialize, - JsonSchema, -)] -#[serde(rename_all = "snake_case")] -pub enum ClickhouseHost { - Ipv6(Ipv6Addr), - Ipv4(Ipv4Addr), - DomainName(String), -} - -impl FromStr for ClickhouseHost { - type Err = Error; - - fn from_str(s: &str) -> Result { - if let Ok(ipv6) = s.parse() { - Ok(ClickhouseHost::Ipv6(ipv6)) - } else if let Ok(ipv4) = s.parse() { - Ok(ClickhouseHost::Ipv4(ipv4)) - // Validating whether a string is a valid domain or - // not is a complex process that isn't necessary for - // this function. In the case of ClickhouseHost, we wil - // only be dealing with our in internal DNS service - // which provides names that always end with `.internal`. - } else if s.ends_with(".internal") { - Ok(ClickhouseHost::DomainName(s.to_string())) - } else { - bail!("{s} is not a valid address or domain name") - } - } -} - -#[derive(Debug, Clone, PartialEq, Eq, JsonSchema, Serialize, Deserialize)] -pub struct KeeperNodeConfig { - pub host: ClickhouseHost, - pub port: u16, -} - -impl KeeperNodeConfig { - /// A new ClickHouse keeper node configuration with default port - pub fn new(host: ClickhouseHost) -> Self { - let port = CLICKHOUSE_KEEPER_TCP_PORT; - Self { host, port } - } -} - -#[derive(Debug, Clone, PartialEq, Eq, JsonSchema, Serialize, Deserialize)] -pub struct ServerNodeConfig { - pub host: ClickhouseHost, - pub port: u16, -} - -impl ServerNodeConfig { - /// A new ClickHouse replica node configuration with default port - pub fn new(host: ClickhouseHost) -> Self { - let port = CLICKHOUSE_TCP_PORT; - Self { host, port } - } -} - -pub enum NodeType { - Server, - Keeper, -} - -#[derive(Debug, Clone, PartialEq, Eq, JsonSchema, Serialize, Deserialize)] -pub struct LogConfig { - pub level: LogLevel, - #[schemars(schema_with = "path_schema")] - pub log: Utf8PathBuf, - #[schemars(schema_with = "path_schema")] - pub errorlog: Utf8PathBuf, - pub size: u16, - pub count: usize, -} - -impl LogConfig { - /// A new logger configuration with default directories - pub fn new(path: Utf8PathBuf, node_type: NodeType) -> Self { - let prefix = match node_type { - NodeType::Server => "clickhouse", - NodeType::Keeper => "clickhouse-keeper", - }; - - let logs: Utf8PathBuf = path.join("log"); - let log = logs.join(format!("{prefix}.log")); - let errorlog = logs.join(format!("{prefix}.err.log")); - - Self { level: LogLevel::default(), log, errorlog, size: 100, count: 1 } - } - - pub fn to_xml(&self) -> String { - let LogConfig { level, log, errorlog, size, count } = &self; - format!( - " - - {level} - {log} - {errorlog} - {size}M - {count} - -" - ) - } -} - -#[derive(Debug, Clone, PartialEq, Eq, JsonSchema, Serialize, Deserialize)] -pub struct KeeperCoordinationSettings { - pub operation_timeout_ms: u32, - pub session_timeout_ms: u32, - pub raft_logs_level: LogLevel, -} - -impl KeeperCoordinationSettings { - pub fn default() -> Self { - Self { - operation_timeout_ms: 10000, - session_timeout_ms: 30000, - raft_logs_level: LogLevel::Trace, - } - } -} - -#[derive(Debug, Clone, PartialEq, Eq, JsonSchema, Serialize, Deserialize)] -pub struct RaftServers { - pub servers: Vec, -} - -impl RaftServers { - pub fn new(servers: Vec) -> Self { - Self { servers } - } - pub fn to_xml(&self) -> String { - let mut s = String::new(); - for server in &self.servers { - let RaftServerConfig { id, hostname, port } = server; - - let sanitised_host = match hostname { - ClickhouseHost::Ipv6(h) => h.to_string(), - ClickhouseHost::Ipv4(h) => h.to_string(), - ClickhouseHost::DomainName(h) => h.to_string(), - }; - - s.push_str(&format!( - " - - {id} - {sanitised_host} - {port} - - " - )); - } - - s - } -} - -#[derive(Debug, Clone, PartialEq, Eq, JsonSchema, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub struct RaftServerSettings { - pub id: KeeperId, - pub host: ClickhouseHost, -} - -#[derive(Debug, Clone, PartialEq, Eq, JsonSchema, Serialize, Deserialize)] -pub struct RaftServerConfig { - pub id: KeeperId, - pub hostname: ClickhouseHost, - pub port: u16, -} - -impl RaftServerConfig { - pub fn new(settings: RaftServerSettings) -> Self { - Self { - id: settings.id, - hostname: settings.host, - port: CLICKHOUSE_KEEPER_RAFT_PORT, - } - } -} - -/// Configuration for a ClickHouse keeper -#[derive(Debug, Clone, PartialEq, Eq, JsonSchema, Serialize, Deserialize)] -pub struct KeeperConfig { - /// Logging settings - pub logger: LogConfig, - /// Address the keeper is listening on - pub listen_host: Ipv6Addr, - /// Port for TCP connections - pub tcp_port: u16, - /// Unique ID for this keeper node - pub server_id: KeeperId, - /// Directory for coordination logs - #[schemars(schema_with = "path_schema")] - pub log_storage_path: Utf8PathBuf, - /// Directory for coordination snapshot storage - #[schemars(schema_with = "path_schema")] - pub snapshot_storage_path: Utf8PathBuf, - /// Internal coordination settings - pub coordination_settings: KeeperCoordinationSettings, - /// Settings for each server in the keeper cluster - pub raft_config: RaftServers, - /// Directory for all files generated by ClickHouse itself - #[schemars(schema_with = "path_schema")] - pub datastore_path: Utf8PathBuf, - /// A unique identifier for the configuration generation. - pub generation: Generation, -} - -impl KeeperConfig { - /// A new ClickHouse keeper node configuration with default ports and directories - pub fn new( - logger: LogConfig, - listen_host: Ipv6Addr, - server_id: KeeperId, - datastore_path: Utf8PathBuf, - raft_config: RaftServers, - generation: Generation, - ) -> Self { - let coordination_path = datastore_path.join("coordination"); - let log_storage_path = coordination_path.join("log"); - let snapshot_storage_path = coordination_path.join("snapshots"); - let coordination_settings = KeeperCoordinationSettings::default(); - Self { - logger, - listen_host, - tcp_port: CLICKHOUSE_KEEPER_TCP_PORT, - server_id, - log_storage_path, - snapshot_storage_path, - coordination_settings, - raft_config, - datastore_path, - generation, - } - } - - pub fn to_xml(&self) -> String { - let KeeperConfig { - logger, - listen_host, - tcp_port, - server_id, - log_storage_path, - snapshot_storage_path, - coordination_settings, - raft_config, - datastore_path, - generation, - } = self; - let logger = logger.to_xml(); - let KeeperCoordinationSettings { - operation_timeout_ms, - session_timeout_ms, - raft_logs_level, - } = coordination_settings; - let raft_servers = raft_config.to_xml(); - format!( - " - -{logger} - {listen_host} - {datastore_path} - - false - {tcp_port} - {server_id} - {log_storage_path} - {snapshot_storage_path} - - {operation_timeout_ms} - {session_timeout_ms} - {raft_logs_level} - - -{raft_servers} - - - - -" - ) - } -} - -#[derive(Debug, Clone, PartialEq, Eq, JsonSchema, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum LogLevel { - Trace, - Debug, -} - -impl LogLevel { - fn default() -> Self { - LogLevel::Trace - } -} - -impl Display for LogLevel { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - let s = match self { - LogLevel::Trace => "trace", - LogLevel::Debug => "debug", - }; - write!(f, "{s}") - } -} - -impl FromStr for LogLevel { - type Err = Error; - - fn from_str(s: &str) -> Result { - if s == "trace" { - Ok(LogLevel::Trace) - } else if s == "debug" { - Ok(LogLevel::Debug) - } else { - bail!("{s} is not a valid log level") - } - } -} +pub use clickhouse_admin_types_versions::latest::config::*; diff --git a/clickhouse-admin/types/src/keeper.rs b/clickhouse-admin/types/src/keeper.rs new file mode 100644 index 00000000000..1da15a5445a --- /dev/null +++ b/clickhouse-admin/types/src/keeper.rs @@ -0,0 +1,5 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +pub use clickhouse_admin_types_versions::latest::keeper::*; diff --git a/clickhouse-admin/types/src/lib.rs b/clickhouse-admin/types/src/lib.rs index 8eda5ce1ee3..e9de4b68e28 100644 --- a/clickhouse-admin/types/src/lib.rs +++ b/clickhouse-admin/types/src/lib.rs @@ -2,2255 +2,20 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this // file, You can obtain one at https://mozilla.org/MPL/2.0/. -use anyhow::{Context, Error, Result, bail}; -use atomicwrites::AtomicFile; -use camino::Utf8PathBuf; -use chrono::{DateTime, Utc}; -use daft::Diffable; -use derive_more::{Add, AddAssign, Display, From}; -use itertools::Itertools; -use omicron_common::api::external::Generation; -use schemars::{ - JsonSchema, - r#gen::SchemaGenerator, - schema::{Schema, SchemaObject}, -}; -use serde::{Deserialize, Serialize}; -use slog::{Logger, info}; -use std::collections::{BTreeMap, BTreeSet}; -use std::fmt; -use std::fs::create_dir; -use std::io::{ErrorKind, Write}; -use std::net::Ipv6Addr; -use std::str::FromStr; +//! Types for the ClickHouse Admin APIs. +//! +//! This crate re-exports types from `clickhouse-admin-types-versions` for use +//! in business logic. For versioned type definitions and API version +//! conversions, see the versions crate directly. -mod config; -pub use config::{ - ClickhouseHost, GenerateConfigResult, KeeperConfig, - KeeperConfigsForReplica, KeeperNodeConfig, LogConfig, LogLevel, Macros, - NodeType, RaftServerConfig, RaftServerSettings, RaftServers, ReplicaConfig, - ServerNodeConfig, -}; +pub mod config; +pub mod keeper; +pub mod server; +// Constants for file paths - not API-published pub const CLICKHOUSE_SERVER_CONFIG_DIR: &str = "/opt/oxide/clickhouse_server/config.d"; pub const CLICKHOUSE_SERVER_CONFIG_FILE: &str = "replica-server-config.xml"; pub const CLICKHOUSE_KEEPER_CONFIG_DIR: &str = "/opt/oxide/clickhouse_keeper"; pub const CLICKHOUSE_KEEPER_CONFIG_FILE: &str = "keeper_config.xml"; pub const OXIMETER_CLUSTER: &str = "oximeter_cluster"; - -// Used for schemars to be able to be used with camino: -// See https://github.com/camino-rs/camino/issues/91#issuecomment-2027908513 -pub fn path_schema(generator: &mut SchemaGenerator) -> Schema { - let mut schema: SchemaObject = ::json_schema(generator).into(); - schema.format = Some("Utf8PathBuf".to_owned()); - schema.into() -} - -/// A unique ID for a ClickHouse Keeper -#[derive( - Debug, - Clone, - Copy, - Eq, - PartialEq, - Ord, - PartialOrd, - From, - Add, - AddAssign, - Display, - JsonSchema, - Serialize, - Deserialize, - Diffable, -)] -pub struct KeeperId(pub u64); - -/// A unique ID for a Clickhouse Server -#[derive( - Debug, - Clone, - Copy, - Eq, - PartialEq, - Ord, - PartialOrd, - From, - Add, - AddAssign, - Display, - JsonSchema, - Serialize, - Deserialize, - Diffable, -)] -pub struct ServerId(pub u64); - -/// The top most type for configuring clickhouse-servers via -/// clickhouse-admin-server-api -#[derive(Debug, Serialize, Deserialize, JsonSchema)] -pub struct ServerConfigurableSettings { - /// A unique identifier for the configuration generation. - pub generation: Generation, - /// Configurable settings for a ClickHouse replica server node. - pub settings: ServerSettings, -} - -impl ServerConfigurableSettings { - /// Generate a configuration file for a replica server node - pub fn generate_xml_file(&self) -> Result { - let logger = LogConfig::new( - self.settings.datastore_path.clone(), - NodeType::Server, - ); - let macros = Macros::new(self.settings.id); - - let keepers: Vec = self - .settings - .keepers - .iter() - .map(|host| KeeperNodeConfig::new(host.clone())) - .collect(); - - let servers: Vec = self - .settings - .remote_servers - .iter() - .map(|host| ServerNodeConfig::new(host.clone())) - .collect(); - - let config = ReplicaConfig::new( - logger, - macros, - self.listen_addr(), - servers.clone(), - keepers.clone(), - self.datastore_path(), - self.generation(), - ); - - match create_dir(self.settings.config_dir.clone()) { - Ok(_) => (), - Err(e) if e.kind() == ErrorKind::AlreadyExists => (), - Err(e) => return Err(e.into()), - }; - - let path = self.settings.config_dir.join("replica-server-config.xml"); - AtomicFile::new( - path.clone(), - atomicwrites::OverwriteBehavior::AllowOverwrite, - ) - .write(|f| f.write_all(config.to_xml().as_bytes())) - .with_context(|| format!("failed to write to `{}`", path))?; - - Ok(config) - } - - pub fn generation(&self) -> Generation { - self.generation - } - - fn listen_addr(&self) -> Ipv6Addr { - self.settings.listen_addr - } - - fn datastore_path(&self) -> Utf8PathBuf { - self.settings.datastore_path.clone() - } -} - -/// The top most type for configuring clickhouse-servers via -/// clickhouse-admin-keeper-api -#[derive(Debug, Serialize, Deserialize, JsonSchema)] -pub struct KeeperConfigurableSettings { - /// A unique identifier for the configuration generation. - pub generation: Generation, - /// Configurable settings for a ClickHouse keeper node. - pub settings: KeeperSettings, -} - -impl KeeperConfigurableSettings { - /// Generate a configuration file for a keeper node - pub fn generate_xml_file(&self) -> Result { - let logger = LogConfig::new( - self.settings.datastore_path.clone(), - NodeType::Keeper, - ); - - let raft_servers = self - .settings - .raft_servers - .iter() - .map(|settings| RaftServerConfig::new(settings.clone())) - .collect(); - let raft_config = RaftServers::new(raft_servers); - - let config = KeeperConfig::new( - logger, - self.listen_addr(), - self.id(), - self.datastore_path(), - raft_config, - self.generation(), - ); - - match create_dir(self.settings.config_dir.clone()) { - Ok(_) => (), - Err(e) if e.kind() == ErrorKind::AlreadyExists => (), - Err(e) => return Err(e.into()), - }; - - let path = self.settings.config_dir.join("keeper_config.xml"); - AtomicFile::new( - path.clone(), - atomicwrites::OverwriteBehavior::AllowOverwrite, - ) - .write(|f| f.write_all(config.to_xml().as_bytes())) - .with_context(|| format!("failed to write to `{}`", path))?; - - Ok(config) - } - - pub fn generation(&self) -> Generation { - self.generation - } - - fn listen_addr(&self) -> Ipv6Addr { - self.settings.listen_addr - } - - fn id(&self) -> KeeperId { - self.settings.id - } - - fn datastore_path(&self) -> Utf8PathBuf { - self.settings.datastore_path.clone() - } -} - -/// Configurable settings for a ClickHouse replica server node. -#[derive(Debug, Clone, PartialEq, Eq, Deserialize, Serialize, JsonSchema)] -#[serde(rename_all = "snake_case")] -pub struct ServerSettings { - /// Directory for the generated server configuration XML file - #[schemars(schema_with = "path_schema")] - pub config_dir: Utf8PathBuf, - /// Unique ID of the server node - pub id: ServerId, - /// Directory for all files generated by ClickHouse itself - #[schemars(schema_with = "path_schema")] - pub datastore_path: Utf8PathBuf, - /// Address the server is listening on - pub listen_addr: Ipv6Addr, - /// Addresses for each of the individual nodes in the Keeper cluster - pub keepers: Vec, - /// Addresses for each of the individual replica servers - pub remote_servers: Vec, -} - -impl ServerSettings { - pub fn new( - config_dir: Utf8PathBuf, - id: ServerId, - datastore_path: Utf8PathBuf, - listen_addr: Ipv6Addr, - keepers: Vec, - remote_servers: Vec, - ) -> Self { - Self { - config_dir, - id, - datastore_path, - listen_addr, - keepers, - remote_servers, - } - } -} - -/// Configurable settings for a ClickHouse keeper node. -#[derive(Debug, Clone, PartialEq, Eq, Deserialize, Serialize, JsonSchema)] -#[serde(rename_all = "snake_case")] -pub struct KeeperSettings { - /// Directory for the generated keeper configuration XML file - #[schemars(schema_with = "path_schema")] - pub config_dir: Utf8PathBuf, - /// Unique ID of the keeper node - pub id: KeeperId, - /// ID and host of each server in the keeper cluster - pub raft_servers: Vec, - /// Directory for all files generated by ClickHouse itself - #[schemars(schema_with = "path_schema")] - pub datastore_path: Utf8PathBuf, - /// Address the keeper is listening on - pub listen_addr: Ipv6Addr, -} - -impl KeeperSettings { - pub fn new( - config_dir: Utf8PathBuf, - id: KeeperId, - raft_servers: Vec, - datastore_path: Utf8PathBuf, - listen_addr: Ipv6Addr, - ) -> Self { - Self { config_dir, id, raft_servers, datastore_path, listen_addr } - } -} - -#[derive(Debug, Clone, PartialEq, Eq, Deserialize, Serialize, JsonSchema)] -#[serde(rename_all = "snake_case")] -/// Logically grouped information file from a keeper node -pub struct Lgif { - /// Index of the first log entry in the current log segment - pub first_log_idx: u64, - /// Term of the leader when the first log entry was created - pub first_log_term: u64, - /// Index of the last log entry in the current log segment - pub last_log_idx: u64, - /// Term of the leader when the last log entry was created - pub last_log_term: u64, - /// Index of the last committed log entry - pub last_committed_log_idx: u64, - /// Index of the last committed log entry from the leader's perspective - pub leader_committed_log_idx: u64, - /// Target index for log commitment during replication or recovery - pub target_committed_log_idx: u64, - /// Index of the most recent snapshot taken - pub last_snapshot_idx: u64, -} - -impl Lgif { - pub fn parse(log: &Logger, data: &[u8]) -> Result { - // The reponse we get from running `clickhouse keeper-client -h {HOST} --q lgif` - // isn't in any known format (e.g. JSON), but rather a series of lines with key-value - // pairs separated by a tab: - // - // ```console - // $ clickhouse keeper-client -h localhost -p 20001 --q lgif - // first_log_idx 1 - // first_log_term 1 - // last_log_idx 10889 - // last_log_term 20 - // last_committed_log_idx 10889 - // leader_committed_log_idx 10889 - // target_committed_log_idx 10889 - // last_snapshot_idx 9465 - // ``` - let s = String::from_utf8_lossy(data); - info!( - log, - "Retrieved data from `clickhouse keeper-client --q lgif`"; - "output" => ?s - ); - - let expected = Lgif::expected_keys(); - - // Verify the output contains the same amount of lines as the expected keys. - // This will ensure we catch any new key-value pairs appended to the lgif output. - let lines = s.trim().lines(); - if expected.len() != lines.count() { - bail!( - "Output from the Keeper differs to the expected output keys \ - Output: {s:?} \ - Expected output keys: {expected:?}" - ); - } - - let mut vals: Vec = Vec::new(); - for (line, expected_key) in s.lines().zip(expected.clone()) { - let mut split = line.split('\t'); - let Some(key) = split.next() else { - bail!("Returned None while attempting to retrieve key"); - }; - if key != expected_key { - bail!( - "Extracted key `{key:?}` from output differs from expected key `{expected_key}`" - ); - } - let Some(val) = split.next() else { - bail!( - "Command output has a line that does not contain a key-value pair: {key:?}" - ); - }; - let val = match u64::from_str(val) { - Ok(v) => v, - Err(e) => bail!( - "Unable to convert value {val:?} into u64 for key {key}: {e}" - ), - }; - vals.push(val); - } - - let mut iter = vals.into_iter(); - Ok(Lgif { - first_log_idx: iter.next().unwrap(), - first_log_term: iter.next().unwrap(), - last_log_idx: iter.next().unwrap(), - last_log_term: iter.next().unwrap(), - last_committed_log_idx: iter.next().unwrap(), - leader_committed_log_idx: iter.next().unwrap(), - target_committed_log_idx: iter.next().unwrap(), - last_snapshot_idx: iter.next().unwrap(), - }) - } - - fn expected_keys() -> Vec<&'static str> { - vec![ - "first_log_idx", - "first_log_term", - "last_log_idx", - "last_log_term", - "last_committed_log_idx", - "leader_committed_log_idx", - "target_committed_log_idx", - "last_snapshot_idx", - ] - } -} - -#[derive( - Debug, - Clone, - PartialEq, - Eq, - Deserialize, - Ord, - PartialOrd, - Serialize, - JsonSchema, -)] -#[serde(rename_all = "snake_case")] -pub enum KeeperServerType { - Participant, - Learner, -} - -impl FromStr for KeeperServerType { - type Err = Error; - - fn from_str(s: &str) -> Result { - match s { - "participant" => Ok(KeeperServerType::Participant), - "learner" => Ok(KeeperServerType::Learner), - _ => bail!("{s} is not a valid keeper server type"), - } - } -} - -#[derive( - Debug, - Clone, - PartialEq, - Eq, - Deserialize, - PartialOrd, - Ord, - Serialize, - JsonSchema, -)] -#[serde(rename_all = "snake_case")] -pub struct KeeperServerInfo { - /// Unique, immutable ID of the keeper server - pub server_id: KeeperId, - /// Host of the keeper server - pub host: ClickhouseHost, - /// Keeper server raft port - pub raft_port: u16, - /// A keeper server either participant or learner - /// (learner does not participate in leader elections). - pub server_type: KeeperServerType, - /// non-negative integer telling which nodes should be - /// prioritised on leader elections. - /// Priority of 0 means server will never be a leader. - pub priority: u16, -} - -#[derive( - Debug, - Clone, - PartialEq, - Eq, - Deserialize, - PartialOrd, - Ord, - Serialize, - JsonSchema, -)] -#[serde(rename_all = "snake_case")] -/// Keeper raft configuration information -pub struct RaftConfig { - pub keeper_servers: BTreeSet, -} - -impl RaftConfig { - pub fn parse(log: &Logger, data: &[u8]) -> Result { - // The response we get from `$ clickhouse keeper-client -h {HOST} --q 'get /keeper/config' - // is a format unique to ClickHouse, where the data for each server is separated by a colon - // - // ```console - // $ clickhouse keeper-client -h localhost --q 'get /keeper/config' - // server.1=::1:21001;participant;1 - // server.2=::1:21002;participant;1 - // server.3=::1:21003;participant;1 - //``` - let s = String::from_utf8_lossy(data); - info!( - log, - "Retrieved data from `clickhouse keeper-client --q 'get /keeper/config'`"; - "output" => ?s - ); - - if s.is_empty() { - bail!("Cannot parse an empty response"); - } - - let mut keeper_servers = BTreeSet::new(); - for line in s.lines() { - let mut split = line.split('='); - let Some(server) = split.next() else { - bail!( - "Returned None while attempting to retrieve raft configuration" - ); - }; - - // Retrieve server ID - let mut split_server = server.split("."); - let Some(s) = split_server.next() else { - bail!( - "Returned None while attempting to retrieve server identifier" - ) - }; - if s != "server" { - bail!( - "Output is not as expected. \ - Server identifier: '{server}' \ - Expected server identifier: 'server.{{SERVER_ID}}'" - ) - }; - let Some(id) = split_server.next() else { - bail!("Returned None while attempting to retrieve server ID"); - }; - let u64_id = match u64::from_str(id) { - Ok(v) => v, - Err(e) => bail!("Unable to convert value {id:?} into u64: {e}"), - }; - let server_id = KeeperId(u64_id); - - // Retrieve server information - let Some(info) = split.next() else { - bail!("Returned None while attempting to retrieve server info"); - }; - let mut split_info = info.split(";"); - - // Retrieve port - let Some(address) = split_info.next() else { - bail!("Returned None while attempting to retrieve address") - }; - let Some(port) = address.split(':').next_back() else { - bail!("A port could not be extracted from {address}") - }; - let raft_port = match u16::from_str(port) { - Ok(v) => v, - Err(e) => { - bail!("Unable to convert value {port:?} into u16: {e}") - } - }; - - // Retrieve host - let p = format!(":{}", port); - let Some(h) = address.split(&p).next() else { - bail!( - "A host could not be extracted from {address}. Missing port {port}" - ) - }; - // The ouput we get from running the clickhouse keeper-client - // command does not add square brackets to an IPv6 address - // that cointains a port: server.1=::1:21001;participant;1 - // Because of this, we can parse `h` directly into an Ipv6Addr - let host = ClickhouseHost::from_str(h)?; - - // Retrieve server_type - let Some(s_type) = split_info.next() else { - bail!("Returned None while attempting to retrieve server type") - }; - let server_type = KeeperServerType::from_str(s_type)?; - - // Retrieve priority - let Some(s_priority) = split_info.next() else { - bail!("Returned None while attempting to retrieve priority") - }; - let priority = match u16::from_str(s_priority) { - Ok(v) => v, - Err(e) => { - bail!( - "Unable to convert value {s_priority:?} into u16: {e}" - ) - } - }; - - keeper_servers.insert(KeeperServerInfo { - server_id, - host, - raft_port, - server_type, - priority, - }); - } - - Ok(RaftConfig { keeper_servers }) - } -} - -// While we generally use "Config", in this case we use "Conf" -// as it is the four letter word command we are invoking: -// `clickhouse keeper-client --q conf` -/// Keeper configuration information -#[derive(Debug, Clone, PartialEq, Eq, Deserialize, Serialize, JsonSchema)] -#[serde(rename_all = "snake_case")] -pub struct KeeperConf { - /// Unique server id, each participant of the ClickHouse Keeper cluster must - /// have a unique number (1, 2, 3, and so on). - pub server_id: KeeperId, - /// Whether Ipv6 is enabled. - pub enable_ipv6: bool, - /// Port for a client to connect. - pub tcp_port: u16, - /// Allow list of 4lw commands. - pub four_letter_word_allow_list: String, - /// Max size of batch in requests count before it will be sent to RAFT. - pub max_requests_batch_size: u64, - /// Min timeout for client session (ms). - pub min_session_timeout_ms: u64, - /// Max timeout for client session (ms). - pub session_timeout_ms: u64, - /// Timeout for a single client operation (ms). - pub operation_timeout_ms: u64, - /// How often ClickHouse Keeper checks for dead sessions and removes them (ms). - pub dead_session_check_period_ms: u64, - /// How often a ClickHouse Keeper leader will send heartbeats to followers (ms). - pub heart_beat_interval_ms: u64, - /// If the follower does not receive a heartbeat from the leader in this interval, - /// then it can initiate leader election. Must be less than or equal to - /// election_timeout_upper_bound_ms. Ideally they shouldn't be equal. - pub election_timeout_lower_bound_ms: u64, - /// If the follower does not receive a heartbeat from the leader in this interval, - /// then it must initiate leader election. - pub election_timeout_upper_bound_ms: u64, - /// How many coordination log records to store before compaction. - pub reserved_log_items: u64, - /// How often ClickHouse Keeper will create new snapshots - /// (in the number of records in logs). - pub snapshot_distance: u64, - /// Allow to forward write requests from followers to the leader. - pub auto_forwarding: bool, - /// Wait to finish internal connections and shutdown (ms). - pub shutdown_timeout: u64, - /// If the server doesn't connect to other quorum participants in the specified - /// timeout it will terminate (ms). - pub startup_timeout: u64, - /// Text logging level about coordination (trace, debug, and so on). - pub raft_logs_level: LogLevel, - /// How many snapshots to keep. - pub snapshots_to_keep: u64, - /// How many log records to store in a single file. - pub rotate_log_storage_interval: u64, - /// Threshold when leader considers follower as stale and sends the snapshot - /// to it instead of logs. - pub stale_log_gap: u64, - /// When the node became fresh. - pub fresh_log_gap: u64, - /// Max size in bytes of batch of requests that can be sent to RAFT. - pub max_requests_batch_bytes_size: u64, - /// Maximum number of requests that can be in queue for processing. - pub max_request_queue_size: u64, - /// Max size of batch of requests to try to get before proceeding with RAFT. - /// Keeper will not wait for requests but take only requests that are already - /// in the queue. - pub max_requests_quick_batch_size: u64, - /// Whether to execute read requests as writes through whole RAFT consesus with - /// similar speed. - pub quorum_reads: bool, - /// Whether to call fsync on each change in RAFT changelog. - pub force_sync: bool, - /// Whether to write compressed coordination logs in ZSTD format. - pub compress_logs: bool, - /// Whether to write compressed snapshots in ZSTD format (instead of custom LZ4). - pub compress_snapshots_with_zstd_format: bool, - /// How many times we will try to apply configuration change (add/remove server) - /// to the cluster. - pub configuration_change_tries_count: u64, - /// If connection to a peer is silent longer than this limit * (heartbeat interval), - /// we re-establish the connection. - pub raft_limits_reconnect_limit: u64, - /// Path to coordination logs, just like ZooKeeper it is best to store logs - /// on non-busy nodes. - #[schemars(schema_with = "path_schema")] - pub log_storage_path: Utf8PathBuf, - /// Name of disk used for logs. - pub log_storage_disk: String, - /// Path to coordination snapshots. - #[schemars(schema_with = "path_schema")] - pub snapshot_storage_path: Utf8PathBuf, - /// Name of disk used for storage. - pub snapshot_storage_disk: String, -} - -impl KeeperConf { - pub fn parse(log: &Logger, data: &[u8]) -> Result { - // Like Lgif, the reponse we get from running `clickhouse keeper-client -h {HOST} --q conf` - // isn't in any known format (e.g. JSON), but rather a series of lines with key-value - // pairs separated by a tab. - let s = String::from_utf8_lossy(data); - info!( - log, - "Retrieved data from `clickhouse keeper-client --q conf`"; - "output" => ?s - ); - - let expected = KeeperConf::expected_keys(); - - // Verify the output contains the same amount of lines as the expected keys. - // This will ensure we catch any new key-value pairs appended to the lgif output. - let lines = s.trim().lines(); - if expected.len() != lines.count() { - bail!( - "Output from the Keeper differs to the expected output keys \ - Output: {s:?} \ - Expected output keys: {expected:?}" - ); - } - - let mut vals: Vec<&str> = Vec::new(); - // The output from the `conf` command contains the `max_requests_batch_size` field - // twice. We make sure to only read it once. - for (line, expected_key) in s.lines().zip(expected.clone()).unique() { - let mut split = line.split('='); - let Some(key) = split.next() else { - bail!("Returned None while attempting to retrieve key"); - }; - if key != expected_key { - bail!( - "Extracted key `{key:?}` from output differs from expected key `{expected_key}`" - ); - } - let Some(val) = split.next() else { - bail!( - "Command output has a line that does not contain a key-value pair: {key:?}" - ); - }; - vals.push(val); - } - - let mut iter = vals.into_iter(); - let server_id = match u64::from_str(iter.next().unwrap()) { - Ok(v) => KeeperId(v), - Err(e) => bail!("Unable to convert value into u64: {e}"), - }; - - let enable_ipv6 = match bool::from_str(iter.next().unwrap()) { - Ok(v) => v, - Err(e) => bail!("Unable to convert value into bool: {e}"), - }; - - let tcp_port = match u16::from_str(iter.next().unwrap()) { - Ok(v) => v, - Err(e) => bail!("Unable to convert value into u16: {e}"), - }; - - let four_letter_word_allow_list = iter.next().unwrap().to_string(); - - let max_requests_batch_size = match u64::from_str(iter.next().unwrap()) - { - Ok(v) => v, - Err(e) => bail!("Unable to convert value into u64: {e}"), - }; - - let min_session_timeout_ms = match u64::from_str(iter.next().unwrap()) { - Ok(v) => v, - Err(e) => bail!("Unable to convert value into u64: {e}"), - }; - - let session_timeout_ms = match u64::from_str(iter.next().unwrap()) { - Ok(v) => v, - Err(e) => bail!("Unable to convert value into u64: {e}"), - }; - - let operation_timeout_ms = match u64::from_str(iter.next().unwrap()) { - Ok(v) => v, - Err(e) => bail!("Unable to convert value into u64: {e}"), - }; - - let dead_session_check_period_ms = - match u64::from_str(iter.next().unwrap()) { - Ok(v) => v, - Err(e) => bail!("Unable to convert value into u64: {e}"), - }; - - let heart_beat_interval_ms = match u64::from_str(iter.next().unwrap()) { - Ok(v) => v, - Err(e) => bail!("Unable to convert value into u64: {e}"), - }; - - let election_timeout_lower_bound_ms = - match u64::from_str(iter.next().unwrap()) { - Ok(v) => v, - Err(e) => bail!("Unable to convert value into u64: {e}"), - }; - - let election_timeout_upper_bound_ms = - match u64::from_str(iter.next().unwrap()) { - Ok(v) => v, - Err(e) => bail!("Unable to convert value into u64: {e}"), - }; - - let reserved_log_items = match u64::from_str(iter.next().unwrap()) { - Ok(v) => v, - Err(e) => bail!("Unable to convert value into u64: {e}"), - }; - - let snapshot_distance = match u64::from_str(iter.next().unwrap()) { - Ok(v) => v, - Err(e) => bail!("Unable to convert value into u64: {e}"), - }; - - let auto_forwarding = match bool::from_str(iter.next().unwrap()) { - Ok(v) => v, - Err(e) => bail!("Unable to convert value into bool: {e}"), - }; - - let shutdown_timeout = match u64::from_str(iter.next().unwrap()) { - Ok(v) => v, - Err(e) => bail!("Unable to convert value into u64 {e}"), - }; - - let startup_timeout = match u64::from_str(iter.next().unwrap()) { - Ok(v) => v, - Err(e) => bail!("Unable to convert value into u64: {e}"), - }; - - let raft_logs_level = match LogLevel::from_str(iter.next().unwrap()) { - Ok(v) => v, - Err(e) => bail!("Unable to convert value into LogLevel: {e}"), - }; - - let snapshots_to_keep = match u64::from_str(iter.next().unwrap()) { - Ok(v) => v, - Err(e) => bail!("Unable to convert value into u64: {e}"), - }; - - let rotate_log_storage_interval = - match u64::from_str(iter.next().unwrap()) { - Ok(v) => v, - Err(e) => bail!("Unable to convert value into u64 {e}"), - }; - - let stale_log_gap = match u64::from_str(iter.next().unwrap()) { - Ok(v) => v, - Err(e) => bail!("Unable to convert value into u64: {e}"), - }; - - let fresh_log_gap = match u64::from_str(iter.next().unwrap()) { - Ok(v) => v, - Err(e) => bail!("Unable to convert value into u64: {e}"), - }; - - let max_requests_batch_bytes_size = - match u64::from_str(iter.next().unwrap()) { - Ok(v) => v, - Err(e) => bail!("Unable to convert value into u64 {e}"), - }; - - let max_request_queue_size = match u64::from_str(iter.next().unwrap()) { - Ok(v) => v, - Err(e) => bail!("Unable to convert value into u64 {e}"), - }; - - let max_requests_quick_batch_size = - match u64::from_str(iter.next().unwrap()) { - Ok(v) => v, - Err(e) => bail!("Unable to convert value into u64 {e}"), - }; - - let quorum_reads = match bool::from_str(iter.next().unwrap()) { - Ok(v) => v, - Err(e) => bail!("Unable to convert value into bool: {e}"), - }; - - let force_sync = match bool::from_str(iter.next().unwrap()) { - Ok(v) => v, - Err(e) => bail!("Unable to convert value into bool: {e}"), - }; - - let compress_logs = match bool::from_str(iter.next().unwrap()) { - Ok(v) => v, - Err(e) => bail!("Unable to convert value into bool: {e}"), - }; - - let compress_snapshots_with_zstd_format = - match bool::from_str(iter.next().unwrap()) { - Ok(v) => v, - Err(e) => bail!("Unable to convert value into bool: {e}"), - }; - - let configuration_change_tries_count = - match u64::from_str(iter.next().unwrap()) { - Ok(v) => v, - Err(e) => bail!("Unable to convert value into u64: {e}"), - }; - - let raft_limits_reconnect_limit = - match u64::from_str(iter.next().unwrap()) { - Ok(v) => v, - Err(e) => bail!("Unable to convert value into u64: {e}"), - }; - - let log_storage_path = match Utf8PathBuf::from_str(iter.next().unwrap()) - { - Ok(v) => v, - Err(e) => bail!("Unable to convert value into Utf8PathBuf: {e}"), - }; - - let log_storage_disk = iter.next().unwrap().to_string(); - - let snapshot_storage_path = - match Utf8PathBuf::from_str(iter.next().unwrap()) { - Ok(v) => v, - Err(e) => { - bail!("Unable to convert value into Utf8PathBuf: {e}") - } - }; - - let snapshot_storage_disk = iter.next().unwrap().to_string(); - - Ok(Self { - server_id, - enable_ipv6, - tcp_port, - four_letter_word_allow_list, - max_requests_batch_size, - min_session_timeout_ms, - session_timeout_ms, - operation_timeout_ms, - dead_session_check_period_ms, - heart_beat_interval_ms, - election_timeout_lower_bound_ms, - election_timeout_upper_bound_ms, - reserved_log_items, - snapshot_distance, - auto_forwarding, - shutdown_timeout, - startup_timeout, - raft_logs_level, - snapshots_to_keep, - rotate_log_storage_interval, - stale_log_gap, - fresh_log_gap, - max_requests_batch_bytes_size, - max_request_queue_size, - max_requests_quick_batch_size, - quorum_reads, - force_sync, - compress_logs, - compress_snapshots_with_zstd_format, - configuration_change_tries_count, - raft_limits_reconnect_limit, - log_storage_path, - log_storage_disk, - snapshot_storage_path, - snapshot_storage_disk, - }) - } - - fn expected_keys() -> Vec<&'static str> { - vec![ - "server_id", - "enable_ipv6", - "tcp_port", - "four_letter_word_allow_list", - "max_requests_batch_size", - "min_session_timeout_ms", - "session_timeout_ms", - "operation_timeout_ms", - "dead_session_check_period_ms", - "heart_beat_interval_ms", - "election_timeout_lower_bound_ms", - "election_timeout_upper_bound_ms", - "reserved_log_items", - "snapshot_distance", - "auto_forwarding", - "shutdown_timeout", - "startup_timeout", - "raft_logs_level", - "snapshots_to_keep", - "rotate_log_storage_interval", - "stale_log_gap", - "fresh_log_gap", - "max_requests_batch_size", - "max_requests_batch_bytes_size", - "max_request_queue_size", - "max_requests_quick_batch_size", - "quorum_reads", - "force_sync", - "compress_logs", - "compress_snapshots_with_zstd_format", - "configuration_change_tries_count", - "raft_limits_reconnect_limit", - "log_storage_path", - "log_storage_disk", - "snapshot_storage_path", - "snapshot_storage_disk", - ] - } -} - -/// The configuration of the clickhouse keeper raft cluster returned from a -/// single keeper node -/// -/// Each keeper is asked for its known raft configuration via `clickhouse-admin` -/// dropshot servers running in `ClickhouseKeeper` zones. state. We include the -/// leader committed log index known to the current keeper node (whether or not -/// it is the leader) to determine which configuration is newest. -#[derive( - Clone, - Debug, - PartialEq, - Eq, - PartialOrd, - Ord, - Deserialize, - Serialize, - JsonSchema, -)] -#[serde(rename_all = "snake_case")] -pub struct ClickhouseKeeperClusterMembership { - /// Keeper ID of the keeper being queried - pub queried_keeper: KeeperId, - /// Index of the last committed log entry from the leader's perspective - pub leader_committed_log_index: u64, - /// Keeper IDs of all keepers in the cluster - pub raft_config: BTreeSet, -} - -#[derive( - Clone, - Debug, - PartialEq, - Eq, - PartialOrd, - Ord, - Deserialize, - Serialize, - JsonSchema, -)] -#[serde(rename_all = "snake_case")] -/// Contains information about distributed ddl queries (ON CLUSTER clause) that were -/// executed on a cluster. -pub struct DistributedDdlQueue { - /// Query id - pub entry: String, - /// Version of the entry - pub entry_version: u64, - /// Host that initiated the DDL operation - pub initiator_host: String, - /// Port used by the initiator - pub initiator_port: u16, - /// Cluster name - pub cluster: String, - /// Query executed - pub query: String, - /// Settings used in the DDL operation - pub settings: BTreeMap, - /// Query created time - pub query_create_time: DateTime, - /// Hostname - pub host: Ipv6Addr, - /// Host Port - pub port: u16, - /// Status of the query - pub status: String, - /// Exception code - pub exception_code: u64, - /// Exception message - pub exception_text: String, - /// Query finish time - pub query_finish_time: DateTime, - /// Duration of query execution (in milliseconds) - pub query_duration_ms: u64, -} - -impl DistributedDdlQueue { - pub fn parse(log: &Logger, data: &[u8]) -> Result> { - let s = String::from_utf8_lossy(data); - info!( - log, - "Retrieved data from `system.distributed_ddl_queue`"; - "output" => ?s - ); - - let mut ddl = vec![]; - - for line in s.lines() { - let item: DistributedDdlQueue = serde_json::from_str(line)?; - ddl.push(item); - } - - Ok(ddl) - } -} - -#[inline] -fn default_interval() -> u64 { - 60 -} - -#[inline] -fn default_time_range() -> u64 { - 86400 -} - -#[inline] -fn default_timestamp_format() -> TimestampFormat { - TimestampFormat::Utc -} - -#[derive(Clone, Copy, Debug, Serialize, Deserialize, JsonSchema)] -#[serde(rename_all = "snake_case")] -/// Available metrics tables in the `system` database -pub enum SystemTable { - AsynchronousMetricLog, - MetricLog, -} - -impl fmt::Display for SystemTable { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - let table = match self { - SystemTable::MetricLog => "metric_log", - SystemTable::AsynchronousMetricLog => "asynchronous_metric_log", - }; - write!(f, "{}", table) - } -} - -#[derive(Clone, Copy, Debug, Serialize, Deserialize, JsonSchema)] -#[serde(rename_all = "snake_case")] -/// Which format should the timestamp be in. -pub enum TimestampFormat { - Utc, - UnixEpoch, -} - -impl fmt::Display for TimestampFormat { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - let table = match self { - TimestampFormat::Utc => "iso", - TimestampFormat::UnixEpoch => "unix_timestamp", - }; - write!(f, "{}", table) - } -} - -#[derive(Debug, Serialize, Deserialize, JsonSchema)] -#[serde(rename_all = "snake_case")] -pub struct MetricInfoPath { - /// Table to query in the `system` database - pub table: SystemTable, - /// Name of the metric to retrieve. - pub metric: String, -} - -#[derive(Debug, Serialize, Deserialize, JsonSchema)] -#[serde(rename_all = "snake_case")] -pub struct TimeSeriesSettingsQuery { - /// The interval to collect monitoring metrics in seconds. - /// Default is 60 seconds. - #[serde(default = "default_interval")] - pub interval: u64, - /// Range of time to collect monitoring metrics in seconds. - /// Default is 86400 seconds (24 hrs). - #[serde(default = "default_time_range")] - pub time_range: u64, - /// Format in which each timeseries timestamp will be in. - /// Default is UTC - #[serde(default = "default_timestamp_format")] - pub timestamp_format: TimestampFormat, -} - -/// Settings to specify which time series to retrieve. -#[derive(Debug, Serialize, Deserialize, JsonSchema)] -#[serde(rename_all = "snake_case")] -pub struct SystemTimeSeriesSettings { - /// Time series retrieval settings (time range and interval) - pub retrieval_settings: TimeSeriesSettingsQuery, - /// Database table and name of the metric to retrieve - pub metric_info: MetricInfoPath, -} - -impl SystemTimeSeriesSettings { - fn interval(&self) -> u64 { - self.retrieval_settings.interval - } - - fn time_range(&self) -> u64 { - self.retrieval_settings.time_range - } - - fn timestamp_format(&self) -> TimestampFormat { - self.retrieval_settings.timestamp_format - } - - fn metric_name(&self) -> &str { - &self.metric_info.metric - } - - fn table(&self) -> SystemTable { - self.metric_info.table - } - - // TODO: Use more aggregate functions than just avg? - pub fn query_avg(&self) -> String { - let interval = self.interval(); - let time_range = self.time_range(); - let metric_name = self.metric_name(); - let table = self.table(); - let ts_fmt = self.timestamp_format(); - - let avg_value = match table { - SystemTable::MetricLog => metric_name, - SystemTable::AsynchronousMetricLog => "value", - }; - - let mut query = format!( - "SELECT toStartOfInterval(event_time, INTERVAL {interval} SECOND) AS time, avg({avg_value}) AS value - FROM system.{table} - WHERE event_date >= toDate(now() - {time_range}) AND event_time >= now() - {time_range} - " - ); - - match table { - SystemTable::MetricLog => (), - SystemTable::AsynchronousMetricLog => query.push_str( - format!( - "AND metric = '{metric_name}' - " - ) - .as_str(), - ), - }; - - query.push_str( - format!( - "GROUP BY time - ORDER BY time WITH FILL STEP {interval} - FORMAT JSONEachRow - SETTINGS date_time_output_format = '{ts_fmt}'" - ) - .as_str(), - ); - query - } -} - -// Our OpenAPI generator does not allow for enums to be of different -// primitive types. Because Utc is a "string" in json, Unix cannot be an int. -// This is why we set it as a `String`. -#[derive(Debug, Display, Serialize, Deserialize, JsonSchema, PartialEq)] -#[serde(untagged)] -pub enum Timestamp { - Utc(DateTime), - Unix(String), -} - -/// Retrieved time series from the internal `system` database. -#[derive(Debug, Serialize, Deserialize, JsonSchema, PartialEq)] -#[serde(rename_all = "snake_case")] -pub struct SystemTimeSeries { - pub time: String, - pub value: f64, - // TODO: Would be really nice to have an enum with possible units (s, ms, bytes) - // Not sure if I can even add this, the system tables don't mention units at all. -} - -impl SystemTimeSeries { - pub fn parse(log: &Logger, data: &[u8]) -> Result> { - let s = String::from_utf8_lossy(data); - info!( - log, - "Retrieved data from `system` database"; - "output" => ?s - ); - - let mut m = vec![]; - - for line in s.lines() { - // serde_json deserialises f64 types with loss of precision at times. - // For example, in our tests some of the values to serialize have a - // fractional value of `.33333`, but once parsed, they become `.33331`. - // - // We do not require this level of precision, so we'll leave as is. - // Just noting that we are aware of this slight inaccuracy. - let item: SystemTimeSeries = serde_json::from_str(line)?; - m.push(item); - } - - Ok(m) - } -} - -#[cfg(test)] -mod tests { - use camino::Utf8PathBuf; - use camino_tempfile::Builder; - use chrono::{DateTime, Utc}; - use omicron_common::api::external::Generation; - use slog::{Drain, o}; - use slog_term::{FullFormat, PlainDecorator, TestStdoutWriter}; - use std::collections::BTreeMap; - use std::net::{Ipv4Addr, Ipv6Addr}; - use std::str::FromStr; - - use crate::{ - ClickhouseHost, DistributedDdlQueue, KeeperConf, - KeeperConfigurableSettings, KeeperId, KeeperServerInfo, - KeeperServerType, KeeperSettings, Lgif, LogLevel, RaftConfig, - RaftServerSettings, ServerConfigurableSettings, ServerId, - ServerSettings, SystemTimeSeries, - }; - - fn log() -> slog::Logger { - let decorator = PlainDecorator::new(TestStdoutWriter); - let drain = FullFormat::new(decorator).build().fuse(); - let drain = slog_async::Async::new(drain).build().fuse(); - slog::Logger::root(drain, o!()) - } - - #[test] - fn test_generate_keeper_config() { - let config_dir = Builder::new() - .tempdir_in( - Utf8PathBuf::try_from(std::env::temp_dir()).unwrap()) - .expect("Could not create directory for ClickHouse configuration generation test" - ); - - let keepers = vec![ - RaftServerSettings { - id: KeeperId(1), - host: ClickhouseHost::Ipv6( - Ipv6Addr::from_str("ff::01").unwrap(), - ), - }, - RaftServerSettings { - id: KeeperId(2), - host: ClickhouseHost::Ipv4( - Ipv4Addr::from_str("127.0.0.1").unwrap(), - ), - }, - RaftServerSettings { - id: KeeperId(3), - host: ClickhouseHost::DomainName("ohai.com".to_string()), - }, - ]; - - let settings = KeeperSettings::new( - Utf8PathBuf::from(config_dir.path()), - KeeperId(1), - keepers, - Utf8PathBuf::from_str("./").unwrap(), - Ipv6Addr::from_str("ff::08").unwrap(), - ); - - let config = KeeperConfigurableSettings { - generation: Generation::new(), - settings, - }; - - config.generate_xml_file().unwrap(); - - let expected_file = Utf8PathBuf::from_str("./testutils") - .unwrap() - .join("keeper_config.xml"); - let generated_file = - Utf8PathBuf::from(config_dir.path()).join("keeper_config.xml"); - let generated_content = std::fs::read_to_string(generated_file) - .expect("Failed to read from generated ClickHouse keeper file"); - - expectorate::assert_contents(expected_file, &generated_content); - } - - #[test] - fn test_generate_replica_config() { - let config_dir = Builder::new() - .tempdir_in( - Utf8PathBuf::try_from(std::env::temp_dir()).unwrap()) - .expect("Could not create directory for ClickHouse configuration generation test" - ); - - let keepers = vec![ - ClickhouseHost::Ipv6(Ipv6Addr::from_str("ff::01").unwrap()), - ClickhouseHost::Ipv4(Ipv4Addr::from_str("127.0.0.1").unwrap()), - ClickhouseHost::DomainName("we.dont.want.brackets.com".to_string()), - ]; - - let servers = vec![ - ClickhouseHost::Ipv6(Ipv6Addr::from_str("ff::09").unwrap()), - ClickhouseHost::DomainName("ohai.com".to_string()), - ]; - - let settings = ServerSettings::new( - Utf8PathBuf::from(config_dir.path()), - ServerId(1), - Utf8PathBuf::from_str("./").unwrap(), - Ipv6Addr::from_str("ff::08").unwrap(), - keepers, - servers, - ); - - let config = ServerConfigurableSettings { - settings, - generation: Generation::new(), - }; - config.generate_xml_file().unwrap(); - - let expected_file = Utf8PathBuf::from_str("./testutils") - .unwrap() - .join("replica-server-config.xml"); - let generated_file = Utf8PathBuf::from(config_dir.path()) - .join("replica-server-config.xml"); - let generated_content = std::fs::read_to_string(generated_file).expect( - "Failed to read from generated ClickHouse replica server file", - ); - - expectorate::assert_contents(expected_file, &generated_content); - } - - #[test] - fn test_full_lgif_parse_success() { - let log = log(); - let data = - "first_log_idx\t1\nfirst_log_term\t1\nlast_log_idx\t4386\nlast_log_term\t1\nlast_committed_log_idx\t4386\nleader_committed_log_idx\t4386\ntarget_committed_log_idx\t4386\nlast_snapshot_idx\t0\n\n" - .as_bytes(); - let lgif = Lgif::parse(&log, data).unwrap(); - - assert!(lgif.first_log_idx == 1); - assert!(lgif.first_log_term == 1); - assert!(lgif.last_log_idx == 4386); - assert!(lgif.last_log_term == 1); - assert!(lgif.last_committed_log_idx == 4386); - assert!(lgif.leader_committed_log_idx == 4386); - assert!(lgif.target_committed_log_idx == 4386); - assert!(lgif.last_snapshot_idx == 0); - } - - #[test] - fn test_missing_keys_lgif_parse_fail() { - let log = log(); - let data = - "first_log_idx\t1\nlast_log_idx\t4386\nlast_log_term\t1\nlast_committed_log_idx\t4386\nleader_committed_log_idx\t4386\ntarget_committed_log_idx\t4386\nlast_snapshot_idx\t0\n\n" - .as_bytes(); - let result = Lgif::parse(&log, data); - let error = result.unwrap_err(); - let root_cause = error.root_cause(); - - assert_eq!( - format!("{}", root_cause), - "Output from the Keeper differs to the expected output keys \ - Output: \"first_log_idx\\t1\\nlast_log_idx\\t4386\\nlast_log_term\\t1\\nlast_committed_log_idx\\t4386\\nleader_committed_log_idx\\t4386\\ntarget_committed_log_idx\\t4386\\nlast_snapshot_idx\\t0\\n\\n\" \ - Expected output keys: [\"first_log_idx\", \"first_log_term\", \"last_log_idx\", \"last_log_term\", \"last_committed_log_idx\", \"leader_committed_log_idx\", \"target_committed_log_idx\", \"last_snapshot_idx\"]" - ); - } - - #[test] - fn test_empty_value_lgif_parse_fail() { - let log = log(); - let data = - "first_log_idx\nfirst_log_term\t1\nlast_log_idx\t4386\nlast_log_term\t1\nlast_committed_log_idx\t4386\nleader_committed_log_idx\t4386\ntarget_committed_log_idx\t4386\nlast_snapshot_idx\t0\n\n" - .as_bytes(); - let result = Lgif::parse(&log, data); - let error = result.unwrap_err(); - let root_cause = error.root_cause(); - - assert_eq!( - format!("{}", root_cause), - "Command output has a line that does not contain a key-value pair: \"first_log_idx\"" - ); - } - - #[test] - fn test_non_u64_value_lgif_parse_fail() { - let log = log(); - let data = - "first_log_idx\t1\nfirst_log_term\tBOB\nlast_log_idx\t4386\nlast_log_term\t1\nlast_committed_log_idx\t4386\nleader_committed_log_idx\t4386\ntarget_committed_log_idx\t4386\nlast_snapshot_idx\t0\n\n" - .as_bytes(); - let result = Lgif::parse(&log, data); - let error = result.unwrap_err(); - let root_cause = error.root_cause(); - - assert_eq!( - format!("{}", root_cause), - "Unable to convert value \"BOB\" into u64 for key first_log_term: invalid digit found in string" - ); - } - - #[test] - fn test_non_existent_key_with_correct_value_lgif_parse_fail() { - let log = log(); - let data = - "first_log_idx\t1\nfirst_log\t1\nlast_log_idx\t4386\nlast_log_term\t1\nlast_committed_log_idx\t4386\nleader_committed_log_idx\t4386\ntarget_committed_log_idx\t4386\nlast_snapshot_idx\t0\n\n" - .as_bytes(); - let result = Lgif::parse(&log, data); - let error = result.unwrap_err(); - let root_cause = error.root_cause(); - - assert_eq!( - format!("{}", root_cause), - "Extracted key `\"first_log\"` from output differs from expected key `first_log_term`" - ); - } - - #[test] - fn test_additional_key_value_pairs_in_output_parse_fail() { - let log = log(); - let data = "first_log_idx\t1\nfirst_log_term\t1\nlast_log_idx\t4386\nlast_log_term\t1\nlast_committed_log_idx\t4386\nleader_committed_log_idx\t4386\ntarget_committed_log_idx\t4386\nlast_snapshot_idx\t0\nlast_snapshot_idx\t3\n\n" - .as_bytes(); - - let result = Lgif::parse(&log, data); - let error = result.unwrap_err(); - let root_cause = error.root_cause(); - - assert_eq!( - format!("{}", root_cause), - "Output from the Keeper differs to the expected output keys \ - Output: \"first_log_idx\\t1\\nfirst_log_term\\t1\\nlast_log_idx\\t4386\\nlast_log_term\\t1\\nlast_committed_log_idx\\t4386\\nleader_committed_log_idx\\t4386\\ntarget_committed_log_idx\\t4386\\nlast_snapshot_idx\\t0\\nlast_snapshot_idx\\t3\\n\\n\" \ - Expected output keys: [\"first_log_idx\", \"first_log_term\", \"last_log_idx\", \"last_log_term\", \"last_committed_log_idx\", \"leader_committed_log_idx\", \"target_committed_log_idx\", \"last_snapshot_idx\"]", - ); - } - - #[test] - fn test_empty_output_parse_fail() { - let log = log(); - let data = "".as_bytes(); - let result = Lgif::parse(&log, data); - let error = result.unwrap_err(); - let root_cause = error.root_cause(); - - assert_eq!( - format!("{}", root_cause), - "Output from the Keeper differs to the expected output keys \ - Output: \"\" \ - Expected output keys: [\"first_log_idx\", \"first_log_term\", \"last_log_idx\", \"last_log_term\", \"last_committed_log_idx\", \"leader_committed_log_idx\", \"target_committed_log_idx\", \"last_snapshot_idx\"]", - ); - } - - #[test] - fn test_full_raft_config_parse_success() { - let log = log(); - let data = - "server.1=::1:21001;participant;1\nserver.2=oxide.internal:21002;participant;1\nserver.3=127.0.0.1:21003;learner;0\n" - .as_bytes(); - let raft_config = RaftConfig::parse(&log, data).unwrap(); - - assert!(raft_config.keeper_servers.contains(&KeeperServerInfo { - server_id: KeeperId(1), - host: ClickhouseHost::Ipv6("::1".parse().unwrap()), - raft_port: 21001, - server_type: KeeperServerType::Participant, - priority: 1, - },)); - assert!(raft_config.keeper_servers.contains(&KeeperServerInfo { - server_id: KeeperId(2), - host: ClickhouseHost::DomainName("oxide.internal".to_string()), - raft_port: 21002, - server_type: KeeperServerType::Participant, - priority: 1, - },)); - assert!(raft_config.keeper_servers.contains(&KeeperServerInfo { - server_id: KeeperId(3), - host: ClickhouseHost::Ipv4("127.0.0.1".parse().unwrap()), - raft_port: 21003, - server_type: KeeperServerType::Learner, - priority: 0, - },)); - } - - #[test] - fn test_misshapen_id_raft_config_parse_fail() { - let log = log(); - let data = "serv.1=::1:21001;participant;1\n".as_bytes(); - let result = RaftConfig::parse(&log, data); - - let error = result.unwrap_err(); - let root_cause = error.root_cause(); - - assert_eq!( - format!("{}", root_cause), - "Output is not as expected. Server identifier: 'serv.1' Expected server identifier: 'server.{SERVER_ID}'", - ); - } - - #[test] - fn test_misshapen_port_raft_config_parse_fail() { - let log = log(); - let data = "server.1=::1:BOB;participant;1".as_bytes(); - let result = RaftConfig::parse(&log, data); - - let error = result.unwrap_err(); - let root_cause = error.root_cause(); - - assert_eq!( - format!("{}", root_cause), - "Unable to convert value \"BOB\" into u16: invalid digit found in string", - ); - } - - #[test] - fn test_empty_output_raft_config_parse_fail() { - let log = log(); - let data = "".as_bytes(); - let result = RaftConfig::parse(&log, data); - - let error = result.unwrap_err(); - let root_cause = error.root_cause(); - - assert_eq!(format!("{}", root_cause), "Cannot parse an empty response",); - } - - #[test] - fn test_missing_server_id_raft_config_parse_fail() { - let log = log(); - let data = "server.=::1:21001;participant;1".as_bytes(); - let result = RaftConfig::parse(&log, data); - - let error = result.unwrap_err(); - let root_cause = error.root_cause(); - - assert_eq!( - format!("{}", root_cause), - "Unable to convert value \"\" into u64: cannot parse integer from empty string", - ); - } - - #[test] - fn test_missing_address_raft_config_parse_fail() { - let log = log(); - let data = "server.1=:21001;participant;1".as_bytes(); - let result = RaftConfig::parse(&log, data); - - let error = result.unwrap_err(); - let root_cause = error.root_cause(); - - assert_eq!( - format!("{}", root_cause), - " is not a valid address or domain name", - ); - } - - #[test] - fn test_invalid_address_raft_config_parse_fail() { - let log = log(); - let data = "server.1=oxide.com:21001;participant;1".as_bytes(); - let result = RaftConfig::parse(&log, data); - - let error = result.unwrap_err(); - let root_cause = error.root_cause(); - - assert_eq!( - format!("{}", root_cause), - "oxide.com is not a valid address or domain name", - ); - } - - #[test] - fn test_missing_port_raft_config_parse_fail() { - let log = log(); - let data = "server.1=::1:;participant;1".as_bytes(); - let result = RaftConfig::parse(&log, data); - - let error = result.unwrap_err(); - let root_cause = error.root_cause(); - - assert_eq!( - format!("{}", root_cause), - "Unable to convert value \"\" into u16: cannot parse integer from empty string", - ); - } - - #[test] - fn test_missing_participant_raft_config_parse_fail() { - let log = log(); - let data = "server.1=::1:21001;1".as_bytes(); - let result = RaftConfig::parse(&log, data); - - let error = result.unwrap_err(); - let root_cause = error.root_cause(); - - assert_eq!( - format!("{}", root_cause), - "1 is not a valid keeper server type", - ); - - let data = "server.1=::1:21001;;1".as_bytes(); - let result = RaftConfig::parse(&log, data); - - let error = result.unwrap_err(); - let root_cause = error.root_cause(); - - assert_eq!( - format!("{}", root_cause), - " is not a valid keeper server type", - ); - } - - #[test] - fn test_misshapen_participant_raft_config_parse_fail() { - let log = log(); - let data = "server.1=::1:21001;runner;1\n".as_bytes(); - let result = RaftConfig::parse(&log, data); - - let error = result.unwrap_err(); - let root_cause = error.root_cause(); - - assert_eq!( - format!("{}", root_cause), - "runner is not a valid keeper server type", - ); - } - - #[test] - fn test_missing_priority_raft_config_parse_fail() { - let log = log(); - let data = "server.1=::1:21001;learner;\n".as_bytes(); - let result = RaftConfig::parse(&log, data); - - let error = result.unwrap_err(); - let root_cause = error.root_cause(); - - assert_eq!( - format!("{}", root_cause), - "Unable to convert value \"\" into u16: cannot parse integer from empty string", - ); - - let data = "server.1=::1:21001;learner\n".as_bytes(); - let result = RaftConfig::parse(&log, data); - - let error = result.unwrap_err(); - let root_cause = error.root_cause(); - - assert_eq!( - format!("{}", root_cause), - "Returned None while attempting to retrieve priority", - ); - } - - #[test] - fn test_misshapen_priority_raft_config_parse_fail() { - let log = log(); - let data = "server.1=::1:21001;learner;BOB\n".as_bytes(); - let result = RaftConfig::parse(&log, data); - - let error = result.unwrap_err(); - let root_cause = error.root_cause(); - - assert_eq!( - format!("{}", root_cause), - "Unable to convert value \"BOB\" into u16: invalid digit found in string", - ); - } - - #[test] - fn test_misshapen_raft_config_parse_fail() { - let log = log(); - let data = "=;;\n".as_bytes(); - let result = RaftConfig::parse(&log, data); - - let error = result.unwrap_err(); - let root_cause = error.root_cause(); - - assert_eq!( - format!("{}", root_cause), - "Output is not as expected. Server identifier: '' Expected server identifier: 'server.{SERVER_ID}'", - ); - } - - #[test] - fn test_full_keeper_conf_parse_success() { - let log = log(); - // This data contains the duplicated "max_requests_batch_size" that occurs in the - // real conf command output - let data = - "server_id=1 -enable_ipv6=true -tcp_port=20001 -four_letter_word_allow_list=conf,cons,crst,envi,ruok,srst,srvr,stat,wchs,dirs,mntr,isro,rcvr,apiv,csnp,lgif,rqld,rclc,clrs,ftfl -max_requests_batch_size=100 -min_session_timeout_ms=10000 -session_timeout_ms=30000 -operation_timeout_ms=10000 -dead_session_check_period_ms=500 -heart_beat_interval_ms=500 -election_timeout_lower_bound_ms=1000 -election_timeout_upper_bound_ms=2000 -reserved_log_items=100000 -snapshot_distance=100000 -auto_forwarding=true -shutdown_timeout=5000 -startup_timeout=180000 -raft_logs_level=trace -snapshots_to_keep=3 -rotate_log_storage_interval=100000 -stale_log_gap=10000 -fresh_log_gap=200 -max_requests_batch_size=100 -max_requests_batch_bytes_size=102400 -max_request_queue_size=100000 -max_requests_quick_batch_size=100 -quorum_reads=false -force_sync=true -compress_logs=true -compress_snapshots_with_zstd_format=true -configuration_change_tries_count=20 -raft_limits_reconnect_limit=50 -log_storage_path=./deployment/keeper-1/coordination/log -log_storage_disk=LocalLogDisk -snapshot_storage_path=./deployment/keeper-1/coordination/snapshots -snapshot_storage_disk=LocalSnapshotDisk -\n" - .as_bytes(); - let conf = KeeperConf::parse(&log, data).unwrap(); - - assert!(conf.server_id == KeeperId(1)); - assert!(conf.enable_ipv6); - assert!(conf.tcp_port == 20001); - assert!( - conf.four_letter_word_allow_list - == *"conf,cons,crst,envi,ruok,srst,srvr,stat,wchs,dirs,mntr,isro,rcvr,apiv,csnp,lgif,rqld,rclc,clrs,ftfl" - ); - assert!(conf.max_requests_batch_size == 100); - assert!(conf.min_session_timeout_ms == 10000); - assert!(conf.session_timeout_ms == 30000); - assert!(conf.operation_timeout_ms == 10000); - assert!(conf.dead_session_check_period_ms == 500); - assert!(conf.heart_beat_interval_ms == 500); - assert!(conf.election_timeout_lower_bound_ms == 1000); - assert!(conf.election_timeout_upper_bound_ms == 2000); - assert!(conf.reserved_log_items == 100000); - assert!(conf.snapshot_distance == 100000); - assert!(conf.auto_forwarding); - assert!(conf.shutdown_timeout == 5000); - assert!(conf.startup_timeout == 180000); - assert!(conf.raft_logs_level == LogLevel::Trace); - assert!(conf.snapshots_to_keep == 3); - assert!(conf.rotate_log_storage_interval == 100000); - assert!(conf.stale_log_gap == 10000); - assert!(conf.fresh_log_gap == 200); - assert!(conf.max_requests_batch_bytes_size == 102400); - assert!(conf.max_request_queue_size == 100000); - assert!(conf.max_requests_quick_batch_size == 100); - assert!(!conf.quorum_reads); - assert!(conf.force_sync); - assert!(conf.compress_logs); - assert!(conf.compress_snapshots_with_zstd_format); - assert!(conf.configuration_change_tries_count == 20); - assert!(conf.raft_limits_reconnect_limit == 50); - assert!( - conf.log_storage_path - == Utf8PathBuf::from_str( - "./deployment/keeper-1/coordination/log" - ) - .unwrap() - ); - assert!(conf.log_storage_disk == *"LocalLogDisk"); - assert!( - conf.snapshot_storage_path - == Utf8PathBuf::from_str( - "./deployment/keeper-1/coordination/snapshots" - ) - .unwrap() - ); - assert!(conf.snapshot_storage_disk == *"LocalSnapshotDisk") - } - - #[test] - fn test_missing_value_keeper_conf_parse_fail() { - let log = log(); - // This data contains the duplicated "max_requests_batch_size" that occurs in the - // real conf command output - let data = - "server_id=1 -enable_ipv6=true -tcp_port=20001 -four_letter_word_allow_list=conf,cons,crst,envi,ruok,srst,srvr,stat,wchs,dirs,mntr,isro,rcvr,apiv,csnp,lgif,rqld,rclc,clrs,ftfl -max_requests_batch_size=100 -min_session_timeout_ms=10000 -session_timeout_ms= -operation_timeout_ms=10000 -dead_session_check_period_ms=500 -heart_beat_interval_ms=500 -election_timeout_lower_bound_ms=1000 -election_timeout_upper_bound_ms=2000 -reserved_log_items=100000 -snapshot_distance=100000 -auto_forwarding=true -shutdown_timeout=5000 -startup_timeout=180000 -raft_logs_level=trace -snapshots_to_keep=3 -rotate_log_storage_interval=100000 -stale_log_gap=10000 -fresh_log_gap=200 -max_requests_batch_size=100 -max_requests_batch_bytes_size=102400 -max_request_queue_size=100000 -max_requests_quick_batch_size=100 -quorum_reads=false -force_sync=true -compress_logs=true -compress_snapshots_with_zstd_format=true -configuration_change_tries_count=20 -raft_limits_reconnect_limit=50 -log_storage_path=./deployment/keeper-1/coordination/log -log_storage_disk=LocalLogDisk -snapshot_storage_path=./deployment/keeper-1/coordination/snapshots -snapshot_storage_disk=LocalSnapshotDisk -\n" - .as_bytes(); - let result = KeeperConf::parse(&log, data); - let error = result.unwrap_err(); - let root_cause = error.root_cause(); - - assert_eq!( - format!("{}", root_cause), - "Unable to convert value into u64: cannot parse integer from empty string" - ); - } - - #[test] - fn test_malformed_output_keeper_conf_parse_fail() { - let log = log(); - // This data contains the duplicated "max_requests_batch_size" that occurs in the - // real conf command output - let data = - "server_id=1 -enable_ipv6=true -tcp_port=20001 -four_letter_word_allow_list=conf,cons,crst,envi,ruok,srst,srvr,stat,wchs,dirs,mntr,isro,rcvr,apiv,csnp,lgif,rqld,rclc,clrs,ftfl -max_requests_batch_size=100 -min_session_timeout_ms=10000 -session_timeout_ms -operation_timeout_ms=10000 -dead_session_check_period_ms=500 -heart_beat_interval_ms=500 -election_timeout_lower_bound_ms=1000 -election_timeout_upper_bound_ms=2000 -reserved_log_items=100000 -snapshot_distance=100000 -auto_forwarding=true -shutdown_timeout=5000 -startup_timeout=180000 -raft_logs_level=trace -snapshots_to_keep=3 -rotate_log_storage_interval=100000 -stale_log_gap=10000 -fresh_log_gap=200 -max_requests_batch_size=100 -max_requests_batch_bytes_size=102400 -max_request_queue_size=100000 -max_requests_quick_batch_size=100 -quorum_reads=false -force_sync=true -compress_logs=true -compress_snapshots_with_zstd_format=true -configuration_change_tries_count=20 -raft_limits_reconnect_limit=50 -log_storage_path=./deployment/keeper-1/coordination/log -log_storage_disk=LocalLogDisk -snapshot_storage_path=./deployment/keeper-1/coordination/snapshots -snapshot_storage_disk=LocalSnapshotDisk -\n" - .as_bytes(); - let result = KeeperConf::parse(&log, data); - let error = result.unwrap_err(); - let root_cause = error.root_cause(); - - assert_eq!( - format!("{}", root_cause), - "Command output has a line that does not contain a key-value pair: \"session_timeout_ms\"" - ); - } - - #[test] - fn test_missing_field_keeper_conf_parse_fail() { - let log = log(); - // This data contains the duplicated "max_requests_batch_size" that occurs in the - // real conf command output - let data = - "server_id=1 -enable_ipv6=true -tcp_port=20001 -four_letter_word_allow_list=conf,cons,crst,envi,ruok,srst,srvr,stat,wchs,dirs,mntr,isro,rcvr,apiv,csnp,lgif,rqld,rclc,clrs,ftfl -max_requests_batch_size=100 -min_session_timeout_ms=10000 -operation_timeout_ms=10000 -dead_session_check_period_ms=500 -heart_beat_interval_ms=500 -election_timeout_lower_bound_ms=1000 -election_timeout_upper_bound_ms=2000 -reserved_log_items=100000 -snapshot_distance=100000 -auto_forwarding=true -shutdown_timeout=5000 -startup_timeout=180000 -raft_logs_level=trace -snapshots_to_keep=3 -rotate_log_storage_interval=100000 -stale_log_gap=10000 -fresh_log_gap=200 -max_requests_batch_size=100 -max_requests_batch_bytes_size=102400 -max_request_queue_size=100000 -max_requests_quick_batch_size=100 -quorum_reads=false -force_sync=true -compress_logs=true -compress_snapshots_with_zstd_format=true -configuration_change_tries_count=20 -raft_limits_reconnect_limit=50 -log_storage_path=./deployment/keeper-1/coordination/log -log_storage_disk=LocalLogDisk -snapshot_storage_path=./deployment/keeper-1/coordination/snapshots -snapshot_storage_disk=LocalSnapshotDisk -\n" - .as_bytes(); - let result = KeeperConf::parse(&log, data); - let error = result.unwrap_err(); - let root_cause = error.root_cause(); - - assert_eq!( - format!("{}", root_cause), - "Output from the Keeper differs to the expected output keys \ - Output: \"server_id=1\\nenable_ipv6=true\\ntcp_port=20001\\nfour_letter_word_allow_list=conf,cons,crst,envi,ruok,srst,srvr,stat,wchs,dirs,mntr,isro,rcvr,apiv,csnp,lgif,rqld,rclc,clrs,ftfl\\nmax_requests_batch_size=100\\nmin_session_timeout_ms=10000\\noperation_timeout_ms=10000\\ndead_session_check_period_ms=500\\nheart_beat_interval_ms=500\\nelection_timeout_lower_bound_ms=1000\\nelection_timeout_upper_bound_ms=2000\\nreserved_log_items=100000\\nsnapshot_distance=100000\\nauto_forwarding=true\\nshutdown_timeout=5000\\nstartup_timeout=180000\\nraft_logs_level=trace\\nsnapshots_to_keep=3\\nrotate_log_storage_interval=100000\\nstale_log_gap=10000\\nfresh_log_gap=200\\nmax_requests_batch_size=100\\nmax_requests_batch_bytes_size=102400\\nmax_request_queue_size=100000\\nmax_requests_quick_batch_size=100\\nquorum_reads=false\\nforce_sync=true\\ncompress_logs=true\\ncompress_snapshots_with_zstd_format=true\\nconfiguration_change_tries_count=20\\nraft_limits_reconnect_limit=50\\nlog_storage_path=./deployment/keeper-1/coordination/log\\nlog_storage_disk=LocalLogDisk\\nsnapshot_storage_path=./deployment/keeper-1/coordination/snapshots\\nsnapshot_storage_disk=LocalSnapshotDisk\\n\\n\" \ - Expected output keys: [\"server_id\", \"enable_ipv6\", \"tcp_port\", \"four_letter_word_allow_list\", \"max_requests_batch_size\", \"min_session_timeout_ms\", \"session_timeout_ms\", \"operation_timeout_ms\", \"dead_session_check_period_ms\", \"heart_beat_interval_ms\", \"election_timeout_lower_bound_ms\", \"election_timeout_upper_bound_ms\", \"reserved_log_items\", \"snapshot_distance\", \"auto_forwarding\", \"shutdown_timeout\", \"startup_timeout\", \"raft_logs_level\", \"snapshots_to_keep\", \"rotate_log_storage_interval\", \"stale_log_gap\", \"fresh_log_gap\", \"max_requests_batch_size\", \"max_requests_batch_bytes_size\", \"max_request_queue_size\", \"max_requests_quick_batch_size\", \"quorum_reads\", \"force_sync\", \"compress_logs\", \"compress_snapshots_with_zstd_format\", \"configuration_change_tries_count\", \"raft_limits_reconnect_limit\", \"log_storage_path\", \"log_storage_disk\", \"snapshot_storage_path\", \"snapshot_storage_disk\"]" - ); - } - - #[test] - fn test_non_existent_key_keeper_conf_parse_fail() { - let log = log(); - // This data contains the duplicated "max_requests_batch_size" that occurs in the - // real conf command output - let data = - "server_id=1 -enable_ipv6=true -tcp_port=20001 -four_letter_word_allow_list=conf,cons,crst,envi,ruok,srst,srvr,stat,wchs,dirs,mntr,isro,rcvr,apiv,csnp,lgif,rqld,rclc,clrs,ftfl -max_requests_batch_size=100 -min_session_timeout_ms=10000 -session_timeout_fake=100 -operation_timeout_ms=10000 -dead_session_check_period_ms=500 -heart_beat_interval_ms=500 -election_timeout_lower_bound_ms=1000 -election_timeout_upper_bound_ms=2000 -reserved_log_items=100000 -snapshot_distance=100000 -auto_forwarding=true -shutdown_timeout=5000 -startup_timeout=180000 -raft_logs_level=trace -snapshots_to_keep=3 -rotate_log_storage_interval=100000 -stale_log_gap=10000 -fresh_log_gap=200 -max_requests_batch_size=100 -max_requests_batch_bytes_size=102400 -max_request_queue_size=100000 -max_requests_quick_batch_size=100 -quorum_reads=false -force_sync=true -compress_logs=true -compress_snapshots_with_zstd_format=true -configuration_change_tries_count=20 -raft_limits_reconnect_limit=50 -log_storage_path=./deployment/keeper-1/coordination/log -log_storage_disk=LocalLogDisk -snapshot_storage_path=./deployment/keeper-1/coordination/snapshots -snapshot_storage_disk=LocalSnapshotDisk -\n" - .as_bytes(); - let result = KeeperConf::parse(&log, data); - let error = result.unwrap_err(); - let root_cause = error.root_cause(); - - assert_eq!( - format!("{}", root_cause), - "Extracted key `\"session_timeout_fake\"` from output differs from expected key `session_timeout_ms`" - ); - } - - #[test] - fn test_distributed_ddl_queries_parse_success() { - let log = log(); - let data = - "{\"entry\":\"query-0000000000\",\"entry_version\":5,\"initiator_host\":\"ixchel\",\"initiator_port\":22001,\"cluster\":\"oximeter_cluster\",\"query\":\"CREATE DATABASE IF NOT EXISTS db1 UUID 'a49757e4-179e-42bd-866f-93ac43136e2d' ON CLUSTER oximeter_cluster\",\"settings\":{\"load_balancing\":\"random\"},\"query_create_time\":\"2024-11-01T16:16:45Z\",\"host\":\"::1\",\"port\":22001,\"status\":\"Finished\",\"exception_code\":0,\"exception_text\":\"\",\"query_finish_time\":\"2024-11-01T16:16:45Z\",\"query_duration_ms\":4} -{\"entry\":\"query-0000000000\",\"entry_version\":5,\"initiator_host\":\"ixchel\",\"initiator_port\":22001,\"cluster\":\"oximeter_cluster\",\"query\":\"CREATE DATABASE IF NOT EXISTS db1 UUID 'a49757e4-179e-42bd-866f-93ac43136e2d' ON CLUSTER oximeter_cluster\",\"settings\":{\"load_balancing\":\"random\"},\"query_create_time\":\"2024-11-01T16:16:45Z\",\"host\":\"::1\",\"port\":22002,\"status\":\"Finished\",\"exception_code\":0,\"exception_text\":\"\",\"query_finish_time\":\"2024-11-01T16:16:45Z\",\"query_duration_ms\":4} -" - .as_bytes(); - let ddl = DistributedDdlQueue::parse(&log, data).unwrap(); - - let expected_result = vec![ - DistributedDdlQueue{ - entry: "query-0000000000".to_string(), - entry_version: 5, - initiator_host: "ixchel".to_string(), - initiator_port: 22001, - cluster: "oximeter_cluster".to_string(), - query: "CREATE DATABASE IF NOT EXISTS db1 UUID 'a49757e4-179e-42bd-866f-93ac43136e2d' ON CLUSTER oximeter_cluster".to_string(), - settings: BTreeMap::from([ - ("load_balancing".to_string(), "random".to_string()), -]), - query_create_time: "2024-11-01T16:16:45Z".parse::>().unwrap(), - host: Ipv6Addr::from_str("::1").unwrap(), - port: 22001, - exception_code: 0, - exception_text: "".to_string(), - status: "Finished".to_string(), - query_finish_time: "2024-11-01T16:16:45Z".parse::>().unwrap(), - query_duration_ms: 4, - }, - DistributedDdlQueue{ - entry: "query-0000000000".to_string(), - entry_version: 5, - initiator_host: "ixchel".to_string(), - initiator_port: 22001, - cluster: "oximeter_cluster".to_string(), - query: "CREATE DATABASE IF NOT EXISTS db1 UUID 'a49757e4-179e-42bd-866f-93ac43136e2d' ON CLUSTER oximeter_cluster".to_string(), - settings: BTreeMap::from([ - ("load_balancing".to_string(), "random".to_string()), -]), - query_create_time: "2024-11-01T16:16:45Z".parse::>().unwrap(), - host: Ipv6Addr::from_str("::1").unwrap(), - port: 22002, - exception_code: 0, - exception_text: "".to_string(), - status: "Finished".to_string(), - query_finish_time: "2024-11-01T16:16:45Z".parse::>().unwrap(), - query_duration_ms: 4, - }, - ]; - assert!(ddl == expected_result); - } - - #[test] - fn test_empty_distributed_ddl_queries_parse_success() { - let log = log(); - let data = "".as_bytes(); - let ddl = DistributedDdlQueue::parse(&log, data).unwrap(); - - let expected_result = vec![]; - assert!(ddl == expected_result); - } - - #[test] - fn test_misshapen_distributed_ddl_queries_parse_fail() { - let log = log(); - let data = - "{\"entry\":\"query-0000000000\",\"initiator_host\":\"ixchel\",\"initiator_port\":22001,\"cluster\":\"oximeter_cluster\",\"query\":\"CREATE DATABASE IF NOT EXISTS db1 UUID 'a49757e4-179e-42bd-866f-93ac43136e2d' ON CLUSTER oximeter_cluster\",\"settings\":{\"load_balancing\":\"random\"},\"query_create_time\":\"2024-11-01T16:16:45Z\",\"host\":\"::1\",\"port\":22001,\"status\":\"Finished\",\"exception_code\":0,\"exception_text\":\"\",\"query_finish_time\":\"2024-11-01T16:16:45Z\",\"query_duration_ms\":4} -" -.as_bytes(); - let result = DistributedDdlQueue::parse(&log, data); - - let error = result.unwrap_err(); - let root_cause = error.root_cause(); - - assert_eq!( - format!("{}", root_cause), - "missing field `entry_version` at line 1 column 454", - ); - } - - #[test] - fn test_unix_epoch_system_timeseries_parse_success() { - let log = log(); - let data = "{\"time\":\"1732494720\",\"value\":110220450825.75238} -{\"time\":\"1732494840\",\"value\":110339992917.33333} -{\"time\":\"1732494960\",\"value\":110421854037.33333}\n" - .as_bytes(); - let timeseries = SystemTimeSeries::parse(&log, data).unwrap(); - - let expected = vec![ - SystemTimeSeries { - time: "1732494720".to_string(), - value: 110220450825.75238, - }, - SystemTimeSeries { - time: "1732494840".to_string(), - value: 110339992917.33331, - }, - SystemTimeSeries { - time: "1732494960".to_string(), - value: 110421854037.33331, - }, - ]; - - assert_eq!(timeseries, expected); - } - - #[test] - fn test_utc_system_timeseries_parse_success() { - let log = log(); - let data = - "{\"time\":\"2024-11-25T00:34:00Z\",\"value\":110220450825.75238} -{\"time\":\"2024-11-25T00:35:00Z\",\"value\":110339992917.33333} -{\"time\":\"2024-11-25T00:36:00Z\",\"value\":110421854037.33333}\n" - .as_bytes(); - let timeseries = SystemTimeSeries::parse(&log, data).unwrap(); - - let expected = vec![ - SystemTimeSeries { - time: "2024-11-25T00:34:00Z".to_string(), - value: 110220450825.75238, - }, - SystemTimeSeries { - time: "2024-11-25T00:35:00Z".to_string(), - value: 110339992917.33331, - }, - SystemTimeSeries { - time: "2024-11-25T00:36:00Z".to_string(), - value: 110421854037.33331, - }, - ]; - - assert_eq!(timeseries, expected); - } - - #[test] - fn test_misshapen_system_timeseries_parse_fail() { - let log = log(); - let data = "{\"bob\":\"1732494720\",\"value\":110220450825.75238}\n" - .as_bytes(); - let result = SystemTimeSeries::parse(&log, data); - - let error = result.unwrap_err(); - let root_cause = error.root_cause(); - - assert_eq!( - format!("{}", root_cause), - "missing field `time` at line 1 column 47", - ); - } - - #[test] - fn test_time_format_system_timeseries_parse_fail() { - let log = log(); - let data = "{\"time\":2024,\"value\":110220450825.75238}\n".as_bytes(); - let result = SystemTimeSeries::parse(&log, data); - - let error = result.unwrap_err(); - let root_cause = error.root_cause(); - - assert_eq!( - format!("{}", root_cause), - "invalid type: integer `2024`, expected a string at line 1 column 12", - ); - } -} diff --git a/clickhouse-admin/types/src/server.rs b/clickhouse-admin/types/src/server.rs new file mode 100644 index 00000000000..8edde786bd9 --- /dev/null +++ b/clickhouse-admin/types/src/server.rs @@ -0,0 +1,5 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +pub use clickhouse_admin_types_versions::latest::server::*; diff --git a/clickhouse-admin/types/versions/Cargo.toml b/clickhouse-admin/types/versions/Cargo.toml new file mode 100644 index 00000000000..f0e6cd1123d --- /dev/null +++ b/clickhouse-admin/types/versions/Cargo.toml @@ -0,0 +1,29 @@ +[package] +name = "clickhouse-admin-types-versions" +version = "0.1.0" +edition.workspace = true +license = "MPL-2.0" + +[lints] +workspace = true + +[dependencies] +anyhow.workspace = true +atomicwrites.workspace = true +camino.workspace = true +camino-tempfile.workspace = true +chrono.workspace = true +daft.workspace = true +derive_more.workspace = true +expectorate.workspace = true +itertools.workspace = true +omicron-common.workspace = true +omicron-workspace-hack.workspace = true +schemars.workspace = true +serde.workspace = true +serde_json.workspace = true +slog.workspace = true + +[dev-dependencies] +slog-async.workspace = true +slog-term.workspace = true diff --git a/clickhouse-admin/types/versions/src/impls/config.rs b/clickhouse-admin/types/versions/src/impls/config.rs new file mode 100644 index 00000000000..1d4b88507ff --- /dev/null +++ b/clickhouse-admin/types/versions/src/impls/config.rs @@ -0,0 +1,536 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! Functional code for config types. + +use crate::latest::config::{ + ClickhouseHost, KeeperConfig, KeeperConfigsForReplica, + KeeperCoordinationSettings, KeeperNodeConfig, LogConfig, LogLevel, Macros, + NodeType, RaftServerConfig, RaftServerSettings, RaftServers, RemoteServers, + ReplicaConfig, ServerNodeConfig, +}; +use crate::latest::keeper::KeeperId; +use crate::latest::server::ServerId; +use anyhow::{Error, bail}; +use camino::Utf8PathBuf; +use omicron_common::address::{ + CLICKHOUSE_HTTP_PORT, CLICKHOUSE_INTERSERVER_PORT, + CLICKHOUSE_KEEPER_RAFT_PORT, CLICKHOUSE_KEEPER_TCP_PORT, + CLICKHOUSE_TCP_PORT, +}; +use omicron_common::api::external::Generation; +use std::fmt::Display; +use std::net::Ipv6Addr; +use std::str::FromStr; + +pub const OXIMETER_CLUSTER: &str = "oximeter_cluster"; + +impl ReplicaConfig { + /// A new ClickHouse replica server configuration with default ports and directories + pub fn new( + logger: LogConfig, + macros: Macros, + listen_host: Ipv6Addr, + remote_servers: Vec, + keepers: Vec, + path: Utf8PathBuf, + generation: Generation, + ) -> Self { + let data_path = path.join("data"); + let remote_servers = RemoteServers::new(remote_servers); + let keepers = KeeperConfigsForReplica::new(keepers); + + Self { + logger, + macros, + listen_host, + http_port: CLICKHOUSE_HTTP_PORT, + tcp_port: CLICKHOUSE_TCP_PORT, + interserver_http_port: CLICKHOUSE_INTERSERVER_PORT, + remote_servers, + keepers, + data_path, + generation, + } + } + + pub fn to_xml(&self) -> String { + let ReplicaConfig { + logger, + macros, + listen_host, + http_port, + tcp_port, + interserver_http_port, + remote_servers, + keepers, + data_path, + generation, + } = self; + let logger = logger.to_xml(); + let cluster = macros.cluster.clone(); + let id = macros.replica; + let macros = macros.to_xml(); + let keepers = keepers.to_xml(); + let remote_servers = remote_servers.to_xml(); + let user_files_path = data_path.clone().join("user_files"); + let temp_files_path = data_path.clone().join("tmp"); + let format_schema_path = data_path.clone().join("format_schemas"); + let backup_path = data_path.clone().join("backup"); + format!( + " + +{logger} + {data_path} + + + + random + + + + + + + + + ::/0 + + default + default + + + + + + + 3600 + 0 + 0 + 0 + 0 + 0 + + + + + + system + query_log
+ Engine = MergeTree ORDER BY event_time TTL event_date + INTERVAL 7 DAY + 10000 + + + + system + metric_log
+ + Engine = MergeTree ORDER BY event_time TTL event_date + INTERVAL 30 DAY + 7500 + 1000 + 1048576 + 8192 + 524288 + false + + + + system + asynchronous_metric_log
+ + Engine = MergeTree ORDER BY event_time TTL event_date + INTERVAL 30 DAY + 7500 + 1000 + 1048576 + 8192 + 524288 + false + + + + system + part_log
+ + + {temp_files_path} + {user_files_path} + default + {format_schema_path} + {cluster}_{id} + {listen_host} + {http_port} + {tcp_port} + {interserver_http_port} + {listen_host} + + + + + 604800 + + + 60 + + + 1000 + + + + {backup_path} + + + + + 1.0 + + + + 1.0 + +{macros} +{remote_servers} +{keepers} + + +" + ) + } +} + +impl Macros { + /// A new macros configuration block with default cluster + pub fn new(replica: ServerId) -> Self { + Self { shard: 1, replica, cluster: OXIMETER_CLUSTER.to_string() } + } + + pub fn to_xml(&self) -> String { + let Macros { shard, replica, cluster } = self; + format!( + " + + {shard} + {replica} + {cluster} + " + ) + } +} + +impl RemoteServers { + /// A new remote_servers configuration block with default cluster + pub fn new(replicas: Vec) -> Self { + Self { + cluster: OXIMETER_CLUSTER.to_string(), + // TODO(https://github.com/oxidecomputer/omicron/issues/3823): secret handling TBD + secret: "some-unique-value".to_string(), + replicas, + } + } + + pub fn to_xml(&self) -> String { + let RemoteServers { cluster, secret, replicas } = self; + + let mut s = format!( + " + + <{cluster}> + + {secret} + + true" + ); + + for r in replicas { + let ServerNodeConfig { host, port } = r; + let sanitised_host = match host { + ClickhouseHost::Ipv6(h) => h.to_string(), + ClickhouseHost::Ipv4(h) => h.to_string(), + ClickhouseHost::DomainName(h) => h.to_string(), + }; + + s.push_str(&format!( + " + + {sanitised_host} + {port} + " + )); + } + + s.push_str(&format!( + " + + + + " + )); + + s + } +} + +impl KeeperConfigsForReplica { + pub fn new(nodes: Vec) -> Self { + Self { nodes } + } + + pub fn to_xml(&self) -> String { + let mut s = String::from(" "); + for node in &self.nodes { + let KeeperNodeConfig { host, port } = node; + + // ClickHouse servers have a small quirk, where when setting the + // keeper hosts as IPv6 addresses in the replica configuration file, + // they must be wrapped in square brackets. + // Otherwise, when running any query, a "Service not found" error + // appears. + // https://github.com/ClickHouse/ClickHouse/blob/a011990fd75628c63c7995c4f15475f1d4125d10/src/Coordination/KeeperStateManager.cpp#L149 + let sanitised_host = match host { + ClickhouseHost::Ipv6(h) => format!("[{h}]"), + ClickhouseHost::Ipv4(h) => h.to_string(), + ClickhouseHost::DomainName(h) => h.to_string(), + }; + + s.push_str(&format!( + " + + {sanitised_host} + {port} + ", + )); + } + s.push_str("\n "); + s + } +} + +impl FromStr for ClickhouseHost { + type Err = Error; + + fn from_str(s: &str) -> Result { + if let Ok(ipv6) = s.parse() { + Ok(ClickhouseHost::Ipv6(ipv6)) + } else if let Ok(ipv4) = s.parse() { + Ok(ClickhouseHost::Ipv4(ipv4)) + // Validating whether a string is a valid domain or + // not is a complex process that isn't necessary for + // this function. In the case of ClickhouseHost, we wil + // only be dealing with our in internal DNS service + // which provides names that always end with `.internal`. + } else if s.ends_with(".internal") { + Ok(ClickhouseHost::DomainName(s.to_string())) + } else { + bail!("{s} is not a valid address or domain name") + } + } +} + +impl KeeperNodeConfig { + /// A new ClickHouse keeper node configuration with default port + pub fn new(host: ClickhouseHost) -> Self { + let port = CLICKHOUSE_KEEPER_TCP_PORT; + Self { host, port } + } +} + +impl ServerNodeConfig { + /// A new ClickHouse replica node configuration with default port + pub fn new(host: ClickhouseHost) -> Self { + let port = CLICKHOUSE_TCP_PORT; + Self { host, port } + } +} + +impl LogConfig { + /// A new logger configuration with default directories + pub fn new(path: Utf8PathBuf, node_type: NodeType) -> Self { + let prefix = match node_type { + NodeType::Server => "clickhouse", + NodeType::Keeper => "clickhouse-keeper", + }; + + let logs: Utf8PathBuf = path.join("log"); + let log = logs.join(format!("{prefix}.log")); + let errorlog = logs.join(format!("{prefix}.err.log")); + + Self { level: LogLevel::default(), log, errorlog, size: 100, count: 1 } + } + + pub fn to_xml(&self) -> String { + let LogConfig { level, log, errorlog, size, count } = &self; + format!( + " + + {level} + {log} + {errorlog} + {size}M + {count} + +" + ) + } +} + +impl KeeperCoordinationSettings { + pub fn default() -> Self { + Self { + operation_timeout_ms: 10000, + session_timeout_ms: 30000, + raft_logs_level: LogLevel::Trace, + } + } +} + +impl RaftServers { + pub fn new(servers: Vec) -> Self { + Self { servers } + } + pub fn to_xml(&self) -> String { + let mut s = String::new(); + for server in &self.servers { + let RaftServerConfig { id, hostname, port } = server; + + let sanitised_host = match hostname { + ClickhouseHost::Ipv6(h) => h.to_string(), + ClickhouseHost::Ipv4(h) => h.to_string(), + ClickhouseHost::DomainName(h) => h.to_string(), + }; + + s.push_str(&format!( + " + + {id} + {sanitised_host} + {port} + + " + )); + } + + s + } +} + +impl RaftServerConfig { + pub fn new(settings: RaftServerSettings) -> Self { + Self { + id: settings.id, + hostname: settings.host, + port: CLICKHOUSE_KEEPER_RAFT_PORT, + } + } +} + +impl KeeperConfig { + /// A new ClickHouse keeper node configuration with default ports and directories + pub fn new( + logger: LogConfig, + listen_host: Ipv6Addr, + server_id: KeeperId, + datastore_path: Utf8PathBuf, + raft_config: RaftServers, + generation: Generation, + ) -> Self { + let coordination_path = datastore_path.join("coordination"); + let log_storage_path = coordination_path.join("log"); + let snapshot_storage_path = coordination_path.join("snapshots"); + let coordination_settings = KeeperCoordinationSettings::default(); + Self { + logger, + listen_host, + tcp_port: CLICKHOUSE_KEEPER_TCP_PORT, + server_id, + log_storage_path, + snapshot_storage_path, + coordination_settings, + raft_config, + datastore_path, + generation, + } + } + + pub fn to_xml(&self) -> String { + let KeeperConfig { + logger, + listen_host, + tcp_port, + server_id, + log_storage_path, + snapshot_storage_path, + coordination_settings, + raft_config, + datastore_path, + generation, + } = self; + let logger = logger.to_xml(); + let KeeperCoordinationSettings { + operation_timeout_ms, + session_timeout_ms, + raft_logs_level, + } = coordination_settings; + let raft_servers = raft_config.to_xml(); + format!( + " + +{logger} + {listen_host} + {datastore_path} + + false + {tcp_port} + {server_id} + {log_storage_path} + {snapshot_storage_path} + + {operation_timeout_ms} + {session_timeout_ms} + {raft_logs_level} + + +{raft_servers} + + + + +" + ) + } +} + +impl LogLevel { + pub(crate) fn default() -> Self { + LogLevel::Trace + } +} + +impl Display for LogLevel { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let s = match self { + LogLevel::Trace => "trace", + LogLevel::Debug => "debug", + }; + write!(f, "{s}") + } +} + +impl FromStr for LogLevel { + type Err = Error; + + fn from_str(s: &str) -> Result { + if s == "trace" { + Ok(LogLevel::Trace) + } else if s == "debug" { + Ok(LogLevel::Debug) + } else { + bail!("{s} is not a valid log level") + } + } +} diff --git a/clickhouse-admin/types/versions/src/impls/keeper.rs b/clickhouse-admin/types/versions/src/impls/keeper.rs new file mode 100644 index 00000000000..34bb618f54a --- /dev/null +++ b/clickhouse-admin/types/versions/src/impls/keeper.rs @@ -0,0 +1,1362 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! Functional code for keeper types. + +use crate::latest::config::{ClickhouseHost, LogLevel}; +use crate::latest::config::{ + KeeperConfig, LogConfig, NodeType, RaftServerConfig, RaftServerSettings, + RaftServers, +}; +use crate::latest::keeper::{ + KeeperConf, KeeperConfigurableSettings, KeeperId, KeeperServerInfo, + KeeperServerType, KeeperSettings, Lgif, RaftConfig, +}; +use anyhow::{Context, Result, bail}; +use atomicwrites::AtomicFile; +use camino::Utf8PathBuf; +use itertools::Itertools; +use omicron_common::api::external::Generation; +use slog::{Logger, info}; +use std::fs::create_dir; +use std::io::{ErrorKind, Write}; +use std::net::Ipv6Addr; +use std::str::FromStr; + +impl KeeperConfigurableSettings { + /// Generate a configuration file for a keeper node + pub fn generate_xml_file(&self) -> Result { + let logger = LogConfig::new( + self.settings.datastore_path.clone(), + NodeType::Keeper, + ); + + let raft_servers = self + .settings + .raft_servers + .iter() + .map(|settings| RaftServerConfig::new(settings.clone())) + .collect(); + let raft_config = RaftServers::new(raft_servers); + + let config = KeeperConfig::new( + logger, + self.listen_addr(), + self.id(), + self.datastore_path(), + raft_config, + self.generation(), + ); + + match create_dir(self.settings.config_dir.clone()) { + Ok(_) => (), + Err(e) if e.kind() == ErrorKind::AlreadyExists => (), + Err(e) => return Err(e.into()), + }; + + let path = self.settings.config_dir.join("keeper_config.xml"); + AtomicFile::new( + path.clone(), + atomicwrites::OverwriteBehavior::AllowOverwrite, + ) + .write(|f| f.write_all(config.to_xml().as_bytes())) + .with_context(|| format!("failed to write to `{}`", path))?; + + Ok(config) + } + + pub fn generation(&self) -> Generation { + self.generation + } + + fn listen_addr(&self) -> Ipv6Addr { + self.settings.listen_addr + } + + fn id(&self) -> KeeperId { + self.settings.id + } + + fn datastore_path(&self) -> Utf8PathBuf { + self.settings.datastore_path.clone() + } +} + +impl KeeperSettings { + pub fn new( + config_dir: Utf8PathBuf, + id: KeeperId, + raft_servers: Vec, + datastore_path: Utf8PathBuf, + listen_addr: Ipv6Addr, + ) -> Self { + Self { config_dir, id, raft_servers, datastore_path, listen_addr } + } +} + +impl Lgif { + pub fn parse(log: &Logger, data: &[u8]) -> Result { + // The reponse we get from running `clickhouse keeper-client -h {HOST} --q lgif` + // isn't in any known format (e.g. JSON), but rather a series of lines with key-value + // pairs separated by a tab: + // + // ```console + // $ clickhouse keeper-client -h localhost -p 20001 --q lgif + // first_log_idx 1 + // first_log_term 1 + // last_log_idx 10889 + // last_log_term 20 + // last_committed_log_idx 10889 + // leader_committed_log_idx 10889 + // target_committed_log_idx 10889 + // last_snapshot_idx 9465 + // ``` + let s = String::from_utf8_lossy(data); + info!( + log, + "Retrieved data from `clickhouse keeper-client --q lgif`"; + "output" => ?s + ); + + let expected = Lgif::expected_keys(); + + // Verify the output contains the same amount of lines as the expected keys. + // This will ensure we catch any new key-value pairs appended to the lgif output. + let lines = s.trim().lines(); + if expected.len() != lines.count() { + bail!( + "Output from the Keeper differs to the expected output keys \ + Output: {s:?} \ + Expected output keys: {expected:?}" + ); + } + + let mut vals: Vec = Vec::new(); + for (line, expected_key) in s.lines().zip(expected.clone()) { + let mut split = line.split('\t'); + let Some(key) = split.next() else { + bail!("Returned None while attempting to retrieve key"); + }; + if key != expected_key { + bail!( + "Extracted key `{key:?}` from output differs from expected key `{expected_key}`" + ); + } + let Some(val) = split.next() else { + bail!( + "Command output has a line that does not contain a key-value pair: {key:?}" + ); + }; + let val = match u64::from_str(val) { + Ok(v) => v, + Err(e) => bail!( + "Unable to convert value {val:?} into u64 for key {key}: {e}" + ), + }; + vals.push(val); + } + + let mut iter = vals.into_iter(); + Ok(Lgif { + first_log_idx: iter.next().unwrap(), + first_log_term: iter.next().unwrap(), + last_log_idx: iter.next().unwrap(), + last_log_term: iter.next().unwrap(), + last_committed_log_idx: iter.next().unwrap(), + leader_committed_log_idx: iter.next().unwrap(), + target_committed_log_idx: iter.next().unwrap(), + last_snapshot_idx: iter.next().unwrap(), + }) + } + + fn expected_keys() -> Vec<&'static str> { + vec![ + "first_log_idx", + "first_log_term", + "last_log_idx", + "last_log_term", + "last_committed_log_idx", + "leader_committed_log_idx", + "target_committed_log_idx", + "last_snapshot_idx", + ] + } +} + +impl FromStr for KeeperServerType { + type Err = anyhow::Error; + + fn from_str(s: &str) -> Result { + match s { + "participant" => Ok(KeeperServerType::Participant), + "learner" => Ok(KeeperServerType::Learner), + _ => bail!("{s} is not a valid keeper server type"), + } + } +} + +impl RaftConfig { + pub fn parse(log: &Logger, data: &[u8]) -> Result { + // The response we get from `$ clickhouse keeper-client -h {HOST} --q 'get /keeper/config' + // is a format unique to ClickHouse, where the data for each server is separated by a colon + // + // ```console + // $ clickhouse keeper-client -h localhost --q 'get /keeper/config' + // server.1=::1:21001;participant;1 + // server.2=::1:21002;participant;1 + // server.3=::1:21003;participant;1 + //``` + let s = String::from_utf8_lossy(data); + info!( + log, + "Retrieved data from `clickhouse keeper-client --q 'get /keeper/config'`"; + "output" => ?s + ); + + if s.is_empty() { + bail!("Cannot parse an empty response"); + } + + let mut keeper_servers = std::collections::BTreeSet::new(); + for line in s.lines() { + let mut split = line.split('='); + let Some(server) = split.next() else { + bail!( + "Returned None while attempting to retrieve raft configuration" + ); + }; + + // Retrieve server ID + let mut split_server = server.split("."); + let Some(s) = split_server.next() else { + bail!( + "Returned None while attempting to retrieve server identifier" + ) + }; + if s != "server" { + bail!( + "Output is not as expected. \ + Server identifier: '{server}' \ + Expected server identifier: 'server.{{SERVER_ID}}'" + ) + }; + let Some(id) = split_server.next() else { + bail!("Returned None while attempting to retrieve server ID"); + }; + let u64_id = match u64::from_str(id) { + Ok(v) => v, + Err(e) => bail!("Unable to convert value {id:?} into u64: {e}"), + }; + let server_id = KeeperId(u64_id); + + // Retrieve server information + let Some(info) = split.next() else { + bail!("Returned None while attempting to retrieve server info"); + }; + let mut split_info = info.split(";"); + + // Retrieve port + let Some(address) = split_info.next() else { + bail!("Returned None while attempting to retrieve address") + }; + let Some(port) = address.split(':').next_back() else { + bail!("A port could not be extracted from {address}") + }; + let raft_port = match u16::from_str(port) { + Ok(v) => v, + Err(e) => { + bail!("Unable to convert value {port:?} into u16: {e}") + } + }; + + // Retrieve host + let p = format!(":{}", port); + let Some(h) = address.split(&p).next() else { + bail!( + "A host could not be extracted from {address}. Missing port {port}" + ) + }; + // The ouput we get from running the clickhouse keeper-client + // command does not add square brackets to an IPv6 address + // that cointains a port: server.1=::1:21001;participant;1 + // Because of this, we can parse `h` directly into an Ipv6Addr + let host = ClickhouseHost::from_str(h)?; + + // Retrieve server_type + let Some(s_type) = split_info.next() else { + bail!("Returned None while attempting to retrieve server type") + }; + let server_type = KeeperServerType::from_str(s_type)?; + + // Retrieve priority + let Some(s_priority) = split_info.next() else { + bail!("Returned None while attempting to retrieve priority") + }; + let priority = match u16::from_str(s_priority) { + Ok(v) => v, + Err(e) => { + bail!( + "Unable to convert value {s_priority:?} into u16: {e}" + ) + } + }; + + keeper_servers.insert(KeeperServerInfo { + server_id, + host, + raft_port, + server_type, + priority, + }); + } + + Ok(RaftConfig { keeper_servers }) + } +} + +impl KeeperConf { + pub fn parse(log: &Logger, data: &[u8]) -> Result { + // Like Lgif, the reponse we get from running `clickhouse keeper-client -h {HOST} --q conf` + // isn't in any known format (e.g. JSON), but rather a series of lines with key-value + // pairs separated by a tab. + let s = String::from_utf8_lossy(data); + info!( + log, + "Retrieved data from `clickhouse keeper-client --q conf`"; + "output" => ?s + ); + + let expected = KeeperConf::expected_keys(); + + // Verify the output contains the same amount of lines as the expected keys. + // This will ensure we catch any new key-value pairs appended to the lgif output. + let lines = s.trim().lines(); + if expected.len() != lines.count() { + bail!( + "Output from the Keeper differs to the expected output keys \ + Output: {s:?} \ + Expected output keys: {expected:?}" + ); + } + + let mut vals: Vec<&str> = Vec::new(); + // The output from the `conf` command contains the `max_requests_batch_size` field + // twice. We make sure to only read it once. + for (line, expected_key) in s.lines().zip(expected.clone()).unique() { + let mut split = line.split('='); + let Some(key) = split.next() else { + bail!("Returned None while attempting to retrieve key"); + }; + if key != expected_key { + bail!( + "Extracted key `{key:?}` from output differs from expected key `{expected_key}`" + ); + } + let Some(val) = split.next() else { + bail!( + "Command output has a line that does not contain a key-value pair: {key:?}" + ); + }; + vals.push(val); + } + + let mut iter = vals.into_iter(); + let server_id = match u64::from_str(iter.next().unwrap()) { + Ok(v) => KeeperId(v), + Err(e) => bail!("Unable to convert value into u64: {e}"), + }; + + let enable_ipv6 = match bool::from_str(iter.next().unwrap()) { + Ok(v) => v, + Err(e) => bail!("Unable to convert value into bool: {e}"), + }; + + let tcp_port = match u16::from_str(iter.next().unwrap()) { + Ok(v) => v, + Err(e) => bail!("Unable to convert value into u16: {e}"), + }; + + let four_letter_word_allow_list = iter.next().unwrap().to_string(); + + let max_requests_batch_size = match u64::from_str(iter.next().unwrap()) + { + Ok(v) => v, + Err(e) => bail!("Unable to convert value into u64: {e}"), + }; + + let min_session_timeout_ms = match u64::from_str(iter.next().unwrap()) { + Ok(v) => v, + Err(e) => bail!("Unable to convert value into u64: {e}"), + }; + + let session_timeout_ms = match u64::from_str(iter.next().unwrap()) { + Ok(v) => v, + Err(e) => bail!("Unable to convert value into u64: {e}"), + }; + + let operation_timeout_ms = match u64::from_str(iter.next().unwrap()) { + Ok(v) => v, + Err(e) => bail!("Unable to convert value into u64: {e}"), + }; + + let dead_session_check_period_ms = + match u64::from_str(iter.next().unwrap()) { + Ok(v) => v, + Err(e) => bail!("Unable to convert value into u64: {e}"), + }; + + let heart_beat_interval_ms = match u64::from_str(iter.next().unwrap()) { + Ok(v) => v, + Err(e) => bail!("Unable to convert value into u64: {e}"), + }; + + let election_timeout_lower_bound_ms = + match u64::from_str(iter.next().unwrap()) { + Ok(v) => v, + Err(e) => bail!("Unable to convert value into u64: {e}"), + }; + + let election_timeout_upper_bound_ms = + match u64::from_str(iter.next().unwrap()) { + Ok(v) => v, + Err(e) => bail!("Unable to convert value into u64: {e}"), + }; + + let reserved_log_items = match u64::from_str(iter.next().unwrap()) { + Ok(v) => v, + Err(e) => bail!("Unable to convert value into u64: {e}"), + }; + + let snapshot_distance = match u64::from_str(iter.next().unwrap()) { + Ok(v) => v, + Err(e) => bail!("Unable to convert value into u64: {e}"), + }; + + let auto_forwarding = match bool::from_str(iter.next().unwrap()) { + Ok(v) => v, + Err(e) => bail!("Unable to convert value into bool: {e}"), + }; + + let shutdown_timeout = match u64::from_str(iter.next().unwrap()) { + Ok(v) => v, + Err(e) => bail!("Unable to convert value into u64 {e}"), + }; + + let startup_timeout = match u64::from_str(iter.next().unwrap()) { + Ok(v) => v, + Err(e) => bail!("Unable to convert value into u64: {e}"), + }; + + let raft_logs_level = match LogLevel::from_str(iter.next().unwrap()) { + Ok(v) => v, + Err(e) => bail!("Unable to convert value into LogLevel: {e}"), + }; + + let snapshots_to_keep = match u64::from_str(iter.next().unwrap()) { + Ok(v) => v, + Err(e) => bail!("Unable to convert value into u64: {e}"), + }; + + let rotate_log_storage_interval = + match u64::from_str(iter.next().unwrap()) { + Ok(v) => v, + Err(e) => bail!("Unable to convert value into u64 {e}"), + }; + + let stale_log_gap = match u64::from_str(iter.next().unwrap()) { + Ok(v) => v, + Err(e) => bail!("Unable to convert value into u64: {e}"), + }; + + let fresh_log_gap = match u64::from_str(iter.next().unwrap()) { + Ok(v) => v, + Err(e) => bail!("Unable to convert value into u64: {e}"), + }; + + let max_requests_batch_bytes_size = + match u64::from_str(iter.next().unwrap()) { + Ok(v) => v, + Err(e) => bail!("Unable to convert value into u64 {e}"), + }; + + let max_request_queue_size = match u64::from_str(iter.next().unwrap()) { + Ok(v) => v, + Err(e) => bail!("Unable to convert value into u64 {e}"), + }; + + let max_requests_quick_batch_size = + match u64::from_str(iter.next().unwrap()) { + Ok(v) => v, + Err(e) => bail!("Unable to convert value into u64 {e}"), + }; + + let quorum_reads = match bool::from_str(iter.next().unwrap()) { + Ok(v) => v, + Err(e) => bail!("Unable to convert value into bool: {e}"), + }; + + let force_sync = match bool::from_str(iter.next().unwrap()) { + Ok(v) => v, + Err(e) => bail!("Unable to convert value into bool: {e}"), + }; + + let compress_logs = match bool::from_str(iter.next().unwrap()) { + Ok(v) => v, + Err(e) => bail!("Unable to convert value into bool: {e}"), + }; + + let compress_snapshots_with_zstd_format = + match bool::from_str(iter.next().unwrap()) { + Ok(v) => v, + Err(e) => bail!("Unable to convert value into bool: {e}"), + }; + + let configuration_change_tries_count = + match u64::from_str(iter.next().unwrap()) { + Ok(v) => v, + Err(e) => bail!("Unable to convert value into u64: {e}"), + }; + + let raft_limits_reconnect_limit = + match u64::from_str(iter.next().unwrap()) { + Ok(v) => v, + Err(e) => bail!("Unable to convert value into u64: {e}"), + }; + + let log_storage_path = match Utf8PathBuf::from_str(iter.next().unwrap()) + { + Ok(v) => v, + Err(e) => bail!("Unable to convert value into Utf8PathBuf: {e}"), + }; + + let log_storage_disk = iter.next().unwrap().to_string(); + + let snapshot_storage_path = + match Utf8PathBuf::from_str(iter.next().unwrap()) { + Ok(v) => v, + Err(e) => { + bail!("Unable to convert value into Utf8PathBuf: {e}") + } + }; + + let snapshot_storage_disk = iter.next().unwrap().to_string(); + + Ok(Self { + server_id, + enable_ipv6, + tcp_port, + four_letter_word_allow_list, + max_requests_batch_size, + min_session_timeout_ms, + session_timeout_ms, + operation_timeout_ms, + dead_session_check_period_ms, + heart_beat_interval_ms, + election_timeout_lower_bound_ms, + election_timeout_upper_bound_ms, + reserved_log_items, + snapshot_distance, + auto_forwarding, + shutdown_timeout, + startup_timeout, + raft_logs_level, + snapshots_to_keep, + rotate_log_storage_interval, + stale_log_gap, + fresh_log_gap, + max_requests_batch_bytes_size, + max_request_queue_size, + max_requests_quick_batch_size, + quorum_reads, + force_sync, + compress_logs, + compress_snapshots_with_zstd_format, + configuration_change_tries_count, + raft_limits_reconnect_limit, + log_storage_path, + log_storage_disk, + snapshot_storage_path, + snapshot_storage_disk, + }) + } + + fn expected_keys() -> Vec<&'static str> { + vec![ + "server_id", + "enable_ipv6", + "tcp_port", + "four_letter_word_allow_list", + "max_requests_batch_size", + "min_session_timeout_ms", + "session_timeout_ms", + "operation_timeout_ms", + "dead_session_check_period_ms", + "heart_beat_interval_ms", + "election_timeout_lower_bound_ms", + "election_timeout_upper_bound_ms", + "reserved_log_items", + "snapshot_distance", + "auto_forwarding", + "shutdown_timeout", + "startup_timeout", + "raft_logs_level", + "snapshots_to_keep", + "rotate_log_storage_interval", + "stale_log_gap", + "fresh_log_gap", + "max_requests_batch_size", + "max_requests_batch_bytes_size", + "max_request_queue_size", + "max_requests_quick_batch_size", + "quorum_reads", + "force_sync", + "compress_logs", + "compress_snapshots_with_zstd_format", + "configuration_change_tries_count", + "raft_limits_reconnect_limit", + "log_storage_path", + "log_storage_disk", + "snapshot_storage_path", + "snapshot_storage_disk", + ] + } +} + +#[cfg(test)] +mod tests { + use crate::latest::config::{ClickhouseHost, LogLevel, RaftServerSettings}; + use crate::latest::keeper::{ + KeeperConf, KeeperConfigurableSettings, KeeperId, KeeperServerInfo, + KeeperServerType, KeeperSettings, Lgif, RaftConfig, + }; + use camino::Utf8PathBuf; + use camino_tempfile::Builder; + use omicron_common::api::external::Generation; + use slog::{Drain, o}; + use slog_term::{FullFormat, PlainDecorator, TestStdoutWriter}; + use std::net::{Ipv4Addr, Ipv6Addr}; + use std::str::FromStr; + + fn log() -> slog::Logger { + let decorator = PlainDecorator::new(TestStdoutWriter); + let drain = FullFormat::new(decorator).build().fuse(); + let drain = slog_async::Async::new(drain).build().fuse(); + slog::Logger::root(drain, o!()) + } + + #[test] + fn test_generate_keeper_config() { + let config_dir = Builder::new() + .tempdir_in( + Utf8PathBuf::try_from(std::env::temp_dir()).unwrap()) + .expect("Could not create directory for ClickHouse configuration generation test" + ); + + let keepers = vec![ + RaftServerSettings { + id: KeeperId(1), + host: ClickhouseHost::Ipv6( + Ipv6Addr::from_str("ff::01").unwrap(), + ), + }, + RaftServerSettings { + id: KeeperId(2), + host: ClickhouseHost::Ipv4( + Ipv4Addr::from_str("127.0.0.1").unwrap(), + ), + }, + RaftServerSettings { + id: KeeperId(3), + host: ClickhouseHost::DomainName("ohai.com".to_string()), + }, + ]; + + let settings = KeeperSettings::new( + Utf8PathBuf::from(config_dir.path()), + KeeperId(1), + keepers, + Utf8PathBuf::from_str("./").unwrap(), + Ipv6Addr::from_str("ff::08").unwrap(), + ); + + let config = KeeperConfigurableSettings { + generation: Generation::new(), + settings, + }; + + config.generate_xml_file().unwrap(); + + let expected_file = + Utf8PathBuf::from("../testutils").join("keeper_config.xml"); + let generated_file = + Utf8PathBuf::from(config_dir.path()).join("keeper_config.xml"); + let generated_content = std::fs::read_to_string(generated_file) + .expect("Failed to read from generated ClickHouse keeper file"); + + expectorate::assert_contents(expected_file, &generated_content); + } + + #[test] + fn test_full_lgif_parse_success() { + let log = log(); + let data = + "first_log_idx\t1\nfirst_log_term\t1\nlast_log_idx\t4386\nlast_log_term\t1\nlast_committed_log_idx\t4386\nleader_committed_log_idx\t4386\ntarget_committed_log_idx\t4386\nlast_snapshot_idx\t0\n\n" + .as_bytes(); + let lgif = Lgif::parse(&log, data).unwrap(); + + assert!(lgif.first_log_idx == 1); + assert!(lgif.first_log_term == 1); + assert!(lgif.last_log_idx == 4386); + assert!(lgif.last_log_term == 1); + assert!(lgif.last_committed_log_idx == 4386); + assert!(lgif.leader_committed_log_idx == 4386); + assert!(lgif.target_committed_log_idx == 4386); + assert!(lgif.last_snapshot_idx == 0); + } + + #[test] + fn test_missing_keys_lgif_parse_fail() { + let log = log(); + let data = + "first_log_idx\t1\nlast_log_idx\t4386\nlast_log_term\t1\nlast_committed_log_idx\t4386\nleader_committed_log_idx\t4386\ntarget_committed_log_idx\t4386\nlast_snapshot_idx\t0\n\n" + .as_bytes(); + let result = Lgif::parse(&log, data); + let error = result.unwrap_err(); + let root_cause = error.root_cause(); + + assert_eq!( + format!("{}", root_cause), + "Output from the Keeper differs to the expected output keys \ + Output: \"first_log_idx\\t1\\nlast_log_idx\\t4386\\nlast_log_term\\t1\\nlast_committed_log_idx\\t4386\\nleader_committed_log_idx\\t4386\\ntarget_committed_log_idx\\t4386\\nlast_snapshot_idx\\t0\\n\\n\" \ + Expected output keys: [\"first_log_idx\", \"first_log_term\", \"last_log_idx\", \"last_log_term\", \"last_committed_log_idx\", \"leader_committed_log_idx\", \"target_committed_log_idx\", \"last_snapshot_idx\"]" + ); + } + + #[test] + fn test_empty_value_lgif_parse_fail() { + let log = log(); + let data = + "first_log_idx\nfirst_log_term\t1\nlast_log_idx\t4386\nlast_log_term\t1\nlast_committed_log_idx\t4386\nleader_committed_log_idx\t4386\ntarget_committed_log_idx\t4386\nlast_snapshot_idx\t0\n\n" + .as_bytes(); + let result = Lgif::parse(&log, data); + let error = result.unwrap_err(); + let root_cause = error.root_cause(); + + assert_eq!( + format!("{}", root_cause), + "Command output has a line that does not contain a key-value pair: \"first_log_idx\"" + ); + } + + #[test] + fn test_non_u64_value_lgif_parse_fail() { + let log = log(); + let data = + "first_log_idx\t1\nfirst_log_term\tBOB\nlast_log_idx\t4386\nlast_log_term\t1\nlast_committed_log_idx\t4386\nleader_committed_log_idx\t4386\ntarget_committed_log_idx\t4386\nlast_snapshot_idx\t0\n\n" + .as_bytes(); + let result = Lgif::parse(&log, data); + let error = result.unwrap_err(); + let root_cause = error.root_cause(); + + assert_eq!( + format!("{}", root_cause), + "Unable to convert value \"BOB\" into u64 for key first_log_term: invalid digit found in string" + ); + } + + #[test] + fn test_non_existent_key_with_correct_value_lgif_parse_fail() { + let log = log(); + let data = + "first_log_idx\t1\nfirst_log\t1\nlast_log_idx\t4386\nlast_log_term\t1\nlast_committed_log_idx\t4386\nleader_committed_log_idx\t4386\ntarget_committed_log_idx\t4386\nlast_snapshot_idx\t0\n\n" + .as_bytes(); + let result = Lgif::parse(&log, data); + let error = result.unwrap_err(); + let root_cause = error.root_cause(); + + assert_eq!( + format!("{}", root_cause), + "Extracted key `\"first_log\"` from output differs from expected key `first_log_term`" + ); + } + + #[test] + fn test_additional_key_value_pairs_in_output_parse_fail() { + let log = log(); + let data = "first_log_idx\t1\nfirst_log_term\t1\nlast_log_idx\t4386\nlast_log_term\t1\nlast_committed_log_idx\t4386\nleader_committed_log_idx\t4386\ntarget_committed_log_idx\t4386\nlast_snapshot_idx\t0\nlast_snapshot_idx\t3\n\n" + .as_bytes(); + + let result = Lgif::parse(&log, data); + let error = result.unwrap_err(); + let root_cause = error.root_cause(); + + assert_eq!( + format!("{}", root_cause), + "Output from the Keeper differs to the expected output keys \ + Output: \"first_log_idx\\t1\\nfirst_log_term\\t1\\nlast_log_idx\\t4386\\nlast_log_term\\t1\\nlast_committed_log_idx\\t4386\\nleader_committed_log_idx\\t4386\\ntarget_committed_log_idx\\t4386\\nlast_snapshot_idx\\t0\\nlast_snapshot_idx\\t3\\n\\n\" \ + Expected output keys: [\"first_log_idx\", \"first_log_term\", \"last_log_idx\", \"last_log_term\", \"last_committed_log_idx\", \"leader_committed_log_idx\", \"target_committed_log_idx\", \"last_snapshot_idx\"]", + ); + } + + #[test] + fn test_empty_output_parse_fail() { + let log = log(); + let data = "".as_bytes(); + let result = Lgif::parse(&log, data); + let error = result.unwrap_err(); + let root_cause = error.root_cause(); + + assert_eq!( + format!("{}", root_cause), + "Output from the Keeper differs to the expected output keys \ + Output: \"\" \ + Expected output keys: [\"first_log_idx\", \"first_log_term\", \"last_log_idx\", \"last_log_term\", \"last_committed_log_idx\", \"leader_committed_log_idx\", \"target_committed_log_idx\", \"last_snapshot_idx\"]", + ); + } + + #[test] + fn test_full_raft_config_parse_success() { + let log = log(); + let data = + "server.1=::1:21001;participant;1\nserver.2=oxide.internal:21002;participant;1\nserver.3=127.0.0.1:21003;learner;0\n" + .as_bytes(); + let raft_config = RaftConfig::parse(&log, data).unwrap(); + + assert!(raft_config.keeper_servers.contains(&KeeperServerInfo { + server_id: KeeperId(1), + host: ClickhouseHost::Ipv6("::1".parse().unwrap()), + raft_port: 21001, + server_type: KeeperServerType::Participant, + priority: 1, + },)); + assert!(raft_config.keeper_servers.contains(&KeeperServerInfo { + server_id: KeeperId(2), + host: ClickhouseHost::DomainName("oxide.internal".to_string()), + raft_port: 21002, + server_type: KeeperServerType::Participant, + priority: 1, + },)); + assert!(raft_config.keeper_servers.contains(&KeeperServerInfo { + server_id: KeeperId(3), + host: ClickhouseHost::Ipv4("127.0.0.1".parse().unwrap()), + raft_port: 21003, + server_type: KeeperServerType::Learner, + priority: 0, + },)); + } + + #[test] + fn test_misshapen_id_raft_config_parse_fail() { + let log = log(); + let data = "serv.1=::1:21001;participant;1\n".as_bytes(); + let result = RaftConfig::parse(&log, data); + + let error = result.unwrap_err(); + let root_cause = error.root_cause(); + + assert_eq!( + format!("{}", root_cause), + "Output is not as expected. Server identifier: 'serv.1' Expected server identifier: 'server.{SERVER_ID}'", + ); + } + + #[test] + fn test_misshapen_port_raft_config_parse_fail() { + let log = log(); + let data = "server.1=::1:BOB;participant;1".as_bytes(); + let result = RaftConfig::parse(&log, data); + + let error = result.unwrap_err(); + let root_cause = error.root_cause(); + + assert_eq!( + format!("{}", root_cause), + "Unable to convert value \"BOB\" into u16: invalid digit found in string", + ); + } + + #[test] + fn test_empty_output_raft_config_parse_fail() { + let log = log(); + let data = "".as_bytes(); + let result = RaftConfig::parse(&log, data); + + let error = result.unwrap_err(); + let root_cause = error.root_cause(); + + assert_eq!(format!("{}", root_cause), "Cannot parse an empty response",); + } + + #[test] + fn test_missing_server_id_raft_config_parse_fail() { + let log = log(); + let data = "server.=::1:21001;participant;1".as_bytes(); + let result = RaftConfig::parse(&log, data); + + let error = result.unwrap_err(); + let root_cause = error.root_cause(); + + assert_eq!( + format!("{}", root_cause), + "Unable to convert value \"\" into u64: cannot parse integer from empty string", + ); + } + + #[test] + fn test_missing_address_raft_config_parse_fail() { + let log = log(); + let data = "server.1=:21001;participant;1".as_bytes(); + let result = RaftConfig::parse(&log, data); + + let error = result.unwrap_err(); + let root_cause = error.root_cause(); + + assert_eq!( + format!("{}", root_cause), + " is not a valid address or domain name", + ); + } + + #[test] + fn test_invalid_address_raft_config_parse_fail() { + let log = log(); + let data = "server.1=oxide.com:21001;participant;1".as_bytes(); + let result = RaftConfig::parse(&log, data); + + let error = result.unwrap_err(); + let root_cause = error.root_cause(); + + assert_eq!( + format!("{}", root_cause), + "oxide.com is not a valid address or domain name", + ); + } + + #[test] + fn test_missing_port_raft_config_parse_fail() { + let log = log(); + let data = "server.1=::1:;participant;1".as_bytes(); + let result = RaftConfig::parse(&log, data); + + let error = result.unwrap_err(); + let root_cause = error.root_cause(); + + assert_eq!( + format!("{}", root_cause), + "Unable to convert value \"\" into u16: cannot parse integer from empty string", + ); + } + + #[test] + fn test_missing_participant_raft_config_parse_fail() { + let log = log(); + let data = "server.1=::1:21001;1".as_bytes(); + let result = RaftConfig::parse(&log, data); + + let error = result.unwrap_err(); + let root_cause = error.root_cause(); + + assert_eq!( + format!("{}", root_cause), + "1 is not a valid keeper server type", + ); + + let data = "server.1=::1:21001;;1".as_bytes(); + let result = RaftConfig::parse(&log, data); + + let error = result.unwrap_err(); + let root_cause = error.root_cause(); + + assert_eq!( + format!("{}", root_cause), + " is not a valid keeper server type", + ); + } + + #[test] + fn test_misshapen_participant_raft_config_parse_fail() { + let log = log(); + let data = "server.1=::1:21001;runner;1\n".as_bytes(); + let result = RaftConfig::parse(&log, data); + + let error = result.unwrap_err(); + let root_cause = error.root_cause(); + + assert_eq!( + format!("{}", root_cause), + "runner is not a valid keeper server type", + ); + } + + #[test] + fn test_missing_priority_raft_config_parse_fail() { + let log = log(); + let data = "server.1=::1:21001;learner;\n".as_bytes(); + let result = RaftConfig::parse(&log, data); + + let error = result.unwrap_err(); + let root_cause = error.root_cause(); + + assert_eq!( + format!("{}", root_cause), + "Unable to convert value \"\" into u16: cannot parse integer from empty string", + ); + + let data = "server.1=::1:21001;learner\n".as_bytes(); + let result = RaftConfig::parse(&log, data); + + let error = result.unwrap_err(); + let root_cause = error.root_cause(); + + assert_eq!( + format!("{}", root_cause), + "Returned None while attempting to retrieve priority", + ); + } + + #[test] + fn test_misshapen_priority_raft_config_parse_fail() { + let log = log(); + let data = "server.1=::1:21001;learner;BOB\n".as_bytes(); + let result = RaftConfig::parse(&log, data); + + let error = result.unwrap_err(); + let root_cause = error.root_cause(); + + assert_eq!( + format!("{}", root_cause), + "Unable to convert value \"BOB\" into u16: invalid digit found in string", + ); + } + + #[test] + fn test_misshapen_raft_config_parse_fail() { + let log = log(); + let data = "=;;\n".as_bytes(); + let result = RaftConfig::parse(&log, data); + + let error = result.unwrap_err(); + let root_cause = error.root_cause(); + + assert_eq!( + format!("{}", root_cause), + "Output is not as expected. Server identifier: '' Expected server identifier: 'server.{SERVER_ID}'", + ); + } + + #[test] + fn test_full_keeper_conf_parse_success() { + let log = log(); + // This data contains the duplicated "max_requests_batch_size" that occurs in the + // real conf command output + let data = + "server_id=1 +enable_ipv6=true +tcp_port=20001 +four_letter_word_allow_list=conf,cons,crst,envi,ruok,srst,srvr,stat,wchs,dirs,mntr,isro,rcvr,apiv,csnp,lgif,rqld,rclc,clrs,ftfl +max_requests_batch_size=100 +min_session_timeout_ms=10000 +session_timeout_ms=30000 +operation_timeout_ms=10000 +dead_session_check_period_ms=500 +heart_beat_interval_ms=500 +election_timeout_lower_bound_ms=1000 +election_timeout_upper_bound_ms=2000 +reserved_log_items=100000 +snapshot_distance=100000 +auto_forwarding=true +shutdown_timeout=5000 +startup_timeout=180000 +raft_logs_level=trace +snapshots_to_keep=3 +rotate_log_storage_interval=100000 +stale_log_gap=10000 +fresh_log_gap=200 +max_requests_batch_size=100 +max_requests_batch_bytes_size=102400 +max_request_queue_size=100000 +max_requests_quick_batch_size=100 +quorum_reads=false +force_sync=true +compress_logs=true +compress_snapshots_with_zstd_format=true +configuration_change_tries_count=20 +raft_limits_reconnect_limit=50 +log_storage_path=./deployment/keeper-1/coordination/log +log_storage_disk=LocalLogDisk +snapshot_storage_path=./deployment/keeper-1/coordination/snapshots +snapshot_storage_disk=LocalSnapshotDisk +\n" + .as_bytes(); + let conf = KeeperConf::parse(&log, data).unwrap(); + + assert!(conf.server_id == KeeperId(1)); + assert!(conf.enable_ipv6); + assert!(conf.tcp_port == 20001); + assert!( + conf.four_letter_word_allow_list + == *"conf,cons,crst,envi,ruok,srst,srvr,stat,wchs,dirs,mntr,isro,rcvr,apiv,csnp,lgif,rqld,rclc,clrs,ftfl" + ); + assert!(conf.max_requests_batch_size == 100); + assert!(conf.min_session_timeout_ms == 10000); + assert!(conf.session_timeout_ms == 30000); + assert!(conf.operation_timeout_ms == 10000); + assert!(conf.dead_session_check_period_ms == 500); + assert!(conf.heart_beat_interval_ms == 500); + assert!(conf.election_timeout_lower_bound_ms == 1000); + assert!(conf.election_timeout_upper_bound_ms == 2000); + assert!(conf.reserved_log_items == 100000); + assert!(conf.snapshot_distance == 100000); + assert!(conf.auto_forwarding); + assert!(conf.shutdown_timeout == 5000); + assert!(conf.startup_timeout == 180000); + assert!(conf.raft_logs_level == LogLevel::Trace); + assert!(conf.snapshots_to_keep == 3); + assert!(conf.rotate_log_storage_interval == 100000); + assert!(conf.stale_log_gap == 10000); + assert!(conf.fresh_log_gap == 200); + assert!(conf.max_requests_batch_bytes_size == 102400); + assert!(conf.max_request_queue_size == 100000); + assert!(conf.max_requests_quick_batch_size == 100); + assert!(!conf.quorum_reads); + assert!(conf.force_sync); + assert!(conf.compress_logs); + assert!(conf.compress_snapshots_with_zstd_format); + assert!(conf.configuration_change_tries_count == 20); + assert!(conf.raft_limits_reconnect_limit == 50); + assert!( + conf.log_storage_path + == Utf8PathBuf::from_str( + "./deployment/keeper-1/coordination/log" + ) + .unwrap() + ); + assert!(conf.log_storage_disk == *"LocalLogDisk"); + assert!( + conf.snapshot_storage_path + == Utf8PathBuf::from_str( + "./deployment/keeper-1/coordination/snapshots" + ) + .unwrap() + ); + assert!(conf.snapshot_storage_disk == *"LocalSnapshotDisk") + } + + #[test] + fn test_missing_value_keeper_conf_parse_fail() { + let log = log(); + // This data contains the duplicated "max_requests_batch_size" that occurs in the + // real conf command output + let data = + "server_id=1 +enable_ipv6=true +tcp_port=20001 +four_letter_word_allow_list=conf,cons,crst,envi,ruok,srst,srvr,stat,wchs,dirs,mntr,isro,rcvr,apiv,csnp,lgif,rqld,rclc,clrs,ftfl +max_requests_batch_size=100 +min_session_timeout_ms=10000 +session_timeout_ms= +operation_timeout_ms=10000 +dead_session_check_period_ms=500 +heart_beat_interval_ms=500 +election_timeout_lower_bound_ms=1000 +election_timeout_upper_bound_ms=2000 +reserved_log_items=100000 +snapshot_distance=100000 +auto_forwarding=true +shutdown_timeout=5000 +startup_timeout=180000 +raft_logs_level=trace +snapshots_to_keep=3 +rotate_log_storage_interval=100000 +stale_log_gap=10000 +fresh_log_gap=200 +max_requests_batch_size=100 +max_requests_batch_bytes_size=102400 +max_request_queue_size=100000 +max_requests_quick_batch_size=100 +quorum_reads=false +force_sync=true +compress_logs=true +compress_snapshots_with_zstd_format=true +configuration_change_tries_count=20 +raft_limits_reconnect_limit=50 +log_storage_path=./deployment/keeper-1/coordination/log +log_storage_disk=LocalLogDisk +snapshot_storage_path=./deployment/keeper-1/coordination/snapshots +snapshot_storage_disk=LocalSnapshotDisk +\n" + .as_bytes(); + let result = KeeperConf::parse(&log, data); + let error = result.unwrap_err(); + let root_cause = error.root_cause(); + + assert_eq!( + format!("{}", root_cause), + "Unable to convert value into u64: cannot parse integer from empty string" + ); + } + + #[test] + fn test_malformed_output_keeper_conf_parse_fail() { + let log = log(); + // This data contains the duplicated "max_requests_batch_size" that occurs in the + // real conf command output + let data = + "server_id=1 +enable_ipv6=true +tcp_port=20001 +four_letter_word_allow_list=conf,cons,crst,envi,ruok,srst,srvr,stat,wchs,dirs,mntr,isro,rcvr,apiv,csnp,lgif,rqld,rclc,clrs,ftfl +max_requests_batch_size=100 +min_session_timeout_ms=10000 +session_timeout_ms +operation_timeout_ms=10000 +dead_session_check_period_ms=500 +heart_beat_interval_ms=500 +election_timeout_lower_bound_ms=1000 +election_timeout_upper_bound_ms=2000 +reserved_log_items=100000 +snapshot_distance=100000 +auto_forwarding=true +shutdown_timeout=5000 +startup_timeout=180000 +raft_logs_level=trace +snapshots_to_keep=3 +rotate_log_storage_interval=100000 +stale_log_gap=10000 +fresh_log_gap=200 +max_requests_batch_size=100 +max_requests_batch_bytes_size=102400 +max_request_queue_size=100000 +max_requests_quick_batch_size=100 +quorum_reads=false +force_sync=true +compress_logs=true +compress_snapshots_with_zstd_format=true +configuration_change_tries_count=20 +raft_limits_reconnect_limit=50 +log_storage_path=./deployment/keeper-1/coordination/log +log_storage_disk=LocalLogDisk +snapshot_storage_path=./deployment/keeper-1/coordination/snapshots +snapshot_storage_disk=LocalSnapshotDisk +\n" + .as_bytes(); + let result = KeeperConf::parse(&log, data); + let error = result.unwrap_err(); + let root_cause = error.root_cause(); + + assert_eq!( + format!("{}", root_cause), + "Command output has a line that does not contain a key-value pair: \"session_timeout_ms\"" + ); + } + + #[test] + fn test_missing_field_keeper_conf_parse_fail() { + let log = log(); + // This data contains the duplicated "max_requests_batch_size" that occurs in the + // real conf command output + let data = + "server_id=1 +enable_ipv6=true +tcp_port=20001 +four_letter_word_allow_list=conf,cons,crst,envi,ruok,srst,srvr,stat,wchs,dirs,mntr,isro,rcvr,apiv,csnp,lgif,rqld,rclc,clrs,ftfl +max_requests_batch_size=100 +min_session_timeout_ms=10000 +operation_timeout_ms=10000 +dead_session_check_period_ms=500 +heart_beat_interval_ms=500 +election_timeout_lower_bound_ms=1000 +election_timeout_upper_bound_ms=2000 +reserved_log_items=100000 +snapshot_distance=100000 +auto_forwarding=true +shutdown_timeout=5000 +startup_timeout=180000 +raft_logs_level=trace +snapshots_to_keep=3 +rotate_log_storage_interval=100000 +stale_log_gap=10000 +fresh_log_gap=200 +max_requests_batch_size=100 +max_requests_batch_bytes_size=102400 +max_request_queue_size=100000 +max_requests_quick_batch_size=100 +quorum_reads=false +force_sync=true +compress_logs=true +compress_snapshots_with_zstd_format=true +configuration_change_tries_count=20 +raft_limits_reconnect_limit=50 +log_storage_path=./deployment/keeper-1/coordination/log +log_storage_disk=LocalLogDisk +snapshot_storage_path=./deployment/keeper-1/coordination/snapshots +snapshot_storage_disk=LocalSnapshotDisk +\n" + .as_bytes(); + let result = KeeperConf::parse(&log, data); + let error = result.unwrap_err(); + let root_cause = error.root_cause(); + + assert_eq!( + format!("{}", root_cause), + "Output from the Keeper differs to the expected output keys \ + Output: \"server_id=1\\nenable_ipv6=true\\ntcp_port=20001\\nfour_letter_word_allow_list=conf,cons,crst,envi,ruok,srst,srvr,stat,wchs,dirs,mntr,isro,rcvr,apiv,csnp,lgif,rqld,rclc,clrs,ftfl\\nmax_requests_batch_size=100\\nmin_session_timeout_ms=10000\\noperation_timeout_ms=10000\\ndead_session_check_period_ms=500\\nheart_beat_interval_ms=500\\nelection_timeout_lower_bound_ms=1000\\nelection_timeout_upper_bound_ms=2000\\nreserved_log_items=100000\\nsnapshot_distance=100000\\nauto_forwarding=true\\nshutdown_timeout=5000\\nstartup_timeout=180000\\nraft_logs_level=trace\\nsnapshots_to_keep=3\\nrotate_log_storage_interval=100000\\nstale_log_gap=10000\\nfresh_log_gap=200\\nmax_requests_batch_size=100\\nmax_requests_batch_bytes_size=102400\\nmax_request_queue_size=100000\\nmax_requests_quick_batch_size=100\\nquorum_reads=false\\nforce_sync=true\\ncompress_logs=true\\ncompress_snapshots_with_zstd_format=true\\nconfiguration_change_tries_count=20\\nraft_limits_reconnect_limit=50\\nlog_storage_path=./deployment/keeper-1/coordination/log\\nlog_storage_disk=LocalLogDisk\\nsnapshot_storage_path=./deployment/keeper-1/coordination/snapshots\\nsnapshot_storage_disk=LocalSnapshotDisk\\n\\n\" \ + Expected output keys: [\"server_id\", \"enable_ipv6\", \"tcp_port\", \"four_letter_word_allow_list\", \"max_requests_batch_size\", \"min_session_timeout_ms\", \"session_timeout_ms\", \"operation_timeout_ms\", \"dead_session_check_period_ms\", \"heart_beat_interval_ms\", \"election_timeout_lower_bound_ms\", \"election_timeout_upper_bound_ms\", \"reserved_log_items\", \"snapshot_distance\", \"auto_forwarding\", \"shutdown_timeout\", \"startup_timeout\", \"raft_logs_level\", \"snapshots_to_keep\", \"rotate_log_storage_interval\", \"stale_log_gap\", \"fresh_log_gap\", \"max_requests_batch_size\", \"max_requests_batch_bytes_size\", \"max_request_queue_size\", \"max_requests_quick_batch_size\", \"quorum_reads\", \"force_sync\", \"compress_logs\", \"compress_snapshots_with_zstd_format\", \"configuration_change_tries_count\", \"raft_limits_reconnect_limit\", \"log_storage_path\", \"log_storage_disk\", \"snapshot_storage_path\", \"snapshot_storage_disk\"]" + ); + } + + #[test] + fn test_non_existent_key_keeper_conf_parse_fail() { + let log = log(); + // This data contains the duplicated "max_requests_batch_size" that occurs in the + // real conf command output + let data = + "server_id=1 +enable_ipv6=true +tcp_port=20001 +four_letter_word_allow_list=conf,cons,crst,envi,ruok,srst,srvr,stat,wchs,dirs,mntr,isro,rcvr,apiv,csnp,lgif,rqld,rclc,clrs,ftfl +max_requests_batch_size=100 +min_session_timeout_ms=10000 +session_timeout_fake=100 +operation_timeout_ms=10000 +dead_session_check_period_ms=500 +heart_beat_interval_ms=500 +election_timeout_lower_bound_ms=1000 +election_timeout_upper_bound_ms=2000 +reserved_log_items=100000 +snapshot_distance=100000 +auto_forwarding=true +shutdown_timeout=5000 +startup_timeout=180000 +raft_logs_level=trace +snapshots_to_keep=3 +rotate_log_storage_interval=100000 +stale_log_gap=10000 +fresh_log_gap=200 +max_requests_batch_size=100 +max_requests_batch_bytes_size=102400 +max_request_queue_size=100000 +max_requests_quick_batch_size=100 +quorum_reads=false +force_sync=true +compress_logs=true +compress_snapshots_with_zstd_format=true +configuration_change_tries_count=20 +raft_limits_reconnect_limit=50 +log_storage_path=./deployment/keeper-1/coordination/log +log_storage_disk=LocalLogDisk +snapshot_storage_path=./deployment/keeper-1/coordination/snapshots +snapshot_storage_disk=LocalSnapshotDisk +\n" + .as_bytes(); + let result = KeeperConf::parse(&log, data); + let error = result.unwrap_err(); + let root_cause = error.root_cause(); + + assert_eq!( + format!("{}", root_cause), + "Extracted key `\"session_timeout_fake\"` from output differs from expected key `session_timeout_ms`" + ); + } +} diff --git a/clickhouse-admin/types/versions/src/impls/mod.rs b/clickhouse-admin/types/versions/src/impls/mod.rs new file mode 100644 index 00000000000..fd87c187fdd --- /dev/null +++ b/clickhouse-admin/types/versions/src/impls/mod.rs @@ -0,0 +1,9 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! Functional code for the latest versions of types. + +mod config; +mod keeper; +mod server; diff --git a/clickhouse-admin/types/versions/src/impls/server.rs b/clickhouse-admin/types/versions/src/impls/server.rs new file mode 100644 index 00000000000..d7f8b0a963b --- /dev/null +++ b/clickhouse-admin/types/versions/src/impls/server.rs @@ -0,0 +1,473 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! Functional code for server types. + +use crate::latest::config::{ + ClickhouseHost, KeeperNodeConfig, LogConfig, Macros, NodeType, + ReplicaConfig, ServerNodeConfig, +}; +use crate::latest::server::{ + DistributedDdlQueue, ServerConfigurableSettings, ServerId, ServerSettings, + SystemTable, SystemTimeSeries, SystemTimeSeriesSettings, TimestampFormat, +}; +use anyhow::{Context, Result}; +use atomicwrites::AtomicFile; +use camino::Utf8PathBuf; +use omicron_common::api::external::Generation; +use slog::{Logger, info}; +use std::fs::create_dir; +use std::io::{ErrorKind, Write}; +use std::net::Ipv6Addr; + +impl ServerConfigurableSettings { + /// Generate a configuration file for a replica server node + pub fn generate_xml_file(&self) -> Result { + let logger = LogConfig::new( + self.settings.datastore_path.clone(), + NodeType::Server, + ); + let macros = Macros::new(self.settings.id); + + let keepers: Vec = self + .settings + .keepers + .iter() + .map(|host| KeeperNodeConfig::new(host.clone())) + .collect(); + + let servers: Vec = self + .settings + .remote_servers + .iter() + .map(|host| ServerNodeConfig::new(host.clone())) + .collect(); + + let config = ReplicaConfig::new( + logger, + macros, + self.listen_addr(), + servers.clone(), + keepers.clone(), + self.datastore_path(), + self.generation(), + ); + + match create_dir(self.settings.config_dir.clone()) { + Ok(_) => (), + Err(e) if e.kind() == ErrorKind::AlreadyExists => (), + Err(e) => return Err(e.into()), + }; + + let path = self.settings.config_dir.join("replica-server-config.xml"); + AtomicFile::new( + path.clone(), + atomicwrites::OverwriteBehavior::AllowOverwrite, + ) + .write(|f| f.write_all(config.to_xml().as_bytes())) + .with_context(|| format!("failed to write to `{}`", path))?; + + Ok(config) + } + + pub fn generation(&self) -> Generation { + self.generation + } + + fn listen_addr(&self) -> Ipv6Addr { + self.settings.listen_addr + } + + fn datastore_path(&self) -> Utf8PathBuf { + self.settings.datastore_path.clone() + } +} + +impl ServerSettings { + pub fn new( + config_dir: Utf8PathBuf, + id: ServerId, + datastore_path: Utf8PathBuf, + listen_addr: Ipv6Addr, + keepers: Vec, + remote_servers: Vec, + ) -> Self { + Self { + config_dir, + id, + datastore_path, + listen_addr, + keepers, + remote_servers, + } + } +} + +impl DistributedDdlQueue { + pub fn parse(log: &Logger, data: &[u8]) -> Result> { + let s = String::from_utf8_lossy(data); + info!( + log, + "Retrieved data from `system.distributed_ddl_queue`"; + "output" => ?s + ); + + let mut ddl = vec![]; + + for line in s.lines() { + let item: DistributedDdlQueue = serde_json::from_str(line)?; + ddl.push(item); + } + + Ok(ddl) + } +} + +impl std::fmt::Display for SystemTable { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + let table = match self { + SystemTable::MetricLog => "metric_log", + SystemTable::AsynchronousMetricLog => "asynchronous_metric_log", + }; + write!(f, "{}", table) + } +} + +impl std::fmt::Display for TimestampFormat { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + let table = match self { + TimestampFormat::Utc => "iso", + TimestampFormat::UnixEpoch => "unix_timestamp", + }; + write!(f, "{}", table) + } +} + +impl SystemTimeSeriesSettings { + fn interval(&self) -> u64 { + self.retrieval_settings.interval + } + + fn time_range(&self) -> u64 { + self.retrieval_settings.time_range + } + + fn timestamp_format(&self) -> TimestampFormat { + self.retrieval_settings.timestamp_format + } + + fn metric_name(&self) -> &str { + &self.metric_info.metric + } + + fn table(&self) -> SystemTable { + self.metric_info.table + } + + // TODO: Use more aggregate functions than just avg? + pub fn query_avg(&self) -> String { + let interval = self.interval(); + let time_range = self.time_range(); + let metric_name = self.metric_name(); + let table = self.table(); + let ts_fmt = self.timestamp_format(); + + let avg_value = match table { + SystemTable::MetricLog => metric_name, + SystemTable::AsynchronousMetricLog => "value", + }; + + let mut query = format!( + "SELECT toStartOfInterval(event_time, INTERVAL {interval} SECOND) AS time, avg({avg_value}) AS value + FROM system.{table} + WHERE event_date >= toDate(now() - {time_range}) AND event_time >= now() - {time_range} + " + ); + + match table { + SystemTable::MetricLog => (), + SystemTable::AsynchronousMetricLog => query.push_str( + format!( + "AND metric = '{metric_name}' + " + ) + .as_str(), + ), + }; + + query.push_str( + format!( + "GROUP BY time + ORDER BY time WITH FILL STEP {interval} + FORMAT JSONEachRow + SETTINGS date_time_output_format = '{ts_fmt}'" + ) + .as_str(), + ); + query + } +} + +impl SystemTimeSeries { + pub fn parse(log: &Logger, data: &[u8]) -> Result> { + let s = String::from_utf8_lossy(data); + info!( + log, + "Retrieved data from `system` database"; + "output" => ?s + ); + + let mut m = vec![]; + + for line in s.lines() { + // serde_json deserialises f64 types with loss of precision at times. + // For example, in our tests some of the values to serialize have a + // fractional value of `.33333`, but once parsed, they become `.33331`. + // + // We do not require this level of precision, so we'll leave as is. + // Just noting that we are aware of this slight inaccuracy. + let item: SystemTimeSeries = serde_json::from_str(line)?; + m.push(item); + } + + Ok(m) + } +} + +#[cfg(test)] +mod tests { + use crate::latest::config::ClickhouseHost; + use crate::latest::server::{ + DistributedDdlQueue, ServerConfigurableSettings, ServerId, + ServerSettings, SystemTimeSeries, + }; + use camino::Utf8PathBuf; + use camino_tempfile::Builder; + use chrono::{DateTime, Utc}; + use omicron_common::api::external::Generation; + use slog::{Drain, o}; + use slog_term::{FullFormat, PlainDecorator, TestStdoutWriter}; + use std::collections::BTreeMap; + use std::net::{Ipv4Addr, Ipv6Addr}; + use std::str::FromStr; + + fn log() -> slog::Logger { + let decorator = PlainDecorator::new(TestStdoutWriter); + let drain = FullFormat::new(decorator).build().fuse(); + let drain = slog_async::Async::new(drain).build().fuse(); + slog::Logger::root(drain, o!()) + } + + #[test] + fn test_generate_replica_config() { + let config_dir = Builder::new() + .tempdir_in( + Utf8PathBuf::try_from(std::env::temp_dir()).unwrap()) + .expect("Could not create directory for ClickHouse configuration generation test" + ); + + let keepers = vec![ + ClickhouseHost::Ipv6(Ipv6Addr::from_str("ff::01").unwrap()), + ClickhouseHost::Ipv4(Ipv4Addr::from_str("127.0.0.1").unwrap()), + ClickhouseHost::DomainName("we.dont.want.brackets.com".to_string()), + ]; + + let servers = vec![ + ClickhouseHost::Ipv6(Ipv6Addr::from_str("ff::09").unwrap()), + ClickhouseHost::DomainName("ohai.com".to_string()), + ]; + + let settings = ServerSettings::new( + Utf8PathBuf::from(config_dir.path()), + ServerId(1), + Utf8PathBuf::from_str("./").unwrap(), + Ipv6Addr::from_str("ff::08").unwrap(), + keepers, + servers, + ); + + let config = ServerConfigurableSettings { + settings, + generation: Generation::new(), + }; + config.generate_xml_file().unwrap(); + + let expected_file = + Utf8PathBuf::from("../testutils").join("replica-server-config.xml"); + let generated_file = Utf8PathBuf::from(config_dir.path()) + .join("replica-server-config.xml"); + let generated_content = std::fs::read_to_string(generated_file).expect( + "Failed to read from generated ClickHouse replica server file", + ); + + expectorate::assert_contents(expected_file, &generated_content); + } + + #[test] + fn test_distributed_ddl_queries_parse_success() { + let log = log(); + let data = + "{\"entry\":\"query-0000000000\",\"entry_version\":5,\"initiator_host\":\"ixchel\",\"initiator_port\":22001,\"cluster\":\"oximeter_cluster\",\"query\":\"CREATE DATABASE IF NOT EXISTS db1 UUID 'a49757e4-179e-42bd-866f-93ac43136e2d' ON CLUSTER oximeter_cluster\",\"settings\":{\"load_balancing\":\"random\"},\"query_create_time\":\"2024-11-01T16:16:45Z\",\"host\":\"::1\",\"port\":22001,\"status\":\"Finished\",\"exception_code\":0,\"exception_text\":\"\",\"query_finish_time\":\"2024-11-01T16:16:45Z\",\"query_duration_ms\":4} +{\"entry\":\"query-0000000000\",\"entry_version\":5,\"initiator_host\":\"ixchel\",\"initiator_port\":22001,\"cluster\":\"oximeter_cluster\",\"query\":\"CREATE DATABASE IF NOT EXISTS db1 UUID 'a49757e4-179e-42bd-866f-93ac43136e2d' ON CLUSTER oximeter_cluster\",\"settings\":{\"load_balancing\":\"random\"},\"query_create_time\":\"2024-11-01T16:16:45Z\",\"host\":\"::1\",\"port\":22002,\"status\":\"Finished\",\"exception_code\":0,\"exception_text\":\"\",\"query_finish_time\":\"2024-11-01T16:16:45Z\",\"query_duration_ms\":4} +" + .as_bytes(); + let ddl = DistributedDdlQueue::parse(&log, data).unwrap(); + + let expected_result = vec![ + DistributedDdlQueue{ + entry: "query-0000000000".to_string(), + entry_version: 5, + initiator_host: "ixchel".to_string(), + initiator_port: 22001, + cluster: "oximeter_cluster".to_string(), + query: "CREATE DATABASE IF NOT EXISTS db1 UUID 'a49757e4-179e-42bd-866f-93ac43136e2d' ON CLUSTER oximeter_cluster".to_string(), + settings: BTreeMap::from([ + ("load_balancing".to_string(), "random".to_string()), +]), + query_create_time: "2024-11-01T16:16:45Z".parse::>().unwrap(), + host: Ipv6Addr::from_str("::1").unwrap(), + port: 22001, + exception_code: 0, + exception_text: "".to_string(), + status: "Finished".to_string(), + query_finish_time: "2024-11-01T16:16:45Z".parse::>().unwrap(), + query_duration_ms: 4, + }, + DistributedDdlQueue{ + entry: "query-0000000000".to_string(), + entry_version: 5, + initiator_host: "ixchel".to_string(), + initiator_port: 22001, + cluster: "oximeter_cluster".to_string(), + query: "CREATE DATABASE IF NOT EXISTS db1 UUID 'a49757e4-179e-42bd-866f-93ac43136e2d' ON CLUSTER oximeter_cluster".to_string(), + settings: BTreeMap::from([ + ("load_balancing".to_string(), "random".to_string()), +]), + query_create_time: "2024-11-01T16:16:45Z".parse::>().unwrap(), + host: Ipv6Addr::from_str("::1").unwrap(), + port: 22002, + exception_code: 0, + exception_text: "".to_string(), + status: "Finished".to_string(), + query_finish_time: "2024-11-01T16:16:45Z".parse::>().unwrap(), + query_duration_ms: 4, + }, + ]; + assert!(ddl == expected_result); + } + + #[test] + fn test_empty_distributed_ddl_queries_parse_success() { + let log = log(); + let data = "".as_bytes(); + let ddl = DistributedDdlQueue::parse(&log, data).unwrap(); + + let expected_result = vec![]; + assert!(ddl == expected_result); + } + + #[test] + fn test_misshapen_distributed_ddl_queries_parse_fail() { + let log = log(); + let data = + "{\"entry\":\"query-0000000000\",\"initiator_host\":\"ixchel\",\"initiator_port\":22001,\"cluster\":\"oximeter_cluster\",\"query\":\"CREATE DATABASE IF NOT EXISTS db1 UUID 'a49757e4-179e-42bd-866f-93ac43136e2d' ON CLUSTER oximeter_cluster\",\"settings\":{\"load_balancing\":\"random\"},\"query_create_time\":\"2024-11-01T16:16:45Z\",\"host\":\"::1\",\"port\":22001,\"status\":\"Finished\",\"exception_code\":0,\"exception_text\":\"\",\"query_finish_time\":\"2024-11-01T16:16:45Z\",\"query_duration_ms\":4} +" +.as_bytes(); + let result = DistributedDdlQueue::parse(&log, data); + + let error = result.unwrap_err(); + let root_cause = error.root_cause(); + + assert_eq!( + format!("{}", root_cause), + "missing field `entry_version` at line 1 column 454", + ); + } + + #[test] + fn test_unix_epoch_system_timeseries_parse_success() { + let log = log(); + let data = "{\"time\":\"1732494720\",\"value\":110220450825.75238} +{\"time\":\"1732494840\",\"value\":110339992917.33333} +{\"time\":\"1732494960\",\"value\":110421854037.33333}\n" + .as_bytes(); + let timeseries = SystemTimeSeries::parse(&log, data).unwrap(); + + let expected = vec![ + SystemTimeSeries { + time: "1732494720".to_string(), + value: 110220450825.75238, + }, + SystemTimeSeries { + time: "1732494840".to_string(), + value: 110339992917.33331, + }, + SystemTimeSeries { + time: "1732494960".to_string(), + value: 110421854037.33331, + }, + ]; + + assert_eq!(timeseries, expected); + } + + #[test] + fn test_utc_system_timeseries_parse_success() { + let log = log(); + let data = + "{\"time\":\"2024-11-25T00:34:00Z\",\"value\":110220450825.75238} +{\"time\":\"2024-11-25T00:35:00Z\",\"value\":110339992917.33333} +{\"time\":\"2024-11-25T00:36:00Z\",\"value\":110421854037.33333}\n" + .as_bytes(); + let timeseries = SystemTimeSeries::parse(&log, data).unwrap(); + + let expected = vec![ + SystemTimeSeries { + time: "2024-11-25T00:34:00Z".to_string(), + value: 110220450825.75238, + }, + SystemTimeSeries { + time: "2024-11-25T00:35:00Z".to_string(), + value: 110339992917.33331, + }, + SystemTimeSeries { + time: "2024-11-25T00:36:00Z".to_string(), + value: 110421854037.33331, + }, + ]; + + assert_eq!(timeseries, expected); + } + + #[test] + fn test_misshapen_system_timeseries_parse_fail() { + let log = log(); + let data = "{\"bob\":\"1732494720\",\"value\":110220450825.75238}\n" + .as_bytes(); + let result = SystemTimeSeries::parse(&log, data); + + let error = result.unwrap_err(); + let root_cause = error.root_cause(); + + assert_eq!( + format!("{}", root_cause), + "missing field `time` at line 1 column 47", + ); + } + + #[test] + fn test_time_format_system_timeseries_parse_fail() { + let log = log(); + let data = "{\"time\":2024,\"value\":110220450825.75238}\n".as_bytes(); + let result = SystemTimeSeries::parse(&log, data); + + let error = result.unwrap_err(); + let root_cause = error.root_cause(); + + assert_eq!( + format!("{}", root_cause), + "invalid type: integer `2024`, expected a string at line 1 column 12", + ); + } +} diff --git a/clickhouse-admin/types/versions/src/initial/config.rs b/clickhouse-admin/types/versions/src/initial/config.rs new file mode 100644 index 00000000000..45694012f66 --- /dev/null +++ b/clickhouse-admin/types/versions/src/initial/config.rs @@ -0,0 +1,186 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! Configuration types shared across ClickHouse Admin APIs. + +use super::keeper::KeeperId; +use super::server::ServerId; +use camino::Utf8PathBuf; +use omicron_common::api::external::Generation; +use schemars::{ + JsonSchema, + r#gen::SchemaGenerator, + schema::{Schema, SchemaObject}, +}; +use serde::{Deserialize, Serialize}; +use std::net::{Ipv4Addr, Ipv6Addr}; + +// Used for schemars to be able to be used with camino: +// See https://github.com/camino-rs/camino/issues/91#issuecomment-2027908513 +pub fn path_schema(generator: &mut SchemaGenerator) -> Schema { + let mut schema: SchemaObject = ::json_schema(generator).into(); + schema.format = Some("Utf8PathBuf".to_owned()); + schema.into() +} + +/// Result after generating a configuration file +#[derive(Debug, Clone, PartialEq, Eq, JsonSchema, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum GenerateConfigResult { + Replica(ReplicaConfig), + Keeper(KeeperConfig), +} + +/// Configuration for a ClickHouse replica server +#[derive(Debug, Clone, PartialEq, Eq, JsonSchema, Serialize, Deserialize)] +pub struct ReplicaConfig { + /// Logging settings + pub logger: LogConfig, + /// Parameter substitutions for replicated tables + pub macros: Macros, + /// Address the server is listening on + pub listen_host: Ipv6Addr, + /// Port for HTTP connections + pub http_port: u16, + /// Port for TCP connections + pub tcp_port: u16, + /// Port for interserver HTTP connections + pub interserver_http_port: u16, + /// Configuration of clusters used by the Distributed table engine and by the cluster + /// table function + pub remote_servers: RemoteServers, + /// Contains settings that allow ClickHouse servers to interact with a Keeper cluster + pub keepers: KeeperConfigsForReplica, + /// Directory for all files generated by ClickHouse itself + #[schemars(schema_with = "path_schema")] + pub data_path: Utf8PathBuf, + /// A unique identifier for the configuration generation. + pub generation: Generation, +} + +#[derive(Debug, Clone, PartialEq, Eq, JsonSchema, Serialize, Deserialize)] +pub struct Macros { + pub shard: u64, + pub replica: ServerId, + pub cluster: String, +} + +#[derive(Debug, Clone, PartialEq, Eq, JsonSchema, Serialize, Deserialize)] +pub struct RemoteServers { + pub cluster: String, + pub secret: String, + pub replicas: Vec, +} + +#[derive(Debug, Clone, PartialEq, Eq, JsonSchema, Serialize, Deserialize)] +pub struct KeeperConfigsForReplica { + pub nodes: Vec, +} + +#[derive( + Debug, + Clone, + PartialEq, + Eq, + Deserialize, + PartialOrd, + Ord, + Serialize, + JsonSchema, +)] +#[serde(rename_all = "snake_case")] +pub enum ClickhouseHost { + Ipv6(Ipv6Addr), + Ipv4(Ipv4Addr), + DomainName(String), +} + +#[derive(Debug, Clone, PartialEq, Eq, JsonSchema, Serialize, Deserialize)] +pub struct KeeperNodeConfig { + pub host: ClickhouseHost, + pub port: u16, +} + +#[derive(Debug, Clone, PartialEq, Eq, JsonSchema, Serialize, Deserialize)] +pub struct ServerNodeConfig { + pub host: ClickhouseHost, + pub port: u16, +} + +pub enum NodeType { + Server, + Keeper, +} + +#[derive(Debug, Clone, PartialEq, Eq, JsonSchema, Serialize, Deserialize)] +pub struct LogConfig { + pub level: LogLevel, + #[schemars(schema_with = "path_schema")] + pub log: Utf8PathBuf, + #[schemars(schema_with = "path_schema")] + pub errorlog: Utf8PathBuf, + pub size: u16, + pub count: usize, +} + +#[derive(Debug, Clone, PartialEq, Eq, JsonSchema, Serialize, Deserialize)] +pub struct KeeperCoordinationSettings { + pub operation_timeout_ms: u32, + pub session_timeout_ms: u32, + pub raft_logs_level: LogLevel, +} + +#[derive(Debug, Clone, PartialEq, Eq, JsonSchema, Serialize, Deserialize)] +pub struct RaftServers { + pub servers: Vec, +} + +#[derive(Debug, Clone, PartialEq, Eq, JsonSchema, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub struct RaftServerSettings { + pub id: KeeperId, + pub host: ClickhouseHost, +} + +#[derive(Debug, Clone, PartialEq, Eq, JsonSchema, Serialize, Deserialize)] +pub struct RaftServerConfig { + pub id: KeeperId, + pub hostname: ClickhouseHost, + pub port: u16, +} + +/// Configuration for a ClickHouse keeper +#[derive(Debug, Clone, PartialEq, Eq, JsonSchema, Serialize, Deserialize)] +pub struct KeeperConfig { + /// Logging settings + pub logger: LogConfig, + /// Address the keeper is listening on + pub listen_host: Ipv6Addr, + /// Port for TCP connections + pub tcp_port: u16, + /// Unique ID for this keeper node + pub server_id: KeeperId, + /// Directory for coordination logs + #[schemars(schema_with = "path_schema")] + pub log_storage_path: Utf8PathBuf, + /// Directory for coordination snapshot storage + #[schemars(schema_with = "path_schema")] + pub snapshot_storage_path: Utf8PathBuf, + /// Internal coordination settings + pub coordination_settings: KeeperCoordinationSettings, + /// Settings for each server in the keeper cluster + pub raft_config: RaftServers, + /// Directory for all files generated by ClickHouse itself + #[schemars(schema_with = "path_schema")] + pub datastore_path: Utf8PathBuf, + /// A unique identifier for the configuration generation. + pub generation: Generation, +} + +#[derive(Debug, Clone, PartialEq, Eq, JsonSchema, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum LogLevel { + Trace, + Debug, +} diff --git a/clickhouse-admin/types/versions/src/initial/keeper.rs b/clickhouse-admin/types/versions/src/initial/keeper.rs new file mode 100644 index 00000000000..d46afaa4302 --- /dev/null +++ b/clickhouse-admin/types/versions/src/initial/keeper.rs @@ -0,0 +1,271 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! Keeper-specific types for the ClickHouse Admin Keeper API. + +use super::config::{ClickhouseHost, RaftServerSettings}; +use camino::Utf8PathBuf; +use daft::Diffable; +use derive_more::{Add, AddAssign, Display, From}; +use omicron_common::api::external::Generation; +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; +use std::collections::BTreeSet; +use std::net::Ipv6Addr; + +use super::config::path_schema; + +/// A unique ID for a ClickHouse Keeper +#[derive( + Debug, + Clone, + Copy, + Eq, + PartialEq, + Ord, + PartialOrd, + From, + Add, + AddAssign, + Display, + JsonSchema, + Serialize, + Deserialize, + Diffable, +)] +pub struct KeeperId(pub u64); + +/// The top most type for configuring clickhouse-servers via +/// clickhouse-admin-keeper-api +#[derive(Debug, Serialize, Deserialize, JsonSchema)] +pub struct KeeperConfigurableSettings { + /// A unique identifier for the configuration generation. + pub generation: Generation, + /// Configurable settings for a ClickHouse keeper node. + pub settings: KeeperSettings, +} + +/// Configurable settings for a ClickHouse keeper node. +#[derive(Debug, Clone, PartialEq, Eq, Deserialize, Serialize, JsonSchema)] +#[serde(rename_all = "snake_case")] +pub struct KeeperSettings { + /// Directory for the generated keeper configuration XML file + #[schemars(schema_with = "path_schema")] + pub config_dir: Utf8PathBuf, + /// Unique ID of the keeper node + pub id: KeeperId, + /// ID and host of each server in the keeper cluster + pub raft_servers: Vec, + /// Directory for all files generated by ClickHouse itself + #[schemars(schema_with = "path_schema")] + pub datastore_path: Utf8PathBuf, + /// Address the keeper is listening on + pub listen_addr: Ipv6Addr, +} + +#[derive(Debug, Clone, PartialEq, Eq, Deserialize, Serialize, JsonSchema)] +#[serde(rename_all = "snake_case")] +/// Logically grouped information file from a keeper node +pub struct Lgif { + /// Index of the first log entry in the current log segment + pub first_log_idx: u64, + /// Term of the leader when the first log entry was created + pub first_log_term: u64, + /// Index of the last log entry in the current log segment + pub last_log_idx: u64, + /// Term of the leader when the last log entry was created + pub last_log_term: u64, + /// Index of the last committed log entry + pub last_committed_log_idx: u64, + /// Index of the last committed log entry from the leader's perspective + pub leader_committed_log_idx: u64, + /// Target index for log commitment during replication or recovery + pub target_committed_log_idx: u64, + /// Index of the most recent snapshot taken + pub last_snapshot_idx: u64, +} + +#[derive( + Debug, + Clone, + PartialEq, + Eq, + Deserialize, + Ord, + PartialOrd, + Serialize, + JsonSchema, +)] +#[serde(rename_all = "snake_case")] +pub enum KeeperServerType { + Participant, + Learner, +} + +#[derive( + Debug, + Clone, + PartialEq, + Eq, + Deserialize, + PartialOrd, + Ord, + Serialize, + JsonSchema, +)] +#[serde(rename_all = "snake_case")] +pub struct KeeperServerInfo { + /// Unique, immutable ID of the keeper server + pub server_id: KeeperId, + /// Host of the keeper server + pub host: ClickhouseHost, + /// Keeper server raft port + pub raft_port: u16, + /// A keeper server either participant or learner + /// (learner does not participate in leader elections). + pub server_type: KeeperServerType, + /// non-negative integer telling which nodes should be + /// prioritised on leader elections. + /// Priority of 0 means server will never be a leader. + pub priority: u16, +} + +#[derive( + Debug, + Clone, + PartialEq, + Eq, + Deserialize, + PartialOrd, + Ord, + Serialize, + JsonSchema, +)] +#[serde(rename_all = "snake_case")] +/// Keeper raft configuration information +pub struct RaftConfig { + pub keeper_servers: BTreeSet, +} + +// While we generally use "Config", in this case we use "Conf" +// as it is the four letter word command we are invoking: +// `clickhouse keeper-client --q conf` +/// Keeper configuration information +#[derive(Debug, Clone, PartialEq, Eq, Deserialize, Serialize, JsonSchema)] +#[serde(rename_all = "snake_case")] +pub struct KeeperConf { + /// Unique server id, each participant of the ClickHouse Keeper cluster must + /// have a unique number (1, 2, 3, and so on). + pub server_id: KeeperId, + /// Whether Ipv6 is enabled. + pub enable_ipv6: bool, + /// Port for a client to connect. + pub tcp_port: u16, + /// Allow list of 4lw commands. + pub four_letter_word_allow_list: String, + /// Max size of batch in requests count before it will be sent to RAFT. + pub max_requests_batch_size: u64, + /// Min timeout for client session (ms). + pub min_session_timeout_ms: u64, + /// Max timeout for client session (ms). + pub session_timeout_ms: u64, + /// Timeout for a single client operation (ms). + pub operation_timeout_ms: u64, + /// How often ClickHouse Keeper checks for dead sessions and removes them (ms). + pub dead_session_check_period_ms: u64, + /// How often a ClickHouse Keeper leader will send heartbeats to followers (ms). + pub heart_beat_interval_ms: u64, + /// If the follower does not receive a heartbeat from the leader in this interval, + /// then it can initiate leader election. Must be less than or equal to + /// election_timeout_upper_bound_ms. Ideally they shouldn't be equal. + pub election_timeout_lower_bound_ms: u64, + /// If the follower does not receive a heartbeat from the leader in this interval, + /// then it must initiate leader election. + pub election_timeout_upper_bound_ms: u64, + /// How many coordination log records to store before compaction. + pub reserved_log_items: u64, + /// How often ClickHouse Keeper will create new snapshots + /// (in the number of records in logs). + pub snapshot_distance: u64, + /// Allow to forward write requests from followers to the leader. + pub auto_forwarding: bool, + /// Wait to finish internal connections and shutdown (ms). + pub shutdown_timeout: u64, + /// If the server doesn't connect to other quorum participants in the specified + /// timeout it will terminate (ms). + pub startup_timeout: u64, + /// Text logging level about coordination (trace, debug, and so on). + pub raft_logs_level: super::config::LogLevel, + /// How many snapshots to keep. + pub snapshots_to_keep: u64, + /// How many log records to store in a single file. + pub rotate_log_storage_interval: u64, + /// Threshold when leader considers follower as stale and sends the snapshot + /// to it instead of logs. + pub stale_log_gap: u64, + /// When the node became fresh. + pub fresh_log_gap: u64, + /// Max size in bytes of batch of requests that can be sent to RAFT. + pub max_requests_batch_bytes_size: u64, + /// Maximum number of requests that can be in queue for processing. + pub max_request_queue_size: u64, + /// Max size of batch of requests to try to get before proceeding with RAFT. + /// Keeper will not wait for requests but take only requests that are already + /// in the queue. + pub max_requests_quick_batch_size: u64, + /// Whether to execute read requests as writes through whole RAFT consesus with + /// similar speed. + pub quorum_reads: bool, + /// Whether to call fsync on each change in RAFT changelog. + pub force_sync: bool, + /// Whether to write compressed coordination logs in ZSTD format. + pub compress_logs: bool, + /// Whether to write compressed snapshots in ZSTD format (instead of custom LZ4). + pub compress_snapshots_with_zstd_format: bool, + /// How many times we will try to apply configuration change (add/remove server) + /// to the cluster. + pub configuration_change_tries_count: u64, + /// If connection to a peer is silent longer than this limit * (heartbeat interval), + /// we re-establish the connection. + pub raft_limits_reconnect_limit: u64, + /// Path to coordination logs, just like ZooKeeper it is best to store logs + /// on non-busy nodes. + #[schemars(schema_with = "path_schema")] + pub log_storage_path: Utf8PathBuf, + /// Name of disk used for logs. + pub log_storage_disk: String, + /// Path to coordination snapshots. + #[schemars(schema_with = "path_schema")] + pub snapshot_storage_path: Utf8PathBuf, + /// Name of disk used for storage. + pub snapshot_storage_disk: String, +} + +/// The configuration of the clickhouse keeper raft cluster returned from a +/// single keeper node +/// +/// Each keeper is asked for its known raft configuration via `clickhouse-admin` +/// dropshot servers running in `ClickhouseKeeper` zones. state. We include the +/// leader committed log index known to the current keeper node (whether or not +/// it is the leader) to determine which configuration is newest. +#[derive( + Clone, + Debug, + PartialEq, + Eq, + PartialOrd, + Ord, + Deserialize, + Serialize, + JsonSchema, +)] +#[serde(rename_all = "snake_case")] +pub struct ClickhouseKeeperClusterMembership { + /// Keeper ID of the keeper being queried + pub queried_keeper: KeeperId, + /// Index of the last committed log entry from the leader's perspective + pub leader_committed_log_index: u64, + /// Keeper IDs of all keepers in the cluster + pub raft_config: BTreeSet, +} diff --git a/clickhouse-admin/types/versions/src/initial/mod.rs b/clickhouse-admin/types/versions/src/initial/mod.rs new file mode 100644 index 00000000000..104a1a42c80 --- /dev/null +++ b/clickhouse-admin/types/versions/src/initial/mod.rs @@ -0,0 +1,9 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! Version `INITIAL` of the ClickHouse Admin APIs. + +pub mod config; +pub mod keeper; +pub mod server; diff --git a/clickhouse-admin/types/versions/src/initial/server.rs b/clickhouse-admin/types/versions/src/initial/server.rs new file mode 100644 index 00000000000..2d201886d49 --- /dev/null +++ b/clickhouse-admin/types/versions/src/initial/server.rs @@ -0,0 +1,200 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! Server-specific types for the ClickHouse Admin Server and Single APIs. + +use super::config::{ClickhouseHost, path_schema}; +use camino::Utf8PathBuf; +use chrono::{DateTime, Utc}; +use daft::Diffable; +use derive_more::{Add, AddAssign, Display, From}; +use omicron_common::api::external::Generation; +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; +use std::collections::BTreeMap; +use std::net::Ipv6Addr; + +/// A unique ID for a Clickhouse Server +#[derive( + Debug, + Clone, + Copy, + Eq, + PartialEq, + Ord, + PartialOrd, + From, + Add, + AddAssign, + Display, + JsonSchema, + Serialize, + Deserialize, + Diffable, +)] +pub struct ServerId(pub u64); + +/// The top most type for configuring clickhouse-servers via +/// clickhouse-admin-server-api +#[derive(Debug, Serialize, Deserialize, JsonSchema)] +pub struct ServerConfigurableSettings { + /// A unique identifier for the configuration generation. + pub generation: Generation, + /// Configurable settings for a ClickHouse replica server node. + pub settings: ServerSettings, +} + +/// Configurable settings for a ClickHouse replica server node. +#[derive(Debug, Clone, PartialEq, Eq, Deserialize, Serialize, JsonSchema)] +#[serde(rename_all = "snake_case")] +pub struct ServerSettings { + /// Directory for the generated server configuration XML file + #[schemars(schema_with = "path_schema")] + pub config_dir: Utf8PathBuf, + /// Unique ID of the server node + pub id: ServerId, + /// Directory for all files generated by ClickHouse itself + #[schemars(schema_with = "path_schema")] + pub datastore_path: Utf8PathBuf, + /// Address the server is listening on + pub listen_addr: Ipv6Addr, + /// Addresses for each of the individual nodes in the Keeper cluster + pub keepers: Vec, + /// Addresses for each of the individual replica servers + pub remote_servers: Vec, +} + +#[derive( + Clone, + Debug, + PartialEq, + Eq, + PartialOrd, + Ord, + Deserialize, + Serialize, + JsonSchema, +)] +#[serde(rename_all = "snake_case")] +/// Contains information about distributed ddl queries (ON CLUSTER clause) that were +/// executed on a cluster. +pub struct DistributedDdlQueue { + /// Query id + pub entry: String, + /// Version of the entry + pub entry_version: u64, + /// Host that initiated the DDL operation + pub initiator_host: String, + /// Port used by the initiator + pub initiator_port: u16, + /// Cluster name + pub cluster: String, + /// Query executed + pub query: String, + /// Settings used in the DDL operation + pub settings: BTreeMap, + /// Query created time + pub query_create_time: DateTime, + /// Hostname + pub host: Ipv6Addr, + /// Host Port + pub port: u16, + /// Status of the query + pub status: String, + /// Exception code + pub exception_code: u64, + /// Exception message + pub exception_text: String, + /// Query finish time + pub query_finish_time: DateTime, + /// Duration of query execution (in milliseconds) + pub query_duration_ms: u64, +} + +#[inline] +fn default_interval() -> u64 { + 60 +} + +#[inline] +fn default_time_range() -> u64 { + 86400 +} + +#[inline] +fn default_timestamp_format() -> TimestampFormat { + TimestampFormat::Utc +} + +#[derive(Clone, Copy, Debug, Serialize, Deserialize, JsonSchema)] +#[serde(rename_all = "snake_case")] +/// Available metrics tables in the `system` database +pub enum SystemTable { + AsynchronousMetricLog, + MetricLog, +} + +#[derive(Clone, Copy, Debug, Serialize, Deserialize, JsonSchema)] +#[serde(rename_all = "snake_case")] +/// Which format should the timestamp be in. +pub enum TimestampFormat { + Utc, + UnixEpoch, +} + +#[derive(Debug, Serialize, Deserialize, JsonSchema)] +#[serde(rename_all = "snake_case")] +pub struct MetricInfoPath { + /// Table to query in the `system` database + pub table: SystemTable, + /// Name of the metric to retrieve. + pub metric: String, +} + +#[derive(Debug, Serialize, Deserialize, JsonSchema)] +#[serde(rename_all = "snake_case")] +pub struct TimeSeriesSettingsQuery { + /// The interval to collect monitoring metrics in seconds. + /// Default is 60 seconds. + #[serde(default = "default_interval")] + pub interval: u64, + /// Range of time to collect monitoring metrics in seconds. + /// Default is 86400 seconds (24 hrs). + #[serde(default = "default_time_range")] + pub time_range: u64, + /// Format in which each timeseries timestamp will be in. + /// Default is UTC + #[serde(default = "default_timestamp_format")] + pub timestamp_format: TimestampFormat, +} + +/// Settings to specify which time series to retrieve. +#[derive(Debug, Serialize, Deserialize, JsonSchema)] +#[serde(rename_all = "snake_case")] +pub struct SystemTimeSeriesSettings { + /// Time series retrieval settings (time range and interval) + pub retrieval_settings: TimeSeriesSettingsQuery, + /// Database table and name of the metric to retrieve + pub metric_info: MetricInfoPath, +} + +// Our OpenAPI generator does not allow for enums to be of different +// primitive types. Because Utc is a "string" in json, Unix cannot be an int. +// This is why we set it as a `String`. +#[derive(Debug, Display, Serialize, Deserialize, JsonSchema, PartialEq)] +#[serde(untagged)] +pub enum Timestamp { + Utc(DateTime), + Unix(String), +} + +/// Retrieved time series from the internal `system` database. +#[derive(Debug, Serialize, Deserialize, JsonSchema, PartialEq)] +#[serde(rename_all = "snake_case")] +pub struct SystemTimeSeries { + pub time: String, + pub value: f64, + // TODO: Would be really nice to have an enum with possible units (s, ms, bytes) + // Not sure if I can even add this, the system tables don't mention units at all. +} diff --git a/clickhouse-admin/types/versions/src/latest.rs b/clickhouse-admin/types/versions/src/latest.rs new file mode 100644 index 00000000000..183ec6ecd05 --- /dev/null +++ b/clickhouse-admin/types/versions/src/latest.rs @@ -0,0 +1,51 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! Re-exports of the latest versions of all types. + +pub mod keeper { + pub use crate::v1::keeper::ClickhouseKeeperClusterMembership; + pub use crate::v1::keeper::KeeperConf; + pub use crate::v1::keeper::KeeperConfigurableSettings; + pub use crate::v1::keeper::KeeperId; + pub use crate::v1::keeper::KeeperServerInfo; + pub use crate::v1::keeper::KeeperServerType; + pub use crate::v1::keeper::KeeperSettings; + pub use crate::v1::keeper::Lgif; + pub use crate::v1::keeper::RaftConfig; +} + +pub mod server { + pub use crate::v1::server::DistributedDdlQueue; + pub use crate::v1::server::MetricInfoPath; + pub use crate::v1::server::ServerConfigurableSettings; + pub use crate::v1::server::ServerId; + pub use crate::v1::server::ServerSettings; + pub use crate::v1::server::SystemTable; + pub use crate::v1::server::SystemTimeSeries; + pub use crate::v1::server::SystemTimeSeriesSettings; + pub use crate::v1::server::TimeSeriesSettingsQuery; + pub use crate::v1::server::Timestamp; + pub use crate::v1::server::TimestampFormat; +} + +pub mod config { + pub use crate::v1::config::ClickhouseHost; + pub use crate::v1::config::GenerateConfigResult; + pub use crate::v1::config::KeeperConfig; + pub use crate::v1::config::KeeperConfigsForReplica; + pub use crate::v1::config::KeeperCoordinationSettings; + pub use crate::v1::config::KeeperNodeConfig; + pub use crate::v1::config::LogConfig; + pub use crate::v1::config::LogLevel; + pub use crate::v1::config::Macros; + pub use crate::v1::config::NodeType; + pub use crate::v1::config::RaftServerConfig; + pub use crate::v1::config::RaftServerSettings; + pub use crate::v1::config::RaftServers; + pub use crate::v1::config::RemoteServers; + pub use crate::v1::config::ReplicaConfig; + pub use crate::v1::config::ServerNodeConfig; + pub use crate::v1::config::path_schema; +} diff --git a/clickhouse-admin/types/versions/src/lib.rs b/clickhouse-admin/types/versions/src/lib.rs new file mode 100644 index 00000000000..89e4189fb3c --- /dev/null +++ b/clickhouse-admin/types/versions/src/lib.rs @@ -0,0 +1,35 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! Versioned types for the ClickHouse Admin APIs. +//! +//! # Adding a new API version +//! +//! When adding a new API version N with added or changed types: +//! +//! 1. Create `/mod.rs`, where `` is the lowercase +//! form of the new version's identifier, as defined in the API trait's +//! `api_versions!` macro. +//! +//! 2. Add to the end of this list: +//! +//! ```rust,ignore +//! #[path = "/mod.rs"] +//! pub mod vN; +//! ``` +//! +//! 3. Add your types to the new module, mirroring the module structure from +//! earlier versions. +//! +//! 4. Update `latest.rs` with new and updated types from the new version. +//! +//! For more information, see the [detailed guide] and [RFD 619]. +//! +//! [detailed guide]: https://github.com/oxidecomputer/dropshot-api-manager/blob/main/guides/new-version.md +//! [RFD 619]: https://rfd.shared.oxide.computer/rfd/619 + +mod impls; +pub mod latest; +#[path = "initial/mod.rs"] +pub mod v1; diff --git a/clients/clickhouse-admin-keeper-client/src/lib.rs b/clients/clickhouse-admin-keeper-client/src/lib.rs index 160b6f4558d..1c86f00b4d7 100644 --- a/clients/clickhouse-admin-keeper-client/src/lib.rs +++ b/clients/clickhouse-admin-keeper-client/src/lib.rs @@ -24,8 +24,8 @@ progenitor::generate_api!( }, derives = [schemars::JsonSchema], replace = { - KeeperConfigurableSettings = clickhouse_admin_types::KeeperConfigurableSettings, - ClickhouseKeeperClusterMembership = clickhouse_admin_types::ClickhouseKeeperClusterMembership, - KeeperId = clickhouse_admin_types::KeeperId + KeeperConfigurableSettings = clickhouse_admin_types::keeper::KeeperConfigurableSettings, + ClickhouseKeeperClusterMembership = clickhouse_admin_types::keeper::ClickhouseKeeperClusterMembership, + KeeperId = clickhouse_admin_types::keeper::KeeperId, } ); diff --git a/clients/clickhouse-admin-server-client/src/lib.rs b/clients/clickhouse-admin-server-client/src/lib.rs index 7df5f735a83..1c45957e3b8 100644 --- a/clients/clickhouse-admin-server-client/src/lib.rs +++ b/clients/clickhouse-admin-server-client/src/lib.rs @@ -24,6 +24,6 @@ progenitor::generate_api!( }, derives = [schemars::JsonSchema], replace = { - ServerConfigurableSettings = clickhouse_admin_types::ServerConfigurableSettings, + ServerConfigurableSettings = clickhouse_admin_types::server::ServerConfigurableSettings, } ); diff --git a/clients/clickhouse-admin-single-client/src/lib.rs b/clients/clickhouse-admin-single-client/src/lib.rs index 93dc4b31c8b..8df01ac72f9 100644 --- a/clients/clickhouse-admin-single-client/src/lib.rs +++ b/clients/clickhouse-admin-single-client/src/lib.rs @@ -24,6 +24,6 @@ progenitor::generate_api!( }, derives = [schemars::JsonSchema], replace = { - ServerConfigurableSettings = clickhouse_admin_types::ServerConfigurableSettings, + ServerConfigurableSettings = clickhouse_admin_types::server::ServerConfigurableSettings, } ); diff --git a/nexus/db-model/src/deployment.rs b/nexus/db-model/src/deployment.rs index 7e8d1b84346..c5d2fedf4f3 100644 --- a/nexus/db-model/src/deployment.rs +++ b/nexus/db-model/src/deployment.rs @@ -15,7 +15,8 @@ use crate::{ }; use anyhow::{Context, Result, anyhow, bail}; use chrono::{DateTime, Utc}; -use clickhouse_admin_types::{KeeperId, ServerId}; +use clickhouse_admin_types::keeper::KeeperId; +use clickhouse_admin_types::server::ServerId; use ipnetwork::IpNetwork; use nexus_db_schema::schema::{ blueprint, bp_clickhouse_cluster_config, diff --git a/nexus/db-model/src/inventory.rs b/nexus/db-model/src/inventory.rs index fba0df6b6a5..7af0efad29b 100644 --- a/nexus/db-model/src/inventory.rs +++ b/nexus/db-model/src/inventory.rs @@ -17,7 +17,9 @@ use crate::{ use anyhow::{Context, Result, anyhow, bail}; use chrono::DateTime; use chrono::Utc; -use clickhouse_admin_types::{ClickhouseKeeperClusterMembership, KeeperId}; +use clickhouse_admin_types::keeper::{ + ClickhouseKeeperClusterMembership, KeeperId, +}; use diesel::backend::Backend; use diesel::deserialize::{self, FromSql}; use diesel::expression::AsExpression; diff --git a/nexus/db-queries/src/db/datastore/deployment.rs b/nexus/db-queries/src/db/datastore/deployment.rs index 4580c6e4257..67c4e162bc5 100644 --- a/nexus/db-queries/src/db/datastore/deployment.rs +++ b/nexus/db-queries/src/db/datastore/deployment.rs @@ -17,7 +17,8 @@ use async_bb8_diesel::AsyncRunQueryDsl; use async_bb8_diesel::AsyncSimpleConnection; use chrono::DateTime; use chrono::Utc; -use clickhouse_admin_types::{KeeperId, ServerId}; +use clickhouse_admin_types::keeper::KeeperId; +use clickhouse_admin_types::server::ServerId; use core::future::Future; use core::pin::Pin; use diesel::BoolExpressionMethods; diff --git a/nexus/db-queries/src/db/datastore/inventory.rs b/nexus/db-queries/src/db/datastore/inventory.rs index d71f3ee0938..9c70c7deb9c 100644 --- a/nexus/db-queries/src/db/datastore/inventory.rs +++ b/nexus/db-queries/src/db/datastore/inventory.rs @@ -11,7 +11,7 @@ use anyhow::Context; use async_bb8_diesel::AsyncConnection; use async_bb8_diesel::AsyncRunQueryDsl; use async_bb8_diesel::AsyncSimpleConnection; -use clickhouse_admin_types::ClickhouseKeeperClusterMembership; +use clickhouse_admin_types::keeper::ClickhouseKeeperClusterMembership; use cockroach_admin_types::NodeId as CockroachNodeId; use diesel::BoolExpressionMethods; use diesel::ExpressionMethods; diff --git a/nexus/inventory/src/builder.rs b/nexus/inventory/src/builder.rs index 83b893657a4..a5f029d77f8 100644 --- a/nexus/inventory/src/builder.rs +++ b/nexus/inventory/src/builder.rs @@ -12,7 +12,7 @@ use anyhow::Context; use anyhow::anyhow; use chrono::DateTime; use chrono::Utc; -use clickhouse_admin_types::ClickhouseKeeperClusterMembership; +use clickhouse_admin_types::keeper::ClickhouseKeeperClusterMembership; use cockroach_admin_types::NodeId; use gateway_client::types::SpComponentCaboose; use gateway_client::types::SpState; diff --git a/nexus/inventory/src/examples.rs b/nexus/inventory/src/examples.rs index 9ad78cfbe13..c3c7760f217 100644 --- a/nexus/inventory/src/examples.rs +++ b/nexus/inventory/src/examples.rs @@ -7,8 +7,8 @@ use crate::CollectionBuilder; use crate::now_db_precision; use camino::Utf8Path; -use clickhouse_admin_types::ClickhouseKeeperClusterMembership; -use clickhouse_admin_types::KeeperId; +use clickhouse_admin_types::keeper::ClickhouseKeeperClusterMembership; +use clickhouse_admin_types::keeper::KeeperId; use gateway_client::types::PowerState; use gateway_client::types::RotState; use gateway_client::types::SpComponentCaboose; diff --git a/nexus/reconfigurator/execution/src/clickhouse.rs b/nexus/reconfigurator/execution/src/clickhouse.rs index b5ec2beb4c5..376430beaaa 100644 --- a/nexus/reconfigurator/execution/src/clickhouse.rs +++ b/nexus/reconfigurator/execution/src/clickhouse.rs @@ -10,14 +10,16 @@ use camino::Utf8PathBuf; use clickhouse_admin_keeper_client::Client as ClickhouseKeeperClient; use clickhouse_admin_server_client::Client as ClickhouseServerClient; use clickhouse_admin_single_client::Client as ClickhouseSingleClient; -use clickhouse_admin_types::CLICKHOUSE_KEEPER_CONFIG_DIR; -use clickhouse_admin_types::CLICKHOUSE_SERVER_CONFIG_DIR; -use clickhouse_admin_types::ClickhouseHost; -use clickhouse_admin_types::KeeperConfigurableSettings; -use clickhouse_admin_types::KeeperSettings; -use clickhouse_admin_types::RaftServerSettings; -use clickhouse_admin_types::ServerConfigurableSettings; -use clickhouse_admin_types::ServerSettings; +use clickhouse_admin_types::config::{ClickhouseHost, RaftServerSettings}; +use clickhouse_admin_types::keeper::{ + KeeperConfigurableSettings, KeeperSettings, +}; +use clickhouse_admin_types::server::{ + ServerConfigurableSettings, ServerSettings, +}; +use clickhouse_admin_types::{ + CLICKHOUSE_KEEPER_CONFIG_DIR, CLICKHOUSE_SERVER_CONFIG_DIR, +}; use futures::future::Either; use futures::stream::FuturesUnordered; use futures::stream::StreamExt; @@ -380,9 +382,8 @@ where #[cfg(test)] mod test { use super::*; - use clickhouse_admin_types::ClickhouseHost; - use clickhouse_admin_types::KeeperId; - use clickhouse_admin_types::ServerId; + use clickhouse_admin_types::keeper::KeeperId; + use clickhouse_admin_types::server::ServerId; use nexus_types::deployment::BlueprintZoneConfig; use nexus_types::deployment::BlueprintZoneImageSource; use nexus_types::deployment::BlueprintZoneType; diff --git a/nexus/reconfigurator/planning/src/blueprint_builder/clickhouse.rs b/nexus/reconfigurator/planning/src/blueprint_builder/clickhouse.rs index 2af9855045b..6fa3380f060 100644 --- a/nexus/reconfigurator/planning/src/blueprint_builder/clickhouse.rs +++ b/nexus/reconfigurator/planning/src/blueprint_builder/clickhouse.rs @@ -5,7 +5,9 @@ //! A mechanism for allocating clickhouse keeper and server nodes for clustered //! clickhouse setups during blueprint planning -use clickhouse_admin_types::{ClickhouseKeeperClusterMembership, KeeperId}; +use clickhouse_admin_types::keeper::{ + ClickhouseKeeperClusterMembership, KeeperId, +}; use nexus_types::deployment::{ BlueprintZoneDisposition, BlueprintZoneType, ClickhouseClusterConfig, }; @@ -291,7 +293,7 @@ impl ClickhouseAllocator { #[cfg(test)] pub mod test { use super::*; - use clickhouse_admin_types::ServerId; + use clickhouse_admin_types::server::ServerId; use omicron_common::api::external::Generation; use omicron_test_utils::dev::test_setup_log; use std::collections::BTreeMap; diff --git a/nexus/reconfigurator/planning/src/system.rs b/nexus/reconfigurator/planning/src/system.rs index 7c530b87206..82c318d3860 100644 --- a/nexus/reconfigurator/planning/src/system.rs +++ b/nexus/reconfigurator/planning/src/system.rs @@ -8,7 +8,7 @@ use anyhow::{Context, anyhow, bail, ensure}; use chrono::DateTime; use chrono::Utc; -use clickhouse_admin_types::ClickhouseKeeperClusterMembership; +use clickhouse_admin_types::keeper::ClickhouseKeeperClusterMembership; use gateway_client::types::RotState; use gateway_client::types::SpComponentCaboose; use gateway_client::types::SpState; diff --git a/nexus/reconfigurator/planning/tests/integration_tests/planner.rs b/nexus/reconfigurator/planning/tests/integration_tests/planner.rs index b9a06e34100..6dc99b71265 100644 --- a/nexus/reconfigurator/planning/tests/integration_tests/planner.rs +++ b/nexus/reconfigurator/planning/tests/integration_tests/planner.rs @@ -5,8 +5,8 @@ use assert_matches::assert_matches; use chrono::DateTime; use chrono::Utc; -use clickhouse_admin_types::ClickhouseKeeperClusterMembership; -use clickhouse_admin_types::KeeperId; +use clickhouse_admin_types::keeper::ClickhouseKeeperClusterMembership; +use clickhouse_admin_types::keeper::KeeperId; use expectorate::assert_contents; use iddqd::IdOrdMap; use nexus_reconfigurator_planning::blueprint_editor::ExternalNetworkingAllocator; diff --git a/nexus/types/src/deployment/blueprint_diff.rs b/nexus/types/src/deployment/blueprint_diff.rs index b2966e18569..6e24d86908d 100644 --- a/nexus/types/src/deployment/blueprint_diff.rs +++ b/nexus/types/src/deployment/blueprint_diff.rs @@ -1272,7 +1272,7 @@ pub struct ClickhouseClusterConfigDiffTables { impl ClickhouseClusterConfigDiffTables { pub fn diff_collection_and_blueprint( - before: &clickhouse_admin_types::ClickhouseKeeperClusterMembership, + before: &clickhouse_admin_types::keeper::ClickhouseKeeperClusterMembership, after: &ClickhouseClusterConfig, ) -> Self { let leader_committed_log_index = if before.leader_committed_log_index @@ -1526,7 +1526,7 @@ impl ClickhouseClusterConfigDiffTables { /// We are diffing a `Collection` and `Blueprint` but the latest blueprint /// does not have a ClickhouseClusterConfig. pub fn removed_from_collection( - before: &clickhouse_admin_types::ClickhouseKeeperClusterMembership, + before: &clickhouse_admin_types::keeper::ClickhouseKeeperClusterMembership, ) -> Self { // There's only so much information in a collection. Show what we can. let metadata = KvList::new( diff --git a/nexus/types/src/deployment/clickhouse.rs b/nexus/types/src/deployment/clickhouse.rs index 0d74ff299b5..122e2885924 100644 --- a/nexus/types/src/deployment/clickhouse.rs +++ b/nexus/types/src/deployment/clickhouse.rs @@ -4,7 +4,8 @@ //! Types used in blueprints related to clickhouse configuration -use clickhouse_admin_types::{KeeperId, ServerId}; +use clickhouse_admin_types::keeper::KeeperId; +use clickhouse_admin_types::server::ServerId; use daft::Diffable; use omicron_common::api::external::Generation; use omicron_uuid_kinds::OmicronZoneUuid; diff --git a/nexus/types/src/inventory.rs b/nexus/types/src/inventory.rs index 34614d545e9..6ce6f4d0828 100644 --- a/nexus/types/src/inventory.rs +++ b/nexus/types/src/inventory.rs @@ -13,7 +13,7 @@ use crate::external_api::params::PhysicalDiskKind; use crate::external_api::params::UninitializedSledId; use chrono::DateTime; use chrono::Utc; -use clickhouse_admin_types::ClickhouseKeeperClusterMembership; +use clickhouse_admin_types::keeper::ClickhouseKeeperClusterMembership; use daft::Diffable; pub use gateway_client::types::PowerState; pub use gateway_client::types::RotImageError;