[nexus] Split Nexus configuration (package vs deployment) (#1174)

This is a major part of #822

This PR splits the Nexus configuration into "things known at package time" and "things known at deployment".

Due to hardcoded configurations, the two have historically been conflated. However, there are a number of
parameters which will only be known at deployment time, including:

- The addresses Nexus should use on the underlay (these are derived from the sled's /64, which is variable)
- The UUID of the Nexus instance (multiple Nexus instances may be executing simultaneously)
- The rack subnet
- (While it is still an IP address) the IP address of CRDB

Author: smklein

Cargo.lock

@@ -29,6 +29,7 @@ steno = { git = "https://github.com/oxidecomputer/steno", branch = "main" }
 thiserror = "1.0"
 tokio = { version = "1.18", features = [ "full" ] }
 tokio-postgres = { version = "0.7", features = [ "with-chrono-0_4", "with-uuid-1" ] }
+toml = "0.5.9"
 uuid = { version = "1.1.0", features = [ "serde", "v4" ] }
 parse-display = "0.5.4"
 progenitor = { git = "https://github.com/oxidecomputer/progenitor" }

common/Cargo.toml

@@ -24,7 +24,8 @@ pub mod address;
 pub mod api;
 pub mod backoff;
 pub mod cmd;
-pub mod config;
+pub mod nexus_config;
+pub mod postgres_config;
 
 #[macro_export]
 macro_rules! generate_logging_api {

common/src/lib.rs

@@ -0,0 +1,128 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at https://mozilla.org/MPL/2.0/.
+
+//! Configuration parameters to Nexus that are usually only known
+//! at deployment time.
+
+use super::address::{Ipv6Subnet, RACK_PREFIX};
+use super::postgres_config::PostgresConfigWithUrl;
+use dropshot::ConfigDropshot;
+use serde::{Deserialize, Serialize};
+use serde_with::serde_as;
+use serde_with::DisplayFromStr;
+use std::fmt;
+use std::path::{Path, PathBuf};
+use uuid::Uuid;
+
+#[derive(Debug)]
+pub struct LoadError {
+    pub path: PathBuf,
+    pub kind: LoadErrorKind,
+}
+
+#[derive(Debug)]
+pub struct InvalidTunable {
+    pub tunable: String,
+    pub message: String,
+}
+
+impl std::fmt::Display for InvalidTunable {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        write!(f, "invalid \"{}\": \"{}\"", self.tunable, self.message)
+    }
+}
+impl std::error::Error for InvalidTunable {}
+
+#[derive(Debug)]
+pub enum LoadErrorKind {
+    Io(std::io::Error),
+    Parse(toml::de::Error),
+    InvalidTunable(InvalidTunable),
+}
+
+impl From<(PathBuf, std::io::Error)> for LoadError {
+    fn from((path, err): (PathBuf, std::io::Error)) -> Self {
+        LoadError { path, kind: LoadErrorKind::Io(err) }
+    }
+}
+
+impl From<(PathBuf, toml::de::Error)> for LoadError {
+    fn from((path, err): (PathBuf, toml::de::Error)) -> Self {
+        LoadError { path, kind: LoadErrorKind::Parse(err) }
+    }
+}
+
+impl std::error::Error for LoadError {}
+
+impl fmt::Display for LoadError {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        match &self.kind {
+            LoadErrorKind::Io(e) => {
+                write!(f, "read \"{}\": {}", self.path.display(), e)
+            }
+            LoadErrorKind::Parse(e) => {
+                write!(f, "parse \"{}\": {}", self.path.display(), e)
+            }
+            LoadErrorKind::InvalidTunable(inner) => {
+                write!(
+                    f,
+                    "invalid tunable \"{}\": {}",
+                    self.path.display(),
+                    inner,
+                )
+            }
+        }
+    }
+}
+
+impl std::cmp::PartialEq<std::io::Error> for LoadError {
+    fn eq(&self, other: &std::io::Error) -> bool {
+        if let LoadErrorKind::Io(e) = &self.kind {
+            e.kind() == other.kind()
+        } else {
+            false
+        }
+    }
+}
+
+#[serde_as]
+#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
+#[serde(tag = "type", rename_all = "snake_case")]
+#[allow(clippy::large_enum_variant)]
+pub enum Database {
+    FromDns,
+    FromUrl {
+        #[serde_as(as = "DisplayFromStr")]
+        url: PostgresConfigWithUrl,
+    },
+}
+
+#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
+pub struct DeploymentConfig {
+    /// Uuid of the Nexus instance
+    pub id: Uuid,
+    /// Dropshot configuration for external API server
+    pub dropshot_external: ConfigDropshot,
+    /// Dropshot configuration for internal API server
+    pub dropshot_internal: ConfigDropshot,
+    /// Portion of the IP space to be managed by the Rack.
+    pub subnet: Ipv6Subnet<RACK_PREFIX>,
+    /// DB configuration.
+    pub database: Database,
+}
+
+impl DeploymentConfig {
+    /// Load a `DeploymentConfig` from the given TOML file
+    ///
+    /// This config object can then be used to create a new `Nexus`.
+    /// The format is described in the README.
+    pub fn from_file<P: AsRef<Path>>(path: P) -> Result<Self, LoadError> {
+        let path = path.as_ref();
+        let file_contents = std::fs::read_to_string(path)
+            .map_err(|e| (path.to_path_buf(), e))?;
+        let config_parsed: Self = toml::from_str(&file_contents)
+            .map_err(|e| (path.to_path_buf(), e))?;
+        Ok(config_parsed)
+    }
+}

common/src/nexus_config.rs

@@ -19,7 +19,7 @@ async fn do_full_setup() {
 // Wraps exclusively the CockroachDB portion of setup/teardown.
 async fn do_crdb_setup() {
     let cfg = nexus_test_utils::load_test_config();
-    let logctx = LogContext::new("crdb_setup", &cfg.log);
+    let logctx = LogContext::new("crdb_setup", &cfg.pkg.log);
     let mut db = test_setup_database(&logctx.log).await;
     db.cleanup().await.unwrap();
 }

common/src/config.rs renamed to common/src/postgres_config.rs

@@ -2,9 +2,6 @@
 # Oxide API: example configuration file
 #
 
-# Identifier for this instance of Nexus
-id = "e6bff1ff-24fb-49dc-a54e-c6a350cd4d6c"
-
 [console]
 # Directory for static assets. Absolute path or relative to CWD.
 static_dir = "nexus/static" # TODO: figure out value
@@ -20,21 +17,6 @@ session_absolute_timeout_minutes = 480
 # TODO(https://github.com/oxidecomputer/omicron/issues/372): Remove "spoof".
 schemes_external = ["spoof", "session_cookie"]
 
-[database]
-# URL for connecting to the database
-url = "postgresql://root@127.0.0.1:32221/omicron?sslmode=disable"
-
-[dropshot_external]
-# IP address and TCP port on which to listen for the external API
-bind_address = "127.0.0.1:12220"
-# Allow larger request bodies (1MiB) to accomodate firewall endpoints (one
-# rule is ~500 bytes)
-request_body_max_bytes = 1048576
-
-[dropshot_internal]
-# IP address and TCP port on which to listen for the internal API
-bind_address = "127.0.0.1:12221"
-
 [log]
 # Show log messages of this level and more severe
 level = "info"
@@ -51,6 +33,29 @@ mode = "stderr-terminal"
 [timeseries_db]
 address = "[::1]:8123"
 
+[deployment]
+# Identifier for this instance of Nexus
+id = "e6bff1ff-24fb-49dc-a54e-c6a350cd4d6c"
+
+[deployment.dropshot_external]
+# IP address and TCP port on which to listen for the external API
+bind_address = "127.0.0.1:12220"
+# Allow larger request bodies (1MiB) to accomodate firewall endpoints (one
+# rule is ~500 bytes)
+request_body_max_bytes = 1048576
+
+[deployment.dropshot_internal]
+# IP address and TCP port on which to listen for the internal API
+bind_address = "127.0.0.1:12221"
+
+[deployment.subnet]
+net = "fd00:1122:3344:0100::/56"
+
+[deployment.database]
+# URL for connecting to the database
+type = "from_url"
+url = "postgresql://root@127.0.0.1:32221/omicron?sslmode=disable"
+
 # Tunable configuration parameters, for testing or experimentation
 [tunables]
 

nexus/benches/setup_benchmark.rs

@@ -112,7 +112,7 @@ impl Nexus {
         authz: Arc<authz::Authz>,
     ) -> Arc<Nexus> {
         let pool = Arc::new(pool);
-        let my_sec_id = db::SecId::from(config.id);
+        let my_sec_id = db::SecId::from(config.deployment.id);
         let db_datastore = Arc::new(db::DataStore::new(Arc::clone(&pool)));
         let sec_store = Arc::new(db::CockroachDbSecStore::new(
             my_sec_id,
@@ -127,7 +127,7 @@ impl Nexus {
             sec_store,
         ));
         let timeseries_client =
-            oximeter_db::Client::new(config.timeseries_db.address, &log);
+            oximeter_db::Client::new(config.pkg.timeseries_db.address, &log);
 
         // TODO-cleanup We may want a first-class subsystem for managing startup
         // background tasks.  It could use a Future for each one, a status enum
@@ -143,7 +143,7 @@ impl Nexus {
             populate_start(populate_ctx, Arc::clone(&db_datastore));
 
         let nexus = Nexus {
-            id: config.id,
+            id: config.deployment.id,
             rack_id,
             log: log.new(o!()),
             api_rack_identity: db::model::RackIdentity::new(rack_id),
@@ -153,8 +153,8 @@ impl Nexus {
             recovery_task: std::sync::Mutex::new(None),
             populate_status,
             timeseries_client,
-            updates_config: config.updates.clone(),
-            tunables: config.tunables.clone(),
+            updates_config: config.pkg.updates.clone(),
+            tunables: config.pkg.tunables.clone(),
             opctx_alloc: OpContext::for_background(
                 log.new(o!("component" => "InstanceAllocator")),
                 Arc::clone(&authz),