[internal-dns] Avoid 'picking ports'

common/Cargo.toml

@@ -29,6 +29,7 @@ steno = { git = "https://github.com/oxidecomputer/steno", branch = "main" }
 thiserror = "1.0"
 tokio = { version = "1.18", features = [ "full" ] }
 tokio-postgres = { version = "0.7", features = [ "with-chrono-0_4", "with-uuid-1" ] }
+toml = "0.5.9"
 uuid = { version = "1.1.0", features = [ "serde", "v4" ] }
 parse-display = "0.5.4"
 progenitor = { git = "https://github.com/oxidecomputer/progenitor" }

common/src/lib.rs

@@ -24,7 +24,8 @@ pub mod address;
 pub mod api;
 pub mod backoff;
 pub mod cmd;
-pub mod config;
+pub mod nexus_config;
+pub mod postgres_config;
 
 #[macro_export]
 macro_rules! generate_logging_api {

common/src/nexus_config.rs

@@ -0,0 +1,128 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at https://mozilla.org/MPL/2.0/.
+
+//! Configuration parameters to Nexus that are usually only known
+//! at deployment time.
+
+use super::address::{Ipv6Subnet, RACK_PREFIX};
+use super::postgres_config::PostgresConfigWithUrl;
+use dropshot::ConfigDropshot;
+use serde::{Deserialize, Serialize};
+use serde_with::serde_as;
+use serde_with::DisplayFromStr;
+use std::fmt;
+use std::path::{Path, PathBuf};
+use uuid::Uuid;
+
+#[derive(Debug)]
+pub struct LoadError {
+    pub path: PathBuf,
+    pub kind: LoadErrorKind,
+}
+
+#[derive(Debug)]
+pub struct InvalidTunable {
+    pub tunable: String,
+    pub message: String,
+}
+
+impl std::fmt::Display for InvalidTunable {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        write!(f, "invalid \"{}\": \"{}\"", self.tunable, self.message)
+    }
+}
+impl std::error::Error for InvalidTunable {}
+
+#[derive(Debug)]
+pub enum LoadErrorKind {
+    Io(std::io::Error),
+    Parse(toml::de::Error),
+    InvalidTunable(InvalidTunable),
+}
+
+impl From<(PathBuf, std::io::Error)> for LoadError {
+    fn from((path, err): (PathBuf, std::io::Error)) -> Self {
+        LoadError { path, kind: LoadErrorKind::Io(err) }
+    }
+}
+
+impl From<(PathBuf, toml::de::Error)> for LoadError {
+    fn from((path, err): (PathBuf, toml::de::Error)) -> Self {
+        LoadError { path, kind: LoadErrorKind::Parse(err) }
+    }
+}
+
+impl std::error::Error for LoadError {}
+
+impl fmt::Display for LoadError {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        match &self.kind {
+            LoadErrorKind::Io(e) => {
+                write!(f, "read \"{}\": {}", self.path.display(), e)
+            }
+            LoadErrorKind::Parse(e) => {
+                write!(f, "parse \"{}\": {}", self.path.display(), e)
+            }
+            LoadErrorKind::InvalidTunable(inner) => {
+                write!(
+                    f,
+                    "invalid tunable \"{}\": {}",
+                    self.path.display(),
+                    inner,
+                )
+            }
+        }
+    }
+}
+
+impl std::cmp::PartialEq<std::io::Error> for LoadError {
+    fn eq(&self, other: &std::io::Error) -> bool {
+        if let LoadErrorKind::Io(e) = &self.kind {
+            e.kind() == other.kind()
+        } else {
+            false
+        }
+    }
+}
+
+#[serde_as]
+#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
+#[serde(tag = "type", rename_all = "snake_case")]
+#[allow(clippy::large_enum_variant)]
+pub enum Database {
+    FromDns,
+    FromUrl {
+        #[serde_as(as = "DisplayFromStr")]
+        url: PostgresConfigWithUrl,
+    },
+}
+
+#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
+pub struct DeploymentConfig {
+    /// Uuid of the Nexus instance
+    pub id: Uuid,
+    /// Dropshot configuration for external API server
+    pub dropshot_external: ConfigDropshot,
+    /// Dropshot configuration for internal API server
+    pub dropshot_internal: ConfigDropshot,
+    /// Portion of the IP space to be managed by the Rack.
+    pub subnet: Ipv6Subnet<RACK_PREFIX>,
+    /// DB configuration.
+    pub database: Database,
+}
+
+impl DeploymentConfig {
+    /// Load a `DeploymentConfig` from the given TOML file
+    ///
+    /// This config object can then be used to create a new `Nexus`.
+    /// The format is described in the README.
+    pub fn from_file<P: AsRef<Path>>(path: P) -> Result<Self, LoadError> {
+        let path = path.as_ref();
+        let file_contents = std::fs::read_to_string(path)
+            .map_err(|e| (path.to_path_buf(), e))?;
+        let config_parsed: Self = toml::from_str(&file_contents)
+            .map_err(|e| (path.to_path_buf(), e))?;
+        Ok(config_parsed)
+    }
+}

internal-dns/Cargo.toml

@@ -30,7 +30,6 @@ expectorate = "1.0.5"
 omicron-test-utils = { path = "../test-utils" }
 openapiv3 = "1.0"
 openapi-lint = { git = "https://github.com/oxidecomputer/openapi-lint", branch = "main" }
-portpicker = "0.1"
 serde_json = "1.0"
 subprocess = "0.2.9"
 trust-dns-resolver = "0.21"

internal-dns/src/bin/dns-server.rs

@@ -54,20 +54,19 @@ async fn main() -> Result<(), anyhow::Error> {
 
     let db = Arc::new(sled::open(&config.data.storage_path)?);
 
-    {
+    let _dns_server = {
         let db = db.clone();
         let log = log.clone();
         let dns_config = internal_dns::dns_server::Config {
             bind_address: dns_address.to_string(),
             zone: zone.to_string(),
         };
-        tokio::spawn(async move {
-            internal_dns::dns_server::run(log, db, dns_config).await
-        });
-    }
+        internal_dns::dns_server::run(log, db, dns_config).await?
+    };
 
-    let server = internal_dns::start_server(config, log, db).await?;
-    server
+    let dropshot_server =
+        internal_dns::start_dropshot_server(config, log, db).await?;
+    dropshot_server
         .await
         .map_err(|error_message| anyhow!("server exiting: {}", error_message))
 }

internal-dns/src/dns_server.rs

@@ -34,23 +34,43 @@ pub struct Config {
     pub zone: String,
 }
 
-pub async fn run(log: Logger, db: Arc<sled::Db>, config: Config) -> Result<()> {
+pub struct Server {
+    pub address: SocketAddr,
+    pub handle: tokio::task::JoinHandle<Result<()>>,
+}
+
+impl Drop for Server {
+    fn drop(&mut self) {
+        self.handle.abort()
+    }
+}
+
+pub async fn run(
+    log: Logger,
+    db: Arc<sled::Db>,
+    config: Config,
+) -> Result<Server> {
     let socket = Arc::new(UdpSocket::bind(config.bind_address).await?);
+    let address = socket.local_addr()?;
 
-    loop {
-        let mut buf = vec![0u8; 16384];
-        let (n, src) = socket.recv_from(&mut buf).await?;
-        buf.resize(n, 0);
+    let handle = tokio::task::spawn(async move {
+        loop {
+            let mut buf = vec![0u8; 16384];
+            let (n, src) = socket.recv_from(&mut buf).await?;
+            buf.resize(n, 0);
 
-        let socket = socket.clone();
-        let log = log.clone();
-        let db = db.clone();
-        let zone = config.zone.clone();
+            let socket = socket.clone();
+            let log = log.clone();
+            let db = db.clone();
+            let zone = config.zone.clone();
 
-        tokio::spawn(async move {
-            handle_req(log, db, socket, src, buf, zone).await
-        });
-    }
+            tokio::spawn(async move {
+                handle_req(log, db, socket, src, buf, zone).await
+            });
+        }
+    });
+
+    Ok(Server { address, handle })
 }
 
 async fn respond_nxdomain(

internal-dns/src/lib.rs

@@ -20,7 +20,7 @@ pub struct Config {
     pub data: dns_data::Config,
 }
 
-pub async fn start_server(
+pub async fn start_dropshot_server(
     config: Config,
     log: slog::Logger,
     db: Arc<sled::Db>,

internal-dns/tests/basic_test.rs

@@ -2,7 +2,7 @@
 // License, v. 2.0. If a copy of the MPL was not distributed with this
 // file, You can obtain one at https://mozilla.org/MPL/2.0/.
 
-use std::net::{Ipv6Addr, SocketAddr, SocketAddrV6};
+use std::net::Ipv6Addr;
 use std::sync::Arc;
 
 use anyhow::{Context, Result};
@@ -280,13 +280,16 @@ pub async fn servfail() -> Result<(), anyhow::Error> {
 struct TestContext {
     client: Client,
     resolver: TokioAsyncResolver,
-    server: dropshot::HttpServer<Arc<internal_dns::dropshot_server::Context>>,
+    dns_server: internal_dns::dns_server::Server,
+    dropshot_server:
+        dropshot::HttpServer<Arc<internal_dns::dropshot_server::Context>>,
     tmp: tempdir::TempDir,
 }
 
 impl TestContext {
     async fn cleanup(self) {
-        self.server.close().await.expect("Failed to clean up server");
+        drop(self.dns_server);
+        self.dropshot_server.close().await.expect("Failed to clean up server");
         self.tmp.close().expect("Failed to clean up tmp directory");
     }
 }
@@ -295,7 +298,7 @@ async fn init_client_server(
     zone: String,
 ) -> Result<TestContext, anyhow::Error> {
     // initialize dns server config
-    let (tmp, config, dropshot_port, dns_port) = test_config()?;
+    let (tmp, config) = test_config()?;
     let log = config
         .log
         .to_logger("internal-dns")
@@ -305,17 +308,21 @@ async fn init_client_server(
     let db = Arc::new(sled::open(&config.data.storage_path)?);
     db.clear()?;
 
-    let client =
-        Client::new(&format!("http://[::1]:{}", dropshot_port), log.clone());
+    // launch a dns server
+    let dns_server = {
+        let db = db.clone();
+        let log = log.clone();
+        let dns_config = internal_dns::dns_server::Config {
+            bind_address: "[::1]:0".into(),
+            zone,
+        };
+
+        internal_dns::dns_server::run(log, db, dns_config).await?
+    };
 
     let mut rc = ResolverConfig::new();
     rc.add_name_server(NameServerConfig {
-        socket_addr: SocketAddr::V6(SocketAddrV6::new(
-            Ipv6Addr::LOCALHOST,
-            dns_port,
-            0,
-            0,
-        )),
+        socket_addr: dns_server.address,
         protocol: Protocol::Udp,
         tls_dns_name: None,
         trust_nx_responses: false,
@@ -325,33 +332,21 @@ async fn init_client_server(
     let resolver =
         TokioAsyncResolver::tokio(rc, ResolverOpts::default()).unwrap();
 
-    // launch a dns server
-    {
-        let db = db.clone();
-        let log = log.clone();
-        let dns_config = internal_dns::dns_server::Config {
-            bind_address: format!("[::1]:{}", dns_port),
-            zone,
-        };
-
-        tokio::spawn(async move {
-            internal_dns::dns_server::run(log, db, dns_config).await
-        });
-    }
-
     // launch a dropshot server
-    let server = internal_dns::start_server(config, log, db).await?;
+    let dropshot_server =
+        internal_dns::start_dropshot_server(config, log.clone(), db).await?;
 
     // wait for server to start
     tokio::time::sleep(tokio::time::Duration::from_millis(250)).await;
 
-    Ok(TestContext { client, resolver, server, tmp })
+    let client =
+        Client::new(&format!("http://{}", dropshot_server.local_addr()), log);
+
+    Ok(TestContext { client, resolver, dns_server, dropshot_server, tmp })
 }
 
 fn test_config(
-) -> Result<(tempdir::TempDir, internal_dns::Config, u16, u16), anyhow::Error> {
-    let dropshot_port = portpicker::pick_unused_port().expect("pick port");
-    let dns_port = portpicker::pick_unused_port().expect("pick port");
+) -> Result<(tempdir::TempDir, internal_dns::Config), anyhow::Error> {
     let tmp_dir = tempdir::TempDir::new("internal-dns-test")?;
     let mut storage_path = tmp_dir.path().to_path_buf();
     storage_path.push("test");
@@ -362,7 +357,7 @@ fn test_config(
             level: dropshot::ConfigLoggingLevel::Info,
         },
         dropshot: dropshot::ConfigDropshot {
-            bind_address: format!("[::1]:{}", dropshot_port).parse().unwrap(),
+            bind_address: format!("[::1]:0").parse().unwrap(),
             request_body_max_bytes: 1024,
             ..Default::default()
         },
@@ -372,5 +367,5 @@ fn test_config(
         },
     };
 
-    Ok((tmp_dir, config, dropshot_port, dns_port))
+    Ok((tmp_dir, config))
 }

nexus/benches/setup_benchmark.rs

@@ -19,7 +19,7 @@ async fn do_full_setup() {
 // Wraps exclusively the CockroachDB portion of setup/teardown.
 async fn do_crdb_setup() {
     let cfg = nexus_test_utils::load_test_config();
-    let logctx = LogContext::new("crdb_setup", &cfg.log);
+    let logctx = LogContext::new("crdb_setup", &cfg.pkg.log);
     let mut db = test_setup_database(&logctx.log).await;
     db.cleanup().await.unwrap();
 }