Discretionary external DNS zones (#6581)

External DNS addresses are not yet in the policy (#3732), so we must
grovel for them in the parent blueprint (and also a bit in the planning
input). Most of the grubbiness here will go away when that's fixed; see
the `TODO-cleanup` notes.

nexus/reconfigurator/planning/src/blueprint_builder/builder.rs

@@ -25,6 +25,7 @@ use nexus_types::deployment::BlueprintZoneType;
 use nexus_types::deployment::BlueprintZonesConfig;
 use nexus_types::deployment::CockroachDbPreserveDowngrade;
 use nexus_types::deployment::DiskFilter;
+use nexus_types::deployment::OmicronZoneExternalFloatingAddr;
 use nexus_types::deployment::OmicronZoneExternalFloatingIp;
 use nexus_types::deployment::OmicronZoneExternalSnatIp;
 use nexus_types::deployment::PlanningInput;
@@ -70,6 +71,7 @@ use std::fmt;
 use std::hash::Hash;
 use std::net::IpAddr;
 use std::net::Ipv6Addr;
+use std::net::SocketAddr;
 use std::net::SocketAddrV6;
 use thiserror::Error;
 use typed_rng::TypedUuidRng;
@@ -96,6 +98,8 @@ pub enum Error {
     NoNexusZonesInParentBlueprint,
     #[error("no Boundary NTP zones exist in parent blueprint")]
     NoBoundaryNtpZonesInParentBlueprint,
+    #[error("no external DNS IP addresses are available")]
+    NoExternalDnsIpAvailable,
     #[error("no external service IP addresses are available")]
     NoExternalServiceIpAvailable,
     #[error("no system MAC addresses are available")]
@@ -118,6 +122,8 @@ pub enum Error {
         "can only have {MAX_INTERNAL_DNS_REDUNDANCY} internal DNS servers"
     )]
     TooManyDnsServers,
+    #[error("planner produced too many {kind:?} zones")]
+    TooManyZones { kind: ZoneKind },
 }
 
 /// Describes whether an idempotent "ensure" operation resulted in action taken
@@ -704,6 +710,97 @@ impl<'a> BlueprintBuilder<'a> {
         Ok(EnsureMultiple::Changed { added: to_add, removed: 0 })
     }
 
+    fn sled_add_zone_external_dns(
+        &mut self,
+        sled_id: SledUuid,
+    ) -> Result<Ensure, Error> {
+        let id = self.rng.zone_rng.next();
+        let ExternalNetworkingChoice {
+            external_ip,
+            nic_ip,
+            nic_subnet,
+            nic_mac,
+        } = self.external_networking.for_new_external_dns()?;
+        let nic = NetworkInterface {
+            id: self.rng.network_interface_rng.next(),
+            kind: NetworkInterfaceKind::Service { id: id.into_untyped_uuid() },
+            name: format!("external-dns-{id}").parse().unwrap(),
+            ip: nic_ip,
+            mac: nic_mac,
+            subnet: nic_subnet,
+            vni: Vni::SERVICES_VNI,
+            primary: true,
+            slot: 0,
+            transit_ips: vec![],
+        };
+
+        let underlay_address = self.sled_alloc_ip(sled_id)?;
+        let http_address =
+            SocketAddrV6::new(underlay_address, DNS_HTTP_PORT, 0, 0);
+        let dns_address = OmicronZoneExternalFloatingAddr {
+            id: self.rng.external_ip_rng.next(),
+            addr: SocketAddr::new(external_ip, DNS_PORT),
+        };
+        let pool_name =
+            self.sled_select_zpool(sled_id, ZoneKind::ExternalDns)?;
+        let zone_type =
+            BlueprintZoneType::ExternalDns(blueprint_zone_type::ExternalDns {
+                dataset: OmicronZoneDataset { pool_name: pool_name.clone() },
+                http_address,
+                dns_address,
+                nic,
+            });
+
+        let zone = BlueprintZoneConfig {
+            disposition: BlueprintZoneDisposition::InService,
+            id,
+            underlay_address,
+            filesystem_pool: Some(pool_name),
+            zone_type,
+        };
+        self.sled_add_zone(sled_id, zone)?;
+        Ok(Ensure::Added)
+    }
+
+    pub fn sled_ensure_zone_multiple_external_dns(
+        &mut self,
+        sled_id: SledUuid,
+        desired_zone_count: usize,
+    ) -> Result<EnsureMultiple, Error> {
+        // How many external DNS zones do we want to add?
+        let count =
+            self.sled_num_running_zones_of_kind(sled_id, ZoneKind::ExternalDns);
+        let to_add = match desired_zone_count.checked_sub(count) {
+            Some(0) => return Ok(EnsureMultiple::NotNeeded),
+            Some(n) => n,
+            None => {
+                return Err(Error::Planner(anyhow!(
+                    "removing an external DNS zone not yet supported \
+                     (sled {sled_id} has {count}; \
+                     planner wants {desired_zone_count})"
+                )));
+            }
+        };
+
+        // Running out of DNS addresses is not a fatal error. This happens,
+        // for instance, when a sled is first marked expunged, since the
+        // available addresses are collected before planning, but the
+        // zones on the sled aren't marked expunged until after planning
+        // has begun. The *next* round of planning will add them back in,
+        // since it will see the zones as expunged and recycle their
+        // addresses.
+        let mut added = 0;
+        for _ in 0..to_add {
+            match self.sled_add_zone_external_dns(sled_id) {
+                Ok(_) => added += 1,
+                Err(Error::NoExternalDnsIpAvailable) => break,
+                Err(e) => return Err(e),
+            }
+        }
+
+        Ok(EnsureMultiple::Changed { added, removed: 0 })
+    }
+
     pub fn sled_ensure_zone_ntp(
         &mut self,
         sled_id: SledUuid,
@@ -1281,6 +1378,32 @@ impl<'a> BlueprintBuilder<'a> {
             })?;
         Ok(&details.resources)
     }
+
+    /// Determine the number of desired external DNS zones by counting
+    /// unique addresses in the parent blueprint.
+    ///
+    /// TODO-cleanup: Remove when external DNS addresses are in the policy.
+    pub fn count_parent_external_dns_zones(&self) -> usize {
+        self.parent_blueprint
+            .all_omicron_zones(BlueprintZoneFilter::All)
+            .filter_map(|(_id, zone)| match &zone.zone_type {
+                BlueprintZoneType::ExternalDns(dns) => {
+                    Some(dns.dns_address.addr.ip())
+                }
+                _ => None,
+            })
+            .collect::<HashSet<IpAddr>>()
+            .len()
+    }
+
+    /// Allow a test to manually add an external DNS address, which could
+    /// ordinarily only come from RSS.
+    ///
+    /// TODO-cleanup: Remove when external DNS addresses are in the policy.
+    #[cfg(test)]
+    pub fn add_external_dns_ip(&mut self, addr: IpAddr) {
+        self.external_networking.add_external_dns_ip(addr);
+    }
 }
 
 #[derive(Debug)]

nexus/reconfigurator/planning/src/blueprint_builder/external_networking.rs

@@ -41,6 +41,12 @@ pub(super) struct BuilderExternalNetworking<'a> {
     boundary_ntp_v6_ips: AvailableIterator<'static, Ipv6Addr>,
     nexus_v4_ips: AvailableIterator<'static, Ipv4Addr>,
     nexus_v6_ips: AvailableIterator<'static, Ipv6Addr>,
+    external_dns_v4_ips: AvailableIterator<'static, Ipv4Addr>,
+    external_dns_v6_ips: AvailableIterator<'static, Ipv6Addr>,
+
+    // External DNS server addresses currently only come from RSS;
+    // see https://github.com/oxidecomputer/omicron/issues/3732
+    available_external_dns_ips: BTreeSet<IpAddr>,
 
     // Allocator for external IPs for service zones
     external_ip_alloc: ExternalIpAllocator<'a>,
@@ -99,6 +105,10 @@ impl<'a> BuilderExternalNetworking<'a> {
             HashSet::new();
         let mut existing_boundary_ntp_v6_ips: HashSet<Ipv6Addr> =
             HashSet::new();
+        let mut existing_external_dns_v4_ips: HashSet<Ipv4Addr> =
+            HashSet::new();
+        let mut existing_external_dns_v6_ips: HashSet<Ipv6Addr> =
+            HashSet::new();
         let mut external_ip_alloc =
             ExternalIpAllocator::new(input.service_ip_pool_ranges());
         let mut used_macs: HashSet<MacAddr> = HashSet::new();
@@ -132,6 +142,18 @@ impl<'a> BuilderExternalNetworking<'a> {
                         }
                     }
                 },
+                BlueprintZoneType::ExternalDns(dns) => match dns.nic.ip {
+                    IpAddr::V4(ip) => {
+                        if !existing_external_dns_v4_ips.insert(ip) {
+                            bail!("duplicate external DNS IP: {ip}");
+                        }
+                    }
+                    IpAddr::V6(ip) => {
+                        if !existing_external_dns_v6_ips.insert(ip) {
+                            bail!("duplicate external DNS IP: {ip}");
+                        }
+                    }
+                },
                 _ => (),
             }
 
@@ -149,6 +171,32 @@ impl<'a> BuilderExternalNetworking<'a> {
             }
         }
 
+        // Recycle the IP addresses of expunged external DNS zones,
+        // ensuring that those addresses aren't currently in use.
+        // TODO: Remove when external DNS addresses come from policy.
+        let used_external_dns_ips = parent_blueprint
+            .all_omicron_zones(BlueprintZoneFilter::ShouldBeRunning)
+            .filter_map(|(_, zone)| {
+                if let BlueprintZoneType::ExternalDns(dns) = &zone.zone_type {
+                    Some(dns.dns_address.addr.ip())
+                } else {
+                    None
+                }
+            })
+            .collect::<BTreeSet<IpAddr>>();
+        let available_external_dns_ips = parent_blueprint
+            .all_omicron_zones(BlueprintZoneFilter::Expunged)
+            .filter_map(|(_, zone)| {
+                if let BlueprintZoneType::ExternalDns(dns) = &zone.zone_type {
+                    let ip = dns.dns_address.addr.ip();
+                    if !used_external_dns_ips.contains(&ip) {
+                        return Some(ip);
+                    }
+                }
+                None
+            })
+            .collect::<BTreeSet<IpAddr>>();
+
         // Check the planning input: there shouldn't be any external networking
         // resources in the database (the source of `input`) that we don't know
         // about from the parent blueprint.
@@ -192,6 +240,16 @@ impl<'a> BuilderExternalNetworking<'a> {
             NTP_OPTE_IPV6_SUBNET.iter().skip(NUM_INITIAL_RESERVED_IP_ADDRESSES),
             existing_boundary_ntp_v6_ips,
         );
+        let external_dns_v4_ips = AvailableIterator::new(
+            DNS_OPTE_IPV4_SUBNET
+                .addr_iter()
+                .skip(NUM_INITIAL_RESERVED_IP_ADDRESSES),
+            existing_external_dns_v4_ips,
+        );
+        let external_dns_v6_ips = AvailableIterator::new(
+            DNS_OPTE_IPV6_SUBNET.iter().skip(NUM_INITIAL_RESERVED_IP_ADDRESSES),
+            existing_external_dns_v6_ips,
+        );
         let available_system_macs =
             AvailableIterator::new(MacAddr::iter_system(), used_macs);
 
@@ -200,6 +258,9 @@ impl<'a> BuilderExternalNetworking<'a> {
             boundary_ntp_v6_ips,
             nexus_v4_ips,
             nexus_v6_ips,
+            external_dns_v4_ips,
+            external_dns_v6_ips,
+            available_external_dns_ips,
             external_ip_alloc,
             available_system_macs,
         })
@@ -274,6 +335,59 @@ impl<'a> BuilderExternalNetworking<'a> {
             nic_mac,
         })
     }
+
+    pub(super) fn for_new_external_dns(
+        &mut self,
+    ) -> Result<ExternalNetworkingChoice, Error> {
+        let external_ip = self
+            .available_external_dns_ips
+            .pop_first()
+            .ok_or(Error::NoExternalDnsIpAvailable)?;
+
+        let (nic_ip, nic_subnet) = match external_ip {
+            IpAddr::V4(_) => (
+                self.external_dns_v4_ips
+                    .next()
+                    .ok_or(Error::ExhaustedOpteIps {
+                        kind: ZoneKind::ExternalDns,
+                    })?
+                    .into(),
+                IpNet::from(*DNS_OPTE_IPV4_SUBNET),
+            ),
+            IpAddr::V6(_) => (
+                self.external_dns_v6_ips
+                    .next()
+                    .ok_or(Error::ExhaustedOpteIps {
+                        kind: ZoneKind::ExternalDns,
+                    })?
+                    .into(),
+                IpNet::from(*DNS_OPTE_IPV6_SUBNET),
+            ),
+        };
+        let nic_mac = self
+            .available_system_macs
+            .next()
+            .ok_or(Error::NoSystemMacAddressAvailable)?;
+
+        Ok(ExternalNetworkingChoice {
+            external_ip,
+            nic_ip,
+            nic_subnet,
+            nic_mac,
+        })
+    }
+
+    /// Allow a test to manually add an external DNS address,
+    /// which could otherwise only be added via RSS.
+    ///
+    /// TODO-cleanup: Remove when external DNS addresses are in the policy.
+    #[cfg(test)]
+    pub fn add_external_dns_ip(&mut self, addr: IpAddr) {
+        assert!(
+            self.available_external_dns_ips.insert(addr),
+            "duplicate external DNS IP address"
+        );
+    }
 }
 
 // Helper to validate that the system hasn't gone off the rails. There should
@@ -314,6 +428,7 @@ fn ensure_input_records_appear_in_parent_blueprint(
     let mut all_macs: HashSet<MacAddr> = HashSet::new();
     let mut all_nexus_nic_ips: HashSet<IpAddr> = HashSet::new();
     let mut all_boundary_ntp_nic_ips: HashSet<IpAddr> = HashSet::new();
+    let mut all_external_dns_nic_ips: HashSet<IpAddr> = HashSet::new();
     let mut all_external_ips: HashSet<OmicronZoneExternalIp> = HashSet::new();
 
     // Unlike the construction of the external IP allocator and existing IPs
@@ -329,7 +444,9 @@ fn ensure_input_records_appear_in_parent_blueprint(
             BlueprintZoneType::Nexus(nexus) => {
                 all_nexus_nic_ips.insert(nexus.nic.ip);
             }
-            // TODO: external-dns
+            BlueprintZoneType::ExternalDns(dns) => {
+                all_external_dns_nic_ips.insert(dns.nic.ip);
+            }
             _ => (),
         }
 
@@ -380,7 +497,12 @@ fn ensure_input_records_appear_in_parent_blueprint(
                 }
             }
             IpAddr::V4(ip) if DNS_OPTE_IPV4_SUBNET.contains(ip) => {
-                // TODO check all_dns_nic_ips, once it exists
+                if !all_external_dns_nic_ips.contains(&ip.into()) {
+                    bail!(
+                        "planning input contains unexpected NIC \
+                         (IP not found in parent blueprint): {nic_entry:?}"
+                    );
+                }
             }
             IpAddr::V6(ip) if NEXUS_OPTE_IPV6_SUBNET.contains(ip) => {
                 if !all_nexus_nic_ips.contains(&ip.into()) {
@@ -399,7 +521,12 @@ fn ensure_input_records_appear_in_parent_blueprint(
                 }
             }
             IpAddr::V6(ip) if DNS_OPTE_IPV6_SUBNET.contains(ip) => {
-                // TODO check all_dns_nic_ips, once it exists
+                if !all_external_dns_nic_ips.contains(&ip.into()) {
+                    bail!(
+                        "planning input contains unexpected NIC \
+                         (IP not found in parent blueprint): {nic_entry:?}"
+                    );
+                }
             }
             _ => {
                 bail!(