Live attach/detach of external IPs

dev-tools/omdb/src/bin/omdb/db.rs

@@ -44,6 +44,7 @@ use nexus_db_model::ExternalIp;
 use nexus_db_model::HwBaseboardId;
 use nexus_db_model::Instance;
 use nexus_db_model::InvCollection;
+use nexus_db_model::IpAttachState;
 use nexus_db_model::Project;
 use nexus_db_model::Region;
 use nexus_db_model::RegionSnapshot;
@@ -1653,6 +1654,7 @@ async fn cmd_db_eips(
         ip: ipnetwork::IpNetwork,
         ports: PortRange,
         kind: String,
+        state: IpAttachState,
         owner: Owner,
     }
 
@@ -1737,6 +1739,7 @@ async fn cmd_db_eips(
                 first: ip.first_port.into(),
                 last: ip.last_port.into(),
             },
+            state: ip.state,
             kind: format!("{:?}", ip.kind),
             owner,
         };

end-to-end-tests/src/instance_launch.rs

@@ -5,9 +5,9 @@ use anyhow::{ensure, Context as _, Result};
 use async_trait::async_trait;
 use omicron_test_utils::dev::poll::{wait_for_condition, CondCheckError};
 use oxide_client::types::{
-    ByteCount, DiskCreate, DiskSource, ExternalIpCreate, InstanceCpuCount,
-    InstanceCreate, InstanceDiskAttachment, InstanceNetworkInterfaceAttachment,
-    SshKeyCreate,
+    ByteCount, DiskCreate, DiskSource, ExternalIp, ExternalIpCreate,
+    InstanceCpuCount, InstanceCreate, InstanceDiskAttachment,
+    InstanceNetworkInterfaceAttachment, SshKeyCreate,
 };
 use oxide_client::{ClientDisksExt, ClientInstancesExt, ClientSessionExt};
 use russh::{ChannelMsg, Disconnect};
@@ -70,7 +70,7 @@ async fn instance_launch() -> Result<()> {
                 name: disk_name.clone(),
             }],
             network_interfaces: InstanceNetworkInterfaceAttachment::Default,
-            external_ips: vec![ExternalIpCreate::Ephemeral { pool_name: None }],
+            external_ips: vec![ExternalIpCreate::Ephemeral { pool: None }],
             user_data: String::new(),
             start: true,
         })
@@ -87,7 +87,10 @@ async fn instance_launch() -> Result<()> {
         .items
         .first()
         .context("no external IPs")?
-        .ip;
+        .clone();
+    let ExternalIp::Ephemeral { ip: ip_addr } = ip_addr else {
+        anyhow::bail!("IP bound to instance was not ephemeral as required.")
+    };
     eprintln!("instance external IP: {}", ip_addr);
 
     // poll serial for login prompt, waiting 5 min max

illumos-utils/src/opte/illumos.rs

@@ -11,6 +11,7 @@ use omicron_common::api::internal::shared::NetworkInterfaceKind;
 use opte_ioctl::OpteHdl;
 use slog::info;
 use slog::Logger;
+use std::net::IpAddr;
 
 #[derive(thiserror::Error, Debug)]
 pub enum Error {
@@ -46,6 +47,15 @@ pub enum Error {
 
     #[error("Tried to release non-existent port ({0}, {1:?})")]
     ReleaseMissingPort(uuid::Uuid, NetworkInterfaceKind),
+
+    #[error("Tried to update external IPs on non-existent port ({0}, {1:?})")]
+    ExternalIpUpdateMissingPort(uuid::Uuid, NetworkInterfaceKind),
+
+    #[error("Could not find Primary NIC")]
+    NoPrimaryNic,
+
+    #[error("Can't attach new ephemeral IP {0}, currently have {1}")]
+    ImplicitEphemeralIpDetach(IpAddr, IpAddr),
 }
 
 /// Delete all xde devices on the system.

illumos-utils/src/opte/non_illumos.rs

@@ -8,6 +8,7 @@ use slog::Logger;
 
 use crate::addrobj::AddrObject;
 use omicron_common::api::internal::shared::NetworkInterfaceKind;
+use std::net::IpAddr;
 
 #[derive(thiserror::Error, Debug)]
 pub enum Error {
@@ -16,6 +17,15 @@ pub enum Error {
 
     #[error("Tried to release non-existent port ({0}, {1:?})")]
     ReleaseMissingPort(uuid::Uuid, NetworkInterfaceKind),
+
+    #[error("Tried to update external IPs on non-existent port ({0}, {1:?})")]
+    ExternalIpUpdateMissingPort(uuid::Uuid, NetworkInterfaceKind),
+
+    #[error("Could not find Primary NIC")]
+    NoPrimaryNic,
+
+    #[error("Can't attach new ephemeral IP {0}, currently have {1}")]
+    ImplicitEphemeralIpDetach(IpAddr, IpAddr),
 }
 
 pub fn initialize_xde_driver(

illumos-utils/src/opte/port_manager.rs

@@ -29,6 +29,7 @@ use oxide_vpc::api::MacAddr;
 use oxide_vpc::api::RouterTarget;
 use oxide_vpc::api::SNat4Cfg;
 use oxide_vpc::api::SNat6Cfg;
+use oxide_vpc::api::SetExternalIpsReq;
 use oxide_vpc::api::VpcCfg;
 use slog::debug;
 use slog::error;
@@ -401,6 +402,121 @@ impl PortManager {
         Ok((port, ticket))
     }
 
+    /// Ensure external IPs for an OPTE port are up to date.
+    #[cfg_attr(not(target_os = "illumos"), allow(unused_variables))]
+    pub fn external_ips_ensure(
+        &self,
+        nic_id: Uuid,
+        nic_kind: NetworkInterfaceKind,
+        source_nat: Option<SourceNatConfig>,
+        ephemeral_ip: Option<IpAddr>,
+        floating_ips: &[IpAddr],
+    ) -> Result<(), Error> {
+        let ports = self.inner.ports.lock().unwrap();
+        let port = ports.get(&(nic_id, nic_kind)).ok_or_else(|| {
+            Error::ExternalIpUpdateMissingPort(nic_id, nic_kind)
+        })?;
+
+        // XXX: duplicates parts of macro logic in `create_port`.
+        macro_rules! ext_ip_cfg {
+            ($ip:expr, $log_prefix:literal, $ip_t:path, $cidr_t:path,
+             $ipcfg_e:path, $ipcfg_t:ident, $snat_t:ident) => {{
+                let snat = match source_nat {
+                    Some(snat) => {
+                        let $ip_t(snat_ip) = snat.ip else {
+                            error!(
+                                self.inner.log,
+                                concat!($log_prefix, " SNAT config");
+                                "snat_ip" => ?snat.ip,
+                            );
+                            return Err(Error::InvalidPortIpConfig);
+                        };
+                        let ports = snat.first_port..=snat.last_port;
+                        Some($snat_t { external_ip: snat_ip.into(), ports })
+                    }
+                    None => None,
+                };
+                let ephemeral_ip = match ephemeral_ip {
+                    Some($ip_t(ip)) => Some(ip.into()),
+                    Some(_) => {
+                        error!(
+                            self.inner.log,
+                            concat!($log_prefix, " ephemeral IP");
+                            "ephemeral_ip" => ?ephemeral_ip,
+                        );
+                        return Err(Error::InvalidPortIpConfig);
+                    }
+                    None => None,
+                };
+                let floating_ips: Vec<_> = floating_ips
+                    .iter()
+                    .copied()
+                    .map(|ip| match ip {
+                        $ip_t(ip) => Ok(ip.into()),
+                        _ => {
+                            error!(
+                                self.inner.log,
+                                concat!($log_prefix, " ephemeral IP");
+                                "ephemeral_ip" => ?ephemeral_ip,
+                            );
+                            Err(Error::InvalidPortIpConfig)
+                        }
+                    })
+                    .collect::<Result<Vec<_>, _>>()?;
+
+                ExternalIpCfg {
+                    ephemeral_ip,
+                    snat,
+                    floating_ips,
+                }
+            }}
+        }
+
+        // TODO-completeness: support dual-stack. We'll need to explicitly store
+        // a v4 and a v6 ephemeral IP + SNat + gateway + ... in `InstanceInner`
+        // to have enough info to build both.
+        let mut v4_cfg = None;
+        let mut v6_cfg = None;
+        match port.gateway().ip {
+            IpAddr::V4(_) => {
+                v4_cfg = Some(ext_ip_cfg!(
+                    ip,
+                    "Expected IPv4",
+                    IpAddr::V4,
+                    IpCidr::Ip4,
+                    IpCfg::Ipv4,
+                    Ipv4Cfg,
+                    SNat4Cfg
+                ))
+            }
+            IpAddr::V6(_) => {
+                v6_cfg = Some(ext_ip_cfg!(
+                    ip,
+                    "Expected IPv6",
+                    IpAddr::V6,
+                    IpCidr::Ip6,
+                    IpCfg::Ipv6,
+                    Ipv6Cfg,
+                    SNat6Cfg
+                ))
+            }
+        }
+
+        let req = SetExternalIpsReq {
+            port_name: port.name().into(),
+            external_ips_v4: v4_cfg,
+            external_ips_v6: v6_cfg,
+        };
+
+        #[cfg(target_os = "illumos")]
+        let hdl = opte_ioctl::OpteHdl::open(opte_ioctl::OpteHdl::XDE_CTL)?;
+
+        #[cfg(target_os = "illumos")]
+        hdl.set_external_ips(&req)?;
+
+        Ok(())
+    }
+
     #[cfg(target_os = "illumos")]
     pub fn firewall_rules_ensure(
         &self,