instance minimum CPU platforms

RFD 505 proposes that instances should be able to set a "minimum
hardware platform" or "minimum CPU platform" that allows uers to
constrain an instance to run on sleds that have a specific set of CPU
features available. Previously, actually-available CPU information was
plumbed from sleds to Nexus. This actually adds a `min_cpu_platform`
setting for instance creation and uses it to drive selection of guest
CPUID leaves.

As-is, this moves VMs on Gimlets away from the byhve-default CPUID
leaves (which are effectively "host CPUID information, but features that
are not or cannot be virtualized are masked out"), instead using the
specific CPUID information set out in RFD 505.

There is no provision for Turin yet, which instead gets CPUID leaves
that look like Milan. Adding a set of CPUID information to advertise for
an `amd_turin` CPU platform, from here, is fairly straightforward.

This does not have a mechanism to enforce specific CPU platform use or
disuse, either in a silo or rack-wide. One could imagine a simple system
oriented around "this silo is permitted to specify these minimum CPU
platforms", but that leaves uncomfortable issues like: if a silo A
permits only Milan, and silo B permits Milan and Turin, all Milan CPUs
are allocated already, and someone is attemting to create a new
Milan-based VM in silo A, should this succeed using Turin CPUs
potentially starving silo B?

Author: gjcolombo

common/src/api/external/mod.rs

@@ -1194,6 +1194,10 @@ pub struct Instance {
 
     #[serde(flatten)]
     pub auto_restart_status: InstanceAutoRestartStatus,
+
+    /// The minimum required CPU platform for this instance. If this is `null`,
+    /// the instance requires no particular CPU platform.
+    pub min_cpu_platform: Option<InstanceMinimumCpuPlatform>,
 }
 
 /// Status of control-plane driven automatic failure recovery for this instance.
@@ -1258,6 +1262,46 @@ pub enum InstanceAutoRestartPolicy {
     BestEffort,
 }
 
+/// A minimum required CPU platform for an instance.
+///
+/// When an instance specifies a minimum required CPU platform:
+///
+/// - The system may expose (to the VM) new CPU features that are only present
+///   on that platform (or on newer platforms of the same lineage that also
+///   support those features).
+/// - The instance must run on hosts that have CPUs that support all the
+///   features of the supplied minimum platform.
+///
+/// That is, the instance is restricted to hosts that have the specified minimum
+/// host CPU type (or a more advanced, but still compatible, CPU), but in
+/// exchange the CPU features exposed by the minimum platform are available for
+/// the guest to use. Note that this may prevent an instance from starting (if
+/// the hosts it requires are full but there is capacity on other incompatible
+/// hosts).
+///
+/// If an instance does not specify a minimum required CPU platform, then when
+/// it starts, the control plane selects a host for the instance and then
+/// supplies the guest with the "minimum" CPU platform supported by that host.
+/// This maximizes the number of hosts that can run the VM if it later needs to
+/// migrate to another host.
+///
+/// In all cases, the CPU features presented by a given CPU platform are a
+/// subset of what the corresponding hardware may actually support; features
+/// which cannot be used from a virtual environment or do not have full
+/// hypervisor support may be masked off. See RFD 314 for specific CPU features
+/// in a CPU platform.
+#[derive(
+    Copy, Clone, Debug, Deserialize, Serialize, JsonSchema, Eq, PartialEq,
+)]
+#[serde(rename_all = "snake_case")]
+pub enum InstanceMinimumCpuPlatform {
+    /// An AMD Milan-like CPU platform.
+    AmdMilan,
+
+    /// An AMD Turin-like CPU platform.
+    AmdTurin,
+}
+
 // AFFINITY GROUPS
 
 /// Affinity policy used to describe "what to do when a request cannot be satisfied"

dev-tools/omdb/src/bin/omdb/db.rs

@@ -4760,6 +4760,7 @@ async fn cmd_db_instance_info(
                     propolis_ip: _,
                     propolis_port: _,
                     instance_id: _,
+                    cpu_platform: _,
                     time_created,
                     time_deleted,
                     runtime:
@@ -7356,6 +7357,7 @@ fn prettyprint_vmm(
     const INSTANCE_ID: &'static str = "instance ID";
     const SLED_ID: &'static str = "sled ID";
     const SLED_SERIAL: &'static str = "sled serial";
+    const CPU_PLATFORM: &'static str = "CPU platform";
     const ADDRESS: &'static str = "propolis address";
     const STATE: &'static str = "state";
     const WIDTH: usize = const_max_len(&[
@@ -7366,6 +7368,7 @@ fn prettyprint_vmm(
         INSTANCE_ID,
         SLED_ID,
         SLED_SERIAL,
+        CPU_PLATFORM,
         STATE,
         ADDRESS,
     ]);
@@ -7379,6 +7382,7 @@ fn prettyprint_vmm(
         sled_id,
         propolis_ip,
         propolis_port,
+        cpu_platform,
         runtime: db::model::VmmRuntimeState { state, r#gen, time_state_updated },
     } = vmm;
 
@@ -7405,6 +7409,7 @@ fn prettyprint_vmm(
     if let Some(serial) = sled_serial {
         println!("{indent}{SLED_SERIAL:>width$}: {serial}");
     }
+    println!("{indent}{CPU_PLATFORM:>width$}: {cpu_platform}");
 }
 
 async fn cmd_db_vmm_list(
@@ -7480,6 +7485,7 @@ async fn cmd_db_vmm_list(
                 sled_id,
                 propolis_ip: _,
                 propolis_port: _,
+                cpu_platform: _,
                 runtime:
                     db::model::VmmRuntimeState {
                         state,

end-to-end-tests/src/instance_launch.rs

@@ -79,6 +79,7 @@ async fn instance_launch() -> Result<()> {
             start: true,
             auto_restart_policy: Default::default(),
             anti_affinity_groups: Vec::new(),
+            min_cpu_platform: None,
         })
         .send()
         .await?;

nexus/db-model/src/instance.rs

@@ -5,7 +5,7 @@
 use super::InstanceIntendedState as IntendedState;
 use super::{
     ByteCount, Disk, ExternalIp, Generation, InstanceAutoRestartPolicy,
-    InstanceCpuCount, InstanceState, Vmm, VmmState,
+    InstanceCpuCount, InstanceMinimumCpuPlatform, InstanceState, Vmm, VmmState,
 };
 use crate::collection::DatastoreAttachTargetConfig;
 use crate::serde_time_delta::optional_time_delta;
@@ -68,6 +68,9 @@ pub struct Instance {
     #[diesel(column_name = boot_disk_id)]
     pub boot_disk_id: Option<Uuid>,
 
+    /// The instance's minimum required CPU platform.
+    pub min_cpu_platform: Option<InstanceMinimumCpuPlatform>,
+
     #[diesel(embed)]
     pub runtime_state: InstanceRuntimeState,
 
@@ -139,6 +142,7 @@ impl Instance {
             // Intentionally ignore `params.boot_disk_id` here: we can't set
             // `boot_disk_id` until the referenced disk is attached.
             boot_disk_id: None,
+            min_cpu_platform: params.min_cpu_platform.map(Into::into),
 
             runtime_state,
             intended_state,
@@ -493,4 +497,6 @@ pub struct InstanceUpdate {
     pub ncpus: InstanceCpuCount,
 
     pub memory: ByteCount,
+
+    pub min_cpu_platform: Option<InstanceMinimumCpuPlatform>,
 }

nexus/db-model/src/instance_minimum_cpu_platform.rs

@@ -0,0 +1,67 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at https://mozilla.org/MPL/2.0/.
+
+use crate::SledCpuFamily;
+
+use super::impl_enum_type;
+use serde::{Deserialize, Serialize};
+
+impl_enum_type!(
+    InstanceMinimumCpuPlatformEnum:
+
+    #[derive(
+        Copy,
+        Clone,
+        Debug,
+        PartialEq,
+        AsExpression,
+        FromSqlRow,
+        Serialize,
+        Deserialize
+    )]
+    pub enum InstanceMinimumCpuPlatform;
+
+    AmdMilan => b"amd_milan"
+    AmdTurin => b"amd_turin"
+);
+
+impl InstanceMinimumCpuPlatform {
+    /// Returns a slice containing the set of sled CPU families that can
+    /// accommodate an instance with this minimum CPU platform.
+    pub fn compatible_sled_cpu_families(&self) -> &'static [SledCpuFamily] {
+        match self {
+            // Turin-based sleds have a superset of the features made available
+            // in a guest's Milan CPU platform
+            Self::AmdMilan => {
+                &[SledCpuFamily::AmdMilan, SledCpuFamily::AmdTurin]
+            }
+            Self::AmdTurin => &[SledCpuFamily::AmdTurin],
+        }
+    }
+}
+
+impl From<omicron_common::api::external::InstanceMinimumCpuPlatform>
+    for InstanceMinimumCpuPlatform
+{
+    fn from(
+        value: omicron_common::api::external::InstanceMinimumCpuPlatform,
+    ) -> Self {
+        use omicron_common::api::external::InstanceMinimumCpuPlatform as ApiPlatform;
+        match value {
+            ApiPlatform::AmdMilan => Self::AmdMilan,
+            ApiPlatform::AmdTurin => Self::AmdTurin,
+        }
+    }
+}
+
+impl From<InstanceMinimumCpuPlatform>
+    for omicron_common::api::external::InstanceMinimumCpuPlatform
+{
+    fn from(value: InstanceMinimumCpuPlatform) -> Self {
+        match value {
+            InstanceMinimumCpuPlatform::AmdMilan => Self::AmdMilan,
+            InstanceMinimumCpuPlatform::AmdTurin => Self::AmdTurin,
+        }
+    }
+}

nexus/db-model/src/lib.rs

@@ -44,6 +44,7 @@ mod instance;
 mod instance_auto_restart_policy;
 mod instance_cpu_count;
 mod instance_intended_state;
+mod instance_minimum_cpu_platform;
 mod instance_state;
 mod internet_gateway;
 mod inventory;
@@ -123,6 +124,7 @@ mod utilization;
 mod virtual_provisioning_collection;
 mod virtual_provisioning_resource;
 mod vmm;
+mod vmm_cpu_platform;
 mod vni;
 mod volume;
 mod volume_repair;
@@ -181,6 +183,7 @@ pub use instance::*;
 pub use instance_auto_restart_policy::*;
 pub use instance_cpu_count::*;
 pub use instance_intended_state::*;
+pub use instance_minimum_cpu_platform::*;
 pub use instance_state::*;
 pub use internet_gateway::*;
 pub use inventory::*;
@@ -248,6 +251,7 @@ pub use v2p_mapping::*;
 pub use virtual_provisioning_collection::*;
 pub use virtual_provisioning_resource::*;
 pub use vmm::*;
+pub use vmm_cpu_platform::*;
 pub use vmm_state::*;
 pub use vni::*;
 pub use volume::*;

nexus/db-model/src/schema_versions.rs

@@ -16,7 +16,7 @@ use std::{collections::BTreeMap, sync::LazyLock};
 ///
 /// This must be updated when you change the database schema.  Refer to
 /// schema/crdb/README.adoc in the root of this repository for details.
-pub const SCHEMA_VERSION: Version = Version::new(174, 0, 0);
+pub const SCHEMA_VERSION: Version = Version::new(175, 0, 0);
 
 /// List of all past database schema versions, in *reverse* order
 ///
@@ -28,6 +28,7 @@ static KNOWN_VERSIONS: LazyLock<Vec<KnownVersion>> = LazyLock::new(|| {
         // |  leaving the first copy as an example for the next person.
         // v
         // KnownVersion::new(next_int, "unique-dirname-with-the-sql-files"),
+        KnownVersion::new(175, "add-instance-minimum-cpu-platform"),
         KnownVersion::new(174, "sled-cpu-family"),
         KnownVersion::new(173, "inv-internal-dns"),
         KnownVersion::new(172, "add-zones-with-mupdate-override"),

nexus/db-model/src/sled.rs

@@ -340,12 +340,13 @@ impl SledUpdate {
 #[derive(Clone, Debug)]
 pub struct SledReservationConstraints {
     must_select_from: Vec<Uuid>,
+    cpu_families: Vec<SledCpuFamily>,
 }
 
 impl SledReservationConstraints {
     /// Creates a constraint set with no constraints in it.
     pub fn none() -> Self {
-        Self { must_select_from: Vec::new() }
+        Self { must_select_from: Vec::new(), cpu_families: Vec::new() }
     }
 
     /// If the constraints include a set of sleds that the caller must select
@@ -359,6 +360,19 @@ impl SledReservationConstraints {
             Some(&self.must_select_from)
         }
     }
+
+    /// If the constraints include a list of acceptable sled CPU families,
+    /// returns `Some` and a slice containing the members of that set.
+    ///
+    /// If no "must select a sled with one of these CPUs" constraint exists,
+    /// returns None.
+    pub fn cpu_families(&self) -> Option<&[SledCpuFamily]> {
+        if self.cpu_families.is_empty() {
+            None
+        } else {
+            Some(&self.cpu_families)
+        }
+    }
 }
 
 #[derive(Debug)]
@@ -381,6 +395,11 @@ impl SledReservationConstraintBuilder {
         self
     }
 
+    pub fn cpu_families(mut self, families: &[SledCpuFamily]) -> Self {
+        self.constraints.cpu_families.extend(families);
+        self
+    }
+
     /// Builds a set of constraints from this builder's current state.
     pub fn build(self) -> SledReservationConstraints {
         self.constraints

nexus/db-model/src/sled_cpu_family.rs

@@ -25,6 +25,25 @@ impl_enum_type!(
     AmdTurin => b"amd_turin"
 );
 
+impl SledCpuFamily {
+    /// Yields the minimum compatible instance CPU platform that can run on this
+    /// sled.
+    ///
+    /// Each instance CPU platform has a set `C` of sled CPU families that can
+    /// host it. The "minimum compatible platform" is the instance CPU platform
+    /// for which (a) `self` is in `C`, and (b) `C` is of maximum cardinality.
+    /// That is: the minimum compatible platform is chosen so that a VMM that
+    /// uses it can run on sleds of this family and as many other families as
+    /// possible.
+    pub fn minimum_compatible_platform(&self) -> crate::VmmCpuPlatform {
+        match self {
+            Self::Unknown => crate::VmmCpuPlatform::SledDefault,
+            Self::AmdMilan => crate::VmmCpuPlatform::AmdMilan,
+            Self::AmdTurin => crate::VmmCpuPlatform::AmdMilan,
+        }
+    }
+}
+
 impl From<nexus_types::internal_api::params::SledCpuFamily> for SledCpuFamily {
     fn from(value: nexus_types::internal_api::params::SledCpuFamily) -> Self {
         use nexus_types::internal_api::params::SledCpuFamily as InputFamily;

nexus/db-model/src/vmm.rs

@@ -13,7 +13,7 @@
 //! sled agent or that sled agent will never update (like the sled ID).
 
 use super::{Generation, VmmState};
-use crate::SqlU16;
+use crate::{SqlU16, VmmCpuPlatform};
 use chrono::{DateTime, Utc};
 use nexus_db_schema::schema::vmm;
 use omicron_uuid_kinds::{GenericUuid, InstanceUuid, PropolisUuid, SledUuid};
@@ -55,6 +55,11 @@ pub struct Vmm {
     /// The socket port on which this VMM is serving the Propolis server API.
     pub propolis_port: SqlU16,
 
+    /// The CPU platform for this VMM. This may be chosen implicitly by the
+    /// control plane if this VMM's instance didn't specify a required platform
+    /// when it was started.
+    pub cpu_platform: VmmCpuPlatform,
+
     /// Runtime state for the VMM.
     #[diesel(embed)]
     pub runtime: VmmRuntimeState,
@@ -71,6 +76,7 @@ impl Vmm {
         sled_id: SledUuid,
         propolis_ip: ipnetwork::IpNetwork,
         propolis_port: u16,
+        cpu_platform: VmmCpuPlatform,
     ) -> Self {
         let now = Utc::now();
 
@@ -82,6 +88,7 @@ impl Vmm {
             sled_id: sled_id.into_untyped_uuid(),
             propolis_ip,
             propolis_port: SqlU16(propolis_port),
+            cpu_platform,
             runtime: VmmRuntimeState {
                 state: VmmState::Creating,
                 time_state_updated: now,