TQ: Plumb initial configuration through handoff

clients/nexus-lockstep-client/Cargo.toml

@@ -24,5 +24,6 @@ reqwest.workspace = true
 schemars.workspace = true
 serde.workspace = true
 serde_json.workspace = true
+sled-hardware-types.workspace = true
 slog.workspace = true
 uuid.workspace = true

clients/nexus-lockstep-client/src/lib.rs

@@ -33,6 +33,7 @@ progenitor::generate_api!(
         "oxnet" = "0.1.0",
     },
     replace = {
+        BaseboardId = sled_hardware_types::BaseboardId,
         // It's kind of unfortunate to pull in such a complex and unstable type
         // as "blueprint" this way, but we have really useful functionality
         // (e.g., diff'ing) that's implemented on our local type.

nexus/db-queries/src/db/datastore/rack.rs

@@ -57,6 +57,7 @@ use nexus_types::external_api::shared;
 use nexus_types::external_api::shared::IpRange;
 use nexus_types::external_api::shared::SiloRole;
 use nexus_types::identity::Resource;
+use nexus_types::internal_api::params::InitialTrustQuorumConfig;
 use nexus_types::inventory::NetworkInterface;
 use omicron_common::api::external::AllowedSourceIps;
 use omicron_common::api::external::DataPageParams;
@@ -70,6 +71,7 @@ use omicron_common::api::external::UserId;
 use omicron_common::api::internal::shared::PrivateIpConfig;
 use omicron_common::bail_unless;
 use omicron_uuid_kinds::GenericUuid;
+use omicron_uuid_kinds::RackUuid;
 use omicron_uuid_kinds::SiloUserUuid;
 use omicron_uuid_kinds::SledUuid;
 use omicron_uuid_kinds::ZpoolUuid;
@@ -96,6 +98,7 @@ pub struct RackInit {
     pub recovery_user_password_hash: omicron_passwords::PasswordHashString,
     pub dns_update: DnsVersionUpdateBuilder,
     pub allowed_source_ips: AllowedSourceIps,
+    pub initial_trust_quorum_configuration: Option<InitialTrustQuorumConfig>,
 }
 
 /// Possible errors while trying to initialize rack
@@ -116,6 +119,7 @@ enum RackInitError {
     Database(DieselError),
     // Error adding initial allowed source IP list
     AllowedSourceIpError(Error),
+    TrustQuorum(Error),
 }
 
 impl From<DieselError> for RackInitError {
@@ -177,6 +181,9 @@ impl From<RackInitError> for Error {
                 err
             )),
             RackInitError::AllowedSourceIpError(err) => err,
+            RackInitError::TrustQuorum(err) => err.internal_context(
+                "failed to insert initial trust quorum configuration",
+            ),
         }
     }
 }
@@ -976,6 +983,20 @@ impl DataStore {
                         DieselError::RollbackTransaction
                     })?;
 
+                    // Insert the initial trust quorum configuration
+                    if let Some(tq_config) = rack_init.initial_trust_quorum_configuration {
+                        Self::tq_insert_rss_config_after_handoff(
+                            opctx,
+                            &conn,
+                            RackUuid::from_untyped_uuid(rack_id),
+                            tq_config.members,
+                            tq_config.coordinator
+                        ).await.map_err(|e| {
+                            err.set(RackInitError::TrustQuorum(e)).unwrap();
+                            DieselError::RollbackTransaction
+                        })?;
+                    }
+
                     let rack = diesel::update(rack_dsl::rack)
                         .filter(rack_dsl::id.eq(rack_id))
                         .set((
@@ -1167,6 +1188,7 @@ mod test {
                     "test suite".to_string(),
                 ),
                 allowed_source_ips: AllowedSourceIps::Any,
+                initial_trust_quorum_configuration: None
             }
         }
     }

nexus/db-queries/src/db/datastore/trust_quorum.rs

@@ -129,7 +129,7 @@ impl DataStore {
     ///
     /// For reconfiguration and lrtq upgrade we always call
     /// `tq_insert_latest_config`.
-    pub async fn insert_rss_config_after_handoff(
+    pub async fn tq_insert_rss_config_after_handoff(
         opctx: &OpContext,
         conn: &async_bb8_diesel::Connection<DbConnection>,
         rack_id: RackUuid,
@@ -1507,7 +1507,7 @@ mod tests {
         let coordinator = members.first().unwrap().clone();
 
         // Insert an initial config
-        DataStore::insert_rss_config_after_handoff(
+        DataStore::tq_insert_rss_config_after_handoff(
             opctx,
             &conn,
             rack_id,
@@ -1590,7 +1590,7 @@ mod tests {
         let coordinator = members.first().unwrap().clone();
 
         // Insert an initial config
-        DataStore::insert_rss_config_after_handoff(
+        DataStore::tq_insert_rss_config_after_handoff(
             opctx,
             &conn,
             rack_id,
@@ -1843,7 +1843,7 @@ mod tests {
         let coordinator = members.first().unwrap().clone();
 
         // Insert an initial config
-        DataStore::insert_rss_config_after_handoff(
+        DataStore::tq_insert_rss_config_after_handoff(
             opctx,
             &conn,
             rack_id,
@@ -2015,7 +2015,7 @@ mod tests {
                 rack2_members = members.clone();
             }
             let coordinator = members.first().unwrap().clone();
-            DataStore::insert_rss_config_after_handoff(
+            DataStore::tq_insert_rss_config_after_handoff(
                 opctx,
                 &conn,
                 rack_id,

nexus/src/app/rack.rs

@@ -733,6 +733,8 @@ impl super::Nexus {
                         .into(),
                     dns_update,
                     allowed_source_ips: request.allowed_source_ips,
+                    initial_trust_quorum_configuration: request
+                        .initial_trust_quorum_configuration,
                 },
             )
             .await?;

nexus/src/lib.rs

@@ -414,6 +414,7 @@ impl nexus_test_interface::NexusServer for Server {
                         bfd: Vec::new(),
                     },
                     allowed_source_ips: AllowedSourceIps::Any,
+                    initial_trust_quorum_configuration: None,
                 },
                 false, // blueprint_execution_enabled
             )

nexus/types/src/internal_api/params.rs

@@ -25,6 +25,8 @@ use schemars::JsonSchema;
 use serde::{Deserialize, Serialize};
 use sled_agent_types_versions::latest::inventory::{SledCpuFamily, SledRole};
 use sled_agent_types_versions::latest::rack_init::RecoverySiloConfig;
+use sled_hardware_types::BaseboardId;
+use std::collections::BTreeSet;
 use std::fmt;
 use std::net::IpAddr;
 use std::net::SocketAddr;
@@ -198,6 +200,13 @@ pub struct RackInitializationRequest {
     pub rack_network_config: RackNetworkConfig,
     /// IPs or subnets allowed to make requests to user-facing services
     pub allowed_source_ips: AllowedSourceIps,
+    /// Data used to write the initial trust quorum configuration to CRDB
+    ///
+    /// This is optional for two reasons:
+    ///   * For clusters fewer than 3 nodes, we don't support trust quorum.
+    ///   * Trust quorum is not fully complete yet, and we only want this to be
+    ///     used in production once it is complete.
+    pub initial_trust_quorum_configuration: Option<InitialTrustQuorumConfig>,
 }
 
 pub type DnsConfigParams = internal_dns_types::config::DnsConfigParams;
@@ -222,3 +231,10 @@ pub struct InstanceMigrateRequest {
     #[schemars(with = "Uuid")]
     pub dst_sled_id: SledUuid,
 }
+
+/// The configuration generated by RSS and used to initialize trust quorum
+#[derive(Debug, Clone, Deserialize, JsonSchema)]
+pub struct InitialTrustQuorumConfig {
+    pub members: BTreeSet<BaseboardId>,
+    pub coordinator: BaseboardId,
+}

openapi/nexus-lockstep.json

@@ -4985,6 +4985,26 @@
           "time_started"
         ]
       },
+      "InitialTrustQuorumConfig": {
+        "description": "The configuration generated by RSS and used to initialize trust quorum",
+        "type": "object",
+        "properties": {
+          "coordinator": {
+            "$ref": "#/components/schemas/BaseboardId"
+          },
+          "members": {
+            "type": "array",
+            "items": {
+              "$ref": "#/components/schemas/BaseboardId"
+            },
+            "uniqueItems": true
+          }
+        },
+        "required": [
+          "coordinator",
+          "members"
+        ]
+      },
       "Instance": {
         "description": "View of an Instance",
         "type": "object",
@@ -7808,6 +7828,15 @@
               }
             ]
           },
+          "initial_trust_quorum_configuration": {
+            "nullable": true,
+            "description": "Data used to write the initial trust quorum configuration to CRDB\n\nThis is optional for two reasons: * For clusters fewer than 3 nodes, we don't support trust quorum. * Trust quorum is not fully complete yet, and we only want this to be used in production once it is complete.",
+            "allOf": [
+              {
+                "$ref": "#/components/schemas/InitialTrustQuorumConfig"
+              }
+            ]
+          },
           "internal_dns_zone_config": {
             "description": "initial internal DNS config",
             "allOf": [

sled-agent/src/rack_setup/service.rs

@@ -81,6 +81,7 @@ use dns_service_client::DnsError;
 use internal_dns_resolver::Resolver as DnsResolver;
 use internal_dns_types::names::ServiceName;
 use itertools::Itertools;
+use nexus_lockstep_client::types::InitialTrustQuorumConfig;
 use nexus_lockstep_client::{
     Client as NexusClient, Error as NexusError, types as NexusTypes,
 };
@@ -771,6 +772,7 @@ impl ServiceInner {
         service_plan: &ServicePlan,
         port_discovery_mode: ExternalPortDiscovery,
         nexus_lockstep_address: SocketAddrV6,
+        initial_trust_quorum_configuration: Option<InitialTrustQuorumConfig>,
     ) -> Result<(), SetupServiceError> {
         info!(self.log, "Handing off control to Nexus");
 
@@ -1050,6 +1052,7 @@ impl ServiceInner {
             rack_network_config,
             external_port_count: port_discovery_mode.into(),
             allowed_source_ips,
+            initial_trust_quorum_configuration,
         };
 
         let notify_nexus = || async {
@@ -1282,7 +1285,10 @@ impl ServiceInner {
 
         rss_step.update(RssStep::InitTrustQuorum);
         // Initialize the trust quorum if there are peers configured.
-        if let Some(peers) = &config.trust_quorum_peers {
+
+        let initial_trust_quorum_configuration = if let Some(peers) =
+            &config.trust_quorum_peers
+        {
             let initial_membership: BTreeSet<_> =
                 peers.iter().cloned().collect();
             bootstore
@@ -1297,10 +1303,24 @@ impl ServiceInner {
                     .collect();
                 let rack_id = RackUuid::from_untyped_uuid(sled_plan.rack_id);
 
-                init_trust_quorum(&self.log, trust_quorum, tq_members, rack_id)
-                    .await?;
+                init_trust_quorum(
+                    &self.log,
+                    trust_quorum.clone(),
+                    tq_members.clone(),
+                    rack_id,
+                )
+                .await?;
+
+                Some(InitialTrustQuorumConfig {
+                    members: tq_members.into_iter().collect(),
+                    coordinator: trust_quorum.baseboard_id().clone(),
+                })
+            } else {
+                None
             }
-        }
+        } else {
+            None
+        };
 
         // Save the relevant network config in the bootstore. We want this to
         // happen before we `initialize_sleds` so each scrimlet (including us)
@@ -1479,6 +1499,7 @@ impl ServiceInner {
             &service_plan,
             ExternalPortDiscovery::Auto(switch_mgmt_addrs),
             nexus_lockstep_address,
+            initial_trust_quorum_configuration,
         )
         .await?;
 

sled-agent/src/sim/server.rs

@@ -644,6 +644,7 @@ pub async fn run_standalone_server(
             bfd: Vec::new(),
         },
         allowed_source_ips: AllowedSourceIps::Any,
+        initial_trust_quorum_configuration: None,
     };
 
     let mut nexus_lockstep_address = config.nexus_address;