TQ: Async Nodes and P2P connections

Builds on #9232

This is the first step in wrapping the `trust_quorum::Node` so that it
can be used in an async context and integrated with sled-agent. Only
the sprockets networking has been fully integrated so far such that
each `NodeTask` has a `ConnMgr` that sets up a full mesh of sprockets
connections. A test for this connectivity behavior has been written but
the code is not wired into the production code yet.

Messages can be sent between `NodeTasks` over sprockets connections.
Each connection exists in it's own task managed by an `EstablishedConn`.
The main `NodeTask` task sends messages to and receives messages from
this task to interact with the outside world via sprockets. Currently
only `Ping` messages are sent over the wire as a means to keep the
connections alive and detect disconnects.

A `NodeHandle` allows one to interact with the `NodeTask`. Currently
only three operations are implemented with messages defined in
`NodeApiRequest`. The user can instruct the node who it's peers
are on the bootstrap network to establish connectivity, can poll
for connectivity status, and can shutdown the node. All of this
functionality is used in the accompanying test.

It's important to re-iterate that this code only implements connectivity
between trust quorum nodes and no actual trust quorum messages are sent.
They can't be as a handle can not yet initiate a reconfiguration or
LRTQ upgrade. That behavior will come in a follow up. This PR is large
enough.

A lot of this code is similar to the LRTQ connection management code,
except that it operates over sprockets rather than TCP channels. This
introduces some complexity, but it is mostly abstracted away into the
`SprocketsConfig`.

Author: andrewjstone

Cargo.lock

@@ -368,6 +368,7 @@ assert_matches = "1.5.0"
 assert_cmd = "2.0.17"
 async-bb8-diesel = "0.2"
 async-trait = "0.1.89"
+attest-mock = { path = "../dice-util/attest-mock" }
 atomicwrites = "0.4.4"
 authz-macros = { path = "nexus/authz-macros" }
 backoff = { version = "0.4.0", features = [ "tokio" ] }
@@ -722,7 +723,8 @@ slog-term = "2.9.1"
 smf = "0.2"
 socket2 = { version = "0.5", features = ["all"] }
 sp-sim = { path = "sp-sim" }
-sprockets-tls = { git = "https://github.com/oxidecomputer/sprockets.git", rev = "6d31fa63217c6a51061dc4afa1ebe175a0021981" }
+sprockets-tls = { git = "https://github.com/oxidecomputer/sprockets.git", rev = "dea3bbfac7d9d3c45f088898fcd05ee5d2ec2210" }
+sprockets-tls-test-utils = { git = "https://github.com/oxidecomputer/sprockets.git", rev = "dea3bbfac7d9d3c45f088898fcd05ee5d2ec2210" }
 sqlformat = "0.3.5"
 sqlparser = { version = "0.45.0", features = [ "visitor" ] }
 static_assertions = "1.1.0"

Cargo.toml

@@ -12,6 +12,7 @@ use crate::bootstrap::views::Response;
 use crate::bootstrap::views::ResponseEnvelope;
 use sled_agent_types::sled::StartSledAgentRequest;
 use slog::Logger;
+use slog_error_chain::SlogInlineError;
 use sprockets_tls::client::Client as SprocketsClient;
 use sprockets_tls::keys::SprocketsConfig;
 use std::borrow::Cow;
@@ -21,34 +22,38 @@ use thiserror::Error;
 use tokio::io::AsyncReadExt;
 use tokio::io::AsyncWriteExt;
 
-#[derive(Debug, Error)]
+#[derive(Debug, Error, SlogInlineError)]
 pub enum Error {
-    #[error("Could not connect to {addr}: {err}")]
-    Connect { addr: SocketAddrV6, err: sprockets_tls::Error },
+    #[error("Could not connect to {addr}")]
+    Connect {
+        addr: SocketAddrV6,
+        #[source]
+        err: sprockets_tls::Error,
+    },
 
-    #[error("Failed serializing request: {0}")]
-    Serialize(serde_json::Error),
+    #[error("Failed serializing request")]
+    Serialize(#[source] serde_json::Error),
 
-    #[error("Failed writing request length prefix: {0}")]
-    WriteLengthPrefix(io::Error),
+    #[error("Failed writing request length prefix")]
+    WriteLengthPrefix(#[source] io::Error),
 
-    #[error("Failed writing request: {0}")]
-    WriteRequest(io::Error),
+    #[error("Failed writing request")]
+    WriteRequest(#[source] io::Error),
 
-    #[error("Failed flushing request: {0}")]
-    FlushRequest(io::Error),
+    #[error("Failed flushing request")]
+    FlushRequest(#[source] io::Error),
 
-    #[error("Failed reading response length prefix: {0}")]
-    ReadLengthPrefix(io::Error),
+    #[error("Failed reading response length prefix")]
+    ReadLengthPrefix(#[source] io::Error),
 
     #[error("Received bogus response length: {0}")]
     BadResponseLength(u32),
 
-    #[error("Failed reading response: {0}")]
-    ReadResponse(io::Error),
+    #[error("Failed reading response")]
+    ReadResponse(#[source] io::Error),
 
-    #[error("Failed deserializing response: {0}")]
-    Deserialize(serde_json::Error),
+    #[error("Failed deserializing response")]
+    Deserialize(#[source] serde_json::Error),
 
     #[error("Unsupported version: {0}")]
     UnsupportedVersion(u32),
@@ -111,9 +116,14 @@ impl Client {
         let log = self.log.new(o!("component" => "SledAgentSprocketsClient"));
         // Establish connection and sprockets connection (if possible).
         // The sprockets client loads the associated root certificates at this point.
+        //
+        // TODO: Use a real corpus
+        let corpus = vec![];
         let stream = SprocketsClient::connect(
             self.sprockets_conf.clone(),
             self.addr,
+            // We don't have corpus files yet
+            vec![],
             log.clone(),
         )
         .await

sled-agent/src/bootstrap/client.rs

@@ -7,3 +7,4 @@
 pub const BOOTSTRAP_AGENT_HTTP_PORT: u16 = 80;
 pub const BOOTSTRAP_AGENT_RACK_INIT_PORT: u16 = 12346;
 pub const BOOTSTORE_PORT: u16 = 12347;
+pub const TRUST_QUORUM_PORT: u16 = 12349;

sled-agent/src/bootstrap/config.rs

@@ -59,22 +59,32 @@ impl SprocketsServer {
     /// which is cancel-safe. Note that cancelling this
     /// server does not necessarily cancel any outstanding requests that it has
     /// already received (and which may still be executing).
-    pub(super) async fn run(mut self) {
+    pub(super) async fn run(self) {
         loop {
             // Sprockets actually _uses_ the key here!
-            let (stream, remote_addr) = match self.listener.accept().await {
-                Ok(conn) => conn,
-                Err(err) => {
-                    error!(self.log, "accept() failed"; "err" => #%err);
-                    continue;
-                }
-            };
-
-            let log = self.log.new(o!("remote_addr" => remote_addr));
-            info!(log, "Accepted connection");
+            // We don't have corpus files yet, so pass in an empty Vec
+            let (stream, remote_addr) =
+                match self.listener.accept(vec![]).await.await {
+                    Ok(conn) => conn,
+                    Err(err) => {
+                        error!(self.log, "accept() failed"; "err" => #%err);
+                        continue;
+                    }
+                };
 
+            let log = self.log.new(o!("remote_addr" => acceptor.addr()));
+            info!(log, "TCP connection accepted");
             let tx_requests = self.tx_requests.clone();
             tokio::spawn(async move {
+                let stream = match acceptor.handshake().await {
+                    Ok((stream, _)) => stream,
+                    Err(err) => {
+                        error!(log, "Sprockets handshake failed"; &err);
+                        return;
+                    }
+                };
+                info!(log, "Sprockets handshake completed");
+
                 match handle_start_sled_agent_request(stream, tx_requests, &log)
                     .await
                 {

sled-agent/src/bootstrap/sprockets_server.rs

@@ -78,7 +78,7 @@ use sled_hardware::{HardwareManager, MemoryReservations, underlay};
 use sled_hardware_types::Baseboard;
 use sled_hardware_types::underlay::BootstrapInterface;
 use slog::Logger;
-use slog_error_chain::InlineErrorChain;
+use slog_error_chain::{InlineErrorChain, SlogInlineError};
 use sprockets_tls::keys::SprocketsConfig;
 use std::collections::BTreeMap;
 use std::net::{Ipv6Addr, SocketAddrV6};
@@ -1191,7 +1191,7 @@ impl SledAgent {
     }
 }
 
-#[derive(From, thiserror::Error, Debug)]
+#[derive(From, thiserror::Error, Debug, SlogInlineError)]
 pub enum AddSledError {
     #[error("Failed to learn bootstrap ip for {sled_id}")]
     BootstrapAgentClient {
@@ -1206,6 +1206,7 @@ pub enum AddSledError {
     #[error("Failed to initialize {sled_id}: {err}")]
     BootstrapTcpClient {
         sled_id: Baseboard,
+        #[source]
         err: crate::bootstrap::client::Error,
     },
 }

sled-agent/src/sled_agent.rs

@@ -76,4 +76,5 @@ if_exists = "append"
 
 [sprockets]
 resolve = { which = "ipcc" }
+attest = { which = "ipcc" }
 roots = ["/usr/share/oxide/idcerts/staging.pem", "/usr/share/oxide/idcerts/production.pem"]

smf/sled-agent/gimlet-standalone/config.toml

@@ -72,4 +72,5 @@ if_exists = "append"
 
 [sprockets]
 resolve = { which = "ipcc" }
+attest = { which = "ipcc" }
 roots = ["/usr/share/oxide/idcerts/staging.pem", "/usr/share/oxide/idcerts/production.pem"]

smf/sled-agent/gimlet/config.toml

@@ -77,46 +77,6 @@ certificate "test-signer-a1" {
     }
 }
 
-key-pair "test-signer-a2" {
-    p384
-}
-
-entity "test-signer-a2" {
-    country-name "US"
-    organization-name "Oxide Computer Company"
-    common-name "test-platformid-1 Signer Staging A2"
-}
-
-certificate "test-signer-a2" {
-    issuer-certificate "test-root-a"
-    issuer-key "test-root-a"
-
-    subject-entity "test-signer-a2"
-    subject-key "test-signer-a2"
-
-    digest-algorithm "sha-384"
-    not-after "9999-12-31T23:59:59Z"
-    serial-number "01"
-
-    extensions {
-        subject-key-identifier critical=false
-        authority-key-identifier critical=false {
-            key-id
-        }
-
-        basic-constraints critical=true ca=true
-        key-usage critical=true {
-            key-cert-sign
-            crl-sign
-        }
-        certificate-policies critical=true {
-            oana-platform-identity
-            tcg-dice-kp-identity-init
-            tcg-dice-kp-attest-init
-            tcg-dice-kp-eca
-        }
-    }
-}
 /// Device 1
 key-pair "test-platformid-1" {
     ed25519
@@ -166,7 +126,7 @@ key-pair "test-deviceid-1" {
 entity "test-deviceid-1" {
     country-name "US"
     organization-name "Oxide Computer Company"
-    common-name "/C=US/O=Oxide Computer Company/CN=test-deviceid-1"
+    common-name "test-deviceid-1"
 }
 
 certificate "test-deviceid-1" {
@@ -207,7 +167,7 @@ key-pair "test-sprockets-auth-1" {
 entity "test-sprockets-auth-1" {
     country-name "US"
     organization-name "Oxide Computer Company"
-    common-name "/C=US/O=Oxide Computer Company/CN=test-sprockets-auth-1"
+    common-name "test-sprockets-auth-1"
 }
 
 certificate "test-sprockets-auth-1" {
@@ -241,6 +201,58 @@ certificate "test-sprockets-auth-1" {
     }
 }
 
+// TODO: sprockets reverses this cert chain before passing it to rustls
+certificate-list "test-sprockets-auth-1" \
+    "test-signer-a1" \
+    "test-platformid-1" \
+    "test-deviceid-1" \
+    "test-sprockets-auth-1"
+
+key-pair "test-alias-1" {
+    ed25519
+}
+
+entity "test-alias-1" {
+    country-name "US"
+    organization-name "Oxide Computer Company"
+    common-name "alias"
+}
+
+certificate "test-alias-1" {
+    issuer-certificate "test-deviceid-1"
+    issuer-key "test-deviceid-1"
+
+    subject-entity "test-alias-1"
+    subject-key "test-alias-1"
+
+    not-after "9999-12-31T23:59:59Z"
+    serial-number "00"
+
+    extensions {
+        basic-constraints critical=true ca=false
+        key-usage critical=true {
+            digital-signature
+        }
+        certificate-policies critical=true {
+            tcg-dice-kp-attest-init
+        }
+        dice-tcb-info critical=true {
+            fwid-list {
+                fwid {
+                    digest-algorithm "sha3-256"
+                    digest "72fa8f8ea84a42251031366002cbb36281d0131f78cd680436116a720cdd9de5"
+                }
+            }
+        }
+    }
+}
+
+certificate-list "test-alias-1" \
+    "test-alias-1" \
+    "test-deviceid-1" \
+    "test-platformid-1" \
+    "test-signer-a1"
+
 /// Device 2
 
 key-pair "test-platformid-2" {
@@ -291,7 +303,7 @@ key-pair "test-deviceid-2" {
 entity "test-deviceid-2" {
     country-name "US"
     organization-name "Oxide Computer Company"
-    common-name "/C=US/O=Oxide Computer Company/CN=test-deviceid-2"
+    common-name "test-deviceid-2"
 }
 
 certificate "test-deviceid-2" {
@@ -332,7 +344,7 @@ key-pair "test-sprockets-auth-2" {
 entity "test-sprockets-auth-2" {
     country-name "US"
     organization-name "Oxide Computer Company"
-    common-name "/C=US/O=Oxide Computer Company/CN=test-sprockets-auth-2"
+    common-name "test-sprockets-auth-2"
 }
 
 certificate "test-sprockets-auth-2" {
@@ -366,3 +378,54 @@ certificate "test-sprockets-auth-2" {
     }
 }
 
+// TODO: sprockets reverses this cert chain before passing it to rustls
+certificate-list "test-sprockets-auth-2" \
+    "test-signer-a1" \
+    "test-platformid-2" \
+    "test-deviceid-2" \
+    "test-sprockets-auth-2"
+
+key-pair "test-alias-2" {
+    ed25519
+}
+
+entity "test-alias-2" {
+    country-name "US"
+    organization-name "Oxide Computer Company"
+    common-name "alias"
+}
+
+certificate "test-alias-2" {
+    issuer-certificate "test-deviceid-2"
+    issuer-key "test-deviceid-2"
+
+    subject-entity "test-alias-2"
+    subject-key "test-alias-2"
+
+    not-after "9999-12-31T23:59:59Z"
+    serial-number "00"
+
+    extensions {
+        basic-constraints critical=true ca=false
+        key-usage critical=true {
+            digital-signature
+        }
+        certificate-policies critical=true {
+            tcg-dice-kp-attest-init
+        }
+        dice-tcb-info critical=true {
+            fwid-list {
+                fwid {
+                    digest-algorithm "sha3-256"
+                    digest "72fa8f8ea84a42251031366002cbb36281d0131f78cd680436116a720cdd9de5"
+                }
+            }
+        }
+    }
+}
+
+certificate-list "test-alias-2" \
+    "test-alias-2" \
+    "test-deviceid-2" \
+    "test-platformid-2" \
+    "test-signer-a1"

smf/sled-agent/non-gimlet/config.kdl

@@ -121,4 +121,5 @@ if_exists = "append"
 # See the .kdl file for use with pki-playground for generating
 [sprockets]
 resolve = { which = "local", priv_key = "/opt/oxide/sled-agent/pkg/sprockets-auth.key.pem", cert_chain = "/opt/oxide/sled-agent/pkg/sprockets-chain.pem" }
+attest = { which = "local", priv_key = "/opt/oxide/sled-agent/pkg/sprockets-attest.key.pem", cert_chain = "/opt/oxide/sled-agent/pkg/sprockets-attest-chain.pem", log = "/opt/oxide/sled-agent/pkg/sprockets-log.bin" }
 roots = ["/opt/oxide/sled-agent/pkg/root.cert.pem"]