Push probe zones to sled-agent instead of pulling them from Nexus

- Add APIs in the sled-agent for creating / deleting probes, and have
  Nexus use them when managing probes from the external API, especially
  replacing the entire set of probes with a PUT.
- Rework the probe manager to accept the list of expected probes from
  Nexus, and drive the state toward that, rather than periodically
  pollling Nexus.
- Add background task for periodically pushing probes to sleds, and omdb
  innards for reporting its state.
- Closes #9157

Author: bnaecker

dev-tools/omdb/src/bin/omdb/nexus.rs

@@ -58,6 +58,7 @@ use nexus_types::internal_api::background::InstanceReincarnationStatus;
 use nexus_types::internal_api::background::InstanceUpdaterStatus;
 use nexus_types::internal_api::background::InventoryLoadStatus;
 use nexus_types::internal_api::background::LookupRegionPortStatus;
+use nexus_types::internal_api::background::ProbeDistributorStatus;
 use nexus_types::internal_api::background::ReadOnlyRegionReplacementStartStatus;
 use nexus_types::internal_api::background::RegionReplacementDriverStatus;
 use nexus_types::internal_api::background::RegionReplacementStatus;
@@ -1189,6 +1190,9 @@ fn print_task_details(bgtask: &BackgroundTask, details: &serde_json::Value) {
         "phantom_disks" => {
             print_task_phantom_disks(details);
         }
+        "probe_distributor" => {
+            print_task_probe_distributor(details);
+        }
         "read_only_region_replacement_start" => {
             print_task_read_only_region_replacement_start(details);
         }
@@ -2110,6 +2114,32 @@ fn print_task_phantom_disks(details: &serde_json::Value) {
     };
 }
 
+fn print_task_probe_distributor(details: &serde_json::Value) {
+    match serde_json::from_value::<ProbeDistributorStatus>(details.clone()) {
+        Err(error) => eprintln!(
+            "warning: failed to interpret task details: {:?}: {:?}",
+            error, details
+        ),
+        Ok(status) => {
+            let n_total_probes: usize = status.probes_by_sled.values().sum();
+            println!("    succesfully-pushed probes: {} total", n_total_probes);
+            for (sled_id, count) in status.probes_by_sled {
+                println!("      sled_id={} n_probes={}", sled_id, count);
+            }
+            println!(
+                "    errors while pushing probes: {} total",
+                status.errors.len()
+            );
+            for err in status.errors {
+                println!(
+                    "      sled_id={} sled_ip={} error={}",
+                    err.sled_id, err.sled_ip, err.error,
+                );
+            }
+        }
+    };
+}
+
 fn print_task_read_only_region_replacement_start(details: &serde_json::Value) {
     match serde_json::from_value::<ReadOnlyRegionReplacementStartStatus>(
         details.clone(),

illumos-utils/src/opte/port.rs

@@ -57,7 +57,7 @@ impl Drop for PortInner {
             Err(e) => e,
         };
         eprintln!(
-            "WARNING: Failed to delete the xde device. It must be deleted
+            "WARNING: Failed to delete the xde device. It must be deleted \
             out of band, and it will not be possible to recreate the xde \
             device until then. Error: {:?}",
             err,

nexus-config/src/nexus_config.rs

@@ -441,6 +441,8 @@ pub struct BackgroundTaskConfig {
     pub webhook_deliverator: WebhookDeliveratorConfig,
     /// configuration for SP ereport ingester task
     pub sp_ereport_ingester: SpEreportIngesterConfig,
+    /// configuration for networking probe distributor
+    pub probe_distributor: ProbeDistributorConfig,
 }
 
 #[serde_as]
@@ -870,6 +872,15 @@ impl Default for SpEreportIngesterConfig {
     }
 }
 
+#[serde_as]
+#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]
+pub struct ProbeDistributorConfig {
+    /// period (in seconds) for periodic activations of the background task that
+    /// distributes networking probe zones to sled-agents.
+    #[serde_as(as = "DurationSeconds<u64>")]
+    pub period_secs: Duration,
+}
+
 /// Configuration for a nexus server
 #[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
 pub struct PackageConfig {
@@ -1172,6 +1183,7 @@ mod test {
             webhook_deliverator.first_retry_backoff_secs = 45
             webhook_deliverator.second_retry_backoff_secs = 46
             sp_ereport_ingester.period_secs = 47
+            probe_distributor.period_secs = 48
             [default_region_allocation_strategy]
             type = "random"
             seed = 0
@@ -1416,6 +1428,9 @@ mod test {
                             period_secs: Duration::from_secs(47),
                             disable: false,
                         },
+                        probe_distributor: ProbeDistributorConfig {
+                            period_secs: Duration::from_secs(48),
+                        },
                     },
                     default_region_allocation_strategy:
                         crate::nexus_config::RegionAllocationStrategy::Random {
@@ -1514,6 +1529,7 @@ mod test {
             alert_dispatcher.period_secs = 42
             webhook_deliverator.period_secs = 43
             sp_ereport_ingester.period_secs = 44
+            probe_distributor.period_secs = 45
 
             [default_region_allocation_strategy]
             type = "random"

nexus/background-task-interface/src/init.rs

@@ -51,6 +51,7 @@ pub struct BackgroundTasks {
     pub task_webhook_deliverator: Activator,
     pub task_sp_ereport_ingester: Activator,
     pub task_reconfigurator_config_loader: Activator,
+    pub task_probe_distributor: Activator,
 
     // Handles to activate background tasks that do not get used by Nexus
     // at-large.  These background tasks are implementation details as far as