[nexus] Use cockroachdb range stats in reconfigurator planner (#8441)

Updates the reconfigurator to evaluate cockroachdb cluster health before
upgrading zones

Only updates zones if:
- Ranges underreplicated == 0, and
- Live nodes == COCKROACH_REDUNDANCY
- At least COCKROACH_REDUNDANCY nodes are reporting their status
successfully

Builds on #8379 and
#8426

Author: smklein

Cargo.lock

@@ -506,7 +506,7 @@ INFO sufficient ExternalDns zones exist in plan, desired_count: 0, current_count
 WARN failed to place all new desired Nexus zones, placed: 0, wanted_to_place: 3
 INFO sufficient Oximeter zones exist in plan, desired_count: 0, current_count: 0
 WARN cannot issue more SP updates (no current artifacts)
-INFO some zones not yet up-to-date, sled_id: 89d02b1b-478c-401a-8e28-7a26f74fa41b
+INFO some zones not yet up-to-date, sled_id: 89d02b1b-478c-401a-8e28-7a26f74fa41b, zones_currently_updating: [(b3c9c041-d2f0-4767-bdaf-0e52e9d7a013 (service), InternalNtp)]
 INFO will ensure cockroachdb setting, setting: cluster.preserve_downgrade_option, value: DoNotModify
 generated blueprint 86db3308-f817-4626-8838-4085949a6a41 based on parent blueprint ade5749d-bdf3-4fab-a8ae-00bea01b3a5a
 

dev-tools/reconfigurator-cli/tests/output/cmds-example-stdout

@@ -981,7 +981,7 @@ INFO added zone to sled, sled_id: 711ac7f8-d19e-4572-bdb9-e9b50f6e362a, kind: Ex
 INFO sufficient Nexus zones exist in plan, desired_count: 3, current_count: 3
 INFO sufficient Oximeter zones exist in plan, desired_count: 0, current_count: 0
 WARN cannot issue more SP updates (no current artifacts)
-INFO some zones not yet up-to-date, sled_id: 711ac7f8-d19e-4572-bdb9-e9b50f6e362a
+INFO some zones not yet up-to-date, sled_id: 711ac7f8-d19e-4572-bdb9-e9b50f6e362a, zones_currently_updating: [(fe2d5287-24e3-4071-b214-2640b097a759 (service), ExternalDns)]
 INFO will ensure cockroachdb setting, setting: cluster.preserve_downgrade_option, value: DoNotModify
 generated blueprint 9c998c1d-1a7b-440a-ae0c-40f781dea6e2 based on parent blueprint 366b0b68-d80e-4bc1-abd3-dc69837847e0
 

dev-tools/reconfigurator-cli/tests/output/cmds-expunge-newly-added-external-dns-stdout

@@ -1014,7 +1014,7 @@ INFO sufficient ExternalDns zones exist in plan, desired_count: 3, current_count
 INFO sufficient Nexus zones exist in plan, desired_count: 3, current_count: 3
 INFO sufficient Oximeter zones exist in plan, desired_count: 0, current_count: 0
 WARN cannot issue more SP updates (no current artifacts)
-INFO some zones not yet up-to-date, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c
+INFO some zones not yet up-to-date, sled_id: 2b8f0cb3-0295-4b3c-bc58-4fe88b57112c, zones_currently_updating: [(e375dd21-320b-43b7-bc92-a2c3dac9d9e1 (service), InternalDns)]
 INFO will ensure cockroachdb setting, setting: cluster.preserve_downgrade_option, value: DoNotModify
 generated blueprint af934083-59b5-4bf6-8966-6fb5292c29e1 based on parent blueprint 58d5e830-0884-47d8-a7cd-b2b3751adeb4
 

dev-tools/reconfigurator-cli/tests/output/cmds-expunge-newly-added-internal-dns-stdout

@@ -47,6 +47,7 @@ omicron-workspace-hack.workspace = true
 dropshot.workspace = true
 expectorate.workspace = true
 maplit.workspace = true
+omicron-cockroach-metrics.workspace = true
 omicron-common = { workspace = true, features = ["testing"] }
 omicron-test-utils.workspace = true
 proptest.workspace = true

nexus/reconfigurator/planning/Cargo.toml

@@ -400,14 +400,12 @@ impl fmt::Display for Operation {
 ///    However, the new blueprint can only be made the system's target if its
 ///    parent is the current target.
 pub struct BlueprintBuilder<'a> {
-    #[allow(dead_code)]
     log: Logger,
 
     /// previous blueprint, on which this one will be based
     parent_blueprint: &'a Blueprint,
 
     /// The latest inventory collection
-    #[allow(unused)]
     collection: &'a Collection,
 
     // These fields are used to allocate resources for sleds.