fix: add nonreporting server metrics (#1763)

<!-- Please make sure there is an issue that this PR is correlated to. -->

## Changes

<!-- If there are frontend changes, please include screenshots. -->

packages/services/cluster/src/lib.rs

@@ -4,6 +4,7 @@ pub mod ops;
 pub mod types;
 pub mod util;
 pub mod workflows;
+pub mod metrics;
 
 pub fn registry() -> WorkflowResult<Registry> {
 	use workflows::*;

packages/services/cluster/src/util/metrics.rs → packages/services/cluster/src/metrics.rs

@@ -84,4 +84,11 @@ lazy_static::lazy_static! {
 		PROVISION_BUCKETS.to_vec(),
 		*REGISTRY,
 	).unwrap();
+
+	pub static ref NONREPORTING_SERVER: IntGaugeVec = register_int_gauge_vec_with_registry!(
+		"provision_nonreporting_server",
+		"Servers without reporting Prometheus metrics.",
+		&["cluster_id", "datacenter_id", "server_id", "provider_datacenter_id", "pool_type"],
+		*REGISTRY,
+	).unwrap();
 }

packages/services/cluster/src/ops/datacenter/topology_get.rs

@@ -87,6 +87,8 @@ pub struct Server {
 	pub pool_type: PoolType,
 	pub usage: Stats,
 	pub limits: Stats,
+	/// Whether or not metrics for this server could not be found and ended up defaulting
+	pub missing: bool,
 }
 
 #[derive(Clone, Debug, Default)]
@@ -290,7 +292,7 @@ pub async fn cluster_datacenter_topology_get(
 		// We assume a server has the default memory
 		// amount (memory of the first hardware in the list) if it is not provisioned yet
 
-		let (usage, limits) = match &server.runtime {
+		let (usage, limits, missing) = match &server.runtime {
 			Runtime::Nomad(nomad_node_id) => {
 				// Gracefully handle if node does not exist in API response
 				if let Some(node) = node_info.iter().find(|node| {
@@ -353,30 +355,33 @@ pub async fn cluster_datacenter_topology_get(
 						bandwidth: 0, // TODO:
 					};
 
-					(usage, limits)
+					(usage, limits, false)
 				} else {
 					tracing::error!(%nomad_node_id, "node not found in nomad response");
 
-					(Stats::default(), Stats::default())
+					(Stats::default(), Stats::default(), true)
 				}
 			}
 			Runtime::Pegboard(pegboard_client_id) => {
 				// Gracefully handle if client usage exists
-				let usage = if let Some(client) = pb_client_usage_res
+				let (usage, missing) = if let Some(client) = pb_client_usage_res
 					.clients
 					.iter()
 					.find(|client| &client.client_id == pegboard_client_id)
 				{
-					Stats {
-						cpu: client.usage.cpu,
-						memory: client.usage.memory,
-						disk: client.usage.disk,
-						bandwidth: 0, // TODO:
-					}
+					(
+						Stats {
+							cpu: client.usage.cpu,
+							memory: client.usage.memory,
+							disk: client.usage.disk,
+							bandwidth: 0, // TODO:
+						},
+						false,
+					)
 				} else {
 					tracing::error!(%pegboard_client_id, "pegboard client not found in response");
 
-					Stats::default()
+					(Stats::default(), true)
 				};
 
 				(
@@ -386,19 +391,20 @@ pub async fn cluster_datacenter_topology_get(
 						server.provider_hardware.as_deref(),
 						default_provider_hardware,
 					)?,
+					missing,
 				)
 			}
 			Runtime::None => {
 				// Gracefully handle if prometheus metrics exist
-				let usage = if let Some(server_metrics) = prometheus_metrics
+				let (usage, missing) = if let Some(server_metrics) = prometheus_metrics
 					.as_ref()
 					.and_then(|x| x.get(&server.server_id))
 				{
-					server_metrics.clone()
+					(server_metrics.clone(), false)
 				} else {
 					tracing::warn!(server_id=%server.server_id, "no prometheus metrics for server");
 
-					Stats::default()
+					(Stats::default(), true)
 				};
 
 				(
@@ -408,6 +414,7 @@ pub async fn cluster_datacenter_topology_get(
 						server.provider_hardware.as_deref(),
 						default_provider_hardware,
 					)?,
+					missing,
 				)
 			}
 		};
@@ -418,6 +425,7 @@ pub async fn cluster_datacenter_topology_get(
 			pool_type: server.pool_type,
 			usage,
 			limits,
+			missing,
 		});
 	}
 

packages/services/cluster/src/util/mod.rs

@@ -3,7 +3,6 @@ use cloudflare::{endpoints as cf, framework as cf_framework};
 
 use crate::types::PoolType;
 
-pub mod metrics;
 pub mod test;
 
 // Use the hash of the server install script in the image variant so that if the install scripts are updated

packages/services/cluster/src/workflows/server/install/mod.rs

@@ -10,7 +10,7 @@ use ssh2::Session;
 
 use crate::{
 	types::{Datacenter, PoolType},
-	util::metrics,
+	metrics,
 };
 
 mod install_scripts;

packages/services/cluster/src/workflows/server/mod.rs

@@ -15,7 +15,7 @@ pub(crate) mod undrain;
 
 use crate::{
 	types::{Pool, PoolType, Provider},
-	util::metrics,
+	metrics,
 };
 
 #[derive(Debug, Serialize, Deserialize)]

packages/services/cluster/standalone/metrics-publish/src/lib.rs

@@ -3,7 +3,7 @@ use std::convert::{TryFrom, TryInto};
 use chirp_workflow::prelude::*;
 use cluster::{
 	types::{Datacenter, PoolType},
-	util::metrics,
+	metrics,
 };
 
 pub async fn start(config: rivet_config::Config, pools: rivet_pools::Pools) -> GlobalResult<()> {

packages/services/pegboard/src/metrics.rs

@@ -1,21 +1,21 @@
 use rivet_metrics::{prometheus::*, REGISTRY};
 
 lazy_static::lazy_static! {
-	pub static ref PEGBOARD_CLIENT_DUPLICATE_EVENT: IntCounterVec = register_int_counter_vec_with_registry!(
+	pub static ref CLIENT_DUPLICATE_EVENT: IntCounterVec = register_int_counter_vec_with_registry!(
 		"pegboard_client_duplicate_event",
 		"Duplicate client event that was attempted to be inserted.",
 		&["client_id", "index"],
 		*REGISTRY
 	).unwrap();
 
-	pub static ref PEGBOARD_CLIENT_LAST_PING: IntGaugeVec = register_int_gauge_vec_with_registry!(
+	pub static ref CLIENT_LAST_PING: IntGaugeVec = register_int_gauge_vec_with_registry!(
 		"pegboard_client_last_ping",
 		"Last client ping timestamp, in ms.",
 		&["client_id"],
 		*REGISTRY
 	).unwrap();
 
-	pub static ref PEGBOARD_CLIENT_ACTORS_ALLOCATED: IntCounterVec = register_int_counter_vec_with_registry!(
+	pub static ref CLIENT_ACTORS_ALLOCATED: IntCounterVec = register_int_counter_vec_with_registry!(
 		"pegboard_client_actors_allocated",
 		"Total actors allocated on a client.",
 		&["client_id"],

packages/services/pegboard/src/workflows/client.rs

@@ -275,7 +275,7 @@ async fn insert_events(ctx: &ActivityCtx, input: &InsertEventsInput) -> GlobalRe
 			continue;
 		}
 
-		metrics::PEGBOARD_CLIENT_DUPLICATE_EVENT
+		metrics::CLIENT_DUPLICATE_EVENT
 			.with_label_values(&[&input.client_id.to_string(), &event.index.to_string()])
 			.inc();
 	}

packages/services/pegboard/standalone/metrics-publish/src/lib.rs

@@ -52,13 +52,13 @@ pub async fn run_from_env(
 	)?;
 
 	for (client_id, last_ping_ts) in client_ping {
-		pegboard::metrics::PEGBOARD_CLIENT_LAST_PING
+		pegboard::metrics::CLIENT_LAST_PING
 			.with_label_values(&[&client_id.to_string()])
 			.set(last_ping_ts);
 	}
 
 	for (client_id, count) in client_actors {
-		pegboard::metrics::PEGBOARD_CLIENT_ACTORS_ALLOCATED
+		pegboard::metrics::CLIENT_ACTORS_ALLOCATED
 			.with_label_values(&[&client_id.to_string()])
 			.set(count.try_into()?);
 	}