From 9bb6a49659d2f05f692d1ccc878288bfcce33860 Mon Sep 17 00:00:00 2001 From: MasterPtato Date: Thu, 2 Jan 2025 20:17:10 +0000 Subject: [PATCH] fix: add pegboard client metrics --- packages/services/pegboard/src/lib.rs | 2 +- packages/services/pegboard/src/metrics.rs | 18 ++++- .../services/pegboard/src/workflows/client.rs | 2 +- .../standalone/metrics-publish/Cargo.toml | 25 +++++++ .../standalone/metrics-publish/src/lib.rs | 67 +++++++++++++++++++ .../metrics-publish/tests/integration.rs | 1 + 6 files changed, 111 insertions(+), 4 deletions(-) create mode 100644 packages/services/pegboard/standalone/metrics-publish/Cargo.toml create mode 100644 packages/services/pegboard/standalone/metrics-publish/src/lib.rs create mode 100644 packages/services/pegboard/standalone/metrics-publish/tests/integration.rs diff --git a/packages/services/pegboard/src/lib.rs b/packages/services/pegboard/src/lib.rs index 09281bebd1..9de915810f 100644 --- a/packages/services/pegboard/src/lib.rs +++ b/packages/services/pegboard/src/lib.rs @@ -2,7 +2,7 @@ use chirp_workflow::prelude::*; pub mod client_config; -mod metrics; +pub mod metrics; #[cfg(feature = "ops")] pub mod ops; pub mod protocol; diff --git a/packages/services/pegboard/src/metrics.rs b/packages/services/pegboard/src/metrics.rs index fc7a8b6f01..fefde52758 100644 --- a/packages/services/pegboard/src/metrics.rs +++ b/packages/services/pegboard/src/metrics.rs @@ -1,10 +1,24 @@ use rivet_metrics::{prometheus::*, REGISTRY}; lazy_static::lazy_static! { - pub static ref PEGBOARD_DUPLICATE_CLIENT_EVENT: IntCounterVec = register_int_counter_vec_with_registry!( - "pegboard_duplicate_client_event", + pub static ref PEGBOARD_CLIENT_DUPLICATE_EVENT: IntCounterVec = register_int_counter_vec_with_registry!( + "pegboard_client_duplicate_event", "Duplicate client event that was attempted to be inserted.", &["client_id", "index"], *REGISTRY ).unwrap(); + + pub static ref PEGBOARD_CLIENT_LAST_PING: IntGaugeVec = register_int_gauge_vec_with_registry!( + "pegboard_client_last_ping", + "Last client ping timestamp, in ms.", + &["client_id"], + *REGISTRY + ).unwrap(); + + pub static ref PEGBOARD_CLIENT_ACTORS_ALLOCATED: IntCounterVec = register_int_counter_vec_with_registry!( + "pegboard_client_actors_allocated", + "Total actors allocated on a client.", + &["client_id"], + *REGISTRY + ).unwrap(); } diff --git a/packages/services/pegboard/src/workflows/client.rs b/packages/services/pegboard/src/workflows/client.rs index 41f104785d..d0333b24d8 100644 --- a/packages/services/pegboard/src/workflows/client.rs +++ b/packages/services/pegboard/src/workflows/client.rs @@ -275,7 +275,7 @@ async fn insert_events(ctx: &ActivityCtx, input: &InsertEventsInput) -> GlobalRe continue; } - metrics::PEGBOARD_DUPLICATE_CLIENT_EVENT + metrics::PEGBOARD_CLIENT_DUPLICATE_EVENT .with_label_values(&[&input.client_id.to_string(), &event.index.to_string()]) .inc(); } diff --git a/packages/services/pegboard/standalone/metrics-publish/Cargo.toml b/packages/services/pegboard/standalone/metrics-publish/Cargo.toml new file mode 100644 index 0000000000..75736b82e8 --- /dev/null +++ b/packages/services/pegboard/standalone/metrics-publish/Cargo.toml @@ -0,0 +1,25 @@ +[package] +name = "pegboard-metrics-publish" +version.workspace = true +authors.workspace = true +license.workspace = true +edition.workspace = true + +[dependencies] +chirp-client.workspace = true +chirp-workflow.workspace = true +rivet-config.workspace = true +rivet-connection.workspace = true +rivet-health-checks.workspace = true +rivet-metrics.workspace = true +rivet-runtime.workspace = true +tokio = { version = "1.40", features = ["full"] } +tracing = "0.1" +tracing-subscriber = { version = "0.3", default-features = false, features = ["fmt", "json", "ansi"] } + +pegboard = { path = "../.." } + +[dependencies.sqlx] +workspace = true + +[dev-dependencies] diff --git a/packages/services/pegboard/standalone/metrics-publish/src/lib.rs b/packages/services/pegboard/standalone/metrics-publish/src/lib.rs new file mode 100644 index 0000000000..ad3e1373bf --- /dev/null +++ b/packages/services/pegboard/standalone/metrics-publish/src/lib.rs @@ -0,0 +1,67 @@ +use chirp_workflow::prelude::*; + +pub async fn start(config: rivet_config::Config, pools: rivet_pools::Pools) -> GlobalResult<()> { + let mut interval = + tokio::time::interval(std::time::Duration::from_secs(15)); + loop { + interval.tick().await; + + run_from_env(config.clone(), pools.clone(), util::timestamp::now()).await?; + } +} + +#[tracing::instrument(skip_all)] +pub async fn run_from_env( + config: rivet_config::Config, + pools: rivet_pools::Pools, + ts: i64, +) -> GlobalResult<()> { + let client = + chirp_client::SharedClient::from_env(pools.clone())?.wrap_new("pegboard-metrics-publish"); + let cache = rivet_cache::CacheInner::from_env(pools.clone())?; + let ctx = StandaloneCtx::new( + chirp_workflow::compat::db_from_pools(&pools).await?, + config, + rivet_connection::Connection::new(client, pools, cache), + "pegboard-metrics-publish", + ) + .await?; + + let (client_ping, client_actors) = tokio::try_join!( + sql_fetch_all!( + [ctx, (Uuid, i64)] + " + SELECT last_ping_ts + FROM db_pegboard.clients AS OF SYSTEM TIME '-1s' + WHERE delete_ts IS NULL + ", + ts - util::duration::seconds(30), + ), + sql_fetch_all!( + [ctx, (Uuid, i64)] + " + SELECT a.client_id, COUNT(*) + FROM db_pegboard.actors AS a + JOIN db_pegboard.clients AS c + ON a.client_id = c.client_id + AS OF SYSTEM TIME '-1s' + WHERE c.delete_ts IS NULL + GROUP BY a.client_id + ", + ), + )?; + + for (client_id, last_ping_ts) in client_ping { + pegboard::metrics::PEGBOARD_CLIENT_LAST_PING + .with_label_values(&[&client_id.to_string()]) + .set(last_ping_ts); + } + + for (client_id, count) in client_actors { + pegboard::metrics::PEGBOARD_CLIENT_ACTORS_ALLOCATED + .with_label_values(&[&client_id.to_string()]) + .set(count.try_into()?); + } + + Ok(()) +} diff --git a/packages/services/pegboard/standalone/metrics-publish/tests/integration.rs b/packages/services/pegboard/standalone/metrics-publish/tests/integration.rs new file mode 100644 index 0000000000..6c8ea4d0f2 --- /dev/null +++ b/packages/services/pegboard/standalone/metrics-publish/tests/integration.rs @@ -0,0 +1 @@ +// TODO: