Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix: add pegboard client metrics #1757

Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion packages/services/pegboard/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
use chirp_workflow::prelude::*;

pub mod client_config;
mod metrics;
pub mod metrics;
#[cfg(feature = "ops")]
pub mod ops;
pub mod protocol;
Expand Down
18 changes: 16 additions & 2 deletions packages/services/pegboard/src/metrics.rs
Original file line number Diff line number Diff line change
@@ -1,10 +1,24 @@
use rivet_metrics::{prometheus::*, REGISTRY};

lazy_static::lazy_static! {
pub static ref PEGBOARD_DUPLICATE_CLIENT_EVENT: IntCounterVec = register_int_counter_vec_with_registry!(
"pegboard_duplicate_client_event",
pub static ref PEGBOARD_CLIENT_DUPLICATE_EVENT: IntCounterVec = register_int_counter_vec_with_registry!(
"pegboard_client_duplicate_event",
"Duplicate client event that was attempted to be inserted.",
&["client_id", "index"],
*REGISTRY
).unwrap();

pub static ref PEGBOARD_CLIENT_LAST_PING: IntGaugeVec = register_int_gauge_vec_with_registry!(
"pegboard_client_last_ping",
"Last client ping timestamp, in ms.",
&["client_id"],
*REGISTRY
).unwrap();

pub static ref PEGBOARD_CLIENT_ACTORS_ALLOCATED: IntCounterVec = register_int_counter_vec_with_registry!(
"pegboard_client_actors_allocated",
"Total actors allocated on a client.",
&["client_id"],
*REGISTRY
).unwrap();
}
2 changes: 1 addition & 1 deletion packages/services/pegboard/src/workflows/client.rs
Original file line number Diff line number Diff line change
Expand Up @@ -275,7 +275,7 @@ async fn insert_events(ctx: &ActivityCtx, input: &InsertEventsInput) -> GlobalRe
continue;
}

metrics::PEGBOARD_DUPLICATE_CLIENT_EVENT
metrics::PEGBOARD_CLIENT_DUPLICATE_EVENT
.with_label_values(&[&input.client_id.to_string(), &event.index.to_string()])
.inc();
}
Expand Down
25 changes: 25 additions & 0 deletions packages/services/pegboard/standalone/metrics-publish/Cargo.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
[package]
name = "pegboard-metrics-publish"
version.workspace = true
authors.workspace = true
license.workspace = true
edition.workspace = true

[dependencies]
chirp-client.workspace = true
chirp-workflow.workspace = true
rivet-config.workspace = true
rivet-connection.workspace = true
rivet-health-checks.workspace = true
rivet-metrics.workspace = true
rivet-runtime.workspace = true
tokio = { version = "1.40", features = ["full"] }
tracing = "0.1"
tracing-subscriber = { version = "0.3", default-features = false, features = ["fmt", "json", "ansi"] }

pegboard = { path = "../.." }

[dependencies.sqlx]
workspace = true

[dev-dependencies]
67 changes: 67 additions & 0 deletions packages/services/pegboard/standalone/metrics-publish/src/lib.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,67 @@
use chirp_workflow::prelude::*;

pub async fn start(config: rivet_config::Config, pools: rivet_pools::Pools) -> GlobalResult<()> {
let mut interval =
tokio::time::interval(std::time::Duration::from_secs(15));
loop {
interval.tick().await;

run_from_env(config.clone(), pools.clone(), util::timestamp::now()).await?;
}
}

#[tracing::instrument(skip_all)]
pub async fn run_from_env(
config: rivet_config::Config,
pools: rivet_pools::Pools,
ts: i64,
) -> GlobalResult<()> {
let client =
chirp_client::SharedClient::from_env(pools.clone())?.wrap_new("pegboard-metrics-publish");
let cache = rivet_cache::CacheInner::from_env(pools.clone())?;
let ctx = StandaloneCtx::new(
chirp_workflow::compat::db_from_pools(&pools).await?,
config,
rivet_connection::Connection::new(client, pools, cache),
"pegboard-metrics-publish",
)
.await?;

let (client_ping, client_actors) = tokio::try_join!(
sql_fetch_all!(
[ctx, (Uuid, i64)]
"
SELECT last_ping_ts
FROM db_pegboard.clients AS OF SYSTEM TIME '-1s'
WHERE delete_ts IS NULL
",
ts - util::duration::seconds(30),
),
sql_fetch_all!(
[ctx, (Uuid, i64)]
"
SELECT a.client_id, COUNT(*)
FROM db_pegboard.actors AS a
JOIN db_pegboard.clients AS c
ON a.client_id = c.client_id
AS OF SYSTEM TIME '-1s'
WHERE c.delete_ts IS NULL
GROUP BY a.client_id
",
),
)?;

for (client_id, last_ping_ts) in client_ping {
pegboard::metrics::PEGBOARD_CLIENT_LAST_PING
.with_label_values(&[&client_id.to_string()])
.set(last_ping_ts);
}

for (client_id, count) in client_actors {
pegboard::metrics::PEGBOARD_CLIENT_ACTORS_ALLOCATED
.with_label_values(&[&client_id.to_string()])
.set(count.try_into()?);
}

Ok(())
}
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
// TODO:
Loading