Skip to content

Commit 6e561b2

Browse files
nikhilsinhaparseablenikhilsinhaparseable
committed
add cluster metrics, update regex for parseable server logs
1 parent 6db4b1c commit 6e561b2

File tree

2 files changed

+160
-1
lines changed

2 files changed

+160
-1
lines changed

‎resources/formats.json‎

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1466,8 +1466,22 @@
14661466
]
14671467
},
14681468
{
1469-
"name": "rust_server_logs",
1469+
"name": "parseable_server_logs",
14701470
"regex": [
1471+
{
1472+
"pattern": "^(?P<customer_id>\\S+)\\s+(?P<deployment_id>\\S+)\\s+(?P<timestamp>\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}\\.\\d+Z?)\\s+(?P<level>\\w+)\\s+(?P<logger_context>\\S+)\\s+(?P<thread_id>ThreadId\\(\\d+\\))\\s+(?P<module>.*?):(?P<line_number>\\d+):\\s+(?P<body>.*)",
1473+
"fields": [
1474+
"customer_id",
1475+
"deployment_id",
1476+
"timestamp",
1477+
"level",
1478+
"logger_context",
1479+
"thread_id",
1480+
"module",
1481+
"line_number",
1482+
"body"
1483+
]
1484+
},
14711485
{
14721486
"pattern": "^(?P<timestamp>\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}\\.\\d+Z)\\s+(?P<level>\\w+)\\s+(?P<logger_context>\\S+)\\s+(?P<thread_id>ThreadId\\(\\d+\\))\\s+(?P<module>.*?):(?P<line_number>\\d+):\\s+(?P<body>.*)",
14731487
"fields": [

‎src/metrics/mod.rs‎

Lines changed: 145 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -336,6 +336,117 @@ pub static TOTAL_FILES_SCANNED_IN_OBJECT_STORE_CALLS_BY_DATE: Lazy<IntCounterVec
336336
.expect("metric can be created")
337337
});
338338

339+
// Cluster Billing Metrics - Gauge type metrics for cluster-wide aggregated billing data
340+
pub static TOTAL_CLUSTER_EVENTS_INGESTED_BY_DATE: Lazy<IntGaugeVec> = Lazy::new(|| {
341+
IntGaugeVec::new(
342+
Opts::new(
343+
"total_cluster_events_ingested_by_date",
344+
"Total cluster events ingested by date (Gauge for cluster billing)",
345+
)
346+
.namespace(METRICS_NAMESPACE),
347+
&["date"],
348+
)
349+
.expect("metric can be created")
350+
});
351+
352+
pub static TOTAL_CLUSTER_EVENTS_INGESTED_SIZE_BY_DATE: Lazy<IntGaugeVec> = Lazy::new(|| {
353+
IntGaugeVec::new(
354+
Opts::new(
355+
"total_cluster_events_ingested_size_by_date",
356+
"Total cluster events ingested size in bytes by date (Gauge for cluster billing)",
357+
)
358+
.namespace(METRICS_NAMESPACE),
359+
&["date"],
360+
)
361+
.expect("metric can be created")
362+
});
363+
364+
pub static TOTAL_CLUSTER_PARQUETS_STORED_BY_DATE: Lazy<IntGaugeVec> = Lazy::new(|| {
365+
IntGaugeVec::new(
366+
Opts::new(
367+
"total_cluster_parquets_stored_by_date",
368+
"Total cluster parquet files stored by date (Gauge for cluster billing)",
369+
)
370+
.namespace(METRICS_NAMESPACE),
371+
&["date"],
372+
)
373+
.expect("metric can be created")
374+
});
375+
376+
pub static TOTAL_CLUSTER_PARQUETS_STORED_SIZE_BY_DATE: Lazy<IntGaugeVec> = Lazy::new(|| {
377+
IntGaugeVec::new(
378+
Opts::new(
379+
"total_cluster_parquets_stored_size_by_date",
380+
"Total cluster parquet files stored size in bytes by date (Gauge for cluster billing)",
381+
)
382+
.namespace(METRICS_NAMESPACE),
383+
&["date"],
384+
)
385+
.expect("metric can be created")
386+
});
387+
388+
pub static TOTAL_CLUSTER_QUERY_CALLS_BY_DATE: Lazy<IntGaugeVec> = Lazy::new(|| {
389+
IntGaugeVec::new(
390+
Opts::new(
391+
"total_cluster_query_calls_by_date",
392+
"Total cluster query calls by date (Gauge for cluster billing)",
393+
)
394+
.namespace(METRICS_NAMESPACE),
395+
&["date"],
396+
)
397+
.expect("metric can be created")
398+
});
399+
400+
pub static TOTAL_CLUSTER_FILES_SCANNED_IN_QUERY_BY_DATE: Lazy<IntGaugeVec> = Lazy::new(|| {
401+
IntGaugeVec::new(
402+
Opts::new(
403+
"total_cluster_files_scanned_in_query_by_date",
404+
"Total cluster files scanned in queries by date (Gauge for cluster billing)",
405+
)
406+
.namespace(METRICS_NAMESPACE),
407+
&["date"],
408+
)
409+
.expect("metric can be created")
410+
});
411+
412+
pub static TOTAL_CLUSTER_BYTES_SCANNED_IN_QUERY_BY_DATE: Lazy<IntGaugeVec> = Lazy::new(|| {
413+
IntGaugeVec::new(
414+
Opts::new(
415+
"total_cluster_bytes_scanned_in_query_by_date",
416+
"Total cluster bytes scanned in queries by date (Gauge for cluster billing)",
417+
)
418+
.namespace(METRICS_NAMESPACE),
419+
&["date"],
420+
)
421+
.expect("metric can be created")
422+
});
423+
424+
pub static TOTAL_CLUSTER_OBJECT_STORE_CALLS_BY_DATE: Lazy<IntGaugeVec> = Lazy::new(|| {
425+
IntGaugeVec::new(
426+
Opts::new(
427+
"total_cluster_object_store_calls_by_date",
428+
"Total cluster object store calls by date (Gauge for cluster billing)",
429+
)
430+
.namespace(METRICS_NAMESPACE),
431+
&["provider", "method", "date"],
432+
)
433+
.expect("metric can be created")
434+
});
435+
436+
pub static TOTAL_CLUSTER_FILES_SCANNED_IN_OBJECT_STORE_CALLS_BY_DATE: Lazy<IntGaugeVec> = Lazy::new(
437+
|| {
438+
IntGaugeVec::new(
439+
Opts::new(
440+
"total_cluster_files_scanned_in_object_store_calls_by_date",
441+
"Total cluster files scanned in object store calls by date (Gauge for cluster billing)",
442+
)
443+
.namespace(METRICS_NAMESPACE),
444+
&["provider", "method", "date"],
445+
)
446+
.expect("metric can be created")
447+
},
448+
);
449+
339450
pub static STORAGE_REQUEST_RESPONSE_TIME: Lazy<HistogramVec> = Lazy::new(|| {
340451
HistogramVec::new(
341452
HistogramOpts::new("storage_request_response_time", "Storage Request Latency")
@@ -433,6 +544,40 @@ fn custom_metrics(registry: &Registry) {
433544
TOTAL_FILES_SCANNED_IN_OBJECT_STORE_CALLS_BY_DATE.clone(),
434545
))
435546
.expect("metric can be registered");
547+
// Register cluster billing metrics
548+
registry
549+
.register(Box::new(TOTAL_CLUSTER_EVENTS_INGESTED_BY_DATE.clone()))
550+
.expect("metric can be registered");
551+
registry
552+
.register(Box::new(TOTAL_CLUSTER_EVENTS_INGESTED_SIZE_BY_DATE.clone()))
553+
.expect("metric can be registered");
554+
registry
555+
.register(Box::new(TOTAL_CLUSTER_PARQUETS_STORED_BY_DATE.clone()))
556+
.expect("metric can be registered");
557+
registry
558+
.register(Box::new(TOTAL_CLUSTER_PARQUETS_STORED_SIZE_BY_DATE.clone()))
559+
.expect("metric can be registered");
560+
registry
561+
.register(Box::new(TOTAL_CLUSTER_QUERY_CALLS_BY_DATE.clone()))
562+
.expect("metric can be registered");
563+
registry
564+
.register(Box::new(
565+
TOTAL_CLUSTER_FILES_SCANNED_IN_QUERY_BY_DATE.clone(),
566+
))
567+
.expect("metric can be registered");
568+
registry
569+
.register(Box::new(
570+
TOTAL_CLUSTER_BYTES_SCANNED_IN_QUERY_BY_DATE.clone(),
571+
))
572+
.expect("metric can be registered");
573+
registry
574+
.register(Box::new(TOTAL_CLUSTER_OBJECT_STORE_CALLS_BY_DATE.clone()))
575+
.expect("metric can be registered");
576+
registry
577+
.register(Box::new(
578+
TOTAL_CLUSTER_FILES_SCANNED_IN_OBJECT_STORE_CALLS_BY_DATE.clone(),
579+
))
580+
.expect("metric can be registered");
436581
registry
437582
.register(Box::new(STORAGE_REQUEST_RESPONSE_TIME.clone()))
438583
.expect("metric can be registered");

0 commit comments

Comments
 (0)