feat: DIS-678 kvbm modularity: standalone metrics endpoint (#3433)

Signed-off-by: richardhuo-nv <rihuo@nvidia.com>

Author: richardhuo-nv

deploy/metrics/grafana_dashboards/grafana-kvbm-dashboard.json

@@ -118,8 +118,8 @@
       "targets": [
         {
           "disableTextWrap": false,
-          "editorMode": "builder",
-          "expr": "dynamo_component_matched_tokens{dynamo_namespace=\"kvbm_connector_leader\"}",
+          "editorMode": "code",
+          "expr": "kvbm_matched_tokens",
           "fullMetaSearch": false,
           "includeNullMetadata": true,
           "legendFormat": "__auto",
@@ -227,8 +227,8 @@
       "targets": [
         {
           "disableTextWrap": false,
-          "editorMode": "builder",
-          "expr": "dynamo_component_offload_requests{dynamo_namespace=\"kvbm_connector_leader\"}",
+          "editorMode": "code",
+          "expr": "kvbm_offload_requests",
           "fullMetaSearch": false,
           "includeNullMetadata": true,
           "legendFormat": "__auto",
@@ -323,8 +323,8 @@
       "targets": [
         {
           "disableTextWrap": false,
-          "editorMode": "builder",
-          "expr": "dynamo_component_offload_blocks_d2h{dynamo_namespace=\"kvbm_connector_leader\"}",
+          "editorMode": "code",
+          "expr": "kvbm_offload_blocks_d2h",
           "fullMetaSearch": false,
           "includeNullMetadata": true,
           "legendFormat": "__auto",
@@ -336,102 +336,6 @@
       "title": "Offload Blocks - Device to Host",
       "type": "timeseries"
     },
-    {
-      "datasource": {
-        "type": "prometheus",
-        "uid": "P1809F7CD0C75ACF3"
-      },
-      "fieldConfig": {
-        "defaults": {
-          "color": {
-            "mode": "palette-classic"
-          },
-          "custom": {
-            "axisBorderShow": false,
-            "axisCenteredZero": false,
-            "axisColorMode": "text",
-            "axisLabel": "",
-            "axisPlacement": "auto",
-            "barAlignment": 0,
-            "barWidthFactor": 0.6,
-            "drawStyle": "line",
-            "fillOpacity": 0,
-            "gradientMode": "none",
-            "hideFrom": {
-              "legend": false,
-              "tooltip": false,
-              "viz": false
-            },
-            "insertNulls": false,
-            "lineInterpolation": "linear",
-            "lineWidth": 1,
-            "pointSize": 5,
-            "scaleDistribution": {
-              "type": "linear"
-            },
-            "showPoints": "auto",
-            "spanNulls": false,
-            "stacking": {
-              "group": "A",
-              "mode": "none"
-            },
-            "thresholdsStyle": {
-              "mode": "off"
-            }
-          },
-          "mappings": [],
-          "thresholds": {
-            "mode": "absolute",
-            "steps": [
-              {
-                "color": "green"
-              },
-              {
-                "color": "red",
-                "value": 80
-              }
-            ]
-          }
-        },
-        "overrides": []
-      },
-      "gridPos": {
-        "h": 8,
-        "w": 12,
-        "x": 0,
-        "y": 18
-      },
-      "id": 1,
-      "options": {
-        "legend": {
-          "calcs": [],
-          "displayMode": "list",
-          "placement": "bottom",
-          "showLegend": true
-        },
-        "tooltip": {
-          "hideZeros": false,
-          "mode": "single",
-          "sort": "none"
-        }
-      },
-      "pluginVersion": "12.0.1",
-      "targets": [
-        {
-          "disableTextWrap": false,
-          "editorMode": "builder",
-          "expr": "dynamo_component_save_kv_layer_requests{dynamo_namespace=\"kvbm_connector_worker\"}",
-          "fullMetaSearch": false,
-          "includeNullMetadata": true,
-          "legendFormat": "__auto",
-          "range": true,
-          "refId": "A",
-          "useBackend": false
-        }
-      ],
-      "title": "Save KV Layer Requests",
-      "type": "timeseries"
-    },
     {
       "collapsed": false,
       "gridPos": {
@@ -528,8 +432,8 @@
       "targets": [
         {
           "disableTextWrap": false,
-          "editorMode": "builder",
-          "expr": "dynamo_component_onboard_requests{dynamo_namespace=\"kvbm_connector_leader\"}",
+          "editorMode": "code",
+          "expr": "kvbm_onboard_requests",
           "fullMetaSearch": false,
           "includeNullMetadata": true,
           "legendFormat": "__auto",
@@ -624,8 +528,8 @@
       "targets": [
         {
           "disableTextWrap": false,
-          "editorMode": "builder",
-          "expr": "dynamo_component_onboard_blocks_h2d{dynamo_namespace=\"kvbm_connector_leader\"}",
+          "editorMode": "code",
+          "expr": "kvbm_onboard_blocks_h2d",
           "fullMetaSearch": false,
           "includeNullMetadata": true,
           "legendFormat": "__auto",
@@ -720,8 +624,8 @@
       "targets": [
         {
           "disableTextWrap": false,
-          "editorMode": "builder",
-          "expr": "dynamo_component_onboard_blocks_d2d{dynamo_namespace=\"kvbm_connector_leader\"}",
+          "editorMode": "code",
+          "expr": "kvbm_onboard_blocks_d2d",
           "fullMetaSearch": false,
           "includeNullMetadata": true,
           "legendFormat": "__auto",
@@ -750,4 +654,4 @@
   "title": "KVBM Dashboard",
   "uid": "3f679257-70a5-402c-92b4-05382337b548",
   "version": 7
-}
+}

deploy/metrics/prometheus.yml

@@ -59,16 +59,10 @@ scrape_configs:
       - targets: ['host.docker.internal:9091']  # metrics aggregation service on host
 
   # KVBM leader related metrics
-  - job_name: 'kvbm-leader-metrics'
+  - job_name: 'kvbm-metrics'
     scrape_interval: 2s
     static_configs:
-      - targets: ['host.docker.internal:6882']
-
-  # KVBM worker related metrics
-  - job_name: 'kvbm-worker-metrics'
-    scrape_interval: 2s
-    static_configs:
-      - targets: ['host.docker.internal:6881']
+      - targets: ['host.docker.internal:6880']
 
   # Uncomment to see its own Prometheus metrics
   # - job_name: 'prometheus'

docs/guides/run_kvbm_in_trtllm.md

@@ -109,18 +109,51 @@ Follow below steps to enable metrics collection and view via Grafana dashboard:
 # Start the basic services (etcd & natsd), along with Prometheus and Grafana
 docker compose -f deploy/docker-compose.yml --profile metrics up -d
 
-# set env var DYN_SYSTEM_ENABLED to true, DYN_SYSTEM_PORT to 6880, DYN_KVBM_SLEEP to 5, when launch via dynamo
-# NOTE: Make sure port 6881 (for KVBM worker metrics) and port 6882 (for KVBM leader metrics) are available.
-# NOTE: DYN_KVBM_SLEEP is needed to avoid metrics port conflict between KVBM leader and worker
-DYN_SYSTEM_ENABLED=true DYN_SYSTEM_PORT=6880 DYN_KVBM_SLEEP=5 \
+# set env var DYN_KVBM_METRICS to true, when launch via dynamo
+# Optionally set DYN_KVBM_METRICS_PORT to choose the /metrics port (default: 6880).
+DYN_KVBM_METRICS=true \
 python3 -m dynamo.trtllm \
   --model-path deepseek-ai/DeepSeek-R1-Distill-Llama-8B \
   --served-model-name deepseek-ai/DeepSeek-R1-Distill-Llama-8B \
   --extra-engine-args /tmp/kvbm_llm_api_config.yaml &
 
 # optional if firewall blocks KVBM metrics ports to send prometheus metrics
-sudo ufw allow 6881/tcp
-sudo ufw allow 6882/tcp
+sudo ufw allow 6880/tcp
 ```
 
 View grafana metrics via http://localhost:3001 (default login: dynamo/dynamo) and look for KVBM Dashboard
+
+## Benchmark KVBM
+
+Once the model is loaded ready, follow below steps to use LMBenchmark to benchmark KVBM performance:
+```bash
+git clone https://github.com/LMCache/LMBenchmark.git
+
+# show case of running the synthetic multi-turn chat dataset.
+# we are passing model, endpoint, output file prefix and qps to the sh script.
+cd LMBenchmark/synthetic-multi-round-qa
+./long_input_short_output_run.sh \
+    "deepseek-ai/DeepSeek-R1-Distill-Llama-8B" \
+    "http://localhost:8000" \
+    "benchmark_kvbm" \
+    1
+
+# Average TTFT and other perf numbers would be in the output from above cmd
+```
+More details about how to use LMBenchmark could be found [here](https://github.com/LMCache/LMBenchmark).
+
+`NOTE`: if metrics are enabled as mentioned in the above section, you can observe KV offloading, and KV onboarding in the grafana dashboard.
+
+To compare, you can remove the `kv_connector_config` section from the LLM API config and run `trtllm-serve` with the updated config as the baseline.
+```bash
+cat > "/tmp/llm_api_config.yaml" <<EOF
+backend: pytorch
+cuda_graph_config: null
+kv_cache_config:
+  enable_partial_reuse: false
+  free_gpu_memory_fraction: 0.80
+EOF
+
+# run trtllm-serve for the baseline for comparison
+trtllm-serve deepseek-ai/DeepSeek-R1-Distill-Llama-8B --host localhost --port 8000 --backend pytorch --extra_llm_api_options /tmp/llm_api_config.yaml &
+```

docs/guides/run_kvbm_in_vllm.md

@@ -77,23 +77,22 @@ Follow below steps to enable metrics collection and view via Grafana dashboard:
 # Start the basic services (etcd & natsd), along with Prometheus and Grafana
 docker compose -f deploy/docker-compose.yml --profile metrics up -d
 
-# set env var DYN_SYSTEM_ENABLED to true, DYN_SYSTEM_PORT to 6880, DYN_KVBM_SLEEP to 5, when launch via dynamo
-# NOTE: Make sure port 6881 (for KVBM worker metrics) and port 6882 (for KVBM leader metrics) are available.
-DYN_SYSTEM_ENABLED=true DYN_SYSTEM_PORT=6880 \
+# set env var DYN_KVBM_METRICS to true, when launch via dynamo
+# Optionally set DYN_KVBM_METRICS_PORT to choose the /metrics port (default: 6880).
+DYN_KVBM_METRICS=true \
 python -m dynamo.vllm \
     --model deepseek-ai/DeepSeek-R1-Distill-Llama-8B \
     --connector kvbm &
 
 # optional if firewall blocks KVBM metrics ports to send prometheus metrics
-sudo ufw allow 6881/tcp
-sudo ufw allow 6882/tcp
+sudo ufw allow 6880/tcp
 ```
 
 View grafana metrics via http://localhost:3001 (default login: dynamo/dynamo) and look for KVBM Dashboard
 
 ## Benchmark KVBM
 
-Once vllm serve is ready, follow below steps to use LMBenchmark to benchmark KVBM performance:
+Once the model is loaded ready, follow below steps to use LMBenchmark to benchmark KVBM performance:
 ```bash
 git clone https://github.com/LMCache/LMBenchmark.git
 

lib/bindings/python/rust/llm/block_manager/vllm/connector/leader.rs

@@ -5,7 +5,7 @@ pub mod recorder;
 pub mod slot;
 
 use super::*;
-use dynamo_llm::block_manager::metrics_kvbm::KvbmMetrics;
+use dynamo_llm::block_manager::metrics_kvbm::{KvbmMetrics, KvbmMetricsRegistry};
 use dynamo_runtime::DistributedRuntime;
 use slot::{ConnectorSlotManager, SlotError, SlotManager, SlotState};
 
@@ -15,7 +15,6 @@ use crate::llm::block_manager::{
     VllmBlockManager, distributed::KvbmLeader as PyKvbmLeader, vllm::KvbmRequest,
     vllm::connector::leader::slot::VllmConnectorSlot,
 };
-use dynamo_runtime::metrics::prometheus_names::kvbm_connector;
 
 use dynamo_llm::block_manager::{
     BasicMetadata, DiskStorage, ImmutableBlock, PinnedStorage,
@@ -103,11 +102,11 @@ impl KvConnectorLeader {
         let drt = drt.inner().clone();
         let handle: Handle = drt.runtime().primary();
 
-        let ns = drt
-            .namespace(kvbm_connector::KVBM_CONNECTOR_LEADER)
-            .unwrap();
-
-        let kvbm_metrics = KvbmMetrics::new(&ns);
+        let kvbm_metrics = KvbmMetrics::new(
+            &KvbmMetricsRegistry::default(),
+            kvbm_metrics_endpoint_enabled(),
+            parse_kvbm_metrics_port(),
+        );
         let kvbm_metrics_clone = kvbm_metrics.clone();
 
         let slot_manager_cell = Arc::new(OnceLock::new());
@@ -615,3 +614,30 @@ impl PyKvConnectorLeader {
             .map_err(to_pyerr)
     }
 }
+
+pub fn kvbm_metrics_endpoint_enabled() -> bool {
+    std::env::var("DYN_KVBM_METRICS")
+        .map(|v| v == "1" || v.eq_ignore_ascii_case("true"))
+        .unwrap_or(false)
+}
+
+pub fn parse_kvbm_metrics_port() -> u16 {
+    match std::env::var("DYN_KVBM_METRICS_PORT") {
+        Ok(val) => match val.trim().parse::<u16>() {
+            Ok(port) => port,
+            Err(_) => {
+                tracing::warn!(
+                    "[kvbm] Invalid DYN_KVBM_METRICS_PORT='{}', falling back to 6880",
+                    val
+                );
+                6880
+            }
+        },
+        Err(_) => {
+            tracing::warn!(
+                "DYN_KVBM_METRICS_PORT not present or couldn’t be interpreted, falling back to 6880"
+            );
+            6880
+        }
+    }
+}

lib/bindings/python/rust/llm/block_manager/vllm/connector/leader/recorder.rs

@@ -100,11 +100,11 @@ impl KvConnectorLeaderRecorder {
         let drt = drt.inner().clone();
         let handle: Handle = drt.runtime().primary();
 
-        let ns = drt
-            .namespace(kvbm_connector::KVBM_CONNECTOR_LEADER)
-            .unwrap();
-
-        let kvbm_metrics = KvbmMetrics::new(&ns);
+        let kvbm_metrics = KvbmMetrics::new(
+            &KvbmMetricsRegistry::default(),
+            kvbm_metrics_endpoint_enabled(),
+            parse_kvbm_metrics_port(),
+        );
         let kvbm_metrics_clone = kvbm_metrics.clone();
 
         let token = CancellationToken::new();

lib/bindings/python/rust/llm/block_manager/vllm/connector/trtllm_leader.rs

@@ -8,10 +8,12 @@ use crate::llm::block_manager::BlockManagerBuilder;
 use crate::llm::block_manager::vllm::connector::leader::slot::{
     ConnectorSlotManager, SlotManager, SlotState,
 };
+use crate::llm::block_manager::vllm::connector::leader::{
+    kvbm_metrics_endpoint_enabled, parse_kvbm_metrics_port,
+};
 use crate::llm::block_manager::{distributed::KvbmLeader as PyKvbmLeader, vllm::KvbmRequest};
 use anyhow;
-use dynamo_llm::block_manager::metrics_kvbm::KvbmMetrics;
-use dynamo_runtime::metrics::prometheus_names::kvbm_connector;
+use dynamo_llm::block_manager::metrics_kvbm::{KvbmMetrics, KvbmMetricsRegistry};
 use std::collections::HashSet;
 use std::sync::{Arc, OnceLock};
 use tokio::runtime::Handle;
@@ -76,11 +78,12 @@ impl KvConnectorLeader {
         let drt = drt.inner().clone();
         let handle: Handle = drt.runtime().primary();
 
-        let ns = drt
-            .namespace(kvbm_connector::KVBM_CONNECTOR_LEADER)
-            .unwrap();
+        let kvbm_metrics = KvbmMetrics::new(
+            &KvbmMetricsRegistry::default(),
+            kvbm_metrics_endpoint_enabled(),
+            parse_kvbm_metrics_port(),
+        );
 
-        let kvbm_metrics = KvbmMetrics::new(&ns);
         let kvbm_metrics_clone = kvbm_metrics.clone();
 
         let slot_manager_cell = Arc::new(OnceLock::new());