fix: sanitize metric prefix and improve test reliability

Author: ryan-lempka

Cargo.lock

@@ -154,7 +154,9 @@ async def async_main():
 
     # Configure Dynamo frontend HTTP service metrics prefix
     if flags.metrics_prefix is not None:
-        os.environ["DYN_METRICS_PREFIX"] = flags.metrics_prefix
+        prefix = flags.metrics_prefix.strip()
+        if prefix:
+            os.environ["DYN_METRICS_PREFIX"] = flags.metrics_prefix
 
     runtime = DistributedRuntime(asyncio.get_running_loop(), is_static)
 

components/frontend/src/dynamo/frontend/main.py

@@ -143,6 +143,7 @@ proptest = "1.5.0"
 reqwest = { workspace = true }
 rstest = "0.18.2"
 rstest_reuse = "0.7.0"
+serial_test = "3"
 tempfile = "3.17.1"
 insta = { version = "1.41", features = [
   "glob",

lib/llm/Cargo.toml

@@ -30,6 +30,25 @@ pub const REQUEST_TYPE_STREAM: &str = "stream";
 /// Partial value for the `type` label in the request counter for unary requests
 pub const REQUEST_TYPE_UNARY: &str = "unary";
 
+fn sanitize_prometheus_prefix(raw: &str) -> String {
+    // Prometheus metric name pattern: [a-zA-Z_:][a-zA-Z0-9_:]*
+    let mut s: String = raw
+        .chars()
+        .map(|c| if c.is_ascii_alphanumeric() || c == '_' || c == ':' { c } else { '_' })
+        .collect();
+
+    if s.is_empty() {
+        return FRONTEND_METRIC_PREFIX.to_string();
+    }
+
+    let first = s.as_bytes()[0];
+    let valid_first = first.is_ascii_alphabetic() || first == b'_' || first == b':';
+    if !valid_first {
+        s.insert(0, '_');
+    }
+    s
+}
+
 pub struct Metrics {
     request_counter: IntCounterVec,
     inflight_gauge: IntGaugeVec,
@@ -117,8 +136,17 @@ impl Metrics {
     /// - `{prefix}_time_to_first_token_seconds` - HistogramVec for time to first token in seconds
     /// - `{prefix}_inter_token_latency_seconds` - HistogramVec for inter-token latency in seconds
     pub fn new() -> Self {
-        let prefix = std::env::var(METRICS_PREFIX_ENV)
+        let raw_prefix = std::env::var(METRICS_PREFIX_ENV)
             .unwrap_or_else(|_| FRONTEND_METRIC_PREFIX.to_string());
+        let prefix = sanitize_prometheus_prefix(&raw_prefix);
+        if prefix != raw_prefix {
+            tracing::warn!(
+                raw=%raw_prefix,
+                sanitized=%prefix,
+                env=%METRICS_PREFIX_ENV,
+                "Sanitized HTTP metrics prefix"
+            );
+        }
         let frontend_metric_name = |suffix: &str| format!("{}_{}", &prefix, suffix);
 
         let request_counter = IntCounterVec::new(

lib/llm/src/http/service/metrics.rs

@@ -1,36 +1,37 @@
 // SPDX-FileCopyrightText: Copyright (c) 2024-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 // SPDX-License-Identifier: Apache-2.0
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
 
 use dynamo_llm::http::service::metrics::{self, Endpoint};
 use dynamo_llm::http::service::service_v2::HttpService;
 use dynamo_runtime::CancellationToken;
+use serial_test::serial;
 use std::{env, time::Duration};
 
 #[tokio::test]
+#[serial]
 async fn metrics_prefix_default_then_env_override() {
     // Case 1: default prefix
     env::remove_var(metrics::METRICS_PREFIX_ENV);
     let svc1 = HttpService::builder().port(9101).build().unwrap();
     let token1 = CancellationToken::new();
     let _h1 = svc1.spawn(token1.clone()).await;
-    tokio::time::sleep(Duration::from_millis(100)).await;
+    wait_for_metrics_ready(9101).await;
 
     // Populate labeled metrics
     let s1 = svc1.state_clone();
-    { let _g = s1.metrics_clone().create_inflight_guard("test-model", Endpoint::ChatCompletions, false); }
-    let body1 = reqwest::get("http://localhost:9101/metrics").await.unwrap().text().await.unwrap();
+    {
+        let _g = s1.metrics_clone().create_inflight_guard(
+            "test-model",
+            Endpoint::ChatCompletions,
+            false,
+        );
+    }
+    let body1 = reqwest::get("http://localhost:9101/metrics")
+        .await
+        .unwrap()
+        .text()
+        .await
+        .unwrap();
     assert!(body1.contains("dynamo_frontend_requests_total"));
     token1.cancel();
 
@@ -39,12 +40,65 @@ async fn metrics_prefix_default_then_env_override() {
     let svc2 = HttpService::builder().port(9102).build().unwrap();
     let token2 = CancellationToken::new();
     let _h2 = svc2.spawn(token2.clone()).await;
-    tokio::time::sleep(Duration::from_millis(100)).await;
+    wait_for_metrics_ready(9102).await;
 
     // Populate labeled metrics
     let s2 = svc2.state_clone();
-    { let _g = s2.metrics_clone().create_inflight_guard("test-model", Endpoint::ChatCompletions, true); }
-    let body2 = reqwest::get("http://localhost:9102/metrics").await.unwrap().text().await.unwrap();
-    assert!(body2.contains("custom_prefix"));
+    {
+        let _g =
+            s2.metrics_clone()
+                .create_inflight_guard("test-model", Endpoint::ChatCompletions, true);
+    }
+    // Single fetch and assert
+    let body2 = reqwest::get("http://localhost:9102/metrics")
+        .await
+        .unwrap()
+        .text()
+        .await
+        .unwrap();
+    assert!(body2.contains("custom_prefix_requests_total"));
+    assert!(!body2.contains("dynamo_frontend_requests_total"));
     token2.cancel();
+
+    // Case 3: invalid env prefix is sanitized
+    env::set_var(metrics::METRICS_PREFIX_ENV, "nv-llm/http service");
+    let svc3 = HttpService::builder().port(9103).build().unwrap();
+    let token3 = CancellationToken::new();
+    let _h3 = svc3.spawn(token3.clone()).await;
+    wait_for_metrics_ready(9103).await;
+
+    let s3 = svc3.state_clone();
+    {
+        let _g =
+            s3.metrics_clone()
+                .create_inflight_guard("test-model", Endpoint::ChatCompletions, true);
+    }
+    let body3 = reqwest::get("http://localhost:9103/metrics")
+        .await
+        .unwrap()
+        .text()
+        .await
+        .unwrap();
+    assert!(body3.contains("nv_llm_http_service_requests_total"));
+    assert!(!body3.contains("dynamo_frontend_requests_total"));
+    token3.cancel();
+
+    // Cleanup env to avoid leaking state
+    env::remove_var(metrics::METRICS_PREFIX_ENV);
+}
+
+// Poll /metrics until ready or timeout
+async fn wait_for_metrics_ready(port: u16) {
+    let url = format!("http://localhost:{}/metrics", port);
+    let start = tokio::time::Instant::now();
+    let timeout = Duration::from_secs(5);
+    loop {
+        if start.elapsed() > timeout {
+            panic!("Timed out waiting for metrics endpoint at {}", url);
+        }
+        match reqwest::get(&url).await {
+            Ok(resp) if resp.status().is_success() => break,
+            _ => tokio::time::sleep(Duration::from_millis(50)).await,
+        }
+    }
 }