feat: automatically setup and inject prometheus configuration (#2912)

julienmancuso · indrajit96 · commit 11d7cd0a9ea2 · 2025-09-09T09:23:11.000-07:00
Signed-off-by: Julien Mancuso &lt;jmancuso@nvidia.com&gt;
Signed-off-by: Indrajit Bhosale &lt;iamindrajitb@gmail.com&gt;
diff --git a/components/backends/vllm/deploy/disagg_planner.yaml b/components/backends/vllm/deploy/disagg_planner.yaml
@@ -53,9 +53,6 @@ spec:
         mainContainer:
           image: nvcr.io/nvidia/ai-dynamo/vllm-runtime:0.4.1
           workingDir: /workspace/components/planner/src/dynamo/planner
-          ports:
-            - name: metrics
-              containerPort: 9085
           command:
             - /bin/sh
             - -c
@@ -66,7 +63,6 @@ spec:
               --backend=vllm
               --adjustment-interval=60
               --profile-results-dir=/workspace/profiling_results
-              --prometheus-port=9085
     Prometheus: # NOTE: this is set on Prometheus to ensure a service is created for the Prometheus component. This is a workaround and should be managed differently.
       dynamoNamespace: vllm-disagg-planner
       componentType: frontend
diff --git a/deploy/cloud/helm/platform/README.md b/deploy/cloud/helm/platform/README.md
@@ -84,9 +84,11 @@ The Dynamo Platform Helm chart deploys the complete Dynamo Cloud infrastructure
 | dynamo-operator.dynamo.istio.gateway | string | `nil` | Istio gateway name for routing |
 | dynamo-operator.dynamo.ingressHostSuffix | string | `""` | Host suffix for generated ingress hostnames |
 | dynamo-operator.dynamo.virtualServiceSupportsHTTPS | bool | `false` | Whether VirtualServices should support HTTPS routing |
+| dynamo-operator.dynamo.metrics.prometheusEndpoint | string | `""` | Endpoint that services can use to retrieve metrics. If set, dynamo operator will automatically inject the PROMETHEUS_ENDPOINT environment variable into services it manages. Users can override the value of the PROMETHEUS_ENDPOINT environment variable by modifying the corresponding deployment's environment variables |
 | grove.enabled | bool | `false` | Whether to enable Grove for multi-node inference coordination, if enabled, the Grove operator will be deployed cluster-wide |
 | kai-scheduler.enabled | bool | `false` | Whether to enable Kai Scheduler for intelligent resource allocation, if enabled, the Kai Scheduler operator will be deployed cluster-wide |
 | etcd.enabled | bool | `true` | Whether to enable etcd deployment, disable if you want to use an external etcd instance |
+| etcd.image.repository | string | `"bitnamilegacy/etcd"` | following bitnami announcement for brownout - https://github.com/bitnami/charts/tree/main/bitnami/etcd#%EF%B8%8F-important-notice-upcoming-changes-to-the-bitnami-catalog, we need to use the legacy repository until we migrate to the new "secure" repository |
 | nats.enabled | bool | `true` | Whether to enable NATS deployment, disable if you want to use an external NATS instance |
 
 ### NATS Configuration
diff --git a/deploy/cloud/helm/platform/components/operator/templates/deployment.yaml b/deploy/cloud/helm/platform/components/operator/templates/deployment.yaml
@@ -107,6 +107,9 @@ spec:
         {{- if .Values.modelExpressURL }}
           - --model-express-url={{ .Values.modelExpressURL }}
         {{- end }}
+        {{- if .Values.dynamo.metrics.prometheusEndpoint }}
+          - --prometheus-endpoint={{ .Values.dynamo.metrics.prometheusEndpoint }}
+        {{- end }}
         command:
         - /manager
         env:
diff --git a/deploy/cloud/helm/platform/components/operator/templates/prometheus.yaml b/deploy/cloud/helm/platform/components/operator/templates/prometheus.yaml
@@ -0,0 +1,81 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+{{- if .Capabilities.APIVersions.Has "monitoring.coreos.com/v1" }}
+apiVersion: monitoring.coreos.com/v1
+kind: PodMonitor
+metadata:
+  name: dynamo-frontend
+spec:
+  {{- if .Values.namespaceRestriction.enabled }}
+  namespaceSelector:
+    matchNames:
+    - {{ .Release.Namespace }}
+  {{- else }}
+  namespaceSelector:
+    any: true
+  {{- end }}
+  podMetricsEndpoints:
+  - interval: 30s
+    path: /metrics
+    port: http
+  selector:
+    matchLabels:
+      nvidia.com/dynamo-component-type: frontend
+      nvidia.com/metrics-enabled: "true"
+---
+apiVersion: monitoring.coreos.com/v1
+kind: PodMonitor
+metadata:
+  name: dynamo-worker
+spec:
+  {{- if .Values.namespaceRestriction.enabled }}
+  namespaceSelector:
+    matchNames:
+    - {{ .Release.Namespace }}
+  {{- else }}
+  namespaceSelector:
+    any: true
+  {{- end }}
+  podMetricsEndpoints:
+  - interval: 30s
+    path: /metrics
+    port: system
+  selector:
+    matchLabels:
+      nvidia.com/dynamo-component-type: worker
+      nvidia.com/metrics-enabled: "true"
+---
+apiVersion: monitoring.coreos.com/v1
+kind: PodMonitor
+metadata:
+  name: dynamo-planner
+spec:
+  {{- if .Values.namespaceRestriction.enabled }}
+  namespaceSelector:
+    matchNames:
+    - {{ .Release.Namespace }}
+  {{- else }}
+  namespaceSelector:
+    any: true
+  {{- end }}
+  podMetricsEndpoints:
+  - interval: 30s
+    path: /metrics
+    port: metrics
+  selector:
+    matchLabels:
+      nvidia.com/dynamo-component-type: planner
+      nvidia.com/metrics-enabled: "true"
+{{- end }}
diff --git a/deploy/cloud/helm/platform/components/operator/values.yaml b/deploy/cloud/helm/platform/components/operator/values.yaml
@@ -75,9 +75,6 @@ controllerManager:
     annotations: {}
 
 dynamo:
-  imageBuilder:
-    serviceAccount:
-      annotations: {}
   components:
     serviceAccount:
       annotations: {}
@@ -99,6 +96,9 @@ dynamo:
     existingSecretName: ''
     secure: true
 
+  metrics:
+    prometheusEndpoint: ""
+
 
 #imagePullSecrets: []
 kubernetesClusterDomain: cluster.local
diff --git a/deploy/cloud/helm/platform/values.yaml b/deploy/cloud/helm/platform/values.yaml
@@ -111,6 +111,11 @@ dynamo-operator:
     # -- Whether VirtualServices should support HTTPS routing
     virtualServiceSupportsHTTPS: false
 
+    # Metrics configuration
+    metrics:
+      # -- Endpoint that services can use to retrieve metrics. If set, dynamo operator will automatically inject the PROMETHEUS_ENDPOINT environment variable into services it manages. Users can override the value of the PROMETHEUS_ENDPOINT environment variable by modifying the corresponding deployment's environment variables
+      prometheusEndpoint: ""
+
 
 # Grove component - distributed inference orchestration
 grove:
@@ -130,8 +135,7 @@ etcd:
   enabled: true
 
   image:
-    # -- following bitnami announcement for brownout - https://github.com/bitnami/charts/tree/main/bitnami/etcd#%EF%B8%8F-important-notice-upcoming-changes-to-the-bitnami-catalog
-    # -- we need to use the legacy repository until we migrate to the new "secure" repository
+    # -- following bitnami announcement for brownout - https://github.com/bitnami/charts/tree/main/bitnami/etcd#%EF%B8%8F-important-notice-upcoming-changes-to-the-bitnami-catalog, we need to use the legacy repository until we migrate to the new "secure" repository
     repository: bitnamilegacy/etcd
     tag: 3.5.18-debian-12-r5
 
diff --git a/deploy/cloud/operator/Makefile b/deploy/cloud/operator/Makefile
@@ -57,7 +57,7 @@ ensure-yq:
 	fi
 
 .PHONY: manifests
-manifests: controller-gen ensure-yq generate-api-docs ## Generate WebhookConfiguration, ClusterRole and CustomResourceDefinition objects.
+manifests: controller-gen ensure-yq ## Generate WebhookConfiguration, ClusterRole and CustomResourceDefinition objects.
 	# Use a large maxDescLen to ensure all field comments are included as OpenAPI descriptions
 	$(CONTROLLER_GEN) rbac:roleName=manager-role crd:maxDescLen=100000 webhook paths="./..." output:crd:artifacts:config=config/crd/bases
 	echo "Removing name from mainContainer required fields"
diff --git a/deploy/cloud/operator/cmd/main.go b/deploy/cloud/operator/cmd/main.go
@@ -132,6 +132,7 @@ func main() {
 	var ingressHostSuffix string
 	var groveTerminationDelay time.Duration
 	var modelExpressURL string
+	var prometheusEndpoint string
 	flag.StringVar(&metricsAddr, "metrics-bind-address", ":8080", "The address the metric endpoint binds to.")
 	flag.StringVar(&probeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.")
 	flag.BoolVar(&enableLeaderElection, "leader-elect", false,
@@ -161,6 +162,8 @@ func main() {
 		"The termination delay for Grove PodGangSets")
 	flag.StringVar(&modelExpressURL, "model-express-url", "",
 		"URL of the Model Express server to inject into all pods")
+	flag.StringVar(&prometheusEndpoint, "prometheus-endpoint", "",
+		"URL of the Prometheus endpoint to use for metrics")
 	opts := zap.Options{
 		Development: true,
 	}
@@ -196,7 +199,8 @@ func main() {
 			IngressControllerTLSSecret: ingressControllerTLSSecretName,
 			IngressHostSuffix:          ingressHostSuffix,
 		},
-		ModelExpressURL: modelExpressURL,
+		ModelExpressURL:    modelExpressURL,
+		PrometheusEndpoint: prometheusEndpoint,
 	}
 
 	mainCtx := ctrl.SetupSignalHandler()
diff --git a/deploy/cloud/operator/internal/consts/consts.go b/deploy/cloud/operator/internal/consts/consts.go
@@ -16,6 +16,9 @@ const (
 	DynamoServicePortName   = "http"
 	DynamoContainerPortName = "http"
 
+	DynamoPlannerMetricsPort = 9085
+	DynamoMetricsPortName    = "metrics"
+
 	DynamoSystemPort     = 9090
 	DynamoSystemPortName = "system"
 
diff --git a/deploy/cloud/operator/internal/controller_common/predicate.go b/deploy/cloud/operator/internal/controller_common/predicate.go
@@ -58,6 +58,8 @@ type Config struct {
 	IngressConfig       IngressConfig
 	// ModelExpressURL is the URL of the Model Express server to inject into all pods
 	ModelExpressURL string
+	// PrometheusEndpoint is the URL of the Prometheus endpoint to use for metrics
+	PrometheusEndpoint string
 }
 
 type IngressConfig struct {
diff --git a/deploy/cloud/operator/internal/dynamo/component_planner.go b/deploy/cloud/operator/internal/dynamo/component_planner.go
@@ -6,6 +6,8 @@
 package dynamo
 
 import (
+	"fmt"
+
 	commonconsts "github.com/ai-dynamo/dynamo/deploy/cloud/operator/internal/consts"
 	corev1 "k8s.io/api/core/v1"
 )
@@ -21,6 +23,19 @@ func NewPlannerDefaults() *PlannerDefaults {
 
 func (p *PlannerDefaults) GetBaseContainer(context ComponentContext) (corev1.Container, error) {
 	container := p.getCommonContainer(context)
+	container.Ports = []corev1.ContainerPort{
+		{
+			Protocol:      corev1.ProtocolTCP,
+			Name:          commonconsts.DynamoMetricsPortName,
+			ContainerPort: int32(commonconsts.DynamoPlannerMetricsPort),
+		},
+	}
+	container.Env = append(container.Env, []corev1.EnvVar{
+		{
+			Name:  "PROMETHEUS_PORT",
+			Value: fmt.Sprintf("%d", commonconsts.DynamoPlannerMetricsPort),
+		},
+	}...)
 	return container, nil
 }
 
diff --git a/deploy/cloud/operator/internal/dynamo/component_planner_test.go b/deploy/cloud/operator/internal/dynamo/component_planner_test.go
@@ -6,8 +6,10 @@
 package dynamo
 
 import (
+	"fmt"
 	"testing"
 
+	commonconsts "github.com/ai-dynamo/dynamo/deploy/cloud/operator/internal/consts"
 	"github.com/google/go-cmp/cmp"
 	corev1 "k8s.io/api/core/v1"
 )
@@ -46,10 +48,14 @@ func TestPlannerDefaults_GetBaseContainer(t *testing.T) {
 					"/bin/sh",
 					"-c",
 				},
+				Ports: []corev1.ContainerPort{
+					{Name: commonconsts.DynamoMetricsPortName, ContainerPort: commonconsts.DynamoPlannerMetricsPort, Protocol: corev1.ProtocolTCP},
+				},
 				Env: []corev1.EnvVar{
 					{Name: "DYN_NAMESPACE", Value: "dynamo-namespace"},
 					{Name: "DYN_PARENT_DGD_K8S_NAME", Value: "name"},
 					{Name: "DYN_PARENT_DGD_K8S_NAMESPACE", Value: "namespace"},
+					{Name: "PROMETHEUS_PORT", Value: fmt.Sprintf("%d", commonconsts.DynamoPlannerMetricsPort)},
 				},
 			},
 		},
diff --git a/deploy/cloud/operator/internal/dynamo/graph.go b/deploy/cloud/operator/internal/dynamo/graph.go
@@ -660,26 +660,35 @@ func isWorkerComponent(componentType string) bool {
 
 // addStandardEnvVars adds the standard environment variables that are common to both Grove and Controller
 func addStandardEnvVars(container *corev1.Container, controllerConfig controller_common.Config) {
+	standardEnvVars := []corev1.EnvVar{}
 	if controllerConfig.NatsAddress != "" {
-		container.Env = append(container.Env, corev1.EnvVar{
+		standardEnvVars = append(standardEnvVars, corev1.EnvVar{
 			Name:  "NATS_SERVER",
 			Value: controllerConfig.NatsAddress,
 		})
 	}
 
 	if controllerConfig.EtcdAddress != "" {
-		container.Env = append(container.Env, corev1.EnvVar{
+		standardEnvVars = append(standardEnvVars, corev1.EnvVar{
 			Name:  "ETCD_ENDPOINTS",
 			Value: controllerConfig.EtcdAddress,
 		})
 	}
 
 	if controllerConfig.ModelExpressURL != "" {
-		container.Env = append(container.Env, corev1.EnvVar{
+		standardEnvVars = append(standardEnvVars, corev1.EnvVar{
 			Name:  "MODEL_EXPRESS_URL",
 			Value: controllerConfig.ModelExpressURL,
 		})
 	}
+	if controllerConfig.PrometheusEndpoint != "" {
+		standardEnvVars = append(standardEnvVars, corev1.EnvVar{
+			Name:  "PROMETHEUS_ENDPOINT",
+			Value: controllerConfig.PrometheusEndpoint,
+		})
+	}
+	// merge the env vars to allow users to override the standard env vars
+	container.Env = MergeEnvs(standardEnvVars, container.Env)
 }
 
 // GenerateBasePodSpec creates a basic PodSpec with common logic shared between controller and grove
diff --git a/deploy/cloud/operator/internal/dynamo/graph_test.go b/deploy/cloud/operator/internal/dynamo/graph_test.go
diff --git a/docs/guides/dynamo_deploy/installation_guide.md b/docs/guides/dynamo_deploy/installation_guide.md
diff --git a/docs/guides/dynamo_deploy/metrics.md b/docs/guides/dynamo_deploy/metrics.md

Original file line number	Diff line number	Diff line change
`@@ -58,6 +58,8 @@ type Config struct {`
`58`	`58`	`IngressConfig IngressConfig`
`59`	`59`	`// ModelExpressURL is the URL of the Model Express server to inject into all pods`
`60`	`60`	`ModelExpressURL string`
	`61`	`+ // PrometheusEndpoint is the URL of the Prometheus endpoint to use for metrics`
	`62`	`+ PrometheusEndpoint string`
`61`	`63`	`}`
`62`	`64`
`63`	`65`	`type IngressConfig struct {`