feat: deploy SLA profiler to k8s (#2030)

Co-authored-by: hongkuan <hongkuanz@nvidia.com>
Co-authored-by: mohammedabdulwahhab <furkhan324@berkeley.edu>
Co-authored-by: Hongkuan Zhou <tedzhouhk@gmail.com>

Author: 3 people

benchmarks/profiler/deploy/profile_sla_binding.yaml

@@ -0,0 +1,15 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: profile-sla-binding
+  namespace: ${NAMESPACE}
+subjects:
+- kind: ServiceAccount
+  name: profile-sla-sa
+  namespace: ${NAMESPACE}
+roleRef:
+  kind: Role
+  name: profile-sla-role
+  apiGroup: rbac.authorization.k8s.io

benchmarks/profiler/deploy/profile_sla_job.yaml

@@ -0,0 +1,48 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: profile-sla
+  namespace: ${NAMESPACE}
+spec:
+  template:
+    spec:
+      serviceAccountName: profile-sla-sa
+      containers:
+      - name: profile-sla
+        image: ${DOCKER_IMAGE}
+        resources:
+          requests:
+            cpu: "1"
+            memory: "2Gi"
+          limits:
+            cpu: "2"
+            memory: "4Gi"
+        env:
+          - name: HUGGING_FACE_HUB_TOKEN
+            valueFrom:
+              secretKeyRef:
+                name: hf-token-secret
+                key: HF_TOKEN
+          - name: NATS_SERVER
+            value: nats://${NAMESPACE}-nats:4222
+          - name: ETCD_ENDPOINTS
+            value: ${NAMESPACE}-etcd:2379
+        command: ["python", "/workspace/benchmarks/profiler/profile_sla.py"]
+        args:
+          - --config
+          - ${DGD_CONFIG_FILE}
+          - --output-dir
+          - /workspace/profiling_results
+          - --namespace
+          - ${NAMESPACE}
+        volumeMounts:
+          - name: output-volume
+            mountPath: /workspace/profiling_results
+      restartPolicy: Never
+      volumes:
+        - name: output-volume
+          persistentVolumeClaim:
+            claimName: profiling-pvc
+  backoffLimit: 0

benchmarks/profiler/deploy/profile_sla_rbac.yaml

@@ -0,0 +1,19 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: profile-sla-role
+  namespace: ${NAMESPACE}
+rules:
+  # DynamoGraphDeployment custom resources - needed for create/get/delete operations
+  - apiGroups: ["nvidia.com"]
+    resources: ["dynamographdeployments"]
+    verbs: ["get", "create", "delete"]
+  # Pods - needed for listing pods by label selector and getting logs
+  - apiGroups: [""]
+    resources: ["pods"]
+    verbs: ["list"]
+  - apiGroups: [""]
+    resources: ["pods/log"]
+    verbs: ["get"]

benchmarks/profiler/deploy/profile_sla_sa.yaml

@@ -0,0 +1,9 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: profile-sla-sa
+  namespace: ${NAMESPACE}
+imagePullSecrets:
+  - name: nvcr-imagepullsecret

benchmarks/profiler/deploy/profiling_pvc.yaml

@@ -0,0 +1,13 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: profiling-pvc
+  namespace: ${NAMESPACE}
+spec:
+  accessModes:
+    - ReadWriteOnce
+  resources:
+    requests:
+      storage: 50Gi