Merge remote-tracking branch 'origin/master' into e2e/ray-service-runtime-env-update

ryankert01 · ryankert01 · commit 3145953baf9b · 2025-11-29T17:31:54.000+08:00
diff --git a/README.md b/README.md
@@ -22,10 +22,10 @@ common workflows when deploying Ray on Kubernetes. If you aren’t familiar with
 plugin simplifies running Ray on Kubernetes. See [kubectl-plugin](https://docs.ray.io/en/latest/cluster/kubernetes/user-guides/kubectl-plugin.html#kubectl-plugin) for more details.
 
 * **KubeRay APIServer** (Alpha): It provides a layer of simplified configuration for KubeRay resources. The KubeRay API server is used internally
-by some organizations to back user interfaces for KubeRay resource management.
+by some organizations to back user interfaces for KubeRay resource management. See [KubeRay APIServer V2](https://github.com/ray-project/kuberay/blob/master/apiserversdk/README.md) for more details.
 
 * **KubeRay Dashboard** (Experimental): Starting from KubeRay v1.4.0, we have introduced a new dashboard that enables users to view and manage KubeRay resources.
-While it is not yet production-ready, we welcome your feedback.
+While it is not yet production-ready, we welcome your feedback. See [KubeRay Dashboard](https://docs.ray.io/en/master/cluster/kubernetes/user-guides/kuberay-dashboard.html) for more details.
 
 ## Documentation
 
@@ -90,7 +90,7 @@ Please read our [CONTRIBUTING](CONTRIBUTING.md) guide before making a pull reque
 
 Join [Ray's Slack workspace](https://docs.google.com/forms/d/e/1FAIpQLSfAcoiLCHOguOm8e7Jnn-JJdZaCxPGjgVCvFijHB5PLaQLeig/viewform), and search the following public channels:
 
-* `#kuberay-questions`: This channel aims to help KubeRay users with their questions. The messages will be closely monitored by the Ray and KubeRay maintainers.
+* `#kuberay`: This channel aims to help KubeRay users with their questions. The messages will be closely monitored by the Ray and KubeRay maintainers.
 
 KubeRay contributors are welcome to join the bi-weekly KubeRay community meetings.
 
diff --git a/dashboard/Dockerfile b/dashboard/Dockerfile
@@ -1,4 +1,4 @@
-FROM node:18-alpine AS base
+FROM node:24-alpine AS base
 
 # Install dependencies only when needed
 FROM base AS deps
diff --git a/historyserver/README.md b/historyserver/README.md
@@ -0,0 +1,6 @@
+# KubeRay History Server
+
+This project is under active development.
+See [#ray-history-server](https://app.slack.com/client/TN4768NRM/C09QLLU8HTL) channel to provide feedback.
+
+<!-- TODO: Add docs for local development -->
diff --git a/historyserver/cmd/collector/Dockerfile b/historyserver/cmd/collector/Dockerfile
diff --git a/historyserver/cmd/collector/main.go b/historyserver/cmd/collector/main.go
@@ -0,0 +1 @@
+package main
diff --git a/historyserver/cmd/historyserver/Dockerfile b/historyserver/cmd/historyserver/Dockerfile
diff --git a/historyserver/cmd/historyserver/main.go b/historyserver/cmd/historyserver/main.go
@@ -0,0 +1 @@
+package main
diff --git a/historyserver/go.mod b/historyserver/go.mod
@@ -0,0 +1,3 @@
+module github.com/ray-project/kuberay/historyserver
+
+go 1.24.8
diff --git a/historyserver/pkg/eventserver/events.go b/historyserver/pkg/eventserver/events.go
@@ -0,0 +1 @@
+package eventserver
diff --git a/historyserver/pkg/storage/storage.go b/historyserver/pkg/storage/storage.go
@@ -0,0 +1 @@
+package storage
diff --git a/historyserver/test/e2e/collector_test.go b/historyserver/test/e2e/collector_test.go
@@ -0,0 +1 @@
+package e2e_test
diff --git a/historyserver/test/e2e/historyserver_test.go b/historyserver/test/e2e/historyserver_test.go
@@ -0,0 +1 @@
+package e2e_test
diff --git a/kuberay.code-workspace b/kuberay.code-workspace
@@ -12,6 +12,9 @@
 		{
 			"path": "kubectl-plugin"
 		},
+		{
+			"path": "historyserver"
+		},
 		{
 			"path": "proto"
 		},
diff --git a/ray-operator/config/samples/ray-cluster-resource-isolation-with-overrides.gke.yaml b/ray-operator/config/samples/ray-cluster-resource-isolation-with-overrides.gke.yaml
@@ -0,0 +1,61 @@
+apiVersion: ray.io/v1
+kind: RayCluster
+metadata:
+  name: raycluster-resource-isolation
+spec:
+  rayVersion: '2.52.0'
+  headGroupSpec:
+    rayStartParams:
+      enable-resource-isolation: "true" # Set --enable-resource-isolation flag in `ray start`
+      system-reserved-cpu: "2.0"
+      system-reserved-memory: "4000000000" # 4GB
+    template:
+      metadata:
+        annotations:
+          node.gke.io/enable-writable-cgroups.ray-head: "true"
+      spec:
+        containers:
+        - name: ray-head
+          image: rayproject/ray:2.52.0
+          resources:
+            limits:
+              cpu: 4
+              memory: 8G
+            requests:
+              cpu: 4
+              memory: 8G
+          ports:
+          - containerPort: 6379
+            name: gcs-server
+          - containerPort: 8265
+            name: dashboard
+          - containerPort: 10001
+            name: client
+        nodeSelector:
+          node.gke.io/enable-writable-cgroups: "true"
+  workerGroupSpecs:
+  - replicas: 1
+    minReplicas: 1
+    maxReplicas: 5
+    groupName: workergroup
+    rayStartParams:
+      enable-resource-isolation: "true" # Set --enable-resource-isolation flag in `ray start`
+      system-reserved-cpu: "1.0"
+      system-reserved-memory: "2000000000" # 2GB
+    template:
+      metadata:
+        annotations:
+          node.gke.io/enable-writable-cgroups.ray-worker: "true"
+      spec:
+        containers:
+        - name: ray-worker
+          image: rayproject/ray:2.52.0
+          resources:
+            limits:
+              cpu: 2
+              memory: 8G
+            requests:
+              cpu: 2
+              memory: 8G
+        nodeSelector:
+          node.gke.io/enable-writable-cgroups: "true"
diff --git a/ray-operator/config/samples/ray-cluster-resource-isolation.gke.yaml b/ray-operator/config/samples/ray-cluster-resource-isolation.gke.yaml
@@ -0,0 +1,57 @@
+apiVersion: ray.io/v1
+kind: RayCluster
+metadata:
+  name: raycluster-resource-isolation
+spec:
+  rayVersion: '2.52.0'
+  headGroupSpec:
+    rayStartParams:
+      enable-resource-isolation: "true" # Set --enable-resource-isolation flag in `ray start`
+    template:
+      metadata:
+        annotations:
+          node.gke.io/enable-writable-cgroups.ray-head: "true"
+      spec:
+        containers:
+        - name: ray-head
+          image: rayproject/ray:2.52.0
+          resources:
+            limits:
+              cpu: 2
+              memory: 8G
+            requests:
+              cpu: 2
+              memory: 8G
+          ports:
+          - containerPort: 6379
+            name: gcs-server
+          - containerPort: 8265
+            name: dashboard
+          - containerPort: 10001
+            name: client
+        nodeSelector:
+          node.gke.io/enable-writable-cgroups: "true"
+  workerGroupSpecs:
+  - replicas: 1
+    minReplicas: 1
+    maxReplicas: 5
+    groupName: workergroup
+    rayStartParams:
+      enable-resource-isolation: "true" # Set --enable-resource-isolation flag in `ray start`
+    template:
+      metadata:
+        annotations:
+          node.gke.io/enable-writable-cgroups.ray-worker: "true"
+      spec:
+        containers:
+        - name: ray-worker
+          image: rayproject/ray:2.52.0
+          resources:
+            limits:
+              cpu: 2
+              memory: 8G
+            requests:
+              cpu: 2
+              memory: 8G
+        nodeSelector:
+          node.gke.io/enable-writable-cgroups: "true"
diff --git a/ray-operator/config/samples/ray-job.light-weight-submitter.yaml b/ray-operator/config/samples/ray-job.light-weight-submitter.yaml
@@ -70,7 +70,7 @@ spec:
       restartPolicy: Never
       containers:
       - name: my-custom-rayjob-submitter
-        image: quay.io/kuberay/submitter:v1.5.0
+        image: quay.io/kuberay/submitter:v1.5.1
         command: ["/submitter"]
         args: ["--runtime-env-json", '{"pip":["requests==2.26.0","pendulum==2.1.2"],"env_vars":{"counter_name":"test_counter"}}', "--", "python", "/home/ray/samples/sample_code.py"]
 
diff --git a/ray-operator/controllers/ray/batchscheduler/volcano/volcano_scheduler.go b/ray-operator/controllers/ray/batchscheduler/volcano/volcano_scheduler.go
@@ -26,7 +26,6 @@ import (
 )
 
 const (
-	PodGroupName                              = "podgroups.scheduling.volcano.sh"
 	pluginName                                = "volcano"
 	QueueNameLabelKey                         = "volcano.sh/queue-name"
 	NetworkTopologyModeLabelKey               = "volcano.sh/network-topology-mode"
@@ -87,10 +86,11 @@ func (v *VolcanoBatchScheduler) handleRayJob(ctx context.Context, rayJob *rayv1.
 }
 
 func getSubmitterResource(rayJob *rayv1.RayJob) corev1.ResourceList {
-	if rayJob.Spec.SubmissionMode == rayv1.K8sJobMode {
+	switch rayJob.Spec.SubmissionMode {
+	case rayv1.K8sJobMode:
 		submitterTemplate := common.GetSubmitterTemplate(&rayJob.Spec, rayJob.Spec.RayClusterSpec)
 		return utils.CalculatePodResource(submitterTemplate.Spec)
-	} else if rayJob.Spec.SubmissionMode == rayv1.SidecarMode {
+	case rayv1.SidecarMode:
 		submitterContainer := common.GetDefaultSubmitterContainer(rayJob.Spec.RayClusterSpec)
 		containerResource := submitterContainer.Resources.Requests
 		for name, quantity := range submitterContainer.Resources.Limits {
@@ -99,8 +99,9 @@ func getSubmitterResource(rayJob *rayv1.RayJob) corev1.ResourceList {
 			}
 		}
 		return containerResource
+	default:
+		return corev1.ResourceList{}
 	}
-	return corev1.ResourceList{}
 }
 
 func getAppPodGroupName(object metav1.Object) string {
diff --git a/ray-operator/test/e2e/raycluster_auth_test.go b/ray-operator/test/e2e/raycluster_auth_test.go
@@ -1,10 +1,13 @@
 package e2e
 
 import (
+	"fmt"
 	"testing"
+	"time"
 
 	. "github.com/onsi/gomega"
 	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 
 	rayv1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1"
 	"github.com/ray-project/kuberay/ray-operator/controllers/ray/utils"
@@ -41,42 +44,92 @@ func TestRayClusterAuthOptions(t *testing.T) {
 		headPod, err := GetHeadPod(test, rayCluster)
 		g.Expect(err).NotTo(HaveOccurred())
 		g.Expect(headPod).NotTo(BeNil())
-		verifyAuthTokenEnvVars(t, rayCluster, *headPod)
 
+		// Verify Ray container has auth token env vars
+		VerifyContainerAuthTokenEnvVars(test, rayCluster, &headPod.Spec.Containers[utils.RayContainerIndex])
+
+		// Verify worker pods have auth token env vars
 		workerPods, err := GetWorkerPods(test, rayCluster)
 		g.Expect(err).NotTo(HaveOccurred())
 		g.Expect(workerPods).ToNot(BeEmpty())
 		for _, workerPod := range workerPods {
-			verifyAuthTokenEnvVars(t, rayCluster, workerPod)
+			VerifyContainerAuthTokenEnvVars(test, rayCluster, &workerPod.Spec.Containers[utils.RayContainerIndex])
 		}
 
-		// TODO(andrewsykim): add job submission test with and without token once a Ray version with token support is released.
+		// Get auth token for job submission tests
+		authToken := getAuthTokenFromPod(test, rayCluster, headPod)
+		g.Expect(authToken).NotTo(BeEmpty(), "Auth token should be present")
+
+		// Test job submission with auth token using Ray Job CLI
+		test.T().Run("Submit job with auth token should succeed", func(_ *testing.T) {
+			LogWithTimestamp(test.T(), "Testing job submission WITH auth token")
+
+			submissionId := fmt.Sprintf("test-job-with-auth-%d", time.Now().Unix())
+
+			// Submit job via Ray Job CLI with auth token
+			// Set RAY_AUTH_TOKEN environment variable for authentication
+			submitCmd := []string{
+				"bash", "-c",
+				fmt.Sprintf("RAY_AUTH_TOKEN=%s ray job submit --address http://127.0.0.1:8265 --submission-id %s --no-wait -- python -c 'import ray; ray.init(); print(\"Job with auth succeeded\")'",
+					authToken, submissionId),
+			}
+
+			stdout, _ := ExecPodCmd(test, headPod, headPod.Spec.Containers[utils.RayContainerIndex].Name, submitCmd)
+
+			// Verify job was submitted successfully
+			g.Expect(stdout.String()).To(ContainSubstring(submissionId), "Job submission should succeed with valid auth token")
+
+			// Verify job status is queryable with auth token (confirms auth works)
+			g.Eventually(func(g Gomega) {
+				statusCmd := []string{
+					"bash", "-c",
+					fmt.Sprintf("RAY_AUTH_TOKEN=%s ray job status --address http://127.0.0.1:8265 %s", authToken, submissionId),
+				}
+				stdout, _ := ExecPodCmd(test, headPod, headPod.Spec.Containers[utils.RayContainerIndex].Name, statusCmd)
+				g.Expect(stdout.String()).To(ContainSubstring("succeeded"))
+			}, TestTimeoutShort).Should(Succeed())
+
+			LogWithTimestamp(test.T(), "Successfully submitted and verified job with auth token")
+		})
+
+		test.T().Run("Submit job with incorrect auth token should fail", func(_ *testing.T) {
+			LogWithTimestamp(test.T(), "Testing job submission WITH incorrect auth token (should fail)")
+
+			submissionId := fmt.Sprintf("test-job-bad-auth-%d", time.Now().Unix())
+
+			// Submit job via Ray Job CLI with INCORRECT auth token
+			incorrectToken := "incorrect-token-12345"
+			submitCmd := []string{
+				"bash", "-c",
+				fmt.Sprintf("RAY_AUTH_TOKEN=%s ray job submit --address http://127.0.0.1:8265 --submission-id %s --no-wait -- python -c 'print(\"Should not run\")'",
+					incorrectToken, submissionId),
+			}
+
+			_, stderr := ExecPodCmd(test, headPod, headPod.Spec.Containers[utils.RayContainerIndex].Name, submitCmd, true)
+
+			// Verify response indicates authentication failure
+			g.Expect(stderr.String()).To(ContainSubstring("Unauthenticated"), "Job submission should fail with Unauthorized when auth token is incorrect")
+
+			LogWithTimestamp(test.T(), "Job submission correctly rejected with incorrect auth token")
+		})
 	})
 }
 
-func verifyAuthTokenEnvVars(t *testing.T, rayCluster *rayv1.RayCluster, pod corev1.Pod) {
-	g := NewWithT(t)
-
-	var rayAuthModeEnvVar *corev1.EnvVar
-	for _, envVar := range pod.Spec.Containers[0].Env {
-		if envVar.Name == utils.RAY_AUTH_MODE_ENV_VAR {
-			rayAuthModeEnvVar = &envVar
-			break
-		}
-	}
-	g.Expect(rayAuthModeEnvVar).NotTo(BeNil(), "RAY_AUTH_MODE environment variable should be set")
-	g.Expect(rayAuthModeEnvVar.Value).To(Equal(string(rayv1.AuthModeToken)), "RAY_AUTH_MODE should be %s", rayv1.AuthModeToken)
+// getAuthTokenFromPod extracts the auth token from the pod's environment variables.
+// It reads the token from the secret referenced by the RAY_AUTH_TOKEN environment variable.
+func getAuthTokenFromPod(test Test, rayCluster *rayv1.RayCluster, pod *corev1.Pod) string {
+	test.T().Helper()
+	g := NewWithT(test.T())
 
-	var rayAuthTokenEnvVar *corev1.EnvVar
-	for _, envVar := range pod.Spec.Containers[0].Env {
+	for _, envVar := range pod.Spec.Containers[utils.RayContainerIndex].Env {
 		if envVar.Name == utils.RAY_AUTH_TOKEN_ENV_VAR {
-			rayAuthTokenEnvVar = &envVar
-			break
+			if envVar.ValueFrom != nil && envVar.ValueFrom.SecretKeyRef != nil {
+				secret, err := test.Client().Core().CoreV1().Secrets(rayCluster.Namespace).
+					Get(test.Ctx(), envVar.ValueFrom.SecretKeyRef.Name, metav1.GetOptions{})
+				g.Expect(err).NotTo(HaveOccurred())
+				return string(secret.Data[envVar.ValueFrom.SecretKeyRef.Key])
+			}
 		}
 	}
-	g.Expect(rayAuthTokenEnvVar).NotTo(BeNil(), "RAY_AUTH_TOKEN environment variable should be set for AuthModeToken")
-	g.Expect(rayAuthTokenEnvVar.ValueFrom).NotTo(BeNil(), "RAY_AUTH_TOKEN should be populated from a secret")
-	g.Expect(rayAuthTokenEnvVar.ValueFrom.SecretKeyRef).NotTo(BeNil(), "RAY_AUTH_TOKEN should be populated from a secret key ref")
-	g.Expect(rayAuthTokenEnvVar.ValueFrom.SecretKeyRef.Name).To(ContainSubstring(rayCluster.Name), "Secret name should contain RayCluster name")
-	g.Expect(rayAuthTokenEnvVar.ValueFrom.SecretKeyRef.Key).To(Equal(utils.RAY_AUTH_TOKEN_SECRET_KEY), "Secret key should be %s", utils.RAY_AUTH_TOKEN_SECRET_KEY)
+	return ""
 }
diff --git a/ray-operator/test/e2erayjob/rayjob_lightweight_test.go b/ray-operator/test/e2erayjob/rayjob_lightweight_test.go
diff --git a/ray-operator/test/e2erayjob/rayjob_sidecar_mode_test.go b/ray-operator/test/e2erayjob/rayjob_sidecar_mode_test.go
diff --git a/ray-operator/test/e2erayjob/rayjob_test.go b/ray-operator/test/e2erayjob/rayjob_test.go
diff --git a/ray-operator/test/support/core.go b/ray-operator/test/support/core.go
diff --git a/ray-operator/test/support/ray.go b/ray-operator/test/support/ray.go

Original file line number	Diff line number	Diff line change
`@@ -1,4 +1,4 @@`
`1`		`-FROM node:18-alpine AS base`
	`1`	`+FROM node:24-alpine AS base`
`2`	`2`
`3`	`3`	`# Install dependencies only when needed`
`4`	`4`	`FROM base AS deps`
Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,3 @@`
	`1`	`+module github.com/ray-project/kuberay/historyserver`
	`2`	`+`
	`3`	`+go 1.24.8`