[ci][7up/2.1] unify default python across rayci (ray-project#42213)

- Unify the default python version across rayci
- Update all rayci test to adapt to the change of the default python version
- Remove explicit python versions from many ci tests that depends on the default version
- Make a few tests agnostic to python version changes

Signed-off-by: can <can@anyscale.com>

.buildkite/build.rayci.yml

@@ -39,7 +39,7 @@ steps:
       - oss
     instance_type: medium
     commands:
-      - bazel run //ci/ray_ci:build_in_docker -- wheel --python-version 3.8 --build-type debug
+      - bazel run //ci/ray_ci:build_in_docker -- wheel --build-type debug
     depends_on:
       - manylinux
       - forge

.buildkite/core.rayci.yml

@@ -308,7 +308,7 @@ steps:
       - docker
     instance_type: medium
     commands:
-      - bazel run //ci/ray_ci:build_in_docker -- docker --python-version 3.8 --platform cpu --canonical-tag ha_integration
+      - bazel run //ci/ray_ci:build_in_docker -- docker --platform cpu --canonical-tag ha_integration
       - bazel run //ci/ray_ci:test_in_docker -- //python/ray/tests/... core --only-tags ha_integration
     depends_on:
       - manylinux
@@ -322,7 +322,7 @@ steps:
       - docker
     instance_type: medium
     commands:
-      - bazel run //ci/ray_ci:build_in_docker -- docker --python-version 3.8 --platform cpu
+      - bazel run //ci/ray_ci:build_in_docker -- docker --platform cpu
         --canonical-tag test_container
       - docker build --build-arg BASE_IMAGE="rayproject/ray:test_container"
         -t rayproject/ray:runtime_env_container -f docker/runtime_env_container/Dockerfile .

.buildkite/ml.rayci.yml

@@ -167,13 +167,13 @@ steps:
     tags: tune
     instance_type: medium
     commands:
-      - bazel run //ci/ray_ci:build_in_docker -- docker --python-version 3.8
-        --platform cpu --image-type ray --canonical-tag multinode-py38
-      - python ./ci/build/build-multinode-image.py rayproject/ray:multinode-py38 rayproject/ray:multinode-py38
+      - bazel run //ci/ray_ci:build_in_docker -- docker
+        --platform cpu --image-type ray --canonical-tag multinode
+      - python ./ci/build/build-multinode-image.py rayproject/ray:multinode rayproject/ray:multinode
       - bazel run //ci/ray_ci:test_in_docker -- //python/ray/tune/... ml 
         --only-tags multinode
         --test-env=RAY_HAS_SSH=1
-        --test-env=RAY_DOCKER_IMAGE=rayproject/ray:multinode-py38
+        --test-env=RAY_DOCKER_IMAGE=rayproject/ray:multinode
         --test-env=RAY_TEMPDIR="/ray-mount"
         --test-env=RAY_HOSTDIR="$${RAYCI_CHECKOUT_DIR}"
         --test-env=RAY_TESTHOST="rayci.localhost"

.buildkite/serve.rayci.yml

@@ -182,7 +182,7 @@ steps:
       - python
     instance_type: medium
     commands:
-      - bazel run //ci/ray_ci:build_in_docker -- docker --python-version 3.8 --platform cpu --canonical-tag ha_integration
+      - bazel run //ci/ray_ci:build_in_docker -- docker --platform cpu --canonical-tag ha_integration
       - bazel run //ci/ray_ci:test_in_docker -- //python/ray/serve/tests/... serve
         --only-tags ha_integration
     depends_on:

ci/k8s/run-chaos-test.sh

@@ -5,7 +5,7 @@ set -euo pipefail
 CHAOS_FAULT="${1:-no_fault}"
 CHAOS_WORKLOAD="${2:-test_potato_passer}"
 
-bazel run //ci/ray_ci:build_in_docker -- docker --python-version 3.8 \
+bazel run //ci/ray_ci:build_in_docker -- docker \
     --platform cpu --canonical-tag kuberay-test
 docker tag rayproject/ray:kuberay-test ray-ci:kuberay-test
 

ci/k8s/run-operator-tests.sh

@@ -6,7 +6,7 @@ set -euo pipefail
 
 echo "--- Build image"
 bazel run //ci/ray_ci:build_in_docker -- docker \
-    --python-version 3.8 --platform cpu --canonical-tag kuberay-test
+    --platform cpu --canonical-tag kuberay-test
 docker tag rayproject/ray:kuberay-test ray-ci:kuberay-test
 
 echo "--- Setup k8s environment"

ci/ray_ci/builder.py

@@ -3,6 +3,7 @@
 import click
 
 from ci.ray_ci.builder_container import (
+    DEFAULT_PYTHON_VERSION,
     PYTHON_VERSIONS,
     BUILD_TYPES,
     ARCHITECTURE,
@@ -34,7 +35,7 @@
 )
 @click.option(
     "--python-version",
-    default="3.8",
+    default=DEFAULT_PYTHON_VERSION,
     type=click.Choice(list(PYTHON_VERSIONS.keys())),
     help=("Python version to build the wheel with"),
 )

ci/ray_ci/docker_container.py

@@ -2,7 +2,7 @@
 from typing import List
 
 from ci.ray_ci.linux_container import LinuxContainer
-from ci.ray_ci.builder_container import DEFAULT_ARCHITECTURE
+from ci.ray_ci.builder_container import DEFAULT_ARCHITECTURE, DEFAULT_PYTHON_VERSION
 
 
 PLATFORM = [
@@ -14,7 +14,6 @@
     "cu12.1.1",
 ]
 GPU_PLATFORM = "cu11.8.0"
-DEFAULT_PYTHON_VERSION = "3.8"
 
 
 class DockerContainer(LinuxContainer):
@@ -73,7 +72,7 @@ def _get_canonical_tag(self) -> str:
         return self.canonical_tag if self.canonical_tag else self._get_image_tags()[0]
 
     def get_python_version_tag(self) -> str:
-        return f"-py{self.python_version.replace('.', '')}"  # 3.8 -> py38
+        return f"-py{self.python_version.replace('.', '')}"  # 3.x -> py3x
 
     def get_platform_tag(self) -> str:
         if self.platform == "cpu":

ci/ray_ci/test_anyscale_docker_container.py

@@ -17,36 +17,37 @@ def test_run(self) -> None:
         def _mock_run_script(input: List[str]) -> None:
             self.cmds.append(input)
 
+        v = self.get_non_default_python()
+        pv = self.get_python_version(v)
+
         with mock.patch(
             "ci.ray_ci.docker_container.LinuxContainer.run_script",
             side_effect=_mock_run_script,
         ), mock.patch.dict(os.environ, {"BUILDKITE_BRANCH": "test_branch"}):
-            container = AnyscaleDockerContainer(
-                "3.9", "cu11.8.0", "ray-ml", upload=True
-            )
+            container = AnyscaleDockerContainer(v, "cu11.8.0", "ray-ml", upload=True)
             container.run()
             cmd = self.cmds[-1]
             aws_ecr = _DOCKER_ECR_REPO.split("/")[0]
             aws_prj = f"{aws_ecr}/anyscale/ray-ml"
             gcp_prj = f"{_DOCKER_GCP_REGISTRY}/anyscale/ray-ml"
             assert cmd == [
                 "./ci/build/build-anyscale-docker.sh "
-                "rayproject/ray-ml:123456-py39-cu118 "
-                f"{aws_prj}:123456-py39-cu118 requirements_ml_byod_3.9.txt {aws_ecr}",
+                f"rayproject/ray-ml:123456-{pv}-cu118 "
+                f"{aws_prj}:123456-{pv}-cu118 requirements_ml_byod_{v}.txt {aws_ecr}",
                 "./release/gcloud_docker_login.sh release/aws2gce_iam.json",
                 "export PATH=$(pwd)/google-cloud-sdk/bin:$PATH",
-                f"docker tag {aws_prj}:123456-py39-cu118 {aws_prj}:123456-py39-cu118",
-                f"docker push {aws_prj}:123456-py39-cu118",
-                f"docker tag {aws_prj}:123456-py39-cu118 {gcp_prj}:123456-py39-cu118",
-                f"docker push {gcp_prj}:123456-py39-cu118",
-                f"docker tag {aws_prj}:123456-py39-cu118 {aws_prj}:123456-py39-gpu",
-                f"docker push {aws_prj}:123456-py39-gpu",
-                f"docker tag {aws_prj}:123456-py39-cu118 {gcp_prj}:123456-py39-gpu",
-                f"docker push {gcp_prj}:123456-py39-gpu",
-                f"docker tag {aws_prj}:123456-py39-cu118 {aws_prj}:123456-py39",
-                f"docker push {aws_prj}:123456-py39",
-                f"docker tag {aws_prj}:123456-py39-cu118 {gcp_prj}:123456-py39",
-                f"docker push {gcp_prj}:123456-py39",
+                f"docker tag {aws_prj}:123456-{pv}-cu118 {aws_prj}:123456-{pv}-cu118",
+                f"docker push {aws_prj}:123456-{pv}-cu118",
+                f"docker tag {aws_prj}:123456-{pv}-cu118 {gcp_prj}:123456-{pv}-cu118",
+                f"docker push {gcp_prj}:123456-{pv}-cu118",
+                f"docker tag {aws_prj}:123456-{pv}-cu118 {aws_prj}:123456-{pv}-gpu",
+                f"docker push {aws_prj}:123456-{pv}-gpu",
+                f"docker tag {aws_prj}:123456-{pv}-cu118 {gcp_prj}:123456-{pv}-gpu",
+                f"docker push {gcp_prj}:123456-{pv}-gpu",
+                f"docker tag {aws_prj}:123456-{pv}-cu118 {aws_prj}:123456-{pv}",
+                f"docker push {aws_prj}:123456-{pv}",
+                f"docker tag {aws_prj}:123456-{pv}-cu118 {gcp_prj}:123456-{pv}",
+                f"docker push {gcp_prj}:123456-{pv}",
             ]
 
 

ci/ray_ci/test_base.py

@@ -2,6 +2,9 @@
 import unittest
 from unittest.mock import patch
 
+from ci.ray_ci.builder_container import PYTHON_VERSIONS
+from ci.ray_ci.builder import DEFAULT_PYTHON_VERSION
+
 
 class RayCITestBase(unittest.TestCase):
     def setUp(self) -> None:
@@ -20,3 +23,14 @@ def setUp(self) -> None:
 
     def tearDown(self) -> None:
         self.patcher.stop()
+
+    def get_non_default_python(self) -> str:
+        for version in PYTHON_VERSIONS.keys():
+            if version != DEFAULT_PYTHON_VERSION:
+                return version
+
+    def get_python_version(self, version: str) -> str:
+        return f"py{version.replace('.', '')}"  # 3.x -> py3x
+
+    def get_cpp_version(self, version: str) -> str:
+        return f"cp{version.replace('.', '')}"  # 3.x -> cp3x

ci/ray_ci/test_ray_docker_container.py

@@ -5,6 +5,7 @@
 
 import pytest
 
+from ci.ray_ci.builder_container import DEFAULT_PYTHON_VERSION
 from ci.ray_ci.container import _DOCKER_ECR_REPO
 from ci.ray_ci.ray_docker_container import RayDockerContainer
 from ci.ray_ci.test_base import RayCITestBase
@@ -24,109 +25,128 @@ def _mock_run_script(input: List[str]) -> None:
             "ci.ray_ci.docker_container.LinuxContainer.run_script",
             side_effect=_mock_run_script,
         ):
-            container = RayDockerContainer("3.8", "cu11.8.0", "ray")
+            v = DEFAULT_PYTHON_VERSION
+            cv = self.get_cpp_version(v)
+            pv = self.get_python_version(v)
+            container = RayDockerContainer(v, "cu11.8.0", "ray")
             container.run()
             cmd = self.cmds[-1]
             assert cmd == (
                 "./ci/build/build-ray-docker.sh "
-                f"ray-{RAY_VERSION}-cp38-cp38-manylinux2014_x86_64.whl "
-                f"{_DOCKER_ECR_REPO}:123-ray-py3.8-cu11.8.0-base "
+                f"ray-{RAY_VERSION}-{cv}-{cv}-manylinux2014_x86_64.whl "
+                f"{_DOCKER_ECR_REPO}:123-ray-py{v}-cu11.8.0-base "
                 "requirements_compiled.txt "
-                "rayproject/ray:123456-py38-cu118 "
-                "ray:123456-py38-cu118_pip-freeze.txt"
+                f"rayproject/ray:123456-{pv}-cu118 "
+                f"ray:123456-{pv}-cu118_pip-freeze.txt"
             )
 
-            container = RayDockerContainer("3.9", "cpu", "ray-ml")
+            v = self.get_non_default_python()
+            cv = self.get_cpp_version(v)
+            pv = self.get_python_version(v)
+            container = RayDockerContainer(v, "cpu", "ray-ml")
             container.run()
             cmd = self.cmds[-1]
             assert cmd == (
                 "./ci/build/build-ray-docker.sh "
-                f"ray-{RAY_VERSION}-cp39-cp39-manylinux2014_x86_64.whl "
-                f"{_DOCKER_ECR_REPO}:123-ray-ml-py3.9-cpu-base "
+                f"ray-{RAY_VERSION}-{cv}-{cv}-manylinux2014_x86_64.whl "
+                f"{_DOCKER_ECR_REPO}:123-ray-ml-py{v}-cpu-base "
                 "requirements_compiled.txt "
-                "rayproject/ray-ml:123456-py39-cpu "
-                "ray-ml:123456-py39-cpu_pip-freeze.txt"
+                f"rayproject/ray-ml:123456-{pv}-cpu "
+                f"ray-ml:123456-{pv}-cpu_pip-freeze.txt"
             )
 
     def test_canonical_tag(self) -> None:
-        container = RayDockerContainer("3.8", "cpu", "ray", canonical_tag="abc")
+        v = DEFAULT_PYTHON_VERSION
+        pv = self.get_python_version(v)
+        container = RayDockerContainer(v, "cpu", "ray", canonical_tag="abc")
         assert container._get_canonical_tag() == "abc"
 
-        container = RayDockerContainer("3.8", "cpu", "ray")
-        assert container._get_canonical_tag() == "123456-py38-cpu"
+        container = RayDockerContainer(v, "cpu", "ray")
+        assert container._get_canonical_tag() == f"123456-{pv}-cpu"
 
-        container = RayDockerContainer("3.8", "cpu", "ray", "aarch64")
-        assert container._get_canonical_tag() == "123456-py38-cpu-aarch64"
+        container = RayDockerContainer(v, "cpu", "ray", "aarch64")
+        assert container._get_canonical_tag() == f"123456-{pv}-cpu-aarch64"
 
-        container = RayDockerContainer("3.8", "cu11.8.0", "ray-ml")
-        assert container._get_canonical_tag() == "123456-py38-cu118"
+        container = RayDockerContainer(v, "cu11.8.0", "ray-ml")
+        assert container._get_canonical_tag() == f"123456-{pv}-cu118"
 
         with mock.patch.dict(os.environ, {"BUILDKITE_BRANCH": "releases/1.0.0"}):
-            container = RayDockerContainer("3.8", "cpu", "ray")
-            assert container._get_canonical_tag() == "1.0.0.123456-py38-cpu"
+            container = RayDockerContainer(v, "cpu", "ray")
+            assert container._get_canonical_tag() == f"1.0.0.123456-{pv}-cpu"
 
         with mock.patch.dict(
             os.environ, {"BUILDKITE_BRANCH": "abc", "BUILDKITE_PULL_REQUEST": "123"}
         ):
-            container = RayDockerContainer("3.8", "cpu", "ray")
-            assert container._get_canonical_tag() == "pr-123.123456-py38-cpu"
+            container = RayDockerContainer(v, "cpu", "ray")
+            assert container._get_canonical_tag() == f"pr-123.123456-{pv}-cpu"
 
     def test_get_image_tags(self) -> None:
         # bulk logic of _get_image_tags is tested in its callers (get_image_name and
         # get_canonical_tag), so we only test the basic cases here
-        container = RayDockerContainer("3.8", "cpu", "ray")
+        v = DEFAULT_PYTHON_VERSION
+        pv = self.get_python_version(v)
+        container = RayDockerContainer(v, "cpu", "ray")
         assert container._get_image_tags() == [
-            "123456-py38-cpu",
+            f"123456-{pv}-cpu",
             "123456-cpu",
-            "123456-py38",
+            f"123456-{pv}",
             "123456",
-            "nightly-py38-cpu",
+            f"nightly-{pv}-cpu",
             "nightly-cpu",
-            "nightly-py38",
+            f"nightly-{pv}",
             "nightly",
         ]
 
     def test_get_image_name(self) -> None:
-        container = RayDockerContainer("3.8", "cpu", "ray")
+        v = DEFAULT_PYTHON_VERSION
+        pv = self.get_python_version(v)
+        container = RayDockerContainer(v, "cpu", "ray")
         assert container._get_image_names() == [
-            "rayproject/ray:123456-py38-cpu",
+            f"rayproject/ray:123456-{pv}-cpu",
             "rayproject/ray:123456-cpu",
-            "rayproject/ray:123456-py38",
+            f"rayproject/ray:123456-{pv}",
             "rayproject/ray:123456",
-            "rayproject/ray:nightly-py38-cpu",
+            f"rayproject/ray:nightly-{pv}-cpu",
             "rayproject/ray:nightly-cpu",
-            "rayproject/ray:nightly-py38",
+            f"rayproject/ray:nightly-{pv}",
             "rayproject/ray:nightly",
         ]
 
-        container = RayDockerContainer("3.9", "cu11.8.0", "ray-ml")
+        v = self.get_non_default_python()
+        pv = self.get_python_version(v)
+        container = RayDockerContainer(v, "cu11.8.0", "ray-ml")
         assert container._get_image_names() == [
-            "rayproject/ray-ml:123456-py39-cu118",
-            "rayproject/ray-ml:123456-py39-gpu",
-            "rayproject/ray-ml:123456-py39",
-            "rayproject/ray-ml:nightly-py39-cu118",
-            "rayproject/ray-ml:nightly-py39-gpu",
-            "rayproject/ray-ml:nightly-py39",
+            f"rayproject/ray-ml:123456-{pv}-cu118",
+            f"rayproject/ray-ml:123456-{pv}-gpu",
+            f"rayproject/ray-ml:123456-{pv}",
+            f"rayproject/ray-ml:nightly-{pv}-cu118",
+            f"rayproject/ray-ml:nightly-{pv}-gpu",
+            f"rayproject/ray-ml:nightly-{pv}",
         ]
 
         with mock.patch.dict(os.environ, {"BUILDKITE_BRANCH": "releases/1.0.0"}):
-            container = RayDockerContainer("3.8", "cpu", "ray")
+            v = DEFAULT_PYTHON_VERSION
+            pv = self.get_python_version(v)
+            container = RayDockerContainer(v, "cpu", "ray")
             assert container._get_image_names() == [
-                "rayproject/ray:1.0.0.123456-py38-cpu",
+                f"rayproject/ray:1.0.0.123456-{pv}-cpu",
                 "rayproject/ray:1.0.0.123456-cpu",
-                "rayproject/ray:1.0.0.123456-py38",
+                f"rayproject/ray:1.0.0.123456-{pv}",
                 "rayproject/ray:1.0.0.123456",
             ]
 
     def test_get_python_version_tag(self) -> None:
-        container = RayDockerContainer("3.8", "cpu", "ray")
-        assert container.get_python_version_tag() == "-py38"
+        v = DEFAULT_PYTHON_VERSION
+        pv = self.get_python_version(v)
+        container = RayDockerContainer(v, "cpu", "ray")
+        assert container.get_python_version_tag() == f"-{pv}"
 
     def test_get_platform_tag(self) -> None:
-        container = RayDockerContainer("3.8", "cpu", "ray")
+        v = DEFAULT_PYTHON_VERSION
+        container = RayDockerContainer(v, "cpu", "ray")
         assert container.get_platform_tag() == "-cpu"
 
-        container = RayDockerContainer("3.8", "cu11.8.0", "ray")
+        container = RayDockerContainer(v, "cu11.8.0", "ray")
         assert container.get_platform_tag() == "-cu118"
 
 

python/ray/_private/test_utils.py

@@ -2117,3 +2117,11 @@ def wrapper(func):
         )(func)
 
     return wrapper
+
+
+def get_ray_default_worker_file_path():
+    py_version = f"{sys.version_info[0]}.{sys.version_info[1]}"
+    return (
+        f"/home/ray/anaconda3/lib/python{py_version}/"
+        "site-packages/ray/_private/workers/default_worker.py"
+    )