Merge branch 'master' into serve-llm-cross-node-parallelism-docs

Signed-off-by: Nikhil Ghosh <nikhil@anyscale.com>

Author: nrghosh

.buildkite/base.rayci.yml

@@ -64,6 +64,7 @@ steps:
     label: "wanda: oss-ci-base_gpu-py{{matrix}}"
     wanda: ci/docker/base.gpu.wanda.yaml
     matrix:
+      - "3.10"
       - "3.11"
       - "3.12"
     env:
@@ -93,6 +94,9 @@ steps:
     tags: cibase
 
   - name: docgpubuild
+    label: "wanda: docgpubuild-py3.10"
     wanda: ci/docker/docgpu.build.wanda.yaml
-    depends_on: oss-ci-base_gpu
+    depends_on: oss-ci-base_gpu-multipy
+    env:
+      PYTHON: "3.10"
     tags: cibase

.buildkite/build.rayci.yml

@@ -97,7 +97,6 @@ steps:
       - raycudabase
       - raycpubase
     matrix:
-      - "3.9"
       - "3.10"
       - "3.11"
       - "3.12"
@@ -123,7 +122,6 @@ steps:
       - raycpubaseextra
       - raycudabaseextra
     matrix:
-      - "3.9"
       - "3.10"
       - "3.11"
       - "3.12"

.buildkite/cicd.rayci.yml

@@ -7,11 +7,11 @@ steps:
           //ci/ray_ci/... //release/... //ci/pipeline/... ci
           --only-tags=release_unit,ci_unit
           --cache-test-results --parallelism-per-worker 2
-          --build-name oss-ci-base_test
+          --build-name oss-ci-base_test-py3.10
           --build-type skip
     instance_type: small
     depends_on:
-      - oss-ci-base_test
+      - oss-ci-base_test-multipy
       - forge
     tags: tools
   - label: ":coral: reef: raydepsets tests"
@@ -20,30 +20,30 @@ steps:
       - bazel run //ci/ray_ci:test_in_docker --
           //ci/raydepsets/... ci
           --cache-test-results
-          --build-name oss-ci-base_test
+          --build-name oss-ci-base_test-py3.10
           --build-type skip
     instance_type: small
     depends_on:
-      - oss-ci-base_test
+      - oss-ci-base_test-multipy
       - forge
     tags: tools
   - label: ":coral: reef: privileged container tests"
     commands:
       - bazel run //ci/ray_ci:test_in_docker --
           //ci/ray_ci:test_privileged ci
           --cache-test-results
-          --build-name oss-ci-base_test
+          --build-name oss-ci-base_test-py3.10
           --build-type cgroup
           --privileged
     instance_type: small
     depends_on:
-      - oss-ci-base_test
+      - oss-ci-base_test-multipy
       - forge
     tags: tools
   - label: ":coral: reef: iwyu tests"
     commands:
       - bazel test --config iwyu //bazel/tests/cpp:example_test
     instance_type: small
-    depends_on: oss-ci-base_build
-    job_env: oss-ci-base_build
+    depends_on: oss-ci-base_build-multipy
+    job_env: oss-ci-base_build-py3.10
     tags: tools

.buildkite/data.rayci.yml

@@ -217,15 +217,17 @@ steps:
     commands:
       # doc tests
       - bazel run //ci/ray_ci:test_in_docker -- //python/ray/data/... //doc/... data
-        --build-name docgpubuild
+        --build-name docgpubuild-py3.10
         --only-tags doctest
         --except-tags cpu
+        --python-version 3.10
       # doc examples
       - bazel run //ci/ray_ci:test_in_docker -- //doc/... data
-        --build-name docgpubuild
+        --build-name docgpubuild-py3.10
         --except-tags doctest
         --only-tags gpu
         --skip-ray-installation
+        --python-version 3.10
     depends_on: docgpubuild
 
   - label: ":database: data: integration tests"
@@ -279,8 +281,9 @@ steps:
     soft_fail: true
     commands:
       - bazel run //ci/ray_ci:test_in_docker -- //... data --run-flaky-tests
-        --build-name docgpubuild
+        --build-name docgpubuild-py3.10
         --only-tags gpu,gpu_only
+        --python-version 3.10
     depends_on: docgpubuild
   - label: ":data: postmerge authenticated tests"
     key: data_postmerge_authenticated_tests

.buildkite/ml.rayci.yml

@@ -6,11 +6,14 @@ depends_on:
 steps:
   # builds
   - name: minbuild-ml
-    label: "wanda: minbuild-ml-py39"
+    label: "wanda: minbuild-ml-py{{matrix}}"
     wanda: ci/docker/min.build.wanda.yaml
     depends_on: oss-ci-base_build
+    matrix:
+      - "3.9"
+      - "3.10"
     env:
-      PYTHON_VERSION: "3.9"
+      PYTHON_VERSION: "{{matrix}}"
       EXTRA_DEPENDENCY: ml
     tags: cibase
 

.buildkite/release-automation/verify-macos-wheels.sh

@@ -7,7 +7,7 @@ set -x
 # TODO(#54047): Python 3.13 is skipped due to the bug
 # we should re-enable it when the bug is fixed.
 
-PYTHON_VERSIONS=("3.9" "3.10" "3.11" "3.12")
+PYTHON_VERSIONS=("3.10" "3.11" "3.12")
 BAZELISK_VERSION="v1.16.0"
 
 export USE_BAZEL_VERSION="${USE_BAZEL_VERSION:-6.5.0}"

.buildkite/release-automation/wheels.rayci.yml

@@ -35,7 +35,6 @@ steps:
       - export RAY_COMMIT="$RAY_COMMIT"
       - bash -i .buildkite/release-automation/verify-linux-wheels.sh
     matrix:
-      - "3.9"
       - "3.10"
       - "3.11"
       - "3.12"
@@ -58,7 +57,6 @@ steps:
       - export RAY_COMMIT="$RAY_COMMIT"
       - bash -i .buildkite/release-automation/verify-linux-wheels.sh
     matrix:
-      - "3.9"
       - "3.10"
       - "3.11"
       - "3.12"

.buildkite/serve.rayci.yml

@@ -214,8 +214,8 @@ steps:
     instance_type: gpu
     commands:
       - bazel run //ci/ray_ci:test_in_docker -- //doc/... serve
-        --build-name docgpubuild
-        --only-tags gpu
+        --build-name docgpubuild-py3.10
+        --only-tags gpu --python-version 3.10
     depends_on: docgpubuild
 
   - label: ":ray-serve: serve: flaky tests"

.pre-commit-config.yaml

@@ -214,6 +214,12 @@ repos:
       - id: vale
         files: ^doc/source/data/.*\.(md|rst)$
 
+  - repo: https://github.com/MarcoGorelli/cython-lint
+    rev: v0.18.1
+    hooks:
+      - id: cython-lint
+        args: [--no-pycodestyle]
+
   - repo: local
     hooks:
       - id: check-import-order

BUILD.bazel

@@ -9,12 +9,11 @@
 load("@bazel_skylib//rules:common_settings.bzl", "bool_flag")
 load("@com_github_grpc_grpc//bazel:cython_library.bzl", "pyx_library")
 load("@hedron_compile_commands//:refresh_compile_commands.bzl", "refresh_compile_commands")
-load("@python3_9//:defs.bzl", python39 = "interpreter")
 load("@rules_cc//cc:defs.bzl", "cc_proto_library")
 load("@rules_pkg//pkg:mappings.bzl", "pkg_attributes", "pkg_files")
 load("@rules_pkg//pkg:zip.bzl", "pkg_zip")
 load("@rules_proto//proto:defs.bzl", "proto_library")
-load("@rules_python//python:defs.bzl", "py_binary", "py_library", "py_runtime", "py_runtime_pair")
+load("@rules_python//python:defs.bzl", "py_binary", "py_library")
 load("//bazel:ray.bzl", "COPTS", "PYX_COPTS", "PYX_SRCS", "ray_cc_library")
 
 package(
@@ -23,34 +22,13 @@ package(
 
 # Hermetic python environment, currently only used for CI infra and scripts.
 
-py_runtime(
-    name = "python3_runtime",
-    interpreter = python39,
-    python_version = "PY3",
-    visibility = ["//visibility:private"],
-)
-
-py_runtime_pair(
-    name = "python_runtime_pair",
-    py2_runtime = None,
-    py3_runtime = ":python3_runtime",
-    visibility = ["//visibility:private"],
-)
-
 constraint_setting(name = "hermetic")
 
 constraint_value(
     name = "hermetic_python",
     constraint_setting = ":hermetic",
 )
 
-toolchain(
-    name = "python_toolchain",
-    exec_compatible_with = [":hermetic_python"],
-    toolchain = ":python_runtime_pair",
-    toolchain_type = "@bazel_tools//tools/python:toolchain_type",
-)
-
 platform(
     name = "hermetic_python_platform",
     constraint_values = [":hermetic_python"],