Merge branch 'master' into ruff_rllib_import_4

Author: pseudo-rnd-thoughts

.buildkite/build.rayci.yml

@@ -1,5 +1,36 @@
 group: build
 steps:
+  - name: ray-core-build
+    label: "wanda: core binary parts py{{matrix}} (x86_64)"
+    wanda: ci/docker/ray-core.wanda.yaml
+    matrix:
+      - "3.9"
+      - "3.10"
+      - "3.11"
+      - "3.12"
+      - "3.13"
+    env:
+      PYTHON_VERSION: "{{matrix}}"
+      ARCH_SUFFIX: ""
+    tags: release_wheels
+    depends_on: manylinux
+
+  - name: ray-dashboard-build
+    label: "wanda: dashboard"
+    wanda: ci/docker/ray-dashboard.wanda.yaml
+    tags: release_wheels
+    depends_on: manylinux
+
+  - name: ray-java-build
+    label: "wanda: java build (x86_64)"
+    wanda: ci/docker/ray-java.wanda.yaml
+    tags:
+      - release_wheels
+      - oss
+    env:
+      ARCH_SUFFIX: ""
+    depends_on: manylinux
+
   - label: ":tapioca: build: wheel {{matrix}} (x86_64)"
     key: linux_wheels
     tags:

.buildkite/core.rayci.yml

@@ -60,7 +60,7 @@ steps:
       - bazel run //ci/ray_ci:test_in_docker -- //python/ray/tests/... //python/ray/_common/tests/... //python/ray/dag/... //python/ray/autoscaler/v2/... core
         --install-mask all-ray-libraries
         --workers "$${BUILDKITE_PARALLEL_JOB_COUNT}" --worker-id "$${BUILDKITE_PARALLEL_JOB}" --parallelism-per-worker 3
-        --except-tags debug_tests,asan_tests,post_wheel_build,ha_integration,mem_pressure,tmpfs,runtime_env_container,manual,multi_gpu,spark_on_ray,ray_client,dask
+        --except-tags custom_setup
         --install-mask all-ray-libraries
 
   - label: ":ray: core: python {{matrix.python}} tests ({{matrix.worker_id}})"
@@ -74,7 +74,7 @@ steps:
         --install-mask all-ray-libraries
         --workers 4 --worker-id "{{matrix.worker_id}}" --parallelism-per-worker 3
         --python-version {{matrix.python}}
-        --except-tags debug_tests,asan_tests,post_wheel_build,ha_integration,mem_pressure,tmpfs,runtime_env_container,manual,multi_gpu,spark_on_ray,ray_client,dask
+        --except-tags custom_setup
     depends_on: corebuild-multipy
     matrix:
       setup:
@@ -103,7 +103,7 @@ steps:
         --install-mask all-ray-libraries
         --workers "$${BUILDKITE_PARALLEL_JOB_COUNT}" --worker-id "$${BUILDKITE_PARALLEL_JOB}" --parallelism-per-worker 3
         --test-env=TEST_EXTERNAL_REDIS=1
-        --except-tags debug_tests,asan_tests,post_wheel_build,ha_integration,mem_pressure,tmpfs,runtime_env_container,manual,multi_gpu,spark_on_ray,ray_client,dask
+        --except-tags custom_setup
 
   - label: ":ray: core: memory pressure tests"
     tags:

.buildkite/linux_aarch64.rayci.yml

@@ -13,6 +13,17 @@ steps:
     wanda: ci/docker/manylinux.aarch64.wanda.yaml
     instance_type: builder-arm64
 
+  - name: ray-java-build-aarch64
+    label: "wanda: java build (aarch64)"
+    wanda: ci/docker/ray-java.wanda.yaml
+    tags:
+      - release_wheels
+      - oss
+    env:
+      ARCH_SUFFIX: "-aarch64"
+    instance_type: builder-arm64
+    depends_on: manylinux-aarch64
+
   - name: raycpubase-aarch64
     label: "wanda: ray.py{{matrix}}.cpu.base (aarch64)"
     tags:

.buildkite/release/build.rayci.yml

@@ -5,12 +5,13 @@ steps:
     wanda: docker/base-extra-testdeps/cpu.wanda.yaml
     matrix:
       - "3.9"
+      - "3.10"
       - "3.11"
       - "3.12"
     env:
       PYTHON_VERSION: "{{matrix}}"
       IMAGE_TYPE: "ray"
-      REQUIREMENTS_FILE: "requirements_byod_{{matrix}}.txt"
+      REQUIREMENTS_FILE: "ray_base_extra_testdeps_py{{matrix}}.lock"
     depends_on:
       - raycpubaseextra
 
@@ -21,6 +22,7 @@ steps:
       setup:
         python:
           - "3.9"
+          - "3.10"
           - "3.11"
           - "3.12"
         cuda:
@@ -29,7 +31,7 @@ steps:
       PYTHON_VERSION: "{{matrix.python}}"
       CUDA_VERSION: "{{matrix.cuda}}"
       IMAGE_TYPE: "ray"
-      REQUIREMENTS_FILE: "requirements_byod_{{matrix.python}}.txt"
+      REQUIREMENTS_FILE: "ray_base_extra_testdeps_py{{matrix.python}}.lock"
     depends_on:
       - raycudabaseextra
 
@@ -57,13 +59,14 @@ steps:
       setup:
         python:
           - "3.9"
+          - "3.10"
         cuda:
           - "12.1.1-cudnn8"
     env:
       PYTHON_VERSION: "{{matrix.python}}"
       CUDA_VERSION: "{{matrix.cuda}}"
       IMAGE_TYPE: "ray-ml"
-      REQUIREMENTS_FILE: "requirements_ml_byod_{{matrix.python}}.txt"
+      REQUIREMENTS_FILE: "ray_ml_base_extra_testdeps_py{{matrix.python}}.lock"
     depends_on:
       - ray-mlcudabaseextra
 
@@ -87,6 +90,7 @@ steps:
           # This list should be kept in sync with the list of supported Python in
           # release test suite. We don't have release tests for Python 3.10 yet.
           - "3.9"
+          - "3.10"
           - "3.11"
           - "3.12"
         platform:
@@ -125,3 +129,4 @@ steps:
       # release test suite. We don't have ray-ml release tests for Python 3.10 and 3.11
       # yet.
       - "3.9"
+      - "3.10"

.buildkite/release/custom-image-build-and-test-init.sh

@@ -0,0 +1,55 @@
+#!/bin/bash
+
+set -euo pipefail
+
+if [[ ${BUILDKITE_COMMIT} == "HEAD" ]]; then
+  BUILDKITE_COMMIT="$(git rev-parse HEAD)"
+  export BUILDKITE_COMMIT
+fi
+
+# Get build ID from environment variables
+BUILD_ID="${RAYCI_BUILD_ID:-}"
+
+if [[ -z "${BUILD_ID}" ]]; then
+    if [[ -n "${BUILDKITE_BUILD_ID:-}" ]]; then
+        # Generate SHA256 hash of BUILDKITE_BUILD_ID and take first 8 chars
+        BUILD_ID=$(echo -n "${BUILDKITE_BUILD_ID}" | sha256sum | cut -c1-8)
+    fi
+fi
+
+export RAYCI_BUILD_ID="${BUILD_ID}"
+echo "RAYCI_BUILD_ID: ${RAYCI_BUILD_ID}"
+
+
+aws ecr get-login-password --region us-west-2 | \
+    docker login --username AWS --password-stdin 029272617770.dkr.ecr.us-west-2.amazonaws.com
+
+bash release/gcloud_docker_login.sh release/aws2gce_iam.json
+export PATH="${PWD}/google-cloud-sdk/bin:$PATH"
+
+echo "Generate custom build steps"
+echo "Downloading Bazel"
+curl -sSfLo /tmp/bazel https://github.com/bazelbuild/bazelisk/releases/download/v1.19.0/bazelisk-linux-amd64
+echo "Making Bazel executable"
+chmod +x /tmp/bazel
+
+if [[ "${AUTOMATIC:-0}" == "1" && "${BUILDKITE_BRANCH}" == "master" ]]; then
+  export REPORT_TO_RAY_TEST_DB=1
+fi
+
+RUN_FLAGS=()
+
+if [[ "${AUTOMATIC:-0}" == "0" || "${BUILDKITE_BRANCH}" == "releases/"* ]]; then
+  RUN_FLAGS+=(--run-jailed-tests)
+fi
+if [[ "${BUILDKITE_BRANCH}" != "releases/"* ]]; then
+  RUN_FLAGS+=(--run-unstable-tests)
+fi
+
+echo "---- Build test steps"
+/tmp/bazel run //release:custom_image_build_and_test_init\
+  -- "${RUN_FLAGS[@]}" \
+  --custom-build-jobs-output-file .buildkite/release/custom_build_jobs.rayci.yaml \
+  --test-jobs-output-file .buildkite/release/release_tests.json \
+
+buildkite-agent pipeline upload .buildkite/release/release_tests.json

.buildkite/release/test.rayci.yml

@@ -180,6 +180,7 @@ steps:
     tags:
       - rllib_gpu
       - gpu
+      - rllib_flaky
       - skip-on-premerge
     instance_type: gpu-large
     commands:
@@ -196,7 +197,7 @@ steps:
     tags:
       - rllib_gpu
       - gpu
-      - flaky
+      - rllib_flaky
       - skip-on-premerge
     instance_type: gpu
     commands:
@@ -212,7 +213,7 @@ steps:
     key: rllib_flaky_tests_01
     tags:
       - rllib
-      - flaky
+      - rllib_flaky
       - skip-on-premerge
     instance_type: large
     commands:
@@ -242,7 +243,7 @@ steps:
     key: rllib_flaky_tests_02
     tags:
       - rllib
-      - flaky
+      - rllib_flaky
       - skip-on-premerge
     instance_type: large
     commands:

.buildkite/rllib.rayci.yml

@@ -44,6 +44,7 @@
 # Ray data.
 /python/ray/data/ @ray-project/ray-data
 /doc/source/data/ @ray-project/ray-data
+/python/ray/dashboard/modules/data/ @ray-project/ray-data
 /python/ray/dashboard/modules/metrics/dashboards/data_dashboard_panels.py @ray-project/ray-data
 
 # Ray workflows.
@@ -72,6 +73,7 @@
 # Ray Serve
 /python/ray/serve/ @ray-project/ray-serve
 /src/ray/protobuf/serve.proto @ray-project/ray-serve
+/python/ray/dashboard/modules/serve/ @ray-project/ray-serve
 /doc/source/serve/ @ray-project/ray-serve @ray-project/ray-docs
 
 # ML Docker Dependencies

.github/CODEOWNERS

@@ -13,7 +13,7 @@
 ## Checks
 
 - [ ] I've signed off every commit(by using the -s flag, i.e., `git commit -s`) in this PR.
-- [ ] I've run `scripts/format.sh` to lint the changes in this PR.
+- [ ] I've run pre-commit jobs to lint the changes in this PR. ([pre-commit setup](https://docs.ray.io/en/latest/ray-contribute/getting-involved.html#lint-and-formatting))
 - [ ] I've included any doc changes needed for https://docs.ray.io/en/master/.
     - [ ] I've added any new APIs to the API Reference. For example, if I added a
            method in Tune, I've added it in `doc/source/tune/api/` under the

.github/PULL_REQUEST_TEMPLATE.md

@@ -1 +1 @@
-0.19.0
+0.20.0