[doc][monitoring] Using existing Grafana instances.

.buildkite/pipeline.arm64.yml

@@ -0,0 +1,56 @@
+- label: ":mechanical_arm: :ferris_wheel: ARM64 Linux wheels"
+  conditions: ["RAY_CI_LINUX_WHEELS_AFFECTED"]
+  instance_size: arm64-medium
+  commands:
+    # Build the wheels
+    - UPLOAD_WHEELS_AS_ARTIFACTS=1 LINUX_WHEELS=1 ./ci/ci.sh build
+    # Upload the wheels
+    # We don't want to push on PRs, in fact, the copy_files will fail because unauthenticated.
+    - if [ "$BUILDKITE_PULL_REQUEST" != "false" ]; then exit 0; fi
+    - pip install -q docker aws_requests_auth boto3
+    - ./ci/env/env_info.sh
+    # Upload to branch directory.
+    - python .buildkite/copy_files.py --destination branch_wheels --path ./.whl
+    # Upload to latest directory.
+    - if [ "$BUILDKITE_BRANCH" == "master" ]; then python .buildkite/copy_files.py --destination wheels --path ./.whl; fi
+
+- label: ":mechanical_arm: :ferris_wheel: ARM64 Post-wheels tests"
+  conditions: ["RAY_CI_LINUX_WHEELS_AFFECTED"]
+  instance_size: arm64-medium
+  commands:
+    - LINUX_WHEELS=1 ./ci/ci.sh build
+    - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT
+    - ./ci/env/env_info.sh
+    - bazel test --config=ci $(./ci/run/bazel_export_options)
+      --test_tag_filters=post_wheel_build
+      --test_env=CONDA_EXE
+      --test_env=CONDA_PYTHON_EXE
+      --test_env=CONDA_SHLVL
+      --test_env=CONDA_PREFIX
+      --test_env=CONDA_DEFAULT_ENV
+      --test_env=CI
+      --test_env=RAY_CI_POST_WHEEL_TESTS=True
+      python/ray/tests/... python/ray/serve/... python/ray/tune/... rllib/... doc/...
+
+
+# This currently takes ~3 hours and times out often.
+# Enable this when ARM debug wheels are actually needed (and look into speeding this up).
+
+#- label: ":mechanical_arm: :ferris_wheel: ARM64 Debug Wheels"
+#  conditions:
+#    [
+#        "RAY_CI_LINUX_WHEELS_AFFECTED",
+#    ]
+#  instance_size: arm64-medium
+#  commands:
+#    # Build the debug wheels
+#    - RAY_DEBUG_BUILD=debug LINUX_WHEELS=1 ./ci/ci.sh build
+#    # Upload the wheels.
+#    # We don't want to push on PRs, in fact, the copy_files will fail because unauthenticated.
+#    - if [ "$BUILDKITE_PULL_REQUEST" != "false" ]; then exit 0; fi
+#    - pip install -q docker aws_requests_auth boto3
+#    - ./ci/env/env_info.sh
+#    # Upload to branch directory.
+#    - python .buildkite/copy_files.py --destination branch_wheels --path ./.whl
+#    # Upload to latest directory.
+#    - if [ "$BUILDKITE_BRANCH" == "master" ]; then python .buildkite/copy_files.py --destination wheels --path ./.whl; fi

.buildkite/pipeline.build.yml

@@ -200,7 +200,9 @@
   instance_size: medium
   commands:
     - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT
-    - bazel test --config=ci --config=ubsan $(./ci/run/bazel_export_options)
+    # Unset CC CXX vars set in Dockerfile. Clang currently runs into problems with ubsan builds, this will revert to
+    # using GCC instead.
+    - unset CC CXX && bazel test --config=ci --config=ubsan $(./ci/run/bazel_export_options)
       --build_tests_only
       --jobs=2
       -- //:all -//:core_worker_test -//:logging_test -//:ray_syncer_test
@@ -215,9 +217,8 @@
       --jobs=2
       -- //:all -//:core_worker_test -//:event_test -//:gcs_actor_manager_test
       -//:gcs_placement_group_manager_test -//:gcs_placement_group_scheduler_test
-      -//:gcs_server_rpc_test -//:gcs_client_test -//:gcs_heartbeat_manager_test
-      -//:metric_exporter_client_test -//:stats_test -//:worker_pool_test
-      -//:ray_syncer_test
+      -//:gcs_server_rpc_test -//:gcs_client_test -//:metric_exporter_client_test
+      -//:stats_test -//:worker_pool_test -//:ray_syncer_test
 
 - label: ":serverless: Dashboard Tests"
   conditions:
@@ -338,6 +339,14 @@
     - ./ci/ci.sh test_minimal 3.10
     - ./ci/ci.sh test_latest_core_dependencies 3.10
 
+- label: ":python: Minimal install 3.11"
+  conditions: ["RAY_CI_PYTHON_AFFECTED"]
+  instance_size: medium
+  commands:
+    - cleanup() { if [ "${BUILDKITE_PULL_REQUEST}" = "false" ]; then ./ci/build/upload_build_info.sh; fi }; trap cleanup EXIT
+    - ./ci/ci.sh test_minimal 3.11
+    - ./ci/ci.sh test_latest_core_dependencies 3.11
+
 - label: ":python: Default install"
   conditions: ["RAY_CI_PYTHON_AFFECTED"]
   instance_size: small

.github/CODEOWNERS

@@ -70,6 +70,7 @@
 # Ray data.
 /python/ray/data/ @ericl @scv119 @clarkzinzow @jjyao @jianoaix @c21
 /python/ray/data/preprocessors/ @clarkzinzow @jiaodong @Yard1 @bveeramani @matthewdeng @amogkam
+/python/ray/data/tests/preprocessors/ @clarkzinzow @jiaodong @Yard1 @bveeramani @matthewdeng @amogkam
 /doc/source/data/ @ericl @scv119 @clarkzinzow @jjyao @jianoaix @maxpumperla @c21 @ray-project/ray-docs
 
 # Ray workflows.
@@ -93,7 +94,7 @@
 /doc/source/serve/ @edoakes @simon-mo @jiaodong @shrekris-anyscale @sihanwang41 @architkulkarni @ray-project/ray-docs
 
 # AIR (docs)
-/doc/source/air/ @richardliaw @gjoliver @krfricke @xwjiang2010 @amogkam @matthewdeng @Yard1 @maxpumperla @ray-project/ray-docs
+/doc/source/ray-air/ @richardliaw @gjoliver @krfricke @xwjiang2010 @amogkam @matthewdeng @Yard1 @maxpumperla @ray-project/ray-docs
 
 # ML Docker Dependencies
 /python/requirements/ml/requirements_dl.txt @amogkam @sven1977 @richardliaw @matthewdeng

.gitignore

@@ -200,10 +200,6 @@ tools/prometheus*
 project-id
 .mypy_cache/
 
-# gitpod cache related
-.pip-cache/
-.bazel-cache/
-
 # release test related
 .anyscale.yaml
 test_state.json
@@ -224,3 +220,6 @@ workflow_data/
 /compile_commands.json
 # Directory where clangd puts its indexing work
 /.cache/
+
+# Auto-generated tag mapping
+tag-mapping.json

BUILD.bazel

@@ -934,17 +934,6 @@ cc_test(
     ],
 )
 
-cc_test(
-    name = "gcs_heartbeat_manager_test",
-    srcs = ["src/ray/gcs/gcs_server/test/gcs_heartbeat_manager_test.cc"],
-    copts = COPTS,
-    tags = ["team:core"],
-    deps = [
-        ":gcs_server_lib",
-        "@com_google_googletest//:gtest_main",
-    ],
-)
-
 cc_test(
     name = "dependency_resolver_test",
     size = "small",

bazel/ray_deps_setup.bzl

@@ -194,8 +194,8 @@ def ray_deps_setup():
     auto_http_archive(
         name = "cython",
         build_file = True,
-        url = "https://github.com/cython/cython/archive/3028e8c7ac296bc848d996e397c3354b3dbbd431.tar.gz",
-        sha256 = "31ea23c2231ddee8572a2a5effd54952e16a1b44e9a4cb3eb645418f8accf20d",
+        url = "https://github.com/cython/cython/archive/c48361d0a0969206e227ec016f654c9d941c2b69.tar.gz",
+        sha256 = "37c466fea398da9785bc37fe16f1455d2645d21a72e402103991d9e2fa1c6ff3",
     )
 
     auto_http_archive(

ci/build/build-docker-images.py

@@ -43,8 +43,9 @@
     "cu112": "nvidia/cuda:11.2.0-cudnn8-devel-ubuntu20.04",
     "cu111": "nvidia/cuda:11.1.1-cudnn8-devel-ubuntu20.04",
     "cu110": "nvidia/cuda:11.0.3-cudnn8-devel-ubuntu20.04",
-    "cu102": "nvidia/cuda:10.2-cudnn8-devel-ubuntu20.04",
-    "cu101": "nvidia/cuda:10.1-cudnn8-devel-ubuntu20.04",
+    # there is no ubuntu20.04 image for cuda 10.2 and 10.1
+    "cu102": "nvidia/cuda:10.2-cudnn8-devel-ubuntu18.04",
+    "cu101": "nvidia/cuda:10.1-cudnn8-devel-ubuntu18.04",
     "cpu": "ubuntu:focal",
 }
 
@@ -619,7 +620,7 @@ def push_readmes(merge_build: bool):
         default="py37",
         nargs="*",
         help="Which python versions to build. "
-        "Must be in (py36, py37, py38, py39, py310)",
+        "Must be in (py36, py37, py38, py39, py310, py311)",
     )
     parser.add_argument(
         "--device-types",

ci/build/test-wheels.sh

@@ -100,7 +100,8 @@ elif [[ "$platform" == "macosx" ]]; then
           "3.7"
           "3.8"
           "3.9"
-          "3.10")
+          "3.10"
+          )
 
   for ((i=0; i<${#PY_MMS[@]}; ++i)); do
     PY_MM="${PY_MMS[i]}"

ci/ci.sh

@@ -414,8 +414,7 @@ validate_wheels_commit_str() {
       continue
     fi
 
-    folder=${basename%%-cp*}
-    WHL_COMMIT=$(unzip -p "$whl" "${folder}.data/purelib/ray/__init__.py" | grep "__commit__" | awk -F'"' '{print $2}')
+    WHL_COMMIT=$(unzip -p "$whl" | grep "^__commit__" | awk -F'"' '{print $2}')
 
     if [ "${WHL_COMMIT}" != "${EXPECTED_COMMIT}" ]; then
       echo "Error: Observed wheel commit (${WHL_COMMIT}) is not expected commit (${EXPECTED_COMMIT}). Aborting."
@@ -455,11 +454,14 @@ build_wheels() {
         -e "RAY_DEBUG_BUILD=${RAY_DEBUG_BUILD:-}"
       )
 
+      IMAGE_NAME="quay.io/pypa/manylinux2014_${HOSTTYPE}"
+      IMAGE_TAG="2022-12-20-b4884d9"
+
       if [ -z "${BUILDKITE-}" ]; then
         # This command should be kept in sync with ray/python/README-building-wheels.md,
         # except the "${MOUNT_BAZEL_CACHE[@]}" part.
         docker run --rm -w /ray -v "${PWD}":/ray "${MOUNT_BAZEL_CACHE[@]}" \
-        quay.io/pypa/manylinux2014_x86_64:2021-11-07-28723f3 /ray/python/build-wheel-manylinux2014.sh
+        "${IMAGE_NAME}:${IMAGE_TAG}" /ray/python/build-wheel-manylinux2014.sh
       else
         rm -rf /ray-mount/*
         rm -rf /ray-mount/.whl || true
@@ -469,7 +471,7 @@ build_wheels() {
         docker run --rm -v /ray:/ray-mounted ubuntu:focal ls /
         docker run --rm -v /ray:/ray-mounted ubuntu:focal ls /ray-mounted
         docker run --rm -w /ray -v /ray:/ray "${MOUNT_BAZEL_CACHE[@]}" \
-          quay.io/pypa/manylinux2014_x86_64:2021-11-07-28723f3 /ray/python/build-wheel-manylinux2014.sh
+          "${IMAGE_NAME}:${IMAGE_TAG}" /ray/python/build-wheel-manylinux2014.sh
         cp -rT /ray-mount /ray # copy new files back here
         find . | grep whl # testing
 

ci/docker/base.test.Dockerfile

@@ -1,4 +1,5 @@
-FROM ubuntu:focal
+ARG DOCKER_IMAGE_BASE_UBUNTU=ubuntu:focal
+FROM $DOCKER_IMAGE_BASE_UBUNTU
 
 ARG REMOTE_CACHE_URL
 ARG BUILDKITE_PULL_REQUEST
@@ -33,6 +34,7 @@ RUN apt-get install -y -qq \
     sudo unzip unrar apt-utils dialog tzdata wget rsync \
     language-pack-en tmux cmake gdb vim htop \
     libgtk2.0-dev zlib1g-dev libgl1-mesa-dev \
+    liblz4-dev libunwind-dev libncurses5 \
     clang-format-12 jq \
     clang-tidy-12 clang-12
 # Make using GCC 9 explicit.

ci/docker/build.Dockerfile

@@ -9,6 +9,9 @@ ENV BUILDKITE_PULL_REQUEST=${BUILDKITE_PULL_REQUEST}
 ENV BUILDKITE_COMMIT=${BUILDKITE_COMMIT}
 ENV BUILDKITE_PULL_REQUEST_BASE_BRANCH=${BUILDKITE_PULL_REQUEST_BASE_BRANCH}
 ENV TRAVIS_COMMIT=${BUILDKITE_COMMIT}
+# Set compiler here to build Ray with CLANG/LLVM
+ENV CC=clang
+ENV CXX=clang++-12
 
 # Move out of working dir /ray
 # Delete stale data
@@ -25,9 +28,8 @@ RUN env
 
 # init also calls install-dependencies.sh
 RUN BUILD=1 bash --login -i ./ci/ci.sh init
-RUN bash --login -i ./ci/ci.sh build
 
-RUN export CC=clang CXX=clang++-12
+RUN bash --login -i ./ci/ci.sh build
 
 # Run determine test to run
 RUN bash --login -i -c "python ./ci/pipeline/determine_tests_to_run.py --output=json > affected_set.json"

ci/env/install-bazel.sh

@@ -5,7 +5,7 @@ ROOT_DIR=$(cd "$(dirname "$0")/$(dirname "$(test -L "$0" && readlink "$0" || ech
 
 arg1="${1-}"
 
-achitecture="${HOSTTYPE}"
+
 platform="unknown"
 case "${OSTYPE}" in
   msys)
@@ -26,6 +26,9 @@ case "${OSTYPE}" in
     exit 1
 esac
 
+architecture="${HOSTTYPE}"
+echo "Architecture is $architecture"
+
 if [ "${BAZEL_CONFIG_ONLY-}" != "1" ]; then
   # Sanity check: Verify we have symlinks where we expect them, or Bazel can produce weird "missing input file" errors.
   # This is most likely to occur on Windows, where symlinks are sometimes disabled by default.
@@ -50,26 +53,57 @@ if [ "${BAZEL_CONFIG_ONLY-}" != "1" ]; then
   if [ "${OSTYPE}" = "msys" ]; then
     target="${MINGW_DIR-/usr}/bin/bazel.exe"
     mkdir -p "${target%/*}"
-    curl -f -s -L -R -o "${target}" "https://github.com/bazelbuild/bazel/releases/download/${version}/bazel-${version}-${platform}-${achitecture}.exe"
+    curl -f -s -L -R -o "${target}" "https://github.com/bazelbuild/bazel/releases/download/${version}/bazel-${version}-${platform}-${architecture}.exe"
   else
-    target="./install.sh"
-    curl -f -s -L -R -o "${target}" "https://github.com/bazelbuild/bazel/releases/download/${version}/bazel-${version}-installer-${platform}-${achitecture}.sh"
-    chmod +x "${target}"
+
+    # Buildkite mac instances
     if [[ -n "${BUILDKITE-}" ]] && [ "${platform}" = "darwin" ]; then
-      "${target}" --user
+      mkdir -p "$HOME/bin"
       # Add bazel to the path.
       # shellcheck disable=SC2016
       printf '\nexport PATH="$HOME/bin:$PATH"\n' >> ~/.zshenv
       # shellcheck disable=SC1090
       source ~/.zshenv
+      INSTALL_USER=1
+    # Buildkite linux instance
     elif [ "${CI-}" = true ] || [ "${arg1-}" = "--system" ]; then
-      "$(command -v sudo || echo command)" "${target}" > /dev/null  # system-wide install for CI
+      INSTALL_USER=0
+    # User
     else
-      "${target}" --user > /dev/null
+      mkdir -p "$HOME/bin"
+      INSTALL_USER=1
       export PATH=$PATH:"$HOME/bin"
     fi
-    which bazel
-    rm -f "${target}"
+
+    if [ "${architecture}" = "aarch64" ]; then
+      # architecture is "aarch64", but the bazel tag is "arm64"
+      url="https://github.com/bazelbuild/bazel/releases/download/${version}/bazel-${version}-${platform}-arm64"
+
+      if [ "$INSTALL_USER" = "1" ]; then
+        target="$HOME/bin/bazel"
+        curl -f -s -L -R -o "${target}" "${url}"
+        chmod +x "${target}"
+      else
+        target="/bin/bazel"
+        sudo curl -f -s -L -R -o "${target}" "${url}"
+        sudo chmod +x "${target}"
+      fi
+
+      which bazel
+
+    else
+      target="./install.sh"
+      curl -f -s -L -R -o "${target}" "https://github.com/bazelbuild/bazel/releases/download/${version}/bazel-${version}-installer-${platform}-${architecture}.sh"
+      chmod +x "${target}"
+
+      if [ "$INSTALL_USER" = "1" ]; then
+        "${target}" --user
+      else
+        "$(command -v sudo || echo command)" "${target}" > /dev/null  # system-wide install for CI
+      fi
+      which bazel
+      rm -f "${target}"
+    fi
   fi
 fi