[core] upgrade opentelemetry-sdk (#53745)

This PR updates the `opentelemetry-sdk` version requirement to a minimum
of 1.30, which is necessary to support histogram metrics as part of our
migration from OpenCensus to OpenTelemetry.

As part of this upgrade, I am also removing
`opentelemetry-exporter-otlp` from the `ray[all]` dependency set. This
package has been notoriously difficult to resolve alongside other
dependencies, and this issue becomes even bigger with the updated SDK
version. Since it is only used for the Ray tracing feature, I propose
making it an optional, user-controlled dependency instead of a default
one.

Additionally, the test environment has been updated to ensure
`test_tracing.py` continues to pass, verifying that tracing
functionality remains intact.

Test:
- CI

Signed-off-by: can <can@anyscale.com>

Author: can-anyscale

.buildkite/build.rayci.yml

@@ -115,12 +115,14 @@ steps:
       - "3.9"
       - "3.10"
       - "3.11"
+
   - label: ":tapioca: smoke test build-docker.sh"
     tags:
       - python_dependencies
       - docker
       - oss
     instance_type: medium
+    soft_fail: true
     commands:
       - export WHEEL_URL="https://files.pythonhosted.org/packages/7e/7c/3544cca730265bb6f7a4900b7b7018c08ce5ec89bf7b6102901fe0bcd67b/ray-2.44.1-cp39-cp39-manylinux2014_x86_64.whl"
       - export CPP_WHEEL_URL="https://files.pythonhosted.org/packages/50/ae/c094818fd526bfb0a361a76bda5708a73cbc888b51edfc7d6aab9de837cd/ray_cpp-2.44.1-cp39-cp39-manylinux2014_x86_64.whl"

.buildkite/serve.rayci.yml

@@ -21,6 +21,9 @@ steps:
     wanda: ci/docker/servepydantic1.build.wanda.yaml
     tags: cibase
 
+  - name: servetracingbuild
+    wanda: ci/docker/servetracing.build.wanda.yaml
+
   - name: minbuild-serve
     label: "wanda: minbuild-{{matrix}}-py39"
     wanda: ci/docker/min.build.wanda.yaml
@@ -41,7 +44,7 @@ steps:
     instance_type: large
     commands:
       - bazel run //ci/ray_ci:test_in_docker -- //python/ray/serve/... //python/ray/tests/...  serve
-        --except-tags post_wheel_build,gpu,ha_integration
+        --except-tags post_wheel_build,gpu,ha_integration,serve_tracing
         --workers "$${BUILDKITE_PARALLEL_JOB_COUNT}" --worker-id "$${BUILDKITE_PARALLEL_JOB}" --parallelism-per-worker 3
         --build-name servebuild --test-env=EXPECTED_PYTHON_VERSION=3.9
     depends_on: "servebuild"
@@ -55,7 +58,7 @@ steps:
     soft_fail: true
     commands:
       - bazel run //ci/ray_ci:test_in_docker -- //python/ray/serve/... //python/ray/tests/...  serve
-        --except-tags post_wheel_build,gpu,ha_integration
+        --except-tags post_wheel_build,gpu,ha_integration,serve_tracing
         --workers "$${BUILDKITE_PARALLEL_JOB_COUNT}" --worker-id "$${BUILDKITE_PARALLEL_JOB}" --parallelism-per-worker 3
         --build-name servepydantic1build --test-env=EXPECTED_PYTHON_VERSION=3.9 --test-env=EXPECTED_PYDANTIC_VERSION=1.10.12
     depends_on: servepydantic1build
@@ -69,7 +72,7 @@ steps:
     soft_fail: true
     commands:
       - bazel run //ci/ray_ci:test_in_docker -- //python/ray/serve/... //python/ray/tests/...  serve
-        --except-tags post_wheel_build,gpu,ha_integration
+        --except-tags post_wheel_build,gpu,ha_integration,serve_tracing
         --workers "$${BUILDKITE_PARALLEL_JOB_COUNT}" --worker-id "$${BUILDKITE_PARALLEL_JOB}" --parallelism-per-worker 3
         --build-name servebuild --test-env=EXPECTED_PYTHON_VERSION=3.9 --test-env=RAY_SERVE_RUN_USER_CODE_IN_SEPARATE_THREAD=0
     depends_on: "servebuild"
@@ -82,7 +85,7 @@ steps:
     instance_type: large
     commands:
       - bazel run //ci/ray_ci:test_in_docker -- //python/ray/serve/... //python/ray/tests/...  serve
-        --except-tags post_wheel_build,gpu,ha_integration
+        --except-tags post_wheel_build,gpu,ha_integration,serve_tracing
         --workers 2 --worker-id {{matrix.worker_id}} --parallelism-per-worker 3
         --python-version {{matrix.python}}
         --test-env=EXPECTED_PYTHON_VERSION={{matrix.python}}
@@ -185,6 +188,16 @@ steps:
       - raycpubase
       - servebuild
 
+  - label: ":ray-serve: serve: tracing tests"
+    tags:
+      - serve
+      - python
+    instance_type: medium
+    commands:
+      - bazel run //ci/ray_ci:test_in_docker -- //python/ray/serve/... //python/ray/tests/...  serve
+        --only-tags serve_tracing --build-name servetracingbuild
+    depends_on: servetracingbuild
+
   - label: ":ray-serve: serve: doc gpu tests"
     tags:
       - serve

ci/docker/serve.build.Dockerfile

@@ -3,6 +3,7 @@
 ARG DOCKER_IMAGE_BASE_BUILD=cr.ray.io/rayproject/oss-ci-base_build
 FROM $DOCKER_IMAGE_BASE_BUILD
 
+ARG ENABLE_TRACING
 ARG PYDANTIC_VERSION
 ARG PYTHON
 
@@ -35,4 +36,13 @@ else
   echo "Not installing Pydantic from source"
 fi
 
+if [[ "${ENABLE_TRACING-}" == "1" ]]; then
+  # Install tracing dependencies if requested. Intentionally, we do not use
+  # requirements_compiled.txt as the constraint file. They are not compatible with
+  # a few packages in that file (e.g. requiring an ugprade to protobuf 5+).
+  pip install opentelemetry-exporter-otlp==1.34.1
+else
+  echo "Not installing tracing dependencies"
+fi
+
 EOF

ci/docker/servetracing.build.wanda.yaml

@@ -0,0 +1,11 @@
+name: "servetracingbuild"
+froms: ["cr.ray.io/rayproject/oss-ci-base_build"]
+dockerfile: ci/docker/serve.build.Dockerfile
+srcs:
+  - python/requirements.txt
+  - python/requirements_compiled.txt
+  - python/requirements/test-requirements.txt
+build_args:
+  - ENABLE_TRACING=1
+tags:
+  - cr.ray.io/rayproject/servetracingbuild

python/ray/tests/BUILD

@@ -1226,6 +1226,7 @@ py_test(
         "exclusive",
         "medium_size_python_tests_k_to_z",
         "no_windows",
+        "serve_tracing",
         "team:serve",
     ],
     deps = [

python/requirements.txt

@@ -30,16 +30,16 @@ smart_open
 lz4
 numpy>=1.20
 aiorwlock
-opentelemetry-exporter-otlp
 scipy
 colorful
 rich
-opentelemetry-sdk
+opentelemetry-sdk>=1.30.0
+opentelemetry-api
 opentelemetry-exporter-prometheus
+opentelemetry-proto
 fastapi
 gymnasium==1.0.0
 virtualenv!=20.21.1,>=20.0.24
-opentelemetry-api
 opencensus
 aiohttp_cors
 dm_tree

python/requirements/cloud-requirements.txt

@@ -1,8 +1,5 @@
 jupyterlab==3.6.1
 ipywidgets
-opentelemetry-api
-opentelemetry-sdk
-opentelemetry-exporter-otlp
 google-cloud-storage
 grpcio>=1.66.1
 grpcio-tools

python/requirements/test-requirements.txt

@@ -36,9 +36,8 @@ moto[s3,server]==4.2.12
 mypy==1.7.0
 numba==0.59.1
 openpyxl==3.0.10
-opentelemetry-api==1.26.0
-opentelemetry-sdk==1.26.0
-opentelemetry-exporter-otlp>=1.26.0
+opentelemetry-api
+opentelemetry-sdk
 opencensus-proto==0.1.0
 pexpect==4.8.0
 Pillow==10.3.0; platform_system != "Windows"

python/requirements_compiled.txt

@@ -407,12 +407,6 @@ delta-sharing==1.0.5
     # via -r python/requirements/ml/data-test-requirements.txt
 deltalake==0.9.0
     # via -r python/requirements/ml/data-test-requirements.txt
-deprecated==1.2.18
-    # via
-    #   opentelemetry-api
-    #   opentelemetry-exporter-otlp-proto-grpc
-    #   opentelemetry-exporter-otlp-proto-http
-    #   opentelemetry-semantic-conventions
 dill==0.3.7
     # via
     #   datasets
@@ -652,8 +646,6 @@ googleapis-common-protos==1.61.0
     # via
     #   google-api-core
     #   grpcio-status
-    #   opentelemetry-exporter-otlp-proto-grpc
-    #   opentelemetry-exporter-otlp-proto-http
     #   tensorflow-metadata
 gpy==1.13.1
     # via -r python/requirements/ml/tune-test-requirements.txt
@@ -686,7 +678,6 @@ grpcio==1.66.2 ; sys_platform != "darwin"
     #   grpcio-tools
     #   mlagents-envs
     #   opencensus-proto
-    #   opentelemetry-exporter-otlp-proto-grpc
     #   tensorboard
     #   tensorflow
 grpcio-status==1.48.2
@@ -1310,47 +1301,25 @@ opencv-python-headless==4.9.0.80
     # via -r python/requirements/ml/rllib-test-requirements.txt
 openpyxl==3.0.10
     # via -r python/requirements/test-requirements.txt
-opentelemetry-api==1.26.0
+opentelemetry-api==1.34.1
     # via
     #   -r python/requirements.txt
-    #   -r python/requirements/cloud-requirements.txt
     #   -r python/requirements/test-requirements.txt
     #   mlflow-skinny
-    #   opentelemetry-exporter-otlp-proto-grpc
-    #   opentelemetry-exporter-otlp-proto-http
     #   opentelemetry-exporter-prometheus
     #   opentelemetry-sdk
     #   opentelemetry-semantic-conventions
-opentelemetry-exporter-otlp==1.26.0
-    # via
-    #   -r python/requirements.txt
-    #   -r python/requirements/cloud-requirements.txt
-    #   -r python/requirements/test-requirements.txt
-opentelemetry-exporter-otlp-proto-common==1.26.0
-    # via
-    #   opentelemetry-exporter-otlp-proto-grpc
-    #   opentelemetry-exporter-otlp-proto-http
-opentelemetry-exporter-otlp-proto-grpc==1.26.0
-    # via opentelemetry-exporter-otlp
-opentelemetry-exporter-otlp-proto-http==1.26.0
-    # via opentelemetry-exporter-otlp
-opentelemetry-exporter-prometheus==0.47b0
+opentelemetry-exporter-prometheus==0.55b1
     # via -r python/requirements.txt
-opentelemetry-proto==1.26.0
-    # via
-    #   opentelemetry-exporter-otlp-proto-common
-    #   opentelemetry-exporter-otlp-proto-grpc
-    #   opentelemetry-exporter-otlp-proto-http
-opentelemetry-sdk==1.26.0
+opentelemetry-proto==1.27.0
+    # via -r python/requirements.txt
+opentelemetry-sdk==1.34.1
     # via
     #   -r python/requirements.txt
-    #   -r python/requirements/cloud-requirements.txt
     #   -r python/requirements/test-requirements.txt
     #   mlflow-skinny
-    #   opentelemetry-exporter-otlp-proto-grpc
-    #   opentelemetry-exporter-otlp-proto-http
     #   opentelemetry-exporter-prometheus
-opentelemetry-semantic-conventions==0.47b0
+opentelemetry-semantic-conventions==0.55b1
     # via opentelemetry-sdk
 opt-einsum==3.3.0
     # via
@@ -1885,7 +1854,6 @@ requests==2.32.3
     #   moto
     #   msal
     #   msrest
-    #   opentelemetry-exporter-otlp-proto-http
     #   pyiceberg
     #   ray
     #   requests-oauthlib
@@ -2389,7 +2357,9 @@ typing-extensions==4.12.2
     #   mypy
     #   myst-nb
     #   nevergrad
+    #   opentelemetry-api
     #   opentelemetry-sdk
+    #   opentelemetry-semantic-conventions
     #   pydantic
     #   pydantic-core
     #   pyopenssl
@@ -2510,7 +2480,6 @@ wrapt==1.14.1
     #   aiobotocore
     #   aws-xray-sdk
     #   comet-ml
-    #   deprecated
     #   tensorflow
     #   tensorflow-datasets
 wurlitzer==3.1.1