Skip to content

Commit b2d11f1

Browse files
dubloomdubloom
authored
fix(ray): errors were not set in child spans (#15097)
Error were not set on long running span if one happened (I thought that it was automatically done when we were leaving the context).
1 parent 5f8d50f commit b2d11f1

File tree

4 files changed

+159
-2
lines changed

4 files changed

+159
-2
lines changed

‎ddtrace/contrib/internal/ray/span_manager.py‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
import atexit
22
from contextlib import contextmanager
33
from itertools import chain
4+
import sys
45
import threading
56
import time
67
from typing import Dict
@@ -54,8 +55,9 @@ def long_running_ray_span(
5455

5556
try:
5657
yield span
57-
except BaseException as e:
58-
raise e
58+
except BaseException:
59+
span.set_exc_info(*sys.exc_info())
60+
raise
5961
finally:
6062
stop_long_running_span(span)
6163

‎releasenotes/notes/fix-ray-no-error-on-child-spans-beb6239ab269b0d9.yaml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
---
2+
fixes:
3+
- |
4+
ray: This fix resolves an issue where exceptions raised in Ray child spans were not properly recorded in the trace.

‎tests/contrib/ray/test_ray.py‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -66,6 +66,17 @@ def add_one(x):
6666
results = ray.get(futures)
6767
assert results == [1, 2], f"Unexpected results: {results}"
6868

69+
@pytest.mark.snapshot(token="tests.contrib.ray.test_ray.test_task_error", ignores=RAY_SNAPSHOT_IGNORES)
70+
def test_task_error(self):
71+
@ray.remote
72+
def add_one(x):
73+
raise ValueError("foo")
74+
return 0
75+
76+
futures = [add_one.remote(i) for i in range(2)] # Reduced from 4 to 2 tasks
77+
with pytest.raises(ValueError):
78+
ray.get(futures)
79+
6980
@pytest.mark.snapshot(token="tests.contrib.ray.test_ray.test_simple_actor", ignores=RAY_SNAPSHOT_IGNORES)
7081
def test_simple_actor(self):
7182
@ray.remote

‎tests/snapshots/tests.contrib.ray.test_ray.test_task_error.json‎

Lines changed: 140 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,140 @@
1+
[[
2+
{
3+
"name": "task.submit",
4+
"service": "tests.contrib.ray",
5+
"resource": "tests.contrib.ray.test_ray.add_one.remote",
6+
"trace_id": 0,
7+
"span_id": 1,
8+
"parent_id": 0,
9+
"type": "ray",
10+
"error": 0,
11+
"meta": {
12+
"_dd.p.dm": "-0",
13+
"_dd.p.tid": "6904f51400000000",
14+
"component": "ray",
15+
"language": "python",
16+
"ray.hostname": "docker-desktop",
17+
"ray.job_id": "01000000",
18+
"ray.node_id": "629c5196ecd37d39eb26d139f7629609aa8991c7eac7f8a364b552c4",
19+
"ray.task.submit_status": "success",
20+
"ray.worker_id": "01000000ffffffffffffffffffffffffffffffffffffffffffffffff",
21+
"runtime-id": "f3eca0ec3aa740d9b664919291660d03",
22+
"span.kind": "producer"
23+
},
24+
"metrics": {
25+
"_dd.ai_obs.enabled": 1,
26+
"_dd.djm.enabled": 1,
27+
"_dd.filter.kept": 1,
28+
"_dd.measured": 1,
29+
"_dd.top_level": 1,
30+
"_dd.tracer_kr": 1.0,
31+
"_sampling_priority_v1": 2,
32+
"process_id": 605
33+
},
34+
"duration": 13672958,
35+
"start": 1761932564841285385
36+
},
37+
{
38+
"name": "task.execute",
39+
"service": "tests.contrib.ray",
40+
"resource": "tests.contrib.ray.test_ray.TestRayIntegration.test_task_error.<locals>.add_one",
41+
"trace_id": 0,
42+
"span_id": 2,
43+
"parent_id": 1,
44+
"type": "ray",
45+
"error": 1,
46+
"meta": {
47+
"component": "ray",
48+
"error.message": "foo",
49+
"error.stack": "Traceback (most recent call last):\n File \"/home/bits/project/ddtrace/contrib/internal/ray/span_manager.py\", line 57, in long_running_ray_span\n yield span\n File \"/home/bits/project/ddtrace/contrib/internal/ray/patch.py\", line 131, in _wrap_task_execution\n result = wrapped(*args, **kwargs)\n File \"/home/bits/project/tests/contrib/ray/test_ray.py\", line 73, in add_one\n raise ValueError(\"foo\")\nValueError: foo\n",
50+
"error.type": "builtins.ValueError",
51+
"ray.hostname": "docker-desktop",
52+
"ray.job_id": "01000000",
53+
"ray.node_id": "629c5196ecd37d39eb26d139f7629609aa8991c7eac7f8a364b552c4",
54+
"ray.task.status": "error",
55+
"ray.worker_id": "01000000ffffffffffffffffffffffffffffffffffffffffffffffff",
56+
"runtime-id": "f3eca0ec3aa740d9b664919291660d03",
57+
"span.kind": "consumer"
58+
},
59+
"metrics": {
60+
"_dd.ai_obs.enabled": 1,
61+
"_dd.djm.enabled": 1,
62+
"_dd.filter.kept": 1,
63+
"_dd.measured": 1,
64+
"_dd.top_level": 1,
65+
"_sampling_priority_v1": 2,
66+
"process_id": 605
67+
},
68+
"duration": 2393708,
69+
"start": 1761932564850504510
70+
}],
71+
[
72+
{
73+
"name": "task.submit",
74+
"service": "tests.contrib.ray",
75+
"resource": "tests.contrib.ray.test_ray.add_one.remote",
76+
"trace_id": 1,
77+
"span_id": 1,
78+
"parent_id": 0,
79+
"type": "ray",
80+
"error": 0,
81+
"meta": {
82+
"_dd.p.dm": "-0",
83+
"_dd.p.tid": "6904f51400000000",
84+
"component": "ray",
85+
"language": "python",
86+
"ray.hostname": "docker-desktop",
87+
"ray.job_id": "01000000",
88+
"ray.node_id": "629c5196ecd37d39eb26d139f7629609aa8991c7eac7f8a364b552c4",
89+
"ray.task.submit_status": "success",
90+
"ray.worker_id": "01000000ffffffffffffffffffffffffffffffffffffffffffffffff",
91+
"runtime-id": "f3eca0ec3aa740d9b664919291660d03",
92+
"span.kind": "producer"
93+
},
94+
"metrics": {
95+
"_dd.ai_obs.enabled": 1,
96+
"_dd.djm.enabled": 1,
97+
"_dd.filter.kept": 1,
98+
"_dd.measured": 1,
99+
"_dd.top_level": 1,
100+
"_dd.tracer_kr": 1.0,
101+
"_sampling_priority_v1": 2,
102+
"process_id": 605
103+
},
104+
"duration": 3147833,
105+
"start": 1761932564855450135
106+
},
107+
{
108+
"name": "task.execute",
109+
"service": "tests.contrib.ray",
110+
"resource": "tests.contrib.ray.test_ray.TestRayIntegration.test_task_error.<locals>.add_one",
111+
"trace_id": 1,
112+
"span_id": 2,
113+
"parent_id": 1,
114+
"type": "ray",
115+
"error": 1,
116+
"meta": {
117+
"component": "ray",
118+
"error.message": "foo",
119+
"error.stack": "Traceback (most recent call last):\n File \"/home/bits/project/ddtrace/contrib/internal/ray/span_manager.py\", line 57, in long_running_ray_span\n yield span\n File \"/home/bits/project/ddtrace/contrib/internal/ray/patch.py\", line 131, in _wrap_task_execution\n result = wrapped(*args, **kwargs)\n File \"/home/bits/project/tests/contrib/ray/test_ray.py\", line 73, in add_one\n raise ValueError(\"foo\")\nValueError: foo\n",
120+
"error.type": "builtins.ValueError",
121+
"ray.hostname": "docker-desktop",
122+
"ray.job_id": "01000000",
123+
"ray.node_id": "629c5196ecd37d39eb26d139f7629609aa8991c7eac7f8a364b552c4",
124+
"ray.task.status": "error",
125+
"ray.worker_id": "01000000ffffffffffffffffffffffffffffffffffffffffffffffff",
126+
"runtime-id": "f3eca0ec3aa740d9b664919291660d03",
127+
"span.kind": "consumer"
128+
},
129+
"metrics": {
130+
"_dd.ai_obs.enabled": 1,
131+
"_dd.djm.enabled": 1,
132+
"_dd.filter.kept": 1,
133+
"_dd.measured": 1,
134+
"_dd.top_level": 1,
135+
"_sampling_priority_v1": 2,
136+
"process_id": 605
137+
},
138+
"duration": 789333,
139+
"start": 1761932564856092760
140+
}]]

0 commit comments

Comments
 (0)