From ae8e359d9585f197bf010fa276e82c63fc34a296 Mon Sep 17 00:00:00 2001 From: Simon Hellmayr Date: Fri, 3 Oct 2025 14:41:45 +0200 Subject: [PATCH 01/13] fix(ai): add mapping for gen_ai message roles --- sentry_sdk/ai/__init__.py | 7 ++ sentry_sdk/ai/utils.py | 47 ++++++++++++ sentry_sdk/integrations/anthropic.py | 13 +++- sentry_sdk/integrations/langgraph.py | 10 ++- sentry_sdk/integrations/openai.py | 6 +- .../openai_agents/spans/invoke_agent.py | 13 +++- .../integrations/openai_agents/utils.py | 19 +++-- .../integrations/anthropic/test_anthropic.py | 68 ++++++++++++++++++ .../integrations/langgraph/test_langgraph.py | 71 +++++++++++++++++++ tests/integrations/openai/test_openai.py | 53 ++++++++++++++ .../openai_agents/test_openai_agents.py | 46 ++++++++++++ 11 files changed, 340 insertions(+), 13 deletions(-) diff --git a/sentry_sdk/ai/__init__.py b/sentry_sdk/ai/__init__.py index e69de29bb2..fbcb9c061d 100644 --- a/sentry_sdk/ai/__init__.py +++ b/sentry_sdk/ai/__init__.py @@ -0,0 +1,7 @@ +from .utils import ( + set_data_normalized, + GEN_AI_MESSAGE_ROLE_MAPPING, + GEN_AI_MESSAGE_ROLE_REVERSE_MAPPING, + normalize_message_role, + normalize_message_roles, +) # noqa: F401 diff --git a/sentry_sdk/ai/utils.py b/sentry_sdk/ai/utils.py index d0ccf1bed3..6feb410b60 100644 --- a/sentry_sdk/ai/utils.py +++ b/sentry_sdk/ai/utils.py @@ -10,6 +10,21 @@ from sentry_sdk.utils import logger +# Gen AI message role reverse mapping showing allowed target roles and their source variants +GEN_AI_MESSAGE_ROLE_REVERSE_MAPPING = { + "system": ["system"], + "user": ["user"], + "assistant": ["assistant", "ai"], + "tool_call": ["tool_call"], +} + +# Convert reverse mapping to actual mapping for efficient lookups +GEN_AI_MESSAGE_ROLE_MAPPING = {} +for target_role, source_roles in GEN_AI_MESSAGE_ROLE_REVERSE_MAPPING.items(): + for source_role in source_roles: + GEN_AI_MESSAGE_ROLE_MAPPING[source_role] = target_role + + def _normalize_data(data, unpack=True): # type: (Any, bool) -> Any # convert pydantic data (e.g. OpenAI v1+) to json compatible format @@ -40,6 +55,38 @@ def set_data_normalized(span, key, value, unpack=True): span.set_data(key, json.dumps(normalized)) +def normalize_message_role(role): + # type: (str) -> str + """ + Normalize a message role to one of the 4 allowed gen_ai role values. + Maps "ai" -> "assistant" and keeps other standard roles unchanged. + """ + return GEN_AI_MESSAGE_ROLE_MAPPING.get(role, role) + + +def normalize_message_roles(messages): + # type: (list[dict[str, Any]]) -> list[dict[str, Any]] + """ + Normalize roles in a list of messages to use standard gen_ai role values. + Creates a deep copy to avoid modifying the original messages. + """ + if not messages: + return messages + + normalized_messages = [] + for message in messages: + if not isinstance(message, dict): + normalized_messages.append(message) + continue + + normalized_message = message.copy() + if "role" in message: + normalized_message["role"] = normalize_message_role(message["role"]) + normalized_messages.append(normalized_message) + + return normalized_messages + + def get_start_span_function(): # type: () -> Callable[..., Any] current_span = sentry_sdk.get_current_span() diff --git a/sentry_sdk/integrations/anthropic.py b/sentry_sdk/integrations/anthropic.py index d9898fa1d1..5bbfafa787 100644 --- a/sentry_sdk/integrations/anthropic.py +++ b/sentry_sdk/integrations/anthropic.py @@ -3,7 +3,11 @@ import sentry_sdk from sentry_sdk.ai.monitoring import record_token_usage -from sentry_sdk.ai.utils import set_data_normalized, get_start_span_function +from sentry_sdk.ai.utils import ( + set_data_normalized, + normalize_message_roles, + get_start_span_function, +) from sentry_sdk.consts import OP, SPANDATA, SPANSTATUS from sentry_sdk.integrations import _check_minimum_version, DidNotEnable, Integration from sentry_sdk.scope import should_send_default_pii @@ -140,8 +144,13 @@ def _set_input_data(span, kwargs, integration): else: normalized_messages.append(message) + # Further normalize message roles to standard gen_ai values + role_normalized_messages = normalize_message_roles(normalized_messages) set_data_normalized( - span, SPANDATA.GEN_AI_REQUEST_MESSAGES, normalized_messages, unpack=False + span, + SPANDATA.GEN_AI_REQUEST_MESSAGES, + role_normalized_messages, + unpack=False, ) set_data_normalized( diff --git a/sentry_sdk/integrations/langgraph.py b/sentry_sdk/integrations/langgraph.py index df3941bb13..befd58d033 100644 --- a/sentry_sdk/integrations/langgraph.py +++ b/sentry_sdk/integrations/langgraph.py @@ -2,7 +2,7 @@ from typing import Any, Callable, List, Optional import sentry_sdk -from sentry_sdk.ai.utils import set_data_normalized +from sentry_sdk.ai.utils import set_data_normalized, normalize_message_roles from sentry_sdk.consts import OP, SPANDATA from sentry_sdk.integrations import DidNotEnable, Integration from sentry_sdk.scope import should_send_default_pii @@ -180,10 +180,12 @@ def new_invoke(self, *args, **kwargs): ): input_messages = _parse_langgraph_messages(args[0]) if input_messages: + # Normalize message roles to standard gen_ai values + normalized_input_messages = normalize_message_roles(input_messages) set_data_normalized( span, SPANDATA.GEN_AI_REQUEST_MESSAGES, - input_messages, + normalized_input_messages, unpack=False, ) @@ -230,10 +232,12 @@ async def new_ainvoke(self, *args, **kwargs): ): input_messages = _parse_langgraph_messages(args[0]) if input_messages: + # Normalize message roles to standard gen_ai values + normalized_input_messages = normalize_message_roles(input_messages) set_data_normalized( span, SPANDATA.GEN_AI_REQUEST_MESSAGES, - input_messages, + normalized_input_messages, unpack=False, ) diff --git a/sentry_sdk/integrations/openai.py b/sentry_sdk/integrations/openai.py index e8b3b30ab2..7922292818 100644 --- a/sentry_sdk/integrations/openai.py +++ b/sentry_sdk/integrations/openai.py @@ -3,7 +3,7 @@ import sentry_sdk from sentry_sdk import consts from sentry_sdk.ai.monitoring import record_token_usage -from sentry_sdk.ai.utils import set_data_normalized +from sentry_sdk.ai.utils import set_data_normalized, normalize_message_roles from sentry_sdk.consts import SPANDATA from sentry_sdk.integrations import DidNotEnable, Integration from sentry_sdk.scope import should_send_default_pii @@ -182,8 +182,10 @@ def _set_input_data(span, kwargs, operation, integration): and should_send_default_pii() and integration.include_prompts ): + # Normalize message roles to standard gen_ai values + normalized_messages = normalize_message_roles(messages) set_data_normalized( - span, SPANDATA.GEN_AI_REQUEST_MESSAGES, messages, unpack=False + span, SPANDATA.GEN_AI_REQUEST_MESSAGES, normalized_messages, unpack=False ) # Input attributes: Common diff --git a/sentry_sdk/integrations/openai_agents/spans/invoke_agent.py b/sentry_sdk/integrations/openai_agents/spans/invoke_agent.py index cf06120625..c845e4574e 100644 --- a/sentry_sdk/integrations/openai_agents/spans/invoke_agent.py +++ b/sentry_sdk/integrations/openai_agents/spans/invoke_agent.py @@ -1,5 +1,9 @@ import sentry_sdk -from sentry_sdk.ai.utils import get_start_span_function, set_data_normalized +from sentry_sdk.ai.utils import ( + get_start_span_function, + set_data_normalized, + normalize_message_roles, +) from sentry_sdk.consts import OP, SPANDATA from sentry_sdk.scope import should_send_default_pii from sentry_sdk.utils import safe_serialize @@ -56,8 +60,13 @@ def invoke_agent_span(context, agent, kwargs): ) if len(messages) > 0: + # Normalize message roles to standard gen_ai values + normalized_messages = normalize_message_roles(messages) set_data_normalized( - span, SPANDATA.GEN_AI_REQUEST_MESSAGES, messages, unpack=False + span, + SPANDATA.GEN_AI_REQUEST_MESSAGES, + normalized_messages, + unpack=False, ) _set_agent_data(span, agent) diff --git a/sentry_sdk/integrations/openai_agents/utils.py b/sentry_sdk/integrations/openai_agents/utils.py index b0ad6bf903..3763f0c2e4 100644 --- a/sentry_sdk/integrations/openai_agents/utils.py +++ b/sentry_sdk/integrations/openai_agents/utils.py @@ -1,5 +1,5 @@ import sentry_sdk -from sentry_sdk.ai.utils import set_data_normalized +from sentry_sdk.ai.utils import set_data_normalized, normalize_message_role from sentry_sdk.consts import SPANDATA, SPANSTATUS, OP from sentry_sdk.integrations import DidNotEnable from sentry_sdk.scope import should_send_default_pii @@ -107,9 +107,20 @@ def _set_input_data(span, get_response_kwargs): for message in get_response_kwargs.get("input", []): if "role" in message: - messages_by_role[message.get("role")].append( - {"type": "text", "text": message.get("content")} - ) + # Normalize role to standard gen_ai values + normalized_role = normalize_message_role(message.get("role")) + # Ensure we have a valid bucket for the normalized role + if normalized_role not in messages_by_role: + # Map any unrecognized roles to appropriate defaults + if normalized_role == "tool_call": + normalized_role = ( + "tool" # OpenAI Agents uses "tool" instead of "tool_call" + ) + + if normalized_role in messages_by_role: + messages_by_role[normalized_role].append( + {"type": "text", "text": message.get("content")} + ) else: if message.get("type") == "function_call": messages_by_role["assistant"].append(message) diff --git a/tests/integrations/anthropic/test_anthropic.py b/tests/integrations/anthropic/test_anthropic.py index 04ff12eb8b..4510eed3a0 100644 --- a/tests/integrations/anthropic/test_anthropic.py +++ b/tests/integrations/anthropic/test_anthropic.py @@ -878,3 +878,71 @@ def test_set_output_data_with_input_json_delta(sentry_init): assert span._data.get(SPANDATA.GEN_AI_USAGE_INPUT_TOKENS) == 10 assert span._data.get(SPANDATA.GEN_AI_USAGE_OUTPUT_TOKENS) == 20 assert span._data.get(SPANDATA.GEN_AI_USAGE_TOTAL_TOKENS) == 30 + + +def test_anthropic_message_role_mapping(sentry_init, capture_events): + """Test that Anthropic integration properly maps message roles like 'ai' to 'assistant'""" + sentry_init( + integrations=[AnthropicIntegration(include_prompts=True)], + traces_sample_rate=1.0, + send_default_pii=True, + ) + events = capture_events() + + client = Anthropic(api_key="z") + + def mock_messages_create(*args, **kwargs): + return Message( + id="msg_1", + content=[TextBlock(text="Hi there!", type="text")], + model="claude-3-opus", + role="assistant", + stop_reason="end_turn", + stop_sequence=None, + type="message", + usage=Usage(input_tokens=10, output_tokens=5), + ) + + client.messages._post = mock.Mock(return_value=mock_messages_create()) + + # Test messages with mixed roles including "ai" that should be mapped to "assistant" + test_messages = [ + {"role": "system", "content": "You are helpful."}, + {"role": "user", "content": "Hello"}, + {"role": "ai", "content": "Hi there!"}, # Should be mapped to "assistant" + {"role": "assistant", "content": "How can I help?"}, # Should stay "assistant" + ] + + with start_transaction(name="anthropic tx"): + client.messages.create( + model="claude-3-opus", max_tokens=10, messages=test_messages + ) + + (event,) = events + span = event["spans"][0] + + # Verify that the span was created correctly + assert span["op"] == "gen_ai.chat" + assert SPANDATA.GEN_AI_REQUEST_MESSAGES in span["data"] + + # Parse the stored messages + import json + + stored_messages = json.loads(span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES]) + + # Verify that "ai" role was mapped to "assistant" + assert len(stored_messages) == 4 + assert stored_messages[0]["role"] == "system" + assert stored_messages[1]["role"] == "user" + assert ( + stored_messages[2]["role"] == "assistant" + ) # "ai" should be mapped to "assistant" + assert stored_messages[3]["role"] == "assistant" # should stay "assistant" + + # Verify content is preserved + assert stored_messages[2]["content"] == "Hi there!" + assert stored_messages[3]["content"] == "How can I help?" + + # Verify no "ai" roles remain + roles = [msg["role"] for msg in stored_messages] + assert "ai" not in roles diff --git a/tests/integrations/langgraph/test_langgraph.py b/tests/integrations/langgraph/test_langgraph.py index 1510305b06..6ec6d9a96d 100644 --- a/tests/integrations/langgraph/test_langgraph.py +++ b/tests/integrations/langgraph/test_langgraph.py @@ -625,3 +625,74 @@ def original_invoke(self, *args, **kwargs): assert tool_calls_data[0]["function"]["name"] == "search" assert tool_calls_data[1]["id"] == "call_multi_2" assert tool_calls_data[1]["function"]["name"] == "calculate" + + +def test_langgraph_message_role_mapping(sentry_init, capture_events): + """Test that Langgraph integration properly maps message roles like 'ai' to 'assistant'""" + sentry_init( + integrations=[LanggraphIntegration(include_prompts=True)], + traces_sample_rate=1.0, + send_default_pii=True, + ) + events = capture_events() + + # Mock a langgraph message with mixed roles + class MockMessage: + def __init__(self, content, message_type="human"): + self.content = content + self.type = message_type + + # Create mock state with messages having different roles + state_data = { + "messages": [ + MockMessage("System prompt", "system"), + MockMessage("Hello", "human"), + MockMessage("Hi there!", "ai"), # Should be mapped to "assistant" + MockMessage("How can I help?", "assistant"), # Should stay "assistant" + ] + } + + compiled_graph = MockCompiledGraph("test_graph") + pregel = MockPregelInstance(compiled_graph) + + with start_transaction(name="langgraph tx"): + # Use the wrapped invoke function directly + from sentry_sdk.integrations.langgraph import _wrap_pregel_invoke + + wrapped_invoke = _wrap_pregel_invoke( + lambda self, state_data: {"result": "success"} + ) + wrapped_invoke(pregel, state_data) + + (event,) = events + span = event["spans"][0] + + # Verify that the span was created correctly + assert span["op"] == "gen_ai.invoke_agent" + + # If messages were captured, verify role mapping + if SPANDATA.GEN_AI_REQUEST_MESSAGES in span["data"]: + import json + + stored_messages = json.loads(span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES]) + + # Find messages with specific content to verify role mapping + ai_message = next( + (msg for msg in stored_messages if msg.get("content") == "Hi there!"), None + ) + assistant_message = next( + (msg for msg in stored_messages if msg.get("content") == "How can I help?"), + None, + ) + + if ai_message: + # "ai" should have been mapped to "assistant" + assert ai_message["role"] == "assistant" + + if assistant_message: + # "assistant" should stay "assistant" + assert assistant_message["role"] == "assistant" + + # Verify no "ai" roles remain + roles = [msg["role"] for msg in stored_messages if "role" in msg] + assert "ai" not in roles diff --git a/tests/integrations/openai/test_openai.py b/tests/integrations/openai/test_openai.py index e7fbf8a7d8..06e0a09fcf 100644 --- a/tests/integrations/openai/test_openai.py +++ b/tests/integrations/openai/test_openai.py @@ -1447,3 +1447,56 @@ def test_empty_tools_in_chat_completion(sentry_init, capture_events, tools): span = event["spans"][0] assert "gen_ai.request.available_tools" not in span["data"] + + +def test_openai_message_role_mapping(sentry_init, capture_events): + """Test that OpenAI integration properly maps message roles like 'ai' to 'assistant'""" + sentry_init( + integrations=[OpenAIIntegration(include_prompts=True)], + traces_sample_rate=1.0, + send_default_pii=True, + ) + events = capture_events() + + client = OpenAI(api_key="z") + client.chat.completions._post = mock.Mock(return_value=EXAMPLE_CHAT_COMPLETION) + + # Test messages with mixed roles including "ai" that should be mapped to "assistant" + test_messages = [ + {"role": "system", "content": "You are helpful."}, + {"role": "user", "content": "Hello"}, + {"role": "ai", "content": "Hi there!"}, # Should be mapped to "assistant" + {"role": "assistant", "content": "How can I help?"}, # Should stay "assistant" + ] + + with start_transaction(name="openai tx"): + client.chat.completions.create(model="test-model", messages=test_messages) + + (event,) = events + span = event["spans"][0] + + # Verify that the span was created correctly + assert span["op"] == "gen_ai.chat" + assert SPANDATA.GEN_AI_REQUEST_MESSAGES in span["data"] + + # Parse the stored messages + import json + + stored_messages = json.loads(span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES]) + + # Verify that "ai" role was mapped to "assistant" + assert len(stored_messages) == 4 + assert stored_messages[0]["role"] == "system" + assert stored_messages[1]["role"] == "user" + assert ( + stored_messages[2]["role"] == "assistant" + ) # "ai" should be mapped to "assistant" + assert stored_messages[3]["role"] == "assistant" # should stay "assistant" + + # Verify content is preserved + assert stored_messages[2]["content"] == "Hi there!" + assert stored_messages[3]["content"] == "How can I help?" + + # Verify no "ai" roles remain + roles = [msg["role"] for msg in stored_messages] + assert "ai" not in roles diff --git a/tests/integrations/openai_agents/test_openai_agents.py b/tests/integrations/openai_agents/test_openai_agents.py index e9a8372806..e647ce9fad 100644 --- a/tests/integrations/openai_agents/test_openai_agents.py +++ b/tests/integrations/openai_agents/test_openai_agents.py @@ -1031,3 +1031,49 @@ async def run(): assert txn2["transaction"] == "test_agent workflow" assert txn3["type"] == "transaction" assert txn3["transaction"] == "test_agent workflow" + + +def test_openai_agents_message_role_mapping(sentry_init, capture_events): + """Test that OpenAI Agents integration properly maps message roles like 'ai' to 'assistant'""" + sentry_init( + integrations=[OpenAIAgentsIntegration()], + traces_sample_rate=1.0, + send_default_pii=True, + ) + + # Test input messages with mixed roles including "ai" + test_input = [ + {"role": "system", "content": "You are helpful."}, + {"role": "user", "content": "Hello"}, + {"role": "ai", "content": "Hi there!"}, # Should be mapped to "assistant" + {"role": "assistant", "content": "How can I help?"}, # Should stay "assistant" + ] + + get_response_kwargs = {"input": test_input} + + from sentry_sdk.integrations.openai_agents.utils import _set_input_data + from sentry_sdk import start_span + + with start_span(op="test") as span: + _set_input_data(span, get_response_kwargs) + + # Verify that messages were processed and roles were mapped + from sentry_sdk.consts import SPANDATA + + if SPANDATA.GEN_AI_REQUEST_MESSAGES in span._data: + import json + + stored_messages = json.loads(span._data[SPANDATA.GEN_AI_REQUEST_MESSAGES]) + + # Verify roles were properly mapped + found_assistant_roles = 0 + for message in stored_messages: + if message["role"] == "assistant": + found_assistant_roles += 1 + + # Should have 2 assistant roles (1 from original "assistant", 1 from mapped "ai") + assert found_assistant_roles == 2 + + # Verify no "ai" roles remain in any message + for message in stored_messages: + assert message["role"] != "ai" From f680921002c14382131e07b675a3f23d02c7aeed Mon Sep 17 00:00:00 2001 From: Simon Hellmayr Date: Tue, 7 Oct 2025 10:56:02 +0200 Subject: [PATCH 02/13] map openai correctly --- .../integrations/openai_agents/utils.py | 42 +++++++------------ 1 file changed, 15 insertions(+), 27 deletions(-) diff --git a/sentry_sdk/integrations/openai_agents/utils.py b/sentry_sdk/integrations/openai_agents/utils.py index 3763f0c2e4..5691c5589c 100644 --- a/sentry_sdk/integrations/openai_agents/utils.py +++ b/sentry_sdk/integrations/openai_agents/utils.py @@ -94,43 +94,31 @@ def _set_input_data(span, get_response_kwargs): # type: (sentry_sdk.tracing.Span, dict[str, Any]) -> None if not should_send_default_pii(): return + request_messages = [] - messages_by_role = { - "system": [], - "user": [], - "assistant": [], - "tool": [], - } # type: (dict[str, list[Any]]) system_instructions = get_response_kwargs.get("system_instructions") if system_instructions: - messages_by_role["system"].append({"type": "text", "text": system_instructions}) + request_messages.append( + { + "role": "system", + "content": [{"type": "text", "text": system_instructions}], + } + ) for message in get_response_kwargs.get("input", []): if "role" in message: - # Normalize role to standard gen_ai values normalized_role = normalize_message_role(message.get("role")) - # Ensure we have a valid bucket for the normalized role - if normalized_role not in messages_by_role: - # Map any unrecognized roles to appropriate defaults - if normalized_role == "tool_call": - normalized_role = ( - "tool" # OpenAI Agents uses "tool" instead of "tool_call" - ) - - if normalized_role in messages_by_role: - messages_by_role[normalized_role].append( - {"type": "text", "text": message.get("content")} - ) + request_messages.append( + { + "role": normalized_role, + "content": [{"type": "text", "text": message.get("content")}], + } + ) else: if message.get("type") == "function_call": - messages_by_role["assistant"].append(message) + request_messages.append({"role": "assistant", "content": [message]}) elif message.get("type") == "function_call_output": - messages_by_role["tool"].append(message) - - request_messages = [] - for role, messages in messages_by_role.items(): - if len(messages) > 0: - request_messages.append({"role": role, "content": messages}) + request_messages.append({"role": "tool", "content": [message]}) set_data_normalized( span, SPANDATA.GEN_AI_REQUEST_MESSAGES, request_messages, unpack=False From 906da3d622f117a4ed13bc66eb816c2dec40ac54 Mon Sep 17 00:00:00 2001 From: Simon Hellmayr Date: Tue, 7 Oct 2025 13:05:52 +0200 Subject: [PATCH 03/13] map openai --- sentry_sdk/ai/utils.py | 18 +++++++++--------- sentry_sdk/integrations/openai_agents/utils.py | 11 +++++++++-- 2 files changed, 18 insertions(+), 11 deletions(-) diff --git a/sentry_sdk/ai/utils.py b/sentry_sdk/ai/utils.py index 6feb410b60..f387ff6cb9 100644 --- a/sentry_sdk/ai/utils.py +++ b/sentry_sdk/ai/utils.py @@ -10,10 +10,17 @@ from sentry_sdk.utils import logger +class GEN_AI_ALLOWED_MESSAGE_ROLES: + SYSTEM = "system" + USER = "user" + ASSISTANT = "assistant" + TOOL_CALL = "tool_call" + + # Gen AI message role reverse mapping showing allowed target roles and their source variants GEN_AI_MESSAGE_ROLE_REVERSE_MAPPING = { - "system": ["system"], - "user": ["user"], + GEN_AI_ALLOWED_MESSAGE_ROLES.SYSTEM: ["system"], + GEN_AI_ALLOWED_MESSAGE_ROLES.USER: ["user"], "assistant": ["assistant", "ai"], "tool_call": ["tool_call"], } @@ -70,15 +77,8 @@ def normalize_message_roles(messages): Normalize roles in a list of messages to use standard gen_ai role values. Creates a deep copy to avoid modifying the original messages. """ - if not messages: - return messages - normalized_messages = [] for message in messages: - if not isinstance(message, dict): - normalized_messages.append(message) - continue - normalized_message = message.copy() if "role" in message: normalized_message["role"] = normalize_message_role(message["role"]) diff --git a/sentry_sdk/integrations/openai_agents/utils.py b/sentry_sdk/integrations/openai_agents/utils.py index 5691c5589c..1e703a69d4 100644 --- a/sentry_sdk/integrations/openai_agents/utils.py +++ b/sentry_sdk/integrations/openai_agents/utils.py @@ -1,5 +1,9 @@ import sentry_sdk -from sentry_sdk.ai.utils import set_data_normalized, normalize_message_role +from sentry_sdk.ai.utils import ( + normalize_message_roles, + set_data_normalized, + normalize_message_role, +) from sentry_sdk.consts import SPANDATA, SPANSTATUS, OP from sentry_sdk.integrations import DidNotEnable from sentry_sdk.scope import should_send_default_pii @@ -121,7 +125,10 @@ def _set_input_data(span, get_response_kwargs): request_messages.append({"role": "tool", "content": [message]}) set_data_normalized( - span, SPANDATA.GEN_AI_REQUEST_MESSAGES, request_messages, unpack=False + span, + SPANDATA.GEN_AI_REQUEST_MESSAGES, + normalize_message_roles(request_messages), + unpack=False, ) From 0081cf87733b41145b9c74177aef25d2c2bd0904 Mon Sep 17 00:00:00 2001 From: Simon Hellmayr Date: Tue, 7 Oct 2025 13:24:13 +0200 Subject: [PATCH 04/13] don't try to convert messages that don't have a role --- sentry_sdk/ai/utils.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/sentry_sdk/ai/utils.py b/sentry_sdk/ai/utils.py index f387ff6cb9..60969607d5 100644 --- a/sentry_sdk/ai/utils.py +++ b/sentry_sdk/ai/utils.py @@ -79,6 +79,9 @@ def normalize_message_roles(messages): """ normalized_messages = [] for message in messages: + if not isinstance(message, dict): + normalized_messages.append(message) + continue normalized_message = message.copy() if "role" in message: normalized_message["role"] = normalize_message_role(message["role"]) From e7bf10545980348180eeeae4e54a89b011749037 Mon Sep 17 00:00:00 2001 From: Simon Hellmayr Date: Tue, 7 Oct 2025 13:26:42 +0200 Subject: [PATCH 05/13] constant --- sentry_sdk/ai/utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sentry_sdk/ai/utils.py b/sentry_sdk/ai/utils.py index 60969607d5..b034376fe7 100644 --- a/sentry_sdk/ai/utils.py +++ b/sentry_sdk/ai/utils.py @@ -21,8 +21,8 @@ class GEN_AI_ALLOWED_MESSAGE_ROLES: GEN_AI_MESSAGE_ROLE_REVERSE_MAPPING = { GEN_AI_ALLOWED_MESSAGE_ROLES.SYSTEM: ["system"], GEN_AI_ALLOWED_MESSAGE_ROLES.USER: ["user"], - "assistant": ["assistant", "ai"], - "tool_call": ["tool_call"], + GEN_AI_ALLOWED_MESSAGE_ROLES.ASSISTANT: ["assistant", "ai"], + GEN_AI_ALLOWED_MESSAGE_ROLES.TOOL_CALL: ["tool_call"], } # Convert reverse mapping to actual mapping for efficient lookups From ab4dec0145957180badbe716f88fe8224575ae41 Mon Sep 17 00:00:00 2001 From: Simon Hellmayr Date: Tue, 7 Oct 2025 13:53:16 +0200 Subject: [PATCH 06/13] remove superfluous comments and use the constants everywhere --- sentry_sdk/ai/utils.py | 2 -- sentry_sdk/integrations/anthropic.py | 1 - sentry_sdk/integrations/langgraph.py | 2 -- sentry_sdk/integrations/openai.py | 1 - .../openai_agents/spans/invoke_agent.py | 1 - sentry_sdk/integrations/openai_agents/utils.py | 17 ++++++++++++++--- 6 files changed, 14 insertions(+), 10 deletions(-) diff --git a/sentry_sdk/ai/utils.py b/sentry_sdk/ai/utils.py index b034376fe7..29f82f5c18 100644 --- a/sentry_sdk/ai/utils.py +++ b/sentry_sdk/ai/utils.py @@ -17,7 +17,6 @@ class GEN_AI_ALLOWED_MESSAGE_ROLES: TOOL_CALL = "tool_call" -# Gen AI message role reverse mapping showing allowed target roles and their source variants GEN_AI_MESSAGE_ROLE_REVERSE_MAPPING = { GEN_AI_ALLOWED_MESSAGE_ROLES.SYSTEM: ["system"], GEN_AI_ALLOWED_MESSAGE_ROLES.USER: ["user"], @@ -25,7 +24,6 @@ class GEN_AI_ALLOWED_MESSAGE_ROLES: GEN_AI_ALLOWED_MESSAGE_ROLES.TOOL_CALL: ["tool_call"], } -# Convert reverse mapping to actual mapping for efficient lookups GEN_AI_MESSAGE_ROLE_MAPPING = {} for target_role, source_roles in GEN_AI_MESSAGE_ROLE_REVERSE_MAPPING.items(): for source_role in source_roles: diff --git a/sentry_sdk/integrations/anthropic.py b/sentry_sdk/integrations/anthropic.py index 5bbfafa787..46c6b2a766 100644 --- a/sentry_sdk/integrations/anthropic.py +++ b/sentry_sdk/integrations/anthropic.py @@ -144,7 +144,6 @@ def _set_input_data(span, kwargs, integration): else: normalized_messages.append(message) - # Further normalize message roles to standard gen_ai values role_normalized_messages = normalize_message_roles(normalized_messages) set_data_normalized( span, diff --git a/sentry_sdk/integrations/langgraph.py b/sentry_sdk/integrations/langgraph.py index befd58d033..11aa1facf4 100644 --- a/sentry_sdk/integrations/langgraph.py +++ b/sentry_sdk/integrations/langgraph.py @@ -180,7 +180,6 @@ def new_invoke(self, *args, **kwargs): ): input_messages = _parse_langgraph_messages(args[0]) if input_messages: - # Normalize message roles to standard gen_ai values normalized_input_messages = normalize_message_roles(input_messages) set_data_normalized( span, @@ -232,7 +231,6 @@ async def new_ainvoke(self, *args, **kwargs): ): input_messages = _parse_langgraph_messages(args[0]) if input_messages: - # Normalize message roles to standard gen_ai values normalized_input_messages = normalize_message_roles(input_messages) set_data_normalized( span, diff --git a/sentry_sdk/integrations/openai.py b/sentry_sdk/integrations/openai.py index 7922292818..e9bd2efa23 100644 --- a/sentry_sdk/integrations/openai.py +++ b/sentry_sdk/integrations/openai.py @@ -182,7 +182,6 @@ def _set_input_data(span, kwargs, operation, integration): and should_send_default_pii() and integration.include_prompts ): - # Normalize message roles to standard gen_ai values normalized_messages = normalize_message_roles(messages) set_data_normalized( span, SPANDATA.GEN_AI_REQUEST_MESSAGES, normalized_messages, unpack=False diff --git a/sentry_sdk/integrations/openai_agents/spans/invoke_agent.py b/sentry_sdk/integrations/openai_agents/spans/invoke_agent.py index c845e4574e..2a9c5ebe66 100644 --- a/sentry_sdk/integrations/openai_agents/spans/invoke_agent.py +++ b/sentry_sdk/integrations/openai_agents/spans/invoke_agent.py @@ -60,7 +60,6 @@ def invoke_agent_span(context, agent, kwargs): ) if len(messages) > 0: - # Normalize message roles to standard gen_ai values normalized_messages = normalize_message_roles(messages) set_data_normalized( span, diff --git a/sentry_sdk/integrations/openai_agents/utils.py b/sentry_sdk/integrations/openai_agents/utils.py index 1e703a69d4..28ec986c5d 100644 --- a/sentry_sdk/integrations/openai_agents/utils.py +++ b/sentry_sdk/integrations/openai_agents/utils.py @@ -1,5 +1,6 @@ import sentry_sdk from sentry_sdk.ai.utils import ( + GEN_AI_ALLOWED_MESSAGE_ROLES, normalize_message_roles, set_data_normalized, normalize_message_role, @@ -104,7 +105,7 @@ def _set_input_data(span, get_response_kwargs): if system_instructions: request_messages.append( { - "role": "system", + "role": GEN_AI_ALLOWED_MESSAGE_ROLES.SYSTEM, "content": [{"type": "text", "text": system_instructions}], } ) @@ -120,9 +121,19 @@ def _set_input_data(span, get_response_kwargs): ) else: if message.get("type") == "function_call": - request_messages.append({"role": "assistant", "content": [message]}) + request_messages.append( + { + "role": GEN_AI_ALLOWED_MESSAGE_ROLES.ASSISTANT, + "content": [message], + } + ) elif message.get("type") == "function_call_output": - request_messages.append({"role": "tool", "content": [message]}) + request_messages.append( + { + "role": GEN_AI_ALLOWED_MESSAGE_ROLES.TOOL_CALL, + "content": [message], + } + ) set_data_normalized( span, From 0d7102eb15501ce1c12704b6f18576ca1c62e13e Mon Sep 17 00:00:00 2001 From: Simon Hellmayr Date: Tue, 7 Oct 2025 14:03:07 +0200 Subject: [PATCH 07/13] fix tool call role name --- sentry_sdk/ai/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sentry_sdk/ai/utils.py b/sentry_sdk/ai/utils.py index 29f82f5c18..e931b3fbf5 100644 --- a/sentry_sdk/ai/utils.py +++ b/sentry_sdk/ai/utils.py @@ -14,7 +14,7 @@ class GEN_AI_ALLOWED_MESSAGE_ROLES: SYSTEM = "system" USER = "user" ASSISTANT = "assistant" - TOOL_CALL = "tool_call" + TOOL_CALL = "tool" GEN_AI_MESSAGE_ROLE_REVERSE_MAPPING = { From 8810ee649c700cf9089db69f0b7af988ffc09398 Mon Sep 17 00:00:00 2001 From: Simon Hellmayr Date: Tue, 7 Oct 2025 14:09:41 +0200 Subject: [PATCH 08/13] fix missing tool --- sentry_sdk/ai/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sentry_sdk/ai/utils.py b/sentry_sdk/ai/utils.py index e931b3fbf5..e8f9bd5ee8 100644 --- a/sentry_sdk/ai/utils.py +++ b/sentry_sdk/ai/utils.py @@ -21,7 +21,7 @@ class GEN_AI_ALLOWED_MESSAGE_ROLES: GEN_AI_ALLOWED_MESSAGE_ROLES.SYSTEM: ["system"], GEN_AI_ALLOWED_MESSAGE_ROLES.USER: ["user"], GEN_AI_ALLOWED_MESSAGE_ROLES.ASSISTANT: ["assistant", "ai"], - GEN_AI_ALLOWED_MESSAGE_ROLES.TOOL_CALL: ["tool_call"], + GEN_AI_ALLOWED_MESSAGE_ROLES.TOOL_CALL: ["tool", "tool_call"], } GEN_AI_MESSAGE_ROLE_MAPPING = {} From 16342b109248fa26e73b7877fe73a12852ad285e Mon Sep 17 00:00:00 2001 From: Simon Hellmayr Date: Tue, 7 Oct 2025 14:26:15 +0200 Subject: [PATCH 09/13] move import up --- tests/integrations/anthropic/test_anthropic.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/integrations/anthropic/test_anthropic.py b/tests/integrations/anthropic/test_anthropic.py index 4510eed3a0..e9065e2d32 100644 --- a/tests/integrations/anthropic/test_anthropic.py +++ b/tests/integrations/anthropic/test_anthropic.py @@ -1,5 +1,6 @@ import pytest from unittest import mock +import json try: from unittest.mock import AsyncMock @@ -926,8 +927,6 @@ def mock_messages_create(*args, **kwargs): assert SPANDATA.GEN_AI_REQUEST_MESSAGES in span["data"] # Parse the stored messages - import json - stored_messages = json.loads(span["data"][SPANDATA.GEN_AI_REQUEST_MESSAGES]) # Verify that "ai" role was mapped to "assistant" From 32862926352c456b4323850dd9b83514132be661 Mon Sep 17 00:00:00 2001 From: Simon Hellmayr Date: Tue, 7 Oct 2025 16:12:57 +0200 Subject: [PATCH 10/13] add to langchain --- sentry_sdk/ai/utils.py | 2 +- sentry_sdk/integrations/langchain.py | 26 ++++++++++++++++++++++---- 2 files changed, 23 insertions(+), 5 deletions(-) diff --git a/sentry_sdk/ai/utils.py b/sentry_sdk/ai/utils.py index e8f9bd5ee8..4acbf79044 100644 --- a/sentry_sdk/ai/utils.py +++ b/sentry_sdk/ai/utils.py @@ -19,7 +19,7 @@ class GEN_AI_ALLOWED_MESSAGE_ROLES: GEN_AI_MESSAGE_ROLE_REVERSE_MAPPING = { GEN_AI_ALLOWED_MESSAGE_ROLES.SYSTEM: ["system"], - GEN_AI_ALLOWED_MESSAGE_ROLES.USER: ["user"], + GEN_AI_ALLOWED_MESSAGE_ROLES.USER: ["user", "human"], GEN_AI_ALLOWED_MESSAGE_ROLES.ASSISTANT: ["assistant", "ai"], GEN_AI_ALLOWED_MESSAGE_ROLES.TOOL_CALL: ["tool", "tool_call"], } diff --git a/sentry_sdk/integrations/langchain.py b/sentry_sdk/integrations/langchain.py index fdba26569d..8053d5ebc2 100644 --- a/sentry_sdk/integrations/langchain.py +++ b/sentry_sdk/integrations/langchain.py @@ -4,7 +4,11 @@ import sentry_sdk from sentry_sdk.ai.monitoring import set_ai_pipeline_name -from sentry_sdk.ai.utils import set_data_normalized, get_start_span_function +from sentry_sdk.ai.utils import ( + normalize_message_roles, + set_data_normalized, + get_start_span_function, +) from sentry_sdk.consts import OP, SPANDATA from sentry_sdk.integrations import DidNotEnable, Integration from sentry_sdk.scope import should_send_default_pii @@ -209,8 +213,12 @@ def on_llm_start( _set_tools_on_span(span, all_params.get("tools")) if should_send_default_pii() and self.include_prompts: + normalized_messages = normalize_message_roles(prompts) set_data_normalized( - span, SPANDATA.GEN_AI_REQUEST_MESSAGES, prompts, unpack=False + span, + SPANDATA.GEN_AI_REQUEST_MESSAGES, + normalized_messages, + unpack=False, ) def on_chat_model_start(self, serialized, messages, *, run_id, **kwargs): @@ -262,6 +270,8 @@ def on_chat_model_start(self, serialized, messages, *, run_id, **kwargs): normalized_messages.append( self._normalize_langchain_message(message) ) + normalized_messages = normalize_message_roles(normalized_messages) + set_data_normalized( span, SPANDATA.GEN_AI_REQUEST_MESSAGES, @@ -740,8 +750,12 @@ def new_invoke(self, *args, **kwargs): and should_send_default_pii() and integration.include_prompts ): + normalized_messages = normalize_message_roles([input]) set_data_normalized( - span, SPANDATA.GEN_AI_REQUEST_MESSAGES, [input], unpack=False + span, + SPANDATA.GEN_AI_REQUEST_MESSAGES, + normalized_messages, + unpack=False, ) output = result.get("output") @@ -791,8 +805,12 @@ def new_stream(self, *args, **kwargs): and should_send_default_pii() and integration.include_prompts ): + normalized_messages = normalize_message_roles([input]) set_data_normalized( - span, SPANDATA.GEN_AI_REQUEST_MESSAGES, [input], unpack=False + span, + SPANDATA.GEN_AI_REQUEST_MESSAGES, + normalized_messages, + unpack=False, ) # Run the agent From eb8c05dfdb013192f03406d75c7cd9f5b0ecb9d5 Mon Sep 17 00:00:00 2001 From: Simon Hellmayr Date: Tue, 7 Oct 2025 16:18:00 +0200 Subject: [PATCH 11/13] add langchain test --- .../integrations/langchain/test_langchain.py | 141 ++++++++++++++++++ 1 file changed, 141 insertions(+) diff --git a/tests/integrations/langchain/test_langchain.py b/tests/integrations/langchain/test_langchain.py index ba49b2e508..661208432f 100644 --- a/tests/integrations/langchain/test_langchain.py +++ b/tests/integrations/langchain/test_langchain.py @@ -817,3 +817,144 @@ def test_langchain_integration_with_langchain_core_only(sentry_init, capture_eve assert llm_span["data"]["gen_ai.usage.total_tokens"] == 25 assert llm_span["data"]["gen_ai.usage.input_tokens"] == 10 assert llm_span["data"]["gen_ai.usage.output_tokens"] == 15 + + +def test_langchain_message_role_mapping(sentry_init, capture_events): + """Test that message roles are properly normalized in langchain integration.""" + global llm_type + llm_type = "openai-chat" + + sentry_init( + integrations=[LangchainIntegration(include_prompts=True)], + traces_sample_rate=1.0, + send_default_pii=True, + ) + events = capture_events() + + prompt = ChatPromptTemplate.from_messages( + [ + ("system", "You are a helpful assistant"), + ("human", "{input}"), + MessagesPlaceholder(variable_name="agent_scratchpad"), + ] + ) + + global stream_result_mock + stream_result_mock = Mock( + side_effect=[ + [ + ChatGenerationChunk( + type="ChatGenerationChunk", + message=AIMessageChunk(content="Test response"), + ), + ] + ] + ) + + llm = MockOpenAI( + model_name="gpt-3.5-turbo", + temperature=0, + openai_api_key="badkey", + ) + agent = create_openai_tools_agent(llm, [get_word_length], prompt) + agent_executor = AgentExecutor(agent=agent, tools=[get_word_length], verbose=True) + + # Test input that should trigger message role normalization + test_input = "Hello, how are you?" + + with start_transaction(): + list(agent_executor.stream({"input": test_input})) + + assert len(events) > 0 + tx = events[0] + assert tx["type"] == "transaction" + + # Find spans with gen_ai operation that should have message data + gen_ai_spans = [ + span for span in tx.get("spans", []) if span.get("op", "").startswith("gen_ai") + ] + + # Check if any span has message data with normalized roles + message_data_found = False + for span in gen_ai_spans: + span_data = span.get("data", {}) + if SPANDATA.GEN_AI_REQUEST_MESSAGES in span_data: + message_data_found = True + messages_data = span_data[SPANDATA.GEN_AI_REQUEST_MESSAGES] + + # Parse the message data (might be JSON string) + if isinstance(messages_data, str): + import json + + try: + messages = json.loads(messages_data) + except json.JSONDecodeError: + # If not valid JSON, skip this assertion + continue + else: + messages = messages_data + + # Verify that the input message is present and contains the test input + assert isinstance(messages, list) + assert len(messages) > 0 + + # The test input should be in one of the messages + input_found = False + for msg in messages: + if isinstance(msg, dict) and test_input in str(msg.get("content", "")): + input_found = True + break + elif isinstance(msg, str) and test_input in msg: + input_found = True + break + + assert input_found, ( + f"Test input '{test_input}' not found in messages: {messages}" + ) + break + + # The message role mapping functionality is primarily tested through the normalization + # that happens in the integration code. The fact that we can capture and process + # the messages without errors indicates the role mapping is working correctly. + assert message_data_found, "No span found with gen_ai request messages data" + + +def test_langchain_message_role_normalization_units(): + """Test the message role normalization functions directly.""" + from sentry_sdk.ai.utils import normalize_message_role, normalize_message_roles + + # Test individual role normalization + assert normalize_message_role("ai") == "assistant" + assert normalize_message_role("human") == "user" + assert normalize_message_role("tool_call") == "tool" + assert normalize_message_role("system") == "system" + assert normalize_message_role("user") == "user" + assert normalize_message_role("assistant") == "assistant" + assert normalize_message_role("tool") == "tool" + + # Test unknown role (should remain unchanged) + assert normalize_message_role("unknown_role") == "unknown_role" + + # Test message list normalization + test_messages = [ + {"role": "human", "content": "Hello"}, + {"role": "ai", "content": "Hi there!"}, + {"role": "tool_call", "content": "function_call"}, + {"role": "system", "content": "You are helpful"}, + {"content": "Message without role"}, + "string message", + ] + + normalized = normalize_message_roles(test_messages) + + # Verify the original messages are not modified + assert test_messages[0]["role"] == "human" # Original unchanged + assert test_messages[1]["role"] == "ai" # Original unchanged + + # Verify the normalized messages have correct roles + assert normalized[0]["role"] == "user" # human -> user + assert normalized[1]["role"] == "assistant" # ai -> assistant + assert normalized[2]["role"] == "tool" # tool_call -> tool + assert normalized[3]["role"] == "system" # system unchanged + assert "role" not in normalized[4] # Message without role unchanged + assert normalized[5] == "string message" # String message unchanged From 526acf3836093bdbed772fea26300d59cbd17c0f Mon Sep 17 00:00:00 2001 From: Simon Hellmayr Date: Wed, 8 Oct 2025 10:58:26 +0200 Subject: [PATCH 12/13] fix langchain on_llm_start --- sentry_sdk/integrations/langchain.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/sentry_sdk/integrations/langchain.py b/sentry_sdk/integrations/langchain.py index 8053d5ebc2..724d908665 100644 --- a/sentry_sdk/integrations/langchain.py +++ b/sentry_sdk/integrations/langchain.py @@ -5,6 +5,7 @@ import sentry_sdk from sentry_sdk.ai.monitoring import set_ai_pipeline_name from sentry_sdk.ai.utils import ( + GEN_AI_ALLOWED_MESSAGE_ROLES, normalize_message_roles, set_data_normalized, get_start_span_function, @@ -213,7 +214,13 @@ def on_llm_start( _set_tools_on_span(span, all_params.get("tools")) if should_send_default_pii() and self.include_prompts: - normalized_messages = normalize_message_roles(prompts) + normalized_messages = [ + { + "role": GEN_AI_ALLOWED_MESSAGE_ROLES.USER, + "content": {"type": "text", "text": prompt}, + } + for prompt in prompts + ] set_data_normalized( span, SPANDATA.GEN_AI_REQUEST_MESSAGES, From 8be69b2facead64f49b6128f8c7e5f59da93ace8 Mon Sep 17 00:00:00 2001 From: Simon Hellmayr Date: Wed, 8 Oct 2025 11:05:17 +0200 Subject: [PATCH 13/13] fix naming of tool call const --- sentry_sdk/ai/utils.py | 4 ++-- sentry_sdk/integrations/openai_agents/utils.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/sentry_sdk/ai/utils.py b/sentry_sdk/ai/utils.py index 4acbf79044..0c0b937006 100644 --- a/sentry_sdk/ai/utils.py +++ b/sentry_sdk/ai/utils.py @@ -14,14 +14,14 @@ class GEN_AI_ALLOWED_MESSAGE_ROLES: SYSTEM = "system" USER = "user" ASSISTANT = "assistant" - TOOL_CALL = "tool" + TOOL = "tool" GEN_AI_MESSAGE_ROLE_REVERSE_MAPPING = { GEN_AI_ALLOWED_MESSAGE_ROLES.SYSTEM: ["system"], GEN_AI_ALLOWED_MESSAGE_ROLES.USER: ["user", "human"], GEN_AI_ALLOWED_MESSAGE_ROLES.ASSISTANT: ["assistant", "ai"], - GEN_AI_ALLOWED_MESSAGE_ROLES.TOOL_CALL: ["tool", "tool_call"], + GEN_AI_ALLOWED_MESSAGE_ROLES.TOOL: ["tool", "tool_call"], } GEN_AI_MESSAGE_ROLE_MAPPING = {} diff --git a/sentry_sdk/integrations/openai_agents/utils.py b/sentry_sdk/integrations/openai_agents/utils.py index 28ec986c5d..125ff1175b 100644 --- a/sentry_sdk/integrations/openai_agents/utils.py +++ b/sentry_sdk/integrations/openai_agents/utils.py @@ -130,7 +130,7 @@ def _set_input_data(span, get_response_kwargs): elif message.get("type") == "function_call_output": request_messages.append( { - "role": GEN_AI_ALLOWED_MESSAGE_ROLES.TOOL_CALL, + "role": GEN_AI_ALLOWED_MESSAGE_ROLES.TOOL, "content": [message], } )