diff --git a/libraries/microsoft-agents-a365-observability-extensions-agentframework/microsoft_agents_a365/observability/extensions/agentframework/__init__.py b/libraries/microsoft-agents-a365-observability-extensions-agentframework/microsoft_agents_a365/observability/extensions/agentframework/__init__.py index 59e481eb..f73f4303 100644 --- a/libraries/microsoft-agents-a365-observability-extensions-agentframework/microsoft_agents_a365/observability/extensions/agentframework/__init__.py +++ b/libraries/microsoft-agents-a365-observability-extensions-agentframework/microsoft_agents_a365/observability/extensions/agentframework/__init__.py @@ -1,2 +1,10 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. + +"""Agent Framework observability extensions for Agent365.""" + +from .trace_instrumentor import AgentFrameworkInstrumentor + +__all__ = [ + "AgentFrameworkInstrumentor", +] diff --git a/libraries/microsoft-agents-a365-observability-extensions-agentframework/microsoft_agents_a365/observability/extensions/agentframework/span_enricher.py b/libraries/microsoft-agents-a365-observability-extensions-agentframework/microsoft_agents_a365/observability/extensions/agentframework/span_enricher.py new file mode 100644 index 00000000..158db985 --- /dev/null +++ b/libraries/microsoft-agents-a365-observability-extensions-agentframework/microsoft_agents_a365/observability/extensions/agentframework/span_enricher.py @@ -0,0 +1,51 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +from microsoft_agents_a365.observability.core.constants import ( + EXECUTE_TOOL_OPERATION_NAME, + GEN_AI_INPUT_MESSAGES_KEY, + GEN_AI_OUTPUT_MESSAGES_KEY, + GEN_AI_TOOL_ARGS_KEY, + GEN_AI_TOOL_CALL_RESULT_KEY, + INVOKE_AGENT_OPERATION_NAME, +) +from microsoft_agents_a365.observability.core.exporters.enriched_span import EnrichedReadableSpan +from opentelemetry.sdk.trace import ReadableSpan + +from .utils import extract_input_content, extract_output_content + +# Agent Framework specific attribute keys +AF_TOOL_CALL_ARGUMENTS_KEY = "gen_ai.tool.call.arguments" +AF_TOOL_CALL_RESULT_KEY = "gen_ai.tool.call.result" + + +def enrich_agent_framework_span(span: ReadableSpan) -> ReadableSpan: + """ + Enricher function for Agent Framework spans. + """ + extra_attributes = {} + attributes = span.attributes or {} + + # Only extract content for invoke_agent spans + if span.name.startswith(INVOKE_AGENT_OPERATION_NAME): + # Extract all text content from input messages + input_messages = attributes.get(GEN_AI_INPUT_MESSAGES_KEY) + if input_messages: + extra_attributes[GEN_AI_INPUT_MESSAGES_KEY] = extract_input_content(input_messages) + + output_messages = attributes.get(GEN_AI_OUTPUT_MESSAGES_KEY) + if output_messages: + extra_attributes[GEN_AI_OUTPUT_MESSAGES_KEY] = extract_output_content(output_messages) + + # Map tool attributes for execute_tool spans + elif span.name.startswith(EXECUTE_TOOL_OPERATION_NAME): + if AF_TOOL_CALL_ARGUMENTS_KEY in attributes: + extra_attributes[GEN_AI_TOOL_ARGS_KEY] = attributes[AF_TOOL_CALL_ARGUMENTS_KEY] + + if AF_TOOL_CALL_RESULT_KEY in attributes: + extra_attributes[GEN_AI_TOOL_CALL_RESULT_KEY] = attributes[AF_TOOL_CALL_RESULT_KEY] + + if extra_attributes: + return EnrichedReadableSpan(span, extra_attributes) + + return span diff --git a/libraries/microsoft-agents-a365-observability-extensions-agentframework/microsoft_agents_a365/observability/extensions/agentframework/span_processor.py b/libraries/microsoft-agents-a365-observability-extensions-agentframework/microsoft_agents_a365/observability/extensions/agentframework/span_processor.py index 0b41b761..a8a6451a 100644 --- a/libraries/microsoft-agents-a365-observability-extensions-agentframework/microsoft_agents_a365/observability/extensions/agentframework/span_processor.py +++ b/libraries/microsoft-agents-a365-observability-extensions-agentframework/microsoft_agents_a365/observability/extensions/agentframework/span_processor.py @@ -1,15 +1,12 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. -# Custom Span Processor - -from opentelemetry.sdk.trace.export import SpanProcessor - from microsoft_agents_a365.observability.core.constants import ( - GEN_AI_OPERATION_NAME_KEY, EXECUTE_TOOL_OPERATION_NAME, GEN_AI_EVENT_CONTENT, + GEN_AI_OPERATION_NAME_KEY, ) +from opentelemetry.sdk.trace.export import SpanProcessor class AgentFrameworkSpanProcessor(SpanProcessor): diff --git a/libraries/microsoft-agents-a365-observability-extensions-agentframework/microsoft_agents_a365/observability/extensions/agentframework/trace_instrumentor.py b/libraries/microsoft-agents-a365-observability-extensions-agentframework/microsoft_agents_a365/observability/extensions/agentframework/trace_instrumentor.py index a0d545a6..41d63247 100644 --- a/libraries/microsoft-agents-a365-observability-extensions-agentframework/microsoft_agents_a365/observability/extensions/agentframework/trace_instrumentor.py +++ b/libraries/microsoft-agents-a365-observability-extensions-agentframework/microsoft_agents_a365/observability/extensions/agentframework/trace_instrumentor.py @@ -7,8 +7,15 @@ from typing import Any from microsoft_agents_a365.observability.core.config import get_tracer_provider, is_configured +from microsoft_agents_a365.observability.core.exporters.enriching_span_processor import ( + register_span_enricher, + unregister_span_enricher, +) from opentelemetry.instrumentation.instrumentor import BaseInstrumentor +from microsoft_agents_a365.observability.extensions.agentframework.span_enricher import ( + enrich_agent_framework_span, +) from microsoft_agents_a365.observability.extensions.agentframework.span_processor import ( AgentFrameworkSpanProcessor, ) @@ -16,13 +23,12 @@ # ----------------------------- # 3) The Instrumentor class # ----------------------------- -_instruments = ("agent-framework-azure-ai >= 1.0.0b251114",) +_instruments = ("agent-framework-azure-ai >= 1.0.0",) class AgentFrameworkInstrumentor(BaseInstrumentor): """ - Instruments Agent Framework: - • Installs your custom OTel SpanProcessor + Instruments Agent Framework with Agent365 observability. """ def __init__(self): @@ -37,13 +43,28 @@ def instrumentation_dependencies(self) -> Collection[str]: def _instrument(self, **kwargs: Any) -> None: """ - kwargs (all optional): - """ + Instrument Agent Framework. + Args: + **kwargs: Optional configuration parameters. + """ # Ensure we have an SDK TracerProvider provider = get_tracer_provider() + + # Add processor for on_start modifications (rename spans, add attributes) self._processor = AgentFrameworkSpanProcessor() provider.add_span_processor(self._processor) + # Register enricher for on_end modifications + register_span_enricher(enrich_agent_framework_span) + def _uninstrument(self, **kwargs: Any) -> None: - pass + """ + Remove Agent Framework instrumentation. + """ + # Unregister the enricher + unregister_span_enricher() + + # Shutdown the processor + if hasattr(self, "_processor"): + self._processor.shutdown() diff --git a/libraries/microsoft-agents-a365-observability-extensions-agentframework/microsoft_agents_a365/observability/extensions/agentframework/utils.py b/libraries/microsoft-agents-a365-observability-extensions-agentframework/microsoft_agents_a365/observability/extensions/agentframework/utils.py new file mode 100644 index 00000000..84f7da1e --- /dev/null +++ b/libraries/microsoft-agents-a365-observability-extensions-agentframework/microsoft_agents_a365/observability/extensions/agentframework/utils.py @@ -0,0 +1,48 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +"""Utility functions for Agent Framework observability extensions.""" + +from __future__ import annotations + +import json + + +def extract_content_as_string_list(messages_json: str, role_filter: str | None = None) -> str: + """Extract content values from messages JSON and return as JSON string list.""" + try: + messages = json.loads(messages_json) + if isinstance(messages, list): + contents = [] + for msg in messages: + if isinstance(msg, dict): + role = msg.get("role", "") + + # Filter by role if specified + if role_filter and role != role_filter: + continue + + # Handle Agent Framework format with "parts" + parts = msg.get("parts") + if parts and isinstance(parts, list): + for part in parts: + if isinstance(part, dict): + part_type = part.get("type", "") + # Only extract text content, not tool_call or tool_call_response + if part_type == "text" and "content" in part: + contents.append(part["content"]) + return json.dumps(contents) + return messages_json + except (json.JSONDecodeError, TypeError): + # If parsing fails, return as-is + return messages_json + + +def extract_input_content(messages_json: str) -> str: + """Extract text content from user messages only.""" + return extract_content_as_string_list(messages_json, role_filter="user") + + +def extract_output_content(messages_json: str) -> str: + """Extract only assistant text content from output messages.""" + return extract_content_as_string_list(messages_json, role_filter="assistant") diff --git a/tests/observability/extensions/agentframework/__init__.py b/tests/observability/extensions/agentframework/__init__.py new file mode 100644 index 00000000..59e481eb --- /dev/null +++ b/tests/observability/extensions/agentframework/__init__.py @@ -0,0 +1,2 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. diff --git a/tests/observability/extensions/agentframework/test_span_enricher.py b/tests/observability/extensions/agentframework/test_span_enricher.py new file mode 100644 index 00000000..49ee706f --- /dev/null +++ b/tests/observability/extensions/agentframework/test_span_enricher.py @@ -0,0 +1,68 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +"""Tests for Agent Framework span enricher.""" + +import unittest +from unittest.mock import Mock + +from microsoft_agents_a365.observability.core.constants import ( + GEN_AI_INPUT_MESSAGES_KEY, + GEN_AI_OUTPUT_MESSAGES_KEY, + GEN_AI_TOOL_ARGS_KEY, + GEN_AI_TOOL_CALL_RESULT_KEY, +) +from microsoft_agents_a365.observability.extensions.agentframework.span_enricher import ( + AF_TOOL_CALL_ARGUMENTS_KEY, + AF_TOOL_CALL_RESULT_KEY, + enrich_agent_framework_span, +) + + +class TestAgentFrameworkSpanEnricher(unittest.TestCase): + """Test suite for enrich_agent_framework_span function.""" + + def test_invoke_agent_span_enrichment(self): + """Test invoke_agent span extracts user input and assistant output text only.""" + span = Mock( + name="invoke_agent Agent365Assistant", + attributes={ + GEN_AI_INPUT_MESSAGES_KEY: '[{"role": "user", "parts": [{"type": "text", "content": "Compute 15 % 4"}]}]', + GEN_AI_OUTPUT_MESSAGES_KEY: '[{"role": "assistant", "parts": [{"type": "tool_call", "id": "c1"}]}, {"role": "tool", "parts": [{"type": "tool_call_response"}]}, {"role": "assistant", "parts": [{"type": "text", "content": "Result is 3."}]}]', + }, + ) + span.name = "invoke_agent Agent365Assistant" + result = enrich_agent_framework_span(span) + self.assertEqual(result.attributes[GEN_AI_INPUT_MESSAGES_KEY], '["Compute 15 % 4"]') + self.assertEqual(result.attributes[GEN_AI_OUTPUT_MESSAGES_KEY], '["Result is 3."]') + + def test_execute_tool_span_enrichment(self): + """Test execute_tool span maps tool arguments and result to standard keys.""" + span = Mock( + name="execute_tool calculate", + attributes={ + AF_TOOL_CALL_ARGUMENTS_KEY: '{"expression": "2 + 2"}', + AF_TOOL_CALL_RESULT_KEY: "Result is 4", + }, + ) + span.name = "execute_tool calculate" + result = enrich_agent_framework_span(span) + self.assertEqual(result.attributes[GEN_AI_TOOL_ARGS_KEY], '{"expression": "2 + 2"}') + self.assertEqual(result.attributes[GEN_AI_TOOL_CALL_RESULT_KEY], "Result is 4") + + def test_non_matching_and_edge_cases_return_original(self): + """Test non-matching, None, and empty attribute spans return unchanged.""" + span = Mock(name="other_op", attributes={"key": "value"}) + span.name = "other_op" + self.assertEqual(enrich_agent_framework_span(span), span) + + span.name = "invoke_agent Test" + span.attributes = None + self.assertEqual(enrich_agent_framework_span(span), span) + + span.attributes = {} + self.assertEqual(enrich_agent_framework_span(span), span) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/observability/extensions/agentframework/test_utils.py b/tests/observability/extensions/agentframework/test_utils.py new file mode 100644 index 00000000..dd6d044e --- /dev/null +++ b/tests/observability/extensions/agentframework/test_utils.py @@ -0,0 +1,35 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +"""Tests for Agent Framework utils.""" + +import unittest + +from microsoft_agents_a365.observability.extensions.agentframework.utils import ( + extract_content_as_string_list, + extract_input_content, + extract_output_content, +) + + +class TestAgentFrameworkUtils(unittest.TestCase): + """Test suite for Agent Framework utility functions.""" + + def test_extract_content_filters_text_by_role(self): + """Test text extraction with role filtering, ignoring tool calls.""" + msgs = '[{"role": "user", "parts": [{"type": "text", "content": "Hi"}]}, {"role": "assistant", "parts": [{"type": "tool_call"}, {"type": "text", "content": "Hello"}]}]' + self.assertEqual(extract_content_as_string_list(msgs), '["Hi", "Hello"]') + self.assertEqual(extract_content_as_string_list(msgs, role_filter="user"), '["Hi"]') + self.assertEqual(extract_input_content(msgs), '["Hi"]') + self.assertEqual(extract_output_content(msgs), '["Hello"]') + + def test_handles_invalid_and_edge_cases(self): + """Test invalid JSON and edge cases return appropriate values.""" + self.assertEqual(extract_content_as_string_list("invalid"), "invalid") + self.assertEqual(extract_content_as_string_list('{"not": "list"}'), '{"not": "list"}') + self.assertEqual(extract_content_as_string_list("[]"), "[]") + self.assertEqual(extract_content_as_string_list('[{"role": "user"}]'), "[]") + + +if __name__ == "__main__": + unittest.main()