Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 3 additions & 2 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -120,18 +120,19 @@ jobs:

- name: Run unit tests
run: |
uv run --frozen python -m pytest tests/ -v --tb=short -m "not integration"
uv run --frozen pytest tests/ -v --tb=short -m "not integration"

- name: Run integration tests
# Only run integration tests if secrets are available
if: ${{ vars.RUN_INTEGRATION_TESTS == 'true' }}
run: |
uv run --frozen python -m pytest tests/integration/ -v --tb=short -m integration
uv run --frozen pytest -m integration -v --tb=short
env:
AZURE_OPENAI_API_KEY: ${{ secrets.AZURE_OPENAI_API_KEY }}
AZURE_OPENAI_ENDPOINT: ${{ vars.AZURE_OPENAI_ENDPOINT }}
AZURE_OPENAI_DEPLOYMENT: ${{ vars.AZURE_OPENAI_DEPLOYMENT }}
AZURE_OPENAI_API_VERSION: ${{ vars.AZURE_OPENAI_API_VERSION }}
ENABLE_OBSERVABILITY: true

# Copy package and samples to drop folder
- name: Copy package and samples to drop folder
Expand Down
Original file line number Diff line number Diff line change
@@ -1,15 +1,42 @@
# Copyright (c) Microsoft. All rights reserved.
# Copyright (c) Microsoft Corporation.
# Licensed under the MIT License.

# Agent details class.
from dataclasses import dataclass
from typing import Optional

from .models.agent_type import AgentType


@dataclass
class AgentDetails:
"""Details about an AI agent."""
"""Details about an AI agent in the system."""

agent_id: str
conversation_id: str | None = None
agent_name: str | None = None
agent_description: str | None = None
icon_uri: str | None = None
"""The unique identifier for the AI agent."""

agent_name: Optional[str] = None
"""The human-readable name of the AI agent."""

agent_description: Optional[str] = None
"""A description of the AI agent's purpose or capabilities."""

agent_auid: Optional[str] = None
"""Optional Agent User ID for the agent."""

agent_upn: Optional[str] = None
"""Optional User Principal Name (UPN) for the agent."""

agent_blueprint_id: Optional[str] = None
"""Optional Blueprint/Application ID for the agent."""

agent_type: Optional[AgentType] = None
"""The agent type."""

tenant_id: Optional[str] = None
"""Optional Tenant ID for the agent."""

conversation_id: Optional[str] = None
"""Optional conversation ID for compatibility."""

icon_uri: Optional[str] = None
"""Optional icon URI for the agent."""
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@
GEN_AI_RESPONSE_MODEL_KEY = "gen_ai.response.model"
GEN_AI_SYSTEM_KEY = "gen_ai.system"
GEN_AI_SYSTEM_VALUE = "az.ai.agent365"
GEN_AI_THOUGHT_PROCESS_KEY = "gen_ai.agent.thought.process"

GEN_AI_AGENT_ID_KEY = "gen_ai.agent.id"
GEN_AI_AGENT_NAME_KEY = "gen_ai.agent.name"
Expand All @@ -45,6 +46,7 @@
GEN_AI_USAGE_OUTPUT_TOKENS_KEY = "gen_ai.usage.output_tokens"
GEN_AI_CHOICE = "gen_ai.choice"
GEN_AI_PROVIDER_NAME_KEY = "gen_ai.provider.name"
GEN_AI_AGENT_TYPE_KEY = "gen_ai.agent.type"

GEN_AI_SYSTEM_INSTRUCTIONS_KEY = "gen_ai.system_instructions"
GEN_AI_INPUT_MESSAGES_KEY = "gen_ai.input.messages"
Expand Down Expand Up @@ -74,6 +76,7 @@
GEN_AI_CALLER_AGENT_NAME_KEY = "gen_ai.caller.agent.name"
GEN_AI_CALLER_AGENT_ID_KEY = "gen_ai.caller.agent.id"
GEN_AI_CALLER_AGENT_APPLICATION_ID_KEY = "gen_ai.caller.agent.applicationid"
GEN_AI_CALLER_AGENT_TYPE_KEY = "gen_ai.caller.agent.type"

# Agent-specific dimensions
AGENT_ID_KEY = "gen_ai.agent.id"
Expand Down
Original file line number Diff line number Diff line change
@@ -1,10 +1,11 @@
# Copyright (c) Microsoft. All rights reserved.

# Execute tool scope for tracing tool execution.
# Copyright (c) Microsoft Corporation.
# Licensed under the MIT License.

from .agent_details import AgentDetails
from .constants import (
EXECUTE_TOOL_OPERATION_NAME,
GEN_AI_EVENT_CONTENT,
GEN_AI_TOOL_ARGS_KEY,
GEN_AI_TOOL_CALL_ID_KEY,
GEN_AI_TOOL_DESCRIPTION_KEY,
GEN_AI_TOOL_NAME_KEY,
Expand All @@ -26,7 +27,7 @@ def start(
agent_details: AgentDetails,
tenant_details: TenantDetails,
) -> "ExecuteToolScope":
"""Create and start a new scope for tool execution tracing.
"""Creates and starts a new scope for tool execution tracing.

Args:
details: The details of the tool call
Expand Down Expand Up @@ -54,18 +55,34 @@ def __init__(
super().__init__(
kind="Internal",
operation_name=EXECUTE_TOOL_OPERATION_NAME,
activity_name=f"execute_tool {details.tool_name}",
activity_name=f"{EXECUTE_TOOL_OPERATION_NAME} {details.tool_name}",
agent_details=agent_details,
tenant_details=tenant_details,
)

self.set_tag_maybe(GEN_AI_TOOL_NAME_KEY, details.tool_name)
self.set_tag_maybe("gen_ai.tool.arguments", details.arguments)
self.set_tag_maybe(GEN_AI_TOOL_TYPE_KEY, details.tool_type)
self.set_tag_maybe(GEN_AI_TOOL_CALL_ID_KEY, details.tool_call_id)
self.set_tag_maybe(GEN_AI_TOOL_DESCRIPTION_KEY, details.description)
# Extract details using deconstruction-like approach
tool_name = details.tool_name
arguments = details.arguments
tool_call_id = details.tool_call_id
description = details.description
tool_type = details.tool_type
endpoint = details.endpoint

self.set_tag_maybe(GEN_AI_TOOL_NAME_KEY, tool_name)
self.set_tag_maybe(GEN_AI_TOOL_ARGS_KEY, arguments)
self.set_tag_maybe(GEN_AI_TOOL_TYPE_KEY, tool_type)
self.set_tag_maybe(GEN_AI_TOOL_CALL_ID_KEY, tool_call_id)
self.set_tag_maybe(GEN_AI_TOOL_DESCRIPTION_KEY, description)

if endpoint:
self.set_tag_maybe(SERVER_ADDRESS_KEY, endpoint.hostname)
if endpoint.port and endpoint.port != 443:
self.set_tag_maybe(SERVER_PORT_KEY, endpoint.port)

if details.endpoint:
self.set_tag_maybe(SERVER_ADDRESS_KEY, details.endpoint.hostname)
if details.endpoint.port and details.endpoint.port != 443:
self.set_tag_maybe(SERVER_PORT_KEY, details.endpoint.port)
def record_response(self, response: str) -> None:
"""Records response information for telemetry tracking.

Args:
response: The response to record
"""
self.set_tag_maybe(GEN_AI_EVENT_CONTENT, response)
Original file line number Diff line number Diff line change
@@ -1,23 +1,30 @@
# Copyright (c) Microsoft. All rights reserved.
# Copyright (c) Microsoft Corporation.
# Licensed under the MIT License.

from typing import List

from .agent_details import AgentDetails
from .constants import (
GEN_AI_INPUT_MESSAGES_KEY,
GEN_AI_OPERATION_NAME_KEY,
GEN_AI_OUTPUT_MESSAGES_KEY,
GEN_AI_PROVIDER_NAME_KEY,
GEN_AI_REQUEST_MODEL_KEY,
GEN_AI_RESPONSE_FINISH_REASONS_KEY,
GEN_AI_RESPONSE_ID_KEY,
GEN_AI_THOUGHT_PROCESS_KEY,
GEN_AI_USAGE_INPUT_TOKENS_KEY,
GEN_AI_USAGE_OUTPUT_TOKENS_KEY,
)
from .inference_call_details import InferenceCallDetails
from .opentelemetry_scope import OpenTelemetryScope
from .request import Request
from .tenant_details import TenantDetails
from .utils import safe_json_dumps


class InferenceScope(OpenTelemetryScope):
"""Provides OpenTelemetry tracing scope for inference call."""
"""Provides OpenTelemetry tracing scope for generative AI inference operations."""

@staticmethod
def start(
Expand All @@ -26,7 +33,7 @@ def start(
tenant_details: TenantDetails,
request: Request | None = None,
) -> "InferenceScope":
"""Create and start a new scope for inference call.
"""Creates and starts a new scope for inference tracing.

Args:
details: The details of the inference call
Expand All @@ -41,40 +48,93 @@ def start(

def __init__(
self,
inference_call_details: InferenceCallDetails,
details: InferenceCallDetails,
agent_details: AgentDetails,
tenant_details: TenantDetails,
request: Request | None = None,
):
"""Initialize the agent invocation scope.
"""Initialize the inference scope.

Args:
inference_call_details: The details of the inference call
details: The details of the inference call
agent_details: The details of the agent making the call
tenant_details: The details of the tenant
request: Optional request details for additional context
"""

super().__init__(
kind="Client",
operation_name=inference_call_details.operationName.value,
activity_name=f"{inference_call_details.operationName.value} {inference_call_details.model}",
operation_name=details.operationName.value,
activity_name=f"{details.operationName.value} {details.model}",
agent_details=agent_details,
tenant_details=tenant_details,
)

# Set request content if provided
if request:
self.set_tag_maybe(GEN_AI_INPUT_MESSAGES_KEY, request.content)

self.set_tag_maybe(GEN_AI_REQUEST_MODEL_KEY, inference_call_details.model)
self.set_tag_maybe(GEN_AI_PROVIDER_NAME_KEY, inference_call_details.providerName)
self.set_tag_maybe(GEN_AI_USAGE_INPUT_TOKENS_KEY, inference_call_details.inputTokens)
self.set_tag_maybe(GEN_AI_USAGE_OUTPUT_TOKENS_KEY, inference_call_details.outputTokens)
self.set_tag_maybe(GEN_AI_OPERATION_NAME_KEY, details.operationName.value)
self.set_tag_maybe(GEN_AI_REQUEST_MODEL_KEY, details.model)
self.set_tag_maybe(GEN_AI_PROVIDER_NAME_KEY, details.providerName)
self.set_tag_maybe(
GEN_AI_USAGE_INPUT_TOKENS_KEY,
str(details.inputTokens) if details.inputTokens is not None else None,
)
self.set_tag_maybe(
GEN_AI_USAGE_OUTPUT_TOKENS_KEY,
str(details.outputTokens) if details.outputTokens is not None else None,
)
self.set_tag_maybe(
GEN_AI_RESPONSE_FINISH_REASONS_KEY,
",".join(inference_call_details.finishReasons)
if inference_call_details.finishReasons
else None,
safe_json_dumps(details.finishReasons) if details.finishReasons else None,
)
self.set_tag_maybe(GEN_AI_RESPONSE_ID_KEY, inference_call_details.responseId)
self.set_tag_maybe(GEN_AI_RESPONSE_ID_KEY, details.responseId)

def record_input_messages(self, messages: List[str]) -> None:
"""Records the input messages for telemetry tracking.

Args:
messages: List of input messages
"""
self.set_tag_maybe(GEN_AI_INPUT_MESSAGES_KEY, safe_json_dumps(messages))

def record_output_messages(self, messages: List[str]) -> None:
"""Records the output messages for telemetry tracking.

Args:
messages: List of output messages
"""
self.set_tag_maybe(GEN_AI_OUTPUT_MESSAGES_KEY, safe_json_dumps(messages))

def record_input_tokens(self, input_tokens: int) -> None:
"""Records the number of input tokens for telemetry tracking.

Args:
input_tokens: Number of input tokens
"""
self.set_tag_maybe(GEN_AI_USAGE_INPUT_TOKENS_KEY, str(input_tokens))

def record_output_tokens(self, output_tokens: int) -> None:
"""Records the number of output tokens for telemetry tracking.

Args:
output_tokens: Number of output tokens
"""
self.set_tag_maybe(GEN_AI_USAGE_OUTPUT_TOKENS_KEY, str(output_tokens))

def record_finish_reasons(self, finish_reasons: List[str]) -> None:
"""Records the finish reasons for telemetry tracking.

Args:
finish_reasons: List of finish reasons
"""
if finish_reasons:
self.set_tag_maybe(GEN_AI_RESPONSE_FINISH_REASONS_KEY, safe_json_dumps(finish_reasons))

def record_thought_process(self, thought_process: str) -> None:
"""Records the thought process.

Args:
thought_process: The thought process to record
"""
self.set_tag_maybe(GEN_AI_THOUGHT_PROCESS_KEY, thought_process)
Loading
Loading