Converter from AI Service threads/runs to evaluator-compatible schema (#40047)

* WIP AIAgentConverter

* Added the v1 of the converter

* Updated the AIAgentConverter with different output schemas.

* ruff format

* Update the top schema to have: query, response, tool_definitions

* "agentic" is not a recognized word, change the wording.

* System message always comes first in query with multiple runs.

* Add support for getting inputs from local files with run_ids.

* Export AIAgentConverter through azure.ai.evaluation, local read updates

* Use from ._models import

* Ruff format again.

* For ComputeInstance and AmlCompute update disableLocalAuth property based on ssh_public_access (#39934)

* add disableLocalAuth for computeInstance

* fix disableLocalAuthAuth issue for amlCompute

* update compute instance

* update recordings

* temp changes

* Revert "temp changes"

This reverts commit 64e3c38.

* update recordings

* fix tests

* Simplify the API by rolling up the static methods and hiding internals.

* Lock the ._converters._ai_services behind an import error.

* Print to install azure-ai-projects if we can't import AIAgentConverter

* By default, include all previous runs' tool calls and results.

* Don't crash if there is no content in historical thread messages.

* Parallelize the calls to get step_details for each run_id.

* Addressing PR comments.

* Use a single underscore to hide internal static members.

---------

Co-authored-by: Prashant Dhote <168401122+pdhotems@users.noreply.github.com>

Author: thecsw

sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/__init__.py

@@ -38,6 +38,16 @@
     OpenAIModelConfiguration,
 )
 
+# The converter from the AI service to the evaluator schema requires a dependency on
+# ai.projects, but we also don't want to force users installing ai.evaluations to pull
+# in ai.projects. So we only import it if it's available and the user has ai.projects.
+try:
+    from ._converters._ai_services import AIAgentConverter
+    _patch_all = ["AIAgentConverter"]
+except ImportError:
+    print("Could not import AIAgentConverter. Please install the dependency with `pip install azure-ai-projects`.")
+    _patch_all = []
+
 __all__ = [
     "evaluate",
     "CoherenceEvaluator",
@@ -72,3 +82,5 @@
     "ISAEvaluator",
     "ToolCallAccuracyEvaluator",
 ]
+
+__all__.extend([p for p in _patch_all if p not in __all__])

sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_converters/__init__.py

@@ -0,0 +1,3 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------

sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_converters/_ai_services.py

@@ -0,0 +1,259 @@
+import datetime
+import json
+
+from pydantic import BaseModel
+
+from azure.ai.projects.models import RunStepFunctionToolCall
+
+from typing import List, Optional, Union
+
+# Message roles constants.
+_SYSTEM = "system"
+_USER = "user"
+_AGENT = "assistant"
+_TOOL = "tool"
+
+# Constant definitions for what tool details include.
+_TOOL_CALL = "tool_call"
+_TOOL_RESULT = "tool_result"
+_FUNCTION = "function"
+
+# This is returned by AI services in the API to filter against tool invocations.
+_TOOL_CALLS = "tool_calls"
+
+
+class Message(BaseModel):
+    """Represents a message in a conversation with agents, assistants, and tools. We need to export these structures
+    to JSON for evaluators and we have custom fields such as createdAt, run_id, and tool_call_id, so we cannot use
+    the standard pydantic models provided by OpenAI.
+
+    :param createdAt: The timestamp when the message was created.
+    :type createdAt: datetime.datetime
+    :param run_id: The ID of the run associated with the message. Optional.
+    :type run_id: Optional[str]
+    :param role: The role of the message sender (e.g., system, user, tool, assistant).
+    :type role: str
+    :param content: The content of the message, which can be a string or a list of dictionaries.
+    :type content: Union[str, List[dict]]
+    """
+
+    createdAt: Optional[Union[datetime.datetime, int]] = None  # SystemMessage wouldn't have this
+    run_id: Optional[str] = None
+    tool_call_id: Optional[str] = None  # see ToolMessage
+    role: str
+    content: Union[str, List[dict]]
+
+
+class SystemMessage(Message):
+    """Represents a system message in a conversation with agents, assistants, and tools.
+
+    :param role: The role of the message sender, which is always 'system'.
+    :type role: str
+    """
+
+    role: str = _SYSTEM
+
+
+class UserMessage(Message):
+    """Represents a user message in a conversation with agents, assistants, and tools.
+
+    :param role: The role of the message sender, which is always 'user'.
+    :type role: str
+    """
+
+    role: str = _USER
+
+
+class ToolMessage(Message):
+    """Represents a tool message in a conversation with agents, assistants, and tools.
+
+    :param run_id: The ID of the run associated with the message.
+    :type run_id: str
+    :param role: The role of the message sender, which is always 'tool'.
+    :type role: str
+    :param tool_call_id: The ID of the tool call associated with the message. Optional.
+    :type tool_call_id: Optional[str]
+    """
+
+    run_id: str
+    role: str = _TOOL
+    tool_call_id: Optional[str] = None
+
+
+class AssistantMessage(Message):
+    """Represents an assistant message.
+
+    :param run_id: The ID of the run associated with the message.
+    :type run_id: str
+    :param role: The role of the message sender, which is always 'assistant'.
+    :type role: str
+    """
+
+    run_id: str
+    role: str = _AGENT
+
+
+class ToolDefinition(BaseModel):
+    """Represents a tool definition that will be used in the agent.
+
+    :param name: The name of the tool.
+    :type name: str
+    :param description: A description of the tool.
+    :type description: str
+    :param parameters: The parameters required by the tool.
+    :type parameters: dict
+    """
+
+    name: str
+    description: Optional[str] = None
+    parameters: dict
+
+
+class ToolCall:
+    """Represents a tool call, used as an intermediate step in the conversion process.
+
+    :param created: The timestamp when the tool call was created.
+    :type created: datetime.datetime
+    :param completed: The timestamp when the tool call was completed.
+    :type completed: datetime.datetime
+    :param details: The details of the tool call.
+    :type details: RunStepFunctionToolCall
+    """
+
+    def __init__(self, created: datetime.datetime, completed: datetime.datetime, details: RunStepFunctionToolCall):
+        self.created = created
+        self.completed = completed
+        self.details = details
+
+
+class EvaluatorData(BaseModel):
+    """Represents the result of a conversion.
+
+    :param query: A list of messages representing the system message, chat history, and user query.
+    :type query: List[Message]
+    :param response: A list of messages representing the assistant's response, including tool calls and results.
+    :type response: List[Message]
+    :param tool_definitions: A list of tool definitions used in the agent.
+    :type tool_definitions: List[ToolDefinition]
+    """
+
+    query: List[Message]
+    response: List[Message]
+    tool_definitions: List[ToolDefinition]
+
+    def to_json(self):
+        """Converts the result to a JSON string.
+
+        :return: The JSON representation of the result.
+        :rtype: str
+        """
+        return self.model_dump_json(exclude={}, exclude_none=True)
+
+
+def break_tool_call_into_messages(tool_call: ToolCall, run_id: str) -> List[Message]:
+    """
+    Breaks a tool call into a list of messages, including the tool call and its result.
+
+    :param tool_call: The tool call to be broken into messages.
+    :type tool_call: ToolCall
+    :param run_id: The ID of the run associated with the messages.
+    :type run_id: str
+    :return: A list of messages representing the tool call and its result.
+    :rtype: List[Message]
+    """
+    # We will use this as our accumulator.
+    messages: List[Message] = []
+
+    # As of March 17th, 2025, we only support custom functions due to built-in code interpreters and bing grounding
+    # tooling not reporting their function calls in the same way. Code interpreters don't include the tool call at
+    # all in most of the cases, and bing would only show the API URL, without arguments or results.
+    # Bing grounding would have "bing_grounding" in details with "requesturl" that will just be the API path with query.
+    # TODO: Work with AI Services to add converter support for BingGrounding and CodeInterpreter.
+    if not hasattr(tool_call.details, _FUNCTION):
+        return messages
+
+    # This is the internals of the content object that will be included with the tool call.
+    tool_call_id = tool_call.details.id
+    content_tool_call = {
+        "type": _TOOL_CALL,
+        _TOOL_CALL: {
+            "id": tool_call_id,
+            "type": _FUNCTION,
+            _FUNCTION: {
+                "name": tool_call.details.function.name,
+                "arguments": safe_loads(tool_call.details.function.arguments),
+            },
+        },
+    }
+
+    # We format it into an assistant message, where the content is a singleton list of the content object.
+    # It should be a tool message, since this is the call, but the given schema treats this message as
+    # assistant's action of calling the tool.
+    messages.append(AssistantMessage(run_id=run_id, content=[to_dict(content_tool_call)], createdAt=tool_call.created))
+
+    # Now, onto the tool result, which only includes the result of the function call.
+    content_tool_call_result = {"type": _TOOL_RESULT, _TOOL_RESULT: safe_loads(tool_call.details.function.output)}
+
+    # Since this is a tool's action of returning, we put it as a tool message.
+    messages.append(
+        ToolMessage(
+            run_id=run_id,
+            tool_call_id=tool_call_id,
+            content=[to_dict(content_tool_call_result)],
+            createdAt=tool_call.completed,
+        )
+    )
+    return messages
+
+
+def to_dict(obj) -> dict:
+    """
+    Converts an object to a dictionary.
+
+    :param obj: The object to be converted.
+    :type obj: Any
+    :return: The dictionary representation of the object.
+    :rtype: dict
+    """
+    return json.loads(json.dumps(obj))
+
+
+def safe_loads(data: str) -> Union[dict, str]:
+    """
+    Safely loads a JSON string into a Python dictionary or returns the original string if loading fails.
+    :param data: The JSON string to be loaded.
+    :type data: str
+    :return: The loaded dictionary or the original string.
+    :rtype: Union[dict, str]
+    """
+    try:
+        return json.loads(data)
+    except json.JSONDecodeError:
+        return data
+
+
+def convert_message(msg: dict) -> Message:
+    """
+    Converts a dictionary to the appropriate Message subclass.
+
+    :param msg: The message dictionary.
+    :type msg: dict
+    :return: The Message object.
+    :rtype: Message
+    """
+    role = msg["role"]
+    if role == "system":
+        return SystemMessage(content=str(msg["content"]))
+    elif role == "user":
+        return UserMessage(content=msg["content"], createdAt=msg["createdAt"])
+    elif role == "assistant":
+        return AssistantMessage(run_id=str(msg["run_id"]), content=msg["content"], createdAt=msg["createdAt"])
+    elif role == "tool":
+        return ToolMessage(
+            run_id=str(msg["run_id"]),
+            tool_call_id=str(msg["tool_call_id"]),
+            content=msg["content"],
+            createdAt=msg["createdAt"],
+        )
+    else:
+        raise ValueError(f"Unknown role: {role}")

sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_converters/_models.py

@@ -3,6 +3,7 @@
 ### Features Added
 
 ### Bugs Fixed
+ - Fix for compute Instance, disableLocalAuth property should be depend on ssh public access enabled.
 
 ## 1.26.0 (2025-03-11)
 

sdk/ml/azure-ai-ml/CHANGELOG.md

@@ -2,5 +2,5 @@
   "AssetsRepo": "Azure/azure-sdk-assets",
   "AssetsRepoPrefixPath": "python",
   "TagPrefix": "python/ml/azure-ai-ml",
-  "Tag": "python/ml/azure-ai-ml_a2c955e6e2"
+  "Tag": "python/ml/azure-ai-ml_305b890d5b"
 }

sdk/ml/azure-ai-ml/assets.json

@@ -251,7 +251,7 @@ def _to_rest_object(self) -> ComputeResource:
             ),
         )
         remote_login_public_access = "Enabled"
-        disableLocalAuth = not (self.ssh_public_access_enabled and self.ssh_settings is not None)
+        disableLocalAuth = not (self.ssh_settings)
         if self.ssh_public_access_enabled is not None:
             remote_login_public_access = "Enabled" if self.ssh_public_access_enabled else "Disabled"
 

sdk/ml/azure-ai-ml/azure/ai/ml/entities/_compute/aml_compute.py

@@ -280,12 +280,14 @@ def _to_rest_object(self) -> ComputeResource:
             subnet_resource = None
 
         ssh_settings = None
+        disable_local_auth = True
         if self.ssh_public_access_enabled is not None or self.ssh_settings is not None:
             ssh_settings = CiSShSettings()
             ssh_settings.ssh_public_access = "Enabled" if self.ssh_public_access_enabled else "Disabled"
             ssh_settings.admin_public_key = (
                 self.ssh_settings.ssh_key_value if self.ssh_settings and self.ssh_settings.ssh_key_value else None
             )
+            disable_local_auth = not self.ssh_public_access_enabled
 
         personal_compute_instance_settings = None
         if self.create_on_behalf_of:
@@ -330,6 +332,7 @@ def _to_rest_object(self) -> ComputeResource:
             description=self.description,
             compute_type=self.type,
             properties=compute_instance_prop,
+            disable_local_auth=disable_local_auth,
         )
         return ComputeResource(
             location=self.location,

sdk/ml/azure-ai-ml/azure/ai/ml/entities/_compute/compute_instance.py

@@ -30,6 +30,8 @@ def test_aml_compute_create_and_delete(self, client: MLClient, rand_compute_name
         assert compute_resource_get.name == compute_name
         assert compute_resource_get.tier == "dedicated"
         assert compute_resource_get.location == compute.location
+        assert compute_resource_get.ssh_public_access_enabled == True
+        assert compute_resource_get.ssh_settings.admin_username == "azureuser"
 
         compute_resource_get.idle_time_before_scale_down = 200
         compute_update_poller = client.compute.begin_update(compute_resource_get)
@@ -46,7 +48,6 @@ def test_aml_compute_create_and_delete(self, client: MLClient, rand_compute_name
         # so this is a preferred approach to assert
         assert isinstance(outcome, LROPoller)
 
-    @pytest.mark.skip(reason="not enough capacity")
     def test_compute_instance_create_and_delete(
         self, client: MLClient, rand_compute_name: Callable[[str], str]
     ) -> None:
@@ -65,20 +66,11 @@ def test_compute_instance_create_and_delete(
         assert isinstance(compute_resource_list, ItemPaged)
         compute_resource_get = client.compute.get(name=compute_name)
         assert compute_resource_get.name == compute_name
-        assert compute_resource_get.identity.type == "system_assigned"
         outcome = client.compute.begin_delete(name=compute_name)
         # the compute is getting deleted , but not waiting on the poller! so immediately returning
         # so this is a preferred approach to assert
         assert isinstance(outcome, LROPoller)
 
-    @pytest.mark.skipif(
-        condition=not is_live(),
-        reason=(
-            "Test takes 5 minutes in automation. "
-            "Already have unit tests verifying correct _restclient method is called. "
-            "Can be validated in live build only."
-        ),
-    )
     def test_compute_instance_stop_start_restart(
         self, client: MLClient, rand_compute_name: Callable[[str], str]
     ) -> None:

sdk/ml/azure-ai-ml/tests/compute/e2etests/test_compute.py

@@ -75,6 +75,8 @@ def test_compute_from_yaml(self):
         assert rest_intermediate.location == compute.location
         assert rest_intermediate.tags is not None
         assert rest_intermediate.tags["test"] == "true"
+        assert rest_intermediate.properties.disable_local_auth is False
+        assert rest_intermediate.properties.properties.remote_login_port_public_access == "Enabled"
 
         serializer = Serializer({"ComputeResource": ComputeResource})
         body = serializer.body(rest_intermediate, "ComputeResource")
@@ -98,6 +100,19 @@ def test_aml_compute_from_yaml_with_disable_public_access(self):
         assert rest_intermediate.properties.properties.enable_node_public_ip
         assert rest_intermediate.properties.disable_local_auth is True
         assert rest_intermediate.location == compute.location
+        assert rest_intermediate.properties.properties.remote_login_port_public_access == "NotSpecified"
+
+    def test_aml_compute_from_yaml_with_creds_and_disable_public_access(self):
+        compute: AmlCompute = load_compute("tests/test_configs/compute/compute-aml-no-identity.yaml")
+        compute.ssh_public_access_enabled = False
+
+        rest_intermediate = compute._to_rest_object()
+
+        assert rest_intermediate.properties.compute_type == "AmlCompute"
+        assert rest_intermediate.properties.properties.enable_node_public_ip
+        assert rest_intermediate.properties.disable_local_auth is False
+        assert rest_intermediate.location == compute.location
+        assert rest_intermediate.properties.properties.remote_login_port_public_access == "Disabled"
 
     def test_aml_compute_from_yaml_with_disable_public_access_when_no_sshSettings(self):
 
@@ -112,6 +127,7 @@ def test_aml_compute_from_yaml_with_disable_public_access_when_no_sshSettings(se
         assert rest_intermediate.properties.compute_type == "AmlCompute"
         assert rest_intermediate.properties.properties.enable_node_public_ip
         assert rest_intermediate.properties.disable_local_auth is True
+        assert rest_intermediate.properties.properties.remote_login_port_public_access == "Enabled"
         assert rest_intermediate.location == compute.location
 
     def test_compute_vm_from_yaml(self):
@@ -126,6 +142,7 @@ def test_compute_vm_from_yaml(self):
         assert compute.ssh_settings.ssh_private_key_file == "tests/test_configs/compute/ssh_fake_key.txt"
 
         rest_intermediate = compute._to_rest_object()
+        assert rest_intermediate.properties.compute_type == "VirtualMachine"
         assert rest_intermediate.properties.resource_id == resource_id
         assert rest_intermediate.properties.properties.ssh_port == 8888
         assert rest_intermediate.properties.properties.administrator_account.password == "azureuserpassword"
@@ -183,6 +200,7 @@ def test_compute_instance_load_from_rest(self):
         compute_instance3: ComputeInstance = load_compute(
             source="tests/test_configs/compute/compute-ci-defaults-unit.yaml",
         )._to_rest_object()
+        assert compute_instance3.properties.compute_type == "ComputeInstance"
         assert compute_instance3.properties.properties.enable_sso is True
         assert compute_instance3.properties.properties.enable_root_access is True
         assert compute_instance3.properties.properties.enable_os_patching is False
@@ -291,14 +309,28 @@ def test_compute_instance_uai_from_yaml(self):
         )
         assert compute_from_rest.ssh_public_access_enabled == False
 
-    def test_compute_instance_sai_from_yaml(self):
+    def test_compute_instace_to_rest(self):
         compute: ComputeInstance = load_compute("tests/test_configs/compute/compute-ci.yaml")
+        compute.ssh_public_access_enabled = True
+
+        compute_rest = compute._to_rest_object()
+
+        assert compute_rest.location == compute.location
+        assert compute_rest.properties.properties.ssh_settings.ssh_public_access == "Enabled"
+        assert compute_rest.properties.properties.ssh_settings.admin_public_key == compute.ssh_settings.ssh_key_value
+        assert compute_rest.properties.description == compute.description
+        assert compute_rest.properties.compute_type == "ComputeInstance"
+        assert compute_rest.properties.disable_local_auth == False
+
+    def test_compute_instance_sai_from_yaml(self):
+        compute: ComputeInstance = load_compute("tests/test_configs/compute/compute-ci-id-sys-assigned.yaml")
         assert compute.name == "banchci"
         assert compute.type == "computeinstance"
         assert compute.identity.type == "system_assigned"
 
         compute_resource = compute._to_rest_object()
         assert compute_resource.identity.type == "SystemAssigned"
+        assert compute_resource.properties.disable_local_auth == True
 
         compute_from_rest = Compute._from_rest_object(compute_resource)
         assert compute_from_rest.type == "computeinstance"

sdk/ml/azure-ai-ml/tests/compute/unittests/test_compute_entity.py

@@ -2,7 +2,7 @@ name: banchaml
 type: amlcompute
 tier: dedicated
 description: some_desc_aml
-size: Standard_DS2_v2
+size: Standard_DS3_v2
 location: eastus
 ssh_public_access_enabled: true
 ssh_settings:

sdk/ml/azure-ai-ml/tests/test_configs/compute/compute-aml-no-identity.yaml

@@ -0,0 +1,10 @@
+name: banchci
+type: computeinstance
+description: some_desc_ci
+size: Standard_DS3_v2
+ssh_public_access_enabled: false
+ssh_settings:
+   admin_username: azureuser
+   ssh_key_value: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDWm/4TTHMZdzZVJcob2aFRWDmIyJLxk028AKh7K03RDgR8bz/Knd9DgB2V3sipjY9wYJ1U9YbsUoAt24+CWZFpnoB79J6vaoqwUb7c6nGkaNrWoGZKI+v26GOX8O2MUjjOyBuPEinJtQ432J4affHfeRI+1iDQXuwFUKhNbRVpxh2h9otXF+J1UvSUaPYggS7Iivyha/x8HJzFcNnIPrAZkPiT/Nb/Qk7FyoFTEw64cIl1ByvmF3ewSOeVXKDpb2d4vrSDTVmXFKRrWduhM3sHO5dckREKc+tSQ0M+SpfBqaBgRTAie4jFVSIiSHCvL2BRQXBMRiyPgNlpiJWTPLp/ administrator@MININT-7IP9G6S
+identity:
+   type: system_assigned

sdk/ml/azure-ai-ml/tests/test_configs/compute/compute-ci-id-sys-assigned.yaml

@@ -1,12 +1,8 @@
 name: banchci
 type: computeinstance
 description: some_desc_ci
-location: eastus
-
-size: Standard_DS2_v2
-
-ssh_public_access_enabled: False
+size: Standard_DS3_v2
+ssh_public_access_enabled: false
 ssh_settings:
+   admin_username: azureuser
    ssh_key_value: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDWm/4TTHMZdzZVJcob2aFRWDmIyJLxk028AKh7K03RDgR8bz/Knd9DgB2V3sipjY9wYJ1U9YbsUoAt24+CWZFpnoB79J6vaoqwUb7c6nGkaNrWoGZKI+v26GOX8O2MUjjOyBuPEinJtQ432J4affHfeRI+1iDQXuwFUKhNbRVpxh2h9otXF+J1UvSUaPYggS7Iivyha/x8HJzFcNnIPrAZkPiT/Nb/Qk7FyoFTEw64cIl1ByvmF3ewSOeVXKDpb2d4vrSDTVmXFKRrWduhM3sHO5dckREKc+tSQ0M+SpfBqaBgRTAie4jFVSIiSHCvL2BRQXBMRiyPgNlpiJWTPLp/ administrator@MININT-7IP9G6S
-identity:
-   type: system_assigned