Users/singankit/evaluation async samples (#38104)

* Adding async sample for evaluation

* Update sample_evaluations_async.py

Author: singankit

sdk/ai/azure-ai-projects/azure/ai/projects/aio/_patch.py

@@ -6,7 +6,10 @@
 
 Follow our quickstart for examples: https://aka.ms/azsdk/python/dpcodegen/python/customize
 """
-from typing import List, Any
+import uuid
+from os import PathLike
+from pathlib import Path
+from typing import List, Any, Union, Dict
 from azure.core import AsyncPipelineClient
 from azure.core.pipeline import policies
 from typing_extensions import Self
@@ -155,7 +158,7 @@ def __init__(
             project_name=project_name,
             credential=credential,
             api_version="2024-07-01-preview",  # TODO: Update me
-            credential_scopes=["https://management.azure.com"],  # TODO: Update once service changes are ready
+            credential_scopes=["https://ml.azure.com"],  # TODO: Update once service changes are ready
             **kwargs3,
         )
         _policies3 = kwargs3.pop("policies", None)
@@ -226,6 +229,49 @@ def from_connection_string(cls, conn_str: str, credential: "AsyncTokenCredential
         project_name = parts[3]
         return cls(endpoint, subscription_id, resource_group_name, project_name, credential, **kwargs)
 
+    def upload_file(self, file_path: Union[Path, str, PathLike]) -> str:
+        """Upload a file to the Azure AI Studio project.
+           This method required *azure-ai-ml* to be installed.
+
+        :param file_path: The path to the file to upload.
+        :type file_path: Union[str, Path, PathLike]
+        :return: The asset id of uploaded file.
+        :rtype: str
+        """
+        try:
+            from azure.ai.ml import MLClient
+            from azure.ai.ml.entities import Data
+            from azure.ai.ml.constants import AssetTypes
+        except ImportError:
+            raise ImportError(
+                "azure-ai-ml must be installed to use this function. Please install it using `pip install azure-ai-ml`")
+
+        data = Data(
+            path=file_path,
+            type=AssetTypes.URI_FILE,
+            name=str(uuid.uuid4()),  # generating random name
+            is_anonymous=True,
+            version="1",
+        )
+
+        ml_client = MLClient(
+            self._config3.credential,
+            self._config3.subscription_id,
+            self._config3.resource_group_name,
+            self._config3.project_name,
+        )
+
+        data_asset = ml_client.data.create_or_update(data)
+
+        return data_asset.id
+
+    @property
+    def scope(self) -> Dict[str, str]:
+        return {
+            "subscription_id": self._config3.subscription_id,
+            "resource_group_name": self._config3.resource_group_name,
+            "project_name": self._config3.project_name,
+        }
 
 __all__: List[str] = [
     "AIProjectClient",

sdk/ai/azure-ai-projects/samples/evaluations/async_samples/evaluate_test_data.jsonl

@@ -0,0 +1,3 @@
+{"query":"How do you create a run?","context":"AML API only","response":"To create a run using the Azure Machine Learning API, you first need to create an Experiment. Once you have an experiment, you can create a Run object that is associated with that experiment. Here is some Python code that demonstrates this process:\n\n```\nfrom azureml.core import Experiment, Run\nfrom azureml.core.workspace import Workspace\n\n# Define workspace and experiment\nws = Workspace.from_config()\nexp = Experiment(workspace=ws, name='my_experiment')\n\n# Create a new run\nrun = exp.start_logging()\n```\n\nIn this code, the `from_config()` method reads the configuration file that you created when you set up your Azure Machine Learning workspace. The `Experiment` constructor creates an Experiment object that is associated with your workspace, and the `start_logging()` method creates a new Run object that is associated with the Experiment. Now you can use the `run` object to log metrics, upload files, and track other information related to your machine learning experiment.", "ground_truth":"Paris is the capital of France."}
+{"query":"How do you log a model?","context":"Logging can be done using any OSS Sdk","response":"There are a few ways to log models in Azure Machine Learning. \n\nOne way is to use the `register_model()` method of the `Run` object. The `register_model()` method logs a model file in the Azure Machine Learning service workspace and makes it available for deployment. Here's an example:\n\n```python\nfrom azureml.core import Model\n\nmodel_path = '.\/outputs\/my_model.pkl'\nmodel = Model.register(workspace=ws, model_path=model_path, model_name='my_model')\n```\n\nThis code registers the model file located at `model_path` to the Azure Machine Learning service workspace with the name `my_model`. \n\nAnother way to log a model is to save it as an output of a `Run`. If your model generation code is part of a script or Jupyter notebook that runs as an Azure Machine Learning experiment, you can save the model file as an output of the `Run` object. Here's an example:\n\n```python\nfrom sklearn.linear_model import LogisticRegression\nfrom azureml.core.run import Run\n\n# Initialize a run object\nrun = Run.get_context()\n\n# Train your model\nX_train, y_train = ...\nclf = LogisticRegression().fit(X_train, y_train)\n\n# Save the model to the Run object's outputs directory\nmodel_path = 'outputs\/model.pkl'\njoblib.dump(value=clf, filename=model_path)\n\n# Log the model as a run artifact\nrun.upload_file(name=model_path, path_or_stream=model_path)\n```\n\nIn this code, `Run.get_context()` retrieves the current run context object, which you can use to track metadata and metrics for the run. After training your model, you can use `joblib.dump()` to save the model to a file, and then log the file as an artifact of the run using `run.upload_file()`.","ground_truth":"Paris is the capital of France."}
+{"query":"What is the capital of France?","context":"France is in Europe","response":"Paris is the capital of France.", "ground_truth":"Paris is the capital of France."}

sdk/ai/azure-ai-projects/samples/evaluations/async_samples/sample_evaluations_async.py

@@ -0,0 +1,92 @@
+# ------------------------------------
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT License.
+# ------------------------------------
+
+"""
+FILE: sample_agents_basics_async.py
+
+DESCRIPTION:
+    This sample demonstrates how to use evaluation operations from
+    the Azure Evaluation service using a asynchronous client.
+
+USAGE:
+    python sample_evaluation_async.py
+
+    Before running the sample:
+
+    pip install azure.ai.projects azure-identity aiohttp
+
+    Set this environment variables with your own values:
+    PROJECT_CONNECTION_STRING - the Azure AI Project connection string, as found in your AI Studio Project.
+"""
+import asyncio
+import time
+import os
+
+from azure.ai.projects.aio import AIProjectClient
+from azure.identity import DefaultAzureCredential
+from azure.ai.projects.models import Evaluation, Dataset, EvaluatorConfiguration, ConnectionType
+from azure.ai.evaluation import F1ScoreEvaluator, RelevanceEvaluator, ViolenceEvaluator
+
+
+async def main():
+    project_client = AIProjectClient.from_connection_string(
+        credential=DefaultAzureCredential(),
+        conn_str=os.environ["PROJECT_CONNECTION_STRING"],
+    )
+
+    # Upload data for evaluation
+    data_id = project_client.upload_file("./evaluate_test_data.jsonl")
+
+    default_connection = await project_client.connections.get_default(connection_type=ConnectionType.AZURE_OPEN_AI)
+
+    deployment_name = "<>"
+    api_version = "<>"
+
+    # Create an evaluation
+    evaluation = Evaluation(
+        display_name="Remote Evaluation",
+        description="Evaluation of dataset",
+        data=Dataset(id=data_id),
+        evaluators={
+            "f1_score": EvaluatorConfiguration(
+                # id=F1ScoreEvaluator.id,
+                id="azureml://registries/azureml-staging/models/F1Score-Evaluator/versions/3",
+            ),
+            "relevance": EvaluatorConfiguration(
+                # id=RelevanceEvaluator.id,
+                id="azureml://registries/azureml-staging/models/Relevance-Evaluator/versions/3",
+                init_params={
+                    "model_config": default_connection.to_evaluator_model_config(deployment_name=deployment_name,
+                                                                                 api_version=api_version)
+                },
+            ),
+            "violence": EvaluatorConfiguration(
+                # id=ViolenceEvaluator.id,
+                id="azureml://registries/azureml-staging/models/Violent-Content-Evaluator/versions/3",
+                init_params={
+                    "azure_ai_project": project_client.scope
+                },
+            ),
+        },
+    )
+
+    async with project_client:
+        # Create evaluation
+        evaluation_response = await project_client.evaluations.create(evaluation)
+
+        # Get evaluation
+        get_evaluation_response = await project_client.evaluations.get(evaluation_response.id)
+
+        print("----------------------------------------------------------------")
+        print("Created evaluation, evaluation ID: ", get_evaluation_response.id)
+        print("Evaluation status: ", get_evaluation_response.status)
+        print("AI Studio URI: ", get_evaluation_response.properties["AiStudioEvaluationUri"])
+        print("----------------------------------------------------------------")
+
+
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

sdk/ai/azure-ai-projects/samples/evaluations/sample_evaluations.py

@@ -39,14 +39,12 @@
 )
 
 # Upload data for evaluation
-# Service side fix needed to make this work
-# data_id = project_client.upload_file("./evaluate_test_data.jsonl")
-data_id = (
-    "azureml://locations/eastus2/workspaces/faa79f3d-91b3-4ed5-afdc-4cc0fe13fb85/data/remote-evals-data/versions/3"
-)
+data_id = project_client.upload_file("./evaluate_test_data.jsonl")
 
 default_connection = project_client.connections.get_default(connection_type=ConnectionType.AZURE_OPEN_AI)
 
+deployment_name = "<>"
+api_version = "<>"
 
 # Create an evaluation
 evaluation = Evaluation(
@@ -55,26 +53,28 @@
     data=Dataset(id=data_id),
     evaluators={
         "f1_score": EvaluatorConfiguration(
-            id=F1ScoreEvaluator.evaluator_id,
+            # id=F1ScoreEvaluator.id,
+            id="azureml://registries/azureml-staging/models/F1Score-Evaluator/versions/3",
         ),
         "relevance": EvaluatorConfiguration(
-            id=RelevanceEvaluator.evaluator_id,
+            # id=RelevanceEvaluator.id,
+            id="azureml://registries/azureml-staging/models/Relevance-Evaluator/versions/3",
             init_params={
-                "model_config": default_connection.to_evaluator_model_config(
-                    deployment_name="GPT-4-Prod", api_version="2024-08-01-preview"
-                )
+                "model_config": default_connection.to_evaluator_model_config(deployment_name=deployment_name,
+                                                                             api_version=api_version)
             },
         ),
-        "hate_unfairness": EvaluatorConfiguration(
-            id=HateUnfairnessEvaluator.evaluator_id,
-            init_params={"azure_ai_project": project_client.scope},
+        "violence": EvaluatorConfiguration(
+            # id=ViolenceEvaluator.id,
+            id="azureml://registries/azureml-staging/models/Violent-Content-Evaluator/versions/3",
+            init_params={
+                "azure_ai_project": project_client.scope
+            },
         ),
     },
-    # This is needed as a workaround until environment gets published to registry
-    properties={"Environment": "azureml://registries/jamahaja-evals-registry/environments/eval-remote-env/versions/6"},
 )
 
-# Create evaluation
+
 evaluation_response = project_client.evaluations.create(
     evaluation=evaluation,
 )