From 6798c588a6c735c8789f12cb21dc2f6862f7cfb7 Mon Sep 17 00:00:00 2001 From: Shyam Namboodiripad Date: Wed, 16 Jul 2025 13:21:27 -0700 Subject: [PATCH] Add support for new Azure AI Foundry project type for Safety evals Fixes #6592 --- .../ContentSafetyChatClient.cs | 27 ++- .../ContentSafetyService.UrlCacheKey.cs | 4 + .../ContentSafetyService.cs | 55 +++-- .../ContentSafetyServiceConfiguration.cs | 220 ++++++++++++++---- .../SafetyEvaluatorTests.cs | 89 ++++++- .../Settings.cs | 5 + .../appsettings.json | 3 +- 7 files changed, 325 insertions(+), 78 deletions(-) diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/ContentSafetyChatClient.cs b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/ContentSafetyChatClient.cs index 347975cb695..a8c1ba889d2 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/ContentSafetyChatClient.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/ContentSafetyChatClient.cs @@ -35,16 +35,27 @@ public ContentSafetyChatClient( ChatClientMetadata? originalMetadata = _originalChatClient?.GetService(); - string providerName = - $"{Moniker} (" + - $"Subscription: {contentSafetyServiceConfiguration.SubscriptionId}, " + - $"Resource Group: {contentSafetyServiceConfiguration.ResourceGroupName}, " + - $"Project: {contentSafetyServiceConfiguration.ProjectName})"; + string providerName; + Uri? providerUri = originalMetadata?.ProviderUri; + + if (contentSafetyServiceConfiguration.IsHubBasedProject) + { + providerName = + $"{Moniker} (" + + $"Subscription: {contentSafetyServiceConfiguration.SubscriptionId}, " + + $"Resource Group: {contentSafetyServiceConfiguration.ResourceGroupName}, " + + $"Project: {contentSafetyServiceConfiguration.ProjectName})"; + } + else + { + providerName = $"{Moniker} (Endpoint: {contentSafetyServiceConfiguration.Endpoint})"; + providerUri = contentSafetyServiceConfiguration.Endpoint; + } if (originalMetadata?.ProviderName is string originalProviderName && !string.IsNullOrWhiteSpace(originalProviderName)) { - providerName = $"{originalProviderName}; {providerName}"; + providerName = $"{providerName}; {originalProviderName}"; } string modelId = Moniker; @@ -52,10 +63,10 @@ public ContentSafetyChatClient( if (originalMetadata?.DefaultModelId is string originalModelId && !string.IsNullOrWhiteSpace(originalModelId)) { - modelId = $"{originalModelId}; {modelId}"; + modelId = $"{modelId}; {originalModelId}"; } - _metadata = new ChatClientMetadata(providerName, originalMetadata?.ProviderUri, modelId); + _metadata = new ChatClientMetadata(providerName, providerUri, modelId); } public async Task GetResponseAsync( diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/ContentSafetyService.UrlCacheKey.cs b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/ContentSafetyService.UrlCacheKey.cs index 41be29e9ed3..c8969a001c4 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/ContentSafetyService.UrlCacheKey.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/ContentSafetyService.UrlCacheKey.cs @@ -26,11 +26,14 @@ public bool Equals(UrlCacheKey? other) } else { +#pragma warning disable S1067 // Expressions should not be too complex return other.Configuration.SubscriptionId == Configuration.SubscriptionId && other.Configuration.ResourceGroupName == Configuration.ResourceGroupName && other.Configuration.ProjectName == Configuration.ProjectName && + other.Configuration.Endpoint == Configuration.Endpoint && other.AnnotationTask == AnnotationTask; +#pragma warning restore S1067 } } @@ -42,6 +45,7 @@ public override int GetHashCode() => Configuration.SubscriptionId, Configuration.ResourceGroupName, Configuration.ProjectName, + Configuration.Endpoint, AnnotationTask); } } diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/ContentSafetyService.cs b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/ContentSafetyService.cs index 6028a82544c..ee9bdf2c926 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/ContentSafetyService.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/ContentSafetyService.cs @@ -22,6 +22,9 @@ namespace Microsoft.Extensions.AI.Evaluation.Safety; internal sealed partial class ContentSafetyService(ContentSafetyServiceConfiguration serviceConfiguration) { + private const string APIVersionForServiceDiscoveryInHubBasedProjects = "?api-version=2023-08-01-preview"; + private const string APIVersionForNonHubBasedProjects = "?api-version=2025-05-15-preview"; + private static HttpClient? _sharedHttpClient; private static HttpClient SharedHttpClient { @@ -168,20 +171,27 @@ private async ValueTask GetServiceUrlAsync( return _serviceUrl; } - string discoveryUrl = - await GetServiceDiscoveryUrlAsync(evaluatorName, cancellationToken).ConfigureAwait(false); - - serviceUrl = - $"{discoveryUrl}/raisvc/v1.0" + - $"/subscriptions/{serviceConfiguration.SubscriptionId}" + - $"/resourceGroups/{serviceConfiguration.ResourceGroupName}" + - $"/providers/Microsoft.MachineLearningServices/workspaces/{serviceConfiguration.ProjectName}"; + if (serviceConfiguration.IsHubBasedProject) + { + string discoveryUrl = + await GetServiceDiscoveryUrlAsync(evaluatorName, cancellationToken).ConfigureAwait(false); + + serviceUrl = + $"{discoveryUrl}/raisvc/v1.0" + + $"/subscriptions/{serviceConfiguration.SubscriptionId}" + + $"/resourceGroups/{serviceConfiguration.ResourceGroupName}" + + $"/providers/Microsoft.MachineLearningServices/workspaces/{serviceConfiguration.ProjectName}"; + } + else + { + serviceUrl = $"{serviceConfiguration.Endpoint.AbsoluteUri}/evaluations"; + } await EnsureServiceAvailabilityAsync( - serviceUrl, - capability: annotationTask, - evaluatorName, - cancellationToken).ConfigureAwait(false); + serviceUrl, + capability: annotationTask, + evaluatorName, + cancellationToken).ConfigureAwait(false); _ = _serviceUrlCache.TryAdd(key, serviceUrl); _serviceUrl = serviceUrl; @@ -196,7 +206,7 @@ private async ValueTask GetServiceDiscoveryUrlAsync( $"https://management.azure.com/subscriptions/{serviceConfiguration.SubscriptionId}" + $"/resourceGroups/{serviceConfiguration.ResourceGroupName}" + $"/providers/Microsoft.MachineLearningServices/workspaces/{serviceConfiguration.ProjectName}" + - $"?api-version=2023-08-01-preview"; + $"{APIVersionForServiceDiscoveryInHubBasedProjects}"; HttpResponseMessage response = await GetResponseAsync( @@ -244,7 +254,10 @@ private async ValueTask EnsureServiceAvailabilityAsync( string evaluatorName, CancellationToken cancellationToken) { - string serviceAvailabilityUrl = $"{serviceUrl}/checkannotation"; + string serviceAvailabilityUrl = + serviceConfiguration.IsHubBasedProject + ? $"{serviceUrl}/checkannotation" + : $"{serviceUrl}/checkannotation{APIVersionForNonHubBasedProjects}"; HttpResponseMessage response = await GetResponseAsync( @@ -297,7 +310,10 @@ private async ValueTask SubmitAnnotationRequestAsync( string evaluatorName, CancellationToken cancellationToken) { - string annotationUrl = $"{serviceUrl}/submitannotation"; + string annotationUrl = + serviceConfiguration.IsHubBasedProject + ? $"{serviceUrl}/submitannotation" + : $"{serviceUrl}/submitannotation{APIVersionForNonHubBasedProjects}"; HttpResponseMessage response = await GetResponseAsync( @@ -426,10 +442,13 @@ private async ValueTask AddHeadersAsync( httpRequestMessage.Headers.Add("User-Agent", userAgent); + TokenRequestContext context = + serviceConfiguration.IsHubBasedProject + ? new TokenRequestContext(scopes: ["https://management.azure.com/.default"]) + : new TokenRequestContext(scopes: ["https://ai.azure.com/.default"]); + AccessToken token = - await serviceConfiguration.Credential.GetTokenAsync( - new TokenRequestContext(scopes: ["https://management.azure.com/.default"]), - cancellationToken).ConfigureAwait(false); + await serviceConfiguration.Credential.GetTokenAsync(context, cancellationToken).ConfigureAwait(false); httpRequestMessage.Headers.Authorization = new AuthenticationHeaderValue("Bearer", token.Token); diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/ContentSafetyServiceConfiguration.cs b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/ContentSafetyServiceConfiguration.cs index ec721fa59c7..06d4a045ced 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/ContentSafetyServiceConfiguration.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/ContentSafetyServiceConfiguration.cs @@ -6,64 +6,62 @@ // We disable this warning because it is a false positive arising from the analyzer's lack of support for C#'s primary // constructor syntax. +using System; +using System.Diagnostics.CodeAnalysis; using System.Net.Http; using Azure.Core; +using Microsoft.Shared.Diagnostics; namespace Microsoft.Extensions.AI.Evaluation.Safety; /// -/// Specifies configuration parameters such as the Azure AI project that should be used, and the credentials that -/// should be used, when a communicates with the Azure AI Foundry Evaluation +/// Specifies configuration parameters such as the Azure AI Foundry project that should be used, and the credentials +/// that should be used, when a communicates with the Azure AI Foundry Evaluation /// service to perform evaluations. /// -/// -/// The Azure that should be used when authenticating requests. -/// -/// -/// The ID of the Azure subscription that contains the project identified by . -/// -/// -/// The name of the Azure resource group that contains the project identified by . -/// -/// -/// The name of the Azure AI project. -/// -/// -/// The that should be used when communicating with the Azure AI Foundry Evaluation service. -/// While the parameter is optional, it is recommended to supply an that is configured with -/// robust resilience and retry policies. -/// -/// -/// The timeout (in seconds) after which a should stop retrying failed attempts -/// to communicate with the Azure AI Foundry Evaluation service when performing evaluations. -/// -public sealed class ContentSafetyServiceConfiguration( - TokenCredential credential, - string subscriptionId, - string resourceGroupName, - string projectName, - HttpClient? httpClient = null, - int timeoutInSecondsForRetries = 300) // 5 minutes +/// +/// +/// Note that Azure AI Foundry supports two kinds of projects - Hub-based projects and non-Hub-based projects (also +/// known simply as Foundry projects). See https://learn.microsoft.com/en-us/azure/ai-foundry/how-to/create-projects. +/// +/// +/// Hub-based projects are configured by specifying the , the +/// , and the for the project. Non-Hub-based projects, on the +/// other hand, are configured by specifying only the for the project. Use the appropriate +/// constructor overload to initialize based on the kind of project you +/// are working with. +/// +/// +public sealed class ContentSafetyServiceConfiguration { + private const int DefaultTimeoutInSecondsForRetries = 300; // 5 minutes + /// /// Gets the Azure that should be used when authenticating requests. /// - public TokenCredential Credential { get; } = credential; + public TokenCredential Credential { get; } + + /// + /// Gets the ID of the Azure subscription that contains the project identified by if the + /// project is a Hub-based project. + /// + public string? SubscriptionId { get; } /// - /// Gets the ID of the Azure subscription that contains the project identified by . + /// Gets the name of the Azure resource group that contains the project identified by if + /// the project is a Hub-based project. /// - public string SubscriptionId { get; } = subscriptionId; + public string? ResourceGroupName { get; } /// - /// Gets the name of the Azure resource group that contains the project identified by . + /// Gets the name of the Azure AI Foundry project if the project is a Hub-based project. /// - public string ResourceGroupName { get; } = resourceGroupName; + public string? ProjectName { get; } /// - /// Gets the name of the Azure AI project. + /// Gets the endpoint for the Azure AI Foundry project if the project is a non-Hub-based project. /// - public string ProjectName { get; } = projectName; + public Uri? Endpoint { get; } /// /// Gets the that should be used when communicating with the Azure AI Foundry Evaluation @@ -73,11 +71,155 @@ public sealed class ContentSafetyServiceConfiguration( /// While supplying an is optional, it is recommended to supply one that is configured /// with robust resilience and retry policies. /// - public HttpClient? HttpClient { get; } = httpClient; + public HttpClient? HttpClient { get; } /// /// Gets the timeout (in seconds) after which a should stop retrying failed /// attempts to communicate with the Azure AI Foundry Evaluation service when performing evaluations. /// - public int TimeoutInSecondsForRetries { get; } = timeoutInSecondsForRetries; + public int TimeoutInSecondsForRetries { get; } + + [MemberNotNullWhen(true, nameof(SubscriptionId), nameof(ResourceGroupName), nameof(ProjectName))] + [MemberNotNullWhen(false, nameof(Endpoint))] + internal bool IsHubBasedProject => + !string.IsNullOrWhiteSpace(SubscriptionId) && + !string.IsNullOrWhiteSpace(ResourceGroupName) && + !string.IsNullOrWhiteSpace(ProjectName) && + Endpoint is null; + + /// + /// Initializes a new instance of the class for a Hub-based Azure + /// AI Foundry project with the specified . + /// + /// + /// The Azure that should be used when authenticating requests. + /// + /// + /// The ID of the Azure subscription that contains the Hub-based AI Foundry project identified by + /// . + /// + /// + /// The name of the Azure resource group that contains the Hub-based AI Foundry project identified by + /// . + /// + /// + /// The name of the Hub-based Azure AI Foundry project. + /// + /// + /// The that should be used when communicating with the Azure AI Foundry Evaluation + /// service. While the parameter is optional, it is recommended to supply an that is + /// configured with robust resilience and retry policies. + /// + /// + /// The timeout (in seconds) after which a should stop retrying failed + /// attempts to communicate with the Azure AI Foundry Evaluation service when performing evaluations. + /// + /// + /// + /// Note that Azure AI Foundry supports two kinds of projects - Hub-based projects and non-Hub-based projects (also + /// known simply as Foundry projects). See + /// https://learn.microsoft.com/en-us/azure/ai-foundry/how-to/create-projects. + /// + /// + /// Use this constructor overload if you are working with a Hub-based project. + /// + /// + public ContentSafetyServiceConfiguration( + TokenCredential credential, + string subscriptionId, + string resourceGroupName, + string projectName, + HttpClient? httpClient = null, + int timeoutInSecondsForRetries = DefaultTimeoutInSecondsForRetries) + { + Credential = Throw.IfNull(credential); + SubscriptionId = Throw.IfNullOrWhitespace(subscriptionId); + ResourceGroupName = Throw.IfNullOrWhitespace(resourceGroupName); + ProjectName = Throw.IfNullOrWhitespace(projectName); + HttpClient = httpClient; + TimeoutInSecondsForRetries = timeoutInSecondsForRetries; + } + + /// + /// Initializes a new instance of the class for a non-Hub-based + /// Azure AI Foundry project with the specified . + /// + /// + /// The Azure that should be used when authenticating requests. + /// + /// + /// The endpoint for the non-Hub-based Azure AI Foundry project. + /// + /// + /// The that should be used when communicating with the Azure AI Foundry Evaluation + /// service. While the parameter is optional, it is recommended to supply an that is + /// configured with robust resilience and retry policies. + /// + /// + /// The timeout (in seconds) after which a should stop retrying failed + /// attempts to communicate with the Azure AI Foundry Evaluation service when performing evaluations. + /// + /// + /// + /// Note that Azure AI Foundry supports two kinds of projects - Hub-based projects and non-Hub-based projects (also + /// known simply as Foundry projects). See + /// https://learn.microsoft.com/en-us/azure/ai-foundry/how-to/create-projects. + /// + /// + /// Use this constructor overload if you are working with a non-Hub-based project. + /// + /// + public ContentSafetyServiceConfiguration( + TokenCredential credential, + Uri endpoint, + HttpClient? httpClient = null, + int timeoutInSecondsForRetries = DefaultTimeoutInSecondsForRetries) + { + Credential = Throw.IfNull(credential); + Endpoint = Throw.IfNull(endpoint); + HttpClient = httpClient; + TimeoutInSecondsForRetries = timeoutInSecondsForRetries; + } + + /// + /// Initializes a new instance of the class for a non-Hub-based + /// Azure AI Foundry project with the specified . + /// + /// + /// The Azure that should be used when authenticating requests. + /// + /// + /// The endpoint URL for the non-Hub-based Azure AI Foundry project. + /// + /// + /// The that should be used when communicating with the Azure AI Foundry Evaluation + /// service. While the parameter is optional, it is recommended to supply an that is + /// configured with robust resilience and retry policies. + /// + /// + /// The timeout (in seconds) after which a should stop retrying failed + /// attempts to communicate with the Azure AI Foundry Evaluation service when performing evaluations. + /// + /// + /// + /// Note that Azure AI Foundry supports two kinds of projects - Hub-based projects and non-Hub-based projects (also + /// known simply as Foundry projects). See + /// https://learn.microsoft.com/en-us/azure/ai-foundry/how-to/create-projects. + /// + /// + /// Use this constructor overload if you are working with a non-Hub-based project. + /// + /// + public ContentSafetyServiceConfiguration( + TokenCredential credential, + string endpointUrl, + HttpClient? httpClient = null, + int timeoutInSecondsForRetries = DefaultTimeoutInSecondsForRetries) + : this( + credential, + endpoint: new Uri(Throw.IfNullOrWhitespace(endpointUrl)), + httpClient, + timeoutInSecondsForRetries) + { + } } diff --git a/test/Libraries/Microsoft.Extensions.AI.Evaluation.Integration.Tests/SafetyEvaluatorTests.cs b/test/Libraries/Microsoft.Extensions.AI.Evaluation.Integration.Tests/SafetyEvaluatorTests.cs index 630adbffd8e..c0f955b8416 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Evaluation.Integration.Tests/SafetyEvaluatorTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Evaluation.Integration.Tests/SafetyEvaluatorTests.cs @@ -24,6 +24,7 @@ public class SafetyEvaluatorTests private static readonly ReportingConfiguration? _imageContentSafetyReportingConfiguration; private static readonly ReportingConfiguration? _codeVulnerabilityReportingConfiguration; private static readonly ReportingConfiguration? _mixedQualityAndSafetyReportingConfiguration; + private static readonly ReportingConfiguration? _hubBasedContentSafetyReportingConfiguration; static SafetyEvaluatorTests() { @@ -37,14 +38,11 @@ static SafetyEvaluatorTests() }; ChatConfiguration llmChatConfiguration = Setup.CreateChatConfiguration(); - ChatClientMetadata? clientMetadata = llmChatConfiguration.ChatClient.GetService(); string version = $"Product Version: {Constants.Version}"; string date = $"Date: {DateTime.UtcNow:dddd, dd MMMM yyyy}"; string projectName = $"Project: Integration Tests"; string testClass = $"Test Class: {nameof(SafetyEvaluatorTests)}"; - string provider = $"Model Provider: {clientMetadata?.ProviderName ?? "Unknown"}"; - string model = $"Model: {clientMetadata?.DefaultModelId ?? "Unknown"}"; string temperature = $"Temperature: {_chatOptions.Temperature}"; string usesContext = $"Feature: Context"; @@ -52,13 +50,17 @@ static SafetyEvaluatorTests() var contentSafetyServiceConfiguration = new ContentSafetyServiceConfiguration( credential, - subscriptionId: Settings.Current.AzureSubscriptionId, - resourceGroupName: Settings.Current.AzureResourceGroupName, - projectName: Settings.Current.AzureAIProjectName); + endpointUrl: Settings.Current.AzureAIProjectEndpoint); ChatConfiguration contentSafetyChatConfiguration = contentSafetyServiceConfiguration.ToChatConfiguration(llmChatConfiguration); + ChatClientMetadata? clientMetadata = + contentSafetyChatConfiguration.ChatClient.GetService(); + + string provider = $"Model Provider: {clientMetadata?.ProviderName ?? "Unknown"}"; + string model = $"Model: {clientMetadata?.DefaultModelId ?? "Unknown"}"; + IEvaluator hateAndUnfairnessEvaluator = new HateAndUnfairnessEvaluator(); IEvaluator selfHarmEvaluator = new SelfHarmEvaluator(); IEvaluator sexualEvaluator = new SexualEvaluator(); @@ -117,20 +119,65 @@ static SafetyEvaluatorTests() chatConfiguration: contentSafetyChatConfiguration, executionName: Constants.Version, tags: [version, date, projectName, testClass, provider, model, temperature]); + + var hubBasedContentSafetyServiceConfiguration = + new ContentSafetyServiceConfiguration( + credential, + subscriptionId: Settings.Current.AzureSubscriptionId, + resourceGroupName: Settings.Current.AzureResourceGroupName, + projectName: Settings.Current.AzureAIProjectName); + + ChatConfiguration hubBasedContentSafetyChatConfiguration = + hubBasedContentSafetyServiceConfiguration.ToChatConfiguration(llmChatConfiguration); + + clientMetadata = hubBasedContentSafetyChatConfiguration.ChatClient.GetService(); + + provider = $"Model Provider: {clientMetadata?.ProviderName ?? "Unknown"}"; + model = $"Model: {clientMetadata?.DefaultModelId ?? "Unknown"}"; + + _hubBasedContentSafetyReportingConfiguration = + DiskBasedReportingConfiguration.Create( + storageRootPath: Settings.Current.StorageRootPath, + evaluators: [ + selfHarmEvaluator, + sexualEvaluator, + protectedMaterialEvaluator, + groundednessProEvaluator, + ungroundedAttributesEvaluator, + indirectAttackEvaluator], + chatConfiguration: hubBasedContentSafetyChatConfiguration, + executionName: Constants.Version, + tags: [version, date, projectName, testClass, provider, model, temperature, usesContext]); } } [ConditionalFact] - public async Task EvaluateConversationWithSingleTurn() + public async Task EvaluateConversationWithSingleTurn_HubBasedProject() + { + SkipIfNotConfigured(); + + await using ScenarioRun scenarioRun = + await _hubBasedContentSafetyReportingConfiguration.CreateScenarioRunAsync( + scenarioName: $"Microsoft.Extensions.AI.Evaluation.Integration.Tests.{nameof(SafetyEvaluatorTests)}.{nameof(EvaluateConversationWithSingleTurn_HubBasedProject)}"); + + await EvaluateConversationWithSingleTurn(scenarioRun); + } + + [ConditionalFact] + public async Task EvaluateConversationWithSingleTurn_NonHubBasedProject() { SkipIfNotConfigured(); await using ScenarioRun scenarioRun = await _contentSafetyReportingConfiguration.CreateScenarioRunAsync( - scenarioName: $"Microsoft.Extensions.AI.Evaluation.Integration.Tests.{nameof(SafetyEvaluatorTests)}.{nameof(EvaluateConversationWithSingleTurn)}"); + scenarioName: $"Microsoft.Extensions.AI.Evaluation.Integration.Tests.{nameof(SafetyEvaluatorTests)}.{nameof(EvaluateConversationWithSingleTurn_NonHubBasedProject)}"); - IChatClient chatClient = scenarioRun.ChatConfiguration!.ChatClient; + await EvaluateConversationWithSingleTurn(scenarioRun); + } + private static async Task EvaluateConversationWithSingleTurn(ScenarioRun scenarioRun) + { + IChatClient chatClient = scenarioRun.ChatConfiguration!.ChatClient; var messages = new List(); string systemPrompt = @@ -183,16 +230,32 @@ The distance varies due to the elliptical orbits of both planets. } [ConditionalFact] - public async Task EvaluateConversationWithMultipleTurns() + public async Task EvaluateConversationWithMultipleTurns_HubBasedProject() + { + SkipIfNotConfigured(); + + await using ScenarioRun scenarioRun = + await _hubBasedContentSafetyReportingConfiguration.CreateScenarioRunAsync( + scenarioName: $"Microsoft.Extensions.AI.Evaluation.Integration.Tests.{nameof(SafetyEvaluatorTests)}.{nameof(EvaluateConversationWithMultipleTurns_HubBasedProject)}"); + + await EvaluateConversationWithMultipleTurns(scenarioRun); + } + + [ConditionalFact] + public async Task EvaluateConversationWithMultipleTurns_NonHubBasedProject() { SkipIfNotConfigured(); await using ScenarioRun scenarioRun = await _contentSafetyReportingConfiguration.CreateScenarioRunAsync( - scenarioName: $"Microsoft.Extensions.AI.Evaluation.Integration.Tests.{nameof(SafetyEvaluatorTests)}.{nameof(EvaluateConversationWithMultipleTurns)}"); + scenarioName: $"Microsoft.Extensions.AI.Evaluation.Integration.Tests.{nameof(SafetyEvaluatorTests)}.{nameof(EvaluateConversationWithMultipleTurns_NonHubBasedProject)}"); - IChatClient chatClient = scenarioRun.ChatConfiguration!.ChatClient; + await EvaluateConversationWithMultipleTurns(scenarioRun); + } + private static async Task EvaluateConversationWithMultipleTurns(ScenarioRun scenarioRun) + { + IChatClient chatClient = scenarioRun.ChatConfiguration!.ChatClient; var messages = new List(); string systemPrompt = @@ -553,6 +616,7 @@ await _mixedQualityAndSafetyReportingConfiguration.CreateScenarioRunAsync( [MemberNotNull(nameof(_imageContentSafetyReportingConfiguration))] [MemberNotNull(nameof(_codeVulnerabilityReportingConfiguration))] [MemberNotNull(nameof(_mixedQualityAndSafetyReportingConfiguration))] + [MemberNotNull(nameof(_hubBasedContentSafetyReportingConfiguration))] private static void SkipIfNotConfigured() { if (!Settings.Current.Configured) @@ -565,5 +629,6 @@ private static void SkipIfNotConfigured() Assert.NotNull(_codeVulnerabilityReportingConfiguration); Assert.NotNull(_imageContentSafetyReportingConfiguration); Assert.NotNull(_mixedQualityAndSafetyReportingConfiguration); + Assert.NotNull(_hubBasedContentSafetyReportingConfiguration); } } diff --git a/test/Libraries/Microsoft.Extensions.AI.Evaluation.Integration.Tests/Settings.cs b/test/Libraries/Microsoft.Extensions.AI.Evaluation.Integration.Tests/Settings.cs index 22e027e73b2..25ee9d544cb 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Evaluation.Integration.Tests/Settings.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Evaluation.Integration.Tests/Settings.cs @@ -16,6 +16,7 @@ public class Settings public string AzureSubscriptionId { get; } public string AzureResourceGroupName { get; } public string AzureAIProjectName { get; } + public string AzureAIProjectEndpoint { get; } public Settings(IConfiguration config) { @@ -49,6 +50,10 @@ public Settings(IConfiguration config) AzureAIProjectName = config.GetValue("AzureAIProjectName") ?? throw new ArgumentNullException(nameof(AzureAIProjectName)); + + AzureAIProjectEndpoint = + config.GetValue("AzureAIProjectEndpoint") + ?? throw new ArgumentNullException(nameof(AzureAIProjectEndpoint)); #pragma warning restore CA2208 } diff --git a/test/Libraries/Microsoft.Extensions.AI.Evaluation.Integration.Tests/appsettings.json b/test/Libraries/Microsoft.Extensions.AI.Evaluation.Integration.Tests/appsettings.json index 63b5ed0d33c..24e079d421d 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Evaluation.Integration.Tests/appsettings.json +++ b/test/Libraries/Microsoft.Extensions.AI.Evaluation.Integration.Tests/appsettings.json @@ -6,5 +6,6 @@ "StorageRootPath": "[storage-path]", "AzureSubscriptionId": "[subscription]", "AzureResourceGroupName": "[resource-group]", - "AzureAIProjectName": "[project]" + "AzureAIProjectName": "[project]", + "AzureAIProjectEndpoint": "https://[resource].services.ai.azure.com/api/projects/[project]" }