diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Console/Commands/CleanCacheCommand.cs b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Console/Commands/CleanCacheCommand.cs index 08f035d55eb..b0d975edb43 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Console/Commands/CleanCacheCommand.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Console/Commands/CleanCacheCommand.cs @@ -18,7 +18,7 @@ internal sealed class CleanCacheCommand(ILogger logger) { internal async Task InvokeAsync(DirectoryInfo? storageRootDir, Uri? endpointUri, CancellationToken cancellationToken = default) { - IResponseCacheProvider cacheProvider; + IEvaluationResponseCacheProvider cacheProvider; if (storageRootDir is not null) { diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Console/Commands/CleanResultsCommand.cs b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Console/Commands/CleanResultsCommand.cs index 59635dc0530..8d6617d8302 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Console/Commands/CleanResultsCommand.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Console/Commands/CleanResultsCommand.cs @@ -23,7 +23,7 @@ internal async Task InvokeAsync( int lastN, CancellationToken cancellationToken = default) { - IResultStore resultStore; + IEvaluationResultStore resultStore; if (storageRootDir is not null) { diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Console/Commands/ReportCommand.cs b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Console/Commands/ReportCommand.cs index f2466923bd8..2611695e542 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Console/Commands/ReportCommand.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Console/Commands/ReportCommand.cs @@ -28,7 +28,7 @@ internal async Task InvokeAsync( Format format, CancellationToken cancellationToken = default) { - IResultStore resultStore; + IEvaluationResultStore resultStore; if (storageRootDir is not null) { diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Azure/JsonSerialization/AzureStorageCamelCaseEnumConverter.cs b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Azure/JsonSerialization/AzureStorageCamelCaseEnumConverter.cs deleted file mode 100644 index 2ec6cdb801f..00000000000 --- a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Azure/JsonSerialization/AzureStorageCamelCaseEnumConverter.cs +++ /dev/null @@ -1,11 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System.Text.Json; -using System.Text.Json.Serialization; - -namespace Microsoft.Extensions.AI.Evaluation.Reporting.JsonSerialization; - -internal sealed class AzureStorageCamelCaseEnumConverter() : - JsonStringEnumConverter(JsonNamingPolicy.CamelCase) - where TEnum : struct, System.Enum; diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Azure/JsonSerialization/AzureStorageJsonUtilities.cs b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Azure/JsonSerialization/AzureStorageJsonUtilities.cs index 23b2ae9c88c..b36c8d8bd56 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Azure/JsonSerialization/AzureStorageJsonUtilities.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Azure/JsonSerialization/AzureStorageJsonUtilities.cs @@ -1,6 +1,7 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. +using System.Diagnostics.CodeAnalysis; using System.Text.Encodings.Web; using System.Text.Json; using System.Text.Json.Serialization; @@ -11,7 +12,7 @@ namespace Microsoft.Extensions.AI.Evaluation.Reporting.JsonSerialization; internal static partial class AzureStorageJsonUtilities { - [System.Diagnostics.CodeAnalysis.SuppressMessage("Naming", "CA1716:Identifiers should not match keywords", Justification = "Default matches the generated source naming convention.")] + [SuppressMessage("Naming", "CA1716:Identifiers should not match keywords", Justification = "Default matches the generated source naming convention.")] internal static class Default { private static JsonSerializerOptions? _options; @@ -24,6 +25,7 @@ internal static class Compact { private static JsonSerializerOptions? _options; internal static JsonSerializerOptions Options => _options ??= CreateJsonSerializerOptions(writeIndented: false); + internal static JsonTypeInfo CacheEntryTypeInfo => Options.GetTypeInfo(); internal static JsonTypeInfo ScenarioRunResultTypeInfo => Options.GetTypeInfo(); } @@ -45,14 +47,14 @@ private static JsonSerializerOptions CreateJsonSerializerOptions(bool writeInden [JsonSerializable(typeof(CacheEntry))] [JsonSourceGenerationOptions( Converters = [ - typeof(AzureStorageCamelCaseEnumConverter), - typeof(AzureStorageCamelCaseEnumConverter), - typeof(AzureStorageTimeSpanConverter) + typeof(CamelCaseEnumConverter), + typeof(CamelCaseEnumConverter), + typeof(TimeSpanConverter), + typeof(EvaluationContextConverter) ], WriteIndented = true, IgnoreReadOnlyProperties = false, DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull, PropertyNamingPolicy = JsonKnownNamingPolicy.CamelCase)] private sealed partial class JsonContext : JsonSerializerContext; - } diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Azure/JsonSerialization/AzureStorageTimeSpanConverter.cs b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Azure/JsonSerialization/AzureStorageTimeSpanConverter.cs deleted file mode 100644 index 0c064ededd3..00000000000 --- a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Azure/JsonSerialization/AzureStorageTimeSpanConverter.cs +++ /dev/null @@ -1,17 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System; -using System.Text.Json; -using System.Text.Json.Serialization; - -namespace Microsoft.Extensions.AI.Evaluation.Reporting.JsonSerialization; - -internal sealed class AzureStorageTimeSpanConverter : JsonConverter -{ - public override TimeSpan Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) - => TimeSpan.FromSeconds(reader.GetDouble()); - - public override void Write(Utf8JsonWriter writer, TimeSpan value, JsonSerializerOptions options) - => writer.WriteNumberValue(value.TotalSeconds); -} diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Azure/Microsoft.Extensions.AI.Evaluation.Reporting.Azure.csproj b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Azure/Microsoft.Extensions.AI.Evaluation.Reporting.Azure.csproj index 669bbab7556..237df014d0d 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Azure/Microsoft.Extensions.AI.Evaluation.Reporting.Azure.csproj +++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Azure/Microsoft.Extensions.AI.Evaluation.Reporting.Azure.csproj @@ -17,6 +17,12 @@ 0 + + + + + + diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Azure/Storage/AzureStorageReportingConfiguration.cs b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Azure/Storage/AzureStorageReportingConfiguration.cs index 9302107b926..fafd8639b34 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Azure/Storage/AzureStorageReportingConfiguration.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Azure/Storage/AzureStorageReportingConfiguration.cs @@ -24,10 +24,6 @@ public static class AzureStorageReportingConfiguration /// /// The set of s that should be invoked to evaluate AI responses. /// - /// - /// An optional that specifies the maximum amount of time that cached AI responses should - /// survive in the cache before they are considered expired and evicted. - /// /// /// A that specifies the that is used by AI-based /// included in the returned . Can be omitted if @@ -36,6 +32,10 @@ public static class AzureStorageReportingConfiguration /// /// to enable caching of AI responses; otherwise. /// + /// + /// An optional that specifies the maximum amount of time that cached AI responses should + /// survive in the cache before they are considered expired and evicted. + /// /// /// An optional collection of unique strings that should be hashed when generating the cache keys for cached AI /// responses. See for more information about this concept. @@ -63,21 +63,21 @@ public static class AzureStorageReportingConfiguration public static ReportingConfiguration Create( DataLakeDirectoryClient client, IEnumerable evaluators, - TimeSpan? timeToLiveForCacheEntries = null, ChatConfiguration? chatConfiguration = null, bool enableResponseCaching = true, + TimeSpan? timeToLiveForCacheEntries = null, IEnumerable? cachingKeys = null, string executionName = Defaults.DefaultExecutionName, Func? evaluationMetricInterpreter = null, IEnumerable? tags = null) #pragma warning restore S107 { - IResponseCacheProvider? responseCacheProvider = + IEvaluationResponseCacheProvider? responseCacheProvider = chatConfiguration is not null && enableResponseCaching ? new AzureStorageResponseCacheProvider(client, timeToLiveForCacheEntries) : null; - IResultStore resultStore = new AzureStorageResultStore(client); + IEvaluationResultStore resultStore = new AzureStorageResultStore(client); return new ReportingConfiguration( evaluators, diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Azure/Storage/AzureStorageResponseCacheProvider.cs b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Azure/Storage/AzureStorageResponseCacheProvider.cs index a890fa80332..6c6d1431a1a 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Azure/Storage/AzureStorageResponseCacheProvider.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Azure/Storage/AzureStorageResponseCacheProvider.cs @@ -15,8 +15,8 @@ namespace Microsoft.Extensions.AI.Evaluation.Reporting.Storage; /// -/// An that returns an that can cache AI responses -/// for a particular under an Azure Storage container. +/// An that returns an that can cache AI +/// responses for a particular under an Azure Storage container. /// /// /// A with access to an Azure Storage container under which the cached AI @@ -28,7 +28,7 @@ namespace Microsoft.Extensions.AI.Evaluation.Reporting.Storage; /// public sealed class AzureStorageResponseCacheProvider( DataLakeDirectoryClient client, - TimeSpan? timeToLiveForCacheEntries = null) : IResponseCacheProvider + TimeSpan? timeToLiveForCacheEntries = null) : IEvaluationResponseCacheProvider { private readonly Func _provideDateTime = () => DateTime.Now; diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Azure/Storage/AzureStorageResultStore.cs b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Azure/Storage/AzureStorageResultStore.cs index 70d988abe74..71682f13651 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Azure/Storage/AzureStorageResultStore.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Azure/Storage/AzureStorageResultStore.cs @@ -20,14 +20,14 @@ namespace Microsoft.Extensions.AI.Evaluation.Reporting.Storage; /// -/// An implementation that stores s under an Azure Storage -/// container. +/// An implementation that stores s under an Azure +/// Storage container. /// /// /// A with access to an Azure Storage container under which the /// s should be stored. /// -public sealed class AzureStorageResultStore(DataLakeDirectoryClient client) : IResultStore +public sealed class AzureStorageResultStore(DataLakeDirectoryClient client) : IEvaluationResultStore { private const string ResultsRootPrefix = "results"; diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/ChatDetailsExtensions.cs b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/ChatDetailsExtensions.cs index 006dfc741e8..71afe53217a 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/ChatDetailsExtensions.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/ChatDetailsExtensions.cs @@ -1,6 +1,7 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. +using System.Collections.Generic; using Microsoft.Shared.Diagnostics; namespace Microsoft.Extensions.AI.Evaluation.Reporting; @@ -11,19 +12,36 @@ namespace Microsoft.Extensions.AI.Evaluation.Reporting; public static class ChatDetailsExtensions { /// - /// Adds for a particular LLM chat conversation turn to the + /// Adds for one or more LLM chat conversation turns to the /// collection. /// /// - /// The object to which the is to be added. + /// The object to which the are to be added. /// /// - /// The for a particular LLM chat conversation turn. + /// The for one or more LLM chat conversation turns. /// - public static void AddTurnDetails(this ChatDetails chatDetails, ChatTurnDetails turnDetails) + public static void AddTurnDetails(this ChatDetails chatDetails, IEnumerable turnDetails) { _ = Throw.IfNull(chatDetails); + _ = Throw.IfNull(turnDetails); - chatDetails.TurnDetails.Add(turnDetails); + foreach (ChatTurnDetails t in turnDetails) + { + chatDetails.TurnDetails.Add(t); + } } + + /// + /// Adds for one or more LLM chat conversation turns to the + /// collection. + /// + /// + /// The object to which the are to be added. + /// + /// + /// The for one or more LLM chat conversation turns. + /// + public static void AddTurnDetails(this ChatDetails chatDetails, params ChatTurnDetails[] turnDetails) + => chatDetails.AddTurnDetails(turnDetails as IEnumerable); } diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/Defaults.cs b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/Defaults.cs index 095bdee575c..f25fa074430 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/Defaults.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/Defaults.cs @@ -27,7 +27,7 @@ public static class Defaults /// /// Gets a that specifies the default amount of time that cached AI responses should survive - /// in the 's cache before they are considered expired and evicted. + /// in the 's cache before they are considered expired and evicted. /// public static TimeSpan DefaultTimeToLiveForCacheEntries { get; } = TimeSpan.FromDays(14); diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/IResponseCacheProvider.cs b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/IEvaluationResponseCacheProvider.cs similarity index 58% rename from src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/IResponseCacheProvider.cs rename to src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/IEvaluationResponseCacheProvider.cs index 6bc8ce25432..1859124a98f 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/IResponseCacheProvider.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/IEvaluationResponseCacheProvider.cs @@ -12,26 +12,28 @@ namespace Microsoft.Extensions.AI.Evaluation.Reporting; /// . /// /// -/// can be used to set up caching of AI-generated responses (both the AI responses -/// under evaluation as well as the AI responses for the evaluations themselves). When caching is enabled, the AI -/// responses associated with each are stored in the that is -/// returned from this . So long as the inputs (such as the content included in the -/// requests, the AI model being invoked etc.) remain unchanged, subsequent evaluations of the same -/// use the cached responses instead of invoking the AI model to generate new ones. Bypassing -/// the AI model when the inputs remain unchanged results in faster execution at a lower cost. +/// can be used to set up caching of AI-generated responses (both the AI +/// responses under evaluation as well as the AI responses for the evaluations themselves). When caching is enabled, +/// the AI responses associated with each are stored in the +/// that is returned from this . So long as the inputs (such as the +/// content included in the requests, the AI model being invoked etc.) remain unchanged, subsequent evaluations of the +/// same use the cached responses instead of invoking the AI model to generate new ones. +/// Bypassing the AI model when the inputs remain unchanged results in faster execution at a lower cost. /// -public interface IResponseCacheProvider +public interface IEvaluationResponseCacheProvider { /// - /// Returns an that caches the AI responses associated with a particular - /// . + /// Returns an that caches all the AI responses associated with the + /// with the supplied and + /// . /// /// The . /// The . /// A that can cancel the operation. /// - /// An that caches the AI responses associated with a particular - /// . + /// An that caches all the AI responses associated with the + /// with the supplied and + /// . /// ValueTask GetCacheAsync( string scenarioName, diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/IResultStore.cs b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/IEvaluationResultStore.cs similarity index 99% rename from src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/IResultStore.cs rename to src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/IEvaluationResultStore.cs index 3f3dea6cc7a..202a6305cd3 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/IResultStore.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/IEvaluationResultStore.cs @@ -10,7 +10,7 @@ namespace Microsoft.Extensions.AI.Evaluation.Reporting; /// /// Represents a store for s. /// -public interface IResultStore +public interface IEvaluationResultStore { /// /// Returns s for s filtered by the specified diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/JsonSerialization/JsonUtilities.cs b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/JsonSerialization/JsonUtilities.cs index 9ba74009433..3a8c2af1ce2 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/JsonSerialization/JsonUtilities.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/JsonSerialization/JsonUtilities.cs @@ -1,6 +1,7 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. +using System.Diagnostics.CodeAnalysis; using System.Text.Encodings.Web; using System.Text.Json; using System.Text.Json.Serialization; @@ -12,7 +13,7 @@ namespace Microsoft.Extensions.AI.Evaluation.Reporting.JsonSerialization; internal static partial class JsonUtilities { - [System.Diagnostics.CodeAnalysis.SuppressMessage("Naming", "CA1716:Identifiers should not match keywords", Justification = "Default matches the generated source naming convention.")] + [SuppressMessage("Naming", "CA1716:Identifiers should not match keywords", Justification = "Default matches the generated source naming convention.")] internal static class Default { private static JsonSerializerOptions? _options; @@ -45,7 +46,7 @@ private static JsonSerializerOptions CreateJsonSerializerOptions(bool writeInden return options; } - [JsonSerializable(typeof(EvaluationResult))] + [JsonSerializable(typeof(ScenarioRunResult))] [JsonSerializable(typeof(Dataset))] [JsonSerializable(typeof(CacheEntry))] [JsonSourceGenerationOptions( diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/ReportingConfiguration.cs b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/ReportingConfiguration.cs index 68a73338d88..130586de930 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/ReportingConfiguration.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/ReportingConfiguration.cs @@ -27,9 +27,10 @@ public sealed class ReportingConfiguration public IReadOnlyList Evaluators { get; } /// - /// Gets the that should be used to persist the s. + /// Gets the that should be used to persist the + /// s. /// - public IResultStore ResultStore { get; } + public IEvaluationResultStore ResultStore { get; } /// /// Gets a that specifies the that is used by @@ -38,9 +39,9 @@ public sealed class ReportingConfiguration public ChatConfiguration? ChatConfiguration { get; } /// - /// Gets the that should be used to cache AI responses. + /// Gets the that should be used to cache AI responses. /// - public IResponseCacheProvider? ResponseCacheProvider { get; } + public IEvaluationResponseCacheProvider? ResponseCacheProvider { get; } /// /// Gets the collection of unique strings that should be hashed when generating the cache keys for cached AI @@ -101,7 +102,7 @@ public sealed class ReportingConfiguration /// The set of s that should be invoked to evaluate AI responses. /// /// - /// The that should be used to persist the s. + /// The that should be used to persist the s. /// /// /// A that specifies the that is used by @@ -109,8 +110,8 @@ public sealed class ReportingConfiguration /// none of the included are AI-based. /// /// - /// The that should be used to cache AI responses. If omitted, AI responses - /// will not be cached. + /// The that should be used to cache AI responses. If omitted, AI + /// responses will not be cached. /// /// /// An optional collection of unique strings that should be hashed when generating the cache keys for cached AI @@ -134,9 +135,9 @@ public sealed class ReportingConfiguration #pragma warning disable S107 // Methods should not have too many parameters public ReportingConfiguration( IEnumerable evaluators, - IResultStore resultStore, + IEvaluationResultStore resultStore, ChatConfiguration? chatConfiguration = null, - IResponseCacheProvider? responseCacheProvider = null, + IEvaluationResponseCacheProvider? responseCacheProvider = null, IEnumerable? cachingKeys = null, string executionName = Defaults.DefaultExecutionName, Func? evaluationMetricInterpreter = null, diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/ScenarioRun.cs b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/ScenarioRun.cs index eb58685bf0c..5fa46e7e4ec 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/ScenarioRun.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/ScenarioRun.cs @@ -93,7 +93,7 @@ public sealed class ScenarioRun : IAsyncDisposable public ChatConfiguration? ChatConfiguration { get; } private readonly CompositeEvaluator _compositeEvaluator; - private readonly IResultStore _resultStore; + private readonly IEvaluationResultStore _resultStore; private readonly Func? _evaluationMetricInterpreter; private readonly ChatDetails? _chatDetails; private readonly IEnumerable? _tags; @@ -106,7 +106,7 @@ internal ScenarioRun( string iterationName, string executionName, IEnumerable evaluators, - IResultStore resultStore, + IEvaluationResultStore resultStore, ChatConfiguration? chatConfiguration = null, Func? evaluationMetricInterpreter = null, ChatDetails? chatDetails = null, @@ -189,7 +189,7 @@ await _compositeEvaluator.EvaluateAsync( /// /// Disposes the and writes the to the configured - /// . + /// . /// /// A that represents the asynchronous operation. public async ValueTask DisposeAsync() diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/ScenarioRunExtensions.cs b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/ScenarioRunExtensions.cs index 3b723a2d258..08822f18d02 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/ScenarioRunExtensions.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/ScenarioRunExtensions.cs @@ -85,16 +85,11 @@ public static ValueTask EvaluateAsync( this ScenarioRun scenarioRun, ChatMessage modelResponse, IEnumerable? additionalContext = null, - CancellationToken cancellationToken = default) - { - _ = Throw.IfNull(scenarioRun); - - return scenarioRun.EvaluateAsync( - messages: [], + CancellationToken cancellationToken = default) => + scenarioRun.EvaluateAsync( new ChatResponse(modelResponse), additionalContext, cancellationToken); - } /// /// Evaluates the supplied and returns an @@ -148,16 +143,12 @@ public static ValueTask EvaluateAsync( ChatMessage userRequest, ChatMessage modelResponse, IEnumerable? additionalContext = null, - CancellationToken cancellationToken = default) - { - _ = Throw.IfNull(scenarioRun); - - return scenarioRun.EvaluateAsync( - messages: [userRequest], + CancellationToken cancellationToken = default) => + scenarioRun.EvaluateAsync( + userRequest, new ChatResponse(modelResponse), additionalContext, cancellationToken); - } /// /// Evaluates the supplied and returns an diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/Storage/DiskBasedReportingConfiguration.cs b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/Storage/DiskBasedReportingConfiguration.cs index 94ab92e177b..e967fdd1db9 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/Storage/DiskBasedReportingConfiguration.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/Storage/DiskBasedReportingConfiguration.cs @@ -32,6 +32,10 @@ public static class DiskBasedReportingConfiguration /// /// to enable caching of AI responses; otherwise. /// + /// + /// An optional that specifies the maximum amount of time that cached AI responses should + /// survive in the cache before they are considered expired and evicted. + /// /// /// An optional collection of unique strings that should be hashed when generating the cache keys for cached AI /// responses. See for more information about this concept. @@ -61,6 +65,7 @@ public static ReportingConfiguration Create( IEnumerable evaluators, ChatConfiguration? chatConfiguration = null, bool enableResponseCaching = true, + TimeSpan? timeToLiveForCacheEntries = null, IEnumerable? cachingKeys = null, string executionName = Defaults.DefaultExecutionName, Func? evaluationMetricInterpreter = null, @@ -69,12 +74,12 @@ public static ReportingConfiguration Create( { storageRootPath = Path.GetFullPath(storageRootPath); - IResponseCacheProvider? responseCacheProvider = + IEvaluationResponseCacheProvider? responseCacheProvider = chatConfiguration is not null && enableResponseCaching - ? new DiskBasedResponseCacheProvider(storageRootPath) + ? new DiskBasedResponseCacheProvider(storageRootPath, timeToLiveForCacheEntries) : null; - IResultStore resultStore = new DiskBasedResultStore(storageRootPath); + IEvaluationResultStore resultStore = new DiskBasedResultStore(storageRootPath); return new ReportingConfiguration( evaluators, diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/Storage/DiskBasedResponseCacheProvider.cs b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/Storage/DiskBasedResponseCacheProvider.cs index feb75df1dba..8b60fe5a272 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/Storage/DiskBasedResponseCacheProvider.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/Storage/DiskBasedResponseCacheProvider.cs @@ -14,8 +14,9 @@ namespace Microsoft.Extensions.AI.Evaluation.Reporting.Storage; /// -/// An that returns an that can cache AI responses -/// for a particular under the specified on disk. +/// An that returns an that can cache +/// AI responses for a particular under the specified on +/// disk. /// /// /// The path to a directory on disk under which the cached AI responses should be stored. @@ -26,15 +27,18 @@ namespace Microsoft.Extensions.AI.Evaluation.Reporting.Storage; /// public sealed class DiskBasedResponseCacheProvider( string storageRootPath, - TimeSpan? timeToLiveForCacheEntries = null) : IResponseCacheProvider + TimeSpan? timeToLiveForCacheEntries = null) : IEvaluationResponseCacheProvider { private readonly Func _provideDateTime = () => DateTime.UtcNow; /// /// Intended for testing purposes only. /// - internal DiskBasedResponseCacheProvider(string storageRootPath, Func provideDateTime) - : this(storageRootPath) + internal DiskBasedResponseCacheProvider( + string storageRootPath, + Func provideDateTime, + TimeSpan? timeToLiveForCacheEntries = null) + : this(storageRootPath, timeToLiveForCacheEntries) { _provideDateTime = provideDateTime; } diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/Storage/DiskBasedResultStore.cs b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/Storage/DiskBasedResultStore.cs index de1517dca99..4662857ec59 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/Storage/DiskBasedResultStore.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/Storage/DiskBasedResultStore.cs @@ -16,9 +16,9 @@ namespace Microsoft.Extensions.AI.Evaluation.Reporting.Storage; /// -/// An implementation that stores s on disk. +/// An implementation that stores s on disk. /// -public sealed class DiskBasedResultStore : IResultStore +public sealed class DiskBasedResultStore : IEvaluationResultStore { private const string DeserializationFailedMessage = "Unable to deserialize the scenario run result file at {0}."; diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/AIContentExtensions.cs b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/AIContentExtensions.cs index 6ec3793d0da..0334d6aa08c 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/AIContentExtensions.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/AIContentExtensions.cs @@ -4,6 +4,7 @@ using System; namespace Microsoft.Extensions.AI.Evaluation.Safety; + internal static class AIContentExtensions { internal static bool IsTextOrUsage(this AIContent content) diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/ContentSafetyServicePayloadFormat.cs b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/ContentSafetyServicePayloadFormat.cs index 428940955ff..b771dc008c8 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/ContentSafetyServicePayloadFormat.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/ContentSafetyServicePayloadFormat.cs @@ -9,5 +9,5 @@ internal enum ContentSafetyServicePayloadFormat QuestionAnswer, QueryResponse, ContextCompletion, - Conversation, + Conversation } diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation/EvaluationRating.cs b/src/Libraries/Microsoft.Extensions.AI.Evaluation/EvaluationRating.cs index 025ef58b809..1ba8ae270e6 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Evaluation/EvaluationRating.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation/EvaluationRating.cs @@ -20,14 +20,14 @@ public enum EvaluationRating Inconclusive, /// - /// A value that indicates that the is interpreted as being exceptional. + /// A value that indicates that the is interpreted as being unacceptable. /// - Exceptional, + Unacceptable, /// - /// A value that indicates that the is interpreted as being good. + /// A value that indicates that the is interpreted as being poor. /// - Good, + Poor, /// /// A value that indicates that the is interpreted as being average. @@ -35,12 +35,12 @@ public enum EvaluationRating Average, /// - /// A value that indicates that the is interpreted as being poor. + /// A value that indicates that the is interpreted as being good. /// - Poor, + Good, /// - /// A value that indicates that the is interpreted as being unacceptable. + /// A value that indicates that the is interpreted as being exceptional. /// - Unacceptable, + Exceptional } diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation/EvaluatorExtensions.cs b/src/Libraries/Microsoft.Extensions.AI.Evaluation/EvaluatorExtensions.cs index 3ffda8fb8f9..8d25085f4e0 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Evaluation/EvaluatorExtensions.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation/EvaluatorExtensions.cs @@ -131,17 +131,12 @@ public static ValueTask EvaluateAsync( ChatMessage modelResponse, ChatConfiguration? chatConfiguration = null, IEnumerable? additionalContext = null, - CancellationToken cancellationToken = default) - { - _ = Throw.IfNull(evaluator); - - return evaluator.EvaluateAsync( - messages: [], + CancellationToken cancellationToken = default) => + evaluator.EvaluateAsync( new ChatResponse(modelResponse), chatConfiguration, additionalContext, cancellationToken); - } /// /// Evaluates the supplied and returns an @@ -225,17 +220,13 @@ public static ValueTask EvaluateAsync( ChatMessage modelResponse, ChatConfiguration? chatConfiguration = null, IEnumerable? additionalContext = null, - CancellationToken cancellationToken = default) - { - _ = Throw.IfNull(evaluator); - - return evaluator.EvaluateAsync( - messages: [userRequest], + CancellationToken cancellationToken = default) => + evaluator.EvaluateAsync( + userRequest, new ChatResponse(modelResponse), chatConfiguration, additionalContext, cancellationToken); - } /// /// Evaluates the supplied and returns an diff --git a/test/Libraries/Microsoft.Extensions.AI.Evaluation.Integration.Tests/QualityEvaluatorTests.cs b/test/Libraries/Microsoft.Extensions.AI.Evaluation.Integration.Tests/QualityEvaluatorTests.cs index 90f7a2b29aa..b56a2673b60 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Evaluation.Integration.Tests/QualityEvaluatorTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Evaluation.Integration.Tests/QualityEvaluatorTests.cs @@ -71,7 +71,7 @@ static QualityEvaluatorTests() DiskBasedReportingConfiguration.Create( storageRootPath: Settings.Current.StorageRootPath, evaluators: [groundednessEvaluator, equivalenceEvaluator, completenessEvaluator, retrievalEvaluator], - chatConfiguration, + chatConfiguration: chatConfiguration, executionName: Constants.Version, tags: [version, date, projectName, testClass, provider, model, temperature, usesContext]); } diff --git a/test/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Tests/AzureStorage/AzureResponseCacheTests.cs b/test/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Tests/AzureStorage/AzureResponseCacheTests.cs index b135a64a04c..2f936621147 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Tests/AzureStorage/AzureResponseCacheTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Tests/AzureStorage/AzureResponseCacheTests.cs @@ -47,9 +47,9 @@ public async Task DisposeAsync() internal override bool IsConfigured => Settings.Current.Configured; - internal override IResponseCacheProvider CreateResponseCacheProvider() + internal override IEvaluationResponseCacheProvider CreateResponseCacheProvider() => new AzureStorageResponseCacheProvider(_dirClient!); - internal override IResponseCacheProvider CreateResponseCacheProvider(Func provideDateTime) + internal override IEvaluationResponseCacheProvider CreateResponseCacheProvider(Func provideDateTime) => new AzureStorageResponseCacheProvider(_dirClient!, provideDateTime); } diff --git a/test/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Tests/AzureStorage/AzureResultStoreTests.cs b/test/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Tests/AzureStorage/AzureResultStoreTests.cs index 610f6345524..62163d5e681 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Tests/AzureStorage/AzureResultStoreTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Tests/AzureStorage/AzureResultStoreTests.cs @@ -47,7 +47,7 @@ public async Task DisposeAsync() public override bool IsConfigured => Settings.Current.Configured; - public override IResultStore CreateResultStore() + public override IEvaluationResultStore CreateResultStore() => new AzureStorageResultStore(_dirClient!); } diff --git a/test/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Tests/DiskBased/DiskBasedResponseCacheTests.cs b/test/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Tests/DiskBased/DiskBasedResponseCacheTests.cs index e0ba0c171d1..8305fe8ddb3 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Tests/DiskBased/DiskBasedResponseCacheTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Tests/DiskBased/DiskBasedResponseCacheTests.cs @@ -45,9 +45,9 @@ public Task DisposeAsync() internal override bool IsConfigured => true; - internal override IResponseCacheProvider CreateResponseCacheProvider() + internal override IEvaluationResponseCacheProvider CreateResponseCacheProvider() => new DiskBasedResponseCacheProvider(UseTempStoragePath()); - internal override IResponseCacheProvider CreateResponseCacheProvider(Func provideDateTime) + internal override IEvaluationResponseCacheProvider CreateResponseCacheProvider(Func provideDateTime) => new DiskBasedResponseCacheProvider(UseTempStoragePath(), provideDateTime); } diff --git a/test/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Tests/DiskBased/DiskBasedResultStoreTests.cs b/test/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Tests/DiskBased/DiskBasedResultStoreTests.cs index 1fee1b9996c..77cabfd7ffd 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Tests/DiskBased/DiskBasedResultStoreTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Tests/DiskBased/DiskBasedResultStoreTests.cs @@ -44,7 +44,7 @@ public Task DisposeAsync() public override bool IsConfigured => true; - public override IResultStore CreateResultStore() + public override IEvaluationResultStore CreateResultStore() => new DiskBasedResultStore(UseTempStoragePath()); } diff --git a/test/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Tests/ResponseCacheTester.cs b/test/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Tests/ResponseCacheTester.cs index 76e50244b92..b69014e631b 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Tests/ResponseCacheTester.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Tests/ResponseCacheTester.cs @@ -18,8 +18,8 @@ public abstract class ResponseCacheTester private static readonly string _keyB = "B Key"; private static readonly byte[] _responseB = Encoding.UTF8.GetBytes("Content B"); - internal abstract IResponseCacheProvider CreateResponseCacheProvider(); - internal abstract IResponseCacheProvider CreateResponseCacheProvider(Func provideDateTime); + internal abstract IEvaluationResponseCacheProvider CreateResponseCacheProvider(); + internal abstract IEvaluationResponseCacheProvider CreateResponseCacheProvider(Func provideDateTime); internal abstract bool IsConfigured { get; } private void SkipIfNotConfigured() @@ -37,7 +37,7 @@ public async Task AddUncachedEntry() string iterationName = "TestIteration"; - IResponseCacheProvider provider = CreateResponseCacheProvider(); + IEvaluationResponseCacheProvider provider = CreateResponseCacheProvider(); IDistributedCache cache = await provider.GetCacheAsync(nameof(AddUncachedEntry), iterationName); Assert.NotNull(cache); @@ -58,7 +58,7 @@ public async Task RemoveCachedEntry() string iterationName = "TestIteration"; - IResponseCacheProvider provider = CreateResponseCacheProvider(); + IEvaluationResponseCacheProvider provider = CreateResponseCacheProvider(); IDistributedCache cache = await provider.GetCacheAsync(nameof(RemoveCachedEntry), iterationName); Assert.NotNull(cache); @@ -85,7 +85,7 @@ public async Task CacheEntryExpiration() DateTime now = DateTime.UtcNow; DateTime provideDateTime() => now; - IResponseCacheProvider provider = CreateResponseCacheProvider(provideDateTime); + IEvaluationResponseCacheProvider provider = CreateResponseCacheProvider(provideDateTime); IDistributedCache cache = await provider.GetCacheAsync(nameof(RemoveCachedEntry), iterationName); Assert.NotNull(cache); @@ -106,7 +106,7 @@ public async Task MultipleCacheInstances() { SkipIfNotConfigured(); - IResponseCacheProvider provider = CreateResponseCacheProvider(); + IEvaluationResponseCacheProvider provider = CreateResponseCacheProvider(); IDistributedCache cache = await provider.GetCacheAsync(nameof(MultipleCacheInstances), "Async"); Assert.NotNull(cache); IDistributedCache cache2 = await provider.GetCacheAsync(nameof(MultipleCacheInstances), "Async"); @@ -133,7 +133,7 @@ public async Task DeleteExpiredEntries() DateTime now = DateTime.UtcNow; DateTime provideDateTime() => now; - IResponseCacheProvider provider = CreateResponseCacheProvider(provideDateTime); + IEvaluationResponseCacheProvider provider = CreateResponseCacheProvider(provideDateTime); IDistributedCache cache = await provider.GetCacheAsync(nameof(RemoveCachedEntry), iterationName); Assert.NotNull(cache); @@ -163,7 +163,7 @@ public async Task ResetCache() string iterationName = "TestIteration"; - IResponseCacheProvider provider = CreateResponseCacheProvider(); + IEvaluationResponseCacheProvider provider = CreateResponseCacheProvider(); IDistributedCache cache = await provider.GetCacheAsync(nameof(RemoveCachedEntry), iterationName); Assert.NotNull(cache); diff --git a/test/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Tests/ResultStoreTester.cs b/test/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Tests/ResultStoreTester.cs index 1ce033b3cd7..995b77a8c5e 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Tests/ResultStoreTester.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Tests/ResultStoreTester.cs @@ -13,7 +13,7 @@ namespace Microsoft.Extensions.AI.Evaluation.Reporting.Tests; public abstract class ResultStoreTester { - public abstract IResultStore CreateResultStore(); + public abstract IEvaluationResultStore CreateResultStore(); public abstract bool IsConfigured { get; } @@ -39,7 +39,8 @@ private static ScenarioRunResult CreateTestResult(string scenarioName, string it private static string ScenarioName(int n) => $"Test.Scenario.{n}"; private static string IterationName(int n) => $"Iteration {n}"; - private static async Task> LoadResultsAsync(int n, IResultStore resultStore) + private static async Task> + LoadResultsAsync(int n, IEvaluationResultStore resultStore) { List<(string executionName, string scenarioName, string iterationName)> results = []; await foreach (string executionName in resultStore.GetLatestExecutionNamesAsync(n)) @@ -69,7 +70,7 @@ public async Task WriteAndReadResults() { SkipIfNotConfigured(); - IResultStore resultStore = CreateResultStore(); + IEvaluationResultStore resultStore = CreateResultStore(); Assert.NotNull(resultStore); string newExecutionName = $"Test Execution {Path.GetRandomFileName()}"; @@ -108,7 +109,7 @@ public async Task WriteAndReadHistoricalResults() { SkipIfNotConfigured(); - IResultStore resultStore = CreateResultStore(); + IEvaluationResultStore resultStore = CreateResultStore(); Assert.NotNull(resultStore); string firstExecutionName = $"Test Execution {Path.GetRandomFileName()}"; @@ -152,7 +153,7 @@ public async Task DeleteExecutions() { SkipIfNotConfigured(); - IResultStore resultStore = CreateResultStore(); + IEvaluationResultStore resultStore = CreateResultStore(); Assert.NotNull(resultStore); string executionName = $"Test Execution {Path.GetRandomFileName()}"; @@ -176,7 +177,7 @@ public async Task DeleteSomeExecutions() { SkipIfNotConfigured(); - IResultStore resultStore = CreateResultStore(); + IEvaluationResultStore resultStore = CreateResultStore(); Assert.NotNull(resultStore); string executionName0 = $"Test Execution {Path.GetRandomFileName()}"; @@ -211,7 +212,7 @@ public async Task DeleteScenarios() { SkipIfNotConfigured(); - IResultStore resultStore = CreateResultStore(); + IEvaluationResultStore resultStore = CreateResultStore(); Assert.NotNull(resultStore); string executionName = $"Test Execution {Path.GetRandomFileName()}"; @@ -246,7 +247,7 @@ public async Task DeleteIterations() { SkipIfNotConfigured(); - IResultStore resultStore = CreateResultStore(); + IEvaluationResultStore resultStore = CreateResultStore(); Assert.NotNull(resultStore); string executionName = $"Test Execution {Path.GetRandomFileName()}";