diff --git a/eng/MSBuild/LegacySupport.props b/eng/MSBuild/LegacySupport.props index 15d34725d84..6b110acaaa1 100644 --- a/eng/MSBuild/LegacySupport.props +++ b/eng/MSBuild/LegacySupport.props @@ -7,6 +7,10 @@ + + + + diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 79b661b0db6..859de22a83f 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -1,196 +1,196 @@ - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - f57e6dc747158ab7ade4e62a75a6750d16b771e8 + e36e4d1a8f8dfb08d7e3a6041459c9791d732c01 - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - f57e6dc747158ab7ade4e62a75a6750d16b771e8 + e36e4d1a8f8dfb08d7e3a6041459c9791d732c01 - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - f57e6dc747158ab7ade4e62a75a6750d16b771e8 + e36e4d1a8f8dfb08d7e3a6041459c9791d732c01 - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - f57e6dc747158ab7ade4e62a75a6750d16b771e8 + e36e4d1a8f8dfb08d7e3a6041459c9791d732c01 - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - f57e6dc747158ab7ade4e62a75a6750d16b771e8 + e36e4d1a8f8dfb08d7e3a6041459c9791d732c01 - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - f57e6dc747158ab7ade4e62a75a6750d16b771e8 + e36e4d1a8f8dfb08d7e3a6041459c9791d732c01 - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - f57e6dc747158ab7ade4e62a75a6750d16b771e8 + e36e4d1a8f8dfb08d7e3a6041459c9791d732c01 - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - f57e6dc747158ab7ade4e62a75a6750d16b771e8 + e36e4d1a8f8dfb08d7e3a6041459c9791d732c01 - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - f57e6dc747158ab7ade4e62a75a6750d16b771e8 + e36e4d1a8f8dfb08d7e3a6041459c9791d732c01 - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - f57e6dc747158ab7ade4e62a75a6750d16b771e8 + e36e4d1a8f8dfb08d7e3a6041459c9791d732c01 - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - f57e6dc747158ab7ade4e62a75a6750d16b771e8 + e36e4d1a8f8dfb08d7e3a6041459c9791d732c01 - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - f57e6dc747158ab7ade4e62a75a6750d16b771e8 + e36e4d1a8f8dfb08d7e3a6041459c9791d732c01 - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - f57e6dc747158ab7ade4e62a75a6750d16b771e8 + e36e4d1a8f8dfb08d7e3a6041459c9791d732c01 - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - f57e6dc747158ab7ade4e62a75a6750d16b771e8 + e36e4d1a8f8dfb08d7e3a6041459c9791d732c01 - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - f57e6dc747158ab7ade4e62a75a6750d16b771e8 + e36e4d1a8f8dfb08d7e3a6041459c9791d732c01 - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - f57e6dc747158ab7ade4e62a75a6750d16b771e8 + e36e4d1a8f8dfb08d7e3a6041459c9791d732c01 - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - f57e6dc747158ab7ade4e62a75a6750d16b771e8 + e36e4d1a8f8dfb08d7e3a6041459c9791d732c01 - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - f57e6dc747158ab7ade4e62a75a6750d16b771e8 + e36e4d1a8f8dfb08d7e3a6041459c9791d732c01 - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - f57e6dc747158ab7ade4e62a75a6750d16b771e8 + e36e4d1a8f8dfb08d7e3a6041459c9791d732c01 - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - f57e6dc747158ab7ade4e62a75a6750d16b771e8 + e36e4d1a8f8dfb08d7e3a6041459c9791d732c01 - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - f57e6dc747158ab7ade4e62a75a6750d16b771e8 + e36e4d1a8f8dfb08d7e3a6041459c9791d732c01 - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - f57e6dc747158ab7ade4e62a75a6750d16b771e8 + e36e4d1a8f8dfb08d7e3a6041459c9791d732c01 - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - f57e6dc747158ab7ade4e62a75a6750d16b771e8 + e36e4d1a8f8dfb08d7e3a6041459c9791d732c01 - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - f57e6dc747158ab7ade4e62a75a6750d16b771e8 + e36e4d1a8f8dfb08d7e3a6041459c9791d732c01 - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - f57e6dc747158ab7ade4e62a75a6750d16b771e8 + e36e4d1a8f8dfb08d7e3a6041459c9791d732c01 - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - f57e6dc747158ab7ade4e62a75a6750d16b771e8 + e36e4d1a8f8dfb08d7e3a6041459c9791d732c01 - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - f57e6dc747158ab7ade4e62a75a6750d16b771e8 + e36e4d1a8f8dfb08d7e3a6041459c9791d732c01 - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - f57e6dc747158ab7ade4e62a75a6750d16b771e8 + e36e4d1a8f8dfb08d7e3a6041459c9791d732c01 - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - f57e6dc747158ab7ade4e62a75a6750d16b771e8 + e36e4d1a8f8dfb08d7e3a6041459c9791d732c01 - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - f57e6dc747158ab7ade4e62a75a6750d16b771e8 + e36e4d1a8f8dfb08d7e3a6041459c9791d732c01 - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - f57e6dc747158ab7ade4e62a75a6750d16b771e8 + e36e4d1a8f8dfb08d7e3a6041459c9791d732c01 - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - f57e6dc747158ab7ade4e62a75a6750d16b771e8 + e36e4d1a8f8dfb08d7e3a6041459c9791d732c01 - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - f57e6dc747158ab7ade4e62a75a6750d16b771e8 + e36e4d1a8f8dfb08d7e3a6041459c9791d732c01 - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - f57e6dc747158ab7ade4e62a75a6750d16b771e8 + e36e4d1a8f8dfb08d7e3a6041459c9791d732c01 - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - f57e6dc747158ab7ade4e62a75a6750d16b771e8 + e36e4d1a8f8dfb08d7e3a6041459c9791d732c01 - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - f57e6dc747158ab7ade4e62a75a6750d16b771e8 + e36e4d1a8f8dfb08d7e3a6041459c9791d732c01 - + https://dev.azure.com/dnceng/internal/_git/dotnet-runtime - f57e6dc747158ab7ade4e62a75a6750d16b771e8 + e36e4d1a8f8dfb08d7e3a6041459c9791d732c01 - + https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore - c15021a04827e7ad60e49aba73df748892e35d25 + ed74665e773dd1ebea3289c5662d71c590305932 - + https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore - c15021a04827e7ad60e49aba73df748892e35d25 + ed74665e773dd1ebea3289c5662d71c590305932 - + https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore - c15021a04827e7ad60e49aba73df748892e35d25 + ed74665e773dd1ebea3289c5662d71c590305932 - + https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore - c15021a04827e7ad60e49aba73df748892e35d25 + ed74665e773dd1ebea3289c5662d71c590305932 - + https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore - c15021a04827e7ad60e49aba73df748892e35d25 + ed74665e773dd1ebea3289c5662d71c590305932 - + https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore - c15021a04827e7ad60e49aba73df748892e35d25 + ed74665e773dd1ebea3289c5662d71c590305932 - + https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore - c15021a04827e7ad60e49aba73df748892e35d25 + ed74665e773dd1ebea3289c5662d71c590305932 - + https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore - c15021a04827e7ad60e49aba73df748892e35d25 + ed74665e773dd1ebea3289c5662d71c590305932 - + https://dev.azure.com/dnceng/internal/_git/dotnet-aspnetcore - c15021a04827e7ad60e49aba73df748892e35d25 + ed74665e773dd1ebea3289c5662d71c590305932 - + https://dev.azure.com/dnceng/internal/_git/dotnet-efcore - 55700ce7d51b40ea546f817fd4947a6bae50be07 + 6765359588e8b38bab2a7974db9398432703828f diff --git a/eng/Versions.props b/eng/Versions.props index 96ec3bac1db..607e12c2c6d 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -11,14 +11,14 @@ - false + true - + release true @@ -34,55 +34,55 @@ --> - 9.0.4 - 9.0.4 - 9.0.4 - 9.0.4 - 9.0.4 - 9.0.4 - 9.0.4 - 9.0.4 - 9.0.4 - 9.0.4 - 9.0.4 - 9.0.4 - 9.0.4 - 9.0.4 - 9.0.4 - 9.0.4 - 9.0.4 - 9.0.4 - 9.0.4 - 9.0.4 - 9.0.4 - 9.0.4 - 9.0.4 - 9.0.4 - 9.0.4 - 9.0.4 - 9.0.4 - 9.0.4 - 9.0.4 - 9.0.4 - 9.0.4 - 9.0.4 - 9.0.4 - 9.0.4 - 9.0.4 - 9.0.4 - 9.0.4 + 9.0.5 + 9.0.5 + 9.0.5 + 9.0.5 + 9.0.5 + 9.0.5 + 9.0.5 + 9.0.5 + 9.0.5 + 9.0.5 + 9.0.5 + 9.0.5 + 9.0.5 + 9.0.5 + 9.0.5 + 9.0.5 + 9.0.5 + 9.0.5 + 9.0.5 + 9.0.5 + 9.0.5 + 9.0.5 + 9.0.5 + 9.0.5 + 9.0.5 + 9.0.5 + 9.0.5 + 9.0.5 + 9.0.5 + 9.0.5 + 9.0.5 + 9.0.5 + 9.0.5 + 9.0.5 + 9.0.5 + 9.0.5 + 9.0.5 - 9.0.4 - 9.0.4 - 9.0.4 - 9.0.4 - 9.0.4 - 9.0.4 - 9.0.4 - 9.0.4 - 9.0.4 + 9.0.5 + 9.0.5 + 9.0.5 + 9.0.5 + 9.0.5 + 9.0.5 + 9.0.5 + 9.0.5 + 9.0.5 - 9.0.4 + 9.0.5 9.0.0-beta.25225.6 @@ -108,8 +108,8 @@ 8.0.1 8.0.0 8.0.2 - 8.0.14 - 8.0.14 + 8.0.16 + 8.0.16 8.0.0 8.0.1 8.0.1 @@ -126,15 +126,17 @@ 8.0.5 8.0.0 - 8.0.15 - 8.0.15 - 8.0.15 - 8.0.15 - 8.0.15 - 8.0.15 - 8.0.15 - 8.0.15 - 8.0.15 + 8.0.16 + 8.0.16 + 8.0.16 + 8.0.16 + 8.0.16 + 8.0.16 + 8.0.16 + 8.0.16 + 8.0.16 + + 8.0.16 - 9.1.0 - 9.1.0-preview.1.25121.10 + 9.2.1 + 9.2.1-preview.1.25222.1 1.0.0-beta.6 2.2.0-beta.4 1.13.2 11.6.0 - 9.3.1-beta.260 - 9.3.1-beta.260 - 9.3.1-beta.260 - 9.3.1-beta.260 + 9.4.1-beta.277 + 9.4.1-beta.277 + 9.4.1-beta.277 + 9.4.1-beta.277 9.2.0 - 1.45.0-preview - 1.45.0-preview - 1.45.0 - 5.1.12 + 1.47.0-preview + 1.47.0-preview + 1.47.0 + 5.1.13 1.9.0 0.1.9 6.0.1 diff --git a/src/LegacySupport/SystemIndex/Index.cs b/src/LegacySupport/SystemIndex/Index.cs new file mode 100644 index 00000000000..7285d669e71 --- /dev/null +++ b/src/LegacySupport/SystemIndex/Index.cs @@ -0,0 +1,160 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Diagnostics.CodeAnalysis; +using System.Runtime.CompilerServices; + +#pragma warning disable CS0436 // Type conflicts with imported type +#pragma warning disable S3427 // Method overloads with default parameter values should not overlap +#pragma warning disable SA1642 // Constructor summary documentation should begin with standard text +#pragma warning disable IDE0011 // Add braces +#pragma warning disable SA1623 // Property summary documentation should match accessors +#pragma warning disable IDE0023 // Use block body for conversion operator +#pragma warning disable S3928 // Parameter names used into ArgumentException constructors should match an existing one +#pragma warning disable LA0001 // Use the 'Microsoft.Shared.Diagnostics.Throws' class instead of explicitly throwing exception for improved performance +#pragma warning disable CA1305 // Specify IFormatProvider + +namespace System +{ + internal readonly struct Index : IEquatable + { + private readonly int _value; + + /// Construct an Index using a value and indicating if the index is from the start or from the end. + /// The index value. it has to be zero or positive number. + /// Indicating if the index is from the start or from the end. + /// + /// If the Index constructed from the end, index value 1 means pointing at the last element and index value 0 means pointing at beyond last element. + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public Index(int value, bool fromEnd = false) + { + if (value < 0) + { + ThrowValueArgumentOutOfRange_NeedNonNegNumException(); + } + + if (fromEnd) + _value = ~value; + else + _value = value; + } + + // The following private constructors mainly created for perf reason to avoid the checks + private Index(int value) + { + _value = value; + } + + /// Create an Index pointing at first element. + public static Index Start => new Index(0); + + /// Create an Index pointing at beyond last element. + public static Index End => new Index(~0); + + /// Create an Index from the start at the position indicated by the value. + /// The index value from the start. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Index FromStart(int value) + { + if (value < 0) + { + ThrowValueArgumentOutOfRange_NeedNonNegNumException(); + } + + return new Index(value); + } + + /// Create an Index from the end at the position indicated by the value. + /// The index value from the end. + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public static Index FromEnd(int value) + { + if (value < 0) + { + ThrowValueArgumentOutOfRange_NeedNonNegNumException(); + } + + return new Index(~value); + } + + /// Returns the index value. + public int Value + { + get + { + if (_value < 0) + return ~_value; + else + return _value; + } + } + + /// Indicates whether the index is from the start or the end. + public bool IsFromEnd => _value < 0; + + /// Calculate the offset from the start using the giving collection length. + /// The length of the collection that the Index will be used with. length has to be a positive value. + /// + /// For performance reason, we don't validate the input length parameter and the returned offset value against negative values. + /// we don't validate either the returned offset is greater than the input length. + /// It is expected Index will be used with collections which always have non negative length/count. If the returned offset is negative and + /// then used to index a collection will get out of range exception which will be same affect as the validation. + /// + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public int GetOffset(int length) + { + int offset = _value; + if (IsFromEnd) + { + // offset = length - (~value) + // offset = length + (~(~value) + 1) + // offset = length + value + 1 + + offset += length + 1; + } + + return offset; + } + + /// Indicates whether the current Index object is equal to another object of the same type. + /// An object to compare with this object. + public override bool Equals([NotNullWhen(true)] object? value) => value is Index && _value == ((Index)value)._value; + + /// Indicates whether the current Index object is equal to another Index object. + /// An object to compare with this object. + public bool Equals(Index other) => _value == other._value; + + /// Returns the hash code for this instance. + public override int GetHashCode() => _value; + + /// Converts integer number to an Index. + public static implicit operator Index(int value) => FromStart(value); + + /// Converts the value of the current Index object to its equivalent string representation. + public override string ToString() + { + if (IsFromEnd) + return ToStringFromEnd(); + + return ((uint)Value).ToString(); + } + + private static void ThrowValueArgumentOutOfRange_NeedNonNegNumException() + { + throw new ArgumentOutOfRangeException("value", "value must be non-negative"); + } + + private string ToStringFromEnd() + { +#if (!NETSTANDARD2_0 && !NETFRAMEWORK) + Span span = stackalloc char[11]; // 1 for ^ and 10 for longest possible uint value + bool formatted = ((uint)Value).TryFormat(span.Slice(1), out int charsWritten); + span[0] = '^'; + return new string(span.Slice(0, charsWritten + 1)); +#else + return '^' + Value.ToString(); +#endif + } + } +} \ No newline at end of file diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/CHANGELOG.md b/src/Libraries/Microsoft.Extensions.AI.Abstractions/CHANGELOG.md index c6639273d70..b4fe9d69a66 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/CHANGELOG.md +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/CHANGELOG.md @@ -1,5 +1,14 @@ # Release History +## 9.4.4-preview.1.25259.16 + +- Added `AIJsonUtilities.TransformSchema` and supporting types. +- Added `BinaryEmbedding` for bit embeddings. +- Added `ChatOptions.RawRepresentationFactory` to make it easier to pass options to the underlying service. +- Added `Base64Data` property to `DataContent`. +- Moved `AIFunctionFactory` to `Microsoft.Extensions.AI.Abstractions`. +- Fixed `AIFunctionFactory` handling of default struct arguments. + ## 9.4.3-preview.1.25230.7 - Renamed `ChatThreadId` to `ConversationId` on `ChatResponse`, `ChatResponseUpdate`, and `ChatOptions`. diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/ChatCompletion/ChatOptions.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/ChatCompletion/ChatOptions.cs index ac5fba99ae5..f2eeffe9dbf 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/ChatCompletion/ChatOptions.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/ChatCompletion/ChatOptions.cs @@ -1,6 +1,7 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. +using System; using System.Collections.Generic; using System.Text.Json.Serialization; @@ -121,6 +122,26 @@ public string? ChatThreadId [JsonIgnore] public IList? Tools { get; set; } + /// + /// Gets or sets a callback responsible of creating the raw representation of the chat options from an underlying implementation. + /// + /// + /// The underlying implementation may have its own representation of options. + /// When or + /// is invoked with a , that implementation may convert the provided options into + /// its own representation in order to use it while performing the operation. For situations where a consumer knows + /// which concrete is being used and how it represents options, a new instance of that + /// implementation-specific options type may be returned by this callback, for the + /// implementation to use instead of creating a new instance. Such implementations may mutate the supplied options + /// instance further based on other settings supplied on this instance or from other inputs, + /// like the enumerable of s, therefore, its **strongly recommended** to not return shared instances + /// and instead make the callback return a new instance per each call. + /// This is typically used to set an implementation-specific setting that isn't otherwise exposed from the strongly-typed + /// properties on . + /// + [JsonIgnore] + public Func? RawRepresentationFactory { get; set; } + /// Gets or sets any additional properties associated with the options. public AdditionalPropertiesDictionary? AdditionalProperties { get; set; } @@ -147,6 +168,7 @@ public virtual ChatOptions Clone() ModelId = ModelId, AllowMultipleToolCalls = AllowMultipleToolCalls, ToolMode = ToolMode, + RawRepresentationFactory = RawRepresentationFactory, AdditionalProperties = AdditionalProperties?.Clone(), }; diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Embeddings/BinaryEmbedding.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Embeddings/BinaryEmbedding.cs new file mode 100644 index 00000000000..2261fd97949 --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Embeddings/BinaryEmbedding.cs @@ -0,0 +1,111 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Buffers; +using System.Collections; +using System.ComponentModel; +using System.Text.Json; +using System.Text.Json.Serialization; +using Microsoft.Shared.Diagnostics; + +namespace Microsoft.Extensions.AI; + +/// Represents an embedding composed of a bit vector. +public sealed class BinaryEmbedding : Embedding +{ + /// The embedding vector this embedding represents. + private BitArray _vector; + + /// Initializes a new instance of the class with the embedding vector. + /// The embedding vector this embedding represents. + /// is . + public BinaryEmbedding(BitArray vector) + { + _vector = Throw.IfNull(vector); + } + + /// Gets or sets the embedding vector this embedding represents. + [JsonConverter(typeof(VectorConverter))] + public BitArray Vector + { + get => _vector; + set => _vector = Throw.IfNull(value); + } + + /// + [JsonIgnore] + public override int Dimensions => _vector.Length; + + /// Provides a for serializing instances. + [EditorBrowsable(EditorBrowsableState.Never)] + public sealed class VectorConverter : JsonConverter + { + /// + public override BitArray Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) + { + _ = Throw.IfNull(typeToConvert); + _ = Throw.IfNull(options); + + if (reader.TokenType != JsonTokenType.String) + { + throw new JsonException("Expected string property."); + } + + ReadOnlySpan utf8; + byte[]? tmpArray = null; + if (!reader.HasValueSequence && !reader.ValueIsEscaped) + { + utf8 = reader.ValueSpan; + } + else + { + // This path should be rare. + int length = reader.HasValueSequence ? checked((int)reader.ValueSequence.Length) : reader.ValueSpan.Length; + tmpArray = ArrayPool.Shared.Rent(length); + utf8 = tmpArray.AsSpan(0, reader.CopyString(tmpArray)); + } + + BitArray result = new(utf8.Length); + + for (int i = 0; i < utf8.Length; i++) + { + result[i] = utf8[i] switch + { + (byte)'0' => false, + (byte)'1' => true, + _ => throw new JsonException("Expected binary character sequence.") + }; + } + + if (tmpArray is not null) + { + ArrayPool.Shared.Return(tmpArray); + } + + return result; + } + + /// + public override void Write(Utf8JsonWriter writer, BitArray value, JsonSerializerOptions options) + { + _ = Throw.IfNull(writer); + _ = Throw.IfNull(value); + _ = Throw.IfNull(options); + + int length = value.Length; + + byte[] tmpArray = ArrayPool.Shared.Rent(length); + + Span utf8 = tmpArray.AsSpan(0, length); + for (int i = 0; i < utf8.Length; i++) + { + utf8[i] = value[i] ? (byte)'1' : (byte)'0'; + } + + writer.WriteStringValue(utf8); + + ArrayPool.Shared.Return(tmpArray); + } + } +} diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Embeddings/Embedding.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Embeddings/Embedding.cs index 19b8feaa182..d6596e1e53e 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Embeddings/Embedding.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Embeddings/Embedding.cs @@ -2,6 +2,7 @@ // The .NET Foundation licenses this file to you under the MIT license. using System; +using System.Diagnostics; using System.Text.Json.Serialization; namespace Microsoft.Extensions.AI; @@ -9,13 +10,15 @@ namespace Microsoft.Extensions.AI; /// Represents an embedding generated by a . /// This base class provides metadata about the embedding. Derived types provide the concrete data contained in the embedding. [JsonPolymorphic(TypeDiscriminatorPropertyName = "$type")] +[JsonDerivedType(typeof(BinaryEmbedding), typeDiscriminator: "binary")] +[JsonDerivedType(typeof(Embedding), typeDiscriminator: "uint8")] +[JsonDerivedType(typeof(Embedding), typeDiscriminator: "int8")] #if NET -[JsonDerivedType(typeof(Embedding), typeDiscriminator: "halves")] +[JsonDerivedType(typeof(Embedding), typeDiscriminator: "float16")] #endif -[JsonDerivedType(typeof(Embedding), typeDiscriminator: "floats")] -[JsonDerivedType(typeof(Embedding), typeDiscriminator: "doubles")] -[JsonDerivedType(typeof(Embedding), typeDiscriminator: "bytes")] -[JsonDerivedType(typeof(Embedding), typeDiscriminator: "sbytes")] +[JsonDerivedType(typeof(Embedding), typeDiscriminator: "float32")] +[JsonDerivedType(typeof(Embedding), typeDiscriminator: "float64")] +[DebuggerDisplay("Dimensions = {Dimensions}")] public class Embedding { /// Initializes a new instance of the class. @@ -26,6 +29,13 @@ protected Embedding() /// Gets or sets a timestamp at which the embedding was created. public DateTimeOffset? CreatedAt { get; set; } + /// Gets the dimensionality of the embedding vector. + /// + /// This value corresponds to the number of elements in the embedding vector. + /// + [JsonIgnore] + public virtual int Dimensions { get; } + /// Gets or sets the model ID using in the creation of the embedding. public string? ModelId { get; set; } diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Embeddings/Embedding{T}.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Embeddings/Embedding{T}.cs index c80e20dfda4..22bc02f2f3f 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Embeddings/Embedding{T}.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Embeddings/Embedding{T}.cs @@ -2,6 +2,7 @@ // The .NET Foundation licenses this file to you under the MIT license. using System; +using System.Text.Json.Serialization; namespace Microsoft.Extensions.AI; @@ -19,4 +20,8 @@ public Embedding(ReadOnlyMemory vector) /// Gets or sets the embedding vector this embedding represents. public ReadOnlyMemory Vector { get; set; } + + /// + [JsonIgnore] + public override int Dimensions => Vector.Length; } diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Functions/AIFunctionArguments.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Functions/AIFunctionArguments.cs index 8fa34e52c08..3238b88e532 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Functions/AIFunctionArguments.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Functions/AIFunctionArguments.cs @@ -4,6 +4,7 @@ using System; using System.Collections; using System.Collections.Generic; +using System.Diagnostics; #pragma warning disable SA1111 // Closing parenthesis should be on line of last parameter #pragma warning disable SA1112 // Closing parenthesis should be on line of opening parenthesis @@ -24,6 +25,7 @@ namespace Microsoft.Extensions.AI; /// an if it needs to resolve any services from a dependency injection /// container. /// +[DebuggerDisplay("Count = {Count}")] public class AIFunctionArguments : IDictionary, IReadOnlyDictionary { /// The nominal arguments. diff --git a/src/Libraries/Microsoft.Extensions.AI/Functions/AIFunctionFactory.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Functions/AIFunctionFactory.cs similarity index 80% rename from src/Libraries/Microsoft.Extensions.AI/Functions/AIFunctionFactory.cs rename to src/Libraries/Microsoft.Extensions.AI.Abstractions/Functions/AIFunctionFactory.cs index 4878239f35b..d5274186645 100644 --- a/src/Libraries/Microsoft.Extensions.AI/Functions/AIFunctionFactory.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Functions/AIFunctionFactory.cs @@ -2,11 +2,12 @@ // The .NET Foundation licenses this file to you under the MIT license. using System; +using System.Buffers; using System.Collections.Concurrent; using System.Collections.Generic; using System.ComponentModel; using System.Diagnostics; -using System.Diagnostics.CodeAnalysis; +using System.IO; #if !NET using System.Linq; #endif @@ -15,23 +16,27 @@ using System.Text.Json; using System.Text.Json.Nodes; using System.Text.Json.Serialization.Metadata; +using System.Text.RegularExpressions; using System.Threading; using System.Threading.Tasks; -using Microsoft.Extensions.DependencyInjection; using Microsoft.Shared.Collections; using Microsoft.Shared.Diagnostics; #pragma warning disable CA1031 // Do not catch general exception types +#pragma warning disable S2333 // Redundant modifiers should not be used #pragma warning disable S3011 // Reflection should not be used to increase accessibility of classes, methods, or fields -#pragma warning disable SA1118 // Parameter should not span multiple lines -#pragma warning disable SA1500 // Braces for multi-line statements should not share line +#pragma warning disable SA1202 // Public members should come before private members namespace Microsoft.Extensions.AI; -/// Provides factory methods for creating commonly used implementations of . +/// Provides factory methods for creating commonly-used implementations of . /// Invoke .NET functions using an AI model. public static partial class AIFunctionFactory { + // NOTE: + // Unlike most library code, AIFunctionFactory uses ConfigureAwait(true) rather than ConfigureAwait(false). This is to + // enable AIFunctionFactory to be used with methods that might be context-aware, such as those employing a UI framework. + /// Holds the default options instance used when creating function. private static readonly AIFunctionFactoryOptions _defaultOptions = new(); @@ -71,25 +76,6 @@ public static partial class AIFunctionFactory /// . /// /// - /// - /// - /// By default, parameters attributed with are resolved from the - /// property and are not included in the JSON schema. If the parameter is optional, such that a default value is provided, - /// is allowed to be ; otherwise, - /// must be non-, or else the invocation will fail with an exception due to the required nature of the parameter. - /// The handling of such parameters may be overridden via . - /// - /// - /// - /// - /// When the is constructed, it may be passed an via - /// . Any parameter that can be satisfied by that - /// according to will not be included in the generated JSON schema and will be resolved - /// from the provided to via , - /// rather than from the argument collection. The handling of such parameters may be overridden via - /// . - /// - /// /// /// All other parameter types are, by default, bound from the dictionary passed into /// and are included in the generated JSON schema. This may be overridden by the provided @@ -170,23 +156,6 @@ public static AIFunction Create(Delegate method, AIFunctionFactoryOptions? optio /// optional or not. /// /// - /// - /// - /// By default, parameters attributed with are resolved from the - /// property and are not included in the JSON schema. If the parameter is optional, such that a default value is provided, - /// is allowed to be ; otherwise, - /// must be non-, or else the invocation will fail with an exception due to the required nature of the parameter. - /// - /// - /// - /// - /// When the is constructed, it may be passed an via - /// . Any parameter that can be satisfied by that - /// according to will not be included in the generated JSON schema and will be resolved - /// from the provided to via , - /// rather than from the argument collection. - /// - /// /// /// All other parameter types are bound from the dictionary passed into /// and are included in the generated JSON schema. @@ -270,25 +239,6 @@ public static AIFunction Create(Delegate method, string? name = null, string? de /// . /// /// - /// - /// - /// By default, parameters attributed with are resolved from the - /// property and are not included in the JSON schema. If the parameter is optional, such that a default value is provided, - /// is allowed to be ; otherwise, - /// must be non-, or else the invocation will fail with an exception due to the required nature of the parameter. - /// The handling of such parameters may be overridden via . - /// - /// - /// - /// - /// When the is constructed, it may be passed an via - /// . Any parameter that can be satisfied by that - /// according to will not be included in the generated JSON schema and will be resolved - /// from the provided to via , - /// rather than from the argument collection. The handling of such parameters may be overridden via - /// . - /// - /// /// /// All other parameter types are, by default, bound from the dictionary passed into /// and are included in the generated JSON schema. This may be overridden by the provided @@ -379,23 +329,6 @@ public static AIFunction Create(MethodInfo method, object? target, AIFunctionFac /// optional or not. /// /// - /// - /// - /// By default, parameters attributed with are resolved from the - /// property and are not included in the JSON schema. If the parameter is optional, such that a default value is provided, - /// is allowed to be ; otherwise, - /// must be non-, or else the invocation will fail with an exception due to the required nature of the parameter. - /// - /// - /// - /// When the is constructed, it may be passed an via - /// . Any parameter that can be satisfied by that - /// according to will not be included in the generated JSON schema and will be resolved - /// from the provided to via , - /// rather than from the argument collection. - /// - /// - /// /// /// All other parameter types are bound from the dictionary passed into /// and are included in the generated JSON schema. @@ -440,17 +373,15 @@ public static AIFunction Create(MethodInfo method, object? target, string? name } /// - /// Creates an instance for a method, specified via an for - /// and instance method, along with a representing the type of the target object to - /// instantiate each time the method is invoked. + /// Creates an instance for a method, specified via a for + /// an instance method and a for constructing an instance of + /// the receiver object each time the is invoked. /// /// The instance method to be represented via the created . - /// - /// The to construct an instance of on which to invoke when - /// the resulting is invoked. If is provided, - /// will be used to construct the instance using those services; otherwise, - /// is used, utilizing the type's public parameterless constructor. - /// If an instance can't be constructed, an exception is thrown during the function's invocation. + /// + /// Callback used on each function invocation to create an instance of the type on which the instance method + /// will be invoked. If the returned instance is or , it will be disposed of + /// after completes its invocation. /// /// Metadata to use to override defaults inferred from . /// The created for invoking . @@ -494,25 +425,6 @@ public static AIFunction Create(MethodInfo method, object? target, string? name /// . /// /// - /// - /// - /// By default, parameters attributed with are resolved from the - /// property and are not included in the JSON schema. If the parameter is optional, such that a default value is provided, - /// is allowed to be ; otherwise, - /// must be non-, or else the invocation will fail with an exception due to the required nature of the parameter. - /// The handling of such parameters may be overridden via . - /// - /// - /// - /// - /// When the is constructed, it may be passed an via - /// . Any parameter that can be satisfied by that - /// according to will not be included in the generated JSON schema and will be resolved - /// from the provided to via , - /// rather than from the argument collection. The handling of such parameters may be overridden via - /// . - /// - /// /// /// All other parameter types are, by default, bound from the dictionary passed into /// and are included in the generated JSON schema. This may be overridden by the provided @@ -543,22 +455,16 @@ public static AIFunction Create(MethodInfo method, object? target, string? name /// /// /// is . - /// is . + /// is . /// represents a static method. /// represents an open generic method. /// contains a parameter without a parameter name. - /// is not assignable to 's declaring type. /// A parameter to or its return type is not serializable. public static AIFunction Create( MethodInfo method, - [DynamicallyAccessedMembers(DynamicallyAccessedMemberTypes.PublicConstructors)] Type targetType, - AIFunctionFactoryOptions? options = null) - { - _ = Throw.IfNull(method); - _ = Throw.IfNull(targetType); - - return ReflectionAIFunction.Build(method, targetType, options ?? _defaultOptions); - } + Func createInstanceFunc, + AIFunctionFactoryOptions? options = null) => + ReflectionAIFunction.Build(method, createInstanceFunc, options ?? _defaultOptions); private sealed class ReflectionAIFunction : AIFunction { @@ -589,10 +495,11 @@ public static ReflectionAIFunction Build(MethodInfo method, object? target, AIFu public static ReflectionAIFunction Build( MethodInfo method, - [DynamicallyAccessedMembers(DynamicallyAccessedMemberTypes.PublicConstructors)] Type targetType, + Func createInstanceFunc, AIFunctionFactoryOptions options) { _ = Throw.IfNull(method); + _ = Throw.IfNull(createInstanceFunc); if (method.ContainsGenericParameters) { @@ -604,13 +511,7 @@ public static ReflectionAIFunction Build( Throw.ArgumentException(nameof(method), "The method must be an instance method."); } - if (method.DeclaringType is { } declaringType && - !declaringType.IsAssignableFrom(targetType)) - { - Throw.ArgumentException(nameof(targetType), "The target type must be assignable to the method's declaring type."); - } - - return new(ReflectionAIFunctionDescriptor.GetOrCreate(method, options), targetType, options); + return new(ReflectionAIFunctionDescriptor.GetOrCreate(method, options), createInstanceFunc, options); } private ReflectionAIFunction(ReflectionAIFunctionDescriptor functionDescriptor, object? target, AIFunctionFactoryOptions options) @@ -622,18 +523,18 @@ private ReflectionAIFunction(ReflectionAIFunctionDescriptor functionDescriptor, private ReflectionAIFunction( ReflectionAIFunctionDescriptor functionDescriptor, - [DynamicallyAccessedMembers(DynamicallyAccessedMemberTypes.PublicConstructors)] Type targetType, + Func createInstanceFunc, AIFunctionFactoryOptions options) { FunctionDescriptor = functionDescriptor; - TargetType = targetType; + CreateInstanceFunc = createInstanceFunc; AdditionalProperties = options.AdditionalProperties ?? EmptyReadOnlyDictionary.Instance; } public ReflectionAIFunctionDescriptor FunctionDescriptor { get; } public object? Target { get; } - [DynamicallyAccessedMembers(DynamicallyAccessedMemberTypes.PublicConstructors)] - public Type? TargetType { get; } + public Func? CreateInstanceFunc { get; } + public override IReadOnlyDictionary AdditionalProperties { get; } public override string Name => FunctionDescriptor.Name; public override string Description => FunctionDescriptor.Description; @@ -649,14 +550,17 @@ private ReflectionAIFunction( object? target = Target; try { - if (TargetType is { } targetType) + if (CreateInstanceFunc is { } func) { Debug.Assert(target is null, "Expected target to be null when we have a non-null target type"); Debug.Assert(!FunctionDescriptor.Method.IsStatic, "Expected an instance method"); - target = arguments.Services is { } services ? - ActivatorUtilities.CreateInstance(services, targetType!) : - Activator.CreateInstance(targetType); + target = func(arguments); + if (target is null) + { + Throw.InvalidOperationException("Unable to create an instance of the target type."); + } + disposeTarget = true; } @@ -669,7 +573,7 @@ private ReflectionAIFunction( } return await FunctionDescriptor.ReturnParameterMarshaller( - ReflectionInvoke(FunctionDescriptor.Method, target, args), cancellationToken); + ReflectionInvoke(FunctionDescriptor.Method, target, args), cancellationToken).ConfigureAwait(true); } finally { @@ -677,7 +581,7 @@ private ReflectionAIFunction( { if (target is IAsyncDisposable ad) { - await ad.DisposeAsync(); + await ad.DisposeAsync().ConfigureAwait(true); } else if (target is IDisposable d) { @@ -709,7 +613,7 @@ public static ReflectionAIFunctionDescriptor GetOrCreate(MethodInfo method, AIFu serializerOptions.MakeReadOnly(); ConcurrentDictionary innerCache = _descriptorCache.GetOrCreateValue(serializerOptions); - DescriptorKey key = new(method, options.Name, options.Description, options.ConfigureParameterBinding, options.MarshalResult, options.Services, schemaOptions); + DescriptorKey key = new(method, options.Name, options.Description, options.ConfigureParameterBinding, options.MarshalResult, schemaOptions); if (innerCache.TryGetValue(key, out ReflectionAIFunctionDescriptor? descriptor)) { return descriptor; @@ -736,8 +640,6 @@ private ReflectionAIFunctionDescriptor(DescriptorKey key, JsonSerializerOptions } } - IServiceProviderIsService? serviceProviderIsService = key.Services?.GetService(); - // Use that binding information to impact the schema generation. AIJsonSchemaCreateOptions schemaOptions = key.SchemaOptions with { @@ -757,21 +659,6 @@ private ReflectionAIFunctionDescriptor(DescriptorKey key, JsonSerializerOptions return false; } - // If the parameter is attributed as [FromKeyedServices], exclude it, as we'll instead - // get its value from the IServiceProvider. - if (parameterInfo.GetCustomAttribute(inherit: true) is not null) - { - return false; - } - - // We assume that if the services used to create the function support a particular type, - // so too do the services that will be passed into InvokeAsync. This is the same basic assumption - // made in ASP.NET. - if (serviceProviderIsService?.IsService(parameterInfo.ParameterType) is true) - { - return false; - } - // If there was an existing IncludeParameter delegate, now defer to it as we've // excluded everything we need to exclude. if (key.SchemaOptions.IncludeParameter is { } existingIncludeParameter) @@ -793,7 +680,7 @@ private ReflectionAIFunctionDescriptor(DescriptorKey key, JsonSerializerOptions options = default; } - ParameterMarshallers[i] = GetParameterMarshaller(serializerOptions, options, parameters[i], serviceProviderIsService); + ParameterMarshallers[i] = GetParameterMarshaller(serializerOptions, options, parameters[i]); } // Get a marshaling delegate for the return value. @@ -863,8 +750,7 @@ static bool IsAsyncMethod(MethodInfo method) private static Func GetParameterMarshaller( JsonSerializerOptions serializerOptions, AIFunctionFactoryOptions.ParameterBindingOptions bindingOptions, - ParameterInfo parameter, - IServiceProviderIsService? serviceProviderIsService) + ParameterInfo parameter) { if (string.IsNullOrWhiteSpace(parameter.Name)) { @@ -911,56 +797,6 @@ static bool IsAsyncMethod(MethodInfo method) }; } - // For [FromKeyedServices] parameters, we resolve from the services passed to InvokeAsync via AIFunctionArguments. - if (parameter.GetCustomAttribute(inherit: true) is { } keyedAttr) - { - return (arguments, _) => - { - if ((arguments.Services as IKeyedServiceProvider)?.GetKeyedService(parameterType, keyedAttr.Key) is { } service) - { - return service; - } - - if (!parameter.HasDefaultValue) - { - if (arguments.Services is null) - { - ThrowNullServices(parameter.Name); - } - - Throw.ArgumentException(nameof(arguments), $"No service of type '{parameterType}' with key '{keyedAttr.Key}' was found for parameter '{parameter.Name}'."); - } - - return parameter.DefaultValue; - }; - } - - // For any parameters that are satisfiable from the IServiceProvider, we resolve from the services passed to InvokeAsync - // via AIFunctionArguments. This is determined by the same same IServiceProviderIsService instance used to determine whether - // the parameter should be included in the schema. - if (serviceProviderIsService?.IsService(parameterType) is true) - { - return (arguments, _) => - { - if (arguments.Services?.GetService(parameterType) is { } service) - { - return service; - } - - if (!parameter.HasDefaultValue) - { - if (arguments.Services is null) - { - ThrowNullServices(parameter.Name); - } - - Throw.ArgumentException(nameof(arguments), $"No service of type '{parameterType}' was found for parameter '{parameter.Name}'."); - } - - return parameter.DefaultValue; - }; - } - // For all other parameters, create a marshaller that tries to extract the value from the arguments dictionary. // Resolve the contract used to marshal the value from JSON -- can throw if not supported or not found. JsonTypeInfo? typeInfo = serializerOptions.GetTypeInfo(parameterType); @@ -1037,14 +873,14 @@ static void ThrowNullServices(string parameterName) => { return async (result, cancellationToken) => { - await ((Task)ThrowIfNullResult(result)); - return await marshalResult(null, null, cancellationToken); + await ((Task)ThrowIfNullResult(result)).ConfigureAwait(true); + return await marshalResult(null, null, cancellationToken).ConfigureAwait(true); }; } return async static (result, _) => { - await ((Task)ThrowIfNullResult(result)); + await ((Task)ThrowIfNullResult(result)).ConfigureAwait(true); return null; }; } @@ -1056,14 +892,14 @@ static void ThrowNullServices(string parameterName) => { return async (result, cancellationToken) => { - await ((ValueTask)ThrowIfNullResult(result)); - return await marshalResult(null, null, cancellationToken); + await ((ValueTask)ThrowIfNullResult(result)).ConfigureAwait(true); + return await marshalResult(null, null, cancellationToken).ConfigureAwait(true); }; } return async static (result, _) => { - await ((ValueTask)ThrowIfNullResult(result)); + await ((ValueTask)ThrowIfNullResult(result)).ConfigureAwait(true); return null; }; } @@ -1078,18 +914,18 @@ static void ThrowNullServices(string parameterName) => { return async (taskObj, cancellationToken) => { - await ((Task)ThrowIfNullResult(taskObj)); + await ((Task)ThrowIfNullResult(taskObj)).ConfigureAwait(true); object? result = ReflectionInvoke(taskResultGetter, taskObj, null); - return await marshalResult(result, taskResultGetter.ReturnType, cancellationToken); + return await marshalResult(result, taskResultGetter.ReturnType, cancellationToken).ConfigureAwait(true); }; } returnTypeInfo = serializerOptions.GetTypeInfo(taskResultGetter.ReturnType); return async (taskObj, cancellationToken) => { - await ((Task)ThrowIfNullResult(taskObj)); + await ((Task)ThrowIfNullResult(taskObj)).ConfigureAwait(true); object? result = ReflectionInvoke(taskResultGetter, taskObj, null); - return await SerializeResultAsync(result, returnTypeInfo, cancellationToken); + return await SerializeResultAsync(result, returnTypeInfo, cancellationToken).ConfigureAwait(true); }; } @@ -1104,9 +940,9 @@ static void ThrowNullServices(string parameterName) => return async (taskObj, cancellationToken) => { var task = (Task)ReflectionInvoke(valueTaskAsTask, ThrowIfNullResult(taskObj), null)!; - await task; + await task.ConfigureAwait(true); object? result = ReflectionInvoke(asTaskResultGetter, task, null); - return await marshalResult(result, asTaskResultGetter.ReturnType, cancellationToken); + return await marshalResult(result, asTaskResultGetter.ReturnType, cancellationToken).ConfigureAwait(true); }; } @@ -1114,9 +950,9 @@ static void ThrowNullServices(string parameterName) => return async (taskObj, cancellationToken) => { var task = (Task)ReflectionInvoke(valueTaskAsTask, ThrowIfNullResult(taskObj), null)!; - await task; + await task.ConfigureAwait(true); object? result = ReflectionInvoke(asTaskResultGetter, task, null); - return await SerializeResultAsync(result, returnTypeInfo, cancellationToken); + return await SerializeResultAsync(result, returnTypeInfo, cancellationToken).ConfigureAwait(true); }; } } @@ -1140,7 +976,7 @@ static void ThrowNullServices(string parameterName) => // Serialize asynchronously to support potential IAsyncEnumerable responses. using PooledMemoryStream stream = new(); - await JsonSerializer.SerializeAsync(stream, result, returnTypeInfo, cancellationToken); + await JsonSerializer.SerializeAsync(stream, result, returnTypeInfo, cancellationToken).ConfigureAwait(true); Utf8JsonReader reader = new(stream.GetBuffer()); return JsonElement.ParseValue(ref reader); } @@ -1169,7 +1005,154 @@ private record struct DescriptorKey( string? Description, Func? GetBindParameterOptions, Func>? MarshalResult, - IServiceProvider? Services, AIJsonSchemaCreateOptions SchemaOptions); } + + /// + /// Removes characters from a .NET member name that shouldn't be used in an AI function name. + /// + /// The .NET member name that should be sanitized. + /// + /// Replaces non-alphanumeric characters in the identifier with the underscore character. + /// Primarily intended to remove characters produced by compiler-generated method name mangling. + /// + private static string SanitizeMemberName(string memberName) => + InvalidNameCharsRegex().Replace(memberName, "_"); + + /// Regex that flags any character other than ASCII digits or letters or the underscore. +#if NET + [GeneratedRegex("[^0-9A-Za-z_]")] + private static partial Regex InvalidNameCharsRegex(); +#else + private static Regex InvalidNameCharsRegex() => _invalidNameCharsRegex; + private static readonly Regex _invalidNameCharsRegex = new("[^0-9A-Za-z_]", RegexOptions.Compiled); +#endif + + /// Invokes the MethodInfo with the specified target object and arguments. + private static object? ReflectionInvoke(MethodInfo method, object? target, object?[]? arguments) + { +#if NET + return method.Invoke(target, BindingFlags.DoNotWrapExceptions, binder: null, arguments, culture: null); +#else + try + { + return method.Invoke(target, BindingFlags.Default, binder: null, arguments, culture: null); + } + catch (TargetInvocationException e) when (e.InnerException is not null) + { + // If we're targeting .NET Framework, such that BindingFlags.DoNotWrapExceptions + // is ignored, the original exception will be wrapped in a TargetInvocationException. + // Unwrap it and throw that original exception, maintaining its stack information. + System.Runtime.ExceptionServices.ExceptionDispatchInfo.Capture(e.InnerException).Throw(); + throw; + } +#endif + } + + /// + /// Implements a simple write-only memory stream that uses pooled buffers. + /// + private sealed class PooledMemoryStream : Stream + { + private const int DefaultBufferSize = 4096; + private byte[] _buffer; + private int _position; + + public PooledMemoryStream(int initialCapacity = DefaultBufferSize) + { + _buffer = ArrayPool.Shared.Rent(initialCapacity); + _position = 0; + } + + public ReadOnlySpan GetBuffer() => _buffer.AsSpan(0, _position); + public override bool CanWrite => true; + public override bool CanRead => false; + public override bool CanSeek => false; + public override long Length => _position; + public override long Position + { + get => _position; + set => throw new NotSupportedException(); + } + + public override void Write(byte[] buffer, int offset, int count) + { + EnsureNotDisposed(); + EnsureCapacity(_position + count); + + Buffer.BlockCopy(buffer, offset, _buffer, _position, count); + _position += count; + } + + public override void Flush() + { + } + + public override Task FlushAsync(CancellationToken cancellationToken) => + Task.CompletedTask; + + public override Task WriteAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) => + WriteAsync(new ReadOnlyMemory(buffer, offset, count), cancellationToken).AsTask(); + +#if NET + public override +#else + private +#endif + ValueTask WriteAsync(ReadOnlyMemory buffer, CancellationToken cancellationToken = default) + { + EnsureNotDisposed(); + + if (cancellationToken.IsCancellationRequested) + { + return new ValueTask(Task.FromCanceled(cancellationToken)); + } + + EnsureCapacity(_position + buffer.Length); + + buffer.Span.CopyTo(_buffer.AsSpan(_position)); + _position += buffer.Length; + + return default; + } + + public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); + public override void SetLength(long value) => throw new NotSupportedException(); + + protected override void Dispose(bool disposing) + { + if (_buffer is not null) + { + ArrayPool.Shared.Return(_buffer); + _buffer = null!; + } + + base.Dispose(disposing); + } + + private void EnsureCapacity(int requiredCapacity) + { + if (requiredCapacity <= _buffer.Length) + { + return; + } + + int newCapacity = Math.Max(requiredCapacity, _buffer.Length * 2); + byte[] newBuffer = ArrayPool.Shared.Rent(newCapacity); + Buffer.BlockCopy(_buffer, 0, newBuffer, 0, _position); + + ArrayPool.Shared.Return(_buffer); + _buffer = newBuffer; + } + + private void EnsureNotDisposed() + { + if (_buffer is null) + { + Throw(); + static void Throw() => throw new ObjectDisposedException(nameof(PooledMemoryStream)); + } + } + } } diff --git a/src/Libraries/Microsoft.Extensions.AI/Functions/AIFunctionFactoryOptions.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Functions/AIFunctionFactoryOptions.cs similarity index 93% rename from src/Libraries/Microsoft.Extensions.AI/Functions/AIFunctionFactoryOptions.cs rename to src/Libraries/Microsoft.Extensions.AI.Abstractions/Functions/AIFunctionFactoryOptions.cs index 80c8b485c59..e71a4687422 100644 --- a/src/Libraries/Microsoft.Extensions.AI/Functions/AIFunctionFactoryOptions.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Functions/AIFunctionFactoryOptions.cs @@ -106,16 +106,6 @@ public AIFunctionFactoryOptions() /// public Func>? MarshalResult { get; set; } - /// - /// Gets or sets optional services used in the construction of the . - /// - /// - /// These services will be used to determine which parameters should be satisifed from dependency injection. As such, - /// what services are satisfied via this provider should match what's satisfied via the provider passed into - /// via . - /// - public IServiceProvider? Services { get; set; } - /// Provides configuration options produced by the delegate. public readonly record struct ParameterBindingOptions { diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Microsoft.Extensions.AI.Abstractions.csproj b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Microsoft.Extensions.AI.Abstractions.csproj index 27a2c5d0513..dc6896b7f53 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Microsoft.Extensions.AI.Abstractions.csproj +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Microsoft.Extensions.AI.Abstractions.csproj @@ -28,6 +28,7 @@ true true true + true diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Utilities/AIJsonSchemaCreateOptions.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Utilities/AIJsonSchemaCreateOptions.cs index 8e3269f7a9c..8c53938f481 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Utilities/AIJsonSchemaCreateOptions.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Utilities/AIJsonSchemaCreateOptions.cs @@ -38,22 +38,33 @@ public sealed record class AIJsonSchemaCreateOptions public Func? IncludeParameter { get; init; } /// - /// Gets a value indicating whether to include the type keyword in inferred schemas for .NET enums. + /// Gets a governing transformations on the JSON schema after it has been generated. /// + public AIJsonSchemaTransformOptions? TransformOptions { get; init; } + + /// + /// Gets a value indicating whether to include the type keyword in created schemas for .NET enums. + /// + [Obsolete("This property has been deprecated.")] + [System.ComponentModel.EditorBrowsable(System.ComponentModel.EditorBrowsableState.Never)] public bool IncludeTypeInEnumSchemas { get; init; } = true; /// /// Gets a value indicating whether to generate schemas with the additionalProperties set to false for .NET objects. /// - public bool DisallowAdditionalProperties { get; init; } = true; + [Obsolete("This property has been deprecated. Use the equivalent property in TransformOptions instead.")] + [System.ComponentModel.EditorBrowsable(System.ComponentModel.EditorBrowsableState.Never)] + public bool DisallowAdditionalProperties { get; init; } /// - /// Gets a value indicating whether to include the $schema keyword in inferred schemas. + /// Gets a value indicating whether to include the $schema keyword in created schemas. /// public bool IncludeSchemaKeyword { get; init; } /// /// Gets a value indicating whether to mark all properties as required in the schema. /// + [Obsolete("This property has been deprecated. Use the equivalent property in TransformOptions instead.")] + [System.ComponentModel.EditorBrowsable(System.ComponentModel.EditorBrowsableState.Never)] public bool RequireAllProperties { get; init; } } diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Utilities/AIJsonSchemaTransformCache.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Utilities/AIJsonSchemaTransformCache.cs new file mode 100644 index 00000000000..a1aaeff26ac --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Utilities/AIJsonSchemaTransformCache.cs @@ -0,0 +1,78 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Runtime.CompilerServices; +using System.Text.Json; +using Microsoft.Shared.Diagnostics; + +namespace Microsoft.Extensions.AI; + +/// +/// Defines a cache for JSON schemas transformed according to the specified policy. +/// +/// +/// +/// This cache stores weak references from AI abstractions that declare JSON schemas such as or +/// to their corresponding JSON schemas transformed according to the specified policy. It is intended for use by +/// implementations that enforce vendor-specific restrictions on what constitutes a valid JSON schema for a given function or response format. +/// +/// +/// It is recommended implementations with schema transformation requirements should create a single static instance of this cache. +/// +/// +public sealed class AIJsonSchemaTransformCache +{ + private readonly ConditionalWeakTable _functionSchemaCache = new(); + private readonly ConditionalWeakTable _responseFormatCache = new(); + + private readonly ConditionalWeakTable.CreateValueCallback _functionSchemaCreateValueCallback; + private readonly ConditionalWeakTable.CreateValueCallback _responseFormatCreateValueCallback; + + /// + /// Initializes a new instance of the class with the specified options. + /// + /// The options governing schema transformation. + public AIJsonSchemaTransformCache(AIJsonSchemaTransformOptions transformOptions) + { + _ = Throw.IfNull(transformOptions); + + if (transformOptions == AIJsonSchemaTransformOptions.Default) + { + Throw.ArgumentException(nameof(transformOptions), "The options instance does not specify any transformations."); + } + + TransformOptions = transformOptions; + _functionSchemaCreateValueCallback = function => AIJsonUtilities.TransformSchema(function.JsonSchema, TransformOptions); + _responseFormatCreateValueCallback = responseFormat => AIJsonUtilities.TransformSchema(responseFormat.Schema!.Value, TransformOptions); + } + + /// + /// Gets the options governing schema transformation. + /// + public AIJsonSchemaTransformOptions TransformOptions { get; } + + /// + /// Gets or creates a transformed JSON schema for the specified instance. + /// + /// The function whose JSON schema we want to transform. + /// The transformed JSON schema corresponding to . + public JsonElement GetOrCreateTransformedSchema(AIFunction function) + { + _ = Throw.IfNull(function); + return (JsonElement)_functionSchemaCache.GetValue(function, _functionSchemaCreateValueCallback); + } + + /// + /// Gets or creates a transformed JSON schema for the specified instance. + /// + /// The response format whose JSON schema we want to transform. + /// The transformed JSON schema corresponding to . + public JsonElement? GetOrCreateTransformedSchema(ChatResponseFormatJson responseFormat) + { + _ = Throw.IfNull(responseFormat); + return responseFormat.Schema is not null + ? (JsonElement?)_responseFormatCache.GetValue(responseFormat, _responseFormatCreateValueCallback) + : null; + } +} diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Utilities/AIJsonSchemaTransformContext.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Utilities/AIJsonSchemaTransformContext.cs new file mode 100644 index 00000000000..4cfd08e160b --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Utilities/AIJsonSchemaTransformContext.cs @@ -0,0 +1,45 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +#pragma warning disable CA1815 // Override equals and operator equals on value types + +using System; + +namespace Microsoft.Extensions.AI; + +/// +/// Defines the context for transforming a schema node withing a larger schema document. +/// +/// +/// This struct is being passed to the user-provided +/// callback by the method and cannot be instantiated directly. +/// +public readonly struct AIJsonSchemaTransformContext +{ + private readonly string[] _path; + + internal AIJsonSchemaTransformContext(string[] path) + { + _path = path; + } + + /// + /// Gets the path to the schema document currently being generated. + /// + public ReadOnlySpan Path => _path; + + /// + /// Gets the containing property name if the current schema is a property of an object. + /// + public string? PropertyName => Path is [.., "properties", string name] ? name : null; + + /// + /// Gets a value indicating whether the current schema is a collection element. + /// + public bool IsCollectionElementSchema => Path is [.., "items"]; + + /// + /// Gets a value indicating whether the current schema is a dictionary value. + /// + public bool IsDictionaryValueSchema => Path is [.., "additionalProperties"]; +} diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Utilities/AIJsonSchemaTransformOptions.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Utilities/AIJsonSchemaTransformOptions.cs new file mode 100644 index 00000000000..46e7476afcf --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Utilities/AIJsonSchemaTransformOptions.cs @@ -0,0 +1,50 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +#pragma warning disable S1067 // Expressions should not be too complex + +using System; +using System.Text.Json.Nodes; + +namespace Microsoft.Extensions.AI; + +/// +/// Provides options for configuring the behavior of JSON schema transformation functionality. +/// +public sealed record class AIJsonSchemaTransformOptions +{ + /// + /// Gets a callback that is invoked for every schema that is generated within the type graph. + /// + public Func? TransformSchemaNode { get; init; } + + /// + /// Gets a value indicating whether to convert boolean schemas to equivalent object-based representations. + /// + public bool ConvertBooleanSchemas { get; init; } + + /// + /// Gets a value indicating whether to generate schemas with the additionalProperties set to false for .NET objects. + /// + public bool DisallowAdditionalProperties { get; init; } + + /// + /// Gets a value indicating whether to mark all properties as required in the schema. + /// + public bool RequireAllProperties { get; init; } + + /// + /// Gets a value indicating whether to substitute nullable "type" keywords with OpenAPI 3.0 style "nullable" keywords in the schema. + /// + public bool UseNullableKeyword { get; init; } + + /// + /// Gets a value indicating whether to move the default keyword to the description field in the schema. + /// + public bool MoveDefaultKeywordToDescription { get; init; } + + /// + /// Gets the default options instance. + /// + internal static AIJsonSchemaTransformOptions Default { get; } = new(); +} diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Utilities/AIJsonUtilities.Defaults.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Utilities/AIJsonUtilities.Defaults.cs index 8ab0152c941..33531661813 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Utilities/AIJsonUtilities.Defaults.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Utilities/AIJsonUtilities.Defaults.cs @@ -116,4 +116,11 @@ private static JsonSerializerOptions CreateDefaultOptions() [JsonSerializable(typeof(AIFunctionArguments))] [EditorBrowsable(EditorBrowsableState.Never)] // Never use JsonContext directly, use DefaultOptions instead. private sealed partial class JsonContext : JsonSerializerContext; + + [JsonSourceGenerationOptions(JsonSerializerDefaults.Web, + UseStringEnumConverter = true, + DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull, + WriteIndented = false)] + [JsonSerializable(typeof(JsonNode))] + private sealed partial class JsonContextNoIndentation : JsonSerializerContext; } diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Utilities/AIJsonUtilities.Schema.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Utilities/AIJsonUtilities.Schema.Create.cs similarity index 79% rename from src/Libraries/Microsoft.Extensions.AI.Abstractions/Utilities/AIJsonUtilities.Schema.cs rename to src/Libraries/Microsoft.Extensions.AI.Abstractions/Utilities/AIJsonUtilities.Schema.Create.cs index a0afa66f98c..a44836d8e96 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Utilities/AIJsonUtilities.Schema.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Utilities/AIJsonUtilities.Schema.Create.cs @@ -5,7 +5,6 @@ using System.ComponentModel; using System.Diagnostics; using System.Diagnostics.CodeAnalysis; -using System.Linq; using System.Reflection; using System.Runtime.CompilerServices; using System.Text.Json; @@ -34,12 +33,13 @@ public static partial class AIJsonUtilities private const string PatternPropertyName = "pattern"; private const string EnumPropertyName = "enum"; private const string PropertiesPropertyName = "properties"; + private const string ItemsPropertyName = "items"; private const string RequiredPropertyName = "required"; private const string AdditionalPropertiesPropertyName = "additionalProperties"; private const string DefaultPropertyName = "default"; private const string RefPropertyName = "$ref"; - /// The uri used when populating the $schema keyword in inferred schemas. + /// The uri used when populating the $schema keyword in created schemas. private const string SchemaKeywordUri = "https://json-schema.org/draft/2020-12/schema"; // List of keywords used by JsonSchemaExporter but explicitly disallowed by some AI vendors. @@ -53,7 +53,7 @@ public static partial class AIJsonUtilities /// The title keyword used by the method schema. /// The description keyword used by the method schema. /// The options used to extract the schema from the specified type. - /// The options controlling schema inference. + /// The options controlling schema creation. /// A JSON schema document encoded as a . /// is . public static JsonElement CreateFunctionJsonSchema( @@ -105,13 +105,13 @@ public static JsonElement CreateFunctionJsonSchema( inferenceOptions); parameterSchemas.Add(parameter.Name, parameterSchema); - if (!parameter.IsOptional || inferenceOptions.RequireAllProperties) + if (!parameter.IsOptional) { (requiredProperties ??= []).Add((JsonNode)parameter.Name); } } - JsonObject schema = new(); + JsonNode schema = new JsonObject(); if (inferenceOptions.IncludeSchemaKeyword) { schema[SchemaPropertyName] = SchemaKeywordUri; @@ -135,7 +135,13 @@ public static JsonElement CreateFunctionJsonSchema( schema[RequiredPropertyName] = requiredProperties; } - return JsonSerializer.SerializeToElement(schema, JsonContext.Default.JsonNode); + // Finally, apply any schema transformations if specified. + if (inferenceOptions.TransformOptions is { } options) + { + schema = TransformSchema(schema, options); + } + + return JsonSerializer.SerializeToElement(schema, JsonContextNoIndentation.Default.JsonNode); } /// Creates a JSON schema for the specified type. @@ -144,7 +150,7 @@ public static JsonElement CreateFunctionJsonSchema( /// if the parameter is optional; otherwise, . /// The default value of the optional parameter, if applicable. /// The options used to extract the schema from the specified type. - /// The options controlling schema inference. + /// The options controlling schema creation. /// A representing the schema. public static JsonElement CreateJsonSchema( Type? type, @@ -157,7 +163,14 @@ public static JsonElement CreateJsonSchema( serializerOptions ??= DefaultOptions; inferenceOptions ??= AIJsonSchemaCreateOptions.Default; JsonNode schema = CreateJsonSchemaCore(type, parameterName: null, description, hasDefaultValue, defaultValue, serializerOptions, inferenceOptions); - return JsonSerializer.SerializeToElement(schema, JsonContext.Default.JsonNode); + + // Finally, apply any schema transformations if specified. + if (inferenceOptions.TransformOptions is { } options) + { + schema = TransformSchema(schema, options); + } + + return JsonSerializer.SerializeToElement(schema, JsonContextNoIndentation.Default.JsonNode); } /// Gets the default JSON schema to be used by types or functions. @@ -202,25 +215,11 @@ private static JsonNode CreateJsonSchemaCore( if (hasDefaultValue) { - if (inferenceOptions.RequireAllProperties) - { - // Default values are only used in the context of optional parameters. - // Do not include a default keyword (since certain AI vendors don't support it) - // and instead embed its JSON in the description as a hint to the LLM. - string defaultValueJson = defaultValue is not null - ? JsonSerializer.Serialize(defaultValue, serializerOptions.GetTypeInfo(defaultValue.GetType())) - : "null"; - - description = CreateDescriptionWithDefaultValue(description, defaultValueJson); - } - else - { - JsonNode? defaultValueNode = defaultValue is not null - ? JsonSerializer.SerializeToNode(defaultValue, serializerOptions.GetTypeInfo(defaultValue.GetType())) - : null; + JsonNode? defaultValueNode = defaultValue is not null + ? JsonSerializer.SerializeToNode(defaultValue, serializerOptions.GetTypeInfo(defaultValue.GetType())) + : null; - (schemaObj ??= [])[DefaultPropertyName] = defaultValueNode; - } + (schemaObj ??= [])[DefaultPropertyName] = defaultValueNode; } if (description is not null) @@ -270,41 +269,11 @@ JsonNode TransformSchemaNode(JsonSchemaExporterContext schemaExporterContext, Js } // Include the type keyword in enum types - if (inferenceOptions.IncludeTypeInEnumSchemas && ctx.TypeInfo.Type.IsEnum && objSchema.ContainsKey(EnumPropertyName) && !objSchema.ContainsKey(TypePropertyName)) + if (ctx.TypeInfo.Type.IsEnum && objSchema.ContainsKey(EnumPropertyName) && !objSchema.ContainsKey(TypePropertyName)) { objSchema.InsertAtStart(TypePropertyName, "string"); } - // Disallow additional properties in object schemas - if (inferenceOptions.DisallowAdditionalProperties && - objSchema.ContainsKey(PropertiesPropertyName) && - !objSchema.ContainsKey(AdditionalPropertiesPropertyName)) - { - objSchema.Add(AdditionalPropertiesPropertyName, (JsonNode)false); - } - - // Mark all properties as required - if (inferenceOptions.RequireAllProperties && - objSchema.TryGetPropertyValue(PropertiesPropertyName, out JsonNode? properties) && - properties is JsonObject propertiesObj) - { - _ = objSchema.TryGetPropertyValue(RequiredPropertyName, out JsonNode? required); - if (required is not JsonArray { } requiredArray || requiredArray.Count != propertiesObj.Count) - { - requiredArray = [.. propertiesObj.Select(prop => (JsonNode)prop.Key)]; - objSchema[RequiredPropertyName] = requiredArray; - } - } - - // Strip default keywords and embed in description where required - if (inferenceOptions.RequireAllProperties && - objSchema.TryGetPropertyValue(DefaultPropertyName, out JsonNode? defaultValue)) - { - _ = objSchema.Remove(DefaultPropertyName); - string defaultValueJson = defaultValue?.ToJsonString() ?? "null"; - localDescription = CreateDescriptionWithDefaultValue(localDescription, defaultValueJson); - } - // Filter potentially disallowed keywords. foreach (string keyword in _schemaKeywordsDisallowedByAIVendors) { @@ -327,20 +296,8 @@ JsonNode TransformSchemaNode(JsonSchemaExporterContext schemaExporterContext, Js if (ctx.Path.IsEmpty && hasDefaultValue) { - // Add root-level default value metadata - if (inferenceOptions.RequireAllProperties) - { - // Default values are only used in the context of optional parameters. - // Do not include a default keyword (since certain AI vendors don't support it) - // and instead embed its JSON in the description as a hint to the LLM. - string defaultValueJson = JsonSerializer.Serialize(defaultValue, ctx.TypeInfo); - localDescription = CreateDescriptionWithDefaultValue(localDescription, defaultValueJson); - } - else - { - JsonNode? defaultValueNode = JsonSerializer.SerializeToNode(defaultValue, ctx.TypeInfo); - ConvertSchemaToObject(ref schema)[DefaultPropertyName] = defaultValueNode; - } + JsonNode? defaultValueNode = JsonSerializer.SerializeToNode(defaultValue, ctx.TypeInfo); + ConvertSchemaToObject(ref schema)[DefaultPropertyName] = defaultValueNode; } if (localDescription is not null) @@ -422,7 +379,7 @@ private static void InsertAtStart(this JsonObject jsonObject, string key, JsonNo jsonObject.Insert(0, key, value); #else jsonObject.Remove(key); - var copiedEntries = jsonObject.ToArray(); + var copiedEntries = System.Linq.Enumerable.ToArray(jsonObject); jsonObject.Clear(); jsonObject.Add(key, value); @@ -433,13 +390,6 @@ private static void InsertAtStart(this JsonObject jsonObject, string key, JsonNo #endif } - private static string CreateDescriptionWithDefaultValue(string? existingDescription, string defaultValueJson) - { - return existingDescription is null - ? $"Default value: {defaultValueJson}" - : $"{existingDescription} (Default value: {defaultValueJson})"; - } - private static JsonElement ParseJsonElement(ReadOnlySpan utf8Json) { Utf8JsonReader reader = new(utf8Json); diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Utilities/AIJsonUtilities.Schema.Transform.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Utilities/AIJsonUtilities.Schema.Transform.cs new file mode 100644 index 00000000000..865b4543abb --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Utilities/AIJsonUtilities.Schema.Transform.cs @@ -0,0 +1,205 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; +using System.Text.Json; +using System.Text.Json.Nodes; +using Microsoft.Shared.Diagnostics; + +namespace Microsoft.Extensions.AI; + +public static partial class AIJsonUtilities +{ + /// + /// Transforms the given JSON schema based on the provided options. + /// + /// The schema document to transform. + /// The options governing schema transformation. + /// A new schema document with transformations applied. + /// The schema and any nested schemas are transformed using depth-first traversal. + public static JsonElement TransformSchema(JsonElement schema, AIJsonSchemaTransformOptions transformOptions) + { + _ = Throw.IfNull(transformOptions); + + if (transformOptions == AIJsonSchemaTransformOptions.Default) + { + Throw.ArgumentException(nameof(transformOptions), "The options instance does not specify any transformations."); + } + + JsonNode? nodeSchema = JsonSerializer.SerializeToNode(schema, JsonContext.Default.JsonElement); + JsonNode transformedSchema = TransformSchema(nodeSchema, transformOptions); + return JsonSerializer.SerializeToElement(transformedSchema, JsonContextNoIndentation.Default.JsonNode); + } + + private static JsonNode TransformSchema(JsonNode? schema, AIJsonSchemaTransformOptions transformOptions) + { + List? path = transformOptions.TransformSchemaNode is not null ? [] : null; + return TransformSchemaCore(schema, transformOptions, path); + } + + private static JsonNode TransformSchemaCore(JsonNode? schema, AIJsonSchemaTransformOptions transformOptions, List? path) + { + switch (schema?.GetValueKind()) + { + case JsonValueKind.False: + if (transformOptions.ConvertBooleanSchemas) + { + schema = new JsonObject { [NotPropertyName] = (JsonNode)true }; + } + + break; + + case JsonValueKind.True: + if (transformOptions.ConvertBooleanSchemas) + { + schema = new JsonObject(); + } + + break; + + case JsonValueKind.Object: + JsonObject schemaObj = (JsonObject)schema; + JsonObject? properties = null; + + // Step 1. Recursively apply transformations to any nested schemas we might be able to detect. + if (schemaObj.TryGetPropertyValue(PropertiesPropertyName, out JsonNode? props) && props is JsonObject propsObj) + { + properties = propsObj; + path?.Add(PropertiesPropertyName); + foreach (var prop in properties.ToArray()) + { + path?.Add(prop.Key); + properties[prop.Key] = TransformSchemaCore(prop.Value, transformOptions, path); + path?.RemoveAt(path.Count - 1); + } + + path?.RemoveAt(path.Count - 1); + } + + if (schemaObj.TryGetPropertyValue(ItemsPropertyName, out JsonNode? itemsSchema)) + { + path?.Add(ItemsPropertyName); + schemaObj[ItemsPropertyName] = TransformSchemaCore(itemsSchema, transformOptions, path); + path?.RemoveAt(path.Count - 1); + } + + if (schemaObj.TryGetPropertyValue(AdditionalPropertiesPropertyName, out JsonNode? additionalProps) && + additionalProps?.GetValueKind() is not JsonValueKind.False) + { + path?.Add(AdditionalPropertiesPropertyName); + schemaObj[AdditionalPropertiesPropertyName] = TransformSchemaCore(additionalProps, transformOptions, path); + path?.RemoveAt(path.Count - 1); + } + + if (schemaObj.TryGetPropertyValue(NotPropertyName, out JsonNode? notSchema)) + { + path?.Add(NotPropertyName); + schemaObj[NotPropertyName] = TransformSchemaCore(notSchema, transformOptions, path); + path?.RemoveAt(path.Count - 1); + } + + // Traverse keywords that contain arrays of schemas + ReadOnlySpan combinatorKeywords = ["anyOf", "oneOf", "allOf"]; + foreach (string combinatorKeyword in combinatorKeywords) + { + if (schemaObj.TryGetPropertyValue(combinatorKeyword, out JsonNode? combinatorSchema) && combinatorSchema is JsonArray combinatorArray) + { + path?.Add(combinatorKeyword); + for (int i = 0; i < combinatorArray.Count; i++) + { + path?.Add($"[{i}]"); + JsonNode element = TransformSchemaCore(combinatorArray[i], transformOptions, path); + if (!ReferenceEquals(element, combinatorArray[i])) + { + combinatorArray[i] = element; + } + + path?.RemoveAt(path.Count - 1); + } + + path?.RemoveAt(path.Count - 1); + } + } + + // Step 2. Apply node-level transformations per the settings. + if (transformOptions.DisallowAdditionalProperties && properties is not null && !schemaObj.ContainsKey(AdditionalPropertiesPropertyName)) + { + schemaObj[AdditionalPropertiesPropertyName] = (JsonNode)false; + } + + if (transformOptions.RequireAllProperties && properties is not null) + { + JsonArray requiredProps = []; + foreach (var prop in properties) + { + requiredProps.Add((JsonNode)prop.Key); + } + + schemaObj[RequiredPropertyName] = requiredProps; + } + + if (transformOptions.UseNullableKeyword && + schemaObj.TryGetPropertyValue(TypePropertyName, out JsonNode? typeSchema) && + typeSchema is JsonArray typeArray) + { + bool isNullable = false; + string? foundType = null; + + foreach (JsonNode? typeNode in typeArray) + { + string typeString = (string)typeNode!; + if (typeString is "null") + { + isNullable = true; + continue; + } + + if (foundType is not null) + { + // The array contains more than one non-null types, abort the transformation. + foundType = null; + break; + } + + foundType = typeString; + } + + if (isNullable && foundType is not null) + { + schemaObj["type"] = (JsonNode)foundType; + schemaObj["nullable"] = (JsonNode)true; + } + } + + if (transformOptions.MoveDefaultKeywordToDescription && + schemaObj.TryGetPropertyValue(DefaultPropertyName, out JsonNode? defaultSchema)) + { + string? description = schemaObj.TryGetPropertyValue(DescriptionPropertyName, out JsonNode? descriptionSchema) ? descriptionSchema?.GetValue() : null; + string defaultValueJson = JsonSerializer.Serialize(defaultSchema, JsonContextNoIndentation.Default.JsonNode!); + description = description is null + ? $"Default value: {defaultValueJson}" + : $"{description} (Default value: {defaultValueJson})"; + schemaObj[DescriptionPropertyName] = description; + _ = schemaObj.Remove(DefaultPropertyName); + } + + break; + + default: + Throw.ArgumentException(nameof(schema), "Schema must be an object or a boolean value."); + break; + } + + // Apply user-defined transformations as the final step. + if (transformOptions.TransformSchemaNode is { } transformer) + { + Debug.Assert(path != null, "Path should not be null when TransformSchemaNode is provided."); + schema = transformer(new AIJsonSchemaTransformContext(path!.ToArray()), schema); + } + + return schema; + } +} diff --git a/src/Libraries/Microsoft.Extensions.AI.AzureAIInference/AzureAIInferenceChatClient.cs b/src/Libraries/Microsoft.Extensions.AI.AzureAIInference/AzureAIInferenceChatClient.cs index ea66cc191e4..6c0acb8ee23 100644 --- a/src/Libraries/Microsoft.Extensions.AI.AzureAIInference/AzureAIInferenceChatClient.cs +++ b/src/Libraries/Microsoft.Extensions.AI.AzureAIInference/AzureAIInferenceChatClient.cs @@ -24,6 +24,15 @@ namespace Microsoft.Extensions.AI; /// Represents an for an Azure AI Inference . internal sealed class AzureAIInferenceChatClient : IChatClient { + /// Gets the JSON schema transform cache conforming to OpenAI restrictions per https://platform.openai.com/docs/guides/structured-outputs?api-mode=responses#supported-schemas. + private static AIJsonSchemaTransformCache SchemaTransformCache { get; } = new(new() + { + RequireAllProperties = true, + DisallowAdditionalProperties = true, + ConvertBooleanSchemas = true, + MoveDefaultKeywordToDescription = true, + }); + /// Metadata about the client. private readonly ChatClientMetadata _metadata; @@ -273,66 +282,74 @@ private static ChatRole ToChatRole(global::Azure.AI.Inference.ChatRole role) => finishReason == CompletionsFinishReason.ToolCalls ? ChatFinishReason.ToolCalls : new(s); + private ChatCompletionsOptions CreateAzureAIOptions(IEnumerable chatContents, ChatOptions? options) => + new(ToAzureAIInferenceChatMessages(chatContents)) + { + Model = options?.ModelId ?? _metadata.DefaultModelId ?? + throw new InvalidOperationException("No model id was provided when either constructing the client or in the chat options.") + }; + /// Converts an extensions options instance to an AzureAI options instance. private ChatCompletionsOptions ToAzureAIOptions(IEnumerable chatContents, ChatOptions? options) { - ChatCompletionsOptions result = new(ToAzureAIInferenceChatMessages(chatContents)) + if (options is null) { - Model = options?.ModelId ?? _metadata.DefaultModelId ?? throw new InvalidOperationException("No model id was provided when either constructing the client or in the chat options.") - }; + return CreateAzureAIOptions(chatContents, options); + } - if (options is not null) + if (options.RawRepresentationFactory?.Invoke(this) is ChatCompletionsOptions result) + { + result.Messages = ToAzureAIInferenceChatMessages(chatContents).ToList(); + result.Model ??= options.ModelId ?? _metadata.DefaultModelId ?? + throw new InvalidOperationException("No model id was provided when either constructing the client or in the chat options."); + } + else { - result.FrequencyPenalty = options.FrequencyPenalty; - result.MaxTokens = options.MaxOutputTokens; - result.NucleusSamplingFactor = options.TopP; - result.PresencePenalty = options.PresencePenalty; - result.Temperature = options.Temperature; - result.Seed = options.Seed; - - if (options.StopSequences is { Count: > 0 } stopSequences) + result = CreateAzureAIOptions(chatContents, options); + } + + result.FrequencyPenalty ??= options.FrequencyPenalty; + result.MaxTokens ??= options.MaxOutputTokens; + result.NucleusSamplingFactor ??= options.TopP; + result.PresencePenalty ??= options.PresencePenalty; + result.Temperature ??= options.Temperature; + result.Seed ??= options.Seed; + + if (options.StopSequences is { Count: > 0 } stopSequences) + { + foreach (string stopSequence in stopSequences) { - foreach (string stopSequence in stopSequences) - { - result.StopSequences.Add(stopSequence); - } + result.StopSequences.Add(stopSequence); } + } + + // This property is strongly typed on ChatOptions but not on ChatCompletionsOptions. + if (options.TopK is int topK && !result.AdditionalProperties.ContainsKey("top_k")) + { + result.AdditionalProperties["top_k"] = new BinaryData(JsonSerializer.SerializeToUtf8Bytes(topK, AIJsonUtilities.DefaultOptions.GetTypeInfo(typeof(int)))); + } - // These properties are strongly typed on ChatOptions but not on ChatCompletionsOptions. - if (options.TopK is int topK) + if (options.AdditionalProperties is { } props) + { + foreach (var prop in props) { - result.AdditionalProperties["top_k"] = new BinaryData(JsonSerializer.SerializeToUtf8Bytes(topK, AIJsonUtilities.DefaultOptions.GetTypeInfo(typeof(int)))); + byte[] data = JsonSerializer.SerializeToUtf8Bytes(prop.Value, AIJsonUtilities.DefaultOptions.GetTypeInfo(typeof(object))); + result.AdditionalProperties[prop.Key] = new BinaryData(data); } + } - if (options.AdditionalProperties is { } props) + if (options.Tools is { Count: > 0 } tools) + { + foreach (AITool tool in tools) { - foreach (var prop in props) + if (tool is AIFunction af) { - switch (prop.Key) - { - // Propagate everything else to the ChatCompletionsOptions' AdditionalProperties. - default: - if (prop.Value is not null) - { - byte[] data = JsonSerializer.SerializeToUtf8Bytes(prop.Value, AIJsonUtilities.DefaultOptions.GetTypeInfo(typeof(object))); - result.AdditionalProperties[prop.Key] = new BinaryData(data); - } - - break; - } + result.Tools.Add(ToAzureAIChatTool(af)); } } - if (options.Tools is { Count: > 0 } tools) + if (result.ToolChoice is null && result.Tools.Count > 0) { - foreach (AITool tool in tools) - { - if (tool is AIFunction af) - { - result.Tools.Add(ToAzureAIChatTool(af)); - } - } - switch (options.ToolMode) { case NoneChatToolMode: @@ -351,14 +368,17 @@ private ChatCompletionsOptions ToAzureAIOptions(IEnumerable chatCon break; } } + } + if (result.ResponseFormat is null) + { if (options.ResponseFormat is ChatResponseFormatText) { result.ResponseFormat = ChatCompletionsResponseFormat.CreateTextFormat(); } else if (options.ResponseFormat is ChatResponseFormatJson json) { - if (json.Schema is { } schema) + if (SchemaTransformCache.GetOrCreateTransformedSchema(json) is { } schema) { var tool = JsonSerializer.Deserialize(schema, JsonContext.Default.AzureAIChatToolJson)!; result.ResponseFormat = ChatCompletionsResponseFormat.CreateJsonFormat( @@ -392,7 +412,7 @@ private ChatCompletionsOptions ToAzureAIOptions(IEnumerable chatCon private static ChatCompletionsToolDefinition ToAzureAIChatTool(AIFunction aiFunction) { // Map to an intermediate model so that redundant properties are skipped. - var tool = JsonSerializer.Deserialize(aiFunction.JsonSchema, JsonContext.Default.AzureAIChatToolJson)!; + var tool = JsonSerializer.Deserialize(SchemaTransformCache.GetOrCreateTransformedSchema(aiFunction), JsonContext.Default.AzureAIChatToolJson)!; var functionParameters = BinaryData.FromBytes(JsonSerializer.SerializeToUtf8Bytes(tool, JsonContext.Default.AzureAIChatToolJson)); return new(new FunctionDefinition(aiFunction.Name) { diff --git a/src/Libraries/Microsoft.Extensions.AI.AzureAIInference/CHANGELOG.md b/src/Libraries/Microsoft.Extensions.AI.AzureAIInference/CHANGELOG.md index aaf1ac1c67c..aeb023efae5 100644 --- a/src/Libraries/Microsoft.Extensions.AI.AzureAIInference/CHANGELOG.md +++ b/src/Libraries/Microsoft.Extensions.AI.AzureAIInference/CHANGELOG.md @@ -1,5 +1,10 @@ # Release History +## 9.4.4-preview.1.25259.16 + +- Added an `AsIEmbeddingGenerator` extension method for `ImageEmbeddingsClient`. +- Updated to accomodate the changes in `Microsoft.Extensions.AI.Abstractions`. + ## 9.4.3-preview.1.25230.7 - Updated to accomodate the changes in `Microsoft.Extensions.AI.Abstractions`. diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Console/Commands/CleanCacheCommand.cs b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Console/Commands/CleanCacheCommand.cs index 08f035d55eb..b0d975edb43 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Console/Commands/CleanCacheCommand.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Console/Commands/CleanCacheCommand.cs @@ -18,7 +18,7 @@ internal sealed class CleanCacheCommand(ILogger logger) { internal async Task InvokeAsync(DirectoryInfo? storageRootDir, Uri? endpointUri, CancellationToken cancellationToken = default) { - IResponseCacheProvider cacheProvider; + IEvaluationResponseCacheProvider cacheProvider; if (storageRootDir is not null) { diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Console/Commands/CleanResultsCommand.cs b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Console/Commands/CleanResultsCommand.cs index 59635dc0530..8d6617d8302 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Console/Commands/CleanResultsCommand.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Console/Commands/CleanResultsCommand.cs @@ -23,7 +23,7 @@ internal async Task InvokeAsync( int lastN, CancellationToken cancellationToken = default) { - IResultStore resultStore; + IEvaluationResultStore resultStore; if (storageRootDir is not null) { diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Console/Commands/ReportCommand.cs b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Console/Commands/ReportCommand.cs index f2466923bd8..2611695e542 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Console/Commands/ReportCommand.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Console/Commands/ReportCommand.cs @@ -28,7 +28,7 @@ internal async Task InvokeAsync( Format format, CancellationToken cancellationToken = default) { - IResultStore resultStore; + IEvaluationResultStore resultStore; if (storageRootDir is not null) { diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Console/Microsoft.Extensions.AI.Evaluation.Console.csproj b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Console/Microsoft.Extensions.AI.Evaluation.Console.csproj index 91b398ce281..a1d9252e8e0 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Console/Microsoft.Extensions.AI.Evaluation.Console.csproj +++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Console/Microsoft.Extensions.AI.Evaluation.Console.csproj @@ -3,9 +3,7 @@ A command line dotnet tool for generating reports and managing evaluation data. Exe - - $(MinimumSupportedTfmForPackaging) + $(NetCoreTargetFrameworks) Microsoft.Extensions.AI.Evaluation.Console $(NoWarn);EA0000 @@ -22,6 +20,15 @@ 0 + + + false + + diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Azure/JsonSerialization/AzureStorageCamelCaseEnumConverter.cs b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Azure/JsonSerialization/AzureStorageCamelCaseEnumConverter.cs deleted file mode 100644 index 2ec6cdb801f..00000000000 --- a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Azure/JsonSerialization/AzureStorageCamelCaseEnumConverter.cs +++ /dev/null @@ -1,11 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System.Text.Json; -using System.Text.Json.Serialization; - -namespace Microsoft.Extensions.AI.Evaluation.Reporting.JsonSerialization; - -internal sealed class AzureStorageCamelCaseEnumConverter() : - JsonStringEnumConverter(JsonNamingPolicy.CamelCase) - where TEnum : struct, System.Enum; diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Azure/JsonSerialization/AzureStorageJsonUtilities.cs b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Azure/JsonSerialization/AzureStorageJsonUtilities.cs index 23b2ae9c88c..b36c8d8bd56 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Azure/JsonSerialization/AzureStorageJsonUtilities.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Azure/JsonSerialization/AzureStorageJsonUtilities.cs @@ -1,6 +1,7 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. +using System.Diagnostics.CodeAnalysis; using System.Text.Encodings.Web; using System.Text.Json; using System.Text.Json.Serialization; @@ -11,7 +12,7 @@ namespace Microsoft.Extensions.AI.Evaluation.Reporting.JsonSerialization; internal static partial class AzureStorageJsonUtilities { - [System.Diagnostics.CodeAnalysis.SuppressMessage("Naming", "CA1716:Identifiers should not match keywords", Justification = "Default matches the generated source naming convention.")] + [SuppressMessage("Naming", "CA1716:Identifiers should not match keywords", Justification = "Default matches the generated source naming convention.")] internal static class Default { private static JsonSerializerOptions? _options; @@ -24,6 +25,7 @@ internal static class Compact { private static JsonSerializerOptions? _options; internal static JsonSerializerOptions Options => _options ??= CreateJsonSerializerOptions(writeIndented: false); + internal static JsonTypeInfo CacheEntryTypeInfo => Options.GetTypeInfo(); internal static JsonTypeInfo ScenarioRunResultTypeInfo => Options.GetTypeInfo(); } @@ -45,14 +47,14 @@ private static JsonSerializerOptions CreateJsonSerializerOptions(bool writeInden [JsonSerializable(typeof(CacheEntry))] [JsonSourceGenerationOptions( Converters = [ - typeof(AzureStorageCamelCaseEnumConverter), - typeof(AzureStorageCamelCaseEnumConverter), - typeof(AzureStorageTimeSpanConverter) + typeof(CamelCaseEnumConverter), + typeof(CamelCaseEnumConverter), + typeof(TimeSpanConverter), + typeof(EvaluationContextConverter) ], WriteIndented = true, IgnoreReadOnlyProperties = false, DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull, PropertyNamingPolicy = JsonKnownNamingPolicy.CamelCase)] private sealed partial class JsonContext : JsonSerializerContext; - } diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Azure/JsonSerialization/AzureStorageTimeSpanConverter.cs b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Azure/JsonSerialization/AzureStorageTimeSpanConverter.cs deleted file mode 100644 index 0c064ededd3..00000000000 --- a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Azure/JsonSerialization/AzureStorageTimeSpanConverter.cs +++ /dev/null @@ -1,17 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System; -using System.Text.Json; -using System.Text.Json.Serialization; - -namespace Microsoft.Extensions.AI.Evaluation.Reporting.JsonSerialization; - -internal sealed class AzureStorageTimeSpanConverter : JsonConverter -{ - public override TimeSpan Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) - => TimeSpan.FromSeconds(reader.GetDouble()); - - public override void Write(Utf8JsonWriter writer, TimeSpan value, JsonSerializerOptions options) - => writer.WriteNumberValue(value.TotalSeconds); -} diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Azure/Microsoft.Extensions.AI.Evaluation.Reporting.Azure.csproj b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Azure/Microsoft.Extensions.AI.Evaluation.Reporting.Azure.csproj index 669bbab7556..237df014d0d 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Azure/Microsoft.Extensions.AI.Evaluation.Reporting.Azure.csproj +++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Azure/Microsoft.Extensions.AI.Evaluation.Reporting.Azure.csproj @@ -17,6 +17,12 @@ 0 + + + + + + diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Azure/Storage/AzureStorageReportingConfiguration.cs b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Azure/Storage/AzureStorageReportingConfiguration.cs index 9302107b926..fafd8639b34 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Azure/Storage/AzureStorageReportingConfiguration.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Azure/Storage/AzureStorageReportingConfiguration.cs @@ -24,10 +24,6 @@ public static class AzureStorageReportingConfiguration /// /// The set of s that should be invoked to evaluate AI responses. /// - /// - /// An optional that specifies the maximum amount of time that cached AI responses should - /// survive in the cache before they are considered expired and evicted. - /// /// /// A that specifies the that is used by AI-based /// included in the returned . Can be omitted if @@ -36,6 +32,10 @@ public static class AzureStorageReportingConfiguration /// /// to enable caching of AI responses; otherwise. /// + /// + /// An optional that specifies the maximum amount of time that cached AI responses should + /// survive in the cache before they are considered expired and evicted. + /// /// /// An optional collection of unique strings that should be hashed when generating the cache keys for cached AI /// responses. See for more information about this concept. @@ -63,21 +63,21 @@ public static class AzureStorageReportingConfiguration public static ReportingConfiguration Create( DataLakeDirectoryClient client, IEnumerable evaluators, - TimeSpan? timeToLiveForCacheEntries = null, ChatConfiguration? chatConfiguration = null, bool enableResponseCaching = true, + TimeSpan? timeToLiveForCacheEntries = null, IEnumerable? cachingKeys = null, string executionName = Defaults.DefaultExecutionName, Func? evaluationMetricInterpreter = null, IEnumerable? tags = null) #pragma warning restore S107 { - IResponseCacheProvider? responseCacheProvider = + IEvaluationResponseCacheProvider? responseCacheProvider = chatConfiguration is not null && enableResponseCaching ? new AzureStorageResponseCacheProvider(client, timeToLiveForCacheEntries) : null; - IResultStore resultStore = new AzureStorageResultStore(client); + IEvaluationResultStore resultStore = new AzureStorageResultStore(client); return new ReportingConfiguration( evaluators, diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Azure/Storage/AzureStorageResponseCache.cs b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Azure/Storage/AzureStorageResponseCache.cs index f7115a37024..5e83b456a0b 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Azure/Storage/AzureStorageResponseCache.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Azure/Storage/AzureStorageResponseCache.cs @@ -40,9 +40,9 @@ internal sealed partial class AzureStorageResponseCache( private const string EntryAndContentsFilesNotFound = "Cache entry file {0} and contents file {1} were not found."; private readonly string _iterationPath = $"cache/{scenarioName}/{iterationName}"; + private readonly Func _provideDateTime = provideDateTime; private readonly TimeSpan _timeToLiveForCacheEntries = timeToLiveForCacheEntries ?? Defaults.DefaultTimeToLiveForCacheEntries; - private readonly Func _provideDateTime = provideDateTime; public byte[]? Get(string key) { diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Azure/Storage/AzureStorageResponseCacheProvider.cs b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Azure/Storage/AzureStorageResponseCacheProvider.cs index a890fa80332..6c6d1431a1a 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Azure/Storage/AzureStorageResponseCacheProvider.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Azure/Storage/AzureStorageResponseCacheProvider.cs @@ -15,8 +15,8 @@ namespace Microsoft.Extensions.AI.Evaluation.Reporting.Storage; /// -/// An that returns an that can cache AI responses -/// for a particular under an Azure Storage container. +/// An that returns an that can cache AI +/// responses for a particular under an Azure Storage container. /// /// /// A with access to an Azure Storage container under which the cached AI @@ -28,7 +28,7 @@ namespace Microsoft.Extensions.AI.Evaluation.Reporting.Storage; /// public sealed class AzureStorageResponseCacheProvider( DataLakeDirectoryClient client, - TimeSpan? timeToLiveForCacheEntries = null) : IResponseCacheProvider + TimeSpan? timeToLiveForCacheEntries = null) : IEvaluationResponseCacheProvider { private readonly Func _provideDateTime = () => DateTime.Now; diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Azure/Storage/AzureStorageResultStore.cs b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Azure/Storage/AzureStorageResultStore.cs index 70d988abe74..71682f13651 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Azure/Storage/AzureStorageResultStore.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Azure/Storage/AzureStorageResultStore.cs @@ -20,14 +20,14 @@ namespace Microsoft.Extensions.AI.Evaluation.Reporting.Storage; /// -/// An implementation that stores s under an Azure Storage -/// container. +/// An implementation that stores s under an Azure +/// Storage container. /// /// /// A with access to an Azure Storage container under which the /// s should be stored. /// -public sealed class AzureStorageResultStore(DataLakeDirectoryClient client) : IResultStore +public sealed class AzureStorageResultStore(DataLakeDirectoryClient client) : IEvaluationResultStore { private const string ResultsRootPrefix = "results"; diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/ChatDetailsExtensions.cs b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/ChatDetailsExtensions.cs index 006dfc741e8..71afe53217a 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/ChatDetailsExtensions.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/ChatDetailsExtensions.cs @@ -1,6 +1,7 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. +using System.Collections.Generic; using Microsoft.Shared.Diagnostics; namespace Microsoft.Extensions.AI.Evaluation.Reporting; @@ -11,19 +12,36 @@ namespace Microsoft.Extensions.AI.Evaluation.Reporting; public static class ChatDetailsExtensions { /// - /// Adds for a particular LLM chat conversation turn to the + /// Adds for one or more LLM chat conversation turns to the /// collection. /// /// - /// The object to which the is to be added. + /// The object to which the are to be added. /// /// - /// The for a particular LLM chat conversation turn. + /// The for one or more LLM chat conversation turns. /// - public static void AddTurnDetails(this ChatDetails chatDetails, ChatTurnDetails turnDetails) + public static void AddTurnDetails(this ChatDetails chatDetails, IEnumerable turnDetails) { _ = Throw.IfNull(chatDetails); + _ = Throw.IfNull(turnDetails); - chatDetails.TurnDetails.Add(turnDetails); + foreach (ChatTurnDetails t in turnDetails) + { + chatDetails.TurnDetails.Add(t); + } } + + /// + /// Adds for one or more LLM chat conversation turns to the + /// collection. + /// + /// + /// The object to which the are to be added. + /// + /// + /// The for one or more LLM chat conversation turns. + /// + public static void AddTurnDetails(this ChatDetails chatDetails, params ChatTurnDetails[] turnDetails) + => chatDetails.AddTurnDetails(turnDetails as IEnumerable); } diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/Defaults.cs b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/Defaults.cs index e5cd6b26ec3..f25fa074430 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/Defaults.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/Defaults.cs @@ -27,13 +27,11 @@ public static class Defaults /// /// Gets a that specifies the default amount of time that cached AI responses should survive - /// in the 's cache before they are considered expired and evicted. + /// in the 's cache before they are considered expired and evicted. /// public static TimeSpan DefaultTimeToLiveForCacheEntries { get; } = TimeSpan.FromDays(14); - /// - /// Defines the version number for the reporting format. If and when the serialized format undergoes - /// breaking changes, this number will be incremented. - /// + // Defines the version number for the reporting format. If and when the serialized format undergoes + // breaking changes, this number should be incremented. internal const int ReportingFormatVersion = 1; } diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/IResponseCacheProvider.cs b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/IEvaluationResponseCacheProvider.cs similarity index 58% rename from src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/IResponseCacheProvider.cs rename to src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/IEvaluationResponseCacheProvider.cs index 6bc8ce25432..1859124a98f 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/IResponseCacheProvider.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/IEvaluationResponseCacheProvider.cs @@ -12,26 +12,28 @@ namespace Microsoft.Extensions.AI.Evaluation.Reporting; /// . /// /// -/// can be used to set up caching of AI-generated responses (both the AI responses -/// under evaluation as well as the AI responses for the evaluations themselves). When caching is enabled, the AI -/// responses associated with each are stored in the that is -/// returned from this . So long as the inputs (such as the content included in the -/// requests, the AI model being invoked etc.) remain unchanged, subsequent evaluations of the same -/// use the cached responses instead of invoking the AI model to generate new ones. Bypassing -/// the AI model when the inputs remain unchanged results in faster execution at a lower cost. +/// can be used to set up caching of AI-generated responses (both the AI +/// responses under evaluation as well as the AI responses for the evaluations themselves). When caching is enabled, +/// the AI responses associated with each are stored in the +/// that is returned from this . So long as the inputs (such as the +/// content included in the requests, the AI model being invoked etc.) remain unchanged, subsequent evaluations of the +/// same use the cached responses instead of invoking the AI model to generate new ones. +/// Bypassing the AI model when the inputs remain unchanged results in faster execution at a lower cost. /// -public interface IResponseCacheProvider +public interface IEvaluationResponseCacheProvider { /// - /// Returns an that caches the AI responses associated with a particular - /// . + /// Returns an that caches all the AI responses associated with the + /// with the supplied and + /// . /// /// The . /// The . /// A that can cancel the operation. /// - /// An that caches the AI responses associated with a particular - /// . + /// An that caches all the AI responses associated with the + /// with the supplied and + /// . /// ValueTask GetCacheAsync( string scenarioName, diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/IResultStore.cs b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/IEvaluationResultStore.cs similarity index 99% rename from src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/IResultStore.cs rename to src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/IEvaluationResultStore.cs index 3f3dea6cc7a..202a6305cd3 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/IResultStore.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/IEvaluationResultStore.cs @@ -10,7 +10,7 @@ namespace Microsoft.Extensions.AI.Evaluation.Reporting; /// /// Represents a store for s. /// -public interface IResultStore +public interface IEvaluationResultStore { /// /// Returns s for s filtered by the specified diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/JsonSerialization/JsonUtilities.cs b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/JsonSerialization/JsonUtilities.cs index e372b7e8434..3a8c2af1ce2 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/JsonSerialization/JsonUtilities.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/JsonSerialization/JsonUtilities.cs @@ -1,6 +1,7 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. +using System.Diagnostics.CodeAnalysis; using System.Text.Encodings.Web; using System.Text.Json; using System.Text.Json.Serialization; @@ -12,14 +13,13 @@ namespace Microsoft.Extensions.AI.Evaluation.Reporting.JsonSerialization; internal static partial class JsonUtilities { - [System.Diagnostics.CodeAnalysis.SuppressMessage("Naming", "CA1716:Identifiers should not match keywords", Justification = "Default matches the generated source naming convention.")] + [SuppressMessage("Naming", "CA1716:Identifiers should not match keywords", Justification = "Default matches the generated source naming convention.")] internal static class Default { private static JsonSerializerOptions? _options; internal static JsonSerializerOptions Options => _options ??= CreateJsonSerializerOptions(writeIndented: true); internal static JsonTypeInfo DatasetTypeInfo => Options.GetTypeInfo(); internal static JsonTypeInfo CacheEntryTypeInfo => Options.GetTypeInfo(); - internal static JsonTypeInfo CacheOptionsTypeInfo => Options.GetTypeInfo(); internal static JsonTypeInfo ScenarioRunResultTypeInfo => Options.GetTypeInfo(); } @@ -29,7 +29,6 @@ internal static class Compact internal static JsonSerializerOptions Options => _options ??= CreateJsonSerializerOptions(writeIndented: false); internal static JsonTypeInfo DatasetTypeInfo => Options.GetTypeInfo(); internal static JsonTypeInfo CacheEntryTypeInfo => Options.GetTypeInfo(); - internal static JsonTypeInfo CacheOptionsTypeInfo => Options.GetTypeInfo(); internal static JsonTypeInfo ScenarioRunResultTypeInfo => Options.GetTypeInfo(); } @@ -47,15 +46,13 @@ private static JsonSerializerOptions CreateJsonSerializerOptions(bool writeInden return options; } - [JsonSerializable(typeof(EvaluationResult))] + [JsonSerializable(typeof(ScenarioRunResult))] [JsonSerializable(typeof(Dataset))] [JsonSerializable(typeof(CacheEntry))] - [JsonSerializable(typeof(CacheOptions))] [JsonSourceGenerationOptions( Converters = [ typeof(CamelCaseEnumConverter), typeof(CamelCaseEnumConverter), - typeof(CamelCaseEnumConverter), typeof(TimeSpanConverter), typeof(EvaluationContextConverter) ], diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/ReportingConfiguration.cs b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/ReportingConfiguration.cs index 68a73338d88..130586de930 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/ReportingConfiguration.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/ReportingConfiguration.cs @@ -27,9 +27,10 @@ public sealed class ReportingConfiguration public IReadOnlyList Evaluators { get; } /// - /// Gets the that should be used to persist the s. + /// Gets the that should be used to persist the + /// s. /// - public IResultStore ResultStore { get; } + public IEvaluationResultStore ResultStore { get; } /// /// Gets a that specifies the that is used by @@ -38,9 +39,9 @@ public sealed class ReportingConfiguration public ChatConfiguration? ChatConfiguration { get; } /// - /// Gets the that should be used to cache AI responses. + /// Gets the that should be used to cache AI responses. /// - public IResponseCacheProvider? ResponseCacheProvider { get; } + public IEvaluationResponseCacheProvider? ResponseCacheProvider { get; } /// /// Gets the collection of unique strings that should be hashed when generating the cache keys for cached AI @@ -101,7 +102,7 @@ public sealed class ReportingConfiguration /// The set of s that should be invoked to evaluate AI responses. /// /// - /// The that should be used to persist the s. + /// The that should be used to persist the s. /// /// /// A that specifies the that is used by @@ -109,8 +110,8 @@ public sealed class ReportingConfiguration /// none of the included are AI-based. /// /// - /// The that should be used to cache AI responses. If omitted, AI responses - /// will not be cached. + /// The that should be used to cache AI responses. If omitted, AI + /// responses will not be cached. /// /// /// An optional collection of unique strings that should be hashed when generating the cache keys for cached AI @@ -134,9 +135,9 @@ public sealed class ReportingConfiguration #pragma warning disable S107 // Methods should not have too many parameters public ReportingConfiguration( IEnumerable evaluators, - IResultStore resultStore, + IEvaluationResultStore resultStore, ChatConfiguration? chatConfiguration = null, - IResponseCacheProvider? responseCacheProvider = null, + IEvaluationResponseCacheProvider? responseCacheProvider = null, IEnumerable? cachingKeys = null, string executionName = Defaults.DefaultExecutionName, Func? evaluationMetricInterpreter = null, diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/ScenarioRun.cs b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/ScenarioRun.cs index eb58685bf0c..5fa46e7e4ec 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/ScenarioRun.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/ScenarioRun.cs @@ -93,7 +93,7 @@ public sealed class ScenarioRun : IAsyncDisposable public ChatConfiguration? ChatConfiguration { get; } private readonly CompositeEvaluator _compositeEvaluator; - private readonly IResultStore _resultStore; + private readonly IEvaluationResultStore _resultStore; private readonly Func? _evaluationMetricInterpreter; private readonly ChatDetails? _chatDetails; private readonly IEnumerable? _tags; @@ -106,7 +106,7 @@ internal ScenarioRun( string iterationName, string executionName, IEnumerable evaluators, - IResultStore resultStore, + IEvaluationResultStore resultStore, ChatConfiguration? chatConfiguration = null, Func? evaluationMetricInterpreter = null, ChatDetails? chatDetails = null, @@ -189,7 +189,7 @@ await _compositeEvaluator.EvaluateAsync( /// /// Disposes the and writes the to the configured - /// . + /// . /// /// A that represents the asynchronous operation. public async ValueTask DisposeAsync() diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/ScenarioRunExtensions.cs b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/ScenarioRunExtensions.cs index 3b723a2d258..08822f18d02 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/ScenarioRunExtensions.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/ScenarioRunExtensions.cs @@ -85,16 +85,11 @@ public static ValueTask EvaluateAsync( this ScenarioRun scenarioRun, ChatMessage modelResponse, IEnumerable? additionalContext = null, - CancellationToken cancellationToken = default) - { - _ = Throw.IfNull(scenarioRun); - - return scenarioRun.EvaluateAsync( - messages: [], + CancellationToken cancellationToken = default) => + scenarioRun.EvaluateAsync( new ChatResponse(modelResponse), additionalContext, cancellationToken); - } /// /// Evaluates the supplied and returns an @@ -148,16 +143,12 @@ public static ValueTask EvaluateAsync( ChatMessage userRequest, ChatMessage modelResponse, IEnumerable? additionalContext = null, - CancellationToken cancellationToken = default) - { - _ = Throw.IfNull(scenarioRun); - - return scenarioRun.EvaluateAsync( - messages: [userRequest], + CancellationToken cancellationToken = default) => + scenarioRun.EvaluateAsync( + userRequest, new ChatResponse(modelResponse), additionalContext, cancellationToken); - } /// /// Evaluates the supplied and returns an diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/Storage/DiskBasedReportingConfiguration.cs b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/Storage/DiskBasedReportingConfiguration.cs index 94ab92e177b..e967fdd1db9 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/Storage/DiskBasedReportingConfiguration.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/Storage/DiskBasedReportingConfiguration.cs @@ -32,6 +32,10 @@ public static class DiskBasedReportingConfiguration /// /// to enable caching of AI responses; otherwise. /// + /// + /// An optional that specifies the maximum amount of time that cached AI responses should + /// survive in the cache before they are considered expired and evicted. + /// /// /// An optional collection of unique strings that should be hashed when generating the cache keys for cached AI /// responses. See for more information about this concept. @@ -61,6 +65,7 @@ public static ReportingConfiguration Create( IEnumerable evaluators, ChatConfiguration? chatConfiguration = null, bool enableResponseCaching = true, + TimeSpan? timeToLiveForCacheEntries = null, IEnumerable? cachingKeys = null, string executionName = Defaults.DefaultExecutionName, Func? evaluationMetricInterpreter = null, @@ -69,12 +74,12 @@ public static ReportingConfiguration Create( { storageRootPath = Path.GetFullPath(storageRootPath); - IResponseCacheProvider? responseCacheProvider = + IEvaluationResponseCacheProvider? responseCacheProvider = chatConfiguration is not null && enableResponseCaching - ? new DiskBasedResponseCacheProvider(storageRootPath) + ? new DiskBasedResponseCacheProvider(storageRootPath, timeToLiveForCacheEntries) : null; - IResultStore resultStore = new DiskBasedResultStore(storageRootPath); + IEvaluationResultStore resultStore = new DiskBasedResultStore(storageRootPath); return new ReportingConfiguration( evaluators, diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/Storage/DiskBasedResponseCache.CacheMode.cs b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/Storage/DiskBasedResponseCache.CacheMode.cs deleted file mode 100644 index cbc8ca2f151..00000000000 --- a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/Storage/DiskBasedResponseCache.CacheMode.cs +++ /dev/null @@ -1,31 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -namespace Microsoft.Extensions.AI.Evaluation.Reporting.Storage; - -internal partial class DiskBasedResponseCache -{ - /// - /// An enum representing the mode in which the cache is operating. - /// - internal enum CacheMode - { - /// - /// In this mode, the cache is disabled. All requests bypass the cache and are forwarded online. - /// - Disabled, - - /// - /// In this mode, the cache is enabled. Requests are handled by the cache first. If a cached response is not - /// available, then the request is forwarded online. - /// - Enabled, - - /// - /// In this mode, the cache is enabled. However, requests are never forwarded online. Instead if a cached response - /// is not available, then an exception is thrown. Additionally in this mode, the cache is considered frozen (or - /// read only) which means that all the cache artifacts (including expired entries) are preserved as is on disk. - /// - EnabledOfflineOnly - } -} diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/Storage/DiskBasedResponseCache.CacheOptions.cs b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/Storage/DiskBasedResponseCache.CacheOptions.cs deleted file mode 100644 index 1e21c59828b..00000000000 --- a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/Storage/DiskBasedResponseCache.CacheOptions.cs +++ /dev/null @@ -1,90 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System; -using System.Globalization; -using System.IO; -using System.Text.Json; -using System.Text.Json.Serialization; -using System.Threading; -using System.Threading.Tasks; -using Microsoft.Extensions.AI.Evaluation.Reporting.JsonSerialization; - -namespace Microsoft.Extensions.AI.Evaluation.Reporting.Storage; - -internal partial class DiskBasedResponseCache -{ - internal sealed class CacheOptions - { - public static CacheOptions Default { get; } = new CacheOptions(); - - private const string DeserializationFailedMessage = "Unable to deserialize the cache options file at {0}."; - - public CacheOptions(CacheMode mode = CacheMode.Enabled, TimeSpan? timeToLiveForCacheEntries = null) - { - Mode = mode; - TimeToLiveForCacheEntries = timeToLiveForCacheEntries ?? Defaults.DefaultTimeToLiveForCacheEntries; - } - - [JsonConstructor] - public CacheOptions(CacheMode mode, TimeSpan timeToLiveForCacheEntries) - { - Mode = mode; - TimeToLiveForCacheEntries = timeToLiveForCacheEntries; - } - - public CacheMode Mode { get; } - - [JsonPropertyName("timeToLiveInSecondsForCacheEntries")] - public TimeSpan TimeToLiveForCacheEntries { get; } - - public static CacheOptions Read(string cacheOptionsFilePath) - { - using FileStream cacheOptionsFile = File.OpenRead(cacheOptionsFilePath); - - CacheOptions cacheOptions = - JsonSerializer.Deserialize( - cacheOptionsFile, - JsonUtilities.Default.CacheOptionsTypeInfo) ?? - throw new JsonException( - string.Format(CultureInfo.CurrentCulture, DeserializationFailedMessage, cacheOptionsFilePath)); - - return cacheOptions; - } - - public static async Task ReadAsync( - string cacheOptionsFilePath, - CancellationToken cancellationToken = default) - { - using FileStream cacheOptionsFile = File.OpenRead(cacheOptionsFilePath); - - CacheOptions cacheOptions = - await JsonSerializer.DeserializeAsync( - cacheOptionsFile, - JsonUtilities.Default.CacheOptionsTypeInfo, - cancellationToken).ConfigureAwait(false) ?? - throw new JsonException( - string.Format(CultureInfo.CurrentCulture, DeserializationFailedMessage, cacheOptionsFilePath)); - - return cacheOptions; - } - - public void Write(string cacheOptionsFilePath) - { - using FileStream cacheOptionsFile = File.Create(cacheOptionsFilePath); - JsonSerializer.Serialize(cacheOptionsFile, this, JsonUtilities.Default.CacheOptionsTypeInfo); - } - - public async Task WriteAsync( - string cacheOptionsFilePath, - CancellationToken cancellationToken = default) - { - using FileStream cacheOptionsFile = File.Create(cacheOptionsFilePath); - await JsonSerializer.SerializeAsync( - cacheOptionsFile, - this, - JsonUtilities.Default.CacheOptionsTypeInfo, - cancellationToken).ConfigureAwait(false); - } - } -} diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/Storage/DiskBasedResponseCache.cs b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/Storage/DiskBasedResponseCache.cs index 51b96739964..d0a107d8710 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/Storage/DiskBasedResponseCache.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/Storage/DiskBasedResponseCache.cs @@ -1,6 +1,11 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. +#pragma warning disable S3604 +// S3604: Member initializer values should not be redundant. +// We disable this warning because it is a false positive arising from the analyzer's lack of support for C#'s primary +// constructor syntax. + #pragma warning disable CA1725 // CA1725: Parameter names should match base declaration. // All functions on 'IDistributedCache' use the parameter name 'token' in place of 'cancellationToken'. However, @@ -26,55 +31,42 @@ internal sealed partial class DiskBasedResponseCache : IDistributedCache private readonly string _scenarioName; private readonly string _iterationName; - private readonly CacheOptions _options; private readonly string _iterationPath; private readonly Func _provideDateTime; + private readonly TimeSpan _timeToLiveForCacheEntries; - public DiskBasedResponseCache( + internal DiskBasedResponseCache( string storageRootPath, string scenarioName, string iterationName, - Func provideDateTime) + Func provideDateTime, + TimeSpan? timeToLiveForCacheEntries = null) { _scenarioName = scenarioName; _iterationName = iterationName; storageRootPath = Path.GetFullPath(storageRootPath); string cacheRootPath = GetCacheRootPath(storageRootPath); - string optionsFilePath = GetOptionsFilePath(cacheRootPath); - _options = File.Exists(optionsFilePath) ? CacheOptions.Read(optionsFilePath) : CacheOptions.Default; + _iterationPath = Path.Combine(cacheRootPath, scenarioName, iterationName); _provideDateTime = provideDateTime; + _timeToLiveForCacheEntries = timeToLiveForCacheEntries ?? Defaults.DefaultTimeToLiveForCacheEntries; } public byte[]? Get(string key) { - if (_options.Mode is CacheMode.Disabled) - { - return null; - } - (_, string entryFilePath, string contentsFilePath, bool filesExist) = GetPaths(key); + if (!filesExist) { - return _options.Mode is CacheMode.EnabledOfflineOnly - ? throw new FileNotFoundException( - string.Format( - CultureInfo.CurrentCulture, - EntryAndContentsFilesNotFound, - entryFilePath, - contentsFilePath)) - : null; + return null; } - if (_options.Mode is not CacheMode.EnabledOfflineOnly) + CacheEntry entry = CacheEntry.Read(entryFilePath); + if (entry.Expiration <= _provideDateTime()) { - CacheEntry entry = CacheEntry.Read(entryFilePath); - if (entry.Expiration <= _provideDateTime()) - { - Remove(key); - return null; - } + Remove(key); + return null; } return File.ReadAllBytes(contentsFilePath); @@ -82,34 +74,20 @@ public DiskBasedResponseCache( public async Task GetAsync(string key, CancellationToken cancellationToken = default) { - if (_options.Mode is CacheMode.Disabled) - { - return null; - } - (string _, string entryFilePath, string contentsFilePath, bool filesExist) = GetPaths(key); + if (!filesExist) { - return _options.Mode is CacheMode.EnabledOfflineOnly - ? throw new FileNotFoundException( - string.Format( - CultureInfo.CurrentCulture, - EntryAndContentsFilesNotFound, - entryFilePath, - contentsFilePath)) - : null; + return null; } - if (_options.Mode is not CacheMode.EnabledOfflineOnly) - { - CacheEntry entry = - await CacheEntry.ReadAsync(entryFilePath, cancellationToken: cancellationToken).ConfigureAwait(false); + CacheEntry entry = + await CacheEntry.ReadAsync(entryFilePath, cancellationToken: cancellationToken).ConfigureAwait(false); - if (entry.Expiration <= _provideDateTime()) - { - await RemoveAsync(key, cancellationToken).ConfigureAwait(false); - return null; - } + if (entry.Expiration <= _provideDateTime()) + { + await RemoveAsync(key, cancellationToken).ConfigureAwait(false); + return null; } #if NET @@ -162,12 +140,8 @@ await stream.ReadAsync( public void Refresh(string key) { - if (_options.Mode is CacheMode.Disabled or CacheMode.EnabledOfflineOnly) - { - return; - } - (_, string entryFilePath, string contentsFilePath, bool filesExist) = GetPaths(key); + if (!filesExist) { throw new FileNotFoundException( @@ -184,12 +158,8 @@ public void Refresh(string key) public async Task RefreshAsync(string key, CancellationToken cancellationToken = default) { - if (_options.Mode is CacheMode.Disabled or CacheMode.EnabledOfflineOnly) - { - return; - } - (_, string entryFilePath, string contentsFilePath, bool filesExist) = GetPaths(key); + if (!filesExist) { throw new FileNotFoundException( @@ -206,33 +176,20 @@ public async Task RefreshAsync(string key, CancellationToken cancellationToken = public void Remove(string key) { - if (_options.Mode is CacheMode.Disabled or CacheMode.EnabledOfflineOnly) - { - return; - } - (string keyPath, _, _, _) = GetPaths(key); + Directory.Delete(keyPath, recursive: true); } public Task RemoveAsync(string key, CancellationToken cancellationToken = default) { - if (_options.Mode is CacheMode.Disabled or CacheMode.EnabledOfflineOnly) - { - return Task.CompletedTask; - } - Remove(key); + return Task.CompletedTask; } public void Set(string key, byte[] value, DistributedCacheEntryOptions options) { - if (_options.Mode is CacheMode.Disabled or CacheMode.EnabledOfflineOnly) - { - return; - } - (string keyPath, string entryFilePath, string contentsFilePath, _) = GetPaths(key); _ = Directory.CreateDirectory(keyPath); @@ -249,11 +206,6 @@ public async Task SetAsync( DistributedCacheEntryOptions options, CancellationToken cancellationToken = default) { - if (_options.Mode is CacheMode.Disabled or CacheMode.EnabledOfflineOnly) - { - return; - } - (string keyPath, string entryFilePath, string contentsFilePath, _) = GetPaths(key); Directory.CreateDirectory(keyPath); @@ -334,9 +286,6 @@ await CacheEntry.ReadAsync( private static string GetCacheRootPath(string storageRootPath) => Path.Combine(storageRootPath, "cache"); - private static string GetOptionsFilePath(string cacheRootPath) - => Path.Combine(cacheRootPath, "options.json"); - private static string GetEntryFilePath(string keyPath) => Path.Combine(keyPath, "entry.json"); @@ -368,7 +317,7 @@ private static string GetContentsFilePath(string keyPath) private CacheEntry CreateEntry() { DateTime creation = _provideDateTime(); - DateTime expiration = creation.Add(_options.TimeToLiveForCacheEntries); + DateTime expiration = creation.Add(_timeToLiveForCacheEntries); return new CacheEntry(_scenarioName, _iterationName, creation, expiration); } diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/Storage/DiskBasedResponseCacheProvider.cs b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/Storage/DiskBasedResponseCacheProvider.cs index d3fce0e5aff..8b60fe5a272 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/Storage/DiskBasedResponseCacheProvider.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/Storage/DiskBasedResponseCacheProvider.cs @@ -14,21 +14,31 @@ namespace Microsoft.Extensions.AI.Evaluation.Reporting.Storage; /// -/// An that returns an that can cache AI responses -/// for a particular under the specified on disk. +/// An that returns an that can cache +/// AI responses for a particular under the specified on +/// disk. /// /// /// The path to a directory on disk under which the cached AI responses should be stored. /// -public sealed class DiskBasedResponseCacheProvider(string storageRootPath) : IResponseCacheProvider +/// +/// An optional that specifies the maximum amount of time that cached AI responses should +/// survive in the cache before they are considered expired and evicted. +/// +public sealed class DiskBasedResponseCacheProvider( + string storageRootPath, + TimeSpan? timeToLiveForCacheEntries = null) : IEvaluationResponseCacheProvider { private readonly Func _provideDateTime = () => DateTime.UtcNow; /// /// Intended for testing purposes only. /// - internal DiskBasedResponseCacheProvider(string storageRootPath, Func provideDateTime) - : this(storageRootPath) + internal DiskBasedResponseCacheProvider( + string storageRootPath, + Func provideDateTime, + TimeSpan? timeToLiveForCacheEntries = null) + : this(storageRootPath, timeToLiveForCacheEntries) { _provideDateTime = provideDateTime; } @@ -39,7 +49,13 @@ public ValueTask GetCacheAsync( string iterationName, CancellationToken cancellationToken = default) { - var cache = new DiskBasedResponseCache(storageRootPath, scenarioName, iterationName, _provideDateTime); + var cache = + new DiskBasedResponseCache( + storageRootPath, + scenarioName, + iterationName, + _provideDateTime, + timeToLiveForCacheEntries); return new ValueTask(cache); } diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/Storage/DiskBasedResultStore.cs b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/Storage/DiskBasedResultStore.cs index de1517dca99..4662857ec59 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/Storage/DiskBasedResultStore.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/Storage/DiskBasedResultStore.cs @@ -16,9 +16,9 @@ namespace Microsoft.Extensions.AI.Evaluation.Reporting.Storage; /// -/// An implementation that stores s on disk. +/// An implementation that stores s on disk. /// -public sealed class DiskBasedResultStore : IResultStore +public sealed class DiskBasedResultStore : IEvaluationResultStore { private const string DeserializationFailedMessage = "Unable to deserialize the scenario run result file at {0}."; diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/TypeScript/html-report/index.html b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/TypeScript/html-report/index.html index 8169711aca6..7f6e82be184 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/TypeScript/html-report/index.html +++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/TypeScript/html-report/index.html @@ -8,8 +8,7 @@ - + AI Evaluation Report diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/TypeScript/package-lock.json b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/TypeScript/package-lock.json index 3e77d76226f..6e76e88aa03 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/TypeScript/package-lock.json +++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/TypeScript/package-lock.json @@ -33,7 +33,7 @@ "tfx-cli": "^0.21.0", "typescript": "^5.5.3", "typescript-eslint": "^8.27.0", - "vite": "^6.2.6", + "vite": "^6.3.4", "vite-plugin-singlefile": "^2.0.2" } }, @@ -8274,6 +8274,51 @@ "node": "*" } }, + "node_modules/tinyglobby": { + "version": "0.2.13", + "resolved": "https://registry.npmjs.org/tinyglobby/-/tinyglobby-0.2.13.tgz", + "integrity": "sha512-mEwzpUgrLySlveBwEVDMKk5B57bhLPYovRfPAXD5gA/98Opn0rCDj3GtLwFvCvH5RK9uPCExUROW5NjDwvqkxw==", + "dev": true, + "license": "MIT", + "dependencies": { + "fdir": "^6.4.4", + "picomatch": "^4.0.2" + }, + "engines": { + "node": ">=12.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/SuperchupuDev" + } + }, + "node_modules/tinyglobby/node_modules/fdir": { + "version": "6.4.4", + "resolved": "https://registry.npmjs.org/fdir/-/fdir-6.4.4.tgz", + "integrity": "sha512-1NZP+GK4GfuAv3PqKvxQRDMjdSRZjnkq7KfhlNrCNNlZ0ygQFpebfrnfnq/W7fpUnAv9aGWmY1zKx7FYL3gwhg==", + "dev": true, + "license": "MIT", + "peerDependencies": { + "picomatch": "^3 || ^4" + }, + "peerDependenciesMeta": { + "picomatch": { + "optional": true + } + } + }, + "node_modules/tinyglobby/node_modules/picomatch": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.2.tgz", + "integrity": "sha512-M7BAV6Rlcy5u+m6oPhAPFgJTzAioX/6B0DxyvDlo9l8+T3nLKbrczg2WLUyzd45L8RqfUMyGPzekbMvX2Ldkwg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/jonschlinkert" + } + }, "node_modules/tinytim": { "version": "0.1.1", "resolved": "https://registry.npmjs.org/tinytim/-/tinytim-0.1.1.tgz", @@ -8764,14 +8809,18 @@ } }, "node_modules/vite": { - "version": "6.2.6", - "resolved": "https://registry.npmjs.org/vite/-/vite-6.2.6.tgz", - "integrity": "sha512-9xpjNl3kR4rVDZgPNdTL0/c6ao4km69a/2ihNQbcANz8RuCOK3hQBmLSJf3bRKVQjVMda+YvizNE8AwvogcPbw==", + "version": "6.3.5", + "resolved": "https://registry.npmjs.org/vite/-/vite-6.3.5.tgz", + "integrity": "sha512-cZn6NDFE7wdTpINgs++ZJ4N49W2vRp8LCKrn3Ob1kYNtOo21vfDoaV5GzBfLU4MovSAB8uNRm4jgzVQZ+mBzPQ==", "dev": true, + "license": "MIT", "dependencies": { "esbuild": "^0.25.0", + "fdir": "^6.4.4", + "picomatch": "^4.0.2", "postcss": "^8.5.3", - "rollup": "^4.30.1" + "rollup": "^4.34.9", + "tinyglobby": "^0.2.13" }, "bin": { "vite": "bin/vite.js" @@ -8850,6 +8899,34 @@ "vite": "^5.4.11 || ^6.0.0" } }, + "node_modules/vite/node_modules/fdir": { + "version": "6.4.4", + "resolved": "https://registry.npmjs.org/fdir/-/fdir-6.4.4.tgz", + "integrity": "sha512-1NZP+GK4GfuAv3PqKvxQRDMjdSRZjnkq7KfhlNrCNNlZ0ygQFpebfrnfnq/W7fpUnAv9aGWmY1zKx7FYL3gwhg==", + "dev": true, + "license": "MIT", + "peerDependencies": { + "picomatch": "^3 || ^4" + }, + "peerDependenciesMeta": { + "picomatch": { + "optional": true + } + } + }, + "node_modules/vite/node_modules/picomatch": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.2.tgz", + "integrity": "sha512-M7BAV6Rlcy5u+m6oPhAPFgJTzAioX/6B0DxyvDlo9l8+T3nLKbrczg2WLUyzd45L8RqfUMyGPzekbMvX2Ldkwg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/jonschlinkert" + } + }, "node_modules/walkdir": { "version": "0.0.11", "resolved": "https://registry.npmjs.org/walkdir/-/walkdir-0.0.11.tgz", diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/TypeScript/package.json b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/TypeScript/package.json index c2505b5d87a..0e32f4ae6f7 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/TypeScript/package.json +++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/TypeScript/package.json @@ -34,7 +34,7 @@ "tfx-cli": "^0.21.0", "typescript": "^5.5.3", "typescript-eslint": "^8.27.0", - "vite": "^6.2.6", + "vite": "^6.3.4", "vite-plugin-singlefile": "^2.0.2" } } diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/AIContentExtensions.cs b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/AIContentExtensions.cs index 6ec3793d0da..0334d6aa08c 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/AIContentExtensions.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/AIContentExtensions.cs @@ -4,6 +4,7 @@ using System; namespace Microsoft.Extensions.AI.Evaluation.Safety; + internal static class AIContentExtensions { internal static bool IsTextOrUsage(this AIContent content) diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/ContentSafetyServicePayloadFormat.cs b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/ContentSafetyServicePayloadFormat.cs index 428940955ff..b771dc008c8 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/ContentSafetyServicePayloadFormat.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/ContentSafetyServicePayloadFormat.cs @@ -9,5 +9,5 @@ internal enum ContentSafetyServicePayloadFormat QuestionAnswer, QueryResponse, ContextCompletion, - Conversation, + Conversation } diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation/EvaluationRating.cs b/src/Libraries/Microsoft.Extensions.AI.Evaluation/EvaluationRating.cs index 025ef58b809..1ba8ae270e6 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Evaluation/EvaluationRating.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation/EvaluationRating.cs @@ -20,14 +20,14 @@ public enum EvaluationRating Inconclusive, /// - /// A value that indicates that the is interpreted as being exceptional. + /// A value that indicates that the is interpreted as being unacceptable. /// - Exceptional, + Unacceptable, /// - /// A value that indicates that the is interpreted as being good. + /// A value that indicates that the is interpreted as being poor. /// - Good, + Poor, /// /// A value that indicates that the is interpreted as being average. @@ -35,12 +35,12 @@ public enum EvaluationRating Average, /// - /// A value that indicates that the is interpreted as being poor. + /// A value that indicates that the is interpreted as being good. /// - Poor, + Good, /// - /// A value that indicates that the is interpreted as being unacceptable. + /// A value that indicates that the is interpreted as being exceptional. /// - Unacceptable, + Exceptional } diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation/EvaluatorExtensions.cs b/src/Libraries/Microsoft.Extensions.AI.Evaluation/EvaluatorExtensions.cs index 3ffda8fb8f9..8d25085f4e0 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Evaluation/EvaluatorExtensions.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation/EvaluatorExtensions.cs @@ -131,17 +131,12 @@ public static ValueTask EvaluateAsync( ChatMessage modelResponse, ChatConfiguration? chatConfiguration = null, IEnumerable? additionalContext = null, - CancellationToken cancellationToken = default) - { - _ = Throw.IfNull(evaluator); - - return evaluator.EvaluateAsync( - messages: [], + CancellationToken cancellationToken = default) => + evaluator.EvaluateAsync( new ChatResponse(modelResponse), chatConfiguration, additionalContext, cancellationToken); - } /// /// Evaluates the supplied and returns an @@ -225,17 +220,13 @@ public static ValueTask EvaluateAsync( ChatMessage modelResponse, ChatConfiguration? chatConfiguration = null, IEnumerable? additionalContext = null, - CancellationToken cancellationToken = default) - { - _ = Throw.IfNull(evaluator); - - return evaluator.EvaluateAsync( - messages: [userRequest], + CancellationToken cancellationToken = default) => + evaluator.EvaluateAsync( + userRequest, new ChatResponse(modelResponse), chatConfiguration, additionalContext, cancellationToken); - } /// /// Evaluates the supplied and returns an diff --git a/src/Libraries/Microsoft.Extensions.AI.Ollama/CHANGELOG.md b/src/Libraries/Microsoft.Extensions.AI.Ollama/CHANGELOG.md index 8822f8ddaea..e90fed2cdba 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Ollama/CHANGELOG.md +++ b/src/Libraries/Microsoft.Extensions.AI.Ollama/CHANGELOG.md @@ -1,5 +1,9 @@ # Release History +## 9.4.4-preview.1.25259.16 + +- Updated to accomodate the changes in `Microsoft.Extensions.AI.Abstractions`. + ## 9.4.3-preview.1.25230.7 - Updated to accomodate the changes in `Microsoft.Extensions.AI.Abstractions`. diff --git a/src/Libraries/Microsoft.Extensions.AI.Ollama/OllamaChatClient.cs b/src/Libraries/Microsoft.Extensions.AI.Ollama/OllamaChatClient.cs index 210d8f7c92d..28f8eb8c3ad 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Ollama/OllamaChatClient.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Ollama/OllamaChatClient.cs @@ -25,6 +25,11 @@ public sealed class OllamaChatClient : IChatClient { private static readonly JsonElement _schemalessJsonResponseFormatValue = JsonDocument.Parse("\"json\"").RootElement; + private static readonly AIJsonSchemaTransformCache _schemaTransformCache = new(new() + { + ConvertBooleanSchemas = true, + }); + /// Metadata about the client. private readonly ChatClientMetadata _metadata; @@ -292,7 +297,7 @@ private static FunctionCallContent ToFunctionCallContent(OllamaFunctionToolCall { if (format is ChatResponseFormatJson jsonFormat) { - return jsonFormat.Schema ?? _schemalessJsonResponseFormatValue; + return _schemaTransformCache.GetOrCreateTransformedSchema(jsonFormat) ?? _schemalessJsonResponseFormatValue; } else { @@ -483,7 +488,7 @@ private static OllamaTool ToOllamaTool(AIFunction function) { Name = function.Name, Description = function.Description, - Parameters = JsonSerializer.Deserialize(function.JsonSchema, JsonContext.Default.OllamaFunctionToolParameters)!, + Parameters = JsonSerializer.Deserialize(_schemaTransformCache.GetOrCreateTransformedSchema(function), JsonContext.Default.OllamaFunctionToolParameters)!, } }; } diff --git a/src/Libraries/Microsoft.Extensions.AI.OpenAI/CHANGELOG.md b/src/Libraries/Microsoft.Extensions.AI.OpenAI/CHANGELOG.md index 05130ba3847..ad915d06aa7 100644 --- a/src/Libraries/Microsoft.Extensions.AI.OpenAI/CHANGELOG.md +++ b/src/Libraries/Microsoft.Extensions.AI.OpenAI/CHANGELOG.md @@ -1,5 +1,11 @@ # Release History +## 9.4.4-preview.1.25259.16 + +- Made `IChatClient` implementation more resilient with non-OpenAI services. +- Added `ErrorContent` to represent refusals. +- Updated to accomodate the changes in `Microsoft.Extensions.AI.Abstractions`. + ## 9.4.3-preview.1.25230.7 - Reverted previous change that enabled `strict` schemas by default. diff --git a/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIChatClient.cs b/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIChatClient.cs index c78b495393b..001f4d1a593 100644 --- a/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIChatClient.cs +++ b/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIChatClient.cs @@ -18,12 +18,22 @@ #pragma warning disable EA0011 // Consider removing unnecessary conditional access operator (?) #pragma warning disable S1067 // Expressions should not be too complex #pragma warning disable S3011 // Reflection should not be used to increase accessibility of classes, methods, or fields +#pragma warning disable SA1204 // Static elements should appear before instance elements namespace Microsoft.Extensions.AI; /// Represents an for an OpenAI or . internal sealed partial class OpenAIChatClient : IChatClient { + /// Gets the JSON schema transformer cache conforming to OpenAI restrictions per https://platform.openai.com/docs/guides/structured-outputs?api-mode=responses#supported-schemas. + internal static AIJsonSchemaTransformCache SchemaTransformCache { get; } = new(new() + { + RequireAllProperties = true, + DisallowAdditionalProperties = true, + ConvertBooleanSchemas = true, + MoveDefaultKeywordToDescription = true, + }); + /// Gets the default OpenAI endpoint. private static Uri DefaultOpenAIEndpoint { get; } = new("https://api.openai.com/v1"); @@ -148,30 +158,55 @@ void IDisposable.Dispose() } else if (input.Role == ChatRole.Assistant) { - AssistantChatMessage message = new(ToOpenAIChatContent(input.Contents)) - { - ParticipantName = input.AuthorName - }; - + List? contentParts = null; + List? toolCalls = null; + string? refusal = null; foreach (var content in input.Contents) { - if (content is FunctionCallContent callRequest) + switch (content) { - message.ToolCalls.Add( - ChatToolCall.CreateFunctionToolCall( - callRequest.CallId, - callRequest.Name, - new(JsonSerializer.SerializeToUtf8Bytes( - callRequest.Arguments, - options.GetTypeInfo(typeof(IDictionary)))))); + case ErrorContent ec when ec.ErrorCode == nameof(AssistantChatMessage.Refusal): + refusal = ec.Message; + break; + + case FunctionCallContent fc: + (toolCalls ??= []).Add( + ChatToolCall.CreateFunctionToolCall(fc.CallId, fc.Name, new(JsonSerializer.SerializeToUtf8Bytes( + fc.Arguments, options.GetTypeInfo(typeof(IDictionary)))))); + break; + + default: + if (ToChatMessageContentPart(content) is { } part) + { + (contentParts ??= []).Add(part); + } + + break; } } - if (input.AdditionalProperties?.TryGetValue(nameof(message.Refusal), out string? refusal) is true) + AssistantChatMessage message; + if (contentParts is not null) + { + message = new(contentParts); + if (toolCalls is not null) + { + foreach (var toolCall in toolCalls) + { + message.ToolCalls.Add(toolCall); + } + } + } + else { - message.Refusal = refusal; + message = toolCalls is not null ? + new(toolCalls) : + new(ChatMessageContentPart.CreateTextPart(string.Empty)); } + message.ParticipantName = input.AuthorName; + message.Refusal = refusal; + yield return message; } } @@ -181,38 +216,12 @@ void IDisposable.Dispose() private static List ToOpenAIChatContent(IList contents) { List parts = []; + foreach (var content in contents) { - switch (content) + if (ToChatMessageContentPart(content) is { } part) { - case TextContent textContent: - parts.Add(ChatMessageContentPart.CreateTextPart(textContent.Text)); - break; - - case UriContent uriContent when uriContent.HasTopLevelMediaType("image"): - parts.Add(ChatMessageContentPart.CreateImagePart(uriContent.Uri, GetImageDetail(content))); - break; - - case DataContent dataContent when dataContent.HasTopLevelMediaType("image"): - parts.Add(ChatMessageContentPart.CreateImagePart(BinaryData.FromBytes(dataContent.Data), dataContent.MediaType, GetImageDetail(content))); - break; - - case DataContent dataContent when dataContent.HasTopLevelMediaType("audio"): - var audioData = BinaryData.FromBytes(dataContent.Data); - if (dataContent.MediaType.Equals("audio/mpeg", StringComparison.OrdinalIgnoreCase)) - { - parts.Add(ChatMessageContentPart.CreateInputAudioPart(audioData, ChatInputAudioFormat.Mp3)); - } - else if (dataContent.MediaType.Equals("audio/wav", StringComparison.OrdinalIgnoreCase)) - { - parts.Add(ChatMessageContentPart.CreateInputAudioPart(audioData, ChatInputAudioFormat.Wav)); - } - - break; - - case DataContent dataContent when dataContent.MediaType.StartsWith("application/pdf", StringComparison.OrdinalIgnoreCase): - parts.Add(ChatMessageContentPart.CreateFilePart(BinaryData.FromBytes(dataContent.Data), dataContent.MediaType, $"{Guid.NewGuid():N}.pdf")); - break; + parts.Add(part); } } @@ -224,6 +233,39 @@ private static List ToOpenAIChatContent(IList return parts; } + private static ChatMessageContentPart? ToChatMessageContentPart(AIContent content) + { + switch (content) + { + case TextContent textContent: + return ChatMessageContentPart.CreateTextPart(textContent.Text); + + case UriContent uriContent when uriContent.HasTopLevelMediaType("image"): + return ChatMessageContentPart.CreateImagePart(uriContent.Uri, GetImageDetail(content)); + + case DataContent dataContent when dataContent.HasTopLevelMediaType("image"): + return ChatMessageContentPart.CreateImagePart(BinaryData.FromBytes(dataContent.Data), dataContent.MediaType, GetImageDetail(content)); + + case DataContent dataContent when dataContent.HasTopLevelMediaType("audio"): + var audioData = BinaryData.FromBytes(dataContent.Data); + if (dataContent.MediaType.Equals("audio/mpeg", StringComparison.OrdinalIgnoreCase)) + { + return ChatMessageContentPart.CreateInputAudioPart(audioData, ChatInputAudioFormat.Mp3); + } + else if (dataContent.MediaType.Equals("audio/wav", StringComparison.OrdinalIgnoreCase)) + { + return ChatMessageContentPart.CreateInputAudioPart(audioData, ChatInputAudioFormat.Wav); + } + + break; + + case DataContent dataContent when dataContent.MediaType.StartsWith("application/pdf", StringComparison.OrdinalIgnoreCase): + return ChatMessageContentPart.CreateFilePart(BinaryData.FromBytes(dataContent.Data), dataContent.MediaType, $"{Guid.NewGuid():N}.pdf"); + } + + return null; + } + private static ChatImageDetailLevel? GetImageDetail(AIContent content) { if (content.AdditionalProperties?.TryGetValue("detail", out object? value) is true) @@ -250,7 +292,6 @@ private static async IAsyncEnumerable FromOpenAIStreamingCha string? responseId = null; DateTimeOffset? createdAt = null; string? modelId = null; - string? fingerprint = null; // Process each update as it arrives await foreach (StreamingChatCompletionUpdate update in updates.WithCancellation(cancellationToken).ConfigureAwait(false)) @@ -261,7 +302,6 @@ private static async IAsyncEnumerable FromOpenAIStreamingCha responseId ??= update.CompletionId; createdAt ??= update.CreatedAt; modelId ??= update.Model; - fingerprint ??= update.SystemFingerprint; // Create the response content object. ChatResponseUpdate responseUpdate = new() @@ -275,22 +315,6 @@ private static async IAsyncEnumerable FromOpenAIStreamingCha Role = streamedRole, }; - // Populate it with any additional metadata from the OpenAI object. - if (update.ContentTokenLogProbabilities is { Count: > 0 } contentTokenLogProbs) - { - (responseUpdate.AdditionalProperties ??= [])[nameof(update.ContentTokenLogProbabilities)] = contentTokenLogProbs; - } - - if (update.RefusalTokenLogProbabilities is { Count: > 0 } refusalTokenLogProbs) - { - (responseUpdate.AdditionalProperties ??= [])[nameof(update.RefusalTokenLogProbabilities)] = refusalTokenLogProbs; - } - - if (fingerprint is not null) - { - (responseUpdate.AdditionalProperties ??= [])[nameof(update.SystemFingerprint)] = fingerprint; - } - // Transfer over content update items. if (update.ContentUpdate is { Count: > 0 }) { @@ -370,13 +394,7 @@ private static async IAsyncEnumerable FromOpenAIStreamingCha // add it to this function calling item. if (refusal is not null) { - (responseUpdate.AdditionalProperties ??= [])[nameof(ChatMessageContentPart.Refusal)] = refusal.ToString(); - } - - // Propagate additional relevant metadata. - if (fingerprint is not null) - { - (responseUpdate.AdditionalProperties ??= [])[nameof(ChatCompletion.SystemFingerprint)] = fingerprint; + responseUpdate.Contents.Add(new ErrorContent(refusal.ToString()) { ErrorCode = "Refusal" }); } yield return responseUpdate; @@ -417,20 +435,7 @@ private static ChatResponse FromOpenAIChatCompletion(ChatCompletion openAIComple "mp3" or _ => "audio/mpeg", }; - var dc = new DataContent(audio.AudioBytes.ToMemory(), mimeType) - { - AdditionalProperties = new() { [nameof(audio.ExpiresAt)] = audio.ExpiresAt }, - }; - - if (audio.Id is string id) - { - dc.AdditionalProperties[nameof(audio.Id)] = id; - } - - if (audio.Transcript is string transcript) - { - dc.AdditionalProperties[nameof(audio.Transcript)] = transcript; - } + var dc = new DataContent(audio.AudioBytes.ToMemory(), mimeType); returnMessage.Contents.Add(dc); } @@ -450,6 +455,12 @@ private static ChatResponse FromOpenAIChatCompletion(ChatCompletion openAIComple } } + // And add error content for any refusals, which represent errors in generating output that conforms to a provided schema. + if (openAICompletion.Refusal is string refusal) + { + returnMessage.Contents.Add(new ErrorContent(refusal) { ErrorCode = nameof(openAICompletion.Refusal) }); + } + // Wrap the content in a ChatResponse to return. var response = new ChatResponse(returnMessage) { @@ -465,152 +476,81 @@ private static ChatResponse FromOpenAIChatCompletion(ChatCompletion openAIComple response.Usage = FromOpenAIUsage(tokenUsage); } - if (openAICompletion.ContentTokenLogProbabilities is { Count: > 0 } contentTokenLogProbs) - { - (response.AdditionalProperties ??= [])[nameof(openAICompletion.ContentTokenLogProbabilities)] = contentTokenLogProbs; - } - - if (openAICompletion.Refusal is string refusal) - { - (response.AdditionalProperties ??= [])[nameof(openAICompletion.Refusal)] = refusal; - } - - if (openAICompletion.RefusalTokenLogProbabilities is { Count: > 0 } refusalTokenLogProbs) - { - (response.AdditionalProperties ??= [])[nameof(openAICompletion.RefusalTokenLogProbabilities)] = refusalTokenLogProbs; - } - - if (openAICompletion.SystemFingerprint is string systemFingerprint) - { - (response.AdditionalProperties ??= [])[nameof(openAICompletion.SystemFingerprint)] = systemFingerprint; - } - return response; } /// Converts an extensions options instance to an OpenAI options instance. - private static ChatCompletionOptions ToOpenAIOptions(ChatOptions? options) + private ChatCompletionOptions ToOpenAIOptions(ChatOptions? options) { - ChatCompletionOptions result = new(); + if (options is null) + { + return new ChatCompletionOptions(); + } - if (options is not null) + if (options.RawRepresentationFactory?.Invoke(this) is not ChatCompletionOptions result) { - result.FrequencyPenalty = options.FrequencyPenalty; - result.MaxOutputTokenCount = options.MaxOutputTokens; - result.TopP = options.TopP; - result.PresencePenalty = options.PresencePenalty; - result.Temperature = options.Temperature; - result.AllowParallelToolCalls = options.AllowMultipleToolCalls; + result = new ChatCompletionOptions(); + } + + result.FrequencyPenalty ??= options.FrequencyPenalty; + result.MaxOutputTokenCount ??= options.MaxOutputTokens; + result.TopP ??= options.TopP; + result.PresencePenalty ??= options.PresencePenalty; + result.Temperature ??= options.Temperature; + result.AllowParallelToolCalls ??= options.AllowMultipleToolCalls; #pragma warning disable OPENAI001 // Type is for evaluation purposes only and is subject to change or removal in future updates. - result.Seed = options.Seed; + result.Seed ??= options.Seed; #pragma warning restore OPENAI001 - if (options.StopSequences is { Count: > 0 } stopSequences) + if (options.StopSequences is { Count: > 0 } stopSequences) + { + foreach (string stopSequence in stopSequences) { - foreach (string stopSequence in stopSequences) - { - result.StopSequences.Add(stopSequence); - } + result.StopSequences.Add(stopSequence); } + } - if (options.AdditionalProperties is { Count: > 0 } additionalProperties) + if (options.Tools is { Count: > 0 } tools) + { + foreach (AITool tool in tools) { - if (additionalProperties.TryGetValue(nameof(result.AudioOptions), out ChatAudioOptions? audioOptions)) - { - result.AudioOptions = audioOptions; - } - - if (additionalProperties.TryGetValue(nameof(result.EndUserId), out string? endUserId)) - { - result.EndUserId = endUserId; - } - - if (additionalProperties.TryGetValue(nameof(result.IncludeLogProbabilities), out bool includeLogProbabilities)) - { - result.IncludeLogProbabilities = includeLogProbabilities; - } - - if (additionalProperties.TryGetValue(nameof(result.LogitBiases), out IDictionary? logitBiases)) - { - foreach (KeyValuePair kvp in logitBiases!) - { - result.LogitBiases[kvp.Key] = kvp.Value; - } - } - - if (additionalProperties.TryGetValue(nameof(result.Metadata), out IDictionary? metadata)) - { - foreach (KeyValuePair kvp in metadata) - { - result.Metadata[kvp.Key] = kvp.Value; - } - } - - if (additionalProperties.TryGetValue(nameof(result.OutputPrediction), out ChatOutputPrediction? outputPrediction)) - { - result.OutputPrediction = outputPrediction; - } - - if (additionalProperties.TryGetValue(nameof(result.ReasoningEffortLevel), out ChatReasoningEffortLevel reasoningEffortLevel)) - { - result.ReasoningEffortLevel = reasoningEffortLevel; - } - - if (additionalProperties.TryGetValue(nameof(result.ResponseModalities), out ChatResponseModalities responseModalities)) - { - result.ResponseModalities = responseModalities; - } - - if (additionalProperties.TryGetValue(nameof(result.StoredOutputEnabled), out bool storeOutputEnabled)) - { - result.StoredOutputEnabled = storeOutputEnabled; - } - - if (additionalProperties.TryGetValue(nameof(result.TopLogProbabilityCount), out int topLogProbabilityCountInt)) + if (tool is AIFunction af) { - result.TopLogProbabilityCount = topLogProbabilityCountInt; + result.Tools.Add(ToOpenAIChatTool(af)); } } - if (options.Tools is { Count: > 0 } tools) + if (result.ToolChoice is null && result.Tools.Count > 0) { - foreach (AITool tool in tools) + switch (options.ToolMode) { - if (tool is AIFunction af) - { - result.Tools.Add(ToOpenAIChatTool(af)); - } - } - - if (result.Tools.Count > 0) - { - switch (options.ToolMode) - { - case NoneChatToolMode: - result.ToolChoice = ChatToolChoice.CreateNoneChoice(); - break; - - case AutoChatToolMode: - case null: - result.ToolChoice = ChatToolChoice.CreateAutoChoice(); - break; - - case RequiredChatToolMode required: - result.ToolChoice = required.RequiredFunctionName is null ? - ChatToolChoice.CreateRequiredChoice() : - ChatToolChoice.CreateFunctionChoice(required.RequiredFunctionName); - break; - } + case NoneChatToolMode: + result.ToolChoice = ChatToolChoice.CreateNoneChoice(); + break; + + case AutoChatToolMode: + case null: + result.ToolChoice = ChatToolChoice.CreateAutoChoice(); + break; + + case RequiredChatToolMode required: + result.ToolChoice = required.RequiredFunctionName is null ? + ChatToolChoice.CreateRequiredChoice() : + ChatToolChoice.CreateFunctionChoice(required.RequiredFunctionName); + break; } } + } + if (result.ResponseFormat is null) + { if (options.ResponseFormat is ChatResponseFormatText) { result.ResponseFormat = OpenAI.Chat.ChatResponseFormat.CreateTextFormat(); } else if (options.ResponseFormat is ChatResponseFormatJson jsonFormat) { - result.ResponseFormat = jsonFormat.Schema is { } jsonSchema ? + result.ResponseFormat = SchemaTransformCache.GetOrCreateTransformedSchema(jsonFormat) is { } jsonSchema ? OpenAI.Chat.ChatResponseFormat.CreateJsonSchemaFormat( jsonFormat.SchemaName ?? "json_schema", BinaryData.FromBytes( @@ -631,8 +571,11 @@ private static ChatTool ToOpenAIChatTool(AIFunction aiFunction) strictObj is bool strictValue ? strictValue : null; + // Perform transformations making the schema legal per OpenAI restrictions + JsonElement jsonSchema = SchemaTransformCache.GetOrCreateTransformedSchema(aiFunction); + // Map to an intermediate model so that redundant properties are skipped. - var tool = JsonSerializer.Deserialize(aiFunction.JsonSchema, ChatClientJsonContext.Default.ChatToolJson)!; + var tool = JsonSerializer.Deserialize(jsonSchema, ChatClientJsonContext.Default.ChatToolJson)!; var functionParameters = BinaryData.FromBytes(JsonSerializer.SerializeToUtf8Bytes(tool, ChatClientJsonContext.Default.ChatToolJson)); return ChatTool.CreateFunctionTool(aiFunction.Name, aiFunction.Description, functionParameters, strict); } diff --git a/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIResponseChatClient.cs b/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIResponseChatClient.cs index 92224379c10..a91ea9abf8f 100644 --- a/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIResponseChatClient.cs +++ b/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIResponseChatClient.cs @@ -263,6 +263,7 @@ public async IAsyncEnumerable GetStreamingResponseAsync( MessageId = lastMessageId, ModelId = modelId, ResponseId = responseId, + Role = lastRole, ConversationId = responseId, Contents = [ @@ -274,6 +275,19 @@ public async IAsyncEnumerable GetStreamingResponseAsync( ], }; break; + + case StreamingResponseRefusalDoneUpdate refusalDone: + yield return new ChatResponseUpdate + { + CreatedAt = createdAt, + MessageId = lastMessageId, + ModelId = modelId, + ResponseId = responseId, + Role = lastRole, + ConversationId = responseId, + Contents = [new ErrorContent(refusalDone.Refusal) { ErrorCode = nameof(ResponseContentPart.Refusal) }], + }; + break; } } } @@ -359,7 +373,7 @@ private static ResponseCreationOptions ToOpenAIResponseCreationOptions(ChatOptio switch (tool) { case AIFunction af: - var oaitool = JsonSerializer.Deserialize(af.JsonSchema, ResponseClientJsonContext.Default.ResponseToolJson)!; + var oaitool = JsonSerializer.Deserialize(SchemaTransformCache.GetOrCreateTransformedSchema(af), ResponseClientJsonContext.Default.ResponseToolJson)!; var functionParameters = BinaryData.FromBytes(JsonSerializer.SerializeToUtf8Bytes(oaitool, ResponseClientJsonContext.Default.ResponseToolJson)); result.Tools.Add(ResponseTool.CreateFunctionTool(af.Name, af.Description, functionParameters)); break; @@ -414,7 +428,7 @@ private static ResponseCreationOptions ToOpenAIResponseCreationOptions(ChatOptio { result.TextOptions = new() { - TextFormat = jsonFormat.Schema is { } jsonSchema ? + TextFormat = SchemaTransformCache.GetOrCreateTransformedSchema(jsonFormat) is { } jsonSchema ? ResponseTextFormat.CreateJsonSchemaFormat( jsonFormat.SchemaName ?? "json_schema", BinaryData.FromBytes(JsonSerializer.SerializeToUtf8Bytes(jsonSchema, ResponseClientJsonContext.Default.JsonElement)), @@ -539,9 +553,15 @@ private static List ToAIContents(IEnumerable con foreach (ResponseContentPart part in contents) { - if (part.Kind == ResponseContentPartKind.OutputText) + switch (part.Kind) { - results.Add(new TextContent(part.Text)); + case ResponseContentPartKind.OutputText: + results.Add(new TextContent(part.Text)); + break; + + case ResponseContentPartKind.Refusal: + results.Add(new ErrorContent(part.Refusal) { ErrorCode = nameof(ResponseContentPartKind.Refusal) }); + break; } } @@ -572,6 +592,10 @@ private static List ToOpenAIResponsesContent(IList` to `useJsonSchemaResponseFormat`. +- Updated `OpenTelemetryChatClient` and `OpenTelemetryEmbeddingGenerator` to conform to the latest 1.33.0 draft specification of the Semantic Conventions for Generative AI systems. + ## 9.4.3-preview.1.25230.7 - Updated the diagnostic spans emitted by `FunctionInvokingChatClient` to include total input and output token counts. diff --git a/src/Libraries/Microsoft.Extensions.AI/ChatCompletion/CachingChatClient.cs b/src/Libraries/Microsoft.Extensions.AI/ChatCompletion/CachingChatClient.cs index 5aa70e4b262..211fc39ec85 100644 --- a/src/Libraries/Microsoft.Extensions.AI/ChatCompletion/CachingChatClient.cs +++ b/src/Libraries/Microsoft.Extensions.AI/ChatCompletion/CachingChatClient.cs @@ -9,6 +9,7 @@ using Microsoft.Shared.Diagnostics; #pragma warning disable S127 // "for" loop stop conditions should be invariant +#pragma warning disable SA1202 // Elements should be ordered by access namespace Microsoft.Extensions.AI; @@ -45,11 +46,19 @@ protected CachingChatClient(IChatClient innerClient) public bool CoalesceStreamingUpdates { get; set; } = true; /// - public override async Task GetResponseAsync( + public override Task GetResponseAsync( IEnumerable messages, ChatOptions? options = null, CancellationToken cancellationToken = default) { _ = Throw.IfNull(messages); + return UseCaching(options) ? + GetCachedResponseAsync(messages, options, cancellationToken) : + base.GetResponseAsync(messages, options, cancellationToken); + } + + private async Task GetCachedResponseAsync( + IEnumerable messages, ChatOptions? options = null, CancellationToken cancellationToken = default) + { // We're only storing the final result, not the in-flight task, so that we can avoid caching failures // or having problems when one of the callers cancels but others don't. This has the drawback that // concurrent callers might trigger duplicate requests, but that's acceptable. @@ -65,11 +74,19 @@ public override async Task GetResponseAsync( } /// - public override async IAsyncEnumerable GetStreamingResponseAsync( - IEnumerable messages, ChatOptions? options = null, [EnumeratorCancellation] CancellationToken cancellationToken = default) + public override IAsyncEnumerable GetStreamingResponseAsync( + IEnumerable messages, ChatOptions? options = null, CancellationToken cancellationToken = default) { _ = Throw.IfNull(messages); + return UseCaching(options) ? + GetCachedStreamingResponseAsync(messages, options, cancellationToken) : + base.GetStreamingResponseAsync(messages, options, cancellationToken); + } + + private async IAsyncEnumerable GetCachedStreamingResponseAsync( + IEnumerable messages, ChatOptions? options = null, [EnumeratorCancellation] CancellationToken cancellationToken = default) + { if (CoalesceStreamingUpdates) { // When coalescing updates, we cache non-streaming results coalesced from streaming ones. That means @@ -178,4 +195,13 @@ public override async IAsyncEnumerable GetStreamingResponseA /// is . /// is . protected abstract Task WriteCacheStreamingAsync(string key, IReadOnlyList value, CancellationToken cancellationToken); + + /// Determine whether to use caching with the request. + private static bool UseCaching(ChatOptions? options) + { + // We want to skip caching if options.ConversationId is set. If it's set, that implies there's + // some state that will impact the response and that's not represented in the messages. Since + // that state could change even with the same ID, we have to assume caching isn't valid. + return options?.ConversationId is null; + } } diff --git a/src/Libraries/Microsoft.Extensions.AI/ChatCompletion/ChatClientStructuredOutputExtensions.cs b/src/Libraries/Microsoft.Extensions.AI/ChatCompletion/ChatClientStructuredOutputExtensions.cs index 0e7b20bade6..69c4cc7ee89 100644 --- a/src/Libraries/Microsoft.Extensions.AI/ChatCompletion/ChatClientStructuredOutputExtensions.cs +++ b/src/Libraries/Microsoft.Extensions.AI/ChatCompletion/ChatClientStructuredOutputExtensions.cs @@ -7,11 +7,13 @@ using System.Reflection; using System.Text.Json; using System.Text.Json.Nodes; +using System.Text.RegularExpressions; using System.Threading; using System.Threading.Tasks; using Microsoft.Shared.Diagnostics; #pragma warning disable SA1118 // Parameter should not span multiple lines +#pragma warning disable S2333 // Redundant modifiers should not be used namespace Microsoft.Extensions.AI; @@ -19,21 +21,24 @@ namespace Microsoft.Extensions.AI; /// Provides extension methods on that simplify working with structured output. /// /// Request a response with structured output. -public static class ChatClientStructuredOutputExtensions +public static partial class ChatClientStructuredOutputExtensions { private static readonly AIJsonSchemaCreateOptions _inferenceOptions = new() { IncludeSchemaKeyword = true, - DisallowAdditionalProperties = true, - IncludeTypeInEnumSchemas = true, - RequireAllProperties = true, + TransformOptions = new AIJsonSchemaTransformOptions + { + DisallowAdditionalProperties = true, + RequireAllProperties = true, + MoveDefaultKeywordToDescription = true, + }, }; /// Sends chat messages, requesting a response matching the type . /// The . /// The chat content to send. /// The chat options to configure the request. - /// + /// /// to set a JSON schema on the ; otherwise, . The default is . /// Using a JSON schema improves reliability if the underlying model supports native structured output with a schema, but might cause an error if the model does not support it. /// @@ -44,19 +49,17 @@ public static Task> GetResponseAsync( this IChatClient chatClient, IEnumerable messages, ChatOptions? options = null, - bool? useJsonSchema = null, + bool? useJsonSchemaResponseFormat = null, CancellationToken cancellationToken = default) => - GetResponseAsync(chatClient, messages, AIJsonUtilities.DefaultOptions, options, useJsonSchema, cancellationToken); + GetResponseAsync(chatClient, messages, AIJsonUtilities.DefaultOptions, options, useJsonSchemaResponseFormat, cancellationToken); /// Sends a user chat text message, requesting a response matching the type . /// The . /// The text content for the chat message to send. /// The chat options to configure the request. - /// + /// /// to set a JSON schema on the ; otherwise, . /// Using a JSON schema improves reliability if the underlying model supports native structured output with a schema, but might cause an error if the model does not support it. - /// If not specified, the default value is determined by the implementation. - /// If a specific value is required, it must be specified by the caller. /// /// The to monitor for cancellation requests. The default is . /// The response messages generated by the client. @@ -66,15 +69,15 @@ public static Task> GetResponseAsync( this IChatClient chatClient, string chatMessage, ChatOptions? options = null, - bool? useJsonSchema = null, + bool? useJsonSchemaResponseFormat = null, CancellationToken cancellationToken = default) => - GetResponseAsync(chatClient, new ChatMessage(ChatRole.User, chatMessage), options, useJsonSchema, cancellationToken); + GetResponseAsync(chatClient, new ChatMessage(ChatRole.User, chatMessage), options, useJsonSchemaResponseFormat, cancellationToken); /// Sends a chat message, requesting a response matching the type . /// The . /// The chat message to send. /// The chat options to configure the request. - /// + /// /// to set a JSON schema on the ; otherwise, . The default is . /// Using a JSON schema improves reliability if the underlying model supports native structured output with a schema, but might cause an error if the model does not support it. /// @@ -85,16 +88,16 @@ public static Task> GetResponseAsync( this IChatClient chatClient, ChatMessage chatMessage, ChatOptions? options = null, - bool? useJsonSchema = null, + bool? useJsonSchemaResponseFormat = null, CancellationToken cancellationToken = default) => - GetResponseAsync(chatClient, [chatMessage], options, useJsonSchema, cancellationToken); + GetResponseAsync(chatClient, [chatMessage], options, useJsonSchemaResponseFormat, cancellationToken); /// Sends a user chat text message, requesting a response matching the type . /// The . /// The text content for the chat message to send. /// The JSON serialization options to use. /// The chat options to configure the request. - /// + /// /// to set a JSON schema on the ; otherwise, . The default is . /// Using a JSON schema improves reliability if the underlying model supports native structured output with a schema, but might cause an error if the model does not support it. /// @@ -106,16 +109,16 @@ public static Task> GetResponseAsync( string chatMessage, JsonSerializerOptions serializerOptions, ChatOptions? options = null, - bool? useJsonSchema = null, + bool? useJsonSchemaResponseFormat = null, CancellationToken cancellationToken = default) => - GetResponseAsync(chatClient, new ChatMessage(ChatRole.User, chatMessage), serializerOptions, options, useJsonSchema, cancellationToken); + GetResponseAsync(chatClient, new ChatMessage(ChatRole.User, chatMessage), serializerOptions, options, useJsonSchemaResponseFormat, cancellationToken); /// Sends a chat message, requesting a response matching the type . /// The . /// The chat message to send. /// The JSON serialization options to use. /// The chat options to configure the request. - /// + /// /// to set a JSON schema on the ; otherwise, . The default is . /// Using a JSON schema improves reliability if the underlying model supports native structured output with a schema, but might cause an error if the model does not support it. /// @@ -127,16 +130,16 @@ public static Task> GetResponseAsync( ChatMessage chatMessage, JsonSerializerOptions serializerOptions, ChatOptions? options = null, - bool? useJsonSchema = null, + bool? useJsonSchemaResponseFormat = null, CancellationToken cancellationToken = default) => - GetResponseAsync(chatClient, [chatMessage], serializerOptions, options, useJsonSchema, cancellationToken); + GetResponseAsync(chatClient, [chatMessage], serializerOptions, options, useJsonSchemaResponseFormat, cancellationToken); /// Sends chat messages, requesting a response matching the type . /// The . /// The chat content to send. /// The JSON serialization options to use. /// The chat options to configure the request. - /// + /// /// to set a JSON schema on the ; otherwise, . The default is . /// Using a JSON schema improves reliability if the underlying model supports native structured output with a schema, but might cause an error if the model does not support it. /// @@ -149,7 +152,7 @@ public static async Task> GetResponseAsync( IEnumerable messages, JsonSerializerOptions serializerOptions, ChatOptions? options = null, - bool? useJsonSchema = null, + bool? useJsonSchemaResponseFormat = null, CancellationToken cancellationToken = default) { _ = Throw.IfNull(chatClient); @@ -192,14 +195,14 @@ public static async Task> GetResponseAsync( // We default to assuming that models support JSON schema because developers will normally use // GetResponseAsync only with models that do. If the model doesn't support JSON schema, it may - // throw or it may ignore the schema. In these cases developers should pass useJsonSchema: false. - if (useJsonSchema.GetValueOrDefault(true)) + // throw or it may ignore the schema. In these cases developers should pass useJsonSchemaResponseFormat: false. + if (useJsonSchemaResponseFormat ?? true) { // When using native structured output, we don't add any additional prompt, because // the LLM backend is meant to do whatever's needed to explain the schema to the LLM. options.ResponseFormat = ChatResponseFormat.ForJsonSchema( schema, - schemaName: AIFunctionFactory.SanitizeMemberName(typeof(T).Name), + schemaName: SanitizeMemberName(typeof(T).Name), schemaDescription: typeof(T).GetCustomAttribute()?.Description); } else @@ -248,4 +251,24 @@ private static bool SchemaRepresentsObject(JsonElement schemaElement) _ => JsonValue.Create(element) }; } + + /// + /// Removes characters from a .NET member name that shouldn't be used in an AI function name. + /// + /// The .NET member name that should be sanitized. + /// + /// Replaces non-alphanumeric characters in the identifier with the underscore character. + /// Primarily intended to remove characters produced by compiler-generated method name mangling. + /// + private static string SanitizeMemberName(string memberName) => + InvalidNameCharsRegex().Replace(memberName, "_"); + + /// Regex that flags any character other than ASCII digits or letters or the underscore. +#if NET + [GeneratedRegex("[^0-9A-Za-z_]")] + private static partial Regex InvalidNameCharsRegex(); +#else + private static Regex InvalidNameCharsRegex() => _invalidNameCharsRegex; + private static readonly Regex _invalidNameCharsRegex = new("[^0-9A-Za-z_]", RegexOptions.Compiled); +#endif } diff --git a/src/Libraries/Microsoft.Extensions.AI/ChatCompletion/LoggingChatClient.cs b/src/Libraries/Microsoft.Extensions.AI/ChatCompletion/LoggingChatClient.cs index b5f43f5385b..aec72eddcdc 100644 --- a/src/Libraries/Microsoft.Extensions.AI/ChatCompletion/LoggingChatClient.cs +++ b/src/Libraries/Microsoft.Extensions.AI/ChatCompletion/LoggingChatClient.cs @@ -13,10 +13,18 @@ namespace Microsoft.Extensions.AI; /// A delegating chat client that logs chat operations to an . +/// /// /// The provided implementation of is thread-safe for concurrent use so long as the /// employed is also thread-safe for concurrent use. /// +/// +/// When the employed enables , the contents of +/// chat messages and options are logged. These messages and options may contain sensitive application data. +/// is disabled by default and should never be enabled in a production environment. +/// Messages and options are not logged at other logging levels. +/// +/// public partial class LoggingChatClient : DelegatingChatClient { /// An instance used for all logging. @@ -145,16 +153,9 @@ public override async IAsyncEnumerable GetStreamingResponseA throw; } - if (_logger.IsEnabled(LogLevel.Debug)) + if (_logger.IsEnabled(LogLevel.Trace)) { - if (_logger.IsEnabled(LogLevel.Trace)) - { - LogStreamingUpdateSensitive(AsJson(update)); - } - else - { - LogStreamingUpdate(); - } + LogStreamingUpdateSensitive(AsJson(update)); } yield return update; @@ -182,9 +183,6 @@ public override async IAsyncEnumerable GetStreamingResponseA [LoggerMessage(LogLevel.Trace, "{MethodName} completed: {ChatResponse}.")] private partial void LogCompletedSensitive(string methodName, string chatResponse); - [LoggerMessage(LogLevel.Debug, "GetStreamingResponseAsync received update.")] - private partial void LogStreamingUpdate(); - [LoggerMessage(LogLevel.Trace, "GetStreamingResponseAsync received update: {ChatResponseUpdate}")] private partial void LogStreamingUpdateSensitive(string chatResponseUpdate); diff --git a/src/Libraries/Microsoft.Extensions.AI/ChatCompletion/LoggingChatClientBuilderExtensions.cs b/src/Libraries/Microsoft.Extensions.AI/ChatCompletion/LoggingChatClientBuilderExtensions.cs index d34716ed886..e2759b6b0a6 100644 --- a/src/Libraries/Microsoft.Extensions.AI/ChatCompletion/LoggingChatClientBuilderExtensions.cs +++ b/src/Libraries/Microsoft.Extensions.AI/ChatCompletion/LoggingChatClientBuilderExtensions.cs @@ -21,6 +21,14 @@ public static class LoggingChatClientBuilderExtensions /// An optional callback that can be used to configure the instance. /// The . /// is . + /// + /// + /// When the employed enables , the contents of + /// chat messages and options are logged. These messages and options may contain sensitive application data. + /// is disabled by default and should never be enabled in a production environment. + /// Messages and options are not logged at other logging levels. + /// + /// public static ChatClientBuilder UseLogging( this ChatClientBuilder builder, ILoggerFactory? loggerFactory = null, diff --git a/src/Libraries/Microsoft.Extensions.AI/Embeddings/LoggingEmbeddingGenerator.cs b/src/Libraries/Microsoft.Extensions.AI/Embeddings/LoggingEmbeddingGenerator.cs index 924ee362633..97e5beb2c42 100644 --- a/src/Libraries/Microsoft.Extensions.AI/Embeddings/LoggingEmbeddingGenerator.cs +++ b/src/Libraries/Microsoft.Extensions.AI/Embeddings/LoggingEmbeddingGenerator.cs @@ -14,10 +14,18 @@ namespace Microsoft.Extensions.AI; /// A delegating embedding generator that logs embedding generation operations to an . /// Specifies the type of the input passed to the generator. /// Specifies the type of the embedding instance produced by the generator. +/// /// /// The provided implementation of is thread-safe for concurrent use /// so long as the employed is also thread-safe for concurrent use. /// +/// +/// When the employed enables , the contents of +/// values and options are logged. These values and options may contain sensitive application data. +/// is disabled by default and should never be enabled in a production environment. +/// Messages and options are not logged at other logging levels. +/// +/// public partial class LoggingEmbeddingGenerator : DelegatingEmbeddingGenerator where TEmbedding : Embedding { diff --git a/src/Libraries/Microsoft.Extensions.AI/Embeddings/LoggingEmbeddingGeneratorBuilderExtensions.cs b/src/Libraries/Microsoft.Extensions.AI/Embeddings/LoggingEmbeddingGeneratorBuilderExtensions.cs index eb472fb1e0e..a7afbdeed85 100644 --- a/src/Libraries/Microsoft.Extensions.AI/Embeddings/LoggingEmbeddingGeneratorBuilderExtensions.cs +++ b/src/Libraries/Microsoft.Extensions.AI/Embeddings/LoggingEmbeddingGeneratorBuilderExtensions.cs @@ -23,6 +23,14 @@ public static class LoggingEmbeddingGeneratorBuilderExtensions /// An optional callback that can be used to configure the instance. /// The . /// is . + /// + /// + /// When the employed enables , the contents of + /// values and options are logged. These values and options may contain sensitive application data. + /// is disabled by default and should never be enabled in a production environment. + /// Messages and options are not logged at other logging levels. + /// + /// public static EmbeddingGeneratorBuilder UseLogging( this EmbeddingGeneratorBuilder builder, ILoggerFactory? loggerFactory = null, diff --git a/src/Libraries/Microsoft.Extensions.AI/Functions/AIFunctionFactory.Utilities.cs b/src/Libraries/Microsoft.Extensions.AI/Functions/AIFunctionFactory.Utilities.cs deleted file mode 100644 index cbafe78e5d3..00000000000 --- a/src/Libraries/Microsoft.Extensions.AI/Functions/AIFunctionFactory.Utilities.cs +++ /dev/null @@ -1,137 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System; -using System.Buffers; -using System.IO; -using System.Reflection; -using System.Text.RegularExpressions; -using Microsoft.Shared.Diagnostics; - -namespace Microsoft.Extensions.AI; - -public static partial class AIFunctionFactory -{ - /// - /// Removes characters from a .NET member name that shouldn't be used in an AI function name. - /// - /// The .NET member name that should be sanitized. - /// - /// Replaces non-alphanumeric characters in the identifier with the underscore character. - /// Primarily intended to remove characters produced by compiler-generated method name mangling. - /// - internal static string SanitizeMemberName(string memberName) - { - _ = Throw.IfNull(memberName); - return InvalidNameCharsRegex().Replace(memberName, "_"); - } - - /// Regex that flags any character other than ASCII digits or letters or the underscore. -#if NET - [GeneratedRegex("[^0-9A-Za-z_]")] - private static partial Regex InvalidNameCharsRegex(); -#else - private static Regex InvalidNameCharsRegex() => _invalidNameCharsRegex; - private static readonly Regex _invalidNameCharsRegex = new("[^0-9A-Za-z_]", RegexOptions.Compiled); -#endif - - /// Invokes the MethodInfo with the specified target object and arguments. - private static object? ReflectionInvoke(MethodInfo method, object? target, object?[]? arguments) - { -#if NET - return method.Invoke(target, BindingFlags.DoNotWrapExceptions, binder: null, arguments, culture: null); -#else - try - { - return method.Invoke(target, BindingFlags.Default, binder: null, arguments, culture: null); - } - catch (TargetInvocationException e) when (e.InnerException is not null) - { - // If we're targeting .NET Framework, such that BindingFlags.DoNotWrapExceptions - // is ignored, the original exception will be wrapped in a TargetInvocationException. - // Unwrap it and throw that original exception, maintaining its stack information. - System.Runtime.ExceptionServices.ExceptionDispatchInfo.Capture(e.InnerException).Throw(); - throw; - } -#endif - } - - /// - /// Implements a simple write-only memory stream that uses pooled buffers. - /// - private sealed class PooledMemoryStream : Stream - { - private const int DefaultBufferSize = 4096; - private byte[] _buffer; - private int _position; - - public PooledMemoryStream(int initialCapacity = DefaultBufferSize) - { - _buffer = ArrayPool.Shared.Rent(initialCapacity); - _position = 0; - } - - public ReadOnlySpan GetBuffer() => _buffer.AsSpan(0, _position); - public override bool CanWrite => true; - public override bool CanRead => false; - public override bool CanSeek => false; - public override long Length => _position; - public override long Position - { - get => _position; - set => throw new NotSupportedException(); - } - - public override void Write(byte[] buffer, int offset, int count) - { - EnsureNotDisposed(); - EnsureCapacity(_position + count); - - Buffer.BlockCopy(buffer, offset, _buffer, _position, count); - _position += count; - } - - public override void Flush() - { - } - - public override int Read(byte[] buffer, int offset, int count) => throw new NotSupportedException(); - public override long Seek(long offset, SeekOrigin origin) => throw new NotSupportedException(); - public override void SetLength(long value) => throw new NotSupportedException(); - - protected override void Dispose(bool disposing) - { - if (_buffer is not null) - { - ArrayPool.Shared.Return(_buffer); - _buffer = null!; - } - - base.Dispose(disposing); - } - - private void EnsureCapacity(int requiredCapacity) - { - if (requiredCapacity <= _buffer.Length) - { - return; - } - - int newCapacity = Math.Max(requiredCapacity, _buffer.Length * 2); - byte[] newBuffer = ArrayPool.Shared.Rent(newCapacity); - Buffer.BlockCopy(_buffer, 0, newBuffer, 0, _position); - - ArrayPool.Shared.Return(_buffer); - _buffer = newBuffer; - } - - private void EnsureNotDisposed() - { - if (_buffer is null) - { - Throw(); - static void Throw() => throw new ObjectDisposedException(nameof(PooledMemoryStream)); - } - } - } -} diff --git a/src/Libraries/Microsoft.Extensions.AI/SpeechToText/LoggingSpeechToTextClient.cs b/src/Libraries/Microsoft.Extensions.AI/SpeechToText/LoggingSpeechToTextClient.cs index 6c5bf0ed929..e7bf7850a94 100644 --- a/src/Libraries/Microsoft.Extensions.AI/SpeechToText/LoggingSpeechToTextClient.cs +++ b/src/Libraries/Microsoft.Extensions.AI/SpeechToText/LoggingSpeechToTextClient.cs @@ -15,10 +15,18 @@ namespace Microsoft.Extensions.AI; /// A delegating speech to text client that logs speech to text operations to an . +/// /// /// The provided implementation of is thread-safe for concurrent use so long as the /// employed is also thread-safe for concurrent use. /// +/// +/// When the employed enables , the contents of +/// messages and options are logged. These messages and options may contain sensitive application data. +/// is disabled by default and should never be enabled in a production environment. +/// Messages and options are not logged at other logging levels. +/// +/// [Experimental("MEAI001")] public partial class LoggingSpeechToTextClient : DelegatingSpeechToTextClient { diff --git a/src/Libraries/Microsoft.Extensions.AI/SpeechToText/SpeechToTextClientBuilderExtensions.cs b/src/Libraries/Microsoft.Extensions.AI/SpeechToText/LoggingSpeechToTextClientBuilderExtensions.cs similarity index 79% rename from src/Libraries/Microsoft.Extensions.AI/SpeechToText/SpeechToTextClientBuilderExtensions.cs rename to src/Libraries/Microsoft.Extensions.AI/SpeechToText/LoggingSpeechToTextClientBuilderExtensions.cs index 7ce2b19ac37..92a67189982 100644 --- a/src/Libraries/Microsoft.Extensions.AI/SpeechToText/SpeechToTextClientBuilderExtensions.cs +++ b/src/Libraries/Microsoft.Extensions.AI/SpeechToText/LoggingSpeechToTextClientBuilderExtensions.cs @@ -12,7 +12,7 @@ namespace Microsoft.Extensions.AI; /// Provides extensions for configuring instances. [Experimental("MEAI001")] -public static class SpeechToTextClientBuilderExtensions +public static class LoggingSpeechToTextClientBuilderExtensions { /// Adds logging to the audio transcription client pipeline. /// The . @@ -22,6 +22,14 @@ public static class SpeechToTextClientBuilderExtensions /// /// An optional callback that can be used to configure the instance. /// The . + /// + /// + /// When the employed enables , the contents of + /// messages and options are logged. These messages and options may contain sensitive application data. + /// is disabled by default and should never be enabled in a production environment. + /// Messages and options are not logged at other logging levels. + /// + /// public static SpeechToTextClientBuilder UseLogging( this SpeechToTextClientBuilder builder, ILoggerFactory? loggerFactory = null, diff --git a/src/ProjectTemplates/GeneratedContent.targets b/src/ProjectTemplates/GeneratedContent.targets index 4767af46a31..5842d5cad74 100644 --- a/src/ProjectTemplates/GeneratedContent.targets +++ b/src/ProjectTemplates/GeneratedContent.targets @@ -15,16 +15,17 @@ - 9.4.0 - 9.4.0-preview.1.25207.5 - 9.0.4 + 9.5.0 + 9.5.0-preview.1.25262.9 + 9.0.5 - false + true + false $(TemplatePinnedRepoPackagesVersion) - $(TemplatePinnedRepoAIPackagesVersion) + $(TemplatePinnedRepoAIPackagesVersion) $(TemplatePinnedMicrosoftEntityFrameworkCoreSqliteVersion) diff --git a/src/ProjectTemplates/Microsoft.Extensions.AI.Templates/src/ChatWithCustomData/ChatWithCustomData-CSharp.Web/README.md b/src/ProjectTemplates/Microsoft.Extensions.AI.Templates/src/ChatWithCustomData/ChatWithCustomData-CSharp.Web/README.md index 44decea800c..37f10b83ce2 100644 --- a/src/ProjectTemplates/Microsoft.Extensions.AI.Templates/src/ChatWithCustomData/ChatWithCustomData-CSharp.Web/README.md +++ b/src/ProjectTemplates/Microsoft.Extensions.AI.Templates/src/ChatWithCustomData/ChatWithCustomData-CSharp.Web/README.md @@ -9,6 +9,16 @@ This project is an AI chat application that demonstrates how to chat with custom ### Prerequisites To use Azure OpenAI or Azure AI Search, you need an Azure account. If you don't already have one, [create an Azure account](https://azure.microsoft.com/free/). +#### ---#endif +#### ---#if (IsOllama) +### Known Issues + +#### Errors running Ollama or Docker + +A recent incompatibility was found between Ollama and Docker Desktop. This issue results in runtime errors when connecting to Ollama, and the workaround for that can lead to Docker not working for Aspire projects. + +This incompatibility can be addressed by upgrading to Docker Desktop 4.41.1. See [ollama/ollama#9509](https://github.com/ollama/ollama/issues/9509#issuecomment-2842461831) for more information and a link to install the version of Docker Desktop with the fix. + #### ---#endif # Configure the AI Model Provider #### ---#if (IsGHModels) diff --git a/src/ProjectTemplates/Microsoft.Extensions.AI.Templates/src/ChatWithCustomData/README.Aspire.md b/src/ProjectTemplates/Microsoft.Extensions.AI.Templates/src/ChatWithCustomData/README.Aspire.md index d1677b7ba78..f7c944dacc8 100644 --- a/src/ProjectTemplates/Microsoft.Extensions.AI.Templates/src/ChatWithCustomData/README.Aspire.md +++ b/src/ProjectTemplates/Microsoft.Extensions.AI.Templates/src/ChatWithCustomData/README.Aspire.md @@ -10,17 +10,14 @@ This project is an AI chat application that demonstrates how to chat with custom To use Azure OpenAI or Azure AI Search, you need an Azure account. If you don't already have one, [create an Azure account](https://azure.microsoft.com/free/). #### ---#endif -#### ---#if (UseQdrant) ### Known Issues -#### Errors After Updating to Aspire Version 9.2.0 -This project is not currently compatible with Aspire 9.2.0, and all Aspire package versions are set to 9.1.0. Updating [Aspire.Qdrant.Client](https://www.nuget.org/packages/Aspire.Qdrant.Client) to version 9.2.0 causes an incompatibility with [Microsoft.SemanticKernel.Connectors.Qdrant](https://www.nuget.org/packages/Microsoft.SemanticKernel.Connectors.Qdrant) where different versions of [Qdrant.Client](https://www.nuget.org/packages/Qdrant.Client) are required. Attempting to run the project with `Aspire.Qdrant.Client` version 9.2.0 will result in the following exception: +#### Errors running Ollama or Docker -> System.MissingMethodException: Method not found: 'Qdrant.Client.Grpc.Vectors Qdrant.Client.Grpc.ScoredPoint.get_Vectors()' +A recent incompatibility was found between Ollama and Docker Desktop. This issue results in runtime errors when connecting to Ollama, and the workaround for that can lead to Docker not working for Aspire projects. -Once a version of `Microsoft.SemanticKernel.Connectors.Qdrant` is published with a dependency on `Qdrant.Client` version `>= 1.13.0`, the Aspire packages can also be updated to version 9.2.0. +This incompatibility can be addressed by upgrading to Docker Desktop 4.41.1. See [ollama/ollama#9509](https://github.com/ollama/ollama/issues/9509#issuecomment-2842461831) for more information and a link to install the version of Docker Desktop with the fix. -#### ---#endif # Configure the AI Model Provider #### ---#if (IsGHModels) diff --git a/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/ChatCompletion/ChatOptionsTests.cs b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/ChatCompletion/ChatOptionsTests.cs index 67bbfb6d3db..cdf1aab09c9 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/ChatCompletion/ChatOptionsTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/ChatCompletion/ChatOptionsTests.cs @@ -1,6 +1,7 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. +using System; using System.Collections.Generic; using System.Text.Json; using Xunit; @@ -28,6 +29,7 @@ public void Constructor_Parameterless_PropsDefaulted() Assert.Null(options.ToolMode); Assert.Null(options.Tools); Assert.Null(options.AdditionalProperties); + Assert.Null(options.RawRepresentationFactory); ChatOptions clone = options.Clone(); Assert.Null(clone.ConversationId); @@ -45,6 +47,7 @@ public void Constructor_Parameterless_PropsDefaulted() Assert.Null(clone.ToolMode); Assert.Null(clone.Tools); Assert.Null(clone.AdditionalProperties); + Assert.Null(clone.RawRepresentationFactory); } [Fact] @@ -69,6 +72,8 @@ public void Properties_Roundtrip() ["key"] = "value", }; + Func rawRepresentationFactory = (c) => null; + options.ConversationId = "12345"; options.Temperature = 0.1f; options.MaxOutputTokens = 2; @@ -83,6 +88,7 @@ public void Properties_Roundtrip() options.AllowMultipleToolCalls = true; options.ToolMode = ChatToolMode.RequireAny; options.Tools = tools; + options.RawRepresentationFactory = rawRepresentationFactory; options.AdditionalProperties = additionalProps; Assert.Equal("12345", options.ConversationId); @@ -99,6 +105,7 @@ public void Properties_Roundtrip() Assert.True(options.AllowMultipleToolCalls); Assert.Same(ChatToolMode.RequireAny, options.ToolMode); Assert.Same(tools, options.Tools); + Assert.Same(rawRepresentationFactory, options.RawRepresentationFactory); Assert.Same(additionalProps, options.AdditionalProperties); ChatOptions clone = options.Clone(); @@ -116,6 +123,7 @@ public void Properties_Roundtrip() Assert.True(clone.AllowMultipleToolCalls); Assert.Same(ChatToolMode.RequireAny, clone.ToolMode); Assert.Equal(tools, clone.Tools); + Assert.Same(rawRepresentationFactory, clone.RawRepresentationFactory); Assert.Equal(additionalProps, clone.AdditionalProperties); } @@ -153,6 +161,7 @@ public void JsonSerialization_Roundtrips() AIFunctionFactory.Create(() => 42), AIFunctionFactory.Create(() => 43), ]; + options.RawRepresentationFactory = (c) => null; options.AdditionalProperties = additionalProps; string json = JsonSerializer.Serialize(options, TestJsonSerializerContext.Default.ChatOptions); @@ -175,6 +184,7 @@ public void JsonSerialization_Roundtrips() Assert.False(deserialized.AllowMultipleToolCalls); Assert.Equal(ChatToolMode.RequireAny, deserialized.ToolMode); Assert.Null(deserialized.Tools); + Assert.Null(deserialized.RawRepresentationFactory); Assert.NotNull(deserialized.AdditionalProperties); Assert.Single(deserialized.AdditionalProperties); diff --git a/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Embeddings/BinaryEmbeddingTests.cs b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Embeddings/BinaryEmbeddingTests.cs new file mode 100644 index 00000000000..c75d715466e --- /dev/null +++ b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Embeddings/BinaryEmbeddingTests.cs @@ -0,0 +1,95 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections; +using System.Linq; +using System.Text.Json; +using Xunit; + +namespace Microsoft.Extensions.AI; + +public class BinaryEmbeddingTests +{ + [Fact] + public void Ctor_Roundtrips() + { + BitArray vector = new BitArray(new bool[] { false, true, false, true }); + + BinaryEmbedding e = new(vector); + Assert.Same(vector, e.Vector); + Assert.Null(e.ModelId); + Assert.Null(e.CreatedAt); + Assert.Null(e.AdditionalProperties); + } + + [Fact] + public void Properties_Roundtrips() + { + BitArray vector = new BitArray(new bool[] { false, true, false, true }); + + BinaryEmbedding e = new(vector); + + Assert.Same(vector, e.Vector); + BitArray newVector = new BitArray(new bool[] { true, false, true, false }); + e.Vector = newVector; + Assert.Same(newVector, e.Vector); + + Assert.Null(e.ModelId); + e.ModelId = "text-embedding-3-small"; + Assert.Equal("text-embedding-3-small", e.ModelId); + + Assert.Null(e.CreatedAt); + DateTimeOffset createdAt = DateTimeOffset.Parse("2022-01-01T00:00:00Z"); + e.CreatedAt = createdAt; + Assert.Equal(createdAt, e.CreatedAt); + + Assert.Null(e.AdditionalProperties); + AdditionalPropertiesDictionary props = new(); + e.AdditionalProperties = props; + Assert.Same(props, e.AdditionalProperties); + } + + [Fact] + public void Serialization_Roundtrips() + { + foreach (int length in Enumerable.Range(0, 64).Concat(new[] { 10_000 })) + { + bool[] bools = new bool[length]; + Random r = new(42); + for (int i = 0; i < length; i++) + { + bools[i] = r.Next(2) != 0; + } + + BitArray vector = new BitArray(bools); + BinaryEmbedding e = new(vector); + + string json = JsonSerializer.Serialize(e, TestJsonSerializerContext.Default.Embedding); + Assert.Equal($$"""{"$type":"binary","vector":"{{string.Concat(vector.Cast().Select(b => b ? '1' : '0'))}}"}""", json); + + BinaryEmbedding result = Assert.IsType(JsonSerializer.Deserialize(json, TestJsonSerializerContext.Default.Embedding)); + Assert.Equal(e.Vector, result.Vector); + } + } + + [Fact] + public void Derialization_SupportsEncodedBits() + { + BinaryEmbedding result = Assert.IsType(JsonSerializer.Deserialize( + """{"$type":"binary","vector":"\u0030\u0031\u0030\u0031\u0030\u0031"}""", + TestJsonSerializerContext.Default.Embedding)); + + Assert.Equal(new BitArray(new[] { false, true, false, true, false, true }), result.Vector); + } + + [Theory] + [InlineData("""{"$type":"binary","vector":"\u0030\u0032"}""")] + [InlineData("""{"$type":"binary","vector":"02"}""")] + [InlineData("""{"$type":"binary","vector":" "}""")] + [InlineData("""{"$type":"binary","vector":10101}""")] + public void Derialization_InvalidBinaryEmbedding_Throws(string json) + { + Assert.Throws(() => JsonSerializer.Deserialize(json, TestJsonSerializerContext.Default.Embedding)); + } +} diff --git a/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Embeddings/EmbeddingTests.cs b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Embeddings/EmbeddingTests.cs index 45fcce8ba63..c3809782006 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Embeddings/EmbeddingTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Embeddings/EmbeddingTests.cs @@ -14,7 +14,7 @@ public class EmbeddingTests public void Embedding_Ctor_Roundtrips() { float[] floats = [1f, 2f, 3f]; - UsageDetails usage = new(); + AdditionalPropertiesDictionary props = []; var createdAt = DateTimeOffset.Parse("2022-01-01T00:00:00Z"); const string Model = "text-embedding-3-small"; @@ -35,6 +35,32 @@ public void Embedding_Ctor_Roundtrips() Assert.Same(floats, array.Array); } + [Fact] + public void Embedding_Byte_SerializationRoundtrips() + { + byte[] bytes = [1, 2, 3]; + Embedding e = new(bytes); + + string json = JsonSerializer.Serialize(e, TestJsonSerializerContext.Default.Embedding); + Assert.Equal("""{"$type":"uint8","vector":"AQID"}""", json); + + Embedding result = Assert.IsType>(JsonSerializer.Deserialize(json, TestJsonSerializerContext.Default.Embedding)); + Assert.Equal(e.Vector.ToArray(), result.Vector.ToArray()); + } + + [Fact] + public void Embedding_SByte_SerializationRoundtrips() + { + sbyte[] bytes = [1, 2, 3]; + Embedding e = new(bytes); + + string json = JsonSerializer.Serialize(e, TestJsonSerializerContext.Default.Embedding); + Assert.Equal("""{"$type":"int8","vector":[1,2,3]}""", json); + + Embedding result = Assert.IsType>(JsonSerializer.Deserialize(json, TestJsonSerializerContext.Default.Embedding)); + Assert.Equal(e.Vector.ToArray(), result.Vector.ToArray()); + } + #if NET [Fact] public void Embedding_Half_SerializationRoundtrips() @@ -43,7 +69,7 @@ public void Embedding_Half_SerializationRoundtrips() Embedding e = new(halfs); string json = JsonSerializer.Serialize(e, TestJsonSerializerContext.Default.Embedding); - Assert.Equal("""{"$type":"halves","vector":[1,2,3]}""", json); + Assert.Equal("""{"$type":"float16","vector":[1,2,3]}""", json); Embedding result = Assert.IsType>(JsonSerializer.Deserialize(json, TestJsonSerializerContext.Default.Embedding)); Assert.Equal(e.Vector.ToArray(), result.Vector.ToArray()); @@ -57,7 +83,7 @@ public void Embedding_Single_SerializationRoundtrips() Embedding e = new(floats); string json = JsonSerializer.Serialize(e, TestJsonSerializerContext.Default.Embedding); - Assert.Equal("""{"$type":"floats","vector":[1,2,3]}""", json); + Assert.Equal("""{"$type":"float32","vector":[1,2,3]}""", json); Embedding result = Assert.IsType>(JsonSerializer.Deserialize(json, TestJsonSerializerContext.Default.Embedding)); Assert.Equal(e.Vector.ToArray(), result.Vector.ToArray()); @@ -70,7 +96,7 @@ public void Embedding_Double_SerializationRoundtrips() Embedding e = new(floats); string json = JsonSerializer.Serialize(e, TestJsonSerializerContext.Default.Embedding); - Assert.Equal("""{"$type":"doubles","vector":[1,2,3]}""", json); + Assert.Equal("""{"$type":"float64","vector":[1,2,3]}""", json); Embedding result = Assert.IsType>(JsonSerializer.Deserialize(json, TestJsonSerializerContext.Default.Embedding)); Assert.Equal(e.Vector.ToArray(), result.Vector.ToArray()); diff --git a/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Utilities/AIJsonSchemaTransformCacheTests.cs b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Utilities/AIJsonSchemaTransformCacheTests.cs new file mode 100644 index 00000000000..4233e5cdbe1 --- /dev/null +++ b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Utilities/AIJsonSchemaTransformCacheTests.cs @@ -0,0 +1,81 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Text.Json; +using Xunit; + +namespace Microsoft.Extensions.AI.Utilities; + +public static class AIJsonSchemaTransformCacheTests +{ + [Fact] + public static void NullOptions_ThrowsArgumentNullException() + { + Assert.Throws(() => new AIJsonSchemaTransformCache(transformOptions: null!)); + } + + [Fact] + public static void EmptyOptions_ThrowsArgumentException() + { + Assert.Throws(() => new AIJsonSchemaTransformCache(transformOptions: new())); + } + + [Fact] + public static void TransformOptions_ReturnsExpectedValue() + { + AIJsonSchemaTransformOptions options = new() { ConvertBooleanSchemas = true }; + AIJsonSchemaTransformCache cache = new(options); + Assert.Same(options, cache.TransformOptions); + } + + [Fact] + public static void NullFunction_ThrowsArgumentNullException() + { + AIJsonSchemaTransformCache cache = new(new() { ConvertBooleanSchemas = true }); + Assert.Throws(() => cache.GetOrCreateTransformedSchema(function: null!)); + } + + [Fact] + public static void NullResponseFormat_ThrowsArgumentNullException() + { + AIJsonSchemaTransformCache cache = new(new() { ConvertBooleanSchemas = true }); + Assert.Throws(() => cache.GetOrCreateTransformedSchema(responseFormat: null!)); + } + + [Fact] + public static void FunctionSchema_ReturnsExpectedResults() + { + AIJsonSchemaTransformCache cache = new(new() { TransformSchemaNode = (_, node) => { node.AsObject().Add("myAwesomeKeyword", 42); return node; } }); + + AIFunction func = AIFunctionFactory.Create((int x, int y) => x + y); + JsonElement transformedSchema = cache.GetOrCreateTransformedSchema(func); + Assert.True(transformedSchema.TryGetProperty("myAwesomeKeyword", out _)); + + JsonElement transformedSchema2 = cache.GetOrCreateTransformedSchema(func); + Assert.Equal(transformedSchema, transformedSchema2); + } + + [Fact] + public static void ChatResponseFormat_ReturnsExpectedResults() + { + AIJsonSchemaTransformCache cache = new(new() { TransformSchemaNode = (_, node) => { node.AsObject().Add("myAwesomeKeyword", 42); return node; } }); + + JsonElement schema = JsonDocument.Parse("{}").RootElement; + ChatResponseFormatJson responseFormat = ChatResponseFormat.ForJsonSchema(schema); + JsonElement? transformedSchema = cache.GetOrCreateTransformedSchema(responseFormat); + Assert.NotNull(transformedSchema); + Assert.True(transformedSchema.Value.TryGetProperty("myAwesomeKeyword", out _)); + + JsonElement? transformedSchema2 = cache.GetOrCreateTransformedSchema(responseFormat); + Assert.Equal(transformedSchema, transformedSchema2); + } + + [Fact] + public static void ChatResponseFormat_NullFormatReturnsNullSchema() + { + AIJsonSchemaTransformCache cache = new(new() { TransformSchemaNode = (_, node) => { node.AsObject().Add("myAwesomeKeyword", 42); return node; } }); + JsonElement? transformedSchema = cache.GetOrCreateTransformedSchema(ChatResponseFormat.Json); + Assert.Null(transformedSchema); + } +} diff --git a/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Utilities/AIJsonUtilitiesTests.cs b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Utilities/AIJsonUtilitiesTests.cs index c4141a4bf0d..0001b8b2125 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Utilities/AIJsonUtilitiesTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Utilities/AIJsonUtilitiesTests.cs @@ -15,6 +15,8 @@ using Microsoft.Extensions.AI.JsonSchemaExporter; using Xunit; +#pragma warning disable 0618 // Suppress obsolete warnings + namespace Microsoft.Extensions.AI; public static partial class AIJsonUtilitiesTests @@ -72,10 +74,11 @@ public static void AIJsonSchemaCreateOptions_DefaultInstance_ReturnsExpectedValu { AIJsonSchemaCreateOptions options = useSingleton ? AIJsonSchemaCreateOptions.Default : new AIJsonSchemaCreateOptions(); Assert.True(options.IncludeTypeInEnumSchemas); - Assert.True(options.DisallowAdditionalProperties); + Assert.False(options.DisallowAdditionalProperties); Assert.False(options.IncludeSchemaKeyword); Assert.False(options.RequireAllProperties); Assert.Null(options.TransformSchemaNode); + Assert.Null(options.TransformOptions); } [Fact] @@ -106,6 +109,12 @@ public static void AIJsonSchemaCreateOptions_UsesStructuralEquality() property.SetValue(options2, includeParameter); break; + case null when property.PropertyType == typeof(AIJsonSchemaTransformOptions): + AIJsonSchemaTransformOptions transformOptions = new AIJsonSchemaTransformOptions { RequireAllProperties = true }; + property.SetValue(options1, transformOptions); + property.SetValue(options2, transformOptions); + break; + default: Assert.Fail($"Unexpected property type: {property.PropertyType}"); break; @@ -152,8 +161,7 @@ public static void CreateJsonSchema_DefaultParameters_GeneratesExpectedJsonSchem "default": "defaultValue" } }, - "required": ["Key", "EnumValue"], - "additionalProperties": false + "required": ["Key", "EnumValue"] } """).RootElement; @@ -176,6 +184,7 @@ public static void CreateJsonSchema_OverriddenParameters_GeneratesExpectedJsonSc "type": "integer" }, "EnumValue": { + "type": "string", "enum": ["A", "B"] }, "Value": { @@ -183,16 +192,20 @@ public static void CreateJsonSchema_OverriddenParameters_GeneratesExpectedJsonSc "type": ["string", "null"] } }, - "required": ["Key", "EnumValue", "Value"] + "required": ["Key", "EnumValue", "Value"], + "additionalProperties": false } """).RootElement; AIJsonSchemaCreateOptions inferenceOptions = new AIJsonSchemaCreateOptions { - IncludeTypeInEnumSchemas = false, - DisallowAdditionalProperties = false, IncludeSchemaKeyword = true, - RequireAllProperties = true, + TransformOptions = new() + { + DisallowAdditionalProperties = true, + RequireAllProperties = true, + MoveDefaultKeywordToDescription = true, + } }; JsonElement actual = AIJsonUtilities.CreateJsonSchema( @@ -227,8 +240,7 @@ public static void CreateJsonSchema_UserDefinedTransformer() "default": "defaultValue" } }, - "required": ["Key", "EnumValue"], - "additionalProperties": false + "required": ["Key", "EnumValue"] } """).RootElement; @@ -268,8 +280,7 @@ public static void CreateJsonSchema_FiltersDisallowedKeywords() "Char" : { "type": "string" } - }, - "additionalProperties": false + } } """).RootElement; @@ -341,6 +352,15 @@ public static void CreateFunctionJsonSchema_OptionalParameters(bool requireAllPr } """).RootElement; + AIJsonSchemaCreateOptions inferenceOptions = new() + { + TransformOptions = new() + { + RequireAllProperties = requireAllProperties, + MoveDefaultKeywordToDescription = requireAllProperties, + } + }; + AIFunction func = AIFunctionFactory.Create(( [Description("The city to get the weather for")] string city, [Description("The unit to calculate the current temperature to")] string unit = "celsius") => "sunny", @@ -348,7 +368,7 @@ public static void CreateFunctionJsonSchema_OptionalParameters(bool requireAllPr { Name = "get_weather", Description = "Gets the current weather for a current location", - JsonSchemaCreateOptions = new AIJsonSchemaCreateOptions { RequireAllProperties = requireAllProperties } + JsonSchemaCreateOptions = inferenceOptions }); Assert.NotNull(func.UnderlyingMethod); @@ -358,7 +378,7 @@ public static void CreateFunctionJsonSchema_OptionalParameters(bool requireAllPr func.UnderlyingMethod, title: func.Name, description: func.Description, - inferenceOptions: new AIJsonSchemaCreateOptions { RequireAllProperties = requireAllProperties }); + inferenceOptions: inferenceOptions); AssertDeepEquals(expected, resolvedSchema); } @@ -423,7 +443,7 @@ public static void CreateJsonSchema_ValidateWithTestData(ITestData testData) JsonTypeInfo typeInfo = options.GetTypeInfo(testData.Type); AIJsonSchemaCreateOptions? createOptions = typeInfo.Properties.Any(prop => prop.IsExtensionData) - ? new() { DisallowAdditionalProperties = false } // Do not append additionalProperties: false to the schema if the type has extension data. + ? new() { TransformOptions = new() { DisallowAdditionalProperties = false } } // Do not append additionalProperties: false to the schema if the type has extension data. : null; JsonElement schema = AIJsonUtilities.CreateJsonSchema(testData.Type, serializerOptions: options, inferenceOptions: createOptions); @@ -566,6 +586,258 @@ public static void CreateFunctionJsonSchema_InvokesIncludeParameterCallbackForEv Assert.Contains("fifth", schemaString); } + [Fact] + public static void TransformJsonSchema_ConvertBooleanSchemas() + { + JsonElement schema = JsonDocument.Parse(""" + { + "properties" : { + "foo": { "type": "string" }, + "bar": { "properties": { "x": false } }, + "baz": true + }, + "required": ["foo"] + } + """).RootElement; + + JsonElement expectedSchema = JsonDocument.Parse(""" + { + "properties" : { + "foo": { "type": "string" }, + "bar": { "properties": { "x": { "not": true } } }, + "baz": { } + }, + "required": ["foo"] + } + """).RootElement; + + AIJsonSchemaTransformOptions options = new() + { + ConvertBooleanSchemas = true, + }; + + JsonElement transformedSchema = AIJsonUtilities.TransformSchema(schema, options); + AssertDeepEquals(expectedSchema, transformedSchema); + } + + [Fact] + public static void TransformJsonSchema_RequireAllProperties() + { + JsonElement schema = JsonDocument.Parse(""" + { + "properties" : { + "foo": { "type": "string" }, + "bar": { "properties": { "x": false } }, + "baz": true + }, + "required": ["foo"] + } + """).RootElement; + + JsonElement expectedSchema = JsonDocument.Parse(""" + { + "properties" : { + "foo": { "type": "string" }, + "bar": { "properties": { "x": false }, "required": ["x"] }, + "baz": true + }, + "required": ["foo", "bar", "baz"] + } + """).RootElement; + + AIJsonSchemaTransformOptions options = new() + { + RequireAllProperties = true, + }; + + JsonElement transformedSchema = AIJsonUtilities.TransformSchema(schema, options); + AssertDeepEquals(expectedSchema, transformedSchema); + } + + [Fact] + public static void TransformJsonSchema_DisallowAdditionalProperties() + { + JsonElement schema = JsonDocument.Parse(""" + { + "properties" : { + "foo": { "type": "string" }, + "bar": { "properties": { "x": false } }, + "baz": true + }, + "required": ["foo"] + } + """).RootElement; + + JsonElement expectedSchema = JsonDocument.Parse(""" + { + "properties" : { + "foo": { "type": "string" }, + "bar": { "properties": { "x": false }, "additionalProperties": false }, + "baz": true + }, + "required": ["foo"], + "additionalProperties": false + } + """).RootElement; + + AIJsonSchemaTransformOptions options = new() + { + DisallowAdditionalProperties = true, + }; + + JsonElement transformedSchema = AIJsonUtilities.TransformSchema(schema, options); + AssertDeepEquals(expectedSchema, transformedSchema); + } + + [Fact] + public static void TransformJsonSchema_UseNullableKeyword() + { + JsonElement schema = JsonDocument.Parse(""" + { + "type": ["object","null"], + "properties" : { + "foo": { "type": ["null","string"] }, + "bar": { "type":"object", "properties": { "x": false } }, + "baz": { "type" : ["string","array","null"] } + }, + "required": ["foo"] + } + """).RootElement; + + JsonElement expectedSchema = JsonDocument.Parse(""" + { + "type": "object", + "properties" : { + "foo": { "type": "string", "nullable": true }, + "bar": { "type":"object", "properties": { "x": false } }, + "baz": { "type" : ["string","array","null"] } + }, + "required": ["foo"], + "nullable": true + } + """).RootElement; + + AIJsonSchemaTransformOptions options = new() + { + UseNullableKeyword = true, + }; + + JsonElement transformedSchema = AIJsonUtilities.TransformSchema(schema, options); + AssertDeepEquals(expectedSchema, transformedSchema); + } + + [Fact] + public static void TransformJsonSchema_MoveDefaultKeywordToDescription() + { + JsonElement schema = JsonDocument.Parse(""" + { + "description": "My awesome schema", + "type": "array", + "default": [1,2,3] + } + """).RootElement; + + JsonElement expectedSchema = JsonDocument.Parse(""" + { + "description": "My awesome schema (Default value: [1,2,3])", + "type": "array" + } + """).RootElement; + + AIJsonSchemaTransformOptions options = new() + { + MoveDefaultKeywordToDescription = true, + }; + + JsonElement transformedSchema = AIJsonUtilities.TransformSchema(schema, options); + AssertDeepEquals(expectedSchema, transformedSchema); + } + + [Theory] + [MemberData(nameof(TestTypes.GetTestDataUsingAllValues), MemberType = typeof(TestTypes))] + public static void TransformJsonSchema_ValidateWithTestData(ITestData testData) + { + // Stress tests the schema generation method using types from the JsonSchemaExporter test battery. + + JsonSerializerOptions options = testData.Options is { } opts + ? new(opts) { TypeInfoResolver = TestTypes.TestTypesContext.Default } + : TestTypes.TestTypesContext.Default.Options; + + JsonTypeInfo typeInfo = options.GetTypeInfo(testData.Type); + AIJsonSchemaCreateOptions? createOptions = typeInfo.Properties.Any(prop => prop.IsExtensionData) + ? new() { TransformOptions = new() { DisallowAdditionalProperties = false } } // Do not append additionalProperties: false to the schema if the type has extension data. + : null; + + JsonElement schema = AIJsonUtilities.CreateJsonSchema(testData.Type, serializerOptions: options, inferenceOptions: createOptions); + + int totalSchemaNodes = 0; + AIJsonSchemaTransformOptions transformOptions = new() + { + ConvertBooleanSchemas = true, + RequireAllProperties = true, + DisallowAdditionalProperties = true, + UseNullableKeyword = true, + TransformSchemaNode = (context, schema) => + { + totalSchemaNodes++; + var schemaObj = Assert.IsType(schema); + schemaObj.Add("myAwesomeKeyword", (JsonNode)42); + return schemaObj; + } + }; + + JsonElement transformedSchema = AIJsonUtilities.TransformSchema(schema, transformOptions); + Assert.True(totalSchemaNodes > 0, "TransformSchema was not invoked."); + + int totalSchemaNodes2 = 0; + transformOptions = new() + { + TransformSchemaNode = (context, schema) => + { + totalSchemaNodes2++; + var schemaObj = Assert.IsType(schema); + Assert.Contains("myAwesomeKeyword", schemaObj); + if (schemaObj.TryGetPropertyValue("properties", out JsonNode? props)) + { + Assert.Contains("required", schemaObj); + Assert.Contains("additionalProperties", schemaObj); + Assert.Equal(((JsonArray)schemaObj["required"]!).Count, ((JsonObject)props!).Count); + } + + if (schemaObj.TryGetPropertyValue("type", out JsonNode? type) && type is JsonArray typeArray) + { + Assert.DoesNotContain("null", typeArray); + } + + return schemaObj; + } + }; + + AIJsonUtilities.TransformSchema(transformedSchema, transformOptions); + Assert.Equal(totalSchemaNodes, totalSchemaNodes2); + } + + [Fact] + public static void TransformJsonSchema_InvalidOptions_ThrowsArgumentException() + { + JsonElement schema = JsonDocument.Parse("{}").RootElement; + Assert.Throws(() => AIJsonUtilities.TransformSchema(schema, transformOptions: null!)); + Assert.Throws(() => AIJsonUtilities.TransformSchema(schema, transformOptions: new())); + } + + [Theory] + [InlineData("null")] + [InlineData("42")] + [InlineData("[1,2,3]")] + [InlineData("""{"properties":{"x": 42 }}""")] + [InlineData("""{"oneOf":[42]}""")] + public static void TransformJsonSchema_InvalidInput_ThrowsArgumentException(string invalidSchema) + { + JsonElement schema = JsonDocument.Parse(invalidSchema).RootElement; + AIJsonSchemaTransformOptions transformOptions = new() { ConvertBooleanSchemas = true }; + Assert.Throws(() => AIJsonUtilities.TransformSchema(schema, transformOptions)); + } + private class DerivedAIContent : AIContent { public int DerivedValue { get; set; } diff --git a/test/Libraries/Microsoft.Extensions.AI.AzureAIInference.Tests/AzureAIInferenceChatClientTests.cs b/test/Libraries/Microsoft.Extensions.AI.AzureAIInference.Tests/AzureAIInferenceChatClientTests.cs index 788b8568607..26cd380ec83 100644 --- a/test/Libraries/Microsoft.Extensions.AI.AzureAIInference.Tests/AzureAIInferenceChatClientTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.AzureAIInference.Tests/AzureAIInferenceChatClientTests.cs @@ -7,6 +7,7 @@ using System.Linq; using System.Net.Http; using System.Text.Json; +using System.Text.Json.Serialization; using System.Threading.Tasks; using Azure; using Azure.AI.Inference; @@ -32,6 +33,19 @@ public void AsIChatClient_InvalidArgs_Throws() Assert.Throws("defaultModelId", () => client.AsIChatClient(" ")); } + [Fact] + public async Task NullModel_Throws() + { + ChatCompletionsClient client = new(new("http://localhost/some/endpoint"), new AzureKeyCredential("key")); + IChatClient chatClient = client.AsIChatClient(modelId: null); + + await Assert.ThrowsAsync(() => chatClient.GetResponseAsync("hello")); + await Assert.ThrowsAsync(() => chatClient.GetStreamingResponseAsync("hello").GetAsyncEnumerator().MoveNextAsync().AsTask()); + + await Assert.ThrowsAsync(() => chatClient.GetResponseAsync("hello", new ChatOptions { ModelId = null })); + await Assert.ThrowsAsync(() => chatClient.GetStreamingResponseAsync("hello", new ChatOptions { ModelId = null }).GetAsyncEnumerator().MoveNextAsync().AsTask()); + } + [Fact] public void AsIChatClient_ProducesExpectedMetadata() { @@ -76,54 +90,54 @@ public void GetService_SuccessfullyReturnsUnderlyingClient() Assert.Null(pipeline.GetService("key")); } - [Theory] - [InlineData(false)] - [InlineData(true)] - public async Task BasicRequestResponse_NonStreaming(bool multiContent) - { - const string Input = """ - { - "messages": [{"role":"user", "content":"hello"}], - "max_tokens":10, - "temperature":0.5, - "model":"gpt-4o-mini" - } - """; + private const string BasicInputNonStreaming = """ + { + "messages": [{"role":"user", "content":"hello"}], + "max_tokens":10, + "temperature":0.5, + "model":"gpt-4o-mini" + } + """; - const string Output = """ + private const string BasicOutputNonStreaming = """ + { + "id": "chatcmpl-ADx3PvAnCwJg0woha4pYsBTi3ZpOI", + "object": "chat.completion", + "created": 1727888631, + "model": "gpt-4o-mini-2024-07-18", + "choices": [ { - "id": "chatcmpl-ADx3PvAnCwJg0woha4pYsBTi3ZpOI", - "object": "chat.completion", - "created": 1727888631, - "model": "gpt-4o-mini-2024-07-18", - "choices": [ - { - "index": 0, - "message": { - "role": "assistant", - "content": "Hello! How can I assist you today?", - "refusal": null - }, - "logprobs": null, - "finish_reason": "stop" - } - ], - "usage": { - "prompt_tokens": 8, - "completion_tokens": 9, - "total_tokens": 17, - "prompt_tokens_details": { - "cached_tokens": 0 + "index": 0, + "message": { + "role": "assistant", + "content": "Hello! How can I assist you today?", + "refusal": null }, - "completion_tokens_details": { - "reasoning_tokens": 0 - } - }, - "system_fingerprint": "fp_f85bea6784" + "logprobs": null, + "finish_reason": "stop" } - """; + ], + "usage": { + "prompt_tokens": 8, + "completion_tokens": 9, + "total_tokens": 17, + "prompt_tokens_details": { + "cached_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0 + } + }, + "system_fingerprint": "fp_f85bea6784" + } + """; - using VerbatimHttpHandler handler = new(Input, Output); + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task BasicRequestResponse_NonStreaming(bool multiContent) + { + using VerbatimHttpHandler handler = new(BasicInputNonStreaming, BasicOutputNonStreaming); using HttpClient httpClient = new(handler); using IChatClient client = CreateChatClient(httpClient, "gpt-4o-mini"); @@ -153,50 +167,50 @@ [new ChatMessage(ChatRole.User, "hello".Select(c => (AIContent)new TextContent(c Assert.Equal(17, response.Usage.TotalTokenCount); } - [Theory] - [InlineData(false)] - [InlineData(true)] - public async Task BasicRequestResponse_Streaming(bool multiContent) - { - const string Input = """ - { - "messages": [{"role":"user", "content":"hello"}], - "max_tokens":20, - "temperature":0.5, - "stream":true, - "model":"gpt-4o-mini"} - """; + private const string BasicInputStreaming = """ + { + "messages": [{"role":"user", "content":"hello"}], + "max_tokens":20, + "temperature":0.5, + "stream":true, + "model":"gpt-4o-mini"} + """; - const string Output = """ - data: {"id":"chatcmpl-ADxFKtX6xIwdWRN42QvBj2u1RZpCK","object":"chat.completion.chunk","created":1727889370,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_f85bea6784","choices":[{"index":0,"delta":{"role":"assistant","content":"","refusal":null},"logprobs":null,"finish_reason":null}],"usage":null} + private const string BasicOutputStreaming = """ + data: {"id":"chatcmpl-ADxFKtX6xIwdWRN42QvBj2u1RZpCK","object":"chat.completion.chunk","created":1727889370,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_f85bea6784","choices":[{"index":0,"delta":{"role":"assistant","content":"","refusal":null},"logprobs":null,"finish_reason":null}],"usage":null} - data: {"id":"chatcmpl-ADxFKtX6xIwdWRN42QvBj2u1RZpCK","object":"chat.completion.chunk","created":1727889370,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_f85bea6784","choices":[{"index":0,"delta":{"content":"Hello"},"logprobs":null,"finish_reason":null}],"usage":null} + data: {"id":"chatcmpl-ADxFKtX6xIwdWRN42QvBj2u1RZpCK","object":"chat.completion.chunk","created":1727889370,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_f85bea6784","choices":[{"index":0,"delta":{"content":"Hello"},"logprobs":null,"finish_reason":null}],"usage":null} - data: {"id":"chatcmpl-ADxFKtX6xIwdWRN42QvBj2u1RZpCK","object":"chat.completion.chunk","created":1727889370,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_f85bea6784","choices":[{"index":0,"delta":{"content":"!"},"logprobs":null,"finish_reason":null}],"usage":null} + data: {"id":"chatcmpl-ADxFKtX6xIwdWRN42QvBj2u1RZpCK","object":"chat.completion.chunk","created":1727889370,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_f85bea6784","choices":[{"index":0,"delta":{"content":"!"},"logprobs":null,"finish_reason":null}],"usage":null} - data: {"id":"chatcmpl-ADxFKtX6xIwdWRN42QvBj2u1RZpCK","object":"chat.completion.chunk","created":1727889370,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_f85bea6784","choices":[{"index":0,"delta":{"content":" How"},"logprobs":null,"finish_reason":null}],"usage":null} + data: {"id":"chatcmpl-ADxFKtX6xIwdWRN42QvBj2u1RZpCK","object":"chat.completion.chunk","created":1727889370,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_f85bea6784","choices":[{"index":0,"delta":{"content":" How"},"logprobs":null,"finish_reason":null}],"usage":null} - data: {"id":"chatcmpl-ADxFKtX6xIwdWRN42QvBj2u1RZpCK","object":"chat.completion.chunk","created":1727889370,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_f85bea6784","choices":[{"index":0,"delta":{"content":" can"},"logprobs":null,"finish_reason":null}],"usage":null} + data: {"id":"chatcmpl-ADxFKtX6xIwdWRN42QvBj2u1RZpCK","object":"chat.completion.chunk","created":1727889370,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_f85bea6784","choices":[{"index":0,"delta":{"content":" can"},"logprobs":null,"finish_reason":null}],"usage":null} - data: {"id":"chatcmpl-ADxFKtX6xIwdWRN42QvBj2u1RZpCK","object":"chat.completion.chunk","created":1727889370,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_f85bea6784","choices":[{"index":0,"delta":{"content":" I"},"logprobs":null,"finish_reason":null}],"usage":null} + data: {"id":"chatcmpl-ADxFKtX6xIwdWRN42QvBj2u1RZpCK","object":"chat.completion.chunk","created":1727889370,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_f85bea6784","choices":[{"index":0,"delta":{"content":" I"},"logprobs":null,"finish_reason":null}],"usage":null} - data: {"id":"chatcmpl-ADxFKtX6xIwdWRN42QvBj2u1RZpCK","object":"chat.completion.chunk","created":1727889370,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_f85bea6784","choices":[{"index":0,"delta":{"content":" assist"},"logprobs":null,"finish_reason":null}],"usage":null} + data: {"id":"chatcmpl-ADxFKtX6xIwdWRN42QvBj2u1RZpCK","object":"chat.completion.chunk","created":1727889370,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_f85bea6784","choices":[{"index":0,"delta":{"content":" assist"},"logprobs":null,"finish_reason":null}],"usage":null} - data: {"id":"chatcmpl-ADxFKtX6xIwdWRN42QvBj2u1RZpCK","object":"chat.completion.chunk","created":1727889370,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_f85bea6784","choices":[{"index":0,"delta":{"content":" you"},"logprobs":null,"finish_reason":null}],"usage":null} + data: {"id":"chatcmpl-ADxFKtX6xIwdWRN42QvBj2u1RZpCK","object":"chat.completion.chunk","created":1727889370,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_f85bea6784","choices":[{"index":0,"delta":{"content":" you"},"logprobs":null,"finish_reason":null}],"usage":null} - data: {"id":"chatcmpl-ADxFKtX6xIwdWRN42QvBj2u1RZpCK","object":"chat.completion.chunk","created":1727889370,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_f85bea6784","choices":[{"index":0,"delta":{"content":" today"},"logprobs":null,"finish_reason":null}],"usage":null} + data: {"id":"chatcmpl-ADxFKtX6xIwdWRN42QvBj2u1RZpCK","object":"chat.completion.chunk","created":1727889370,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_f85bea6784","choices":[{"index":0,"delta":{"content":" today"},"logprobs":null,"finish_reason":null}],"usage":null} - data: {"id":"chatcmpl-ADxFKtX6xIwdWRN42QvBj2u1RZpCK","object":"chat.completion.chunk","created":1727889370,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_f85bea6784","choices":[{"index":0,"delta":{"content":"?"},"logprobs":null,"finish_reason":null}],"usage":null} + data: {"id":"chatcmpl-ADxFKtX6xIwdWRN42QvBj2u1RZpCK","object":"chat.completion.chunk","created":1727889370,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_f85bea6784","choices":[{"index":0,"delta":{"content":"?"},"logprobs":null,"finish_reason":null}],"usage":null} - data: {"id":"chatcmpl-ADxFKtX6xIwdWRN42QvBj2u1RZpCK","object":"chat.completion.chunk","created":1727889370,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_f85bea6784","choices":[{"index":0,"delta":{},"logprobs":null,"finish_reason":"stop"}],"usage":null} + data: {"id":"chatcmpl-ADxFKtX6xIwdWRN42QvBj2u1RZpCK","object":"chat.completion.chunk","created":1727889370,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_f85bea6784","choices":[{"index":0,"delta":{},"logprobs":null,"finish_reason":"stop"}],"usage":null} - data: {"id":"chatcmpl-ADxFKtX6xIwdWRN42QvBj2u1RZpCK","object":"chat.completion.chunk","created":1727889370,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_f85bea6784","choices":[],"usage":{"prompt_tokens":8,"completion_tokens":9,"total_tokens":17,"prompt_tokens_details":{"cached_tokens":0},"completion_tokens_details":{"reasoning_tokens":0}}} + data: {"id":"chatcmpl-ADxFKtX6xIwdWRN42QvBj2u1RZpCK","object":"chat.completion.chunk","created":1727889370,"model":"gpt-4o-mini-2024-07-18","system_fingerprint":"fp_f85bea6784","choices":[],"usage":{"prompt_tokens":8,"completion_tokens":9,"total_tokens":17,"prompt_tokens_details":{"cached_tokens":0},"completion_tokens_details":{"reasoning_tokens":0}}} - data: [DONE] + data: [DONE] - """; + """; - using VerbatimHttpHandler handler = new(Input, Output); + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task BasicRequestResponse_Streaming(bool multiContent) + { + using VerbatimHttpHandler handler = new(BasicInputStreaming, BasicOutputStreaming); using HttpClient httpClient = new(handler); using IChatClient client = CreateChatClient(httpClient, "gpt-4o-mini"); @@ -230,6 +244,420 @@ [new ChatMessage(ChatRole.User, "hello".Select(c => (AIContent)new TextContent(c } } + [Fact] + public async Task IChatClient_WithNullModel_ChatOptions_WithNotNullModel_NonStreaming() + { + using VerbatimHttpHandler handler = new(BasicInputNonStreaming, BasicOutputNonStreaming); + using HttpClient httpClient = new(handler); + using IChatClient client = CreateChatClient(httpClient, modelId: null!); + + var response = await client.GetResponseAsync("hello", new ChatOptions + { + ModelId = "gpt-4o-mini", + MaxOutputTokens = 10, + Temperature = 0.5f, + }); + Assert.NotNull(response); + Assert.Equal("Hello! How can I assist you today?", response.Text); + } + + [Fact] + public async Task IChatClient_WithNullModel_ChatOptions_WithNotNullModel_Streaming() + { + using VerbatimHttpHandler handler = new(BasicInputStreaming, BasicOutputStreaming); + using HttpClient httpClient = new(handler); + using IChatClient client = CreateChatClient(httpClient, modelId: null!); + + string responseText = string.Empty; + await foreach (var update in client.GetStreamingResponseAsync("hello", new ChatOptions + { + ModelId = "gpt-4o-mini", + MaxOutputTokens = 20, + Temperature = 0.5f, + })) + { + responseText += update.Text; + } + + Assert.Equal("Hello! How can I assist you today?", responseText); + } + + [Fact] + public async Task ChatOptions_DoNotOverwrite_NotNullPropertiesInRawRepresentation_NonStreaming() + { + const string Input = """ + { + "messages":[{"role":"user","content":"hello"}], + "model":"gpt-4o-mini", + "frequency_penalty":0.75, + "max_tokens":10, + "top_p":0.5, + "presence_penalty":0.5, + "temperature":0.5, + "seed":42, + "stop":["hello","world"], + "response_format":{"type":"text"}, + "tools":[ + {"type":"function","function":{"name":"GetPersonAge","description":"Gets the age of the specified person.","parameters":{"type":"object","required":["personName"],"properties":{"personName":{"description":"The person whose age is being requested","type":"string"}}}}}, + {"type":"function","function":{"name":"GetPersonAge","description":"Gets the age of the specified person.","parameters":{"type": "object","required": ["personName"],"properties": {"personName": {"description": "The person whose age is being requested","type": "string"}}}}} + ], + "tool_choice":"auto", + "additional_property_from_raw_representation":42, + "additional_property_from_MEAI_options":42 + } + """; + + const string Output = """ + { + "id": "chatcmpl-ADx3PvAnCwJg0woha4pYsBTi3ZpOI", + "object": "chat.completion", + "choices": [ + { + "message": { + "role": "assistant", + "content": "Hello! How can I assist you today?" + } + } + ] + } + """; + + using VerbatimHttpHandler handler = new(Input, Output); + using HttpClient httpClient = new(handler); + using IChatClient client = CreateChatClient(httpClient, modelId: null!); + AIFunction tool = AIFunctionFactory.Create(([Description("The person whose age is being requested")] string personName) => 42, "GetPersonAge", "Gets the age of the specified person."); + + ChatCompletionsOptions azureAIOptions = new() + { + Model = "gpt-4o-mini", + FrequencyPenalty = 0.75f, + MaxTokens = 10, + NucleusSamplingFactor = 0.5f, + PresencePenalty = 0.5f, + Temperature = 0.5f, + Seed = 42, + ToolChoice = ChatCompletionsToolChoice.Auto, + ResponseFormat = ChatCompletionsResponseFormat.CreateTextFormat() + }; + azureAIOptions.StopSequences.Add("hello"); + azureAIOptions.Tools.Add(ToAzureAIChatTool(tool)); + azureAIOptions.AdditionalProperties["additional_property_from_raw_representation"] = new BinaryData("42"); + + ChatOptions chatOptions = new ChatOptions + { + RawRepresentationFactory = (c) => + { + ChatCompletionsOptions azureAIOptions = new() + { + Model = "gpt-4o-mini", + FrequencyPenalty = 0.75f, + MaxTokens = 10, + NucleusSamplingFactor = 0.5f, + PresencePenalty = 0.5f, + Temperature = 0.5f, + Seed = 42, + ToolChoice = ChatCompletionsToolChoice.Auto, + ResponseFormat = ChatCompletionsResponseFormat.CreateTextFormat() + }; + azureAIOptions.StopSequences.Add("hello"); + azureAIOptions.Tools.Add(ToAzureAIChatTool(tool)); + azureAIOptions.AdditionalProperties["additional_property_from_raw_representation"] = new BinaryData("42"); + return azureAIOptions; + }, + ModelId = null, + FrequencyPenalty = 0.125f, + MaxOutputTokens = 1, + TopP = 0.125f, + PresencePenalty = 0.125f, + Temperature = 0.125f, + Seed = 1, + StopSequences = ["world"], + Tools = [tool], + ToolMode = ChatToolMode.None, + ResponseFormat = ChatResponseFormat.Json, + AdditionalProperties = new AdditionalPropertiesDictionary + { + ["additional_property_from_MEAI_options"] = 42 + } + }; + + var response = await client.GetResponseAsync("hello", chatOptions); + Assert.NotNull(response); + Assert.Equal("Hello! How can I assist you today?", response.Text); + } + + [Fact] + public async Task ChatOptions_DoNotOverwrite_NotNullPropertiesInRawRepresentation_Streaming() + { + const string Input = """ + { + "messages":[{"role":"user","content":"hello"}], + "model":"gpt-4o-mini", + "frequency_penalty":0.75, + "max_tokens":10, + "top_p":0.5, + "presence_penalty":0.5, + "temperature":0.5, + "seed":42, + "stop":["hello","world"], + "response_format":{"type":"text"}, + "tools":[ + {"type":"function","function":{"name":"GetPersonAge","description":"Gets the age of the specified person.","parameters":{"type":"object","required":["personName"],"properties":{"personName":{"description":"The person whose age is being requested","type":"string"}}}}}, + {"type":"function","function":{"name":"GetPersonAge","description":"Gets the age of the specified person.","parameters":{"type": "object","required": ["personName"],"properties": {"personName": {"description": "The person whose age is being requested","type": "string"}}}}} + ], + "tool_choice":"auto", + "additional_property_from_raw_representation":42, + "additional_property_from_MEAI_options":42, + "stream":true + } + """; + + const string Output = """ + data: {"id":"chatcmpl-ADx3PvAnCwJg0woha4pYsBTi3ZpOI","object":"chat.completion.chunk","choices":[{"delta":{"role":"assistant","content":"Hello! "}}]} + + data: {"id":"chatcmpl-ADx3PvAnCwJg0woha4pYsBTi3ZpOI","object":"chat.completion.chunk","choices":[{"delta":{"content":"How can I assist you today?"}}]} + + data: {"id":"chatcmpl-ADx3PvAnCwJg0woha4pYsBTi3ZpOI","object":"chat.completion.chunk","choices":[{"delta":{},"finish_reason":"stop"}]} + + data: [DONE] + """; + + using VerbatimHttpHandler handler = new(Input, Output); + using HttpClient httpClient = new(handler); + using IChatClient client = CreateChatClient(httpClient, modelId: null!); + AIFunction tool = AIFunctionFactory.Create(([Description("The person whose age is being requested")] string personName) => 42, "GetPersonAge", "Gets the age of the specified person."); + + ChatOptions chatOptions = new ChatOptions + { + RawRepresentationFactory = (c) => + { + ChatCompletionsOptions azureAIOptions = new() + { + Model = "gpt-4o-mini", + FrequencyPenalty = 0.75f, + MaxTokens = 10, + NucleusSamplingFactor = 0.5f, + PresencePenalty = 0.5f, + Temperature = 0.5f, + Seed = 42, + ToolChoice = ChatCompletionsToolChoice.Auto, + ResponseFormat = ChatCompletionsResponseFormat.CreateTextFormat() + }; + azureAIOptions.StopSequences.Add("hello"); + azureAIOptions.Tools.Add(ToAzureAIChatTool(tool)); + azureAIOptions.AdditionalProperties["additional_property_from_raw_representation"] = new BinaryData("42"); + return azureAIOptions; + }, + ModelId = null, + FrequencyPenalty = 0.125f, + MaxOutputTokens = 1, + TopP = 0.125f, + PresencePenalty = 0.125f, + Temperature = 0.125f, + Seed = 1, + StopSequences = ["world"], + Tools = [tool], + ToolMode = ChatToolMode.None, + ResponseFormat = ChatResponseFormat.Json, + AdditionalProperties = new AdditionalPropertiesDictionary + { + ["additional_property_from_MEAI_options"] = 42 + } + }; + + string responseText = string.Empty; + await foreach (var update in client.GetStreamingResponseAsync("hello", chatOptions)) + { + responseText += update.Text; + } + + Assert.Equal("Hello! How can I assist you today?", responseText); + } + + [Fact] + public async Task ChatOptions_Overwrite_NullPropertiesInRawRepresentation_NonStreaming() + { + const string Input = """ + { + "messages":[{"role":"user","content":"hello"}], + "model":"gpt-4o-mini", + "frequency_penalty":0.125, + "max_tokens":1, + "top_p":0.125, + "presence_penalty":0.125, + "temperature":0.125, + "seed":1, + "stop":["world"], + "response_format":{"type":"json_object"}, + "tools":[ + {"type":"function","function":{"name":"GetPersonAge","description":"Gets the age of the specified person.","parameters":{"type":"object","required":["personName"],"properties":{"personName":{"description":"The person whose age is being requested","type":"string"}}}}} + ], + "tool_choice":"none" + } + """; + + const string Output = """ + { + "id": "chatcmpl-ADx3PvAnCwJg0woha4pYsBTi3ZpOI", + "object": "chat.completion", + "choices": [ + { + "message": { + "role": "assistant", + "content": "Hello! How can I assist you today?" + } + } + ] + } + """; + + using VerbatimHttpHandler handler = new(Input, Output); + using HttpClient httpClient = new(handler); + using IChatClient client = CreateChatClient(httpClient, modelId: null!); + AIFunction tool = AIFunctionFactory.Create(([Description("The person whose age is being requested")] string personName) => 42, "GetPersonAge", "Gets the age of the specified person."); + + ChatOptions chatOptions = new ChatOptions + { + RawRepresentationFactory = (c) => + { + ChatCompletionsOptions azureAIOptions = new(); + Assert.Empty(azureAIOptions.Messages); + Assert.Null(azureAIOptions.Model); + Assert.Null(azureAIOptions.FrequencyPenalty); + Assert.Null(azureAIOptions.MaxTokens); + Assert.Null(azureAIOptions.NucleusSamplingFactor); + Assert.Null(azureAIOptions.PresencePenalty); + Assert.Null(azureAIOptions.Temperature); + Assert.Null(azureAIOptions.Seed); + Assert.Empty(azureAIOptions.StopSequences); + Assert.Empty(azureAIOptions.Tools); + Assert.Null(azureAIOptions.ToolChoice); + Assert.Null(azureAIOptions.ResponseFormat); + return azureAIOptions; + }, + ModelId = "gpt-4o-mini", + FrequencyPenalty = 0.125f, + MaxOutputTokens = 1, + TopP = 0.125f, + PresencePenalty = 0.125f, + Temperature = 0.125f, + Seed = 1, + StopSequences = ["world"], + Tools = [tool], + ToolMode = ChatToolMode.None, + ResponseFormat = ChatResponseFormat.Json + }; + + var response = await client.GetResponseAsync("hello", chatOptions); + Assert.NotNull(response); + Assert.Equal("Hello! How can I assist you today?", response.Text); + } + + [Fact] + public async Task ChatOptions_Overwrite_NullPropertiesInRawRepresentation_Streaming() + { + const string Input = """ + { + "messages":[{"role":"user","content":"hello"}], + "model":"gpt-4o-mini", + "frequency_penalty":0.125, + "max_tokens":1, + "top_p":0.125, + "presence_penalty":0.125, + "temperature":0.125, + "seed":1, + "stop":["world"], + "response_format":{"type":"json_object"}, + "tools":[ + {"type":"function","function":{"name":"GetPersonAge","description":"Gets the age of the specified person.","parameters":{"type":"object","required":["personName"],"properties":{"personName":{"description":"The person whose age is being requested","type":"string"}}}}} + ], + "tool_choice":"none", + "stream":true + } + """; + + const string Output = """ + data: {"id":"chatcmpl-ADx3PvAnCwJg0woha4pYsBTi3ZpOI","object":"chat.completion.chunk","choices":[{"delta":{"role":"assistant","content":"Hello! "}}]} + + data: {"id":"chatcmpl-ADx3PvAnCwJg0woha4pYsBTi3ZpOI","object":"chat.completion.chunk","choices":[{"delta":{"content":"How can I assist you today?"}}]} + + data: {"id":"chatcmpl-ADx3PvAnCwJg0woha4pYsBTi3ZpOI","object":"chat.completion.chunk","choices":[{"delta":{},"finish_reason":"stop"}]} + + data: [DONE] + """; + + using VerbatimHttpHandler handler = new(Input, Output); + using HttpClient httpClient = new(handler); + using IChatClient client = CreateChatClient(httpClient, modelId: null!); + AIFunction tool = AIFunctionFactory.Create(([Description("The person whose age is being requested")] string personName) => 42, "GetPersonAge", "Gets the age of the specified person."); + + ChatOptions chatOptions = new ChatOptions + { + RawRepresentationFactory = (c) => + { + ChatCompletionsOptions azureAIOptions = new(); + Assert.Empty(azureAIOptions.Messages); + Assert.Null(azureAIOptions.Model); + Assert.Null(azureAIOptions.FrequencyPenalty); + Assert.Null(azureAIOptions.MaxTokens); + Assert.Null(azureAIOptions.NucleusSamplingFactor); + Assert.Null(azureAIOptions.PresencePenalty); + Assert.Null(azureAIOptions.Temperature); + Assert.Null(azureAIOptions.Seed); + Assert.Empty(azureAIOptions.StopSequences); + Assert.Empty(azureAIOptions.Tools); + Assert.Null(azureAIOptions.ToolChoice); + Assert.Null(azureAIOptions.ResponseFormat); + return azureAIOptions; + }, + ModelId = "gpt-4o-mini", + FrequencyPenalty = 0.125f, + MaxOutputTokens = 1, + TopP = 0.125f, + PresencePenalty = 0.125f, + Temperature = 0.125f, + Seed = 1, + StopSequences = ["world"], + Tools = [tool], + ToolMode = ChatToolMode.None, + ResponseFormat = ChatResponseFormat.Json + }; + + string responseText = string.Empty; + await foreach (var update in client.GetStreamingResponseAsync("hello", chatOptions)) + { + responseText += update.Text; + } + + Assert.Equal("Hello! How can I assist you today?", responseText); + } + + /// Converts an Extensions function to an AzureAI chat tool. + private static ChatCompletionsToolDefinition ToAzureAIChatTool(AIFunction aiFunction) + { + // Map to an intermediate model so that redundant properties are skipped. + var tool = JsonSerializer.Deserialize(aiFunction.JsonSchema)!; + var functionParameters = BinaryData.FromBytes(JsonSerializer.SerializeToUtf8Bytes(tool)); + return new(new FunctionDefinition(aiFunction.Name) + { + Description = aiFunction.Description, + Parameters = functionParameters, + }); + } + + /// Used to create the JSON payload for an AzureAI chat tool description. + private sealed class AzureAIChatToolJson + { + [JsonPropertyName("type")] + public string Type { get; set; } = "object"; + + [JsonPropertyName("required")] + public List Required { get; set; } = []; + + [JsonPropertyName("properties")] + public Dictionary Properties { get; set; } = []; + } + [Fact] public async Task AdditionalOptions_NonStreaming() { @@ -279,10 +707,72 @@ public async Task AdditionalOptions_NonStreaming() PresencePenalty = 0.5f, Seed = 42, StopSequences = ["yes", "no"], - AdditionalProperties = new() + RawRepresentationFactory = (c) => + { + ChatCompletionsOptions azureAIOptions = new(); + azureAIOptions.AdditionalProperties.Add("something_else", new BinaryData(JsonSerializer.SerializeToUtf8Bytes("value1", typeof(object)))); + azureAIOptions.AdditionalProperties.Add("and_something_further", new BinaryData(JsonSerializer.SerializeToUtf8Bytes(123, typeof(object)))); + return azureAIOptions; + }, + })); + } + + [Fact] + public async Task TopK_DoNotOverwrite_NonStreaming() + { + const string Input = """ + { + "messages":[{"role":"user", "content":"hello"}], + "max_tokens":10, + "temperature":0.5, + "top_p":0.5, + "stop":["yes","no"], + "presence_penalty":0.5, + "frequency_penalty":0.75, + "seed":42, + "model":"gpt-4o-mini", + "top_k":40, + "something_else":"value1", + "and_something_further":123 + } + """; + + const string Output = """ + { + "id": "chatcmpl-ADx3PvAnCwJg0woha4pYsBTi3ZpOI", + "object": "chat.completion", + "choices": [ + { + "message": { + "role": "assistant", + "content": "Hello! How can I assist you today?" + } + } + ] + } + """; + + using VerbatimHttpHandler handler = new(Input, Output); + using HttpClient httpClient = new(handler); + using IChatClient client = CreateChatClient(httpClient, "gpt-4o-mini"); + + Assert.NotNull(await client.GetResponseAsync("hello", new() + { + MaxOutputTokens = 10, + Temperature = 0.5f, + TopP = 0.5f, + TopK = 20, // will be ignored because the raw representation already specifies it. + FrequencyPenalty = 0.75f, + PresencePenalty = 0.5f, + Seed = 42, + StopSequences = ["yes", "no"], + RawRepresentationFactory = (c) => { - ["something_else"] = "value1", - ["and_something_further"] = 123, + ChatCompletionsOptions azureAIOptions = new(); + azureAIOptions.AdditionalProperties.Add("top_k", new BinaryData(JsonSerializer.SerializeToUtf8Bytes(40, typeof(object)))); + azureAIOptions.AdditionalProperties.Add("something_else", new BinaryData(JsonSerializer.SerializeToUtf8Bytes("value1", typeof(object)))); + azureAIOptions.AdditionalProperties.Add("and_something_further", new BinaryData(JsonSerializer.SerializeToUtf8Bytes(123, typeof(object)))); + return azureAIOptions; }, })); } diff --git a/test/Libraries/Microsoft.Extensions.AI.AzureAIInference.Tests/Microsoft.Extensions.AI.AzureAIInference.Tests.csproj b/test/Libraries/Microsoft.Extensions.AI.AzureAIInference.Tests/Microsoft.Extensions.AI.AzureAIInference.Tests.csproj index e129c6cf26f..a0f9abaf589 100644 --- a/test/Libraries/Microsoft.Extensions.AI.AzureAIInference.Tests/Microsoft.Extensions.AI.AzureAIInference.Tests.csproj +++ b/test/Libraries/Microsoft.Extensions.AI.AzureAIInference.Tests/Microsoft.Extensions.AI.AzureAIInference.Tests.csproj @@ -6,6 +6,7 @@ true + $(NoWarn);S104 diff --git a/test/Libraries/Microsoft.Extensions.AI.Evaluation.Integration.Tests/QualityEvaluatorTests.cs b/test/Libraries/Microsoft.Extensions.AI.Evaluation.Integration.Tests/QualityEvaluatorTests.cs index 90f7a2b29aa..b56a2673b60 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Evaluation.Integration.Tests/QualityEvaluatorTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Evaluation.Integration.Tests/QualityEvaluatorTests.cs @@ -71,7 +71,7 @@ static QualityEvaluatorTests() DiskBasedReportingConfiguration.Create( storageRootPath: Settings.Current.StorageRootPath, evaluators: [groundednessEvaluator, equivalenceEvaluator, completenessEvaluator, retrievalEvaluator], - chatConfiguration, + chatConfiguration: chatConfiguration, executionName: Constants.Version, tags: [version, date, projectName, testClass, provider, model, temperature, usesContext]); } diff --git a/test/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Tests/AzureStorage/AzureResponseCacheTests.cs b/test/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Tests/AzureStorage/AzureResponseCacheTests.cs index b135a64a04c..2f936621147 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Tests/AzureStorage/AzureResponseCacheTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Tests/AzureStorage/AzureResponseCacheTests.cs @@ -47,9 +47,9 @@ public async Task DisposeAsync() internal override bool IsConfigured => Settings.Current.Configured; - internal override IResponseCacheProvider CreateResponseCacheProvider() + internal override IEvaluationResponseCacheProvider CreateResponseCacheProvider() => new AzureStorageResponseCacheProvider(_dirClient!); - internal override IResponseCacheProvider CreateResponseCacheProvider(Func provideDateTime) + internal override IEvaluationResponseCacheProvider CreateResponseCacheProvider(Func provideDateTime) => new AzureStorageResponseCacheProvider(_dirClient!, provideDateTime); } diff --git a/test/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Tests/AzureStorage/AzureResultStoreTests.cs b/test/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Tests/AzureStorage/AzureResultStoreTests.cs index 610f6345524..62163d5e681 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Tests/AzureStorage/AzureResultStoreTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Tests/AzureStorage/AzureResultStoreTests.cs @@ -47,7 +47,7 @@ public async Task DisposeAsync() public override bool IsConfigured => Settings.Current.Configured; - public override IResultStore CreateResultStore() + public override IEvaluationResultStore CreateResultStore() => new AzureStorageResultStore(_dirClient!); } diff --git a/test/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Tests/CacheOptionsTests.cs b/test/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Tests/CacheOptionsTests.cs deleted file mode 100644 index b8351f45695..00000000000 --- a/test/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Tests/CacheOptionsTests.cs +++ /dev/null @@ -1,73 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System; -using System.IO; -using System.Text.Json; -using System.Threading.Tasks; -using Microsoft.Extensions.AI.Evaluation.Reporting.JsonSerialization; -using Xunit; -using CacheMode = Microsoft.Extensions.AI.Evaluation.Reporting.Storage.DiskBasedResponseCache.CacheMode; -using CacheOptions = Microsoft.Extensions.AI.Evaluation.Reporting.Storage.DiskBasedResponseCache.CacheOptions; - -namespace Microsoft.Extensions.AI.Evaluation.Reporting.Tests; - -public class CacheOptionsTests -{ - [Fact] - public void SerializeCacheOptions() - { - var options = new CacheOptions(CacheMode.Disabled, TimeSpan.FromDays(300)); - - string json = JsonSerializer.Serialize(options, JsonUtilities.Default.CacheOptionsTypeInfo); - CacheOptions? deserialized = JsonSerializer.Deserialize(json, JsonUtilities.Default.CacheOptionsTypeInfo); - - Assert.NotNull(deserialized); - Assert.Equal(options.Mode, deserialized!.Mode); - Assert.Equal(options.TimeToLiveForCacheEntries, deserialized.TimeToLiveForCacheEntries); - - } - - [Fact] - public void SerializeCacheOptionsCompact() - { - var options = new CacheOptions(CacheMode.Disabled, TimeSpan.FromDays(300)); - - string json = JsonSerializer.Serialize(options, JsonUtilities.Compact.CacheOptionsTypeInfo); - CacheOptions? deserialized = JsonSerializer.Deserialize(json, JsonUtilities.Default.CacheOptionsTypeInfo); - - Assert.NotNull(deserialized); - Assert.Equal(options.Mode, deserialized!.Mode); - Assert.Equal(options.TimeToLiveForCacheEntries, deserialized.TimeToLiveForCacheEntries); - - } - - [Fact] - public void SerializeCacheOptionsToFile() - { - var options = new CacheOptions(CacheMode.Enabled, TimeSpan.FromSeconds(10)); - - string tempFilePath = Path.GetTempFileName(); - options.Write(tempFilePath); - CacheOptions deserialized = CacheOptions.Read(tempFilePath); - - Assert.NotNull(deserialized); - Assert.Equal(options.Mode, deserialized.Mode); - Assert.Equal(options.TimeToLiveForCacheEntries, deserialized.TimeToLiveForCacheEntries); - } - - [Fact] - public async Task SerializeCacheOptionsToFileAsync() - { - var options = new CacheOptions(CacheMode.Enabled, TimeSpan.FromSeconds(10)); - - string tempFilePath = Path.GetTempFileName(); - await options.WriteAsync(tempFilePath); - CacheOptions deserialized = await CacheOptions.ReadAsync(tempFilePath); - - Assert.NotNull(deserialized); - Assert.Equal(options.Mode, deserialized.Mode); - Assert.Equal(options.TimeToLiveForCacheEntries, deserialized.TimeToLiveForCacheEntries); - } - -} diff --git a/test/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Tests/DiskBased/DiskBasedResponseCacheTests.cs b/test/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Tests/DiskBased/DiskBasedResponseCacheTests.cs index e0ba0c171d1..8305fe8ddb3 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Tests/DiskBased/DiskBasedResponseCacheTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Tests/DiskBased/DiskBasedResponseCacheTests.cs @@ -45,9 +45,9 @@ public Task DisposeAsync() internal override bool IsConfigured => true; - internal override IResponseCacheProvider CreateResponseCacheProvider() + internal override IEvaluationResponseCacheProvider CreateResponseCacheProvider() => new DiskBasedResponseCacheProvider(UseTempStoragePath()); - internal override IResponseCacheProvider CreateResponseCacheProvider(Func provideDateTime) + internal override IEvaluationResponseCacheProvider CreateResponseCacheProvider(Func provideDateTime) => new DiskBasedResponseCacheProvider(UseTempStoragePath(), provideDateTime); } diff --git a/test/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Tests/DiskBased/DiskBasedResultStoreTests.cs b/test/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Tests/DiskBased/DiskBasedResultStoreTests.cs index 1fee1b9996c..77cabfd7ffd 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Tests/DiskBased/DiskBasedResultStoreTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Tests/DiskBased/DiskBasedResultStoreTests.cs @@ -44,7 +44,7 @@ public Task DisposeAsync() public override bool IsConfigured => true; - public override IResultStore CreateResultStore() + public override IEvaluationResultStore CreateResultStore() => new DiskBasedResultStore(UseTempStoragePath()); } diff --git a/test/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Tests/ResponseCacheTester.cs b/test/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Tests/ResponseCacheTester.cs index 60e4e6f21ed..b69014e631b 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Tests/ResponseCacheTester.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Tests/ResponseCacheTester.cs @@ -5,7 +5,6 @@ using System.Linq; using System.Text; using System.Threading.Tasks; -using Microsoft.Extensions.AI.Evaluation.Reporting.Storage; using Microsoft.Extensions.Caching.Distributed; using Microsoft.TestUtilities; using Xunit; @@ -19,8 +18,8 @@ public abstract class ResponseCacheTester private static readonly string _keyB = "B Key"; private static readonly byte[] _responseB = Encoding.UTF8.GetBytes("Content B"); - internal abstract IResponseCacheProvider CreateResponseCacheProvider(); - internal abstract IResponseCacheProvider CreateResponseCacheProvider(Func provideDateTime); + internal abstract IEvaluationResponseCacheProvider CreateResponseCacheProvider(); + internal abstract IEvaluationResponseCacheProvider CreateResponseCacheProvider(Func provideDateTime); internal abstract bool IsConfigured { get; } private void SkipIfNotConfigured() @@ -38,7 +37,7 @@ public async Task AddUncachedEntry() string iterationName = "TestIteration"; - IResponseCacheProvider provider = CreateResponseCacheProvider(); + IEvaluationResponseCacheProvider provider = CreateResponseCacheProvider(); IDistributedCache cache = await provider.GetCacheAsync(nameof(AddUncachedEntry), iterationName); Assert.NotNull(cache); @@ -59,7 +58,7 @@ public async Task RemoveCachedEntry() string iterationName = "TestIteration"; - IResponseCacheProvider provider = CreateResponseCacheProvider(); + IEvaluationResponseCacheProvider provider = CreateResponseCacheProvider(); IDistributedCache cache = await provider.GetCacheAsync(nameof(RemoveCachedEntry), iterationName); Assert.NotNull(cache); @@ -86,7 +85,7 @@ public async Task CacheEntryExpiration() DateTime now = DateTime.UtcNow; DateTime provideDateTime() => now; - IResponseCacheProvider provider = CreateResponseCacheProvider(provideDateTime); + IEvaluationResponseCacheProvider provider = CreateResponseCacheProvider(provideDateTime); IDistributedCache cache = await provider.GetCacheAsync(nameof(RemoveCachedEntry), iterationName); Assert.NotNull(cache); @@ -96,7 +95,7 @@ public async Task CacheEntryExpiration() cache.Set(_keyB, _responseB); Assert.True(_responseB.SequenceEqual(cache.Get(_keyB) ?? [])); - now = DateTime.UtcNow + DiskBasedResponseCache.CacheOptions.Default.TimeToLiveForCacheEntries; + now = DateTime.UtcNow + Defaults.DefaultTimeToLiveForCacheEntries; Assert.Null(await cache.GetAsync(_keyA)); Assert.Null(cache.Get(_keyB)); @@ -107,7 +106,7 @@ public async Task MultipleCacheInstances() { SkipIfNotConfigured(); - IResponseCacheProvider provider = CreateResponseCacheProvider(); + IEvaluationResponseCacheProvider provider = CreateResponseCacheProvider(); IDistributedCache cache = await provider.GetCacheAsync(nameof(MultipleCacheInstances), "Async"); Assert.NotNull(cache); IDistributedCache cache2 = await provider.GetCacheAsync(nameof(MultipleCacheInstances), "Async"); @@ -134,7 +133,7 @@ public async Task DeleteExpiredEntries() DateTime now = DateTime.UtcNow; DateTime provideDateTime() => now; - IResponseCacheProvider provider = CreateResponseCacheProvider(provideDateTime); + IEvaluationResponseCacheProvider provider = CreateResponseCacheProvider(provideDateTime); IDistributedCache cache = await provider.GetCacheAsync(nameof(RemoveCachedEntry), iterationName); Assert.NotNull(cache); @@ -144,7 +143,7 @@ public async Task DeleteExpiredEntries() cache.Set(_keyB, _responseB); Assert.True(_responseB.SequenceEqual(cache.Get(_keyB) ?? [])); - now = DateTime.UtcNow + DiskBasedResponseCache.CacheOptions.Default.TimeToLiveForCacheEntries; + now = DateTime.UtcNow + Defaults.DefaultTimeToLiveForCacheEntries; await provider.DeleteExpiredCacheEntriesAsync(); @@ -164,7 +163,7 @@ public async Task ResetCache() string iterationName = "TestIteration"; - IResponseCacheProvider provider = CreateResponseCacheProvider(); + IEvaluationResponseCacheProvider provider = CreateResponseCacheProvider(); IDistributedCache cache = await provider.GetCacheAsync(nameof(RemoveCachedEntry), iterationName); Assert.NotNull(cache); diff --git a/test/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Tests/ResultStoreTester.cs b/test/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Tests/ResultStoreTester.cs index 1ce033b3cd7..995b77a8c5e 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Tests/ResultStoreTester.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Tests/ResultStoreTester.cs @@ -13,7 +13,7 @@ namespace Microsoft.Extensions.AI.Evaluation.Reporting.Tests; public abstract class ResultStoreTester { - public abstract IResultStore CreateResultStore(); + public abstract IEvaluationResultStore CreateResultStore(); public abstract bool IsConfigured { get; } @@ -39,7 +39,8 @@ private static ScenarioRunResult CreateTestResult(string scenarioName, string it private static string ScenarioName(int n) => $"Test.Scenario.{n}"; private static string IterationName(int n) => $"Iteration {n}"; - private static async Task> LoadResultsAsync(int n, IResultStore resultStore) + private static async Task> + LoadResultsAsync(int n, IEvaluationResultStore resultStore) { List<(string executionName, string scenarioName, string iterationName)> results = []; await foreach (string executionName in resultStore.GetLatestExecutionNamesAsync(n)) @@ -69,7 +70,7 @@ public async Task WriteAndReadResults() { SkipIfNotConfigured(); - IResultStore resultStore = CreateResultStore(); + IEvaluationResultStore resultStore = CreateResultStore(); Assert.NotNull(resultStore); string newExecutionName = $"Test Execution {Path.GetRandomFileName()}"; @@ -108,7 +109,7 @@ public async Task WriteAndReadHistoricalResults() { SkipIfNotConfigured(); - IResultStore resultStore = CreateResultStore(); + IEvaluationResultStore resultStore = CreateResultStore(); Assert.NotNull(resultStore); string firstExecutionName = $"Test Execution {Path.GetRandomFileName()}"; @@ -152,7 +153,7 @@ public async Task DeleteExecutions() { SkipIfNotConfigured(); - IResultStore resultStore = CreateResultStore(); + IEvaluationResultStore resultStore = CreateResultStore(); Assert.NotNull(resultStore); string executionName = $"Test Execution {Path.GetRandomFileName()}"; @@ -176,7 +177,7 @@ public async Task DeleteSomeExecutions() { SkipIfNotConfigured(); - IResultStore resultStore = CreateResultStore(); + IEvaluationResultStore resultStore = CreateResultStore(); Assert.NotNull(resultStore); string executionName0 = $"Test Execution {Path.GetRandomFileName()}"; @@ -211,7 +212,7 @@ public async Task DeleteScenarios() { SkipIfNotConfigured(); - IResultStore resultStore = CreateResultStore(); + IEvaluationResultStore resultStore = CreateResultStore(); Assert.NotNull(resultStore); string executionName = $"Test Execution {Path.GetRandomFileName()}"; @@ -246,7 +247,7 @@ public async Task DeleteIterations() { SkipIfNotConfigured(); - IResultStore resultStore = CreateResultStore(); + IEvaluationResultStore resultStore = CreateResultStore(); Assert.NotNull(resultStore); string executionName = $"Test Execution {Path.GetRandomFileName()}"; diff --git a/test/Libraries/Microsoft.Extensions.AI.Integration.Tests/BinaryEmbedding.cs b/test/Libraries/Microsoft.Extensions.AI.Integration.Tests/BinaryEmbedding.cs deleted file mode 100644 index f538d1476b0..00000000000 --- a/test/Libraries/Microsoft.Extensions.AI.Integration.Tests/BinaryEmbedding.cs +++ /dev/null @@ -1,16 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System; - -namespace Microsoft.Extensions.AI; - -internal sealed class BinaryEmbedding : Embedding -{ - public BinaryEmbedding(ReadOnlyMemory bits) - { - Bits = bits; - } - - public ReadOnlyMemory Bits { get; } -} diff --git a/test/Libraries/Microsoft.Extensions.AI.Integration.Tests/ChatClientIntegrationTests.cs b/test/Libraries/Microsoft.Extensions.AI.Integration.Tests/ChatClientIntegrationTests.cs index 22531e14e22..994fef47517 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Integration.Tests/ChatClientIntegrationTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Integration.Tests/ChatClientIntegrationTests.cs @@ -77,7 +77,9 @@ public virtual async Task GetResponseAsync_WithEmptyMessage() var response = await _chatClient.GetResponseAsync( [ + new(ChatRole.System, []), new(ChatRole.User, []), + new(ChatRole.Assistant, []), new(ChatRole.User, "What is 1 + 2? Reply with a single number."), ]); @@ -229,16 +231,7 @@ public virtual async Task FunctionInvocation_AutomaticallyInvokeFunction_Paramet }); Assert.Contains(secretNumber.ToString(), response.Text); - - // If the underlying IChatClient provides usage data, function invocation should aggregate the - // usage data across all calls to produce a single Usage value on the final response - if (response.Usage is { } finalUsage) - { - var totalInputTokens = activities.Sum(a => (int?)a.GetTagItem("gen_ai.response.input_tokens")!); - var totalOutputTokens = activities.Sum(a => (int?)a.GetTagItem("gen_ai.response.output_tokens")!); - Assert.Equal(totalInputTokens, finalUsage.InputTokenCount); - Assert.Equal(totalOutputTokens, finalUsage.OutputTokenCount); - } + AssertUsageAgainstActivities(response, activities); } [ConditionalFact] @@ -306,16 +299,7 @@ public virtual async Task FunctionInvocation_OptionalParameter() }); Assert.Contains(secretNumber.ToString(), response.Text); - - // If the underlying IChatClient provides usage data, function invocation should aggregate the - // usage data across all calls to produce a single Usage value on the final response - if (response.Usage is { } finalUsage) - { - var totalInputTokens = activities.Sum(a => (int?)a.GetTagItem("gen_ai.response.input_tokens")!); - var totalOutputTokens = activities.Sum(a => (int?)a.GetTagItem("gen_ai.response.output_tokens")!); - Assert.Equal(totalInputTokens, finalUsage.InputTokenCount); - Assert.Equal(totalOutputTokens, finalUsage.OutputTokenCount); - } + AssertUsageAgainstActivities(response, activities); } [ConditionalFact] @@ -347,15 +331,21 @@ public virtual async Task FunctionInvocation_NestedParameters() }); Assert.Contains((secretNumber + 19).ToString(), response.Text); + AssertUsageAgainstActivities(response, activities); + } + private static void AssertUsageAgainstActivities(ChatResponse response, List activities) + { // If the underlying IChatClient provides usage data, function invocation should aggregate the - // usage data across all calls to produce a single Usage value on the final response + // usage data across all calls to produce a single Usage value on the final response. + // The FunctionInvokingChatClient then itself creates a span that will also be tagged with a sum + // across all consituent calls, which means our final answer will be double. if (response.Usage is { } finalUsage) { var totalInputTokens = activities.Sum(a => (int?)a.GetTagItem("gen_ai.response.input_tokens")!); var totalOutputTokens = activities.Sum(a => (int?)a.GetTagItem("gen_ai.response.output_tokens")!); - Assert.Equal(totalInputTokens, finalUsage.InputTokenCount); - Assert.Equal(totalOutputTokens, finalUsage.OutputTokenCount); + Assert.Equal(totalInputTokens, finalUsage.InputTokenCount * 2); + Assert.Equal(totalOutputTokens, finalUsage.OutputTokenCount * 2); } } @@ -630,9 +620,11 @@ public virtual async Task Caching_AfterFunctionInvocation_FunctionOutputUnchange var secondResponse = await chatClient.GetResponseAsync([message]); Assert.Equal(response.Text, secondResponse.Text); Assert.Equal(2, functionCallCount); - Assert.Equal(2, llmCallCount!.CallCount); + Assert.Equal(FunctionInvokingChatClientSetsConversationId ? 3 : 2, llmCallCount!.CallCount); } + public virtual bool FunctionInvokingChatClientSetsConversationId => false; + [ConditionalFact] public virtual async Task Caching_AfterFunctionInvocation_FunctionOutputChangedAsync() { @@ -942,7 +934,7 @@ public virtual async Task GetResponseAsync_StructuredOutput_NonNative() var response = await captureOutputChatClient.GetResponseAsync(""" Supply an object to represent Jimbo Smith from Cardiff. - """, useJsonSchema: false); + """, useJsonSchemaResponseFormat: false); Assert.Equal("Jimbo Smith", response.Result.FullName); Assert.Contains("Cardiff", response.Result.HomeTown); diff --git a/test/Libraries/Microsoft.Extensions.AI.Integration.Tests/EmbeddingGeneratorIntegrationTests.cs b/test/Libraries/Microsoft.Extensions.AI.Integration.Tests/EmbeddingGeneratorIntegrationTests.cs index 1188e899e4d..1504d0d2488 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Integration.Tests/EmbeddingGeneratorIntegrationTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Integration.Tests/EmbeddingGeneratorIntegrationTests.cs @@ -2,6 +2,9 @@ // The .NET Foundation licenses this file to you under the MIT license. using System; +#if NET +using System.Collections; +#endif using System.Collections.Generic; using System.Diagnostics; using System.Diagnostics.CodeAnalysis; @@ -148,7 +151,14 @@ public async Task Quantization_Binary_EmbeddingsCompareSuccessfully() { for (int j = 0; j < embeddings.Count; j++) { - distances[i, j] = TensorPrimitives.HammingBitDistance(embeddings[i].Bits.Span, embeddings[j].Bits.Span); + distances[i, j] = TensorPrimitives.HammingBitDistance(ToArray(embeddings[i].Vector), ToArray(embeddings[j].Vector)); + + static byte[] ToArray(BitArray array) + { + byte[] result = new byte[(array.Length + 7) / 8]; + array.CopyTo(result, 0); + return result; + } } } diff --git a/test/Libraries/Microsoft.Extensions.AI.Integration.Tests/QuantizationEmbeddingGenerator.cs b/test/Libraries/Microsoft.Extensions.AI.Integration.Tests/QuantizationEmbeddingGenerator.cs index 3bf33988146..ea87408da38 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Integration.Tests/QuantizationEmbeddingGenerator.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Integration.Tests/QuantizationEmbeddingGenerator.cs @@ -2,6 +2,7 @@ // The .NET Foundation licenses this file to you under the MIT license. using System; +using System.Collections; using System.Collections.Generic; using System.Linq; #if NET @@ -46,12 +47,12 @@ private static BinaryEmbedding QuantizeToBinary(Embedding embedding) { ReadOnlySpan vector = embedding.Vector.Span; - var result = new byte[(int)Math.Ceiling(vector.Length / 8.0)]; + var result = new BitArray(vector.Length); for (int i = 0; i < vector.Length; i++) { if (vector[i] > 0) { - result[i / 8] |= (byte)(1 << (i % 8)); + result[i / 8] = true; } } diff --git a/test/Libraries/Microsoft.Extensions.AI.OpenAI.Tests/OpenAIChatClientTests.cs b/test/Libraries/Microsoft.Extensions.AI.OpenAI.Tests/OpenAIChatClientTests.cs index 4fdc36b5280..9ba9c743166 100644 --- a/test/Libraries/Microsoft.Extensions.AI.OpenAI.Tests/OpenAIChatClientTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.OpenAI.Tests/OpenAIChatClientTests.cs @@ -8,6 +8,8 @@ using System.ComponentModel; using System.Linq; using System.Net.Http; +using System.Text.Json; +using System.Text.Json.Serialization; using System.Threading.Tasks; using Azure.AI.OpenAI; using Microsoft.Extensions.Caching.Distributed; @@ -184,9 +186,6 @@ public async Task BasicRequestResponse_NonStreaming() { "OutputTokenDetails.AcceptedPredictionTokenCount", 0 }, { "OutputTokenDetails.RejectedPredictionTokenCount", 0 }, }, response.Usage.AdditionalCounts); - - Assert.NotNull(response.AdditionalProperties); - Assert.Equal("fp_f85bea6784", response.AdditionalProperties[nameof(ChatCompletion.SystemFingerprint)]); } [Fact] @@ -257,8 +256,6 @@ public async Task BasicRequestResponse_Streaming() Assert.Equal(createdAt, updates[i].CreatedAt); Assert.Equal("gpt-4o-mini-2024-07-18", updates[i].ModelId); Assert.Equal(ChatRole.Assistant, updates[i].Role); - Assert.NotNull(updates[i].AdditionalProperties); - Assert.Equal("fp_f85bea6784", updates[i].AdditionalProperties![nameof(ChatCompletion.SystemFingerprint)]); Assert.Equal(i == 10 ? 0 : 1, updates[i].Contents.Count); Assert.Equal(i < 10 ? null : ChatFinishReason.Stop, updates[i].FinishReason); } @@ -280,7 +277,361 @@ public async Task BasicRequestResponse_Streaming() } [Fact] - public async Task NonStronglyTypedOptions_AllSent() + public async Task ChatOptions_DoNotOverwrite_NotNullPropertiesInRawRepresentation_NonStreaming() + { + const string Input = """ + { + "messages":[{"role":"user","content":"hello"}], + "model":"gpt-4o-mini", + "frequency_penalty":0.75, + "max_completion_tokens":10, + "top_p":0.5, + "presence_penalty":0.5, + "temperature":0.5, + "seed":42, + "stop":["hello","world"], + "response_format":{"type":"text"}, + "tools":[ + {"type":"function","function":{"name":"GetPersonAge","description":"Gets the age of the specified person.","parameters":{"additionalProperties":false,"type":"object","required":["personName"],"properties":{"personName":{"description":"The person whose age is being requested","type":"string"}}}}}, + {"type":"function","function":{"name":"GetPersonAge","description":"Gets the age of the specified person.","parameters":{"additionalProperties":false,"type":"object","required":["personName"],"properties":{"personName":{"description":"The person whose age is being requested","type":"string"}}}}} + ], + "tool_choice":"auto" + } + """; + + const string Output = """ + { + "id": "chatcmpl-123", + "object": "chat.completion", + "choices": [ + { + "message": { + "role": "assistant", + "content": "Hello! How can I assist you today?" + } + } + ] + } + """; + + using VerbatimHttpHandler handler = new(Input, Output); + using HttpClient httpClient = new(handler); + using IChatClient client = CreateChatClient(httpClient, modelId: "gpt-4o-mini"); + AIFunction tool = AIFunctionFactory.Create(([Description("The person whose age is being requested")] string personName) => 42, "GetPersonAge", "Gets the age of the specified person."); + + ChatOptions chatOptions = new() + { + RawRepresentationFactory = (c) => + { + ChatCompletionOptions openAIOptions = new() + { + FrequencyPenalty = 0.75f, + MaxOutputTokenCount = 10, + TopP = 0.5f, + PresencePenalty = 0.5f, + Temperature = 0.5f, +#pragma warning disable OPENAI001 // Type is for evaluation purposes only and is subject to change or removal in future updates. + Seed = 42, +#pragma warning restore OPENAI001 + ToolChoice = ChatToolChoice.CreateAutoChoice(), + ResponseFormat = OpenAI.Chat.ChatResponseFormat.CreateTextFormat() + }; + openAIOptions.StopSequences.Add("hello"); + openAIOptions.Tools.Add(ToOpenAIChatTool(tool)); + return openAIOptions; + }, + ModelId = null, + FrequencyPenalty = 0.125f, + MaxOutputTokens = 1, + TopP = 0.125f, + PresencePenalty = 0.125f, + Temperature = 0.125f, + Seed = 1, + StopSequences = ["world"], + Tools = [tool], + ToolMode = ChatToolMode.None, + ResponseFormat = ChatResponseFormat.Json + }; + + var response = await client.GetResponseAsync("hello", chatOptions); + Assert.NotNull(response); + Assert.Equal("Hello! How can I assist you today?", response.Text); + } + + [Fact] + public async Task ChatOptions_DoNotOverwrite_NotNullPropertiesInRawRepresentation_Streaming() + { + const string Input = """ + { + "messages":[{"role":"user","content":"hello"}], + "model":"gpt-4o-mini", + "frequency_penalty":0.75, + "max_completion_tokens":10, + "top_p":0.5, + "presence_penalty":0.5, + "temperature":0.5, + "seed":42, + "stop":["hello","world"], + "response_format":{"type":"text"}, + "tools":[ + {"type":"function","function":{"name":"GetPersonAge","description":"Gets the age of the specified person.","parameters":{"type":"object","required":["personName"],"properties":{"personName":{"description":"The person whose age is being requested","type":"string"}},"additionalProperties":false}}}, + {"type":"function","function":{"name":"GetPersonAge","description":"Gets the age of the specified person.","parameters":{"type":"object","required":["personName"],"properties":{"personName":{"description":"The person whose age is being requested","type":"string"}},"additionalProperties":false}}} + ], + "tool_choice":"auto", + "stream":true, + "stream_options":{"include_usage":true} + } + """; + + const string Output = """ + data: {"id":"chatcmpl-123","object":"chat.completion.chunk","choices":[{"delta":{"role":"assistant","content":"Hello! "}}]} + + data: {"id":"chatcmpl-123","object":"chat.completion.chunk","choices":[{"delta":{"content":"How can I assist you today?"}}]} + + data: {"id":"chatcmpl-123","object":"chat.completion.chunk","choices":[{"delta":{},"finish_reason":"stop"}]} + + data: [DONE] + """; + + using VerbatimHttpHandler handler = new(Input, Output); + using HttpClient httpClient = new(handler); + using IChatClient client = CreateChatClient(httpClient, modelId: "gpt-4o-mini"); + AIFunction tool = AIFunctionFactory.Create(([Description("The person whose age is being requested")] string personName) => 42, "GetPersonAge", "Gets the age of the specified person."); + + ChatOptions chatOptions = new() + { + RawRepresentationFactory = (c) => + { + ChatCompletionOptions openAIOptions = new() + { + FrequencyPenalty = 0.75f, + MaxOutputTokenCount = 10, + TopP = 0.5f, + PresencePenalty = 0.5f, + Temperature = 0.5f, +#pragma warning disable OPENAI001 // Type is for evaluation purposes only and is subject to change or removal in future updates. + Seed = 42, +#pragma warning restore OPENAI001 + ToolChoice = ChatToolChoice.CreateAutoChoice(), + ResponseFormat = OpenAI.Chat.ChatResponseFormat.CreateTextFormat() + }; + openAIOptions.StopSequences.Add("hello"); + openAIOptions.Tools.Add(ToOpenAIChatTool(tool)); + return openAIOptions; + }, + ModelId = null, // has no effect, you cannot change the model of an OpenAI's ChatClient. + FrequencyPenalty = 0.125f, + MaxOutputTokens = 1, + TopP = 0.125f, + PresencePenalty = 0.125f, + Temperature = 0.125f, + Seed = 1, + StopSequences = ["world"], + Tools = [tool], + ToolMode = ChatToolMode.None, + ResponseFormat = ChatResponseFormat.Json + }; + + string responseText = string.Empty; + await foreach (var update in client.GetStreamingResponseAsync("hello", chatOptions)) + { + responseText += update.Text; + } + + Assert.Equal("Hello! How can I assist you today?", responseText); + } + + [Fact] + public async Task ChatOptions_Overwrite_NullPropertiesInRawRepresentation_NonStreaming() + { + const string Input = """ + { + "messages":[{"role":"user","content":"hello"}], + "model":"gpt-4o-mini", + "frequency_penalty":0.125, + "max_completion_tokens":1, + "top_p":0.125, + "presence_penalty":0.125, + "temperature":0.125, + "seed":1, + "stop":["world"], + "response_format":{"type":"json_object"}, + "tools":[ + {"type":"function","function":{"name":"GetPersonAge","description":"Gets the age of the specified person.","parameters":{"additionalProperties":false,"type":"object","required":["personName"],"properties":{"personName":{"description":"The person whose age is being requested","type":"string"}}}}} + ], + "tool_choice":"none" + } + """; + + const string Output = """ + { + "id": "chatcmpl-123", + "object": "chat.completion", + "choices": [ + { + "message": { + "role": "assistant", + "content": "Hello! How can I assist you today?" + } + } + ] + } + """; + + using VerbatimHttpHandler handler = new(Input, Output); + using HttpClient httpClient = new(handler); + using IChatClient client = CreateChatClient(httpClient, modelId: "gpt-4o-mini"); + AIFunction tool = AIFunctionFactory.Create(([Description("The person whose age is being requested")] string personName) => 42, "GetPersonAge", "Gets the age of the specified person."); + + ChatOptions chatOptions = new() + { + RawRepresentationFactory = (c) => + { + ChatCompletionOptions openAIOptions = new(); + Assert.Null(openAIOptions.FrequencyPenalty); + Assert.Null(openAIOptions.MaxOutputTokenCount); + Assert.Null(openAIOptions.TopP); + Assert.Null(openAIOptions.PresencePenalty); + Assert.Null(openAIOptions.Temperature); +#pragma warning disable OPENAI001 // Type is for evaluation purposes only and is subject to change or removal in future updates. + Assert.Null(openAIOptions.Seed); +#pragma warning restore OPENAI001 + Assert.Empty(openAIOptions.StopSequences); + Assert.Empty(openAIOptions.Tools); + Assert.Null(openAIOptions.ToolChoice); + Assert.Null(openAIOptions.ResponseFormat); + return openAIOptions; + }, + ModelId = null, // has no effect, you cannot change the model of an OpenAI's ChatClient. + FrequencyPenalty = 0.125f, + MaxOutputTokens = 1, + TopP = 0.125f, + PresencePenalty = 0.125f, + Temperature = 0.125f, + Seed = 1, + StopSequences = ["world"], + Tools = [tool], + ToolMode = ChatToolMode.None, + ResponseFormat = ChatResponseFormat.Json + }; + + var response = await client.GetResponseAsync("hello", chatOptions); + Assert.NotNull(response); + Assert.Equal("Hello! How can I assist you today?", response.Text); + } + + [Fact] + public async Task ChatOptions_Overwrite_NullPropertiesInRawRepresentation_Streaming() + { + const string Input = """ + { + "messages":[{"role":"user","content":"hello"}], + "model":"gpt-4o-mini", + "frequency_penalty":0.125, + "max_completion_tokens":1, + "top_p":0.125, + "presence_penalty":0.125, + "temperature":0.125, + "seed":1, + "stop":["world"], + "response_format":{"type":"json_object"}, + "tools":[ + {"type":"function","function":{"name":"GetPersonAge","description":"Gets the age of the specified person.","parameters":{"additionalProperties":false,"type":"object","required":["personName"],"properties":{"personName":{"description":"The person whose age is being requested","type":"string"}}}}} + ], + "tool_choice":"none", + "stream":true, + "stream_options":{"include_usage":true} + } + """; + + const string Output = """ + data: {"id":"chatcmpl-123","object":"chat.completion.chunk","choices":[{"delta":{"role":"assistant","content":"Hello! "}}]} + + data: {"id":"chatcmpl-123","object":"chat.completion.chunk","choices":[{"delta":{"content":"How can I assist you today?"}}]} + + data: {"id":"chatcmpl-123","object":"chat.completion.chunk","choices":[{"delta":{},"finish_reason":"stop"}]} + + data: [DONE] + """; + + using VerbatimHttpHandler handler = new(Input, Output); + using HttpClient httpClient = new(handler); + using IChatClient client = CreateChatClient(httpClient, modelId: "gpt-4o-mini"); + AIFunction tool = AIFunctionFactory.Create(([Description("The person whose age is being requested")] string personName) => 42, "GetPersonAge", "Gets the age of the specified person."); + + ChatOptions chatOptions = new() + { + RawRepresentationFactory = (c) => + { + ChatCompletionOptions openAIOptions = new(); + Assert.Null(openAIOptions.FrequencyPenalty); + Assert.Null(openAIOptions.MaxOutputTokenCount); + Assert.Null(openAIOptions.TopP); + Assert.Null(openAIOptions.PresencePenalty); + Assert.Null(openAIOptions.Temperature); +#pragma warning disable OPENAI001 // Type is for evaluation purposes only and is subject to change or removal in future updates. + Assert.Null(openAIOptions.Seed); +#pragma warning restore OPENAI001 + Assert.Empty(openAIOptions.StopSequences); + Assert.Empty(openAIOptions.Tools); + Assert.Null(openAIOptions.ToolChoice); + Assert.Null(openAIOptions.ResponseFormat); + return openAIOptions; + }, + ModelId = null, + FrequencyPenalty = 0.125f, + MaxOutputTokens = 1, + TopP = 0.125f, + PresencePenalty = 0.125f, + Temperature = 0.125f, + Seed = 1, + StopSequences = ["world"], + Tools = [tool], + ToolMode = ChatToolMode.None, + ResponseFormat = ChatResponseFormat.Json + }; + + string responseText = string.Empty; + await foreach (var update in client.GetStreamingResponseAsync("hello", chatOptions)) + { + responseText += update.Text; + } + + Assert.Equal("Hello! How can I assist you today?", responseText); + } + + /// Converts an Extensions function to an OpenAI chat tool. + private static ChatTool ToOpenAIChatTool(AIFunction aiFunction) + { + bool? strict = + aiFunction.AdditionalProperties.TryGetValue("strictJsonSchema", out object? strictObj) && + strictObj is bool strictValue ? + strictValue : null; + + // Map to an intermediate model so that redundant properties are skipped. + var tool = JsonSerializer.Deserialize(aiFunction.JsonSchema)!; + var functionParameters = BinaryData.FromBytes(JsonSerializer.SerializeToUtf8Bytes(tool)); + return ChatTool.CreateFunctionTool(aiFunction.Name, aiFunction.Description, functionParameters, strict); + } + + /// Used to create the JSON payload for an OpenAI chat tool description. + private sealed class ChatToolJson + { + [JsonPropertyName("type")] + public string Type { get; set; } = "object"; + + [JsonPropertyName("required")] + public HashSet Required { get; set; } = []; + + [JsonPropertyName("properties")] + public Dictionary Properties { get; set; } = []; + + [JsonPropertyName("additionalProperties")] + public bool AdditionalProperties { get; set; } + } + + [Fact] + public async Task StronglyTypedOptions_AllSent() { const string Input = """ { @@ -320,17 +671,18 @@ public async Task NonStronglyTypedOptions_AllSent() Assert.NotNull(await client.GetResponseAsync("hello", new() { AllowMultipleToolCalls = false, - AdditionalProperties = new() + RawRepresentationFactory = (c) => { - ["StoredOutputEnabled"] = true, - ["Metadata"] = new Dictionary + var openAIOptions = new ChatCompletionOptions { - ["something"] = "else", - }, - ["LogitBiases"] = new Dictionary { { 12, 34 } }, - ["IncludeLogProbabilities"] = true, - ["TopLogProbabilityCount"] = 42, - ["EndUserId"] = "12345", + StoredOutputEnabled = true, + IncludeLogProbabilities = true, + TopLogProbabilityCount = 42, + EndUserId = "12345", + }; + openAIOptions.Metadata.Add("something", "else"); + openAIOptions.LogitBiases.Add(12, 34); + return openAIOptions; }, })); } @@ -446,9 +798,6 @@ public async Task MultipleMessages_NonStreaming() { "OutputTokenDetails.AcceptedPredictionTokenCount", 0 }, { "OutputTokenDetails.RejectedPredictionTokenCount", 0 }, }, response.Usage.AdditionalCounts); - - Assert.NotNull(response.AdditionalProperties); - Assert.Equal("fp_f85bea6784", response.AdditionalProperties[nameof(ChatCompletion.SystemFingerprint)]); } [Fact] @@ -546,9 +895,6 @@ public async Task MultiPartSystemMessage_NonStreaming() { "OutputTokenDetails.AcceptedPredictionTokenCount", 0 }, { "OutputTokenDetails.RejectedPredictionTokenCount", 0 }, }, response.Usage.AdditionalCounts); - - Assert.NotNull(response.AdditionalProperties); - Assert.Equal("fp_f85bea6784", response.AdditionalProperties[nameof(ChatCompletion.SystemFingerprint)]); } [Fact] @@ -647,9 +993,6 @@ public async Task EmptyAssistantMessage_NonStreaming() { "OutputTokenDetails.AcceptedPredictionTokenCount", 0 }, { "OutputTokenDetails.RejectedPredictionTokenCount", 0 }, }, response.Usage.AdditionalCounts); - - Assert.NotNull(response.AdditionalProperties); - Assert.Equal("fp_f85bea6784", response.AdditionalProperties[nameof(ChatCompletion.SystemFingerprint)]); } [Fact] @@ -767,9 +1110,6 @@ public async Task FunctionCallContent_NonStreaming() FunctionCallContent fcc = Assert.IsType(response.Messages.Single().Contents[0]); Assert.Equal("GetPersonAge", fcc.Name); AssertExtensions.EqualFunctionCallParameters(new Dictionary { ["personName"] = "Alice" }, fcc.Arguments); - - Assert.NotNull(response.AdditionalProperties); - Assert.Equal("fp_f85bea6784", response.AdditionalProperties[nameof(ChatCompletion.SystemFingerprint)]); } [Fact] @@ -852,9 +1192,6 @@ public async Task UnavailableBuiltInFunctionCall_NonStreaming() Assert.Single(response.Messages.Single().Contents); TextContent fcc = Assert.IsType(response.Messages.Single().Contents[0]); - - Assert.NotNull(response.AdditionalProperties); - Assert.Equal("fp_f85bea6784", response.AdditionalProperties[nameof(ChatCompletion.SystemFingerprint)]); } [Fact] @@ -946,8 +1283,6 @@ public async Task FunctionCallContent_Streaming() Assert.Equal(createdAt, updates[i].CreatedAt); Assert.Equal("gpt-4o-mini-2024-07-18", updates[i].ModelId); Assert.Equal(ChatRole.Assistant, updates[i].Role); - Assert.NotNull(updates[i].AdditionalProperties); - Assert.Equal("fp_f85bea6784", updates[i].AdditionalProperties![nameof(ChatCompletion.SystemFingerprint)]); Assert.Equal(i < 7 ? null : ChatFinishReason.ToolCalls, updates[i].FinishReason); } @@ -1111,9 +1446,6 @@ public async Task AssistantMessageWithBothToolsAndContent_NonStreaming() { "OutputTokenDetails.AcceptedPredictionTokenCount", 0 }, { "OutputTokenDetails.RejectedPredictionTokenCount", 0 }, }, response.Usage.AdditionalCounts); - - Assert.NotNull(response.AdditionalProperties); - Assert.Equal("fp_f85bea6784", response.AdditionalProperties[nameof(ChatCompletion.SystemFingerprint)]); } [Fact] @@ -1229,9 +1561,6 @@ private static async Task DataContentMessage_Image_AdditionalPropertyDetail_NonS { "OutputTokenDetails.AcceptedPredictionTokenCount", 0 }, { "OutputTokenDetails.RejectedPredictionTokenCount", 0 }, }, response.Usage.AdditionalCounts); - - Assert.NotNull(response.AdditionalProperties); - Assert.Equal("fp_b705f0c291", response.AdditionalProperties[nameof(ChatCompletion.SystemFingerprint)]); } private static IChatClient CreateChatClient(HttpClient httpClient, string modelId) => diff --git a/test/Libraries/Microsoft.Extensions.AI.OpenAI.Tests/OpenAIResponseClientIntegrationTests.cs b/test/Libraries/Microsoft.Extensions.AI.OpenAI.Tests/OpenAIResponseClientIntegrationTests.cs index bbfad1c571d..2c1d6cdc80e 100644 --- a/test/Libraries/Microsoft.Extensions.AI.OpenAI.Tests/OpenAIResponseClientIntegrationTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.OpenAI.Tests/OpenAIResponseClientIntegrationTests.cs @@ -9,4 +9,6 @@ public class OpenAIResponseClientIntegrationTests : ChatClientIntegrationTests IntegrationTestHelpers.GetOpenAIClient() ?.GetOpenAIResponseClient(TestRunnerConfiguration.Instance["OpenAI:ChatModel"] ?? "gpt-4o-mini") .AsIChatClient(); + + public override bool FunctionInvokingChatClientSetsConversationId => true; } diff --git a/test/Libraries/Microsoft.Extensions.AI.Tests/ChatCompletion/ChatClientStructuredOutputExtensionsTests.cs b/test/Libraries/Microsoft.Extensions.AI.Tests/ChatCompletion/ChatClientStructuredOutputExtensionsTests.cs index 557eecc3c29..edd22edc41e 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Tests/ChatCompletion/ChatClientStructuredOutputExtensionsTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Tests/ChatCompletion/ChatClientStructuredOutputExtensionsTests.cs @@ -34,7 +34,8 @@ public async Task SuccessUsage_Default() GetResponseAsyncCallback = (messages, options, cancellationToken) => { var responseFormat = Assert.IsType(options!.ResponseFormat); - Assert.Equal(""" + Assert.NotNull(responseFormat.Schema); + AssertDeepEquals(JsonDocument.Parse(""" { "$schema": "https://json-schema.org/draft/2020-12/schema", "description": "Some test description", @@ -65,7 +66,7 @@ public async Task SuccessUsage_Default() "species" ] } - """, responseFormat.Schema.ToString()); + """).RootElement, responseFormat.Schema.Value); Assert.Equal(nameof(Animal), responseFormat.SchemaName); Assert.Equal("Some test description", responseFormat.SchemaDescription); @@ -139,7 +140,7 @@ public async Task SuccessUsage_NoJsonSchema() }; var chatHistory = new List { new(ChatRole.User, "Hello") }; - var response = await client.GetResponseAsync(chatHistory, useJsonSchema: false, serializerOptions: JsonContext2.Default.Options); + var response = await client.GetResponseAsync(chatHistory, useJsonSchemaResponseFormat: false, serializerOptions: JsonContext2.Default.Options); // The response contains the deserialized result and other response properties Assert.Equal(1, response.Result.Id); @@ -332,7 +333,8 @@ public async Task CanSpecifyCustomJsonSerializationOptions() // - The property is named full_name, because we specified SnakeCaseLower // - The species value is an integer instead of a string, because we didn't use enum-to-string conversion var responseFormat = Assert.IsType(options!.ResponseFormat); - Assert.Equal(""" + Assert.NotNull(responseFormat.Schema); + AssertDeepEquals(JsonDocument.Parse(""" { "$schema": "https://json-schema.org/draft/2020-12/schema", "description": "Some test description", @@ -358,7 +360,7 @@ public async Task CanSpecifyCustomJsonSerializationOptions() "species" ] } - """, responseFormat.Schema.ToString()); + """).RootElement, responseFormat.Schema.Value); return Task.FromResult(expectedResponse); }, @@ -432,4 +434,28 @@ private class Envelope [JsonSerializable(typeof(Envelope))] [JsonSerializable(typeof(Data))] private partial class JsonContext2 : JsonSerializerContext; + + private static void AssertDeepEquals(JsonElement element1, JsonElement element2) + { +#pragma warning disable SA1118 // Parameter should not span multiple lines + Assert.True(DeepEquals(element1, element2), $""" + Elements are not equal. + Expected: + {element1} + Actual: + {element2} + """); +#pragma warning restore SA1118 // Parameter should not span multiple lines + } + + private static bool DeepEquals(JsonElement element1, JsonElement element2) + { +#if NET9_0_OR_GREATER + return JsonElement.DeepEquals(element1, element2); +#else + return System.Text.Json.Nodes.JsonNode.DeepEquals( + JsonSerializer.SerializeToNode(element1, AIJsonUtilities.DefaultOptions), + JsonSerializer.SerializeToNode(element2, AIJsonUtilities.DefaultOptions)); +#endif + } } diff --git a/test/Libraries/Microsoft.Extensions.AI.Tests/ChatCompletion/DistributedCachingChatClientTest.cs b/test/Libraries/Microsoft.Extensions.AI.Tests/ChatCompletion/DistributedCachingChatClientTest.cs index 374e617adba..4f2427d133c 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Tests/ChatCompletion/DistributedCachingChatClientTest.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Tests/ChatCompletion/DistributedCachingChatClientTest.cs @@ -32,10 +32,13 @@ public void Ctor_ExpectedDefaults() Assert.True(cachingClient.CoalesceStreamingUpdates); } - [Fact] - public async Task CachesSuccessResultsAsync() + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task CachesSuccessResultsAsync(bool conversationIdSet) { // Arrange + ChatOptions options = new() { ConversationId = conversationIdSet ? "123" : null }; // Verify that all the expected properties will round-trip through the cache, // even if this involves serialization @@ -82,20 +85,20 @@ public async Task CachesSuccessResultsAsync() }; // Make the initial request and do a quick sanity check - var result1 = await outer.GetResponseAsync("some input"); + var result1 = await outer.GetResponseAsync("some input", options); Assert.Same(expectedResponse, result1); Assert.Equal(1, innerCallCount); // Act - var result2 = await outer.GetResponseAsync("some input"); + var result2 = await outer.GetResponseAsync("some input", options); // Assert - Assert.Equal(1, innerCallCount); + Assert.Equal(conversationIdSet ? 2 : 1, innerCallCount); AssertResponsesEqual(expectedResponse, result2); // Act/Assert 2: Cache misses do not return cached results - await outer.GetResponseAsync("some modified input"); - Assert.Equal(2, innerCallCount); + await outer.GetResponseAsync("some modified input", options); + Assert.Equal(conversationIdSet ? 3 : 2, innerCallCount); } [Fact] @@ -207,10 +210,13 @@ public async Task DoesNotCacheCanceledResultsAsync() Assert.Equal("A good result", result2.Text); } - [Fact] - public async Task StreamingCachesSuccessResultsAsync() + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task StreamingCachesSuccessResultsAsync(bool conversationIdSet) { // Arrange + ChatOptions options = new() { ConversationId = conversationIdSet ? "123" : null }; // Verify that all the expected properties will round-trip through the cache, // even if this involves serialization @@ -255,20 +261,20 @@ public async Task StreamingCachesSuccessResultsAsync() }; // Make the initial request and do a quick sanity check - var result1 = outer.GetStreamingResponseAsync("some input"); + var result1 = outer.GetStreamingResponseAsync("some input", options); await AssertResponsesEqualAsync(actualUpdate, result1); Assert.Equal(1, innerCallCount); // Act - var result2 = outer.GetStreamingResponseAsync("some input"); + var result2 = outer.GetStreamingResponseAsync("some input", options); // Assert - Assert.Equal(1, innerCallCount); - await AssertResponsesEqualAsync(expectedCachedResponse, result2); + Assert.Equal(conversationIdSet ? 2 : 1, innerCallCount); + await AssertResponsesEqualAsync(conversationIdSet ? actualUpdate : expectedCachedResponse, result2); // Act/Assert 2: Cache misses do not return cached results - await ToListAsync(outer.GetStreamingResponseAsync("some modified input")); - Assert.Equal(2, innerCallCount); + await ToListAsync(outer.GetStreamingResponseAsync("some modified input", options)); + Assert.Equal(conversationIdSet ? 3 : 2, innerCallCount); } [Theory] diff --git a/test/Libraries/Microsoft.Extensions.AI.Tests/ChatCompletion/LoggingChatClientTests.cs b/test/Libraries/Microsoft.Extensions.AI.Tests/ChatCompletion/LoggingChatClientTests.cs index 51638d1a252..cd383381c06 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Tests/ChatCompletion/LoggingChatClientTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Tests/ChatCompletion/LoggingChatClientTests.cs @@ -134,8 +134,6 @@ static async IAsyncEnumerable GetUpdatesAsync() { Assert.Collection(logs, entry => Assert.True(entry.Message.Contains("GetStreamingResponseAsync invoked.") && !entry.Message.Contains("biggest animal")), - entry => Assert.True(entry.Message.Contains("GetStreamingResponseAsync received update.") && !entry.Message.Contains("blue")), - entry => Assert.True(entry.Message.Contains("GetStreamingResponseAsync received update.") && !entry.Message.Contains("whale")), entry => Assert.Contains("GetStreamingResponseAsync completed.", entry.Message)); } else diff --git a/test/Libraries/Microsoft.Extensions.AI.Tests/Functions/AIFunctionFactoryTest.cs b/test/Libraries/Microsoft.Extensions.AI.Tests/Functions/AIFunctionFactoryTest.cs index 4b5ff9a0600..6d448efb710 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Tests/Functions/AIFunctionFactoryTest.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Tests/Functions/AIFunctionFactoryTest.cs @@ -29,7 +29,8 @@ public void InvalidArguments_Throw() Assert.Throws("method", () => AIFunctionFactory.Create(method: null!, target: new object())); Assert.Throws("method", () => AIFunctionFactory.Create(method: null!, target: new object(), name: "myAiFunk")); Assert.Throws("target", () => AIFunctionFactory.Create(typeof(AIFunctionFactoryTest).GetMethod(nameof(InvalidArguments_Throw))!, (object?)null)); - Assert.Throws("targetType", () => AIFunctionFactory.Create(typeof(AIFunctionFactoryTest).GetMethod(nameof(InvalidArguments_Throw))!, (Type)null!)); + Assert.Throws("createInstanceFunc", () => + AIFunctionFactory.Create(typeof(AIFunctionFactoryTest).GetMethod(nameof(InvalidArguments_Throw))!, (Func)null!)); Assert.Throws("method", () => AIFunctionFactory.Create(typeof(List<>).GetMethod("Add")!, new List())); } @@ -299,55 +300,6 @@ public async Task AIFunctionArguments_MissingServicesMayBeOptional() Assert.Equal("", result?.ToString()); } - [Fact] - public async Task IServiceProvider_ServicesInOptionsImpactsFunctionCreation() - { - ServiceCollection sc = new(); - sc.AddSingleton(new MyService(123)); - IServiceProvider sp = sc.BuildServiceProvider(); - - AIFunction func; - - // Services not provided to Create, non-optional argument - if (JsonSerializer.IsReflectionEnabledByDefault) - { - func = AIFunctionFactory.Create((MyService myService) => myService.Value); - Assert.Contains("myService", func.JsonSchema.ToString()); - await Assert.ThrowsAsync("arguments", () => func.InvokeAsync(new()).AsTask()); - await Assert.ThrowsAsync("arguments", () => func.InvokeAsync(new() { Services = sp }).AsTask()); - } - else - { - Assert.Throws(() => AIFunctionFactory.Create((MyService myService) => myService.Value)); - } - - // Services not provided to Create, optional argument - if (JsonSerializer.IsReflectionEnabledByDefault) - { - func = AIFunctionFactory.Create((MyService? myService = null) => myService?.Value ?? 456); - Assert.Contains("myService", func.JsonSchema.ToString()); - Assert.Contains("456", (await func.InvokeAsync(new()))?.ToString()); - Assert.Contains("456", (await func.InvokeAsync(new() { Services = sp }))?.ToString()); - } - else - { - Assert.Throws(() => AIFunctionFactory.Create((MyService myService) => myService.Value)); - } - - // Services provided to Create, non-optional argument - func = AIFunctionFactory.Create((MyService myService) => myService.Value, new() { Services = sp }); - Assert.DoesNotContain("myService", func.JsonSchema.ToString()); - await Assert.ThrowsAsync("arguments.Services", () => func.InvokeAsync(new()).AsTask()); - await Assert.ThrowsAsync("arguments", () => func.InvokeAsync(new() { Services = new ServiceCollection().BuildServiceProvider() }).AsTask()); - Assert.Contains("123", (await func.InvokeAsync(new() { Services = sp }))?.ToString()); - - // Services provided to Create, optional argument - func = AIFunctionFactory.Create((MyService? myService = null) => myService?.Value ?? 456, new() { Services = sp }); - Assert.DoesNotContain("myService", func.JsonSchema.ToString()); - Assert.Contains("456", (await func.InvokeAsync(new()))?.ToString()); - Assert.Contains("123", (await func.InvokeAsync(new() { Services = sp }))?.ToString()); - } - [Fact] public async Task Create_NoInstance_UsesActivatorUtilitiesWhenServicesAvailable() { @@ -361,11 +313,12 @@ public async Task Create_NoInstance_UsesActivatorUtilitiesWhenServicesAvailable( AIFunction func = AIFunctionFactory.Create( typeof(MyFunctionTypeWithOneArg).GetMethod(nameof(MyFunctionTypeWithOneArg.InstanceMethod))!, - typeof(MyFunctionTypeWithOneArg), - new() + static arguments => { - MarshalResult = (result, type, cancellationToken) => new ValueTask(result), - }); + Assert.NotNull(arguments.Services); + return ActivatorUtilities.CreateInstance(arguments.Services, typeof(MyFunctionTypeWithOneArg)); + }, + new() { MarshalResult = (result, type, cancellationToken) => new ValueTask(result) }); Assert.NotNull(func); var result = (Tuple?)await func.InvokeAsync(new() { Services = sp }); @@ -374,31 +327,25 @@ public async Task Create_NoInstance_UsesActivatorUtilitiesWhenServicesAvailable( } [Fact] - public async Task Create_NoInstance_UsesActivatorWhenServicesUnavailable() + public async Task Create_CreateInstanceReturnsNull_ThrowsDuringInvocation() { AIFunction func = AIFunctionFactory.Create( - typeof(MyFunctionTypeWithNoArgs).GetMethod(nameof(MyFunctionTypeWithNoArgs.InstanceMethod))!, - typeof(MyFunctionTypeWithNoArgs), - new() - { - MarshalResult = (result, type, cancellationToken) => new ValueTask(result), - }); + typeof(MyFunctionTypeWithOneArg).GetMethod(nameof(MyFunctionTypeWithOneArg.InstanceMethod))!, + static _ => null!); Assert.NotNull(func); - Assert.Equal("42", await func.InvokeAsync()); + await Assert.ThrowsAsync(async () => await func.InvokeAsync()); } [Fact] - public async Task Create_NoInstance_ThrowsWhenCantConstructInstance() + public async Task Create_WrongConstructedType_ThrowsDuringInvocation() { - var sp = new ServiceCollection().BuildServiceProvider(); - AIFunction func = AIFunctionFactory.Create( typeof(MyFunctionTypeWithOneArg).GetMethod(nameof(MyFunctionTypeWithOneArg.InstanceMethod))!, - typeof(MyFunctionTypeWithOneArg)); + static _ => new MyFunctionTypeWithNoArgs()); Assert.NotNull(func); - await Assert.ThrowsAsync(async () => await func.InvokeAsync(new() { Services = sp })); + await Assert.ThrowsAsync(async () => await func.InvokeAsync()); } [Fact] @@ -406,15 +353,7 @@ public void Create_NoInstance_ThrowsForStaticMethod() { Assert.Throws("method", () => AIFunctionFactory.Create( typeof(MyFunctionTypeWithNoArgs).GetMethod(nameof(MyFunctionTypeWithNoArgs.StaticMethod))!, - typeof(MyFunctionTypeWithNoArgs))); - } - - [Fact] - public void Create_NoInstance_ThrowsForMismatchedMethod() - { - Assert.Throws("targetType", () => AIFunctionFactory.Create( - typeof(MyFunctionTypeWithNoArgs).GetMethod(nameof(MyFunctionTypeWithNoArgs.InstanceMethod))!, - typeof(MyFunctionTypeWithOneArg))); + static _ => new MyFunctionTypeWithNoArgs())); } [Fact] @@ -422,7 +361,7 @@ public async Task Create_NoInstance_DisposableInstanceCreatedDisposedEachInvocat { AIFunction func = AIFunctionFactory.Create( typeof(DisposableService).GetMethod(nameof(DisposableService.GetThis))!, - typeof(DisposableService), + static _ => new DisposableService(), new() { MarshalResult = (result, type, cancellationToken) => new ValueTask(result), @@ -441,7 +380,7 @@ public async Task Create_NoInstance_AsyncDisposableInstanceCreatedDisposedEachIn { AIFunction func = AIFunctionFactory.Create( typeof(AsyncDisposableService).GetMethod(nameof(AsyncDisposableService.GetThis))!, - typeof(AsyncDisposableService), + static _ => new AsyncDisposableService(), new() { MarshalResult = (result, type, cancellationToken) => new ValueTask(result), @@ -460,7 +399,7 @@ public async Task Create_NoInstance_DisposableAndAsyncDisposableInstanceCreatedD { AIFunction func = AIFunctionFactory.Create( typeof(DisposableAndAsyncDisposableService).GetMethod(nameof(DisposableAndAsyncDisposableService.GetThis))!, - typeof(DisposableAndAsyncDisposableService), + static _ => new DisposableAndAsyncDisposableService(), new() { MarshalResult = (result, type, cancellationToken) => new ValueTask(result), @@ -485,13 +424,13 @@ public async Task FromKeyedServices_ResolvesFromServiceProvider() sc.AddKeyedSingleton("key", service); IServiceProvider sp = sc.BuildServiceProvider(); - AIFunction f = AIFunctionFactory.Create(([FromKeyedServices("key")] MyService service, int myInteger) => service.Value + myInteger); + AIFunction f = AIFunctionFactory.Create(([FromKeyedServices("key")] MyService service, int myInteger) => service.Value + myInteger, + CreateKeyedServicesSupportOptions()); Assert.Contains("myInteger", f.JsonSchema.ToString()); Assert.DoesNotContain("service", f.JsonSchema.ToString()); - Exception e = await Assert.ThrowsAsync("arguments.Services", () => f.InvokeAsync(new() { ["myInteger"] = 1 }).AsTask()); - Assert.Contains("Services are required", e.Message); + Exception e = await Assert.ThrowsAsync("arguments.Services", () => f.InvokeAsync(new() { ["myInteger"] = 1 }).AsTask()); var result = await f.InvokeAsync(new() { ["myInteger"] = 1, Services = sp }); Assert.Contains("43", result?.ToString()); @@ -506,13 +445,13 @@ public async Task FromKeyedServices_NullKeysBindToNonKeyedServices() sc.AddSingleton(service); IServiceProvider sp = sc.BuildServiceProvider(); - AIFunction f = AIFunctionFactory.Create(([FromKeyedServices(null!)] MyService service, int myInteger) => service.Value + myInteger); + AIFunction f = AIFunctionFactory.Create(([FromKeyedServices(null!)] MyService service, int myInteger) => service.Value + myInteger, + CreateKeyedServicesSupportOptions()); Assert.Contains("myInteger", f.JsonSchema.ToString()); Assert.DoesNotContain("service", f.JsonSchema.ToString()); - Exception e = await Assert.ThrowsAsync("arguments.Services", () => f.InvokeAsync(new() { ["myInteger"] = 1 }).AsTask()); - Assert.Contains("Services are required", e.Message); + Exception e = await Assert.ThrowsAsync("arguments.Services", () => f.InvokeAsync(new() { ["myInteger"] = 1 }).AsTask()); var result = await f.InvokeAsync(new() { ["myInteger"] = 1, Services = sp }); Assert.Contains("43", result?.ToString()); @@ -528,7 +467,8 @@ public async Task FromKeyedServices_OptionalDefaultsToNull() IServiceProvider sp = sc.BuildServiceProvider(); AIFunction f = AIFunctionFactory.Create(([FromKeyedServices("key")] MyService? service = null, int myInteger = 0) => - service is null ? "null " + 1 : (service.Value + myInteger).ToString()); + service is null ? "null " + 1 : (service.Value + myInteger).ToString(), + CreateKeyedServicesSupportOptions()); Assert.Contains("myInteger", f.JsonSchema.ToString()); Assert.DoesNotContain("service", f.JsonSchema.ToString()); @@ -864,11 +804,7 @@ public ValueTask DisposeAsync() private sealed class MyFunctionTypeWithNoArgs { - private string _value = "42"; - public static void StaticMethod() => throw new NotSupportedException(); - - public string InstanceMethod() => _value; } private sealed class MyFunctionTypeWithOneArg(MyArgumentType arg) @@ -891,6 +827,27 @@ public StructWithDefaultCtor() } } + private static AIFunctionFactoryOptions CreateKeyedServicesSupportOptions() => + new AIFunctionFactoryOptions + { + ConfigureParameterBinding = p => + { + if (p.GetCustomAttribute() is { } attr) + { + return new() + { + BindParameter = (p, a) => + (a.Services as IKeyedServiceProvider)?.GetKeyedService(p.ParameterType, attr.Key) is { } s ? s : + p.HasDefaultValue ? p.DefaultValue : + throw new ArgumentException($"Unable to resolve argument for '{p.Name}'.", "arguments.Services"), + ExcludeFromSchema = true + }; + } + + return default; + }, + }; + [JsonSerializable(typeof(IAsyncEnumerable))] [JsonSerializable(typeof(int[]))] [JsonSerializable(typeof(string))] diff --git a/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Snapshots/aichatweb.Basic.verified/aichatweb/aichatweb.csproj b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Snapshots/aichatweb.Basic.verified/aichatweb/aichatweb.csproj index 2d9ade69626..78946f3f313 100644 --- a/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Snapshots/aichatweb.Basic.verified/aichatweb/aichatweb.csproj +++ b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Snapshots/aichatweb.Basic.verified/aichatweb/aichatweb.csproj @@ -9,9 +9,9 @@ - + - + diff --git a/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Snapshots/aichatweb.BasicAspire.verified/aichatweb/README.md b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Snapshots/aichatweb.BasicAspire.verified/aichatweb/README.md index 0a467b898bd..c05c18281ef 100644 --- a/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Snapshots/aichatweb.BasicAspire.verified/aichatweb/README.md +++ b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Snapshots/aichatweb.BasicAspire.verified/aichatweb/README.md @@ -5,6 +5,14 @@ This project is an AI chat application that demonstrates how to chat with custom >[!NOTE] > Before running this project you need to configure the API keys or endpoints for the providers you have chosen. See below for details specific to your choices. +### Known Issues + +#### Errors running Ollama or Docker + +A recent incompatibility was found between Ollama and Docker Desktop. This issue results in runtime errors when connecting to Ollama, and the workaround for that can lead to Docker not working for Aspire projects. + +This incompatibility can be addressed by upgrading to Docker Desktop 4.41.1. See [ollama/ollama#9509](https://github.com/ollama/ollama/issues/9509#issuecomment-2842461831) for more information and a link to install the version of Docker Desktop with the fix. + # Configure the AI Model Provider ## Using GitHub Models diff --git a/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Snapshots/aichatweb.BasicAspire.verified/aichatweb/aichatweb.AppHost/aichatweb.AppHost.csproj b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Snapshots/aichatweb.BasicAspire.verified/aichatweb/aichatweb.AppHost/aichatweb.AppHost.csproj index 09439522151..a74ef7b7f3b 100644 --- a/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Snapshots/aichatweb.BasicAspire.verified/aichatweb/aichatweb.AppHost/aichatweb.AppHost.csproj +++ b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Snapshots/aichatweb.BasicAspire.verified/aichatweb/aichatweb.AppHost/aichatweb.AppHost.csproj @@ -1,6 +1,6 @@  - + Exe @@ -12,8 +12,8 @@ - - + + diff --git a/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Snapshots/aichatweb.BasicAspire.verified/aichatweb/aichatweb.Web/aichatweb.Web.csproj b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Snapshots/aichatweb.BasicAspire.verified/aichatweb/aichatweb.Web/aichatweb.Web.csproj index 85c0c9cae35..c7b40c7b575 100644 --- a/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Snapshots/aichatweb.BasicAspire.verified/aichatweb/aichatweb.Web/aichatweb.Web.csproj +++ b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Snapshots/aichatweb.BasicAspire.verified/aichatweb/aichatweb.Web/aichatweb.Web.csproj @@ -8,11 +8,11 @@ - + - + - + diff --git a/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Snapshots/aichatweb.OpenAI_AzureAISearch.verified/aichatweb/aichatweb.csproj b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Snapshots/aichatweb.OpenAI_AzureAISearch.verified/aichatweb/aichatweb.csproj index 93bebb61f9f..09532951c55 100644 --- a/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Snapshots/aichatweb.OpenAI_AzureAISearch.verified/aichatweb/aichatweb.csproj +++ b/test/ProjectTemplates/Microsoft.Extensions.AI.Templates.IntegrationTests/Snapshots/aichatweb.OpenAI_AzureAISearch.verified/aichatweb/aichatweb.csproj @@ -10,13 +10,13 @@ - + - + - +