diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/ChatCompletion/ChatResponseExtensions.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/ChatCompletion/ChatResponseExtensions.cs index 6691c665c54..667a9d9aea1 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/ChatCompletion/ChatResponseExtensions.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/ChatCompletion/ChatResponseExtensions.cs @@ -3,6 +3,7 @@ using System; using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; using System.Linq; using System.Text; using System.Threading; @@ -197,14 +198,16 @@ static void Coalesce(List contents, Func int start = 0; while (start < contents.Count - 1) { - // We need at least two TextContents in a row to be able to coalesce. - if (contents[start] is not TContent firstText) + // We need at least two TextContents in a row to be able to coalesce. We also avoid touching contents + // that have annotations, as we want to ensure the annotations (and in particular any start/end indices + // into the text content) remain accurate. + if (!TryAsCoalescable(contents[start], out var firstText)) { start++; continue; } - if (contents[start + 1] is not TContent secondText) + if (!TryAsCoalescable(contents[start + 1], out var secondText)) { start += 2; continue; @@ -216,7 +219,7 @@ static void Coalesce(List contents, Func _ = coalescedText.Clear().Append(firstText).Append(secondText); contents[start + 1] = null!; int i = start + 2; - for (; i < contents.Count && contents[i] is TContent next; i++) + for (; i < contents.Count && TryAsCoalescable(contents[i], out TContent? next); i++) { _ = coalescedText.Append(next); contents[i] = null!; @@ -230,6 +233,18 @@ static void Coalesce(List contents, Func newContent.AdditionalProperties = firstText.AdditionalProperties?.Clone(); start = i; + + static bool TryAsCoalescable(AIContent content, [NotNullWhen(true)] out TContent? coalescable) + { + if (content is TContent && (content is not TextContent tc || tc.Annotations is not { Count: > 0 })) + { + coalescable = (TContent)content; + return true; + } + + coalescable = null!; + return false; + } } // Remove all of the null slots left over from the coalescing process. diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Contents/AIAnnotation.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Contents/AIAnnotation.cs index e69de29bb2d..73fdff81aa2 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Contents/AIAnnotation.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Contents/AIAnnotation.cs @@ -0,0 +1,43 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Collections.Generic; +using System.Text.Json.Serialization; + +namespace Microsoft.Extensions.AI; + +/// +/// Represents an annotation on content. +/// +[JsonPolymorphic(TypeDiscriminatorPropertyName = "$type")] +[JsonDerivedType(typeof(CitationAnnotation), typeDiscriminator: "citation")] +public class AIAnnotation +{ + /// + /// Initializes a new instance of the class. + /// + public AIAnnotation() + { + } + + /// Gets or sets any target regions for the annotation, pointing to where in the associated this annotation applies. + /// + /// The most common form of is , which provides starting and ending character indices + /// for . + /// + public IList? AnnotatedRegions { get; set; } + + /// Gets or sets the raw representation of the annotation from an underlying implementation. + /// + /// If an is created to represent some underlying object from another object + /// model, this property can be used to store that original object. This can be useful for debugging or + /// for enabling a consumer to access the underlying object model, if needed. + /// + [JsonIgnore] + public object? RawRepresentation { get; set; } + + /// + /// Gets or sets additional metadata specific to the provider or source type. + /// + public AdditionalPropertiesDictionary? AdditionalProperties { get; set; } +} diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Contents/AIAnnotationExtensions.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Contents/AIAnnotationExtensions.cs deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Contents/AIAnnotationKind.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Contents/AIAnnotationKind.cs deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Contents/AIAnnotationReference.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Contents/AIAnnotationReference.cs deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Contents/AIContent.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Contents/AIContent.cs index 06798f10f3d..5d0baf93957 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Contents/AIContent.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Contents/AIContent.cs @@ -1,6 +1,7 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. +using System.Collections.Generic; using System.Text.Json.Serialization; namespace Microsoft.Extensions.AI; @@ -24,6 +25,11 @@ public AIContent() { } + /// + /// Gets or sets a list of annotations on this content. + /// + public IList? Annotations { get; set; } + /// Gets or sets the raw representation of the content from an underlying implementation. /// /// If an is created to represent some underlying object from another object diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Contents/AnnotatedRegion.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Contents/AnnotatedRegion.cs new file mode 100644 index 00000000000..fed6dc886b7 --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Contents/AnnotatedRegion.cs @@ -0,0 +1,23 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Text.Json.Serialization; + +namespace Microsoft.Extensions.AI; + +/// Describes the portion of an associated to which an annotation applies. +/// +/// Details about the region is provided by derived types based on how the region is described. For example, starting +/// and ending indices into text content are provided by . +/// +[JsonPolymorphic(TypeDiscriminatorPropertyName = "$type")] +[JsonDerivedType(typeof(TextSpanAnnotatedRegion), typeDiscriminator: "textSpan")] +public class AnnotatedRegion +{ + /// + /// Initializes a new instance of the class. + /// + public AnnotatedRegion() + { + } +} diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Contents/CitationAnnotation.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Contents/CitationAnnotation.cs new file mode 100644 index 00000000000..5d1d2f88b30 --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Contents/CitationAnnotation.cs @@ -0,0 +1,53 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; + +namespace Microsoft.Extensions.AI; + +/// +/// Represents an annotation that links content to source references, +/// such as documents, URLs, files, or tool outputs. +/// +public class CitationAnnotation : AIAnnotation +{ + /// + /// Initializes a new instance of the class. + /// + public CitationAnnotation() + { + } + + /// + /// Gets or sets the title or name of the source. + /// + /// + /// This could be the title of a document, a title from a web page, a name of a file, or similarly descriptive text. + /// + public string? Title { get; set; } + + /// + /// Gets or sets a URI from which the source material was retrieved. + /// + public Uri? Url { get; set; } + + /// Gets or sets a source identifier associated with the annotation. + /// + /// This is a provider-specific identifier that can be used to reference the source material by + /// an ID. This may be a document ID, or a file ID, or some other identifier for the source material + /// that can be used to uniquely identify it with the provider. + /// + public string? FileId { get; set; } + + /// Gets or sets the name of any tool involved in the production of the associated content. + /// + /// This might be a function name, such as one from , or the name of a built-in tool + /// from the provider, such as "code_interpreter" or "file_search". + /// + public string? ToolName { get; set; } + + /// + /// Gets or sets a snippet or excerpt from the source that was cited. + /// + public string? Snippet { get; set; } +} diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Contents/TextSpanAnnotatedRegion.cs b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Contents/TextSpanAnnotatedRegion.cs new file mode 100644 index 00000000000..8ce3dbfa3c5 --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Contents/TextSpanAnnotatedRegion.cs @@ -0,0 +1,32 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Diagnostics; +using System.Text.Json.Serialization; + +namespace Microsoft.Extensions.AI; + +/// Describes a location in the associated based on starting and ending character indices. +/// This typically applies to . +[DebuggerDisplay("[{StartIndex}, {EndIndex})")] +public sealed class TextSpanAnnotatedRegion : AnnotatedRegion +{ + /// + /// Initializes a new instance of the class. + /// + public TextSpanAnnotatedRegion() + { + } + + /// + /// Gets or sets the start character index (inclusive) of the annotated span in the . + /// + [JsonPropertyName("start")] + public int? StartIndex { get; set; } + + /// + /// Gets or sets the end character index (exclusive) of the annotated span in the . + /// + [JsonPropertyName("end")] + public int? EndIndex { get; set; } +} diff --git a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Microsoft.Extensions.AI.Abstractions.json b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Microsoft.Extensions.AI.Abstractions.json index 9562de0d93f..81dcda0bc8d 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Abstractions/Microsoft.Extensions.AI.Abstractions.json +++ b/src/Libraries/Microsoft.Extensions.AI.Abstractions/Microsoft.Extensions.AI.Abstractions.json @@ -123,6 +123,30 @@ } ] }, + { + "Type": "class Microsoft.Extensions.AI.AIAnnotation", + "Stage": "Stable", + "Methods": [ + { + "Member": "Microsoft.Extensions.AI.AIAnnotation.AIAnnotation();", + "Stage": "Stable" + } + ], + "Properties": [ + { + "Member": "System.Collections.Generic.IList? Microsoft.Extensions.AI.AIAnnotation.AnnotatedRegions { get; set; }", + "Stage": "Stable" + }, + { + "Member": "Microsoft.Extensions.AI.AdditionalPropertiesDictionary? Microsoft.Extensions.AI.AIAnnotation.AdditionalProperties { get; set; }", + "Stage": "Stable" + }, + { + "Member": "object? Microsoft.Extensions.AI.AIAnnotation.RawRepresentation { get; set; }", + "Stage": "Stable" + } + ] + }, { "Type": "class Microsoft.Extensions.AI.AIContent", "Stage": "Stable", @@ -137,6 +161,10 @@ "Member": "Microsoft.Extensions.AI.AdditionalPropertiesDictionary? Microsoft.Extensions.AI.AIContent.AdditionalProperties { get; set; }", "Stage": "Stable" }, + { + "Member": "System.Collections.Generic.IList? Microsoft.Extensions.AI.AIContent.Annotations { get; set; }", + "Stage": "Stable" + }, { "Member": "object? Microsoft.Extensions.AI.AIContent.RawRepresentation { get; set; }", "Stage": "Stable" @@ -653,6 +681,16 @@ } ] }, + { + "Type": "class Microsoft.Extensions.AI.AnnotatedRegion", + "Stage": "Stable", + "Methods": [ + { + "Member": "Microsoft.Extensions.AI.AnnotatedRegion.AnnotatedRegion();", + "Stage": "Stable" + } + ] + }, { "Type": "sealed class Microsoft.Extensions.AI.AutoChatToolMode : Microsoft.Extensions.AI.ChatToolMode", "Stage": "Stable", @@ -1323,6 +1361,38 @@ } ] }, + { + "Type": "class Microsoft.Extensions.AI.CitationAnnotation : Microsoft.Extensions.AI.AIAnnotation", + "Stage": "Stable", + "Methods": [ + { + "Member": "Microsoft.Extensions.AI.CitationAnnotation.CitationAnnotation();", + "Stage": "Stable" + } + ], + "Properties": [ + { + "Member": "string? Microsoft.Extensions.AI.CitationAnnotation.Title { get; set; }", + "Stage": "Stable" + }, + { + "Member": "string? Microsoft.Extensions.AI.CitationAnnotation.ToolName { get; set; }", + "Stage": "Stable" + }, + { + "Member": "System.Uri? Microsoft.Extensions.AI.CitationAnnotation.Url { get; set; }", + "Stage": "Stable" + }, + { + "Member": "string? Microsoft.Extensions.AI.CitationAnnotation.FileId { get; set; }", + "Stage": "Stable" + }, + { + "Member": "string? Microsoft.Extensions.AI.CitationAnnotation.Snippet { get; set; }", + "Stage": "Stable" + } + ] + }, { "Type": "class Microsoft.Extensions.AI.DataContent : Microsoft.Extensions.AI.AIContent", "Stage": "Stable", @@ -2273,6 +2343,26 @@ } ] }, + { + "Type": "sealed class Microsoft.Extensions.AI.TextSpanAnnotatedRegion : Microsoft.Extensions.AI.AnnotatedRegion", + "Stage": "Stable", + "Methods": [ + { + "Member": "Microsoft.Extensions.AI.TextSpanAnnotatedRegion.TextSpanAnnotatedRegion();", + "Stage": "Stable" + } + ], + "Properties": [ + { + "Member": "int? Microsoft.Extensions.AI.TextSpanAnnotatedRegion.StartIndex { get; set; }", + "Stage": "Stable" + }, + { + "Member": "int? Microsoft.Extensions.AI.TextSpanAnnotatedRegion.EndIndex { get; set; }", + "Stage": "Stable" + } + ] + }, { "Type": "class Microsoft.Extensions.AI.UriContent : Microsoft.Extensions.AI.AIContent", "Stage": "Stable", diff --git a/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIAssistantsChatClient.cs b/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIAssistantsChatClient.cs index 46bc39ee278..c437a553d28 100644 --- a/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIAssistantsChatClient.cs +++ b/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIAssistantsChatClient.cs @@ -210,7 +210,7 @@ public async IAsyncEnumerable GetStreamingResponseAsync( break; case MessageContentUpdate mcu: - yield return new(mcu.Role == MessageRole.User ? ChatRole.User : ChatRole.Assistant, mcu.Text) + ChatResponseUpdate textUpdate = new(mcu.Role == MessageRole.User ? ChatRole.User : ChatRole.Assistant, mcu.Text) { AuthorName = _assistantId, ConversationId = threadId, @@ -218,10 +218,42 @@ public async IAsyncEnumerable GetStreamingResponseAsync( RawRepresentation = mcu, ResponseId = responseId, }; + + // Add any annotations from the text update. The OpenAI Assistants API does not support passing these back + // into the model (MessageContent.FromXx does not support providing annotations), so they end up being one way and are dropped + // on subsequent requests. + if (mcu.TextAnnotation is { } tau) + { + string? fileId = null; + string? toolName = null; + if (!string.IsNullOrWhiteSpace(tau.InputFileId)) + { + fileId = tau.InputFileId; + toolName = "file_search"; + } + else if (!string.IsNullOrWhiteSpace(tau.OutputFileId)) + { + fileId = tau.OutputFileId; + toolName = "code_interpreter"; + } + + if (fileId is not null) + { + (((TextContent)textUpdate.Contents[0]).Annotations ??= []).Add(new CitationAnnotation + { + RawRepresentation = tau, + AnnotatedRegions = [new TextSpanAnnotatedRegion { StartIndex = tau.StartIndex, EndIndex = tau.EndIndex }], + FileId = fileId, + ToolName = toolName, + }); + } + } + + yield return textUpdate; break; default: - yield return new ChatResponseUpdate + yield return new() { AuthorName = _assistantId, ConversationId = threadId, diff --git a/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIChatClient.cs b/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIChatClient.cs index a70fcec2a8f..8a4e08cac9f 100644 --- a/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIChatClient.cs +++ b/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIChatClient.cs @@ -3,6 +3,7 @@ using System; using System.Collections.Generic; +using System.Linq; using System.Reflection; using System.Runtime.CompilerServices; using System.Text; @@ -480,6 +481,30 @@ internal static ChatResponse FromOpenAIChatCompletion(ChatCompletion openAICompl returnMessage.Contents.Add(new ErrorContent(refusal) { ErrorCode = nameof(openAICompletion.Refusal) }); } + // And add annotations. OpenAI chat completion specifies annotations at the message level (and as such they can't be + // roundtripped back); we store them either on the first text content, assuming there is one, or on a dedicated content + // instance if not. + if (openAICompletion.Annotations is { Count: > 0 }) + { + TextContent? annotationContent = returnMessage.Contents.OfType().FirstOrDefault(); + if (annotationContent is null) + { + annotationContent = new(null); + returnMessage.Contents.Add(annotationContent); + } + + foreach (var annotation in openAICompletion.Annotations) + { + (annotationContent.Annotations ??= []).Add(new CitationAnnotation + { + RawRepresentation = annotation, + AnnotatedRegions = [new TextSpanAnnotatedRegion { StartIndex = annotation.StartIndex, EndIndex = annotation.EndIndex }], + Title = annotation.WebResourceTitle, + Url = annotation.WebResourceUri, + }); + } + } + // Wrap the content in a ChatResponse to return. var response = new ChatResponse(returnMessage) { diff --git a/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIResponsesChatClient.cs b/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIResponsesChatClient.cs index dfe0c50d374..5bd3529e292 100644 --- a/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIResponsesChatClient.cs +++ b/src/Libraries/Microsoft.Extensions.AI.OpenAI/OpenAIResponsesChatClient.cs @@ -608,10 +608,27 @@ private static List ToAIContents(IEnumerable con switch (part.Kind) { case ResponseContentPartKind.OutputText: - results.Add(new TextContent(part.Text) + TextContent text = new(part.Text) { RawRepresentation = part, - }); + }; + + if (part.OutputTextAnnotations is { Count: > 0 }) + { + foreach (var ota in part.OutputTextAnnotations) + { + (text.Annotations ??= []).Add(new CitationAnnotation + { + RawRepresentation = ota, + AnnotatedRegions = [new TextSpanAnnotatedRegion { StartIndex = ota.UriCitationStartIndex, EndIndex = ota.UriCitationEndIndex }], + Title = ota.UriCitationTitle, + Url = ota.UriCitationUri, + FileId = ota.FileCitationFileId ?? ota.FilePathFileId, + }); + } + } + + results.Add(text); break; case ResponseContentPartKind.Refusal: diff --git a/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/ChatCompletion/ChatResponseUpdateExtensionsTests.cs b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/ChatCompletion/ChatResponseUpdateExtensionsTests.cs index 2eb8db9b477..45a82542da8 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/ChatCompletion/ChatResponseUpdateExtensionsTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/ChatCompletion/ChatResponseUpdateExtensionsTests.cs @@ -242,6 +242,48 @@ public async Task ToChatResponse_CoalescesTextContentAndTextReasoningContentSepa Assert.Equal("OP", Assert.IsType(message.Contents[7]).Text); } + [Theory] + [InlineData(false)] + [InlineData(true)] + public async Task ToChatResponse_DoesNotCoalesceAnnotatedContent(bool useAsync) + { + ChatResponseUpdate[] updates = + { + new(null, "A"), + new(null, "B"), + new(null, "C"), + new() { Contents = [new TextContent("D") { Annotations = [new()] }] }, + new() { Contents = [new TextContent("E") { Annotations = [new()] }] }, + new() { Contents = [new TextContent("F") { Annotations = [new()] }] }, + new() { Contents = [new TextContent("G") { Annotations = [] }] }, + new() { Contents = [new TextContent("H") { Annotations = [] }] }, + new() { Contents = [new TextContent("I") { Annotations = [new()] }] }, + new() { Contents = [new TextContent("J") { Annotations = [new()] }] }, + new(null, "K"), + new() { Contents = [new TextContent("L") { Annotations = [new()] }] }, + new(null, "M"), + new(null, "N"), + new() { Contents = [new TextContent("O") { Annotations = [new()] }] }, + new() { Contents = [new TextContent("P") { Annotations = [new()] }] }, + }; + + ChatResponse response = useAsync ? await YieldAsync(updates).ToChatResponseAsync() : updates.ToChatResponse(); + ChatMessage message = Assert.Single(response.Messages); + Assert.Equal(12, message.Contents.Count); + Assert.Equal("ABC", Assert.IsType(message.Contents[0]).Text); + Assert.Equal("D", Assert.IsType(message.Contents[1]).Text); + Assert.Equal("E", Assert.IsType(message.Contents[2]).Text); + Assert.Equal("F", Assert.IsType(message.Contents[3]).Text); + Assert.Equal("GH", Assert.IsType(message.Contents[4]).Text); + Assert.Equal("I", Assert.IsType(message.Contents[5]).Text); + Assert.Equal("J", Assert.IsType(message.Contents[6]).Text); + Assert.Equal("K", Assert.IsType(message.Contents[7]).Text); + Assert.Equal("L", Assert.IsType(message.Contents[8]).Text); + Assert.Equal("MN", Assert.IsType(message.Contents[9]).Text); + Assert.Equal("O", Assert.IsType(message.Contents[10]).Text); + Assert.Equal("P", Assert.IsType(message.Contents[11]).Text); + } + [Fact] public async Task ToChatResponse_UsageContentExtractedFromContents() { diff --git a/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Contents/AIAnnotationTests.cs b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Contents/AIAnnotationTests.cs index e69de29bb2d..2b4b23f0a72 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Contents/AIAnnotationTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Contents/AIAnnotationTests.cs @@ -0,0 +1,71 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.Text.Json; +using Xunit; + +namespace Microsoft.Extensions.AI; + +public class AIAnnotationTests +{ + [Fact] + public void Constructor_PropsDefault() + { + AIAnnotation a = new(); + Assert.Null(a.AdditionalProperties); + Assert.Null(a.RawRepresentation); + Assert.Null(a.AnnotatedRegions); + } + + [Fact] + public void Constructor_PropsRoundtrip() + { + AIAnnotation a = new(); + + Assert.Null(a.AdditionalProperties); + AdditionalPropertiesDictionary props = new() { { "key", "value" } }; + a.AdditionalProperties = props; + Assert.Same(props, a.AdditionalProperties); + + Assert.Null(a.AnnotatedRegions); + List regions = [new TextSpanAnnotatedRegion { StartIndex = 10, EndIndex = 42 }]; + a.AnnotatedRegions = regions; + Assert.Same(regions, a.AnnotatedRegions); + + Assert.Null(a.RawRepresentation); + object raw = new(); + a.RawRepresentation = raw; + Assert.Same(raw, a.RawRepresentation); + } + + [Fact] + public void Serialization_Roundtrips() + { + AIAnnotation original = new() + { + AdditionalProperties = new AdditionalPropertiesDictionary { { "key", "value" } }, + AnnotatedRegions = [new TextSpanAnnotatedRegion { StartIndex = 10, EndIndex = 42 }], + RawRepresentation = new object(), + }; + + string json = JsonSerializer.Serialize(original, AIJsonUtilities.DefaultOptions.GetTypeInfo(typeof(AIAnnotation))); + Assert.NotNull(json); + + var deserialized = (AIAnnotation?)JsonSerializer.Deserialize(json, AIJsonUtilities.DefaultOptions.GetTypeInfo(typeof(AIAnnotation))); + Assert.NotNull(deserialized); + + Assert.NotNull(deserialized.AdditionalProperties); + Assert.Single(deserialized.AdditionalProperties); + Assert.Equal(JsonSerializer.Deserialize("\"value\"", AIJsonUtilities.DefaultOptions).ToString(), deserialized.AdditionalProperties["key"]!.ToString()); + + Assert.Null(deserialized.RawRepresentation); + + Assert.NotNull(deserialized.AnnotatedRegions); + TextSpanAnnotatedRegion? region = Assert.IsType(Assert.Single(deserialized.AnnotatedRegions)); + Assert.NotNull(region); + Assert.Equal(10, region.StartIndex); + Assert.Equal(42, region.EndIndex); + } +} diff --git a/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Contents/CitationAnnotationTests.cs b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Contents/CitationAnnotationTests.cs new file mode 100644 index 00000000000..08097f3e05e --- /dev/null +++ b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Contents/CitationAnnotationTests.cs @@ -0,0 +1,100 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.Text.Json; +using Xunit; + +namespace Microsoft.Extensions.AI; + +public class CitationAnnotationTests +{ + [Fact] + public void Constructor_PropsDefault() + { + CitationAnnotation a = new(); + Assert.Null(a.AdditionalProperties); + Assert.Null(a.AnnotatedRegions); + Assert.Null(a.RawRepresentation); + Assert.Null(a.Snippet); + Assert.Null(a.Title); + Assert.Null(a.ToolName); + Assert.Null(a.Url); + } + + [Fact] + public void Constructor_PropsRoundtrip() + { + CitationAnnotation a = new(); + + Assert.Null(a.AdditionalProperties); + AdditionalPropertiesDictionary props = new() { { "key", "value" } }; + a.AdditionalProperties = props; + Assert.Same(props, a.AdditionalProperties); + + Assert.Null(a.RawRepresentation); + object raw = new(); + a.RawRepresentation = raw; + Assert.Same(raw, a.RawRepresentation); + + Assert.Null(a.AnnotatedRegions); + List regions = [new TextSpanAnnotatedRegion { StartIndex = 10, EndIndex = 42 }]; + a.AnnotatedRegions = regions; + Assert.Same(regions, a.AnnotatedRegions); + + Assert.Null(a.Snippet); + a.Snippet = "snippet"; + Assert.Equal("snippet", a.Snippet); + + Assert.Null(a.Title); + a.Title = "title"; + Assert.Equal("title", a.Title); + + Assert.Null(a.ToolName); + a.ToolName = "toolName"; + Assert.Equal("toolName", a.ToolName); + + Assert.Null(a.Url); + Uri url = new("https://example.com"); + a.Url = url; + Assert.Same(url, a.Url); + } + + [Fact] + public void Serialization_Roundtrips() + { + CitationAnnotation original = new() + { + AdditionalProperties = new AdditionalPropertiesDictionary { { "key", "value" } }, + RawRepresentation = new object(), + Snippet = "snippet", + Title = "title", + ToolName = "toolName", + Url = new("https://example.com"), + AnnotatedRegions = [new TextSpanAnnotatedRegion { StartIndex = 10, EndIndex = 42 }], + }; + + string json = JsonSerializer.Serialize(original, AIJsonUtilities.DefaultOptions.GetTypeInfo(typeof(CitationAnnotation))); + Assert.NotNull(json); + + var deserialized = (CitationAnnotation?)JsonSerializer.Deserialize(json, AIJsonUtilities.DefaultOptions.GetTypeInfo(typeof(CitationAnnotation))); + Assert.NotNull(deserialized); + + Assert.NotNull(deserialized.AdditionalProperties); + Assert.Single(deserialized.AdditionalProperties); + Assert.Equal(JsonSerializer.Deserialize("\"value\"", AIJsonUtilities.DefaultOptions).ToString(), deserialized.AdditionalProperties["key"]!.ToString()); + + Assert.Null(deserialized.RawRepresentation); + Assert.Equal("snippet", deserialized.Snippet); + Assert.Equal("title", deserialized.Title); + Assert.Equal("toolName", deserialized.ToolName); + Assert.NotNull(deserialized.AnnotatedRegions); + TextSpanAnnotatedRegion region = Assert.IsType(Assert.Single(deserialized.AnnotatedRegions)); + Assert.Equal(10, region.StartIndex); + Assert.Equal(42, region.EndIndex); + + Assert.NotNull(deserialized.Url); + Assert.Equal(original.Url, deserialized.Url); + } +} diff --git a/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Utilities/AIJsonUtilitiesTests.cs b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Utilities/AIJsonUtilitiesTests.cs index c2177486fea..acbb5515085 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Utilities/AIJsonUtilitiesTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Abstractions.Tests/Utilities/AIJsonUtilitiesTests.cs @@ -943,13 +943,14 @@ public static void AddAIContentType_DerivedAIContent() JsonSerializerOptions options = new() { TypeInfoResolver = JsonTypeInfoResolver.Combine(AIJsonUtilities.DefaultOptions.TypeInfoResolver, JsonContext.Default), + DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull, }; options.AddAIContentType("derivativeContent"); AIContent c = new DerivedAIContent { DerivedValue = 42 }; string json = JsonSerializer.Serialize(c, options); - Assert.Equal("""{"$type":"derivativeContent","DerivedValue":42,"AdditionalProperties":null}""", json); + Assert.Equal("""{"$type":"derivativeContent","DerivedValue":42}""", json); AIContent? deserialized = JsonSerializer.Deserialize(json, options); Assert.IsType(deserialized); diff --git a/test/Libraries/Microsoft.Extensions.AI.Integration.Tests/ChatClientIntegrationTests.cs b/test/Libraries/Microsoft.Extensions.AI.Integration.Tests/ChatClientIntegrationTests.cs index c9f0e09abf3..5f7f3769d21 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Integration.Tests/ChatClientIntegrationTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Integration.Tests/ChatClientIntegrationTests.cs @@ -34,16 +34,16 @@ namespace Microsoft.Extensions.AI; public abstract class ChatClientIntegrationTests : IDisposable { - private readonly IChatClient? _chatClient; - protected ChatClientIntegrationTests() { - _chatClient = CreateChatClient(); + ChatClient = CreateChatClient(); } + protected IChatClient? ChatClient { get; } + public void Dispose() { - _chatClient?.Dispose(); + ChatClient?.Dispose(); GC.SuppressFinalize(this); } @@ -54,7 +54,7 @@ public virtual async Task GetResponseAsync_SingleRequestMessage() { SkipIfNotEnabled(); - var response = await _chatClient.GetResponseAsync("What's the biggest animal?"); + var response = await ChatClient.GetResponseAsync("What's the biggest animal?"); Assert.Contains("whale", response.Text, StringComparison.OrdinalIgnoreCase); } @@ -64,7 +64,7 @@ public virtual async Task GetResponseAsync_MultipleRequestMessages() { SkipIfNotEnabled(); - var response = await _chatClient.GetResponseAsync( + var response = await ChatClient.GetResponseAsync( [ new(ChatRole.User, "Pick a city, any city"), new(ChatRole.Assistant, "Seattle"), @@ -82,7 +82,7 @@ public virtual async Task GetResponseAsync_WithEmptyMessage() { SkipIfNotEnabled(); - var response = await _chatClient.GetResponseAsync( + var response = await ChatClient.GetResponseAsync( [ new(ChatRole.System, []), new(ChatRole.User, []), @@ -104,7 +104,7 @@ public virtual async Task GetStreamingResponseAsync() ]; StringBuilder sb = new(); - await foreach (var chunk in _chatClient.GetStreamingResponseAsync(chatHistory)) + await foreach (var chunk in ChatClient.GetStreamingResponseAsync(chatHistory)) { sb.Append(chunk.Text); } @@ -119,7 +119,7 @@ public virtual async Task GetResponseAsync_UsageDataAvailable() { SkipIfNotEnabled(); - var response = await _chatClient.GetResponseAsync("Explain in 10 words how AI works"); + var response = await ChatClient.GetResponseAsync("Explain in 10 words how AI works"); Assert.True(response.Usage?.InputTokenCount > 1); Assert.True(response.Usage?.OutputTokenCount > 1); @@ -131,7 +131,7 @@ public virtual async Task GetStreamingResponseAsync_UsageDataAvailable() { SkipIfNotEnabled(); - var response = _chatClient.GetStreamingResponseAsync("Explain in 10 words how AI works", new() + var response = ChatClient.GetStreamingResponseAsync("Explain in 10 words how AI works", new() { AdditionalProperties = new() { @@ -160,7 +160,7 @@ public virtual async Task GetStreamingResponseAsync_AppendToHistory() List history = [new(ChatRole.User, "Explain in 100 words how AI works")]; - var streamingResponse = _chatClient.GetStreamingResponseAsync(history); + var streamingResponse = ChatClient.GetStreamingResponseAsync(history); Assert.Single(history); await history.AddMessagesAsync(streamingResponse); @@ -179,7 +179,7 @@ public virtual async Task MultiModal_DescribeImage() { SkipIfNotEnabled(); - var response = await _chatClient.GetResponseAsync( + var response = await ChatClient.GetResponseAsync( [ new(ChatRole.User, [ @@ -197,7 +197,7 @@ public virtual async Task MultiModal_DescribePdf() { SkipIfNotEnabled(); - var response = await _chatClient.GetResponseAsync( + var response = await ChatClient.GetResponseAsync( [ new(ChatRole.User, [ @@ -223,7 +223,7 @@ public virtual async Task FunctionInvocation_AutomaticallyInvokeFunction_Paramet .Build(); using var chatClient = new FunctionInvokingChatClient( - new OpenTelemetryChatClient(_chatClient, sourceName: sourceName)); + new OpenTelemetryChatClient(ChatClient, sourceName: sourceName)); int secretNumber = 42; @@ -246,7 +246,7 @@ public virtual async Task FunctionInvocation_AutomaticallyInvokeFunction_WithPar { SkipIfNotEnabled(); - using var chatClient = new FunctionInvokingChatClient(_chatClient); + using var chatClient = new FunctionInvokingChatClient(ChatClient); var response = await chatClient.GetResponseAsync("What is the result of SecretComputation on 42 and 84?", new() { @@ -261,7 +261,7 @@ public virtual async Task FunctionInvocation_AutomaticallyInvokeFunction_WithPar { SkipIfNotEnabled(); - using var chatClient = new FunctionInvokingChatClient(_chatClient); + using var chatClient = new FunctionInvokingChatClient(ChatClient); var response = chatClient.GetStreamingResponseAsync("What is the result of SecretComputation on 42 and 84?", new() { @@ -290,7 +290,7 @@ public virtual async Task FunctionInvocation_OptionalParameter() .Build(); using var chatClient = new FunctionInvokingChatClient( - new OpenTelemetryChatClient(_chatClient, sourceName: sourceName)); + new OpenTelemetryChatClient(ChatClient, sourceName: sourceName)); int secretNumber = 42; @@ -322,7 +322,7 @@ public virtual async Task FunctionInvocation_NestedParameters() .Build(); using var chatClient = new FunctionInvokingChatClient( - new OpenTelemetryChatClient(_chatClient, sourceName: sourceName)); + new OpenTelemetryChatClient(ChatClient, sourceName: sourceName)); int secretNumber = 42; @@ -354,7 +354,7 @@ public virtual async Task FunctionInvocation_ArrayParameter() .Build(); using var chatClient = new FunctionInvokingChatClient( - new OpenTelemetryChatClient(_chatClient, sourceName: sourceName)); + new OpenTelemetryChatClient(ChatClient, sourceName: sourceName)); List messages = [ @@ -411,7 +411,7 @@ private async Task AvailableTools_SchemasAreAccepted(bool strict) .Build(); using var chatClient = new FunctionInvokingChatClient( - new OpenTelemetryChatClient(_chatClient, sourceName: sourceName)); + new OpenTelemetryChatClient(ChatClient, sourceName: sourceName)); int methodCount = 1; Func createOptions = () => @@ -567,7 +567,7 @@ public virtual async Task FunctionInvocation_SupportsMultipleParallelRequests() throw new SkipTestException("Parallel function calling is not supported by this chat client"); } - using var chatClient = new FunctionInvokingChatClient(_chatClient); + using var chatClient = new FunctionInvokingChatClient(ChatClient); // The service/model isn't guaranteed to request two calls to GetPersonAge in the same turn, but it's common that it will. var response = await chatClient.GetResponseAsync("How much older is Elsa than Anna? Return the age difference as a single number.", new() @@ -600,7 +600,7 @@ public virtual async Task FunctionInvocation_RequireAny() return 123; }, "GetSecretNumber"); - using var chatClient = new FunctionInvokingChatClient(_chatClient); + using var chatClient = new FunctionInvokingChatClient(ChatClient); var response = await chatClient.GetResponseAsync("Are birds real?", new() { @@ -620,7 +620,7 @@ public virtual async Task FunctionInvocation_RequireSpecific() var getSecretNumberTool = AIFunctionFactory.Create(() => 123, "GetSecretNumber"); var shieldsUpTool = AIFunctionFactory.Create(() => shieldsUp = true, "ShieldsUp"); - using var chatClient = new FunctionInvokingChatClient(_chatClient); + using var chatClient = new FunctionInvokingChatClient(ChatClient); // Even though the user doesn't ask for the shields to be activated, verify that the tool is invoked var response = await chatClient.GetResponseAsync("What's the current secret number?", new() @@ -638,9 +638,9 @@ public virtual async Task Caching_OutputVariesWithoutCaching() SkipIfNotEnabled(); var message = new ChatMessage(ChatRole.User, "Pick a random number, uniformly distributed between 1 and 1000000"); - var firstResponse = await _chatClient.GetResponseAsync([message]); + var firstResponse = await ChatClient.GetResponseAsync([message]); - var secondResponse = await _chatClient.GetResponseAsync([message]); + var secondResponse = await ChatClient.GetResponseAsync([message]); Assert.NotEqual(firstResponse.Text, secondResponse.Text); } @@ -650,7 +650,7 @@ public virtual async Task Caching_SamePromptResultsInCacheHit_NonStreaming() SkipIfNotEnabled(); using var chatClient = new DistributedCachingChatClient( - _chatClient, + ChatClient, new MemoryDistributedCache(Options.Options.Create(new MemoryDistributedCacheOptions()))); var message = new ChatMessage(ChatRole.User, "Pick a random number, uniformly distributed between 1 and 1000000"); @@ -675,7 +675,7 @@ public virtual async Task Caching_SamePromptResultsInCacheHit_Streaming() SkipIfNotEnabled(); using var chatClient = new DistributedCachingChatClient( - _chatClient, + ChatClient, new MemoryDistributedCache(Options.Options.Create(new MemoryDistributedCacheOptions()))); var message = new ChatMessage(ChatRole.User, "Pick a random number, uniformly distributed between 1 and 1000000"); @@ -957,7 +957,7 @@ public virtual async Task GetResponseAsync_StructuredOutput() { SkipIfNotEnabled(); - var response = await _chatClient.GetResponseAsync(""" + var response = await ChatClient.GetResponseAsync(""" Who is described in the following sentence? Jimbo Smith is a 35-year-old programmer from Cardiff, Wales. """); @@ -973,7 +973,7 @@ public virtual async Task GetResponseAsync_StructuredOutputArray() { SkipIfNotEnabled(); - var response = await _chatClient.GetResponseAsync(""" + var response = await ChatClient.GetResponseAsync(""" Who are described in the following sentence? Jimbo Smith is a 35-year-old software developer from Cardiff, Wales. Josh Simpson is a 25-year-old software developer from Newport, Wales. @@ -989,7 +989,7 @@ public virtual async Task GetResponseAsync_StructuredOutputInteger() { SkipIfNotEnabled(); - var response = await _chatClient.GetResponseAsync(""" + var response = await ChatClient.GetResponseAsync(""" There were 14 abstractions for AI programming, which was too many. To fix this we added another one. How many are there now? """); @@ -1002,7 +1002,7 @@ public virtual async Task GetResponseAsync_StructuredOutputString() { SkipIfNotEnabled(); - var response = await _chatClient.GetResponseAsync(""" + var response = await ChatClient.GetResponseAsync(""" The software developer, Jimbo Smith, is a 35-year-old from Cardiff, Wales. What's his full name? """); @@ -1015,7 +1015,7 @@ public virtual async Task GetResponseAsync_StructuredOutputBool_True() { SkipIfNotEnabled(); - var response = await _chatClient.GetResponseAsync(""" + var response = await ChatClient.GetResponseAsync(""" Jimbo Smith is a 35-year-old software developer from Cardiff, Wales. Is there at least one software developer from Cardiff? """); @@ -1028,7 +1028,7 @@ public virtual async Task GetResponseAsync_StructuredOutputBool_False() { SkipIfNotEnabled(); - var response = await _chatClient.GetResponseAsync(""" + var response = await ChatClient.GetResponseAsync(""" Jimbo Smith is a 35-year-old software developer from Cardiff, Wales. Reply true if the previous statement indicates that he is a medical doctor, otherwise false. """); @@ -1041,7 +1041,7 @@ public virtual async Task GetResponseAsync_StructuredOutputEnum() { SkipIfNotEnabled(); - var response = await _chatClient.GetResponseAsync(""" + var response = await ChatClient.GetResponseAsync(""" Taylor Swift is a famous singer and songwriter. What is her job? """); @@ -1061,7 +1061,7 @@ public virtual async Task GetResponseAsync_StructuredOutput_WithFunctions() Job = JobType.Programmer, }; - using var chatClient = new FunctionInvokingChatClient(_chatClient); + using var chatClient = new FunctionInvokingChatClient(ChatClient); var response = await chatClient.GetResponseAsync( "Who is person with ID 123?", new ChatOptions { @@ -1085,7 +1085,7 @@ public virtual async Task GetResponseAsync_StructuredOutput_NonNative() SkipIfNotEnabled(); var capturedOptions = new List(); - var captureOutputChatClient = _chatClient.AsBuilder() + var captureOutputChatClient = ChatClient.AsBuilder() .Use((messages, options, nextAsync, cancellationToken) => { capturedOptions.Add(options); @@ -1125,12 +1125,12 @@ private enum JobType Unknown, } - [MemberNotNull(nameof(_chatClient))] + [MemberNotNull(nameof(ChatClient))] protected void SkipIfNotEnabled() { string? skipIntegration = TestRunnerConfiguration.Instance["SkipIntegrationTests"]; - if (skipIntegration is not null || _chatClient is null) + if (skipIntegration is not null || ChatClient is null) { throw new SkipTestException("Client is not enabled."); } diff --git a/test/Libraries/Microsoft.Extensions.AI.OpenAI.Tests/IntegrationTestHelpers.cs b/test/Libraries/Microsoft.Extensions.AI.OpenAI.Tests/IntegrationTestHelpers.cs index 9d8f806ca8a..a794460a9bd 100644 --- a/test/Libraries/Microsoft.Extensions.AI.OpenAI.Tests/IntegrationTestHelpers.cs +++ b/test/Libraries/Microsoft.Extensions.AI.OpenAI.Tests/IntegrationTestHelpers.cs @@ -18,7 +18,7 @@ internal static class IntegrationTestHelpers { var configuration = TestRunnerConfiguration.Instance; - string? apiKey = configuration["OpenAI:Key"]; + string? apiKey = configuration["AI:OpenAI:ApiKey"]; string? mode = configuration["OpenAI:Mode"]; if (string.Equals(mode, "AzureOpenAI", StringComparison.OrdinalIgnoreCase)) diff --git a/test/Libraries/Microsoft.Extensions.AI.OpenAI.Tests/OpenAIResponseClientIntegrationTests.cs b/test/Libraries/Microsoft.Extensions.AI.OpenAI.Tests/OpenAIResponseClientIntegrationTests.cs index f8e835bdb81..630af9e34b0 100644 --- a/test/Libraries/Microsoft.Extensions.AI.OpenAI.Tests/OpenAIResponseClientIntegrationTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.OpenAI.Tests/OpenAIResponseClientIntegrationTests.cs @@ -1,7 +1,11 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. +using System.Collections.Generic; +using System.Linq; using System.Threading.Tasks; +using Microsoft.TestUtilities; +using Xunit; namespace Microsoft.Extensions.AI; @@ -16,4 +20,33 @@ public class OpenAIResponseClientIntegrationTests : ChatClientIntegrationTests // Test structure doesn't make sense with Respones. public override Task Caching_AfterFunctionInvocation_FunctionOutputUnchangedAsync() => Task.CompletedTask; + + [ConditionalFact] + public async Task UseWebSearch_AnnotationsReflectResults() + { + SkipIfNotEnabled(); + + var response = await ChatClient.GetResponseAsync( + "Write a paragraph about the three most recent blog posts on the .NET blog. Cite your sources.", + new() { Tools = [new HostedWebSearchTool()] }); + + ChatMessage m = Assert.Single(response.Messages); + TextContent tc = m.Contents.OfType().First(); + Assert.NotNull(tc.Annotations); + Assert.NotEmpty(tc.Annotations); + Assert.All(tc.Annotations, a => + { + CitationAnnotation ca = Assert.IsType(a); + var regions = Assert.IsType>(ca.AnnotatedRegions); + Assert.NotNull(regions); + Assert.Single(regions); + var region = Assert.IsType(regions[0]); + Assert.NotNull(region); + Assert.NotNull(region.StartIndex); + Assert.NotNull(region.EndIndex); + Assert.NotNull(ca.Url); + Assert.NotNull(ca.Title); + Assert.NotEmpty(ca.Title); + }); + } }