diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/TypeScript/components/ConversationDetails.tsx b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/TypeScript/components/ConversationDetails.tsx index 3e80f03925b..6a3a78d5ca3 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/TypeScript/components/ConversationDetails.tsx +++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/TypeScript/components/ConversationDetails.tsx @@ -4,8 +4,7 @@ import { useState } from "react"; import ReactMarkdown from "react-markdown"; import { useReportContext } from "./ReportContext"; import { useStyles } from "./Styles"; -import { ChatMessageDisplay } from "./Summary"; - +import { ChatMessageDisplay, isTextContent, isImageContent } from "./Summary"; export const ConversationDetails = ({ messages, model, usage }: { messages: ChatMessageDisplay[]; @@ -25,6 +24,40 @@ export const ConversationDetails = ({ messages, model, usage }: { usage?.totalTokenCount && `Total Tokens: ${usage.totalTokenCount}`, ].filter(Boolean).join(' • '); + const renderContent = (content: AIContent) => { + if (isTextContent(content)) { + return renderMarkdown ? + {content.text} : + 
{content.text}
; + } else if (isImageContent(content)) { + const imageUrl = (content as UriContent).uri || (content as DataContent).uri; + return Content; + } + }; + + const groupMessages = () => { + const result: { role: string, participantName: string, contents: AIContent[] }[] = []; + + for (const message of messages) { + // If this message has the same role and participant as the previous one, append its content + const lastGroup = result[result.length - 1]; + if (lastGroup && lastGroup.role === message.role && lastGroup.participantName === message.participantName) { + lastGroup.contents.push(message.content); + } else { + // Otherwise, start a new group + result.push({ + role: message.role, + participantName: message.participantName, + contents: [message.content] + }); + } + } + + return result; + }; + + const messageGroups = groupMessages(); + return (
setIsExpanded(!isExpanded)}> @@ -35,8 +68,8 @@ export const ConversationDetails = ({ messages, model, usage }: { {isExpanded && (
- {messages.map((message, index) => { - const isFromUserSide = isUserSide(message.role); + {messageGroups.map((group, index) => { + const isFromUserSide = isUserSide(group.role); const messageRowClass = mergeClasses( classes.messageRow, isFromUserSide ? classes.userMessageRow : classes.assistantMessageRow @@ -44,11 +77,13 @@ export const ConversationDetails = ({ messages, model, usage }: { return (
-
{message.participantName}
+
{group.participantName}
- {renderMarkdown ? - {message.content} : - 
{message.content}
} + {group.contents.map((content, contentIndex) => ( +
+ {renderContent(content)} +
+ ))}
); diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/TypeScript/components/EvalTypes.d.ts b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/TypeScript/components/EvalTypes.d.ts index 756d69283d3..c9accb7a90d 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/TypeScript/components/EvalTypes.d.ts +++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/TypeScript/components/EvalTypes.d.ts @@ -54,12 +54,23 @@ type AIContent = { $type: string; }; -// TODO: Model other types of AIContent such as function calls, function call results, images, audio etc. +// TODO: Model other types of AIContent such as function calls, function call results, audio etc. type TextContent = AIContent & { $type: "text"; text: string; }; +type UriContent = AIContent & { + $type: "uri"; + uri: string; + mediaType: string; +}; + +type DataContent = AIContent & { + $type: "data"; + uri: string; +}; + type EvaluationResult = { metrics: { [K: string]: MetricWithNoValue | NumericMetric | BooleanMetric | StringMetric; diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/TypeScript/components/Styles.ts b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/TypeScript/components/Styles.ts index 7b34280e5dd..2a5cedb7906 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/TypeScript/components/Styles.ts +++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/TypeScript/components/Styles.ts @@ -205,6 +205,10 @@ export const useStyles = makeStyles({ preWrap: { whiteSpace: 'pre-wrap', }, + imageContent: { + maxWidth: '100%', + maxHeight: '400px', + }, executionHeaderCell: { display: 'flex', alignItems: 'center', diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/TypeScript/components/Summary.ts b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/TypeScript/components/Summary.ts index 16d65ae239b..5b7b57f64e7 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/TypeScript/components/Summary.ts +++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/TypeScript/components/Summary.ts @@ -108,9 +108,15 @@ export class ScoreNode { const { messages } = getConversationDisplay(lastMessage ? [lastMessage] : [], this.scenario?.modelResponse); let history = ""; if (messages.length === 1) { - history = messages[0].content; + const content = messages[0].content; + if (isTextContent(content)) { + history = content.text; + } } else if (messages.length > 1) { - history = messages.map(m => `[${m.participantName}] ${m.content}`).join("\n\n"); + history = messages + .filter(m => isTextContent(m.content)) + .map(m => `[${m.participantName}] ${(m.content as TextContent).text}`) + .join("\n\n"); } this.shortenedPrompt = shortenPrompt(history); @@ -284,10 +290,25 @@ const flattener = function* (node: ScoreNode): Iterable { } }; -const isTextContent = (content: AIContent): content is TextContent => { +export const isTextContent = (content: AIContent): content is TextContent => { return (content as TextContent).text !== undefined; }; +export const isImageContent = (content: AIContent): content is UriContent | DataContent => { + if ((content as UriContent).uri !== undefined && (content as UriContent).mediaType) { + return (content as UriContent).mediaType.startsWith("image/"); + } + + if ((content as DataContent).uri !== undefined) { + const dataContent = content as DataContent; + if (dataContent.uri.startsWith('data:image/')) { + return true; + } + } + + return false; +}; + export type ConversationDisplay = { messages: ChatMessageDisplay[]; model?: string; @@ -297,7 +318,7 @@ export type ConversationDisplay = { export type ChatMessageDisplay = { role: string; participantName: string; - content: string; + content: AIContent; }; export const getConversationDisplay = (messages: ChatMessage[], modelResponse?: ChatResponse): ConversationDisplay => { @@ -305,28 +326,24 @@ export const getConversationDisplay = (messages: ChatMessage[], modelResponse?: for (const m of messages) { for (const c of m.contents) { - if (isTextContent(c)) { - const participantName = m.authorName ? `${m.authorName} (${m.role})` : m.role; - chatMessages.push({ - role: m.role, - participantName: participantName, - content: c.text - }); - } + const participantName = m.authorName ? `${m.authorName} (${m.role})` : m.role; + chatMessages.push({ + role: m.role, + participantName: participantName, + content: c + }); } } if (modelResponse?.messages) { for (const m of modelResponse.messages) { for (const c of m.contents) { - if (isTextContent(c)) { - const participantName = m.authorName ? `${m.authorName} (${m.role})` : m.role || 'Assistant'; - chatMessages.push({ - role: m.role, - participantName: participantName, - content: c.text - }); - } + const participantName = m.authorName ? `${m.authorName} (${m.role})` : m.role || 'Assistant'; + chatMessages.push({ + role: m.role, + participantName: participantName, + content: c + }); } } } diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/AIContentExtensions.cs b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/AIContentExtensions.cs new file mode 100644 index 00000000000..4e33c64c305 --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/AIContentExtensions.cs @@ -0,0 +1,42 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; + +namespace Microsoft.Extensions.AI.Evaluation.Safety; +internal static class AIContentExtensions +{ + internal static bool IsTextOrUsage(this AIContent content) + => content is TextContent || content is UsageContent; + + internal static bool IsImageWithSupportedFormat(this AIContent content) => + (content is UriContent uriContent && IsSupportedImageFormat(uriContent.MediaType)) || + (content is DataContent dataContent && IsSupportedImageFormat(dataContent.MediaType)); + + internal static bool IsUriBase64Encoded(this DataContent dataContent) + { + ReadOnlyMemory uri = dataContent.Uri.AsMemory(); + + int commaIndex = uri.Span.IndexOf(','); + if (commaIndex == -1) + { + return false; + } + + ReadOnlyMemory metadata = uri.Slice(0, commaIndex); + + bool isBase64Encoded = metadata.Span.EndsWith(";base64".AsSpan(), StringComparison.OrdinalIgnoreCase); + return isBase64Encoded; + } + + private static bool IsSupportedImageFormat(string mediaType) + { + // 'image/jpeg' is the official MIME type for JPEG. However, some systems recognize 'image/jpg' as well. + + return + mediaType.Equals("image/jpeg", StringComparison.OrdinalIgnoreCase) || + mediaType.Equals("image/jpg", StringComparison.OrdinalIgnoreCase) || + mediaType.Equals("image/png", StringComparison.OrdinalIgnoreCase) || + mediaType.Equals("image/gif", StringComparison.OrdinalIgnoreCase); + } +} diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/ChatMessageExtensions.cs b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/ChatMessageExtensions.cs new file mode 100644 index 00000000000..0634a6bdcba --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/ChatMessageExtensions.cs @@ -0,0 +1,16 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Collections.Generic; +using System.Linq; + +namespace Microsoft.Extensions.AI.Evaluation.Safety; + +internal static class ChatMessageExtensions +{ + internal static bool ContainsImageWithSupportedFormat(this ChatMessage message) + => message.Contents.Any(c => c.IsImageWithSupportedFormat()); + + internal static bool ContainsImageWithSupportedFormat(this IEnumerable conversation) + => conversation.Any(ContainsImageWithSupportedFormat); +} diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/ChatResponseExtensions.cs b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/ChatResponseExtensions.cs new file mode 100644 index 00000000000..d8171aaf191 --- /dev/null +++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/ChatResponseExtensions.cs @@ -0,0 +1,10 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +namespace Microsoft.Extensions.AI.Evaluation.Safety; + +internal static class ChatResponseExtensions +{ + internal static bool ContainsImageWithSupportedFormat(this ChatResponse response) + => response.Messages.ContainsImageWithSupportedFormat(); +} diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/ContentSafetyServicePayloadUtilities.cs b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/ContentSafetyServicePayloadUtilities.cs index bb12bc6afec..a2694669106 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/ContentSafetyServicePayloadUtilities.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/ContentSafetyServicePayloadUtilities.cs @@ -12,19 +12,6 @@ namespace Microsoft.Extensions.AI.Evaluation.Safety; internal static class ContentSafetyServicePayloadUtilities { - internal static bool IsImage(this AIContent content) => - (content is UriContent uriContent && uriContent.HasTopLevelMediaType("image")) || - (content is DataContent dataContent && dataContent.HasTopLevelMediaType("image")); - - internal static bool ContainsImage(this ChatMessage message) - => message.Contents.Any(IsImage); - - internal static bool ContainsImage(this ChatResponse response) - => response.Messages.ContainsImage(); - - internal static bool ContainsImage(this IEnumerable conversation) - => conversation.Any(ContainsImage); - internal static (string payload, IReadOnlyList? diagnostics) GetPayload( ContentSafetyServicePayloadFormat payloadFormat, IEnumerable conversation, @@ -356,8 +343,17 @@ IEnumerable GetContents(ChatMessage message) } else if (content is DataContent dataContent && dataContent.HasTopLevelMediaType("image")) { - BinaryData imageBytes = BinaryData.FromBytes(dataContent.Data); - string base64ImageData = Convert.ToBase64String(imageBytes.ToArray()); + string url; + if (dataContent.IsUriBase64Encoded()) + { + url = dataContent.Uri; + } + else + { + BinaryData imageBytes = BinaryData.FromBytes(dataContent.Data); + string base64ImageData = Convert.ToBase64String(imageBytes.ToArray()); + url = $"data:{dataContent.MediaType};base64,{base64ImageData}"; + } yield return new JsonObject { @@ -365,7 +361,7 @@ IEnumerable GetContents(ChatMessage message) ["image_url"] = new JsonObject { - ["url"] = $"data:{dataContent.MediaType};base64,{base64ImageData}" + ["url"] = url } }; } @@ -475,7 +471,7 @@ void ValidateContents(ChatMessage message) if (areImagesSupported) { - if (content.IsImage()) + if (content.IsImageWithSupportedFormat()) { ++imagesCount; } @@ -533,7 +529,7 @@ void ValidateContents(ChatMessage message) EvaluationDiagnostic.Warning( $"The supplied conversation contained {unsupportedContentCount} instances of unsupported content within messages. " + $"The current evaluation being performed by {evaluatorName} only supports content of type '{nameof(TextContent)}', '{nameof(UriContent)}' and '{nameof(DataContent)}'. " + - $"For '{nameof(UriContent)}' and '{nameof(DataContent)}', only content with media type 'image/*' is supported. " + + $"For '{nameof(UriContent)}' and '{nameof(DataContent)}', only content with media type 'image/png', 'image/jpeg' and 'image/gif' are supported. " + $"The unsupported contents were ignored for this evaluation.")); } else @@ -582,7 +578,4 @@ void ValidateContents(ChatMessage message) return (turns, normalizedPerTurnContext, diagnostics, contentType); } - - private static bool IsTextOrUsage(this AIContent content) - => content is TextContent || content is UsageContent; } diff --git a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/ProtectedMaterialEvaluator.cs b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/ProtectedMaterialEvaluator.cs index d37bc17c94b..25c99306c32 100644 --- a/src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/ProtectedMaterialEvaluator.cs +++ b/src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/ProtectedMaterialEvaluator.cs @@ -87,7 +87,7 @@ await EvaluateContentSafetyAsync( // If images are present in the conversation, do a second evaluation for protected material in images. // The content safety service does not support evaluating both text and images in the same request currently. - if (messages.ContainsImage() || modelResponse.ContainsImage()) + if (messages.ContainsImageWithSupportedFormat() || modelResponse.ContainsImageWithSupportedFormat()) { EvaluationResult imageResult = await EvaluateContentSafetyAsync( diff --git a/test/Libraries/Microsoft.Extensions.AI.Evaluation.Integration.Tests/Microsoft.Extensions.AI.Evaluation.Integration.Tests.csproj b/test/Libraries/Microsoft.Extensions.AI.Evaluation.Integration.Tests/Microsoft.Extensions.AI.Evaluation.Integration.Tests.csproj index aff6aadaa2a..4679b04a338 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Evaluation.Integration.Tests/Microsoft.Extensions.AI.Evaluation.Integration.Tests.csproj +++ b/test/Libraries/Microsoft.Extensions.AI.Evaluation.Integration.Tests/Microsoft.Extensions.AI.Evaluation.Integration.Tests.csproj @@ -2,9 +2,18 @@ $(LatestTargetFramework) - Microsoft.Extensions.AI.Evaluation.Integration.Tests + Microsoft.Extensions.AI Integration tests for Microsoft.Extensions.AI.Evaluation. + + + + + + + + + diff --git a/test/Libraries/Microsoft.Extensions.AI.Evaluation.Integration.Tests/SafetyEvaluatorTests.cs b/test/Libraries/Microsoft.Extensions.AI.Evaluation.Integration.Tests/SafetyEvaluatorTests.cs index 270c091ecb2..3e4a7867ad5 100644 --- a/test/Libraries/Microsoft.Extensions.AI.Evaluation.Integration.Tests/SafetyEvaluatorTests.cs +++ b/test/Libraries/Microsoft.Extensions.AI.Evaluation.Integration.Tests/SafetyEvaluatorTests.cs @@ -247,15 +247,15 @@ public async Task EvaluateConversationWithImageInAnswer() await _imageContentSafetyReportingConfiguration.CreateScenarioRunAsync( scenarioName: $"Microsoft.Extensions.AI.Evaluation.Integration.Tests.{nameof(SafetyEvaluatorTests)}.{nameof(EvaluateConversationWithImageInAnswer)}"); - ChatMessage question = "Can you show me an image pertaining to Microsoft Copilot?".ToUserMessage(); + ChatMessage question = "Can you show me an image pertaining to DotNet?".ToUserMessage(); ChatMessage answer = new ChatMessage { Role = ChatRole.Assistant, Contents = [ - new TextContent("Here's an image pertaining to Microsoft Copilot:"), - new UriContent("https://uhf.microsoft.com/images/banners/RW1iGSh.png", "image/png")], + new TextContent("Here's an image pertaining to DotNet:"), + new DataContent(ImageDataUri.GetImageDataUri())], }; EvaluationResult result = await scenarioRun.EvaluateAsync(question, answer); @@ -280,10 +280,10 @@ await _imageContentSafetyReportingConfiguration.CreateScenarioRunAsync( Role = ChatRole.User, Contents = [ new TextContent("What does this image depict?"), - new UriContent("https://uhf.microsoft.com/images/microsoft/RE1Mu3b.png", "image/png")], + new DataContent(ImageDataUri.GetImageDataUri())], }; - ChatMessage answer1 = "The image depicts a logo for Microsoft Corporation.".ToAssistantMessage(); + ChatMessage answer1 = "The image depicts a logo for DotNet.".ToAssistantMessage(); ChatMessage question2 = "Can you show me an image pertaining to Microsoft Copilot?".ToUserMessage();