Remove the abstraction for token counting from the main evaluation API

This change is being made because there is still some uncertainty around what a general purpose token counting abstraction (that supports all kinds of future models, and all kinds of input modalities) should look like at the moment. We do not want to bake in an API that only supports text based inputs for the models and use cases that are prevalent today, since it would be a potential breaking change to change this API after we release a stable version of the evaluation APIs.

We can always reintroduce the token counting support in a non-breaking fashion in the future if and when there is more clarity on what a general purpose token counting abstraction should look like, or if and when a general purpose token counting abstraction is introduced in a lower layer (Microsoft.Extensions.AI) in the future.

In the meanwhile, callers should still be able to use the Microsoft.ML.Tokenizers library directly to count tokens in text-based content and trim down the conversation history before calling EvaluateAsync() if needed.

Author: shyamnamboodiripad

src/Libraries/Microsoft.Extensions.AI.Evaluation.Quality/ChatConversationEvaluator.cs

@@ -57,99 +57,6 @@ public virtual async ValueTask<EvaluationResult> EvaluateAsync(
 
         (ChatMessage? userRequest, List<ChatMessage> history) = GetUserRequestAndHistory(messages);
 
-        int inputTokenLimit = 0;
-        int ignoredMessagesCount = 0;
-
-        if (chatConfiguration.TokenCounter is not null)
-        {
-            IEvaluationTokenCounter tokenCounter = chatConfiguration.TokenCounter;
-            inputTokenLimit = tokenCounter.InputTokenLimit;
-            int tokenBudget = inputTokenLimit;
-
-            void OnTokenBudgetExceeded()
-            {
-                EvaluationDiagnostic tokenBudgetExceeded =
-                    EvaluationDiagnostic.Error(
-                        $"Evaluation failed because the specified limit of {inputTokenLimit} input tokens was exceeded.");
-
-                result.AddDiagnosticsToAllMetrics(tokenBudgetExceeded);
-            }
-
-            if (!string.IsNullOrWhiteSpace(SystemPrompt))
-            {
-                tokenBudget -= tokenCounter.CountTokens(SystemPrompt!);
-                if (tokenBudget < 0)
-                {
-                    OnTokenBudgetExceeded();
-                    return result;
-                }
-            }
-
-            string baseEvaluationPrompt =
-                await RenderEvaluationPromptAsync(
-                    userRequest,
-                    modelResponse,
-                    includedHistory: [],
-                    additionalContext,
-                    cancellationToken).ConfigureAwait(false);
-
-            tokenBudget -= tokenCounter.CountTokens(baseEvaluationPrompt);
-            if (tokenBudget < 0)
-            {
-                OnTokenBudgetExceeded();
-                return result;
-            }
-
-            if (history.Count > 0 && !IgnoresHistory)
-            {
-                if (history.Count == 1)
-                {
-                    (bool canRender, tokenBudget) =
-                        await CanRenderAsync(
-                            history[0],
-                            tokenBudget,
-                            chatConfiguration,
-                            cancellationToken).ConfigureAwait(false);
-
-                    if (!canRender)
-                    {
-                        ignoredMessagesCount = 1;
-                        history = [];
-                    }
-                }
-                else
-                {
-                    int totalMessagesCount = history.Count;
-                    int includedMessagesCount = 0;
-
-                    history.Reverse();
-
-                    foreach (ChatMessage message in history)
-                    {
-                        cancellationToken.ThrowIfCancellationRequested();
-
-                        (bool canRender, tokenBudget) =
-                            await CanRenderAsync(
-                                message,
-                                tokenBudget,
-                                chatConfiguration,
-                                cancellationToken).ConfigureAwait(false);
-
-                        if (!canRender)
-                        {
-                            ignoredMessagesCount = totalMessagesCount - includedMessagesCount;
-                            history.RemoveRange(index: includedMessagesCount, count: ignoredMessagesCount);
-                            break;
-                        }
-
-                        includedMessagesCount++;
-                    }
-
-                    history.Reverse();
-                }
-            }
-        }
-
         var evaluationMessages = new List<ChatMessage>();
         if (!string.IsNullOrWhiteSpace(SystemPrompt))
         {
@@ -172,84 +79,9 @@ await PerformEvaluationAsync(
             result,
             cancellationToken).ConfigureAwait(false);
 
-        if (inputTokenLimit > 0 && ignoredMessagesCount > 0)
-        {
-#pragma warning disable S103 // Lines should not be too long
-            result.AddDiagnosticsToAllMetrics(
-                EvaluationDiagnostic.Warning(
-                    $"The evaluation may be inconclusive because the oldest {ignoredMessagesCount} messages in the supplied conversation history were ignored in order to stay under the specified limit of {inputTokenLimit} input tokens."));
-#pragma warning restore S103
-        }
-
         return result;
     }
 
-    /// <summary>
-    /// Determines if there is sufficient <paramref name="tokenBudget"/> remaining to render the
-    /// supplied <paramref name="message"/> as part of the evaluation prompt that this <see cref="IEvaluator"/> uses.
-    /// </summary>
-    /// <param name="message">
-    /// A message that is part of the conversation history for the response being evaluated and that is to be rendered
-    /// as part of the evaluation prompt.
-    /// </param>
-    /// <param name="tokenBudget">
-    /// The number of tokens available for the rendering additional content as part of the evaluation prompt.
-    /// </param>
-    /// <param name="chatConfiguration">
-    /// A <see cref="ChatConfiguration"/> that specifies the <see cref="IChatClient"/> and the
-    /// <see cref="IEvaluationTokenCounter"/> that this <see cref="IEvaluator"/> uses to perform the evaluation.
-    /// </param>
-    /// <param name="cancellationToken">A <see cref="CancellationToken"/> that can cancel the operation.</param>
-    /// <returns>
-    /// A tuple containing a <see langword="bool"/> indicating whether there is sufficient
-    /// <paramref name="tokenBudget"/> remaining to render the supplied <paramref name="message"/> as part of the
-    /// evaluation prompt, and an <see langword="int"/> containing the remaining token budget that would be available
-    /// once this <paramref name="message"/> is rendered.
-    /// </returns>
-    protected virtual ValueTask<(bool canRender, int remainingTokenBudget)> CanRenderAsync(
-        ChatMessage message,
-        int tokenBudget,
-        ChatConfiguration chatConfiguration,
-        CancellationToken cancellationToken)
-    {
-        _ = Throw.IfNull(message);
-        _ = Throw.IfNull(chatConfiguration);
-
-        IEvaluationTokenCounter? tokenCounter = chatConfiguration.TokenCounter;
-        if (tokenCounter is null)
-        {
-            return new ValueTask<(bool, int)>((true, tokenBudget));
-        }
-
-        string? author = message.AuthorName;
-        string role = message.Role.Value;
-        string content = message.Text ?? string.Empty;
-
-        int tokenCount =
-            string.IsNullOrWhiteSpace(author)
-                ? tokenCounter.CountTokens("[") +
-                    tokenCounter.CountTokens(role) +
-                    tokenCounter.CountTokens("] ") +
-                    tokenCounter.CountTokens(content) +
-                    tokenCounter.CountTokens("\n")
-                : tokenCounter.CountTokens("[") +
-                    tokenCounter.CountTokens(author!) +
-                    tokenCounter.CountTokens(" (") +
-                    tokenCounter.CountTokens(role) +
-                    tokenCounter.CountTokens(")] ") +
-                    tokenCounter.CountTokens(content) +
-                    tokenCounter.CountTokens("\n");
-
-        if (tokenCount > tokenBudget)
-        {
-            return new ValueTask<(bool, int)>((false, tokenBudget));
-        }
-        else
-        {
-            return new ValueTask<(bool, int)>((true, tokenBudget - tokenCount));
-        }
-    }
-
     /// <summary>
     /// Renders the supplied <paramref name="response"/> to a string that can be included as part of the evaluation
     /// prompt that this <see cref="IEvaluator"/> uses.
@@ -351,8 +183,8 @@ protected abstract ValueTask<string> RenderEvaluationPromptAsync(
     /// <see cref="EvaluationMetric"/>s in the supplied <paramref name="result"/>.
     /// </summary>
     /// <param name="chatConfiguration">
-    /// A <see cref="ChatConfiguration"/> that specifies the <see cref="IChatClient"/> and the
-    /// <see cref="IEvaluationTokenCounter"/> that this <see cref="IEvaluator"/> uses to perform the evaluation.
+    /// A <see cref="ChatConfiguration"/> that specifies the <see cref="IChatClient"/> that should be used if one or
+    /// more composed <see cref="IEvaluator"/>s use an AI model to perform evaluation.
     /// </param>
     /// <param name="evaluationMessages">
     /// The set of messages that are to be sent to the supplied <see cref="ChatConfiguration.ChatClient"/> to perform

src/Libraries/Microsoft.Extensions.AI.Evaluation.Quality/Utilities/JsonOutputFixer.cs

@@ -24,7 +24,7 @@ internal static ReadOnlySpan<char> TrimMarkdownDelimiters(string json)
         // Trim 'json' marker from markdown if it exists.
         const string JsonMarker = "json";
         int markerLength = JsonMarker.Length;
-        if (trimmed.Length > markerLength && trimmed[0..markerLength].SequenceEqual(JsonMarker.AsSpan()))
+        if (trimmed.Length > markerLength && trimmed.Slice(0, markerLength).SequenceEqual(JsonMarker.AsSpan()))
         {
             trimmed = trimmed.Slice(markerLength);
         }

src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting.Azure/Storage/AzureStorageReportingConfiguration.cs

@@ -29,10 +29,9 @@ public static class AzureStorageReportingConfiguration
     /// survive in the cache before they are considered expired and evicted.
     /// </param>
     /// <param name="chatConfiguration">
-    /// A <see cref="ChatConfiguration"/> that specifies the <see cref="IChatClient"/> and the
-    /// <see cref="IEvaluationTokenCounter"/> that are used by AI-based <paramref name="evaluators"/> included in the
-    /// returned <see cref="ReportingConfiguration"/>. Can be omitted if none of the included
-    /// <paramref name="evaluators"/> are AI-based.
+    /// A <see cref="ChatConfiguration"/> that specifies the <see cref="IChatClient"/> that is used by AI-based
+    /// <paramref name="evaluators"/> included in the returned <see cref="ReportingConfiguration"/>. Can be omitted if
+    /// none of the included <paramref name="evaluators"/> are AI-based.
     /// </param>
     /// <param name="enableResponseCaching">
     /// <see langword="true"/> to enable caching of AI responses; <see langword="false"/> otherwise.

src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/ReportingConfiguration.cs

@@ -30,9 +30,8 @@ public sealed class ReportingConfiguration
     public IResultStore ResultStore { get; }
 
     /// <summary>
-    /// Gets a <see cref="Evaluation.ChatConfiguration"/> that specifies the <see cref="IChatClient"/> and the
-    /// <see cref="IEvaluationTokenCounter"/> that are used by AI-based <see cref="Evaluators"/> included in this
-    /// <see cref="ReportingConfiguration"/>.
+    /// Gets a <see cref="Evaluation.ChatConfiguration"/> that specifies the <see cref="IChatClient"/> that is used by
+    /// AI-based <see cref="Evaluators"/> included in this <see cref="ReportingConfiguration"/>.
     /// </summary>
     public ChatConfiguration? ChatConfiguration { get; }
 
@@ -103,10 +102,9 @@ public sealed class ReportingConfiguration
     /// The <see cref="IResultStore"/> that should be used to persist the <see cref="ScenarioRunResult"/>s.
     /// </param>
     /// <param name="chatConfiguration">
-    /// A <see cref="Evaluation.ChatConfiguration"/> that specifies the <see cref="IChatClient"/> and the
-    /// <see cref="IEvaluationTokenCounter"/> that are used by AI-based <paramref name="evaluators"/> included in this
-    /// <see cref="ReportingConfiguration"/>. Can be omitted if none of the included <paramref name="evaluators"/> are
-    /// AI-based.
+    /// A <see cref="Evaluation.ChatConfiguration"/> that specifies the <see cref="IChatClient"/> that is used by
+    /// AI-based <paramref name="evaluators"/> included in this <see cref="ReportingConfiguration"/>. Can be omitted if
+    /// none of the included <paramref name="evaluators"/> are AI-based.
     /// </param>
     /// <param name="responseCacheProvider">
     /// The <see cref="IResponseCacheProvider"/> that should be used to cache AI responses. If omitted, AI responses
@@ -246,7 +244,7 @@ await ResponseCacheProvider.GetCacheAsync(
             }
 #pragma warning restore CA2000
 
-            chatConfiguration = new ChatConfiguration(chatClient, chatConfiguration.TokenCounter);
+            chatConfiguration = new ChatConfiguration(chatClient);
         }
 
         return new ScenarioRun(

src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/ScenarioRun.cs

@@ -80,9 +80,9 @@ public sealed class ScenarioRun : IAsyncDisposable
     public string ExecutionName { get; }
 
     /// <summary>
-    /// Gets a <see cref="Evaluation.ChatConfiguration"/> that specifies the <see cref="IChatClient"/> and the
-    /// <see cref="IEvaluationTokenCounter"/> that are used by AI-based <see cref="IEvaluator"/>s that are invoked as
-    /// part of the evaluation of this <see cref="ScenarioRun"/>.
+    /// Gets a <see cref="Evaluation.ChatConfiguration"/> that specifies the <see cref="IChatClient"/> that is used by
+    /// AI-based <see cref="IEvaluator"/>s that are invoked as part of the evaluation of this
+    /// <see cref="ScenarioRun"/>.
     /// </summary>
     public ChatConfiguration? ChatConfiguration { get; }
 

src/Libraries/Microsoft.Extensions.AI.Evaluation.Reporting/CSharp/Storage/DiskBasedReportingConfiguration.cs

@@ -25,10 +25,9 @@ public static class DiskBasedReportingConfiguration
     /// The set of <see cref="IEvaluator"/>s that should be invoked to evaluate AI responses.
     /// </param>
     /// <param name="chatConfiguration">
-    /// A <see cref="ChatConfiguration"/> that specifies the <see cref="IChatClient"/> and the
-    /// <see cref="IEvaluationTokenCounter"/> that are used by AI-based <paramref name="evaluators"/> included in the
-    /// returned <see cref="ReportingConfiguration"/>. Can be omitted if none of the included
-    /// <paramref name="evaluators"/> are AI-based.
+    /// A <see cref="ChatConfiguration"/> that specifies the <see cref="IChatClient"/> that is used by AI-based
+    /// <paramref name="evaluators"/> included in the returned <see cref="ReportingConfiguration"/>. Can be omitted if
+    /// none of the included <paramref name="evaluators"/> are AI-based.
     /// </param>
     /// <param name="enableResponseCaching">
     /// <see langword="true"/> to enable caching of AI responses; <see langword="false"/> otherwise.

src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/ContentSafetyServiceConfigurationExtensions.cs

@@ -46,7 +46,7 @@ public static ChatConfiguration ToChatConfiguration(
                 originalChatClient: originalChatConfiguration?.ChatClient);
 #pragma warning restore CA2000
 
-        return new ChatConfiguration(newChatClient, originalChatConfiguration?.TokenCounter);
+        return new ChatConfiguration(newChatClient);
     }
 
     /// <summary>

src/Libraries/Microsoft.Extensions.AI.Evaluation.Safety/Microsoft.Extensions.AI.Evaluation.Safety.csproj

@@ -15,6 +15,10 @@
     <MinCodeCoverage>0</MinCodeCoverage>
     <MinMutationScore>0</MinMutationScore>
   </PropertyGroup>
+  
+  <ItemGroup>
+    <PackageReference Include="Microsoft.Bcl.HashCode" />
+  </ItemGroup>
 
   <ItemGroup>
     <ProjectReference Include="..\Microsoft.Extensions.AI.Evaluation\Microsoft.Extensions.AI.Evaluation.csproj" />

src/Libraries/Microsoft.Extensions.AI.Evaluation/ChatConfiguration.cs

@@ -9,27 +9,13 @@
 namespace Microsoft.Extensions.AI.Evaluation;
 
 /// <summary>
-/// Specifies the <see cref="IChatClient"/> and the <see cref="IEvaluationTokenCounter"/> that should be used when
-/// evaluation is performed using an AI model.
+/// Specifies the <see cref="IChatClient"/> that should be used when evaluation is performed using an AI model.
 /// </summary>
 /// <param name="chatClient">An <see cref="IChatClient"/> that can be used to communicate with an AI model.</param>
-/// <param name="tokenCounter">
-/// An <see cref="IEvaluationTokenCounter"/> that can be used to counts tokens present in evaluation prompts, or
-/// <see langword="null"/> if the AI model / deployment being used does not impose an input token limit.
-/// </param>
-public sealed class ChatConfiguration(IChatClient chatClient, IEvaluationTokenCounter? tokenCounter = null)
+public sealed class ChatConfiguration(IChatClient chatClient)
 {
     /// <summary>
     /// Gets an <see cref="IChatClient"/> that can be used to communicate with an AI model.
     /// </summary>
     public IChatClient ChatClient { get; } = chatClient;
-
-    /// <summary>
-    /// Gets an <see cref="IEvaluationTokenCounter"/> that can be used to counts tokens present in evaluation prompts.
-    /// </summary>
-    /// <remarks>
-    /// <see cref="TokenCounter"/> can be set to <see langword="null"/> if the AI model / deployment being used does
-    /// not impose an input token limit.
-    /// </remarks>
-    public IEvaluationTokenCounter? TokenCounter { get; } = tokenCounter;
 }

src/Libraries/Microsoft.Extensions.AI.Evaluation/CompositeEvaluator.cs

@@ -88,9 +88,8 @@ public CompositeEvaluator(IEnumerable<IEvaluator> evaluators)
     /// </param>
     /// <param name="modelResponse">The response that is to be evaluated.</param>
     /// <param name="chatConfiguration">
-    /// A <see cref="ChatConfiguration"/> that specifies the <see cref="IChatClient"/> and the
-    /// <see cref="IEvaluationTokenCounter"/> that should be used if one or more composed <see cref="IEvaluator"/>s use
-    /// an AI model to perform evaluation.
+    /// A <see cref="ChatConfiguration"/> that specifies the <see cref="IChatClient"/> that should be used if one or
+    /// more composed <see cref="IEvaluator"/>s use an AI model to perform evaluation.
     /// </param>
     /// <param name="additionalContext">
     /// Additional contextual information (beyond that which is available in <paramref name="messages"/>) that composed