From 9858a75e678bd3fcebc502a07c4e4d4a9334a6ca Mon Sep 17 00:00:00 2001
From: Roger Barreto <19890735+RogerBarreto@users.noreply.github.com>
Date: Sat, 19 Apr 2025 17:28:49 +0100
Subject: [PATCH 01/24] Created potential extensible
FunctionInvokingChatClientV2
---
.../FunctionInvokingChatClientV2.Extra.cs | 123 ++
.../FunctionInvokingChatClientV2.cs | 886 ++++++++++++++
.../KernelFunctionInvokingChatClient.cs | 754 +-----------
.../KernelFunctionInvokingChatClientOld.cs | 1023 +++++++++++++++++
4 files changed, 2077 insertions(+), 709 deletions(-)
create mode 100644 dotnet/src/SemanticKernel.Abstractions/AI/ChatClient/FunctionInvokingChatClientV2.Extra.cs
create mode 100644 dotnet/src/SemanticKernel.Abstractions/AI/ChatClient/FunctionInvokingChatClientV2.cs
create mode 100644 dotnet/src/SemanticKernel.Abstractions/AI/ChatClient/KernelFunctionInvokingChatClientOld.cs
diff --git a/dotnet/src/SemanticKernel.Abstractions/AI/ChatClient/FunctionInvokingChatClientV2.Extra.cs b/dotnet/src/SemanticKernel.Abstractions/AI/ChatClient/FunctionInvokingChatClientV2.Extra.cs
new file mode 100644
index 000000000000..12edbbb4d15e
--- /dev/null
+++ b/dotnet/src/SemanticKernel.Abstractions/AI/ChatClient/FunctionInvokingChatClientV2.Extra.cs
@@ -0,0 +1,123 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+// Intended for deletion
+
+using System;
+using System.Diagnostics.CodeAnalysis;
+using System.Runtime.CompilerServices;
+
+namespace Microsoft.Extensions.AI;
+
+public partial class FunctionInvokingChatClientV2 : DelegatingChatClient
+{
+ private static class Throw
+ {
+ ///
+ /// Throws an if the specified argument is .
+ ///
+ /// Argument type to be checked for .
+ /// Object to be checked for .
+ /// The name of the parameter being checked.
+ /// The original value of .
+ [MethodImpl(MethodImplOptions.AggressiveInlining)]
+ [return: NotNull]
+ public static T IfNull([NotNull] T argument, [CallerArgumentExpression(nameof(argument))] string paramName = "")
+ {
+ if (argument is null)
+ {
+ throw new ArgumentNullException(paramName);
+ }
+
+ return argument;
+ }
+
+ ///
+ /// Throws an .
+ ///
+ /// A message that describes the error.
+#if !NET6_0_OR_GREATER
+ [MethodImpl(MethodImplOptions.NoInlining)]
+#endif
+ [DoesNotReturn]
+ public static void InvalidOperationException(string message)
+ => throw new InvalidOperationException(message);
+
+ ///
+ /// Throws an .
+ ///
+ /// The name of the parameter that caused the exception.
+#if !NET6_0_OR_GREATER
+ [MethodImpl(MethodImplOptions.NoInlining)]
+#endif
+ [DoesNotReturn]
+ public static void ArgumentOutOfRangeException(string paramName)
+ => throw new ArgumentOutOfRangeException(paramName);
+
+ ///
+ /// Throws an .
+ ///
+ /// The name of the parameter that caused the exception.
+ /// A message that describes the error.
+#if !NET6_0_OR_GREATER
+ [MethodImpl(MethodImplOptions.NoInlining)]
+#endif
+ [DoesNotReturn]
+ public static void ArgumentOutOfRangeException(string paramName, string? message)
+ => throw new ArgumentOutOfRangeException(paramName, message);
+
+ ///
+ /// Throws an .
+ ///
+ /// The name of the parameter that caused the exception.
+ /// The value of the argument that caused this exception.
+ /// A message that describes the error.
+#if !NET6_0_OR_GREATER
+ [MethodImpl(MethodImplOptions.NoInlining)]
+#endif
+ [DoesNotReturn]
+ public static void ArgumentOutOfRangeException(string paramName, object? actualValue, string? message)
+ => throw new ArgumentOutOfRangeException(paramName, actualValue, message);
+
+ ///
+ /// Throws an if the specified number is less than min.
+ ///
+ /// Number to be expected being less than min.
+ /// The number that must be less than the argument.
+ /// The name of the parameter being checked.
+ /// The original value of .
+ [MethodImpl(MethodImplOptions.AggressiveInlining)]
+ public static int IfLessThan(int argument, int min, [CallerArgumentExpression(nameof(argument))] string paramName = "")
+ {
+ if (argument < min)
+ {
+ ArgumentOutOfRangeException(paramName, argument, $"Argument less than minimum value {min}");
+ }
+
+ return argument;
+ }
+ }
+
+ private static class LoggingHelpers
+ {
+ /// Serializes as JSON for logging purposes.
+ public static string AsJson(T value, System.Text.Json.JsonSerializerOptions? options)
+ {
+ if (options?.TryGetTypeInfo(typeof(T), out var typeInfo) is true ||
+ AIJsonUtilities.DefaultOptions.TryGetTypeInfo(typeof(T), out typeInfo))
+ {
+ try
+ {
+ return System.Text.Json.JsonSerializer.Serialize(value, typeInfo);
+ }
+#pragma warning disable CA1031 // Do not catch general exception types
+ catch
+#pragma warning restore CA1031 // Do not catch general exception types
+ {
+ }
+ }
+
+ return "{}";
+ }
+ }
+}
diff --git a/dotnet/src/SemanticKernel.Abstractions/AI/ChatClient/FunctionInvokingChatClientV2.cs b/dotnet/src/SemanticKernel.Abstractions/AI/ChatClient/FunctionInvokingChatClientV2.cs
new file mode 100644
index 000000000000..f2f3d9f12a62
--- /dev/null
+++ b/dotnet/src/SemanticKernel.Abstractions/AI/ChatClient/FunctionInvokingChatClientV2.cs
@@ -0,0 +1,886 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.Diagnostics.CodeAnalysis;
+using System.Linq;
+using System.Runtime.CompilerServices;
+using System.Runtime.ExceptionServices;
+using System.Threading;
+using System.Threading.Tasks;
+using Microsoft.Extensions.Logging;
+using Microsoft.Extensions.Logging.Abstractions;
+
+#pragma warning disable CA2213 // Disposable fields should be disposed
+#pragma warning disable EA0002 // Use 'System.TimeProvider' to make the code easier to test
+#pragma warning disable SA1202 // 'protected' members should come before 'private' members
+#pragma warning disable VSTHRD111 // Use ConfigureAwait(bool)
+#pragma warning disable CA2007 // Use ConfigureAwait
+
+namespace Microsoft.Extensions.AI;
+
+///
+/// A delegating chat client that invokes functions defined on .
+/// Include this in a chat pipeline to resolve function calls automatically.
+///
+///
+///
+/// When this client receives a in a chat response, it responds
+/// by calling the corresponding defined in ,
+/// producing a that it sends back to the inner client. This loop
+/// is repeated until there are no more function calls to make, or until another stop condition is met,
+/// such as hitting .
+///
+///
+/// The provided implementation of is thread-safe for concurrent use so long as the
+/// instances employed as part of the supplied are also safe.
+/// The property can be used to control whether multiple function invocation
+/// requests as part of the same request are invocable concurrently, but even with that set to
+/// (the default), multiple concurrent requests to this same instance and using the same tools could result in those
+/// tools being used concurrently (one per request). For example, a function that accesses the HttpContext of a specific
+/// ASP.NET web request should only be used as part of a single at a time, and only with
+/// set to , in case the inner client decided to issue multiple
+/// invocation requests to that same function.
+///
+///
+public partial class FunctionInvokingChatClientV2 : DelegatingChatClient
+{
+ /// The for the current function invocation.
+ private static readonly AsyncLocal _currentContext = new();
+
+ /// Optional services used for function invocation.
+ private readonly IServiceProvider? _functionInvocationServices;
+
+ /// The logger to use for logging information about function invocation.
+ protected readonly ILogger _logger;
+
+ /// The to use for telemetry.
+ /// This component does not own the instance and should not dispose it.
+ protected readonly ActivitySource? _activitySource;
+
+ /// Maximum number of roundtrips allowed to the inner client.
+ private int _maximumIterationsPerRequest = 10;
+
+ /// Maximum number of consecutive iterations that are allowed contain at least one exception result. If the limit is exceeded, we rethrow the exception instead of continuing.
+ private int _maximumConsecutiveErrorsPerRequest = 3;
+
+ ///
+ /// Initializes a new instance of the class.
+ ///
+ /// The underlying , or the next instance in a chain of clients.
+ /// An to use for logging information about function invocation.
+ /// An optional to use for resolving services required by the instances being invoked.
+ public FunctionInvokingChatClientV2(IChatClient innerClient, ILoggerFactory? loggerFactory = null, IServiceProvider? functionInvocationServices = null)
+ : base(innerClient)
+ {
+ _logger = (ILogger?)loggerFactory?.CreateLogger() ?? NullLogger.Instance;
+ _activitySource = innerClient.GetService();
+ _functionInvocationServices = functionInvocationServices;
+ }
+
+ ///
+ /// Gets or sets the for the current function invocation.
+ ///
+ ///
+ /// This value flows across async calls.
+ ///
+ public static FunctionInvocationContext? CurrentContext
+ {
+ get => _currentContext.Value;
+ protected set => _currentContext.Value = value;
+ }
+
+ ///
+ /// Gets or sets a value indicating whether detailed exception information should be included
+ /// in the chat history when calling the underlying .
+ ///
+ ///
+ /// if the full exception message is added to the chat history
+ /// when calling the underlying .
+ /// if a generic error message is included in the chat history.
+ /// The default value is .
+ ///
+ ///
+ ///
+ /// Setting the value to prevents the underlying language model from disclosing
+ /// raw exception details to the end user, since it doesn't receive that information. Even in this
+ /// case, the raw object is available to application code by inspecting
+ /// the property.
+ ///
+ ///
+ /// Setting the value to can help the underlying bypass problems on
+ /// its own, for example by retrying the function call with different arguments. However it might
+ /// result in disclosing the raw exception information to external users, which can be a security
+ /// concern depending on the application scenario.
+ ///
+ ///
+ /// Changing the value of this property while the client is in use might result in inconsistencies
+ /// as to whether detailed errors are provided during an in-flight request.
+ ///
+ ///
+ public bool IncludeDetailedErrors { get; set; }
+
+ ///
+ /// Gets or sets a value indicating whether to allow concurrent invocation of functions.
+ ///
+ ///
+ /// if multiple function calls can execute in parallel.
+ /// if function calls are processed serially.
+ /// The default value is .
+ ///
+ ///
+ /// An individual response from the inner client might contain multiple function call requests.
+ /// By default, such function calls are processed serially. Set to
+ /// to enable concurrent invocation such that multiple function calls can execute in parallel.
+ ///
+ public bool AllowConcurrentInvocation { get; set; }
+
+ ///
+ /// Gets or sets the maximum number of iterations per request.
+ ///
+ ///
+ /// The maximum number of iterations per request.
+ /// The default value is 10.
+ ///
+ ///
+ ///
+ /// Each request to this might end up making
+ /// multiple requests to the inner client. Each time the inner client responds with
+ /// a function call request, this client might perform that invocation and send the results
+ /// back to the inner client in a new request. This property limits the number of times
+ /// such a roundtrip is performed. The value must be at least one, as it includes the initial request.
+ ///
+ ///
+ /// Changing the value of this property while the client is in use might result in inconsistencies
+ /// as to how many iterations are allowed for an in-flight request.
+ ///
+ ///
+ public int MaximumIterationsPerRequest
+ {
+ get => _maximumIterationsPerRequest;
+ set
+ {
+ if (value < 1)
+ {
+ Throw.ArgumentOutOfRangeException(nameof(value));
+ }
+
+ _maximumIterationsPerRequest = value;
+ }
+ }
+
+ ///
+ /// Gets or sets the maximum number of consecutive iterations that are allowed to fail with an error.
+ ///
+ ///
+ /// The maximum number of consecutive iterations that are allowed to fail with an error.
+ /// The default value is 3.
+ ///
+ ///
+ ///
+ /// When function invocations fail with an exception, the
+ /// continues to make requests to the inner client, optionally supplying exception information (as
+ /// controlled by ). This allows the to
+ /// recover from errors by trying other function parameters that may succeed.
+ ///
+ ///
+ /// However, in case function invocations continue to produce exceptions, this property can be used to
+ /// limit the number of consecutive failing attempts. When the limit is reached, the exception will be
+ /// rethrown to the caller.
+ ///
+ ///
+ /// If the value is set to zero, all function calling exceptions immediately terminate the function
+ /// invocation loop and the exception will be rethrown to the caller.
+ ///
+ ///
+ /// Changing the value of this property while the client is in use might result in inconsistencies
+ /// as to how many iterations are allowed for an in-flight request.
+ ///
+ ///
+ public int MaximumConsecutiveErrorsPerRequest
+ {
+ get => _maximumConsecutiveErrorsPerRequest;
+ set => _maximumConsecutiveErrorsPerRequest = Throw.IfLessThan(value, 0);
+ }
+
+ ///
+ public override async Task GetResponseAsync(
+ IEnumerable messages, ChatOptions? options = null, CancellationToken cancellationToken = default)
+ {
+ _ = Throw.IfNull(messages);
+
+ // A single request into this GetResponseAsync may result in multiple requests to the inner client.
+ // Create an activity to group them together for better observability.
+ using Activity? activity = _activitySource?.StartActivity(nameof(FunctionInvokingChatClient));
+
+ // Copy the original messages in order to avoid enumerating the original messages multiple times.
+ // The IEnumerable can represent an arbitrary amount of work.
+ List originalMessages = [.. messages];
+ messages = originalMessages;
+
+ List? augmentedHistory = null; // the actual history of messages sent on turns other than the first
+ ChatResponse? response = null; // the response from the inner client, which is possibly modified and then eventually returned
+ List? responseMessages = null; // tracked list of messages, across multiple turns, to be used for the final response
+ UsageDetails? totalUsage = null; // tracked usage across all turns, to be used for the final response
+ List? functionCallContents = null; // function call contents that need responding to in the current turn
+ bool lastIterationHadThreadId = false; // whether the last iteration's response had a ChatThreadId set
+ int consecutiveErrorCount = 0;
+
+ for (int iteration = 0; ; iteration++)
+ {
+ functionCallContents?.Clear();
+
+ // Make the call to the inner client.
+ response = await base.GetResponseAsync(messages, options, cancellationToken);
+ if (response is null)
+ {
+ Throw.InvalidOperationException($"The inner {nameof(IChatClient)} returned a null {nameof(ChatResponse)}.");
+ }
+
+ // Any function call work to do? If yes, ensure we're tracking that work in functionCallContents.
+ bool requiresFunctionInvocation =
+ options?.Tools is { Count: > 0 } &&
+ iteration < MaximumIterationsPerRequest &&
+ CopyFunctionCalls(response.Messages, ref functionCallContents);
+
+ // In a common case where we make a request and there's no function calling work required,
+ // fast path out by just returning the original response.
+ if (iteration == 0 && !requiresFunctionInvocation)
+ {
+ return response;
+ }
+
+ // Track aggregatable details from the response, including all of the response messages and usage details.
+ (responseMessages ??= []).AddRange(response.Messages);
+ if (response.Usage is not null)
+ {
+ if (totalUsage is not null)
+ {
+ totalUsage.Add(response.Usage);
+ }
+ else
+ {
+ totalUsage = response.Usage;
+ }
+ }
+
+ // If there are no tools to call, or for any other reason we should stop, we're done.
+ // Break out of the loop and allow the handling at the end to configure the response
+ // with aggregated data from previous requests.
+ if (!requiresFunctionInvocation)
+ {
+ break;
+ }
+
+ // Prepare the history for the next iteration.
+ FixupHistories(originalMessages, ref messages, ref augmentedHistory, response, responseMessages, ref lastIterationHadThreadId);
+
+ // Add the responses from the function calls into the augmented history and also into the tracked
+ // list of response messages.
+ var modeAndMessages = await ProcessFunctionCallsAsync(response, augmentedHistory, options!, functionCallContents!, iteration, consecutiveErrorCount, isStreaming: false, _functionInvocationServices, cancellationToken);
+ responseMessages.AddRange(modeAndMessages.MessagesAdded);
+ consecutiveErrorCount = modeAndMessages.NewConsecutiveErrorCount;
+
+ if (modeAndMessages.ShouldTerminate)
+ {
+ break;
+ }
+
+ UpdateOptionsForNextIteration(ref options!, response.ChatThreadId);
+ }
+
+ Debug.Assert(responseMessages is not null, "Expected to only be here if we have response messages.");
+ response.Messages = responseMessages!;
+ response.Usage = totalUsage;
+
+ return response;
+ }
+
+ ///
+ public override async IAsyncEnumerable GetStreamingResponseAsync(
+ IEnumerable messages, ChatOptions? options = null, [EnumeratorCancellation] CancellationToken cancellationToken = default)
+ {
+ _ = Throw.IfNull(messages);
+
+ // A single request into this GetStreamingResponseAsync may result in multiple requests to the inner client.
+ // Create an activity to group them together for better observability.
+ using Activity? activity = _activitySource?.StartActivity(nameof(FunctionInvokingChatClient));
+
+ // Copy the original messages in order to avoid enumerating the original messages multiple times.
+ // The IEnumerable can represent an arbitrary amount of work.
+ List originalMessages = [.. messages];
+ messages = originalMessages;
+
+ List? augmentedHistory = null; // the actual history of messages sent on turns other than the first
+ List? functionCallContents = null; // function call contents that need responding to in the current turn
+ List? responseMessages = null; // tracked list of messages, across multiple turns, to be used in fallback cases to reconstitute history
+ bool lastIterationHadThreadId = false; // whether the last iteration's response had a ChatThreadId set
+ List updates = []; // updates from the current response
+ int consecutiveErrorCount = 0;
+
+ for (int iteration = 0; ; iteration++)
+ {
+ updates.Clear();
+ functionCallContents?.Clear();
+
+ await foreach (var update in base.GetStreamingResponseAsync(messages, options, cancellationToken))
+ {
+ if (update is null)
+ {
+ Throw.InvalidOperationException($"The inner {nameof(IChatClient)} streamed a null {nameof(ChatResponseUpdate)}.");
+ }
+
+ updates.Add(update);
+
+ _ = CopyFunctionCalls(update.Contents, ref functionCallContents);
+
+ yield return update;
+ Activity.Current = activity; // workaround for https://github.com/dotnet/runtime/issues/47802
+ }
+
+ // If there are no tools to call, or for any other reason we should stop, return the response.
+ if (functionCallContents is not { Count: > 0 } ||
+ options?.Tools is not { Count: > 0 } ||
+ iteration >= _maximumIterationsPerRequest)
+ {
+ break;
+ }
+
+ // Reconsistitue a response from the response updates.
+ var response = updates.ToChatResponse();
+ (responseMessages ??= []).AddRange(response.Messages);
+
+ // Prepare the history for the next iteration.
+ FixupHistories(originalMessages, ref messages, ref augmentedHistory, response, responseMessages, ref lastIterationHadThreadId);
+
+ // Process all of the functions, adding their results into the history.
+ var modeAndMessages = await ProcessFunctionCallsAsync(response, augmentedHistory, options, functionCallContents, iteration, consecutiveErrorCount, isStreaming: true, _functionInvocationServices, cancellationToken);
+ responseMessages.AddRange(modeAndMessages.MessagesAdded);
+ consecutiveErrorCount = modeAndMessages.NewConsecutiveErrorCount;
+
+ // This is a synthetic ID since we're generating the tool messages instead of getting them from
+ // the underlying provider. When emitting the streamed chunks, it's perfectly valid for us to
+ // use the same message ID for all of them within a given iteration, as this is a single logical
+ // message with multiple content items. We could also use different message IDs per tool content,
+ // but there's no benefit to doing so.
+ string toolResponseId = Guid.NewGuid().ToString("N");
+
+ // Stream any generated function results. This mirrors what's done for GetResponseAsync, where the returned messages
+ // includes all activitys, including generated function results.
+ foreach (var message in modeAndMessages.MessagesAdded)
+ {
+ var toolResultUpdate = new ChatResponseUpdate
+ {
+ AdditionalProperties = message.AdditionalProperties,
+ AuthorName = message.AuthorName,
+ ChatThreadId = response.ChatThreadId,
+ CreatedAt = DateTimeOffset.UtcNow,
+ Contents = message.Contents,
+ RawRepresentation = message.RawRepresentation,
+ ResponseId = toolResponseId,
+ MessageId = toolResponseId, // See above for why this can be the same as ResponseId
+ Role = message.Role,
+ };
+
+ yield return toolResultUpdate;
+ Activity.Current = activity; // workaround for https://github.com/dotnet/runtime/issues/47802
+ }
+
+ if (modeAndMessages.ShouldTerminate)
+ {
+ yield break;
+ }
+
+ UpdateOptionsForNextIteration(ref options, response.ChatThreadId);
+ }
+ }
+
+ /// Prepares the various chat message lists after a response from the inner client and before invoking functions.
+ /// The original messages provided by the caller.
+ /// The messages reference passed to the inner client.
+ /// The augmented history containing all the messages to be sent.
+ /// The most recent response being handled.
+ /// A list of all response messages received up until this point.
+ /// Whether the previous iteration's response had a thread id.
+ private static void FixupHistories(
+ IEnumerable originalMessages,
+ ref IEnumerable messages,
+ [NotNull] ref List? augmentedHistory,
+ ChatResponse response,
+ List allTurnsResponseMessages,
+ ref bool lastIterationHadThreadId)
+ {
+ // We're now going to need to augment the history with function result contents.
+ // That means we need a separate list to store the augmented history.
+ if (response.ChatThreadId is not null)
+ {
+ // The response indicates the inner client is tracking the history, so we don't want to send
+ // anything we've already sent or received.
+ if (augmentedHistory is not null)
+ {
+ augmentedHistory.Clear();
+ }
+ else
+ {
+ augmentedHistory = [];
+ }
+
+ lastIterationHadThreadId = true;
+ }
+ else if (lastIterationHadThreadId)
+ {
+ // In the very rare case where the inner client returned a response with a thread ID but then
+ // returned a subsequent response without one, we want to reconstitue the full history. To do that,
+ // we can populate the history with the original chat messages and then all of the response
+ // messages up until this point, which includes the most recent ones.
+ augmentedHistory ??= [];
+ augmentedHistory.Clear();
+ augmentedHistory.AddRange(originalMessages);
+ augmentedHistory.AddRange(allTurnsResponseMessages);
+
+ lastIterationHadThreadId = false;
+ }
+ else
+ {
+ // If augmentedHistory is already non-null, then we've already populated it with everything up
+ // until this point (except for the most recent response). If it's null, we need to seed it with
+ // the chat history provided by the caller.
+ augmentedHistory ??= originalMessages.ToList();
+
+ // Now add the most recent response messages.
+ augmentedHistory.AddMessages(response);
+
+ lastIterationHadThreadId = false;
+ }
+
+ // Use the augmented history as the new set of messages to send.
+ messages = augmentedHistory;
+ }
+
+ /// Copies any from to .
+ private static bool CopyFunctionCalls(
+ IList messages, [NotNullWhen(true)] ref List? functionCalls)
+ {
+ bool any = false;
+ int count = messages.Count;
+ for (int i = 0; i < count; i++)
+ {
+ any |= CopyFunctionCalls(messages[i].Contents, ref functionCalls);
+ }
+
+ return any;
+ }
+
+ /// Copies any from to .
+ private static bool CopyFunctionCalls(
+ IList content, [NotNullWhen(true)] ref List? functionCalls)
+ {
+ bool any = false;
+ int count = content.Count;
+ for (int i = 0; i < count; i++)
+ {
+ if (content[i] is FunctionCallContent functionCall)
+ {
+ (functionCalls ??= []).Add(functionCall);
+ any = true;
+ }
+ }
+
+ return any;
+ }
+
+ private static void UpdateOptionsForNextIteration(ref ChatOptions options, string? chatThreadId)
+ {
+ if (options.ToolMode is RequiredChatToolMode)
+ {
+ // We have to reset the tool mode to be non-required after the first iteration,
+ // as otherwise we'll be in an infinite loop.
+ options = options.Clone();
+ options.ToolMode = null;
+ options.ChatThreadId = chatThreadId;
+ }
+ else if (options.ChatThreadId != chatThreadId)
+ {
+ // As with the other modes, ensure we've propagated the chat thread ID to the options.
+ // We only need to clone the options if we're actually mutating it.
+ options = options.Clone();
+ options.ChatThreadId = chatThreadId;
+ }
+ }
+
+ ///
+ /// Processes the function calls in the list.
+ ///
+ /// The response being processed.
+ /// The current chat contents, inclusive of the function call contents being processed.
+ /// The options used for the response being processed.
+ /// The function call contents representing the functions to be invoked.
+ /// The iteration number of how many roundtrips have been made to the inner client.
+ /// The number of consecutive iterations, prior to this one, that were recorded as having function invocation errors.
+ /// Whether the function calls are being processed in a streaming context.
+ /// The services to use for function invocation.
+ /// The to monitor for cancellation requests.
+ /// A value indicating how the caller should proceed.
+ protected virtual async Task<(bool ShouldTerminate, int NewConsecutiveErrorCount, IList MessagesAdded)> ProcessFunctionCallsAsync(
+ ChatResponse response, List messages, ChatOptions options, List functionCallContents, int iteration, int consecutiveErrorCount, bool isStreaming, IServiceProvider functionInvocationServices, CancellationToken cancellationToken)
+ {
+ // We must add a response for every tool call, regardless of whether we successfully executed it or not.
+ // If we successfully execute it, we'll add the result. If we don't, we'll add an error.
+
+ Debug.Assert(functionCallContents.Count > 0, "Expected at least one function call.");
+
+ var captureCurrentIterationExceptions = consecutiveErrorCount < _maximumConsecutiveErrorsPerRequest;
+
+ // Process all functions. If there's more than one and concurrent invocation is enabled, do so in parallel.
+ if (functionCallContents.Count == 1)
+ {
+ FunctionInvocationResult result = await ProcessFunctionCallAsync(
+ messages, options, functionCallContents, iteration, 0, captureCurrentIterationExceptions, isStreaming, functionInvocationServices, cancellationToken);
+
+ IList added = CreateResponseMessages([result]);
+ ThrowIfNoFunctionResultsAdded(added);
+ UpdateConsecutiveErrorCountOrThrow(added, ref consecutiveErrorCount);
+
+ messages.AddRange(added);
+ return (result.ShouldTerminate, consecutiveErrorCount, added);
+ }
+ else
+ {
+ FunctionInvocationResult[] results;
+
+ if (AllowConcurrentInvocation)
+ {
+ // Rather than await'ing each function before invoking the next, invoke all of them
+ // and then await all of them. We avoid forcibly introducing parallelism via Task.Run,
+ // but if a function invocation completes asynchronously, its processing can overlap
+ // with the processing of other the other invocation invocations.
+ results = await Task.WhenAll(
+ from i in Enumerable.Range(0, functionCallContents.Count)
+ select ProcessFunctionCallAsync(
+ messages, options, functionCallContents,
+ iteration, i, captureExceptions: true, isStreaming, functionInvocationServices, cancellationToken));
+ }
+ else
+ {
+ // Invoke each function serially.
+ results = new FunctionInvocationResult[functionCallContents.Count];
+ for (int i = 0; i < results.Length; i++)
+ {
+ results[i] = await ProcessFunctionCallAsync(
+ messages, options, functionCallContents,
+ iteration, i, captureCurrentIterationExceptions, isStreaming, functionInvocationServices, cancellationToken);
+ }
+ }
+
+ var shouldTerminate = false;
+
+ IList added = CreateResponseMessages(results);
+ ThrowIfNoFunctionResultsAdded(added);
+ UpdateConsecutiveErrorCountOrThrow(added, ref consecutiveErrorCount);
+
+ messages.AddRange(added);
+ foreach (FunctionInvocationResult fir in results)
+ {
+ shouldTerminate = shouldTerminate || fir.ShouldTerminate;
+ }
+
+ return (shouldTerminate, consecutiveErrorCount, added);
+ }
+ }
+
+ protected void UpdateConsecutiveErrorCountOrThrow(IList added, ref int consecutiveErrorCount)
+ {
+ var allExceptions = added.SelectMany(m => m.Contents.OfType())
+ .Select(frc => frc.Exception!)
+ .Where(e => e is not null);
+
+ if (allExceptions.Any())
+ {
+ consecutiveErrorCount++;
+ if (consecutiveErrorCount > _maximumConsecutiveErrorsPerRequest)
+ {
+ var allExceptionsArray = allExceptions.ToArray();
+ if (allExceptionsArray.Length == 1)
+ {
+ ExceptionDispatchInfo.Capture(allExceptionsArray[0]).Throw();
+ }
+ else
+ {
+ throw new AggregateException(allExceptionsArray);
+ }
+ }
+ }
+ else
+ {
+ consecutiveErrorCount = 0;
+ }
+ }
+
+ ///
+ /// Throws an exception if doesn't create any messages.
+ ///
+ protected void ThrowIfNoFunctionResultsAdded(IList? messages)
+ {
+ if (messages is null || messages.Count == 0)
+ {
+ Throw.InvalidOperationException($"{GetType().Name}.{nameof(CreateResponseMessages)} returned null or an empty collection of messages.");
+ }
+ }
+
+ /// Processes the function call described in [].
+ /// The current chat contents, inclusive of the function call contents being processed.
+ /// The options used for the response being processed.
+ /// The function call contents representing all the functions being invoked.
+ /// The iteration number of how many roundtrips have been made to the inner client.
+ /// The 0-based index of the function being called out of .
+ /// If true, handles function-invocation exceptions by returning a value with . Otherwise, rethrows.
+ /// Whether the function calls are being processed in a streaming context.
+ /// The services to use for function invocation.
+ /// The to monitor for cancellation requests.
+ /// A value indicating how the caller should proceed.
+ protected virtual async Task ProcessFunctionCallAsync(
+ List messages, ChatOptions options, List callContents,
+ int iteration, int functionCallIndex, bool captureExceptions, bool isStreaming, IServiceProvider functionInvocationServices, CancellationToken cancellationToken)
+ {
+ var callContent = callContents[functionCallIndex];
+
+ // Look up the AIFunction for the function call. If the requested function isn't available, send back an error.
+ AIFunction? function = options.Tools!.OfType().FirstOrDefault(t => t.Name == callContent.Name);
+ if (function is null)
+ {
+ return new(shouldTerminate: false, FunctionInvocationStatus.NotFound, callContent, result: null, exception: null);
+ }
+
+ FunctionInvocationContext context = new()
+ {
+ Function = function,
+ Arguments = new(callContent.Arguments) { Services = functionInvocationServices },
+
+ Messages = messages,
+ Options = options,
+
+ CallContent = callContent,
+ Iteration = iteration,
+ FunctionCallIndex = functionCallIndex,
+ FunctionCount = callContents.Count,
+ };
+
+ object? result;
+ try
+ {
+ result = await InvokeFunctionAsync(context, cancellationToken);
+ }
+ catch (Exception e) when (!cancellationToken.IsCancellationRequested)
+ {
+ if (!captureExceptions)
+ {
+ throw;
+ }
+
+ return new(
+ shouldTerminate: false,
+ FunctionInvocationStatus.Exception,
+ callContent,
+ result: null,
+ exception: e);
+ }
+
+ return new(
+ shouldTerminate: context.Terminate,
+ FunctionInvocationStatus.RanToCompletion,
+ callContent,
+ result,
+ exception: null);
+ }
+
+ /// Creates one or more response messages for function invocation results.
+ /// Information about the function call invocations and results.
+ /// A list of all chat messages created from .
+ protected virtual IList CreateResponseMessages(
+ ReadOnlySpan results)
+ {
+ var contents = new List(results.Length);
+ for (int i = 0; i < results.Length; i++)
+ {
+ contents.Add(CreateFunctionResultContent(results[i]));
+ }
+
+ return [new(ChatRole.Tool, contents)];
+
+ FunctionResultContent CreateFunctionResultContent(FunctionInvocationResult result)
+ {
+ _ = Throw.IfNull(result);
+
+ object? functionResult;
+ if (result.Status == FunctionInvocationStatus.RanToCompletion)
+ {
+ functionResult = result.Result ?? "Success: Function completed.";
+ }
+ else
+ {
+ string message = result.Status switch
+ {
+ FunctionInvocationStatus.NotFound => $"Error: Requested function \"{result.CallContent.Name}\" not found.",
+ FunctionInvocationStatus.Exception => "Error: Function failed.",
+ _ => "Error: Unknown error.",
+ };
+
+ if (IncludeDetailedErrors && result.Exception is not null)
+ {
+ message = $"{message} Exception: {result.Exception.Message}";
+ }
+
+ functionResult = message;
+ }
+
+ return new FunctionResultContent(result.CallContent.CallId, functionResult) { Exception = result.Exception };
+ }
+ }
+
+ /// Invokes the function asynchronously.
+ ///
+ /// The function invocation context detailing the function to be invoked and its arguments along with additional request information.
+ ///
+ /// The to monitor for cancellation requests. The default is .
+ /// The result of the function invocation, or if the function invocation returned .
+ /// is .
+ protected virtual async Task InvokeFunctionAsync(FunctionInvocationContext context, CancellationToken cancellationToken)
+ {
+ _ = Throw.IfNull(context);
+
+ using Activity? activity = _activitySource?.StartActivity(context.Function.Name);
+
+ long startingTimestamp = 0;
+ if (_logger.IsEnabled(LogLevel.Debug))
+ {
+ startingTimestamp = Stopwatch.GetTimestamp();
+ if (_logger.IsEnabled(LogLevel.Trace))
+ {
+ LogInvokingSensitive(context.Function.Name, LoggingHelpers.AsJson(context.Arguments, context.Function.JsonSerializerOptions));
+ }
+ else
+ {
+ LogInvoking(context.Function.Name);
+ }
+ }
+
+ object? result = null;
+ try
+ {
+ CurrentContext = context; // doesn't need to be explicitly reset after, as that's handled automatically at async method exit
+ result = await TryInvokeFunctionAsync(context, cancellationToken);
+ }
+ catch (Exception e)
+ {
+ if (activity is not null)
+ {
+ _ = activity.SetTag("error.type", e.GetType().FullName)
+ .SetStatus(ActivityStatusCode.Error, e.Message);
+ }
+
+ if (e is OperationCanceledException)
+ {
+ LogInvocationCanceled(context.Function.Name);
+ }
+ else
+ {
+ LogInvocationFailed(context.Function.Name, e);
+ }
+
+ throw;
+ }
+ finally
+ {
+ if (_logger.IsEnabled(LogLevel.Debug))
+ {
+ TimeSpan elapsed = GetElapsedTime(startingTimestamp);
+
+ if (result is not null && _logger.IsEnabled(LogLevel.Trace))
+ {
+ LogInvocationCompletedSensitive(context.Function.Name, elapsed, LoggingHelpers.AsJson(result, context.Function.JsonSerializerOptions));
+ }
+ else
+ {
+ LogInvocationCompleted(context.Function.Name, elapsed);
+ }
+ }
+ }
+
+ return result;
+ }
+
+ protected virtual async Task<(FunctionInvocationContext Context, object? Result)> TryInvokeFunctionAsync(FunctionInvocationContext context, CancellationToken cancellationToken)
+ {
+ var result = await context.Function.InvokeAsync(context.Arguments, cancellationToken);
+
+ return (context, result);
+ }
+
+ private static TimeSpan GetElapsedTime(long startingTimestamp) =>
+#if NET
+ Stopwatch.GetElapsedTime(startingTimestamp);
+#else
+ new((long)((Stopwatch.GetTimestamp() - startingTimestamp) * ((double)TimeSpan.TicksPerSecond / Stopwatch.Frequency)));
+#endif
+
+ [LoggerMessage(LogLevel.Debug, "Invoking {MethodName}.", SkipEnabledCheck = true)]
+ private partial void LogInvoking(string methodName);
+
+ [LoggerMessage(LogLevel.Trace, "Invoking {MethodName}({Arguments}).", SkipEnabledCheck = true)]
+ private partial void LogInvokingSensitive(string methodName, string arguments);
+
+ [LoggerMessage(LogLevel.Debug, "{MethodName} invocation completed. Duration: {Duration}", SkipEnabledCheck = true)]
+ private partial void LogInvocationCompleted(string methodName, TimeSpan duration);
+
+ [LoggerMessage(LogLevel.Trace, "{MethodName} invocation completed. Duration: {Duration}. Result: {Result}", SkipEnabledCheck = true)]
+ private partial void LogInvocationCompletedSensitive(string methodName, TimeSpan duration, string result);
+
+ [LoggerMessage(LogLevel.Debug, "{MethodName} invocation canceled.")]
+ private partial void LogInvocationCanceled(string methodName);
+
+ [LoggerMessage(LogLevel.Error, "{MethodName} invocation failed.")]
+ private partial void LogInvocationFailed(string methodName, Exception error);
+
+ /// Provides information about the invocation of a function call.
+ public sealed class FunctionInvocationResult
+ {
+ internal FunctionInvocationResult(bool shouldTerminate, FunctionInvocationStatus status, FunctionCallContent callContent, object? result, Exception? exception)
+ {
+ ShouldTerminate = shouldTerminate;
+ Status = status;
+ CallContent = callContent;
+ Result = result;
+ Exception = exception;
+ }
+
+ /// Gets status about how the function invocation completed.
+ public FunctionInvocationStatus Status { get; }
+
+ /// Gets the function call content information associated with this invocation.
+ public FunctionCallContent CallContent { get; }
+
+ /// Gets the result of the function call.
+ public object? Result { get; }
+
+ /// Gets any exception the function call threw.
+ public Exception? Exception { get; }
+
+ /// Gets a value indicating whether the caller should terminate the processing loop.
+ internal bool ShouldTerminate { get; }
+ }
+
+ /// Provides error codes for when errors occur as part of the function calling loop.
+ public enum FunctionInvocationStatus
+ {
+ /// The operation completed successfully.
+ RanToCompletion,
+
+ /// The requested function could not be found.
+ NotFound,
+
+ /// The function call failed with an exception.
+ Exception,
+ }
+}
diff --git a/dotnet/src/SemanticKernel.Abstractions/AI/ChatClient/KernelFunctionInvokingChatClient.cs b/dotnet/src/SemanticKernel.Abstractions/AI/ChatClient/KernelFunctionInvokingChatClient.cs
index ea2dce48fc62..fc4968a390ed 100644
--- a/dotnet/src/SemanticKernel.Abstractions/AI/ChatClient/KernelFunctionInvokingChatClient.cs
+++ b/dotnet/src/SemanticKernel.Abstractions/AI/ChatClient/KernelFunctionInvokingChatClient.cs
@@ -4,15 +4,10 @@
#pragma warning restore IDE0073 // The file header does not match the required text
using System.Collections.Generic;
using System.Diagnostics;
-using System.Diagnostics.CodeAnalysis;
using System.Linq;
-using System.Runtime.CompilerServices;
-using System.Runtime.ExceptionServices;
-using System.Text.Json;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Extensions.Logging;
-using Microsoft.Extensions.Logging.Abstractions;
using Microsoft.SemanticKernel;
using Microsoft.SemanticKernel.ChatCompletion;
@@ -51,469 +46,12 @@ namespace Microsoft.Extensions.AI;
/// invocation requests to that same function.
///
///
-public partial class KernelFunctionInvokingChatClient : DelegatingChatClient
+public partial class KernelFunctionInvokingChatClient : FunctionInvokingChatClientV2
{
- /// The for the current function invocation.
- private static readonly AsyncLocal _currentContext = new();
-
- /// Optional services used for function invocation.
- private readonly IServiceProvider? _functionInvocationServices;
-
- /// The logger to use for logging information about function invocation.
- private readonly ILogger _logger;
-
- /// The to use for telemetry.
- /// This component does not own the instance and should not dispose it.
- private readonly ActivitySource? _activitySource;
-
- /// Maximum number of roundtrips allowed to the inner client.
- private int _maximumIterationsPerRequest = 10;
-
- /// Maximum number of consecutive iterations that are allowed contain at least one exception result. If the limit is exceeded, we rethrow the exception instead of continuing.
- private int _maximumConsecutiveErrorsPerRequest = 3;
-
- ///
- /// Initializes a new instance of the class.
- ///
- /// The underlying , or the next instance in a chain of clients.
- /// An to use for logging information about function invocation.
- /// An optional to use for resolving services required by the instances being invoked.
- public KernelFunctionInvokingChatClient(IChatClient innerClient, ILoggerFactory? loggerFactory = null, IServiceProvider? functionInvocationServices = null)
- : base(innerClient)
- {
- _logger = (ILogger?)loggerFactory?.CreateLogger() ?? NullLogger.Instance;
- _activitySource = innerClient.GetService();
- _functionInvocationServices = functionInvocationServices;
- }
-
- ///
- /// Gets or sets the for the current function invocation.
- ///
- ///
- /// This value flows across async calls.
- ///
- public static AutoFunctionInvocationContext? CurrentContext
- {
- get => _currentContext.Value;
- protected set => _currentContext.Value = value;
- }
-
- ///
- /// Gets or sets a value indicating whether detailed exception information should be included
- /// in the chat history when calling the underlying .
- ///
- ///
- /// if the full exception message is added to the chat history
- /// when calling the underlying .
- /// if a generic error message is included in the chat history.
- /// The default value is .
- ///
- ///
- ///
- /// Setting the value to prevents the underlying language model from disclosing
- /// raw exception details to the end user, since it doesn't receive that information. Even in this
- /// case, the raw object is available to application code by inspecting
- /// the property.
- ///
- ///
- /// Setting the value to can help the underlying bypass problems on
- /// its own, for example by retrying the function call with different arguments. However, it might
- /// result in disclosing the raw exception information to external users, which can be a security
- /// concern depending on the application scenario.
- ///
- ///
- /// Changing the value of this property while the client is in use might result in inconsistencies
- /// whether detailed errors are provided during an in-flight request.
- ///
- ///
- public bool IncludeDetailedErrors { get; set; }
-
- ///
- /// Gets or sets a value indicating whether to allow concurrent invocation of functions.
- ///
- ///
- /// if multiple function calls can execute in parallel.
- /// if function calls are processed serially.
- /// The default value is .
- ///
- ///
- /// An individual response from the inner client might contain multiple function call requests.
- /// By default, such function calls are processed serially. Set to
- /// to enable concurrent invocation such that multiple function calls can execute in parallel.
- ///
- public bool AllowConcurrentInvocation { get; set; }
-
- ///
- /// Gets or sets the maximum number of iterations per request.
- ///
- ///
- /// The maximum number of iterations per request.
- /// The default value is 10.
- ///
- ///
- ///
- /// Each request to this might end up making
- /// multiple requests to the inner client. Each time the inner client responds with
- /// a function call request, this client might perform that invocation and send the results
- /// back to the inner client in a new request. This property limits the number of times
- /// such a roundtrip is performed. The value must be at least one, as it includes the initial request.
- ///
- ///
- /// Changing the value of this property while the client is in use might result in inconsistencies
- /// as to how many iterations are allowed for an in-flight request.
- ///
- ///
- public int MaximumIterationsPerRequest
- {
- get => _maximumIterationsPerRequest;
- set
- {
- if (value < 1)
- {
- throw new ArgumentOutOfRangeException(nameof(value));
- }
-
- _maximumIterationsPerRequest = value;
- }
- }
-
- ///
- /// Gets or sets the maximum number of consecutive iterations that are allowed to fail with an error.
- ///
- ///
- /// The maximum number of consecutive iterations that are allowed to fail with an error.
- /// The default value is 3.
- ///
- ///
- ///
- /// When function invocations fail with an exception, the
- /// continues to make requests to the inner client, optionally supplying exception information (as
- /// controlled by ). This allows the to
- /// recover from errors by trying other function parameters that may succeed.
- ///
- ///
- /// However, in case function invocations continue to produce exceptions, this property can be used to
- /// limit the number of consecutive failing attempts. When the limit is reached, the exception will be
- /// rethrown to the caller.
- ///
- ///
- /// If the value is set to zero, all function calling exceptions immediately terminate the function
- /// invocation loop and the exception will be rethrown to the caller.
- ///
- ///
- /// Changing the value of this property while the client is in use might result in inconsistencies
- /// as to how many iterations are allowed for an in-flight request.
- ///
- ///
- public int MaximumConsecutiveErrorsPerRequest
- {
- get => _maximumConsecutiveErrorsPerRequest;
- set
- {
- if (value < 0)
- {
- throw new ArgumentOutOfRangeException(nameof(value), "Argument less than minimum value 0");
- }
- _maximumConsecutiveErrorsPerRequest = value;
- }
- }
-
///
- public override async Task GetResponseAsync(
- IEnumerable messages, ChatOptions? options = null, CancellationToken cancellationToken = default)
- {
- Verify.NotNull(messages);
-
- // A single request into this GetResponseAsync may result in multiple requests to the inner client.
- // Create an activity to group them together for better observability.
- using Activity? activity = _activitySource?.StartActivity(nameof(KernelFunctionInvokingChatClient));
-
- // Copy the original messages in order to avoid enumerating the original messages multiple times.
- // The IEnumerable can represent an arbitrary amount of work.
- List originalMessages = [.. messages];
- messages = originalMessages;
-
- List? augmentedHistory = null; // the actual history of messages sent on turns other than the first
- ChatResponse? response = null; // the response from the inner client, which is possibly modified and then eventually returned
- List? responseMessages = null; // tracked list of messages, across multiple turns, to be used for the final response
- UsageDetails? totalUsage = null; // tracked usage across all turns, to be used for the final response
- List? functionCallContents = null; // function call contents that need responding to in the current turn
- bool lastIterationHadThreadId = false; // whether the last iteration's response had a ChatThreadId set
- int consecutiveErrorCount = 0;
-
- for (int iteration = 0; ; iteration++)
- {
- functionCallContents?.Clear();
-
- // Make the call to the inner client.
- response = await base.GetResponseAsync(messages, options, cancellationToken).ConfigureAwait(false);
- if (response is null)
- {
- throw new InvalidOperationException($"The inner {nameof(IChatClient)} returned a null {nameof(ChatResponse)}.");
- }
-
- // Any function call work to do? If yes, ensure we're tracking that work in functionCallContents.
- bool requiresFunctionInvocation =
- options?.Tools is { Count: > 0 } &&
- iteration < MaximumIterationsPerRequest &&
- CopyFunctionCalls(response.Messages, ref functionCallContents);
-
- // In a common case where we make a request and there's no function calling work required,
- // fast path out by just returning the original response.
- if (iteration == 0 && !requiresFunctionInvocation)
- {
- return response;
- }
-
- // Track aggregate details from the response, including all the response messages and usage details.
- (responseMessages ??= []).AddRange(response.Messages);
- if (response.Usage is not null)
- {
- if (totalUsage is not null)
- {
- totalUsage.Add(response.Usage);
- }
- else
- {
- totalUsage = response.Usage;
- }
- }
-
- // If there are no tools to call, or for any other reason we should stop, we're done.
- // Break out of the loop and allow the handling at the end to configure the response
- // with aggregated data from previous requests.
- if (!requiresFunctionInvocation)
- {
- break;
- }
-
- // Prepare the history for the next iteration.
- FixupHistories(originalMessages, ref messages, ref augmentedHistory, response, responseMessages, ref lastIterationHadThreadId);
-
- // Prepare the options for the next auto function invocation iteration.
- UpdateOptionsForAutoFunctionInvocation(ref options!, response.Messages.Last().ToChatMessageContent(), isStreaming: false);
-
- // Add the responses from the function calls into the augmented history and also into the tracked
- // list of response messages.
- var modeAndMessages = await ProcessFunctionCallsAsync(augmentedHistory, options!, functionCallContents!, iteration, consecutiveErrorCount, isStreaming: false, cancellationToken).ConfigureAwait(false);
- responseMessages.AddRange(modeAndMessages.MessagesAdded);
- consecutiveErrorCount = modeAndMessages.NewConsecutiveErrorCount;
-
- if (modeAndMessages.ShouldTerminate)
- {
- break;
- }
-
- // Clear the auto function invocation options.
- ClearOptionsForAutoFunctionInvocation(ref options);
-
- UpdateOptionsForNextIteration(ref options!, response.ChatThreadId);
- }
-
- Debug.Assert(responseMessages is not null, "Expected to only be here if we have response messages.");
- response.Messages = responseMessages!;
- response.Usage = totalUsage;
-
- return response;
- }
-
- ///
- public override async IAsyncEnumerable GetStreamingResponseAsync(
- IEnumerable messages, ChatOptions? options = null, [EnumeratorCancellation] CancellationToken cancellationToken = default)
- {
- Verify.NotNull(messages);
-
- // A single request into this GetStreamingResponseAsync may result in multiple requests to the inner client.
- // Create an activity to group them together for better observability.
- using Activity? activity = _activitySource?.StartActivity(nameof(FunctionInvokingChatClient));
-
- // Copy the original messages in order to avoid enumerating the original messages multiple times.
- // The IEnumerable can represent an arbitrary amount of work.
- List originalMessages = [.. messages];
- messages = originalMessages;
-
- List? augmentedHistory = null; // the actual history of messages sent on turns other than the first
- List? functionCallContents = null; // function call contents that need responding to in the current turn
- List? responseMessages = null; // tracked list of messages, across multiple turns, to be used in fallback cases to reconstitute history
- bool lastIterationHadThreadId = false; // whether the last iteration's response had a ChatThreadId set
- List updates = []; // updates from the current response
- int consecutiveErrorCount = 0;
-
- for (int iteration = 0; ; iteration++)
- {
- updates.Clear();
- functionCallContents?.Clear();
-
- await foreach (var update in base.GetStreamingResponseAsync(messages, options, cancellationToken).ConfigureAwait(false))
- {
- if (update is null)
- {
- throw new InvalidOperationException($"The inner {nameof(IChatClient)} streamed a null {nameof(ChatResponseUpdate)}.");
- }
-
- updates.Add(update);
-
- _ = CopyFunctionCalls(update.Contents, ref functionCallContents);
-
- yield return update;
- Activity.Current = activity; // workaround for https://github.com/dotnet/runtime/issues/47802
- }
-
- // If there are no tools to call, or for any other reason we should stop, return the response.
- if (functionCallContents is not { Count: > 0 } ||
- options?.Tools is not { Count: > 0 } ||
- iteration >= _maximumIterationsPerRequest)
- {
- break;
- }
-
- // Reconstitute a response from the response updates.
- var response = updates.ToChatResponse();
- (responseMessages ??= []).AddRange(response.Messages);
-
- // Prepare the history for the next iteration.
- FixupHistories(originalMessages, ref messages, ref augmentedHistory, response, responseMessages, ref lastIterationHadThreadId);
-
- // Prepare the options for the next auto function invocation iteration.
- UpdateOptionsForAutoFunctionInvocation(ref options, response.Messages.Last().ToChatMessageContent(), isStreaming: true);
-
- // Process all the functions, adding their results into the history.
- var modeAndMessages = await ProcessFunctionCallsAsync(augmentedHistory, options, functionCallContents, iteration, consecutiveErrorCount, isStreaming: true, cancellationToken).ConfigureAwait(false);
- responseMessages.AddRange(modeAndMessages.MessagesAdded);
- consecutiveErrorCount = modeAndMessages.NewConsecutiveErrorCount;
-
- // Clear the auto function invocation options.
- ClearOptionsForAutoFunctionInvocation(ref options);
-
- // This is a synthetic ID since we're generating the tool messages instead of getting them from
- // the underlying provider. When emitting the streamed chunks, it's perfectly valid for us to
- // use the same message ID for all of them within a given iteration, as this is a single logical
- // message with multiple content items. We could also use different message IDs per tool content,
- // but there's no benefit to doing so.
- string toolResponseId = Guid.NewGuid().ToString("N");
-
- // Stream any generated function results. This mirrors what's done for GetResponseAsync, where the returned messages
- // include all activity, including generated function results.
- foreach (var message in modeAndMessages.MessagesAdded)
- {
- var toolResultUpdate = new ChatResponseUpdate
- {
- AdditionalProperties = message.AdditionalProperties,
- AuthorName = message.AuthorName,
- ChatThreadId = response.ChatThreadId,
- CreatedAt = DateTimeOffset.UtcNow,
- Contents = message.Contents,
- RawRepresentation = message.RawRepresentation,
- ResponseId = toolResponseId,
- MessageId = toolResponseId, // See above for why this can be the same as ResponseId
- Role = message.Role,
- };
-
- yield return toolResultUpdate;
- Activity.Current = activity; // workaround for https://github.com/dotnet/runtime/issues/47802
- }
-
- if (modeAndMessages.ShouldTerminate)
- {
- yield break;
- }
-
- UpdateOptionsForNextIteration(ref options, response.ChatThreadId);
- }
- }
-
- /// Prepares the various chat message lists after a response from the inner client and before invoking functions.
- /// The original messages provided by the caller.
- /// The messages reference passed to the inner client.
- /// The augmented history containing all the messages to be sent.
- /// The most recent response being handled.
- /// A list of all response messages received up until this point.
- /// Whether the previous iteration's response had a thread id.
- private static void FixupHistories(
- IEnumerable originalMessages,
- ref IEnumerable messages,
- [NotNull] ref List? augmentedHistory,
- ChatResponse response,
- List allTurnsResponseMessages,
- ref bool lastIterationHadThreadId)
- {
- // We're now going to need to augment the history with function result contents.
- // That means we need a separate list to store the augmented history.
- if (response.ChatThreadId is not null)
- {
- // The response indicates the inner client is tracking the history, so we don't want to send
- // anything we've already sent or received.
- if (augmentedHistory is not null)
- {
- augmentedHistory.Clear();
- }
- else
- {
- augmentedHistory = [];
- }
-
- lastIterationHadThreadId = true;
- }
- else if (lastIterationHadThreadId)
- {
- // In the very rare case where the inner client returned a response with a thread ID but then
- // returned a subsequent response without one, we want to reconstitute the full history. To do that,
- // we can populate the history with the original chat messages and then all the response
- // messages up until this point, which includes the most recent ones.
- augmentedHistory ??= [];
- augmentedHistory.Clear();
- augmentedHistory.AddRange(originalMessages);
- augmentedHistory.AddRange(allTurnsResponseMessages);
-
- lastIterationHadThreadId = false;
- }
- else
- {
- // If augmentedHistory is already non-null, then we've already populated it with everything up
- // until this point (except for the most recent response). If it's null, we need to seed it with
- // the chat history provided by the caller.
- augmentedHistory ??= originalMessages.ToList();
-
- // Now add the most recent response messages.
- augmentedHistory.AddMessages(response);
-
- lastIterationHadThreadId = false;
- }
-
- // Use the augmented history as the new set of messages to send.
- messages = augmentedHistory;
- }
-
- /// Copies any from to .
- private static bool CopyFunctionCalls(
- IList messages, [NotNullWhen(true)] ref List? functionCalls)
- {
- bool any = false;
- int count = messages.Count;
- for (int i = 0; i < count; i++)
- {
- any |= CopyFunctionCalls(messages[i].Contents, ref functionCalls);
- }
-
- return any;
- }
-
- /// Copies any from to .
- private static bool CopyFunctionCalls(
- IList content, [NotNullWhen(true)] ref List? functionCalls)
+ public KernelFunctionInvokingChatClient(IChatClient innerClient, ILoggerFactory? loggerFactory = null, IServiceProvider? functionInvocationServices = null)
+ : base(innerClient, loggerFactory, functionInvocationServices)
{
- bool any = false;
- int count = content.Count;
- for (int i = 0; i < count; i++)
- {
- if (content[i] is FunctionCallContent functionCall)
- {
- (functionCalls ??= []).Add(functionCall);
- any = true;
- }
- }
-
- return any;
}
private static void UpdateOptionsForAutoFunctionInvocation(ref ChatOptions options, ChatMessageContent content, bool isStreaming)
@@ -547,67 +85,40 @@ private static void ClearOptionsForAutoFunctionInvocation(ref ChatOptions option
}
}
- private static void UpdateOptionsForNextIteration(ref ChatOptions options, string? chatThreadId)
+ ///
+ protected override async Task<(bool ShouldTerminate, int NewConsecutiveErrorCount, IList MessagesAdded)> ProcessFunctionCallsAsync(
+ ChatResponse response, List messages, ChatOptions options, List functionCallContents, int iteration, int consecutiveErrorCount, bool isStreaming, IServiceProvider functionInvocationServices, CancellationToken cancellationToken)
{
- if (options.ToolMode is RequiredChatToolMode)
- {
- // We have to reset the tool mode to be non-required after the first iteration,
- // as otherwise we'll be in an infinite loop.
- options = options.Clone();
- options.ToolMode = null;
- options.ChatThreadId = chatThreadId;
- }
- else if (options.ChatThreadId != chatThreadId)
- {
- // As with the other modes, ensure we've propagated the chat thread ID to the options.
- // We only need to clone the options if we're actually mutating it.
- options = options.Clone();
- options.ChatThreadId = chatThreadId;
- }
- }
+ // Prepare the options for the next auto function invocation iteration.
+ UpdateOptionsForAutoFunctionInvocation(ref options!, response.Messages.Last().ToChatMessageContent(), isStreaming: false);
- ///
- /// Processes the function calls in the list.
- ///
- /// The current chat contents, inclusive of the function call contents being processed.
- /// The options used for the response being processed.
- /// The function call contents representing the functions to be invoked.
- /// The iteration number of how many roundtrips have been made to the inner client.
- /// The number of consecutive iterations, prior to this one, that were recorded as having function invocation errors.
- /// Whether the function calls are being processed in a streaming context.
- /// The to monitor for cancellation requests.
- /// A value indicating how the caller should proceed.
- private async Task<(bool ShouldTerminate, int NewConsecutiveErrorCount, IList MessagesAdded)> ProcessFunctionCallsAsync(
- List messages, ChatOptions options, List functionCallContents,
- int iteration, int consecutiveErrorCount, bool isStreaming, CancellationToken cancellationToken)
- {
// We must add a response for every tool call, regardless of whether we successfully executed it or not.
// If we successfully execute it, we'll add the result. If we don't, we'll add an error.
-
+ (bool ShouldTerminate, int NewConsecutiveErrorCount, IList MessagesAdded) result;
Debug.Assert(functionCallContents.Count > 0, "Expected at least one function call.");
var shouldTerminate = false;
- var captureCurrentIterationExceptions = consecutiveErrorCount < _maximumConsecutiveErrorsPerRequest;
+ var captureCurrentIterationExceptions = consecutiveErrorCount < this.MaximumConsecutiveErrorsPerRequest;
// Process all functions. If there's more than one and concurrent invocation is enabled, do so in parallel.
if (functionCallContents.Count == 1)
{
- FunctionInvocationResult result = await ProcessFunctionCallAsync(
- messages, options, functionCallContents, iteration, 0, captureCurrentIterationExceptions, isStreaming, cancellationToken).ConfigureAwait(false);
+ FunctionInvocationResult functionResult = await this.ProcessFunctionCallAsync(
+ messages, options, functionCallContents, iteration, 0, captureCurrentIterationExceptions, isStreaming, functionInvocationServices, cancellationToken).ConfigureAwait(false);
- IList added = CreateResponseMessages([result]);
- ThrowIfNoFunctionResultsAdded(added);
- UpdateConsecutiveErrorCountOrThrow(added, ref consecutiveErrorCount);
+ IList added = this.CreateResponseMessages([functionResult]);
+ this.ThrowIfNoFunctionResultsAdded(added);
+ this.UpdateConsecutiveErrorCountOrThrow(added, ref consecutiveErrorCount);
messages.AddRange(added);
- return (result.ShouldTerminate, consecutiveErrorCount, added);
+ result = (functionResult.ShouldTerminate, consecutiveErrorCount, added);
}
else
{
List results = [];
var terminationRequested = false;
- if (AllowConcurrentInvocation)
+ if (this.AllowConcurrentInvocation)
{
// Rather than awaiting each function before invoking the next, invoke all of them
// and then await all of them. We avoid forcibly introducing parallelism via Task.Run,
@@ -615,9 +126,9 @@ private static void UpdateOptionsForNextIteration(ref ChatOptions options, strin
// with the processing of other the other invocation invocations.
results.AddRange(await Task.WhenAll(
from i in Enumerable.Range(0, functionCallContents.Count)
- select ProcessFunctionCallAsync(
+ select this.ProcessFunctionCallAsync(
messages, options, functionCallContents,
- iteration, i, captureExceptions: true, isStreaming, cancellationToken)).ConfigureAwait(false));
+ iteration, i, captureExceptions: true, isStreaming, functionInvocationServices, cancellationToken)).ConfigureAwait(false));
terminationRequested = results.Any(r => r.ShouldTerminate);
}
@@ -626,13 +137,15 @@ select ProcessFunctionCallAsync(
// Invoke each function serially.
for (int i = 0; i < functionCallContents.Count; i++)
{
- var result = await ProcessFunctionCallAsync(
+ var functionResult = await this.ProcessFunctionCallAsync(
messages, options, functionCallContents,
- iteration, i, captureCurrentIterationExceptions, isStreaming, cancellationToken).ConfigureAwait(false);
+ iteration, i, captureCurrentIterationExceptions, isStreaming, functionInvocationServices, cancellationToken).ConfigureAwait(false);
- results.Add(result);
+ results.Add(functionResult);
- if (result.ShouldTerminate)
+ // Differently from the original FunctionInvokingChatClient, as soon the termination is requested,
+ // we stop and don't continue
+ if (functionResult.ShouldTerminate)
{
shouldTerminate = true;
terminationRequested = true;
@@ -641,9 +154,9 @@ select ProcessFunctionCallAsync(
}
}
- IList added = CreateResponseMessages(results);
- ThrowIfNoFunctionResultsAdded(added);
- UpdateConsecutiveErrorCountOrThrow(added, ref consecutiveErrorCount);
+ IList added = this.CreateResponseMessages(results);
+ this.ThrowIfNoFunctionResultsAdded(added);
+ this.UpdateConsecutiveErrorCountOrThrow(added, ref consecutiveErrorCount);
messages.AddRange(added);
@@ -657,64 +170,19 @@ select ProcessFunctionCallAsync(
}
}
- return (shouldTerminate, consecutiveErrorCount, added);
+ result = (shouldTerminate, consecutiveErrorCount, added);
}
- }
- private void UpdateConsecutiveErrorCountOrThrow(IList added, ref int consecutiveErrorCount)
- {
- var allExceptions = added.SelectMany(m => m.Contents.OfType())
- .Select(frc => frc.Exception!)
- .Where(e => e is not null);
+ // Clear the auto function invocation options.
+ ClearOptionsForAutoFunctionInvocation(ref options);
-#pragma warning disable CA1851 // Possible multiple enumerations of 'IEnumerable' collection
- if (allExceptions.Any())
- {
- consecutiveErrorCount++;
- if (consecutiveErrorCount > _maximumConsecutiveErrorsPerRequest)
- {
- var allExceptionsArray = allExceptions.ToArray();
- if (allExceptionsArray.Length == 1)
- {
- ExceptionDispatchInfo.Capture(allExceptionsArray[0]).Throw();
- }
- else
- {
- throw new AggregateException(allExceptionsArray);
- }
- }
- }
- else
- {
- consecutiveErrorCount = 0;
- }
-#pragma warning restore CA1851 // Possible multiple enumerations of 'IEnumerable' collection
- }
-
- ///
- /// Throws an exception if doesn't create any messages.
- ///
- private void ThrowIfNoFunctionResultsAdded(IList? messages)
- {
- if (messages is null || messages.Count == 0)
- {
- throw new InvalidOperationException($"{this.GetType().Name}.{nameof(this.CreateResponseMessages)} returned null or an empty collection of messages.");
- }
+ return result;
}
- /// Processes the function call described in [].
- /// The current chat contents, inclusive of the function call contents being processed.
- /// The options used for the response being processed.
- /// The function call contents representing all the functions being invoked.
- /// The iteration number of how many roundtrips have been made to the inner client.
- /// The 0-based index of the function being called out of .
- /// If true, handles function-invocation exceptions by returning a value with . Otherwise, rethrows.
- /// Whether the function calls are being processed in a streaming context.
- /// The to monitor for cancellation requests.
- /// A value indicating how the caller should proceed.
- private async Task ProcessFunctionCallAsync(
+ ///
+ protected override async Task ProcessFunctionCallAsync(
List messages, ChatOptions options, List callContents,
- int iteration, int functionCallIndex, bool captureExceptions, bool isStreaming, CancellationToken cancellationToken)
+ int iteration, int functionCallIndex, bool captureExceptions, bool isStreaming, IServiceProvider functionInvocationServices, CancellationToken cancellationToken)
{
var callContent = callContents[functionCallIndex];
@@ -722,7 +190,7 @@ private async Task ProcessFunctionCallAsync(
AIFunction? function = options.Tools!.OfType().FirstOrDefault(t => t.Name == callContent.Name);
if (function is null)
{
- return new(shouldTerminate: false, FunctionInvokingChatClient.FunctionInvocationStatus.NotFound, callContent, result: null, exception: null);
+ return new(shouldTerminate: false, FunctionInvocationStatus.NotFound, callContent, result: null, exception: null);
}
if (callContent.Arguments is not null)
@@ -733,7 +201,7 @@ private async Task ProcessFunctionCallAsync(
var context = new AutoFunctionInvocationContext(new()
{
Function = function,
- Arguments = new(callContent.Arguments) { Services = _functionInvocationServices },
+ Arguments = new(callContent.Arguments) { Services = functionInvocationServices },
Messages = messages,
Options = options,
@@ -858,39 +326,13 @@ await autoFunctionInvocationFilters[index].OnAutoFunctionInvocationAsync(
}
}
- /// Invokes the function asynchronously.
- ///
- /// The function invocation context detailing the function to be invoked and its arguments along with additional request information.
- ///
- /// The to monitor for cancellation requests. The default is .
- /// The result of the function invocation, or if the function invocation returned .
- /// is .
- private async Task InvokeFunctionAsync(AutoFunctionInvocationContext context, CancellationToken cancellationToken)
+ protected override async Task<(FunctionInvocationContext context, object? result)> TryInvokeFunctionAsync(FunctionInvocationContext context, CancellationToken cancellationToken)
{
- Verify.NotNull(context);
-
- using Activity? activity = _activitySource?.StartActivity(context.Function.Name);
-
- long startingTimestamp = 0;
- if (_logger.IsEnabled(LogLevel.Debug))
- {
- startingTimestamp = Stopwatch.GetTimestamp();
- if (_logger.IsEnabled(LogLevel.Trace))
- {
- LogInvokingSensitive(context.Function.Name, LoggingAsJson(context.CallContent.Arguments, context.AIFunction.JsonSerializerOptions));
- }
- else
- {
- LogInvoking(context.Function.Name);
- }
- }
-
object? result = null;
- try
+ if (context is AutoFunctionInvocationContext autoContext)
{
- CurrentContext = context; // doesn't need to be explicitly reset after, as that's handled automatically at async method exit
context = await this.OnAutoFunctionInvocationAsync(
- context,
+ autoContext,
async (ctx) =>
{
// Check if filter requested termination
@@ -902,122 +344,16 @@ await autoFunctionInvocationFilters[index].OnAutoFunctionInvocationAsync(
// Note that we explicitly do not use executionSettings here; those pertain to the all-up operation and not necessarily to any
// further calls made as part of this function invocation. In particular, we must not use function calling settings naively here,
// as the called function could in turn telling the model about itself as a possible candidate for invocation.
- result = await context.AIFunction.InvokeAsync(new(context.Arguments), cancellationToken).ConfigureAwait(false);
+ result = await autoContext.AIFunction.InvokeAsync(new(context.Arguments), cancellationToken).ConfigureAwait(false);
ctx.Result = new FunctionResult(ctx.Function, result);
}).ConfigureAwait(false);
- result = context.Result.GetValue