From b22d56b26f478096eefcd2ef8c37736492e1bfe0 Mon Sep 17 00:00:00 2001 From: Roger Barreto <19890735+RogerBarreto@users.noreply.github.com> Date: Fri, 20 Oct 2023 18:18:35 +0100 Subject: [PATCH 01/46] Streaming Initial ADR Draft --- docs/decisions/0013-kernel-streaming.md | 62 +++++++++++++++++++++++++ 1 file changed, 62 insertions(+) create mode 100644 docs/decisions/0013-kernel-streaming.md diff --git a/docs/decisions/0013-kernel-streaming.md b/docs/decisions/0013-kernel-streaming.md new file mode 100644 index 000000000000..fb36f2795e0e --- /dev/null +++ b/docs/decisions/0013-kernel-streaming.md @@ -0,0 +1,62 @@ +--- +# These are optional elements. Feel free to remove any of them. +status: proposed +date: 2023-09-26 +deciders: rbarreto,markwallace,sergey,dmytro +consulted: +informed: +--- + +# Streaming Capability for Kernel and Functions usage + +## Context and Problem Statement + +Its very common in co-pilot implementations to have a streamlined output of messages from the LLM and currently that's not possible while using ISKFunctions.InvokeAsync or Kernel.RunAsync methods, which enforces users to work around the Kernel and Functions to use `ITextCompletion` and `IChatCompletion` services directly. + +So this ADR propose a solution for the above fore mentioned problem. + +## Decision Drivers + +- Architecture changes and the associated decision making process should be transparent to the community. +- Decision records are stored in the repository and are easily discoverable for teams involved in the various language ports. +- The solution should be backward compatible with the existing implementations. +- The design should be extensible to support future requirements. +- Keep the design as simple and easy to understand. + +## Considered Options + +### Streaming Setting + New Streaming Interfaces + +This option is to add a new setting `Streaming` to the `AIRequestSettings` definition to flag the caller would like to stream the output from the kernel and functions if possible. + +Seggregate old Text/Chat Completion interfaces the into new `ITextStreamingCompletion` and `IChatStreamingCompletion` dedicated for streaming (interface segragation). **This becomes crutial to the Kernel knows if streaming is supported by the service.** + +#### Backward compatibility aspects: + +- SKContext was built on top of a non streaming functionality, so to keep the compatibility for streaming scenarios, SKContext needs to be updated into a Lazy loading strategy, where when used will enforce the buffering of the whole stream to be able to return to final result. + +- Streaming by defaullt will be `false` + 1. It wont force streaming results over legacy implementations. + 2. Providers like OpenAI/Azure OpenAI don't provide additional data like (Usage, Completion Tokens, Prompt Tokens, Total Tokens, ...) in streaming results. + +Enable streaming behavior set `Streaming = true` in a `AIRequestSettings` specialized class like `OpenAIRequestSettings`, once this is set for text/chat based models like OpenAi's GPT you can get streaming data using `GetValue>()` in the result of `Kernel.RunAsync` or `Function.InvokeAsync`. + +Example: + +``` +myFunction = kernel.CreateSemanticFunction($"your prompt", + requestSettings: new OpenAIRequestSettings + { + Streaming = true, + }); + +var result = await kernel.RunAsync(myFunction); +await foreach (string token in result.GetValue>()) +{ + Console.Write(token); +} +``` + +## Decision Outcome + +As for now, the only proposed solution is the #1. From 188bbcaa6fef9adbb5ed61d3a63e61e5c64b06a3 Mon Sep 17 00:00:00 2001 From: Roger Barreto <19890735+RogerBarreto@users.noreply.github.com> Date: Tue, 24 Oct 2023 08:40:17 +0100 Subject: [PATCH 02/46] Update ADR number --- docs/decisions/0014-kernel-streaming.md | 62 +++++++++++++++++++++++++ 1 file changed, 62 insertions(+) create mode 100644 docs/decisions/0014-kernel-streaming.md diff --git a/docs/decisions/0014-kernel-streaming.md b/docs/decisions/0014-kernel-streaming.md new file mode 100644 index 000000000000..fb36f2795e0e --- /dev/null +++ b/docs/decisions/0014-kernel-streaming.md @@ -0,0 +1,62 @@ +--- +# These are optional elements. Feel free to remove any of them. +status: proposed +date: 2023-09-26 +deciders: rbarreto,markwallace,sergey,dmytro +consulted: +informed: +--- + +# Streaming Capability for Kernel and Functions usage + +## Context and Problem Statement + +Its very common in co-pilot implementations to have a streamlined output of messages from the LLM and currently that's not possible while using ISKFunctions.InvokeAsync or Kernel.RunAsync methods, which enforces users to work around the Kernel and Functions to use `ITextCompletion` and `IChatCompletion` services directly. + +So this ADR propose a solution for the above fore mentioned problem. + +## Decision Drivers + +- Architecture changes and the associated decision making process should be transparent to the community. +- Decision records are stored in the repository and are easily discoverable for teams involved in the various language ports. +- The solution should be backward compatible with the existing implementations. +- The design should be extensible to support future requirements. +- Keep the design as simple and easy to understand. + +## Considered Options + +### Streaming Setting + New Streaming Interfaces + +This option is to add a new setting `Streaming` to the `AIRequestSettings` definition to flag the caller would like to stream the output from the kernel and functions if possible. + +Seggregate old Text/Chat Completion interfaces the into new `ITextStreamingCompletion` and `IChatStreamingCompletion` dedicated for streaming (interface segragation). **This becomes crutial to the Kernel knows if streaming is supported by the service.** + +#### Backward compatibility aspects: + +- SKContext was built on top of a non streaming functionality, so to keep the compatibility for streaming scenarios, SKContext needs to be updated into a Lazy loading strategy, where when used will enforce the buffering of the whole stream to be able to return to final result. + +- Streaming by defaullt will be `false` + 1. It wont force streaming results over legacy implementations. + 2. Providers like OpenAI/Azure OpenAI don't provide additional data like (Usage, Completion Tokens, Prompt Tokens, Total Tokens, ...) in streaming results. + +Enable streaming behavior set `Streaming = true` in a `AIRequestSettings` specialized class like `OpenAIRequestSettings`, once this is set for text/chat based models like OpenAi's GPT you can get streaming data using `GetValue>()` in the result of `Kernel.RunAsync` or `Function.InvokeAsync`. + +Example: + +``` +myFunction = kernel.CreateSemanticFunction($"your prompt", + requestSettings: new OpenAIRequestSettings + { + Streaming = true, + }); + +var result = await kernel.RunAsync(myFunction); +await foreach (string token in result.GetValue>()) +{ + Console.Write(token); +} +``` + +## Decision Outcome + +As for now, the only proposed solution is the #1. From a3df39ebeaf7fd46fe3f15d7b8050bb75af995c2 Mon Sep 17 00:00:00 2001 From: Roger Barreto <19890735+RogerBarreto@users.noreply.github.com> Date: Tue, 24 Oct 2023 16:11:06 +0100 Subject: [PATCH 03/46] Typo fix --- docs/decisions/0013-kernel-streaming.md | 62 ------------------------- docs/decisions/0014-kernel-streaming.md | 4 +- 2 files changed, 2 insertions(+), 64 deletions(-) delete mode 100644 docs/decisions/0013-kernel-streaming.md diff --git a/docs/decisions/0013-kernel-streaming.md b/docs/decisions/0013-kernel-streaming.md deleted file mode 100644 index fb36f2795e0e..000000000000 --- a/docs/decisions/0013-kernel-streaming.md +++ /dev/null @@ -1,62 +0,0 @@ ---- -# These are optional elements. Feel free to remove any of them. -status: proposed -date: 2023-09-26 -deciders: rbarreto,markwallace,sergey,dmytro -consulted: -informed: ---- - -# Streaming Capability for Kernel and Functions usage - -## Context and Problem Statement - -Its very common in co-pilot implementations to have a streamlined output of messages from the LLM and currently that's not possible while using ISKFunctions.InvokeAsync or Kernel.RunAsync methods, which enforces users to work around the Kernel and Functions to use `ITextCompletion` and `IChatCompletion` services directly. - -So this ADR propose a solution for the above fore mentioned problem. - -## Decision Drivers - -- Architecture changes and the associated decision making process should be transparent to the community. -- Decision records are stored in the repository and are easily discoverable for teams involved in the various language ports. -- The solution should be backward compatible with the existing implementations. -- The design should be extensible to support future requirements. -- Keep the design as simple and easy to understand. - -## Considered Options - -### Streaming Setting + New Streaming Interfaces - -This option is to add a new setting `Streaming` to the `AIRequestSettings` definition to flag the caller would like to stream the output from the kernel and functions if possible. - -Seggregate old Text/Chat Completion interfaces the into new `ITextStreamingCompletion` and `IChatStreamingCompletion` dedicated for streaming (interface segragation). **This becomes crutial to the Kernel knows if streaming is supported by the service.** - -#### Backward compatibility aspects: - -- SKContext was built on top of a non streaming functionality, so to keep the compatibility for streaming scenarios, SKContext needs to be updated into a Lazy loading strategy, where when used will enforce the buffering of the whole stream to be able to return to final result. - -- Streaming by defaullt will be `false` - 1. It wont force streaming results over legacy implementations. - 2. Providers like OpenAI/Azure OpenAI don't provide additional data like (Usage, Completion Tokens, Prompt Tokens, Total Tokens, ...) in streaming results. - -Enable streaming behavior set `Streaming = true` in a `AIRequestSettings` specialized class like `OpenAIRequestSettings`, once this is set for text/chat based models like OpenAi's GPT you can get streaming data using `GetValue>()` in the result of `Kernel.RunAsync` or `Function.InvokeAsync`. - -Example: - -``` -myFunction = kernel.CreateSemanticFunction($"your prompt", - requestSettings: new OpenAIRequestSettings - { - Streaming = true, - }); - -var result = await kernel.RunAsync(myFunction); -await foreach (string token in result.GetValue>()) -{ - Console.Write(token); -} -``` - -## Decision Outcome - -As for now, the only proposed solution is the #1. diff --git a/docs/decisions/0014-kernel-streaming.md b/docs/decisions/0014-kernel-streaming.md index fb36f2795e0e..16c5708bfbcf 100644 --- a/docs/decisions/0014-kernel-streaming.md +++ b/docs/decisions/0014-kernel-streaming.md @@ -29,13 +29,13 @@ So this ADR propose a solution for the above fore mentioned problem. This option is to add a new setting `Streaming` to the `AIRequestSettings` definition to flag the caller would like to stream the output from the kernel and functions if possible. -Seggregate old Text/Chat Completion interfaces the into new `ITextStreamingCompletion` and `IChatStreamingCompletion` dedicated for streaming (interface segragation). **This becomes crutial to the Kernel knows if streaming is supported by the service.** +Seggregate old Text/Chat Completion interfaces the into new `ITextStreamingCompletion` and `IChatStreamingCompletion` dedicated for streaming (interface segregation). **This becomes crucial to the Kernel knows if streaming is supported by the service.** #### Backward compatibility aspects: - SKContext was built on top of a non streaming functionality, so to keep the compatibility for streaming scenarios, SKContext needs to be updated into a Lazy loading strategy, where when used will enforce the buffering of the whole stream to be able to return to final result. -- Streaming by defaullt will be `false` +- Streaming by default will be `false` 1. It wont force streaming results over legacy implementations. 2. Providers like OpenAI/Azure OpenAI don't provide additional data like (Usage, Completion Tokens, Prompt Tokens, Total Tokens, ...) in streaming results. From ebaf812399765bfaf13b95879e9aeced9b70fac9 Mon Sep 17 00:00:00 2001 From: Roger Barreto <19890735+RogerBarreto@users.noreply.github.com> Date: Mon, 6 Nov 2023 21:53:44 +0000 Subject: [PATCH 04/46] WIP Streaming ADR --- docs/decisions/0014-kernel-streaming.md | 62 ----------- docs/decisions/0020-kernel-streaming.md | 140 ++++++++++++++++++++++++ 2 files changed, 140 insertions(+), 62 deletions(-) delete mode 100644 docs/decisions/0014-kernel-streaming.md create mode 100644 docs/decisions/0020-kernel-streaming.md diff --git a/docs/decisions/0014-kernel-streaming.md b/docs/decisions/0014-kernel-streaming.md deleted file mode 100644 index 16c5708bfbcf..000000000000 --- a/docs/decisions/0014-kernel-streaming.md +++ /dev/null @@ -1,62 +0,0 @@ ---- -# These are optional elements. Feel free to remove any of them. -status: proposed -date: 2023-09-26 -deciders: rbarreto,markwallace,sergey,dmytro -consulted: -informed: ---- - -# Streaming Capability for Kernel and Functions usage - -## Context and Problem Statement - -Its very common in co-pilot implementations to have a streamlined output of messages from the LLM and currently that's not possible while using ISKFunctions.InvokeAsync or Kernel.RunAsync methods, which enforces users to work around the Kernel and Functions to use `ITextCompletion` and `IChatCompletion` services directly. - -So this ADR propose a solution for the above fore mentioned problem. - -## Decision Drivers - -- Architecture changes and the associated decision making process should be transparent to the community. -- Decision records are stored in the repository and are easily discoverable for teams involved in the various language ports. -- The solution should be backward compatible with the existing implementations. -- The design should be extensible to support future requirements. -- Keep the design as simple and easy to understand. - -## Considered Options - -### Streaming Setting + New Streaming Interfaces - -This option is to add a new setting `Streaming` to the `AIRequestSettings` definition to flag the caller would like to stream the output from the kernel and functions if possible. - -Seggregate old Text/Chat Completion interfaces the into new `ITextStreamingCompletion` and `IChatStreamingCompletion` dedicated for streaming (interface segregation). **This becomes crucial to the Kernel knows if streaming is supported by the service.** - -#### Backward compatibility aspects: - -- SKContext was built on top of a non streaming functionality, so to keep the compatibility for streaming scenarios, SKContext needs to be updated into a Lazy loading strategy, where when used will enforce the buffering of the whole stream to be able to return to final result. - -- Streaming by default will be `false` - 1. It wont force streaming results over legacy implementations. - 2. Providers like OpenAI/Azure OpenAI don't provide additional data like (Usage, Completion Tokens, Prompt Tokens, Total Tokens, ...) in streaming results. - -Enable streaming behavior set `Streaming = true` in a `AIRequestSettings` specialized class like `OpenAIRequestSettings`, once this is set for text/chat based models like OpenAi's GPT you can get streaming data using `GetValue>()` in the result of `Kernel.RunAsync` or `Function.InvokeAsync`. - -Example: - -``` -myFunction = kernel.CreateSemanticFunction($"your prompt", - requestSettings: new OpenAIRequestSettings - { - Streaming = true, - }); - -var result = await kernel.RunAsync(myFunction); -await foreach (string token in result.GetValue>()) -{ - Console.Write(token); -} -``` - -## Decision Outcome - -As for now, the only proposed solution is the #1. diff --git a/docs/decisions/0020-kernel-streaming.md b/docs/decisions/0020-kernel-streaming.md new file mode 100644 index 000000000000..8996fd04f55f --- /dev/null +++ b/docs/decisions/0020-kernel-streaming.md @@ -0,0 +1,140 @@ +--- +# These are optional elements. Feel free to remove any of them. +status: proposed +date: 2023-09-26 +deciders: rbarreto,markwallace,sergey,dmytro +consulted: +informed: +--- + +# Streaming Capability for Kernel and Functions usage + +## Context and Problem Statement + +Its very common in co-pilot implementations to have a streamlined output of messages from the LLM and currently that's not possible while using ISKFunctions.InvokeAsync or Kernel.RunAsync methods, which enforces users to work around the Kernel and Functions to use `ITextCompletion` and `IChatCompletion` services directly which currently support this feature. + +But notably streaming in its current state is not supported by all providers and this should be added as part or our design to ensure it as a capability the service with a proper abstraction to support also streaming of other types of data like images, audio, video, etc. + +Needs to be clear for Kernel when a connector that supports a specific input/output mimetype is available to be used in the function to function pipeline. + +Needs to be clear for the Kernel when a connector returns a stream how to build the final result from the bits of data returned by the connector in streaming mode, so this information can be appended and passed down to the next function in the pipeline and the streaming can continue. + +## Decision Drivers + +- Connectors should be able to flag they have Streaming capability and the Kernel should be able to use it. +- Users should be able to use Kernel to get streaming results. +- Users should be able to use Functions to get streaming results. +- Streaming results can be represented as non streaming results, so getting streaming first approach eventually will be able to support non streaming results. +- Having streaming dedicated methods in Kernel or Functions allow the caller to know what result to expect and implement accordingly. + +## User experience goal + +abstract StreamingResultBit +string mimeType; +object Value; + +StreamingFunctionResult +StreamingResultBuilder Builder; +string mimeType; +Dictionary Metadata; +IAsyncEnumerable StreamingValue; + +// Abstractions to make clear how to build a streaming result from streaming bits enumerations +StreamingResultBuilder +string mimeType; +TComplete Build(); +void Append(TBit streamingBit); + +// Abstraction of a streaming bit +StreamingResultBit +Type streamingBuilder +string mimeType; +TBit streamingBit; + +// Abstract connector modality interface +IConnectorModalityService +string string[] inputMimeTypes; +string string outputMimeType; +AsyncEnumerable GetStreamingResult(object input); +StreamingResultBuilder GetStreamingResultBuilder(string inputMimeType, string outputMimeType); + +// Generic version of connector modality interface +IConnectorModalityService : IConnectorModalityService +string string[] inputMimeTypes; +string string outputMimeType; +AsyncEnumerable GetStreamingResult(T input); +StreamingResultBuilder GetStreamingResultBuilder(); + +// +IServiceSelector +IConnectorModalityService GetServiceByModality(config.inputMimeType ?? "text/plain", config.outputMimeType ?? "text/plain"); + +// Function + +StreamingFunctionResult StreamingInvokeAsync() +{ + + (IConnectorModalityService service, var defaultRequestSettings) = serviceSelector.GetServiceByModality(renderedPrompt, context.ServiceProvider, this._modelSettings); + + var resultBuilder = service.GetStreamingResultBuilder(defaultRequestSettings); + FunctionResult result = new StreamingFunctionResult(this.Name, this.PluginName, context, service.GetStreamingResult) { + Metadata = new() { + { "Usage", defaultRequestSettings.InputMimeType }, + { "outputMimeType", defaultRequestSettings.OutputMimeType }, + } + }; + } + +} + +``` +var kernel = new KernelBuilder() + .WithLoggerFactory(ConsoleLogger.LoggerFactory) + .WithOpenAIChatCompletionService(TestConfiguration.OpenAI.ChatModelId, openAIApiKey) + .Build(); + +var summarizeFunctions = kernel.ImportSemanticFunctionsFromDirectory(RepoFiles.SamplePluginsPath(), "SummarizePlugin"); + +await foreach (var resultBit = await kernel.StreamingRunAsync(ask, searchFunctions["Search"])) +{ + Console.Write(resultBit); +} +``` + +## Considered Options + +### Streaming Setting + New Streaming Interfaces + +This option is to add a new setting `Streaming` to the `AIRequestSettings` definition to flag the caller would like to stream the output from the kernel and functions if possible. + +Seggregate old Text/Chat Completion interfaces the into new `ITextStreamingCompletion` and `IChatStreamingCompletion` dedicated for streaming (interface segregation). **This becomes crucial to the Kernel knows if streaming is supported by the service.** + +#### Backward compatibility aspects: + +- SKContext was built on top of a non streaming functionality, so to keep the compatibility for streaming scenarios, SKContext needs to be updated into a Lazy loading strategy, where when used will enforce the buffering of the whole stream to be able to return to final result. + +- Streaming by default will be `false` + 1. It wont force streaming results over legacy implementations. + 2. Providers like OpenAI/Azure OpenAI don't provide additional data like (Usage, Completion Tokens, Prompt Tokens, Total Tokens, ...) in streaming results. + +Enable streaming behavior set `Streaming = true` in a `AIRequestSettings` specialized class like `OpenAIRequestSettings`, once this is set for text/chat based models like OpenAi's GPT you can get streaming data using `GetValue>()` in the result of `Kernel.RunAsync` or `Function.InvokeAsync`. + +Example: + +``` +myFunction = kernel.CreateSemanticFunction($"your prompt", + requestSettings: new OpenAIRequestSettings + { + Streaming = true, + }); + +var result = await kernel.RunAsync(myFunction); +await foreach (string token in result.GetValue>()) +{ + Console.Write(token); +} +``` + +## Decision Outcome + +As for now, the only proposed solution is the #1. From ebeec438d2a1e2d4d0e8ce905b750486b7025371 Mon Sep 17 00:00:00 2001 From: Roger Barreto <19890735+RogerBarreto@users.noreply.github.com> Date: Mon, 13 Nov 2023 20:44:13 +0000 Subject: [PATCH 05/46] Move ADR number to 23 --- .../{0020-kernel-streaming.md => 0023-kernel-streaming.md} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename docs/decisions/{0020-kernel-streaming.md => 0023-kernel-streaming.md} (100%) diff --git a/docs/decisions/0020-kernel-streaming.md b/docs/decisions/0023-kernel-streaming.md similarity index 100% rename from docs/decisions/0020-kernel-streaming.md rename to docs/decisions/0023-kernel-streaming.md From 9c06b5f37d09c682bf30be3be3b555c90020a844 Mon Sep 17 00:00:00 2001 From: Roger Barreto <19890735+RogerBarreto@users.noreply.github.com> Date: Mon, 13 Nov 2023 22:03:38 +0000 Subject: [PATCH 06/46] Streaming Update --- docs/decisions/0023-kernel-streaming.md | 192 ++++++++++++------------ 1 file changed, 97 insertions(+), 95 deletions(-) diff --git a/docs/decisions/0023-kernel-streaming.md b/docs/decisions/0023-kernel-streaming.md index 8996fd04f55f..051216306263 100644 --- a/docs/decisions/0023-kernel-streaming.md +++ b/docs/decisions/0023-kernel-streaming.md @@ -11,130 +11,132 @@ informed: ## Context and Problem Statement -Its very common in co-pilot implementations to have a streamlined output of messages from the LLM and currently that's not possible while using ISKFunctions.InvokeAsync or Kernel.RunAsync methods, which enforces users to work around the Kernel and Functions to use `ITextCompletion` and `IChatCompletion` services directly which currently support this feature. +Its very common in co-pilot implementations to have a streamlined output of messages from the LLM and currently that's not possible while using ISKFunctions.InvokeAsync or Kernel.RunAsync methods, which enforces users to work around the Kernel and Functions to use `ITextCompletion` and `IChatCompletion` services directly as the only interfaces that currently support streaming. -But notably streaming in its current state is not supported by all providers and this should be added as part or our design to ensure it as a capability the service with a proper abstraction to support also streaming of other types of data like images, audio, video, etc. +Currently streaming is a capability that not all providers do support and this as part of our design we try to ensure the services will have the proper abstractions to support streaming not only of text but be open to other types of data like images, audio, video, etc. -Needs to be clear for Kernel when a connector that supports a specific input/output mimetype is available to be used in the function to function pipeline. - -Needs to be clear for the Kernel when a connector returns a stream how to build the final result from the bits of data returned by the connector in streaming mode, so this information can be appended and passed down to the next function in the pipeline and the streaming can continue. +Needs to be clear for the sk developer when he is attempting to get a streaming data. ## Decision Drivers -- Connectors should be able to flag they have Streaming capability and the Kernel should be able to use it. -- Users should be able to use Kernel to get streaming results. -- Users should be able to use Functions to get streaming results. -- Streaming results can be represented as non streaming results, so getting streaming first approach eventually will be able to support non streaming results. -- Having streaming dedicated methods in Kernel or Functions allow the caller to know what result to expect and implement accordingly. - -## User experience goal - -abstract StreamingResultBit -string mimeType; -object Value; - -StreamingFunctionResult -StreamingResultBuilder Builder; -string mimeType; -Dictionary Metadata; -IAsyncEnumerable StreamingValue; - -// Abstractions to make clear how to build a streaming result from streaming bits enumerations -StreamingResultBuilder -string mimeType; -TComplete Build(); -void Append(TBit streamingBit); - -// Abstraction of a streaming bit -StreamingResultBit -Type streamingBuilder -string mimeType; -TBit streamingBit; - -// Abstract connector modality interface -IConnectorModalityService -string string[] inputMimeTypes; -string string outputMimeType; -AsyncEnumerable GetStreamingResult(object input); -StreamingResultBuilder GetStreamingResultBuilder(string inputMimeType, string outputMimeType); - -// Generic version of connector modality interface -IConnectorModalityService : IConnectorModalityService -string string[] inputMimeTypes; -string string outputMimeType; -AsyncEnumerable GetStreamingResult(T input); -StreamingResultBuilder GetStreamingResultBuilder(); - -// -IServiceSelector -IConnectorModalityService GetServiceByModality(config.inputMimeType ?? "text/plain", config.outputMimeType ?? "text/plain"); - -// Function - -StreamingFunctionResult StreamingInvokeAsync() -{ +1. The sk developer should be able to get streaming data from the Kernel and Functions using Kernel.RunAsync or ISKFunctions.InvokeAsync methods - (IConnectorModalityService service, var defaultRequestSettings) = serviceSelector.GetServiceByModality(renderedPrompt, context.ServiceProvider, this._modelSettings); +2. The sk developer should be able to get the data in a generic way, so the Kernel and Functions can be able to stream data of any type, not limited to text. - var resultBuilder = service.GetStreamingResultBuilder(defaultRequestSettings); - FunctionResult result = new StreamingFunctionResult(this.Name, this.PluginName, context, service.GetStreamingResult) { - Metadata = new() { - { "Usage", defaultRequestSettings.InputMimeType }, - { "outputMimeType", defaultRequestSettings.OutputMimeType }, - } - }; - } +3. The sk developer when using streaming from a model that don't support streaming should still be able to use it with only one streaming update representing the whole data. -} +## User Experience Goal -``` -var kernel = new KernelBuilder() - .WithLoggerFactory(ConsoleLogger.LoggerFactory) - .WithOpenAIChatCompletionService(TestConfiguration.OpenAI.ChatModelId, openAIApiKey) - .Build(); +```csharp +//(providing the type at as generic parameter) + +// Getting a Raw Streaming data from Kernel +await foreach(string update in kernel.StreamingRunAsync(variables, function)) -var summarizeFunctions = kernel.ImportSemanticFunctionsFromDirectory(RepoFiles.SamplePluginsPath(), "SummarizePlugin"); +// Getting a String as Streaming data from Kernel +await foreach(string update in kernel.StreamingRunAsync(variables, function)) -await foreach (var resultBit = await kernel.StreamingRunAsync(ask, searchFunctions["Search"])) +// Getting a StreamingResultUpdate as Streaming data from Kernel +await foreach(StreamingResultUpdate update in kernel.StreamingRunAsync(variables, function)) +// OR +await foreach(StreamingResultUpdate update in kernel.StreamingRunAsync(variables, function)) // defaults to Generic above) { - Console.Write(resultBit); + Console.WriteLine(update); } ``` ## Considered Options -### Streaming Setting + New Streaming Interfaces +### Option 1 - Dedicated Streaming Interfaces -This option is to add a new setting `Streaming` to the `AIRequestSettings` definition to flag the caller would like to stream the output from the kernel and functions if possible. +Using dedicated streaming interfaces that allow the sk developer to get the streaming data in a generic way, including string, byte array directly from the connector as well as allowing the Kernel and Functions implementations to be able to stream data of any type, not limited to text. -Seggregate old Text/Chat Completion interfaces the into new `ITextStreamingCompletion` and `IChatStreamingCompletion` dedicated for streaming (interface segregation). **This becomes crucial to the Kernel knows if streaming is supported by the service.** +This approach also exposes dedicated interfaces in the kernel and functions to use streaming making it clear to the sk developer what is the type of data being returned in IAsyncEnumerable format. -#### Backward compatibility aspects: +`ITextCompletion` and `IChatCompletion` will have new APIs to get `byte[]` and `string` streaming data directly as well as the specialized `StreamingResultUpdate` return. -- SKContext was built on top of a non streaming functionality, so to keep the compatibility for streaming scenarios, SKContext needs to be updated into a Lazy loading strategy, where when used will enforce the buffering of the whole stream to be able to return to final result. +The sk developer will be able to specify a generic type to the `Kernel.StreamingRunAsync()` and `ISKFunction.StreamingInvokeAsync` to get the streaming data. If the type is not specified, the Kernel and Functions will return the data as StreamingResultUpdate. -- Streaming by default will be `false` - 1. It wont force streaming results over legacy implementations. - 2. Providers like OpenAI/Azure OpenAI don't provide additional data like (Usage, Completion Tokens, Prompt Tokens, Total Tokens, ...) in streaming results. +If the type isn't the specified or if the string representation can't be casted an exception will be thrown. -Enable streaming behavior set `Streaming = true` in a `AIRequestSettings` specialized class like `OpenAIRequestSettings`, once this is set for text/chat based models like OpenAi's GPT you can get streaming data using `GetValue>()` in the result of `Kernel.RunAsync` or `Function.InvokeAsync`. +If the type specified is `StreamingResultUpdate`, `string` or `byte[]` no error will be thrown as the connectors will have interface methods that garantee the data to be given in at least those types. -Example: +```csharp -``` -myFunction = kernel.CreateSemanticFunction($"your prompt", - requestSettings: new OpenAIRequestSettings - { - Streaming = true, - }); - -var result = await kernel.RunAsync(myFunction); -await foreach (string token in result.GetValue>()) +// Depending on the underlying function model and connector used the streaming +// data will be of a specialization of StreamingResultUpdate exposing useful +// properties including Type, the raw data in byte[] and string representation. +abstract class StreamingResultUpdate +{ + public abstract string Type { get; } + [JsonIgnore] + public abstract string Value { get; } + [JsonIgnore] + public abstract byte[] RawValue { get; } + + // In a scenario of multiple results, this represents zero-based index of the result in the streaming sequence + public abstract int ResultIndex { get; } +} + +// Specialization example of a ChatMessageUpdate +public class ChatMessageUpdate : StreamingResultUpdate +{ + public override string Type => "ChatMessage"; + public override string Value => JsonSerialize(this); + public override byte[] RawValue => Encoding.UTF8.GetBytes(Value); + + public string Message { get; } + public string Role { get; } + + public ChatMessageUpdate(string message, string role, int resultIndex = 0) + { + Message = message; + Role = role; + ResultIndex = resultIndex; + } +} + +interface IChatCompletion { - Console.Write(token); + IAsyncEnumerable GetStreamingResultAsync(); + IAsyncEnumerable GetStringStreamingResultAsync(); + IAsyncEnumerable GetByteStreamingResultAsync(); +} + +interface ITextCompletion +{ + IAsyncEnumerable GetStreamingResultAsync(); + IAsyncEnumerable GetStringStreamingResultAsync(); + IAsyncEnumerable GetByteStreamingResultAsync(); +} + +interface IKernel +{ + // When the develper provides a T, the Kernel will try to get the streaming data as T + IAsyncEnumerable StreamingRunAsync(ContextVariables variables, ISKFunction function); +} + +interface ISKFunction +{ + // When the develper provides a T, the Kernel will try to get the streaming data as T + IAsyncEnumerable StreamingInvokeAsync(SKContext context); } ``` +## Pros + +1. All the User Experience Goal section options will be possible. +2. Kernel and Functions implementations will be able to stream data of any type, not limited to text +3. The sk developer will be able to provide the type it expects from the `GetStreamingResultAsync` method. +4. The above will allow the sk developer to get the streaming data in a generic way, including `string`, `byte array`, or the `StreamingResultUpdate` abstraction directly from the connector. + +5. IChatCompletion, IKernel and ISKFunction + +## Cons + +1. If the sk developer want to use the specialized type of `StreamingResultUpdate` he will need to know what is the connector being used to use the correct **StreamingResultUpdate extension method** or to provide directly type in ``. +2. Connectors will have greater responsibility to provide the correct type of `StreamingResultUpdate` for the connector being used and implementations for both byte[] and string streaming data. + ## Decision Outcome As for now, the only proposed solution is the #1. From 58f9c78af209ee90170702163af7012a9027f282 Mon Sep 17 00:00:00 2001 From: Roger Barreto <19890735+RogerBarreto@users.noreply.github.com> Date: Mon, 13 Nov 2023 22:13:14 +0000 Subject: [PATCH 07/46] Fix typos --- docs/decisions/0023-kernel-streaming.md | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/docs/decisions/0023-kernel-streaming.md b/docs/decisions/0023-kernel-streaming.md index 051216306263..eb1b665a6507 100644 --- a/docs/decisions/0023-kernel-streaming.md +++ b/docs/decisions/0023-kernel-streaming.md @@ -11,11 +11,11 @@ informed: ## Context and Problem Statement -Its very common in co-pilot implementations to have a streamlined output of messages from the LLM and currently that's not possible while using ISKFunctions.InvokeAsync or Kernel.RunAsync methods, which enforces users to work around the Kernel and Functions to use `ITextCompletion` and `IChatCompletion` services directly as the only interfaces that currently support streaming. +It is quite common in co-pilot implementations to have a streamlined output of messages from the LLM (large language models)M and currently that is not possible while using ISKFunctions.InvokeAsync or Kernel.RunAsync methods, which enforces users to work around the Kernel and Functions to use `ITextCompletion` and `IChatCompletion` services directly as the only interfaces that currently support streaming. Currently streaming is a capability that not all providers do support and this as part of our design we try to ensure the services will have the proper abstractions to support streaming not only of text but be open to other types of data like images, audio, video, etc. -Needs to be clear for the sk developer when he is attempting to get a streaming data. +Needs to be clear for the sk developer when he is attempting to get streaming data. ## Decision Drivers @@ -23,7 +23,7 @@ Needs to be clear for the sk developer when he is attempting to get a streaming 2. The sk developer should be able to get the data in a generic way, so the Kernel and Functions can be able to stream data of any type, not limited to text. -3. The sk developer when using streaming from a model that don't support streaming should still be able to use it with only one streaming update representing the whole data. +3. The sk developer when using streaming from a model that doesn't support streaming should still be able to use it with only one streaming update representing the whole data. ## User Experience Goal @@ -57,9 +57,9 @@ This approach also exposes dedicated interfaces in the kernel and functions to u The sk developer will be able to specify a generic type to the `Kernel.StreamingRunAsync()` and `ISKFunction.StreamingInvokeAsync` to get the streaming data. If the type is not specified, the Kernel and Functions will return the data as StreamingResultUpdate. -If the type isn't the specified or if the string representation can't be casted an exception will be thrown. +If the type isn't specified or if the string representation can't be cast, an exception will be thrown. -If the type specified is `StreamingResultUpdate`, `string` or `byte[]` no error will be thrown as the connectors will have interface methods that garantee the data to be given in at least those types. +If the type specified is `StreamingResultUpdate`, `string` or `byte[]` no error will be thrown as the connectors will have interface methods that guarantee the data to be given in at least those types. ```csharp @@ -112,13 +112,13 @@ interface ITextCompletion interface IKernel { - // When the develper provides a T, the Kernel will try to get the streaming data as T + // When the developer provides a T, the Kernel will try to get the streaming data as T IAsyncEnumerable StreamingRunAsync(ContextVariables variables, ISKFunction function); } interface ISKFunction { - // When the develper provides a T, the Kernel will try to get the streaming data as T + // When the developer provides a T, the Kernel will try to get the streaming data as T IAsyncEnumerable StreamingInvokeAsync(SKContext context); } ``` @@ -134,7 +134,7 @@ interface ISKFunction ## Cons -1. If the sk developer want to use the specialized type of `StreamingResultUpdate` he will need to know what is the connector being used to use the correct **StreamingResultUpdate extension method** or to provide directly type in ``. +1. If the sk developer wants to use the specialized type of `StreamingResultUpdate` he will need to know what the connector is being used to use the correct **StreamingResultUpdate extension method** or to provide directly type in ``. 2. Connectors will have greater responsibility to provide the correct type of `StreamingResultUpdate` for the connector being used and implementations for both byte[] and string streaming data. ## Decision Outcome From 0da7ff35a3eed524432767f34dd1c6cf64502c80 Mon Sep 17 00:00:00 2001 From: Roger Barreto <19890735+RogerBarreto@users.noreply.github.com> Date: Tue, 14 Nov 2023 13:54:51 +0000 Subject: [PATCH 08/46] Streaming implementation + Plans suggestion --- docs/decisions/0023-kernel-streaming.md | 32 ++-- .../AzureSdk/ClientBase.cs | 60 +++++++ .../AzureSdk/StreamingChatResultUpdate.cs | 67 ++++++++ .../AzureSdk/StreamingTextResultUpdate.cs | 46 ++++++ .../AzureOpenAIChatCompletion.cs | 27 ++++ .../AI/ChatCompletion/IChatCompletion.cs | 36 +++++ .../AI/StreamingResultUpdate.cs | 40 +++++ .../AI/TextCompletion/ITextCompletion.cs | 38 ++++- .../Functions/ISKFunction.cs | 12 ++ .../Orchestration/IFunctionRunner.cs | 34 ++++ .../Functions/SemanticFunction.cs | 65 +++++++- dotnet/src/SemanticKernel.Core/Kernel.cs | 45 ++++++ .../src/SemanticKernel.Core/Planning/Plan.cs | 149 ++++++++++++++++++ 13 files changed, 631 insertions(+), 20 deletions(-) create mode 100644 dotnet/src/Connectors/Connectors.AI.OpenAI/AzureSdk/StreamingChatResultUpdate.cs create mode 100644 dotnet/src/Connectors/Connectors.AI.OpenAI/AzureSdk/StreamingTextResultUpdate.cs create mode 100644 dotnet/src/SemanticKernel.Abstractions/AI/StreamingResultUpdate.cs diff --git a/docs/decisions/0023-kernel-streaming.md b/docs/decisions/0023-kernel-streaming.md index eb1b665a6507..d457be15debf 100644 --- a/docs/decisions/0023-kernel-streaming.md +++ b/docs/decisions/0023-kernel-streaming.md @@ -1,8 +1,8 @@ --- # These are optional elements. Feel free to remove any of them. status: proposed -date: 2023-09-26 -deciders: rbarreto,markwallace,sergey,dmytro +date: 2023-11-13 +deciders: rogerbarreto,markwallace-microsoft,SergeyMenshykh,dmytrostruk consulted: informed: --- @@ -23,7 +23,7 @@ Needs to be clear for the sk developer when he is attempting to get streaming da 2. The sk developer should be able to get the data in a generic way, so the Kernel and Functions can be able to stream data of any type, not limited to text. -3. The sk developer when using streaming from a model that doesn't support streaming should still be able to use it with only one streaming update representing the whole data. +3. The sk developer when using streaming from a model that does not support streaming should still be able to use it with only one streaming update representing the whole data. ## User Experience Goal @@ -57,7 +57,7 @@ This approach also exposes dedicated interfaces in the kernel and functions to u The sk developer will be able to specify a generic type to the `Kernel.StreamingRunAsync()` and `ISKFunction.StreamingInvokeAsync` to get the streaming data. If the type is not specified, the Kernel and Functions will return the data as StreamingResultUpdate. -If the type isn't specified or if the string representation can't be cast, an exception will be thrown. +If the type is not specified or if the string representation cannot be cast, an exception will be thrown. If the type specified is `StreamingResultUpdate`, `string` or `byte[]` no error will be thrown as the connectors will have interface methods that guarantee the data to be given in at least those types. @@ -69,21 +69,19 @@ If the type specified is `StreamingResultUpdate`, `string` or `byte[]` no error abstract class StreamingResultUpdate { public abstract string Type { get; } - [JsonIgnore] - public abstract string Value { get; } - [JsonIgnore] - public abstract byte[] RawValue { get; } - // In a scenario of multiple results, this represents zero-based index of the result in the streaming sequence public abstract int ResultIndex { get; } + + public abstract override string ToString(); + public abstract byte[] ToByteArray(); } // Specialization example of a ChatMessageUpdate public class ChatMessageUpdate : StreamingResultUpdate { public override string Type => "ChatMessage"; - public override string Value => JsonSerialize(this); - public override byte[] RawValue => Encoding.UTF8.GetBytes(Value); + public override string ToString() => JsonSerializer.Serialize(this); + public override byte[] ToByteArray() => Encoding.UTF8.GetBytes(Value); public string Message { get; } public string Role { get; } @@ -98,16 +96,16 @@ public class ChatMessageUpdate : StreamingResultUpdate interface IChatCompletion { - IAsyncEnumerable GetStreamingResultAsync(); - IAsyncEnumerable GetStringStreamingResultAsync(); - IAsyncEnumerable GetByteStreamingResultAsync(); + IAsyncEnumerable GetStreamingUpdatesAsync(); + IAsyncEnumerable GetStringStreamingUpdatesAsync(); + IAsyncEnumerable GetByteStreamingUpdatesAsync(); } interface ITextCompletion { - IAsyncEnumerable GetStreamingResultAsync(); - IAsyncEnumerable GetStringStreamingResultAsync(); - IAsyncEnumerable GetByteStreamingResultAsync(); + IAsyncEnumerable GetStreamingUpdatesAsync(); + IAsyncEnumerable GetStringStreamingUpdatesAsync(); + IAsyncEnumerable GetByteStreamingUpdatesAsync(); } interface IKernel diff --git a/dotnet/src/Connectors/Connectors.AI.OpenAI/AzureSdk/ClientBase.cs b/dotnet/src/Connectors/Connectors.AI.OpenAI/AzureSdk/ClientBase.cs index 35dae4906177..d63804d66a47 100644 --- a/dotnet/src/Connectors/Connectors.AI.OpenAI/AzureSdk/ClientBase.cs +++ b/dotnet/src/Connectors/Connectors.AI.OpenAI/AzureSdk/ClientBase.cs @@ -152,6 +152,33 @@ private protected async IAsyncEnumerable InternalGetTextStr } } + private protected async IAsyncEnumerable InternalGetTextStreamingUpdatesAsync( + string prompt, + AIRequestSettings? requestSettings, + [EnumeratorCancellation] CancellationToken cancellationToken = default) + { + OpenAIRequestSettings textRequestSettings = OpenAIRequestSettings.FromRequestSettings(requestSettings, OpenAIRequestSettings.DefaultTextMaxTokens); + + ValidateMaxTokens(textRequestSettings.MaxTokens); + + var options = CreateCompletionsOptions(prompt, textRequestSettings); + + Response? response = await RunRequestAsync>( + () => this.Client.GetCompletionsStreamingAsync(this.DeploymentOrModelName, options, cancellationToken)).ConfigureAwait(false); + + using StreamingCompletions streamingChatCompletions = response.Value; + + int choiceIndex = 0; + await foreach (StreamingChoice choice in streamingChatCompletions.GetChoicesStreaming(cancellationToken).ConfigureAwait(false)) + { + await foreach (string textUpdate in choice.GetTextStreaming(cancellationToken).ConfigureAwait(false)) + { + yield return new StreamingTextResultUpdate(textUpdate, choiceIndex); + } + choiceIndex++; + } + } + /// /// Generates an embedding from the given . /// @@ -261,6 +288,39 @@ private protected async IAsyncEnumerable InternalGetChatSt } } + private protected async IAsyncEnumerable InternalGetChatStreamingUpdatesAsync( + IEnumerable chat, + AIRequestSettings? requestSettings, + [EnumeratorCancellation] CancellationToken cancellationToken = default) + { + Verify.NotNull(chat); + + OpenAIRequestSettings chatRequestSettings = OpenAIRequestSettings.FromRequestSettings(requestSettings); + + ValidateMaxTokens(chatRequestSettings.MaxTokens); + + var options = CreateChatCompletionsOptions(chatRequestSettings, chat); + + Response? response = await RunRequestAsync>( + () => this.Client.GetChatCompletionsStreamingAsync(this.DeploymentOrModelName, options, cancellationToken)).ConfigureAwait(false); + + if (response is null) + { + throw new SKException("Chat completions null response"); + } + + using StreamingChatCompletions streamingChatCompletions = response.Value; + int choiceIndex = 0; + await foreach (StreamingChatChoice choice in streamingChatCompletions.GetChoicesStreaming(cancellationToken).ConfigureAwait(false)) + { + await foreach (ChatMessage chatMessage in choice.GetMessageStreaming(cancellationToken).ConfigureAwait(false)) + { + yield return new StreamingChatResultUpdate(chatMessage, choiceIndex); + } + choiceIndex++; + } + } + /// /// Create a new empty chat instance /// diff --git a/dotnet/src/Connectors/Connectors.AI.OpenAI/AzureSdk/StreamingChatResultUpdate.cs b/dotnet/src/Connectors/Connectors.AI.OpenAI/AzureSdk/StreamingChatResultUpdate.cs new file mode 100644 index 000000000000..8aa13e7f73d1 --- /dev/null +++ b/dotnet/src/Connectors/Connectors.AI.OpenAI/AzureSdk/StreamingChatResultUpdate.cs @@ -0,0 +1,67 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Text; +using System.Text.Json; +using Azure.AI.OpenAI; +using Microsoft.SemanticKernel.AI; +using Microsoft.SemanticKernel.AI.ChatCompletion; + +namespace Microsoft.SemanticKernel.Connectors.AI.OpenAI.AzureSdk; + +/// +/// Streaming chat result update. +/// +public class StreamingChatResultUpdate : StreamingResultUpdate +{ + /// + public override string Type => "openai_chat_message_update"; + + /// + public override int ResultIndex { get; } + + /// + /// Function call associated to the message payload + /// + public FunctionCall FunctionCall { get; } + + /// + /// Text associated to the message payload + /// + public string Content { get; } + + /// + /// Role of the author of the message + /// + public AuthorRole Role { get; } + + /// + /// Name of the author of the message. Name is required if the role is 'function'. + /// + public string Name { get; } + + /// + /// Create a new instance of the class. + /// + /// Original Azure SDK Message update representation + /// Index of the choice + public StreamingChatResultUpdate(ChatMessage chatMessage, int resultIndex) + { + this.ResultIndex = resultIndex; + this.FunctionCall = chatMessage.FunctionCall; + this.Content = chatMessage.Content; + this.Role = new AuthorRole(chatMessage.Role.ToString()); + this.Name = chatMessage.Name; + } + + /// + public override byte[] ToByteArray() + { + return Encoding.UTF8.GetBytes(this.ToString()); + } + + /// + public override string ToString() + { + return JsonSerializer.Serialize(this); + } +} diff --git a/dotnet/src/Connectors/Connectors.AI.OpenAI/AzureSdk/StreamingTextResultUpdate.cs b/dotnet/src/Connectors/Connectors.AI.OpenAI/AzureSdk/StreamingTextResultUpdate.cs new file mode 100644 index 000000000000..37eebefaec51 --- /dev/null +++ b/dotnet/src/Connectors/Connectors.AI.OpenAI/AzureSdk/StreamingTextResultUpdate.cs @@ -0,0 +1,46 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Text; +using Microsoft.SemanticKernel.AI; + +namespace Microsoft.SemanticKernel.Connectors.AI.OpenAI.AzureSdk; + +/// +/// Streaming text result update. +/// +public class StreamingTextResultUpdate : StreamingResultUpdate +{ + /// + public override string Type => "openai_text_update"; + + /// + public override int ResultIndex { get; } + + /// + /// Text associated to the update + /// + public string Content { get; } + + /// + /// Create a new instance of the class. + /// + /// Text update + /// Index of the choice + public StreamingTextResultUpdate(string text, int resultIndex) + { + this.ResultIndex = resultIndex; + this.Content = text; + } + + /// + public override byte[] ToByteArray() + { + return Encoding.UTF8.GetBytes(this.ToString()); + } + + /// + public override string ToString() + { + return this.Content; + } +} diff --git a/dotnet/src/Connectors/Connectors.AI.OpenAI/ChatCompletion/AzureOpenAIChatCompletion.cs b/dotnet/src/Connectors/Connectors.AI.OpenAI/ChatCompletion/AzureOpenAIChatCompletion.cs index c40a039c965c..5cb33b4ade8d 100644 --- a/dotnet/src/Connectors/Connectors.AI.OpenAI/ChatCompletion/AzureOpenAIChatCompletion.cs +++ b/dotnet/src/Connectors/Connectors.AI.OpenAI/ChatCompletion/AzureOpenAIChatCompletion.cs @@ -2,6 +2,8 @@ using System.Collections.Generic; using System.Net.Http; +using System.Runtime.CompilerServices; +using System.Text.Json; using System.Threading; using System.Threading.Tasks; using Azure.AI.OpenAI; @@ -125,4 +127,29 @@ public Task> GetCompletionsAsync( this.LogActionDetails(); return this.InternalGetChatResultsAsTextAsync(text, requestSettings, cancellationToken); } + + /// + public IAsyncEnumerable GetStreamingUpdatesAsync(string input, AIRequestSettings? requestSettings = null, CancellationToken cancellationToken = default) + { + var chatHistory = JsonSerializer.Deserialize(input) ?? new(); + return this.InternalGetChatStreamingUpdatesAsync(chatHistory, requestSettings, cancellationToken); + } + + /// + public async IAsyncEnumerable GetStringStreamingUpdatesAsync(string input, AIRequestSettings? requestSettings = null, [EnumeratorCancellation] CancellationToken cancellationToken = default) + { + await foreach (var update in this.GetStreamingUpdatesAsync(input, requestSettings, cancellationToken).ConfigureAwait(false)) + { + yield return update.ToString(); + } + } + + /// + public async IAsyncEnumerable GetByteStreamingUpdatesAsync(string input, AIRequestSettings? requestSettings = null, [EnumeratorCancellation] CancellationToken cancellationToken = default) + { + await foreach (var update in this.GetStreamingUpdatesAsync(input, requestSettings, cancellationToken).ConfigureAwait(false)) + { + yield return update.ToByteArray(); + } + } } diff --git a/dotnet/src/SemanticKernel.Abstractions/AI/ChatCompletion/IChatCompletion.cs b/dotnet/src/SemanticKernel.Abstractions/AI/ChatCompletion/IChatCompletion.cs index 6d713703c93d..88bb835796f8 100644 --- a/dotnet/src/SemanticKernel.Abstractions/AI/ChatCompletion/IChatCompletion.cs +++ b/dotnet/src/SemanticKernel.Abstractions/AI/ChatCompletion/IChatCompletion.cs @@ -42,4 +42,40 @@ IAsyncEnumerable GetStreamingChatCompletionsAsync( ChatHistory chat, AIRequestSettings? requestSettings = null, CancellationToken cancellationToken = default); + + /// + /// Get streaming completion results for the prompt and settings. + /// + /// The input string. (May be a JSON for complex objects, Byte64 for binary, will depend on the connector spec). + /// Request settings for the completion API + /// The to monitor for cancellation requests. The default is . + /// Streaming list of different completion streaming result updates generated by the remote model + IAsyncEnumerable GetStreamingUpdatesAsync( + string input, + AIRequestSettings? requestSettings = null, + CancellationToken cancellationToken = default); + + /// + /// Get streaming results for the prompt and settings. + /// + /// The input string. (May be a JSON for complex objects, Byte64 for binary, will depend on the connector spec). + /// Request settings for the completion API + /// The to monitor for cancellation requests. The default is . + /// Streaming list of different completion streaming string updates generated by the remote model + IAsyncEnumerable GetStringStreamingUpdatesAsync( + string input, + AIRequestSettings? requestSettings = null, + CancellationToken cancellationToken = default); + + /// + /// Get streaming results for the prompt and settings. + /// + /// The input string. (May be a JSON for complex objects, Byte64 for binary, will depend on the connector spec). + /// Request settings for the completion API + /// The to monitor for cancellation requests. The default is . + /// Streaming list of different completion streaming byte array updates generated by the remote model + IAsyncEnumerable GetByteStreamingUpdatesAsync( + string input, + AIRequestSettings? requestSettings = null, + CancellationToken cancellationToken = default); } diff --git a/dotnet/src/SemanticKernel.Abstractions/AI/StreamingResultUpdate.cs b/dotnet/src/SemanticKernel.Abstractions/AI/StreamingResultUpdate.cs new file mode 100644 index 000000000000..ce3010c47fae --- /dev/null +++ b/dotnet/src/SemanticKernel.Abstractions/AI/StreamingResultUpdate.cs @@ -0,0 +1,40 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Text.Json.Serialization; +using Microsoft.SemanticKernel.Orchestration; + +namespace Microsoft.SemanticKernel.AI; + +/// +/// Represents a single update to a streaming result. +/// +public abstract class StreamingResultUpdate +{ + /// + /// Type of the update. + /// + public abstract string Type { get; } + + /// + /// In a scenario of multiple results, this represents zero-based index of the result in the streaming sequence + /// + public abstract int ResultIndex { get; } + + /// + /// Converts the update class to string. + /// + /// String representation of the update + public abstract override string ToString(); + + /// + /// Converts the update class to byte array. + /// + /// Byte array representation of the update + public abstract byte[] ToByteArray(); + + /// + /// The current context associated the function call. + /// + [JsonIgnore] + internal SKContext? Context { get; set; } +} diff --git a/dotnet/src/SemanticKernel.Abstractions/AI/TextCompletion/ITextCompletion.cs b/dotnet/src/SemanticKernel.Abstractions/AI/TextCompletion/ITextCompletion.cs index 42d8f295ef65..6de89e76e8d1 100644 --- a/dotnet/src/SemanticKernel.Abstractions/AI/TextCompletion/ITextCompletion.cs +++ b/dotnet/src/SemanticKernel.Abstractions/AI/TextCompletion/ITextCompletion.cs @@ -30,9 +30,45 @@ Task> GetCompletionsAsync( /// The prompt to complete. /// Request settings for the completion API /// The to monitor for cancellation requests. The default is . - /// List of different completion streaming results generated by the remote model + /// Streaming list of different completion streaming results generated by the remote model IAsyncEnumerable GetStreamingCompletionsAsync( string text, AIRequestSettings? requestSettings = null, CancellationToken cancellationToken = default); + + /// + /// Get streaming completion results for the prompt and settings. + /// + /// The prompt to complete. + /// Request settings for the completion API + /// The to monitor for cancellation requests. The default is . + /// Streaming list of different completion streaming result updates generated by the remote model + IAsyncEnumerable GetStreamingUpdatesAsync( + string text, + AIRequestSettings? requestSettings = null, + CancellationToken cancellationToken = default); + + /// + /// Get streaming results for the prompt and settings. + /// + /// The prompt to complete. + /// Request settings for the completion API + /// The to monitor for cancellation requests. The default is . + /// Streaming list of different completion streaming string updates generated by the remote model + IAsyncEnumerable GetStringStreamingUpdatesAsync( + string text, + AIRequestSettings? requestSettings = null, + CancellationToken cancellationToken = default); + + /// + /// Get streaming results for the prompt and settings. + /// + /// The prompt to complete. + /// Request settings for the completion API + /// The to monitor for cancellation requests. The default is . + /// Streaming list of different completion streaming byte array updates generated by the remote model + IAsyncEnumerable GetByteStreamingUpdatesAsync( + string text, + AIRequestSettings? requestSettings = null, + CancellationToken cancellationToken = default); } diff --git a/dotnet/src/SemanticKernel.Abstractions/Functions/ISKFunction.cs b/dotnet/src/SemanticKernel.Abstractions/Functions/ISKFunction.cs index cb33723d9539..8023ee254e03 100644 --- a/dotnet/src/SemanticKernel.Abstractions/Functions/ISKFunction.cs +++ b/dotnet/src/SemanticKernel.Abstractions/Functions/ISKFunction.cs @@ -57,6 +57,18 @@ Task InvokeAsync( AIRequestSettings? requestSettings = null, CancellationToken cancellationToken = default); + /// + /// Invoke the in streaming mode. + /// + /// SK context + /// LLM completion settings (for semantic functions only) + /// The updated context, potentially a new one if context switching is implemented. + /// The to monitor for cancellation requests. The default is . + IAsyncEnumerable StreamingInvokeAsync( + SKContext context, + AIRequestSettings? requestSettings = null, + CancellationToken cancellationToken = default); + #region Obsolete /// diff --git a/dotnet/src/SemanticKernel.Abstractions/Orchestration/IFunctionRunner.cs b/dotnet/src/SemanticKernel.Abstractions/Orchestration/IFunctionRunner.cs index 9a9056a0737a..c1391d321e96 100644 --- a/dotnet/src/SemanticKernel.Abstractions/Orchestration/IFunctionRunner.cs +++ b/dotnet/src/SemanticKernel.Abstractions/Orchestration/IFunctionRunner.cs @@ -1,7 +1,9 @@ // Copyright (c) Microsoft. All rights reserved. +using System.Collections.Generic; using System.Threading; using System.Threading.Tasks; +using Microsoft.SemanticKernel.AI; namespace Microsoft.SemanticKernel.Orchestration; @@ -41,4 +43,36 @@ public interface IFunctionRunner string functionName, ContextVariables? variables = null, CancellationToken cancellationToken = default); + + /// + /// Execute a function using the resources loaded in the context in streaming mode. + /// + /// + /// It may return no updates if function was cancelled by a hook + /// + /// Target function to run + /// Input to process + /// The to monitor for cancellation requests. The default is . + /// Result of the function composition + IAsyncEnumerable StreamingRunAsync( + ISKFunction skFunction, + ContextVariables? variables = null, + CancellationToken cancellationToken = default); + + /// + /// Execute a function using the resources loaded in the context in streaming mode. + /// + /// + /// It may return no updates if function was cancelled by a hook + /// + /// The name of the plugin containing the function to run + /// The name of the function to run + /// Input to process + /// The to monitor for cancellation requests. The default is . + /// Result of the function composition + IAsyncEnumerable StreamingRunAsync( + string pluginName, + string functionName, + ContextVariables? variables = null, + CancellationToken cancellationToken = default); } diff --git a/dotnet/src/SemanticKernel.Core/Functions/SemanticFunction.cs b/dotnet/src/SemanticKernel.Core/Functions/SemanticFunction.cs index 52c05cbb434a..35f6760a7d54 100644 --- a/dotnet/src/SemanticKernel.Core/Functions/SemanticFunction.cs +++ b/dotnet/src/SemanticKernel.Core/Functions/SemanticFunction.cs @@ -5,6 +5,8 @@ using System.ComponentModel; using System.Diagnostics; using System.Linq; +using System.Runtime.CompilerServices; +using System.Text; using System.Text.Json; using System.Threading; using System.Threading.Tasks; @@ -85,14 +87,25 @@ public FunctionView Describe() } /// - public async Task InvokeAsync( + public Task InvokeAsync( SKContext context, AIRequestSettings? requestSettings = null, CancellationToken cancellationToken = default) { this.AddDefaultValues(context.Variables); - return await this.RunPromptAsync(requestSettings, context, cancellationToken).ConfigureAwait(false); + return this.RunPromptAsync(requestSettings, context, cancellationToken); + } + + /// + public IAsyncEnumerable StreamingInvokeAsync( + SKContext context, + AIRequestSettings? requestSettings = null, + CancellationToken cancellationToken = default) + { + this.AddDefaultValues(context.Variables); + + return this.RunStreamingPromptAsync(requestSettings, context, cancellationToken); } /// @@ -168,6 +181,43 @@ private void AddDefaultValues(ContextVariables variables) } } + private async IAsyncEnumerable RunStreamingPromptAsync( + AIRequestSettings? requestSettings, + SKContext context, + [EnumeratorCancellation] CancellationToken cancellationToken = default) + { + string renderedPrompt = await this._promptTemplate.RenderAsync(context, cancellationToken).ConfigureAwait(false); + + var serviceSelector = this._serviceSelector ?? context.ServiceSelector; + (var textCompletion, var defaultRequestSettings) = serviceSelector.SelectAIService(context, this); + Verify.NotNull(textCompletion); + + this.CallFunctionInvoking(context, renderedPrompt); + if (SKFunction.IsInvokingCancelOrSkipRequested(context)) + { + yield break; + } + + renderedPrompt = this.GetPromptFromEventArgsMetadataOrDefault(context, renderedPrompt); + + StringBuilder fullCompletion = new(); + await foreach (StreamingResultUpdate update in textCompletion.GetStreamingUpdatesAsync(renderedPrompt, requestSettings ?? defaultRequestSettings, cancellationToken).ConfigureAwait(false)) + { + fullCompletion.Append(update); + + // This currently is needed as plans need use it to update the variables created after the stream ends. + update.Context = context; + + yield return update; + } + + // Update the result with the completion + context.Variables.Update(fullCompletion.ToString()); + this.CallFunctionInvoked(context, renderedPrompt); + + // Post cancellation is not supported as the stream data was already sent. + } + private async Task RunPromptAsync( AIRequestSettings? requestSettings, SKContext context, @@ -267,6 +317,17 @@ private void CallFunctionInvoked(FunctionResult result, SKContext context, strin result.Metadata = eventWrapper.EventArgs.Metadata; } + /// + /// Handles the FunctionInvoked event + /// + /// Execution context + /// Prompt used by the function + private void CallFunctionInvoked(SKContext context, string prompt) => + this.CallFunctionInvoked( + new FunctionResult(this.Name, this.PluginName, context), + context, + prompt); + /// /// Try to get the prompt from the event args metadata. /// diff --git a/dotnet/src/SemanticKernel.Core/Kernel.cs b/dotnet/src/SemanticKernel.Core/Kernel.cs index 38dbf7ebe8d0..7cdb37e89a18 100644 --- a/dotnet/src/SemanticKernel.Core/Kernel.cs +++ b/dotnet/src/SemanticKernel.Core/Kernel.cs @@ -5,10 +5,12 @@ using System.ComponentModel; using System.Globalization; using System.Linq; +using System.Runtime.CompilerServices; using System.Threading; using System.Threading.Tasks; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.SemanticKernel.AI; using Microsoft.SemanticKernel.Diagnostics; using Microsoft.SemanticKernel.Events; using Microsoft.SemanticKernel.Http; @@ -168,6 +170,49 @@ public async Task RunAsync(ContextVariables variables, Cancellatio return KernelResult.FromFunctionResults(allFunctionResults.LastOrDefault()?.Value, allFunctionResults); } + /// + /// Run a function in streaming mode. + /// + /// Input to process + /// Target function to run + /// The to monitor for cancellation requests. + /// Result of the function composition + public async IAsyncEnumerable StreamingRunAsync(ContextVariables variables, ISKFunction skFunction, [EnumeratorCancellation] CancellationToken cancellationToken) + { + var context = this.CreateNewContext(); + + var repeatRequested = false; + + do + { + repeatRequested = false; + + var functionDetails = skFunction.Describe(); + await foreach (StreamingResultUpdate update in skFunction.StreamingInvokeAsync(context, null, cancellationToken).ConfigureAwait(false)) + { + cancellationToken.ThrowIfCancellationRequested(); + + if (this.IsCancelRequested(skFunction, context, 1)) + { + break; + } + + if (this.IsSkipRequested(skFunction, context, 1)) + { + break; + } + + yield return update; + } + + if (this.IsRepeatRequested(skFunction, context, 1)) + { + repeatRequested = true; + } + } + while (repeatRequested); + } + /// public SKContext CreateNewContext( ContextVariables? variables = null, diff --git a/dotnet/src/SemanticKernel.Core/Planning/Plan.cs b/dotnet/src/SemanticKernel.Core/Planning/Plan.cs index 37b5d75a1a54..c4cb2fdef0ed 100644 --- a/dotnet/src/SemanticKernel.Core/Planning/Plan.cs +++ b/dotnet/src/SemanticKernel.Core/Planning/Plan.cs @@ -5,6 +5,8 @@ using System.ComponentModel; using System.Diagnostics; using System.Linq; +using System.Runtime.CompilerServices; +using System.Text; using System.Text.Json; using System.Text.Json.Serialization; using System.Text.RegularExpressions; @@ -342,6 +344,83 @@ public async Task InvokeAsync( return result; } + /// + public async IAsyncEnumerable StreamingInvokeAsync( + SKContext context, + AIRequestSettings? requestSettings = null, + [EnumeratorCancellation] CancellationToken cancellationToken = default) + { + // Keep this implementation for a future refactoring + throw new NotSupportedException("StreamingInvokeAsync is not supported for plans"); + + // Suggested change bellow... + +#pragma warning disable CS0162 // Unreachable code detected + if (this.Function is not null) + { + // Merge state with the current context variables. + // Then filter the variables to only those needed for the next step. + // This is done to prevent the function from having access to variables that it shouldn't. + AddVariablesToContext(this.State, context); + var functionVariables = this.GetNextStepVariables(context.Variables, this); + var functionContext = context.Clone(functionVariables, context.Functions); + + // Execute the step + await foreach (var update in this.Function + .WithInstrumentation(context.LoggerFactory) + .StreamingInvokeAsync(functionContext, requestSettings, cancellationToken) + .ConfigureAwait(false)) + { + yield return update; + } + + // Ensure after the stream ended the data processed in skcontext is passed further + var functionResult = new FunctionResult(this.Function.Name, this.Function.PluginName, context); + this.UpdateFunctionResultWithOutputs(functionResult); + + yield break; + } + + this.CallFunctionInvoking(context); + if (SKFunction.IsInvokingCancelOrSkipRequested(context)) + { + yield break; + } + + // loop through steps and execute until completion + while (this.HasNextStep) + { + AddVariablesToContext(this.State, context); + SKContext? resultContext = null; + + int streamingUpdates = 0; + await foreach (var update in this.InternalStreamingInvokeNextStepAsync(context, cancellationToken).ConfigureAwait(false)) + { + // This currently is needed as plans need use it to update the variables created after the stream ends. + resultContext ??= update.Context; + + streamingUpdates++; + + yield return update; + }; + + if (context.FunctionInvokingHandler?.EventArgs?.IsSkipRequested ?? false) + { + continue; + } + + if (streamingUpdates > 0) + { + this.UpdateContextWithOutputs(resultContext!); + } + } + + this.CallFunctionInvoked(context); + + // Post cancellation is not supported as the stream data was already sent. +#pragma warning restore CS0162 // Unreachable code detected + } + #endregion ISKFunction implementation /// @@ -435,6 +514,67 @@ internal string ExpandFromVariables(ContextVariables variables, string input) throw new InvalidOperationException("There isn't a next step"); } + private async IAsyncEnumerable InternalStreamingInvokeNextStepAsync(SKContext context, [EnumeratorCancellation] CancellationToken cancellationToken = default) + { + if (!this.HasNextStep) + { + throw new InvalidOperationException("There isn't a next step"); + } + + var step = this.Steps[this.NextStepIndex]; + + // Merge the state with the current context variables for step execution + var functionVariables = this.GetNextStepVariables(context.Variables, step); + + // Execute the step + StringBuilder fullResult = new(); + SKContext? resultContext = null; + await foreach (var update in context.Runner.StreamingRunAsync(step, functionVariables, cancellationToken).ConfigureAwait(false)) + { + resultContext ??= update.Context; + + fullResult.Append(update); + yield return update; + } + + var resultValue = fullResult.ToString().Trim(); + + #region Update State + + // Update state with result + this.State.Update(resultValue); + + // Update Plan Result in State with matching outputs (if any) + if (this.Outputs.Intersect(step.Outputs).Any()) + { + if (this.State.TryGetValue(DefaultResultKey, out string? currentPlanResult)) + { + this.State.Set(DefaultResultKey, $"{currentPlanResult}\n{resultValue}"); + } + else + { + this.State.Set(DefaultResultKey, resultValue); + } + } + + // Update state with outputs (if any) + foreach (var item in step.Outputs) + { + if (resultContext?.Variables.TryGetValue(item, out string? val) ?? false) + { + this.State.Set(item, val); + } + else + { + this.State.Set(item, resultValue); + } + } + + #endregion Update State + + this.NextStepIndex++; + } + private void CallFunctionInvoking(SKContext context) { var eventWrapper = context.FunctionInvokingHandler; @@ -465,6 +605,15 @@ private void CallFunctionInvoked(FunctionResult result, SKContext context) result.Metadata = eventWrapper.EventArgs.Metadata; } + /// + /// Handles the FunctionInvoked event + /// + /// Execution context + private void CallFunctionInvoked(SKContext context) => + this.CallFunctionInvoked( + new FunctionResult(this.Name, this.PluginName, context), + context); + /// /// Set functions for a plan and its steps. /// From 8c518180fe0e5c70025428c91c3931e3f81b3af3 Mon Sep 17 00:00:00 2001 From: Roger Barreto <19890735+RogerBarreto@users.noreply.github.com> Date: Tue, 14 Nov 2023 15:20:16 +0000 Subject: [PATCH 09/46] Native methods support for streaming + Removal of Plan suggested impl --- docs/decisions/0023-kernel-streaming.md | 19 ++- .../SemanticKernel.Abstractions/IKernel.cs | 10 ++ .../Functions/InstrumentedSKFunction.cs | 38 +++++ .../Functions/NativeFunction.cs | 122 +++++++++++++++- .../Functions/StreamingNativeResultUpdate.cs | 52 +++++++ dotnet/src/SemanticKernel.Core/Kernel.cs | 6 +- .../Planning/InstrumentedPlan.cs | 8 + .../src/SemanticKernel.Core/Planning/Plan.cs | 138 +----------------- 8 files changed, 253 insertions(+), 140 deletions(-) create mode 100644 dotnet/src/SemanticKernel.Core/Functions/StreamingNativeResultUpdate.cs diff --git a/docs/decisions/0023-kernel-streaming.md b/docs/decisions/0023-kernel-streaming.md index d457be15debf..163769850c4c 100644 --- a/docs/decisions/0023-kernel-streaming.md +++ b/docs/decisions/0023-kernel-streaming.md @@ -7,7 +7,7 @@ consulted: informed: --- -# Streaming Capability for Kernel and Functions usage +# Streaming Capability for Kernel and Functions usage - Phase 1 ## Context and Problem Statement @@ -45,6 +45,11 @@ await foreach(StreamingResultUpdate update in kernel.StreamingRunAsync(variables } ``` +## Out of Scope + +- Streaming with plans will not be supported in this phase. Attempting to do so will throw an exception. +- Kernel streaming will not support multiple functions (pipeline). + ## Considered Options ### Option 1 - Dedicated Streaming Interfaces @@ -121,6 +126,18 @@ interface ISKFunction } ``` +## Semantic Functions Behavior + +When Semantic Functions are invoked using the Streaming API, they will attempt to use the Connectors streaming implementation. The connector will be responsible to provide the specialized type of `StreamingResultUpdate` as well as to keep the API streaming working (streaming the single complete result) even when the backend don't support it. + +## Native Functions Behavior + +NativeFunctions will support StreamingResults automatically with a StreamingNativeResultUpdate wrapping the object returned in the iterator. + +If NativeFunctions are already IAsyncEnumerable methods, the result will be automatically wrapped in the StreamingNativeResultUpdate keeping the streaming behavior and the overall abstraction consistent. + +If NativeFunctions don't return IAsyncEnumerable, the result will be wrapped in a StreamingNativeResultUpdate and the result will be returned as a single result. + ## Pros 1. All the User Experience Goal section options will be possible. diff --git a/dotnet/src/SemanticKernel.Abstractions/IKernel.cs b/dotnet/src/SemanticKernel.Abstractions/IKernel.cs index 6dbd645141f3..5c849cf46800 100644 --- a/dotnet/src/SemanticKernel.Abstractions/IKernel.cs +++ b/dotnet/src/SemanticKernel.Abstractions/IKernel.cs @@ -7,6 +7,7 @@ using System.Threading; using System.Threading.Tasks; using Microsoft.Extensions.Logging; +using Microsoft.SemanticKernel.AI; using Microsoft.SemanticKernel.Events; using Microsoft.SemanticKernel.Http; using Microsoft.SemanticKernel.Memory; @@ -55,6 +56,15 @@ Task RunAsync( CancellationToken cancellationToken, params ISKFunction[] pipeline); + /// + /// Run a function in streaming mode. + /// + /// Target function to run + /// Input to process + /// The to monitor for cancellation requests. + /// Result of the function composition + public IAsyncEnumerable StreamingRunAsync(ISKFunction skFunction, ContextVariables? variables, CancellationToken cancellationToken); + /// /// Create a new instance of a context, linked to the kernel internal state. /// diff --git a/dotnet/src/SemanticKernel.Core/Functions/InstrumentedSKFunction.cs b/dotnet/src/SemanticKernel.Core/Functions/InstrumentedSKFunction.cs index 84c670376a64..4511c72645c5 100644 --- a/dotnet/src/SemanticKernel.Core/Functions/InstrumentedSKFunction.cs +++ b/dotnet/src/SemanticKernel.Core/Functions/InstrumentedSKFunction.cs @@ -79,6 +79,14 @@ public async Task InvokeAsync( this._function.InvokeAsync(context, requestSettings, cancellationToken)).ConfigureAwait(false); } + /// + public IAsyncEnumerable StreamingInvokeAsync( + SKContext context, + AIRequestSettings? requestSettings = null, + CancellationToken cancellationToken = default) + => this.StreamingInvokeWithInstrumentationAsync( + () => this._function.StreamingInvokeAsync(context, requestSettings, cancellationToken)); + #region private ================================================================================ private readonly ISKFunction _function; @@ -163,6 +171,36 @@ private async Task InvokeWithInstrumentationAsync(Func + /// Wrapper for instrumentation to be used in multiple invocation places. + /// + /// Delegate to instrument. + private async IAsyncEnumerable StreamingInvokeWithInstrumentationAsync(Func> func) + { + using var activity = s_activitySource.StartActivity($"{this.PluginName}.{this.Name}"); + + this._logger.LogInformation("{PluginName}.{FunctionName}: Function execution started.", this.PluginName, this.Name); + + var stopwatch = new Stopwatch(); + stopwatch.Start(); + + await foreach (var update in func().ConfigureAwait(false)) + { + yield return update; + } + + stopwatch.Stop(); + this._executionTotalCounter.Add(1); + this._executionTimeHistogram.Record(stopwatch.ElapsedMilliseconds); + + this._logger.LogInformation("{PluginName}.{FunctionName}: Function execution status: {Status}", + this.PluginName, this.Name, "Success"); + + this._logger.LogInformation("{PluginName}.{FunctionName}: Function execution finished in {ExecutionTime}ms", + this.PluginName, this.Name, stopwatch.ElapsedMilliseconds); + + this._executionSuccessCounter.Add(1); + } #endregion #region Obsolete ======================================================================= diff --git a/dotnet/src/SemanticKernel.Core/Functions/NativeFunction.cs b/dotnet/src/SemanticKernel.Core/Functions/NativeFunction.cs index fc3e5627b4a1..1a1a28c9b07a 100644 --- a/dotnet/src/SemanticKernel.Core/Functions/NativeFunction.cs +++ b/dotnet/src/SemanticKernel.Core/Functions/NativeFunction.cs @@ -9,6 +9,7 @@ using System.Globalization; using System.Linq; using System.Reflection; +using System.Runtime.CompilerServices; using System.Text.Json; using System.Text.RegularExpressions; using System.Threading; @@ -71,6 +72,7 @@ public static ISKFunction Create( MethodDetails methodDetails = GetMethodDetails(method, target, pluginName!, logger); var result = new NativeFunction( methodDetails.Function, + methodDetails.StreamingFunc, pluginName!, functionName ?? methodDetails.Name, description ?? methodDetails.Description, @@ -152,6 +154,42 @@ public async Task InvokeAsync( } } + public async IAsyncEnumerable StreamingInvokeAsync(SKContext context, + AIRequestSettings? requestSettings = null, + [EnumeratorCancellation] CancellationToken cancellationToken = default) + { + // Invoke pre hook, and stop if skipping requested. + this.CallFunctionInvoking(context); + if (SKFunction.IsInvokingCancelOrSkipRequested(context)) + { + if (this._logger.IsEnabled(LogLevel.Trace)) + { + this._logger.LogTrace("Function {Plugin}.{Name} canceled or skipped prior to invocation.", this.PluginName, this.Name); + } + + yield break; + } + + if (this._logger.IsEnabled(LogLevel.Trace)) + { + this._logger.LogTrace("Function {Plugin}.{Name} streaming invoke.", this.PluginName, this.Name); + } + + await foreach (var update in this._streamingFunction(null, requestSettings, context, cancellationToken).ConfigureAwait(false)) + { + yield return update; + } + + var result = this.CallFunctionInvoked(context); + + if (this._logger.IsEnabled(LogLevel.Trace)) + { + this._logger.LogTrace("Function {Plugin}.{Name} invocation {Completion}: {Result}", + this.PluginName, this.Name, + SKFunction.IsInvokedCancelRequested(context) ? "canceled" : "completed", + result.Value); + } + } private void CallFunctionInvoking(SKContext context) { var eventWrapper = context.FunctionInvokingHandler; @@ -162,6 +200,9 @@ private void CallFunctionInvoking(SKContext context) } } + private FunctionResult CallFunctionInvoked(SKContext context) => + this.CallFunctionInvoked(new FunctionResult(this.Name, this.PluginName, context), context); + private FunctionResult CallFunctionInvoked(FunctionResult result, SKContext context) { var eventWrapper = context.FunctionInvokedHandler; @@ -203,17 +244,25 @@ private delegate ValueTask ImplementationFunc( SKContext context, CancellationToken cancellationToken); + private delegate IAsyncEnumerable ImplementationStreamingFunc( + ITextCompletion? textCompletion, + AIRequestSettings? requestSettingsk, + SKContext context, + CancellationToken cancellationToken); + private static readonly JsonSerializerOptions s_toStringStandardSerialization = new(); private static readonly JsonSerializerOptions s_toStringIndentedSerialization = new() { WriteIndented = true }; private readonly ImplementationFunc _function; + private readonly ImplementationStreamingFunc _streamingFunction; private readonly IReadOnlyList _parameters; private readonly ReturnParameterView _returnParameter; private readonly ILogger _logger; - private record struct MethodDetails(string Name, string Description, ImplementationFunc Function, List Parameters, ReturnParameterView ReturnParameter); + private record struct MethodDetails(string Name, string Description, ImplementationFunc Function, ImplementationStreamingFunc StreamingFunc, List Parameters, ReturnParameterView ReturnParameter); private NativeFunction( ImplementationFunc implementationFunc, + ImplementationStreamingFunc implementationStreamingFunc, string pluginName, string functionName, string description, @@ -227,6 +276,7 @@ private NativeFunction( this._logger = logger; this._function = implementationFunc; + this._streamingFunction = implementationStreamingFunc; this._parameters = parameters.ToArray(); Verify.ParametersUniqueness(this._parameters); this._returnParameter = returnParameter; @@ -298,11 +348,65 @@ ValueTask Function(ITextCompletion? text, AIRequestSettings? req return returnFunc(functionName!, pluginName, result, context); } + // Create the streaming func + async IAsyncEnumerable StreamingFunction( + ITextCompletion? text, + AIRequestSettings? requestSettings, + SKContext context, + [EnumeratorCancellation] CancellationToken cancellationToken) + { + // Create the arguments. + object?[] args = parameterFuncs!.Length != 0 ? new object?[parameterFuncs.Length] : Array.Empty(); + for (int i = 0; i < args.Length; i++) + { + args[i] = parameterFuncs[i](context, cancellationToken); + } + + if (IsAsyncEnumerable(method, out var enumeratedTypes)) + { + // Invoke the method to get the IAsyncEnumerable instance + object asyncEnumerable = method.Invoke(target, null); + if (asyncEnumerable == null) + { + yield break; + } + + // Get the method for GetAsyncEnumerator() + MethodInfo getAsyncEnumeratorMethod = typeof(IAsyncEnumerable<>) + .MakeGenericType(enumeratedTypes) + .GetMethod("GetAsyncEnumerator"); + + object asyncEnumerator = getAsyncEnumeratorMethod.Invoke(asyncEnumerable, null); + + // Get the MoveNextAsync() and Current properties + MethodInfo moveNextAsyncMethod = asyncEnumerator.GetType().GetMethod("MoveNextAsync"); + PropertyInfo currentProperty = asyncEnumerator.GetType().GetProperty("Current"); + + // Iterate over the items + while (await ((ValueTask)moveNextAsyncMethod.Invoke(asyncEnumerator, null)).ConfigureAwait(false)) + { + object currentItem = currentProperty.GetValue(asyncEnumerator); + + yield return new StreamingNativeResultUpdate(currentItem); + } + } + else + { + // Invoke the method. + object? result = method.Invoke(target, args); + + if (result is not null) + { + yield return new StreamingNativeResultUpdate(result); + } + } + } + // And return the details. return new MethodDetails { Function = Function, - + StreamingFunc = StreamingFunction, Name = functionName!, Description = method.GetCustomAttribute(inherit: true)?.Description ?? "", Parameters = stringParameterViews, @@ -332,6 +436,20 @@ private static bool IsAsyncMethod(MethodInfo method) return false; } + private static bool IsAsyncEnumerable(MethodInfo method, out Type[] enumeratedTypes) + { + Type returnType = method.ReturnType; + + if (returnType.IsGenericType && returnType.GetGenericTypeDefinition() == typeof(IAsyncEnumerable<>)) + { + enumeratedTypes = returnType.GetGenericArguments(); + return true; + } + + enumeratedTypes = Array.Empty(); + return false; + } + /// /// Gets a delegate for handling the marshaling of a parameter. /// diff --git a/dotnet/src/SemanticKernel.Core/Functions/StreamingNativeResultUpdate.cs b/dotnet/src/SemanticKernel.Core/Functions/StreamingNativeResultUpdate.cs new file mode 100644 index 000000000000..ae9d64aeb89d --- /dev/null +++ b/dotnet/src/SemanticKernel.Core/Functions/StreamingNativeResultUpdate.cs @@ -0,0 +1,52 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Text; +using Microsoft.SemanticKernel.AI; + +#pragma warning disable IDE0130 +// ReSharper disable once CheckNamespace - Using the main namespace +namespace Microsoft.SemanticKernel; +#pragma warning restore IDE0130 + +/// +/// Native function streaming result update. +/// +public sealed class StreamingNativeResultUpdate : StreamingResultUpdate +{ + /// + public override string Type => "native_result_update"; + + /// + public override int ResultIndex => 0; + + /// + /// Native object that represents the update + /// + public object Update { get; } + + /// + public override byte[] ToByteArray() + { + if (this.Update is byte[]) + { + return (byte[])this.Update; + } + + return Encoding.UTF8.GetBytes(this.Update?.ToString()); + } + + /// + public override string ToString() + { + return this.Update.ToString(); + } + + /// + /// Initializes a new instance of the class. + /// + /// Underlying object that represents the update + public StreamingNativeResultUpdate(object update) + { + this.Update = update; + } +} diff --git a/dotnet/src/SemanticKernel.Core/Kernel.cs b/dotnet/src/SemanticKernel.Core/Kernel.cs index 7cdb37e89a18..30948b0475f8 100644 --- a/dotnet/src/SemanticKernel.Core/Kernel.cs +++ b/dotnet/src/SemanticKernel.Core/Kernel.cs @@ -173,13 +173,13 @@ public async Task RunAsync(ContextVariables variables, Cancellatio /// /// Run a function in streaming mode. /// - /// Input to process /// Target function to run + /// Input to process /// The to monitor for cancellation requests. /// Result of the function composition - public async IAsyncEnumerable StreamingRunAsync(ContextVariables variables, ISKFunction skFunction, [EnumeratorCancellation] CancellationToken cancellationToken) + public async IAsyncEnumerable StreamingRunAsync(ISKFunction skFunction, ContextVariables? variables, [EnumeratorCancellation] CancellationToken cancellationToken) { - var context = this.CreateNewContext(); + var context = this.CreateNewContext(variables); var repeatRequested = false; diff --git a/dotnet/src/SemanticKernel.Core/Planning/InstrumentedPlan.cs b/dotnet/src/SemanticKernel.Core/Planning/InstrumentedPlan.cs index c04d0bf7a171..35af83e89cc1 100644 --- a/dotnet/src/SemanticKernel.Core/Planning/InstrumentedPlan.cs +++ b/dotnet/src/SemanticKernel.Core/Planning/InstrumentedPlan.cs @@ -61,6 +61,14 @@ public async Task InvokeAsync( this._plan.InvokeAsync(context, requestSettings, cancellationToken)).ConfigureAwait(false); } + /// + public IAsyncEnumerable StreamingInvokeAsync(SKContext context, + AIRequestSettings? requestSettings = null, + CancellationToken cancellationToken = default) + { + // Implementation will be added in future streaming feature iteration + throw new NotSupportedException("Streaming currently is not supported for plans"); + } #region private ================================================================================ private readonly ISKFunction _plan; diff --git a/dotnet/src/SemanticKernel.Core/Planning/Plan.cs b/dotnet/src/SemanticKernel.Core/Planning/Plan.cs index c4cb2fdef0ed..210842f80f4e 100644 --- a/dotnet/src/SemanticKernel.Core/Planning/Plan.cs +++ b/dotnet/src/SemanticKernel.Core/Planning/Plan.cs @@ -5,8 +5,6 @@ using System.ComponentModel; using System.Diagnostics; using System.Linq; -using System.Runtime.CompilerServices; -using System.Text; using System.Text.Json; using System.Text.Json.Serialization; using System.Text.RegularExpressions; @@ -345,80 +343,13 @@ public async Task InvokeAsync( } /// - public async IAsyncEnumerable StreamingInvokeAsync( + public IAsyncEnumerable StreamingInvokeAsync( SKContext context, AIRequestSettings? requestSettings = null, - [EnumeratorCancellation] CancellationToken cancellationToken = default) + CancellationToken cancellationToken = default) { - // Keep this implementation for a future refactoring - throw new NotSupportedException("StreamingInvokeAsync is not supported for plans"); - - // Suggested change bellow... - -#pragma warning disable CS0162 // Unreachable code detected - if (this.Function is not null) - { - // Merge state with the current context variables. - // Then filter the variables to only those needed for the next step. - // This is done to prevent the function from having access to variables that it shouldn't. - AddVariablesToContext(this.State, context); - var functionVariables = this.GetNextStepVariables(context.Variables, this); - var functionContext = context.Clone(functionVariables, context.Functions); - - // Execute the step - await foreach (var update in this.Function - .WithInstrumentation(context.LoggerFactory) - .StreamingInvokeAsync(functionContext, requestSettings, cancellationToken) - .ConfigureAwait(false)) - { - yield return update; - } - - // Ensure after the stream ended the data processed in skcontext is passed further - var functionResult = new FunctionResult(this.Function.Name, this.Function.PluginName, context); - this.UpdateFunctionResultWithOutputs(functionResult); - - yield break; - } - - this.CallFunctionInvoking(context); - if (SKFunction.IsInvokingCancelOrSkipRequested(context)) - { - yield break; - } - - // loop through steps and execute until completion - while (this.HasNextStep) - { - AddVariablesToContext(this.State, context); - SKContext? resultContext = null; - - int streamingUpdates = 0; - await foreach (var update in this.InternalStreamingInvokeNextStepAsync(context, cancellationToken).ConfigureAwait(false)) - { - // This currently is needed as plans need use it to update the variables created after the stream ends. - resultContext ??= update.Context; - - streamingUpdates++; - - yield return update; - }; - - if (context.FunctionInvokingHandler?.EventArgs?.IsSkipRequested ?? false) - { - continue; - } - - if (streamingUpdates > 0) - { - this.UpdateContextWithOutputs(resultContext!); - } - } - - this.CallFunctionInvoked(context); - - // Post cancellation is not supported as the stream data was already sent. -#pragma warning restore CS0162 // Unreachable code detected + // Implementation will be added in future streaming feature iteration + throw new NotSupportedException("Streaming currently is not supported for plans"); } #endregion ISKFunction implementation @@ -514,67 +445,6 @@ internal string ExpandFromVariables(ContextVariables variables, string input) throw new InvalidOperationException("There isn't a next step"); } - private async IAsyncEnumerable InternalStreamingInvokeNextStepAsync(SKContext context, [EnumeratorCancellation] CancellationToken cancellationToken = default) - { - if (!this.HasNextStep) - { - throw new InvalidOperationException("There isn't a next step"); - } - - var step = this.Steps[this.NextStepIndex]; - - // Merge the state with the current context variables for step execution - var functionVariables = this.GetNextStepVariables(context.Variables, step); - - // Execute the step - StringBuilder fullResult = new(); - SKContext? resultContext = null; - await foreach (var update in context.Runner.StreamingRunAsync(step, functionVariables, cancellationToken).ConfigureAwait(false)) - { - resultContext ??= update.Context; - - fullResult.Append(update); - yield return update; - } - - var resultValue = fullResult.ToString().Trim(); - - #region Update State - - // Update state with result - this.State.Update(resultValue); - - // Update Plan Result in State with matching outputs (if any) - if (this.Outputs.Intersect(step.Outputs).Any()) - { - if (this.State.TryGetValue(DefaultResultKey, out string? currentPlanResult)) - { - this.State.Set(DefaultResultKey, $"{currentPlanResult}\n{resultValue}"); - } - else - { - this.State.Set(DefaultResultKey, resultValue); - } - } - - // Update state with outputs (if any) - foreach (var item in step.Outputs) - { - if (resultContext?.Variables.TryGetValue(item, out string? val) ?? false) - { - this.State.Set(item, val); - } - else - { - this.State.Set(item, resultValue); - } - } - - #endregion Update State - - this.NextStepIndex++; - } - private void CallFunctionInvoking(SKContext context) { var eventWrapper = context.FunctionInvokingHandler; From f174f5b1314329f5cc054a6030d48ba36e032feb Mon Sep 17 00:00:00 2001 From: Roger Barreto <19890735+RogerBarreto@users.noreply.github.com> Date: Tue, 14 Nov 2023 16:33:15 +0000 Subject: [PATCH 10/46] Adjusting WithData Completion + Examples --- .../Example16_CustomLLM.cs | 53 +++++++++++++++++++ .../Example34_CustomChatModel.cs | 50 +++++++++++++++++ .../HuggingFaceTextCompletion.cs | 37 +++++++++++++ .../StreamingTextResultUpdate.cs | 46 ++++++++++++++++ .../AzureSdk/StreamingChatResultUpdate.cs | 22 ++++++-- .../AzureSdk/StreamingTextResultUpdate.cs | 2 +- .../ChatCompletion/OpenAIChatCompletion.cs | 27 ++++++++++ .../AzureOpenAIChatCompletionWithData.cs | 37 +++++++++++++ .../ChatWithDataStreamingResult.cs | 2 +- .../TextCompletion/AzureTextCompletion.cs | 31 +++++++++-- .../TextCompletion/OpenAITextCompletion.cs | 25 +++++++++ .../KernelSemanticFunctionExtensionsTests.cs | 14 +++++ .../AI/TextCompletion/ITextCompletion.cs | 12 ++--- .../Orchestration/FunctionRunner.cs | 12 +++++ ...redIAIServiceConfigurationProviderTests.cs | 15 ++++++ 15 files changed, 370 insertions(+), 15 deletions(-) create mode 100644 dotnet/src/Connectors/Connectors.AI.HuggingFace/TextCompletion/StreamingTextResultUpdate.cs diff --git a/dotnet/samples/KernelSyntaxExamples/Example16_CustomLLM.cs b/dotnet/samples/KernelSyntaxExamples/Example16_CustomLLM.cs index e0077418362b..6eb4fed7074b 100644 --- a/dotnet/samples/KernelSyntaxExamples/Example16_CustomLLM.cs +++ b/dotnet/samples/KernelSyntaxExamples/Example16_CustomLLM.cs @@ -2,7 +2,9 @@ using System; using System.Collections.Generic; +using System.Linq; using System.Runtime.CompilerServices; +using System.Text; using System.Text.Json; using System.Threading; using System.Threading.Tasks; @@ -47,6 +49,57 @@ public async IAsyncEnumerable GetStreamingCompletionsAsync { yield return new MyTextCompletionStreamingResult(); } + + public async IAsyncEnumerable GetByteStreamingUpdatesAsync(string input, AIRequestSettings? requestSettings = null, [EnumeratorCancellation] CancellationToken cancellationToken = default) + { + await foreach (var update in this.GetStreamingUpdatesAsync(input, requestSettings, cancellationToken)) + { + yield return update.ToByteArray(); + } + } + + public IAsyncEnumerable GetStreamingUpdatesAsync(string input, AIRequestSettings? requestSettings = null, CancellationToken cancellationToken = default) + { + var list = new List() + { + new MyStreamingResultUpdate("llm content update 1"), + new MyStreamingResultUpdate("llm content update 2") + }; + + return list.ToAsyncEnumerable(); + } + + public async IAsyncEnumerable GetStringStreamingUpdatesAsync(string input, AIRequestSettings? requestSettings = null, [EnumeratorCancellation] CancellationToken cancellationToken = default) + { + await foreach (var update in this.GetStreamingUpdatesAsync(input, requestSettings, cancellationToken)) + { + yield return update.ToString(); + } + } +} + +public class MyStreamingResultUpdate : StreamingResultUpdate +{ + public override string Type => "my_text_type"; + + public override int ResultIndex => 0; + + public string Content { get; } + + public MyStreamingResultUpdate(string content) + { + this.Content = content; + } + + public override byte[] ToByteArray() + { + return Encoding.UTF8.GetBytes(this.Content); + } + + public override string ToString() + { + return this.Content; + } } public class MyTextCompletionStreamingResult : ITextStreamingResult, ITextResult diff --git a/dotnet/samples/KernelSyntaxExamples/Example34_CustomChatModel.cs b/dotnet/samples/KernelSyntaxExamples/Example34_CustomChatModel.cs index 752385346771..30bb22d3cdcd 100644 --- a/dotnet/samples/KernelSyntaxExamples/Example34_CustomChatModel.cs +++ b/dotnet/samples/KernelSyntaxExamples/Example34_CustomChatModel.cs @@ -4,6 +4,7 @@ using System.Collections.Generic; using System.Linq; using System.Runtime.CompilerServices; +using System.Text; using System.Threading; using System.Threading.Tasks; using Microsoft.SemanticKernel.AI; @@ -51,8 +52,57 @@ public IAsyncEnumerable GetStreamingChatCompletionsAsync(C new MyChatStreamingResult(MyRoles.Bot, "Hi I'm your SK Custom Assistant and I'm here to help you to create custom chats like this. :)") }).ToAsyncEnumerable(); } + public async IAsyncEnumerable GetByteStreamingUpdatesAsync(string input, AIRequestSettings? requestSettings = null, [EnumeratorCancellation] CancellationToken cancellationToken = default) + { + await foreach (var update in this.GetStreamingUpdatesAsync(input, requestSettings, cancellationToken)) + { + yield return update.ToByteArray(); + } + } + + public IAsyncEnumerable GetStreamingUpdatesAsync(string input, AIRequestSettings? requestSettings = null, CancellationToken cancellationToken = default) + { + var list = new List() + { + new MyStreamingChatResultUpdate("llm content update 1"), + new MyStreamingChatResultUpdate("llm content update 2") + }; + + return list.ToAsyncEnumerable(); + } + + public async IAsyncEnumerable GetStringStreamingUpdatesAsync(string input, AIRequestSettings? requestSettings = null, [EnumeratorCancellation] CancellationToken cancellationToken = default) + { + await foreach (var update in this.GetStreamingUpdatesAsync(input, requestSettings, cancellationToken)) + { + yield return update.ToString(); + } + } } +public class MyStreamingChatResultUpdate : StreamingResultUpdate +{ + public override string Type => "my_chat_type"; + + public override int ResultIndex => 0; + + public string Content { get; } + + public MyStreamingChatResultUpdate(string content) + { + this.Content = content; + } + + public override byte[] ToByteArray() + { + return Encoding.UTF8.GetBytes(this.Content); + } + + public override string ToString() + { + return this.Content; + } +} public class MyChatStreamingResult : IChatStreamingResult { private readonly ChatMessageBase _message; diff --git a/dotnet/src/Connectors/Connectors.AI.HuggingFace/TextCompletion/HuggingFaceTextCompletion.cs b/dotnet/src/Connectors/Connectors.AI.HuggingFace/TextCompletion/HuggingFaceTextCompletion.cs index 168e4e0d8bb4..cb23fe868c5d 100644 --- a/dotnet/src/Connectors/Connectors.AI.HuggingFace/TextCompletion/HuggingFaceTextCompletion.cs +++ b/dotnet/src/Connectors/Connectors.AI.HuggingFace/TextCompletion/HuggingFaceTextCompletion.cs @@ -3,6 +3,7 @@ using System; using System.Collections.Generic; using System.Net.Http; +using System.Runtime.CompilerServices; using System.Text.Json; using System.Threading; using System.Threading.Tasks; @@ -90,6 +91,42 @@ public async Task> GetCompletionsAsync( return await this.ExecuteGetCompletionsAsync(text, cancellationToken).ConfigureAwait(false); } + /// + public async IAsyncEnumerable GetStreamingUpdatesAsync( + string input, + AIRequestSettings? requestSettings = null, + [EnumeratorCancellation] CancellationToken cancellationToken = default) + { + var resultIndex = 0; + foreach (var result in await this.ExecuteGetCompletionsAsync(input, cancellationToken).ConfigureAwait(false)) + { + cancellationToken.ThrowIfCancellationRequested(); + + var completion = await result.GetCompletionAsync(cancellationToken).ConfigureAwait(false); + yield return new StreamingTextResultUpdate(completion, resultIndex); + + resultIndex++; + } + } + + /// + public async IAsyncEnumerable GetStringStreamingUpdatesAsync(string input, AIRequestSettings? requestSettings = null, [EnumeratorCancellation] CancellationToken cancellationToken = default) + { + await foreach (var update in this.GetStreamingUpdatesAsync(input, requestSettings, cancellationToken).ConfigureAwait(false)) + { + yield return update.ToString(); + } + } + + /// + public async IAsyncEnumerable GetByteStreamingUpdatesAsync(string input, AIRequestSettings? requestSettings = null, [EnumeratorCancellation] CancellationToken cancellationToken = default) + { + await foreach (var update in this.GetStreamingUpdatesAsync(input, requestSettings, cancellationToken).ConfigureAwait(false)) + { + yield return update.ToByteArray(); + } + } + #region private ================================================================================ private async Task> ExecuteGetCompletionsAsync(string text, CancellationToken cancellationToken = default) diff --git a/dotnet/src/Connectors/Connectors.AI.HuggingFace/TextCompletion/StreamingTextResultUpdate.cs b/dotnet/src/Connectors/Connectors.AI.HuggingFace/TextCompletion/StreamingTextResultUpdate.cs new file mode 100644 index 000000000000..45b53b648784 --- /dev/null +++ b/dotnet/src/Connectors/Connectors.AI.HuggingFace/TextCompletion/StreamingTextResultUpdate.cs @@ -0,0 +1,46 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Text; +using Microsoft.SemanticKernel.AI; + +namespace Microsoft.SemanticKernel.Connectors.AI.HuggingFace.TextCompletion; + +/// +/// Streaming text result update. +/// +public class StreamingTextResultUpdate : StreamingResultUpdate +{ + /// + public override string Type => "huggingface_text_update"; + + /// + public override int ResultIndex { get; } + + /// + /// Text associated to the update + /// + public string Content { get; } + + /// + /// Create a new instance of the class. + /// + /// Text update + /// Index of the choice + public StreamingTextResultUpdate(string text, int resultIndex) + { + this.ResultIndex = resultIndex; + this.Content = text; + } + + /// + public override byte[] ToByteArray() + { + return Encoding.UTF8.GetBytes(this.ToString()); + } + + /// + public override string ToString() + { + return this.Content; + } +} diff --git a/dotnet/src/Connectors/Connectors.AI.OpenAI/AzureSdk/StreamingChatResultUpdate.cs b/dotnet/src/Connectors/Connectors.AI.OpenAI/AzureSdk/StreamingChatResultUpdate.cs index 8aa13e7f73d1..b07452edaaa2 100644 --- a/dotnet/src/Connectors/Connectors.AI.OpenAI/AzureSdk/StreamingChatResultUpdate.cs +++ b/dotnet/src/Connectors/Connectors.AI.OpenAI/AzureSdk/StreamingChatResultUpdate.cs @@ -22,22 +22,22 @@ public class StreamingChatResultUpdate : StreamingResultUpdate /// /// Function call associated to the message payload /// - public FunctionCall FunctionCall { get; } + public FunctionCall? FunctionCall { get; } /// /// Text associated to the message payload /// - public string Content { get; } + public string? Content { get; } /// /// Role of the author of the message /// - public AuthorRole Role { get; } + public AuthorRole? Role { get; } /// /// Name of the author of the message. Name is required if the role is 'function'. /// - public string Name { get; } + public string? Name { get; } /// /// Create a new instance of the class. @@ -53,6 +53,20 @@ public StreamingChatResultUpdate(ChatMessage chatMessage, int resultIndex) this.Name = chatMessage.Name; } + /// + /// Create a new instance of the class. + /// + /// Original Azure SDK Message update representation + /// Index of the choice + public StreamingChatResultUpdate(SKChatMessage chatMessage, int resultIndex) + { + this.ResultIndex = resultIndex; + this.FunctionCall = chatMessage.FunctionCall; + this.Content = chatMessage.Content; + this.Role = new AuthorRole(chatMessage.Role.ToString()); + this.Name = chatMessage.FunctionCall?.Name; + } + /// public override byte[] ToByteArray() { diff --git a/dotnet/src/Connectors/Connectors.AI.OpenAI/AzureSdk/StreamingTextResultUpdate.cs b/dotnet/src/Connectors/Connectors.AI.OpenAI/AzureSdk/StreamingTextResultUpdate.cs index 37eebefaec51..d73fb3043e8d 100644 --- a/dotnet/src/Connectors/Connectors.AI.OpenAI/AzureSdk/StreamingTextResultUpdate.cs +++ b/dotnet/src/Connectors/Connectors.AI.OpenAI/AzureSdk/StreamingTextResultUpdate.cs @@ -22,7 +22,7 @@ public class StreamingTextResultUpdate : StreamingResultUpdate public string Content { get; } /// - /// Create a new instance of the class. + /// Create a new instance of the class. /// /// Text update /// Index of the choice diff --git a/dotnet/src/Connectors/Connectors.AI.OpenAI/ChatCompletion/OpenAIChatCompletion.cs b/dotnet/src/Connectors/Connectors.AI.OpenAI/ChatCompletion/OpenAIChatCompletion.cs index ce84c8f4b22f..b50e8cdecf27 100644 --- a/dotnet/src/Connectors/Connectors.AI.OpenAI/ChatCompletion/OpenAIChatCompletion.cs +++ b/dotnet/src/Connectors/Connectors.AI.OpenAI/ChatCompletion/OpenAIChatCompletion.cs @@ -2,6 +2,8 @@ using System.Collections.Generic; using System.Net.Http; +using System.Runtime.CompilerServices; +using System.Text.Json; using System.Threading; using System.Threading.Tasks; using Azure.AI.OpenAI; @@ -101,4 +103,29 @@ public Task> GetCompletionsAsync( this.LogActionDetails(); return this.InternalGetChatResultsAsTextAsync(text, requestSettings, cancellationToken); } + + /// + public IAsyncEnumerable GetStreamingUpdatesAsync(string input, AIRequestSettings? requestSettings = null, CancellationToken cancellationToken = default) + { + var chatHistory = JsonSerializer.Deserialize(input) ?? new(); + return this.InternalGetChatStreamingUpdatesAsync(chatHistory, requestSettings, cancellationToken); + } + + /// + public async IAsyncEnumerable GetStringStreamingUpdatesAsync(string input, AIRequestSettings? requestSettings = null, [EnumeratorCancellation] CancellationToken cancellationToken = default) + { + await foreach (var update in this.GetStreamingUpdatesAsync(input, requestSettings, cancellationToken).ConfigureAwait(false)) + { + yield return update.ToString(); + } + } + + /// + public async IAsyncEnumerable GetByteStreamingUpdatesAsync(string input, AIRequestSettings? requestSettings = null, [EnumeratorCancellation] CancellationToken cancellationToken = default) + { + await foreach (var update in this.GetStreamingUpdatesAsync(input, requestSettings, cancellationToken).ConfigureAwait(false)) + { + yield return update.ToByteArray(); + } + } } diff --git a/dotnet/src/Connectors/Connectors.AI.OpenAI/ChatCompletionWithData/AzureOpenAIChatCompletionWithData.cs b/dotnet/src/Connectors/Connectors.AI.OpenAI/ChatCompletionWithData/AzureOpenAIChatCompletionWithData.cs index 83d38df59595..c8740458e45a 100644 --- a/dotnet/src/Connectors/Connectors.AI.OpenAI/ChatCompletionWithData/AzureOpenAIChatCompletionWithData.cs +++ b/dotnet/src/Connectors/Connectors.AI.OpenAI/ChatCompletionWithData/AzureOpenAIChatCompletionWithData.cs @@ -14,6 +14,7 @@ using Microsoft.SemanticKernel.AI; using Microsoft.SemanticKernel.AI.ChatCompletion; using Microsoft.SemanticKernel.AI.TextCompletion; +using Microsoft.SemanticKernel.Connectors.AI.OpenAI.AzureSdk; using Microsoft.SemanticKernel.Connectors.AI.OpenAI.ChatCompletion; using Microsoft.SemanticKernel.Diagnostics; using Microsoft.SemanticKernel.Services; @@ -118,6 +119,42 @@ public async IAsyncEnumerable GetStreamingCompletionsAsync } } + /// + public async IAsyncEnumerable GetStreamingUpdatesAsync(string input, AIRequestSettings? requestSettings = null, [EnumeratorCancellation] CancellationToken cancellationToken = default) + { + OpenAIRequestSettings chatRequestSettings = OpenAIRequestSettings.FromRequestSettings(requestSettings); + + var chat = this.PrepareChatHistory(input, chatRequestSettings); + + var resultIndex = 0; + await foreach (var result in this.GetStreamingChatCompletionsAsync(chat, requestSettings, cancellationToken).ConfigureAwait(false)) + { + await foreach (var message in result.GetStreamingChatMessageAsync(cancellationToken).ConfigureAwait(false)) + { + yield return new StreamingChatResultUpdate((SKChatMessage)message, resultIndex); + } + resultIndex++; + } + } + + /// + public async IAsyncEnumerable GetStringStreamingUpdatesAsync(string input, AIRequestSettings? requestSettings = null, [EnumeratorCancellation] CancellationToken cancellationToken = default) + { + await foreach (var update in this.GetStreamingUpdatesAsync(input, requestSettings, cancellationToken).ConfigureAwait(false)) + { + yield return update.ToString(); + } + } + + /// + public async IAsyncEnumerable GetByteStreamingUpdatesAsync(string input, AIRequestSettings? requestSettings = null, [EnumeratorCancellation] CancellationToken cancellationToken = default) + { + await foreach (var update in this.GetStreamingUpdatesAsync(input, requestSettings, cancellationToken).ConfigureAwait(false)) + { + yield return update.ToByteArray(); + } + } + #region private ================================================================================ private const string DefaultApiVersion = "2023-06-01-preview"; diff --git a/dotnet/src/Connectors/Connectors.AI.OpenAI/ChatCompletionWithData/ChatWithDataStreamingResult.cs b/dotnet/src/Connectors/Connectors.AI.OpenAI/ChatCompletionWithData/ChatWithDataStreamingResult.cs index 92ec4e463544..6d5f3ba1b31f 100644 --- a/dotnet/src/Connectors/Connectors.AI.OpenAI/ChatCompletionWithData/ChatWithDataStreamingResult.cs +++ b/dotnet/src/Connectors/Connectors.AI.OpenAI/ChatCompletionWithData/ChatWithDataStreamingResult.cs @@ -37,7 +37,7 @@ public async Task GetChatMessageAsync(CancellationToken cancell var result = new SKChatMessage(AuthorRole.Assistant.Label, message?.Delta?.Content ?? string.Empty); - return await Task.FromResult(result).ConfigureAwait(false); + return await Task.FromResult(result).ConfigureAwait(false); } public async IAsyncEnumerable GetStreamingChatMessageAsync([EnumeratorCancellation] CancellationToken cancellationToken = default) diff --git a/dotnet/src/Connectors/Connectors.AI.OpenAI/TextCompletion/AzureTextCompletion.cs b/dotnet/src/Connectors/Connectors.AI.OpenAI/TextCompletion/AzureTextCompletion.cs index 99e1faded63d..18334d5bd8e2 100644 --- a/dotnet/src/Connectors/Connectors.AI.OpenAI/TextCompletion/AzureTextCompletion.cs +++ b/dotnet/src/Connectors/Connectors.AI.OpenAI/TextCompletion/AzureTextCompletion.cs @@ -2,6 +2,7 @@ using System.Collections.Generic; using System.Net.Http; +using System.Runtime.CompilerServices; using System.Threading; using System.Threading.Tasks; using Azure.AI.OpenAI; @@ -20,6 +21,9 @@ namespace Microsoft.SemanticKernel.Connectors.AI.OpenAI.TextCompletion; /// public sealed class AzureTextCompletion : AzureOpenAIClientBase, ITextCompletion { + /// + public IReadOnlyDictionary Attributes => this.InternalAttributes; + /// /// Creates a new AzureTextCompletion client instance using API Key auth /// @@ -76,9 +80,6 @@ public AzureTextCompletion( this.AddAttribute(IAIServiceExtensions.ModelIdKey, modelId); } - /// - public IReadOnlyDictionary Attributes => this.InternalAttributes; - /// public IAsyncEnumerable GetStreamingCompletionsAsync( string text, @@ -98,4 +99,28 @@ public Task> GetCompletionsAsync( this.LogActionDetails(); return this.InternalGetTextResultsAsync(text, requestSettings, cancellationToken); } + + /// + public IAsyncEnumerable GetStreamingUpdatesAsync(string input, AIRequestSettings? requestSettings = null, CancellationToken cancellationToken = default) + { + return this.InternalGetTextStreamingUpdatesAsync(input, requestSettings, cancellationToken); + } + + /// + public async IAsyncEnumerable GetStringStreamingUpdatesAsync(string input, AIRequestSettings? requestSettings = null, [EnumeratorCancellation] CancellationToken cancellationToken = default) + { + await foreach (var update in this.GetStreamingUpdatesAsync(input, requestSettings, cancellationToken).ConfigureAwait(false)) + { + yield return update.ToString(); + } + } + + /// + public async IAsyncEnumerable GetByteStreamingUpdatesAsync(string input, AIRequestSettings? requestSettings = null, [EnumeratorCancellation] CancellationToken cancellationToken = default) + { + await foreach (var update in this.GetStreamingUpdatesAsync(input, requestSettings, cancellationToken).ConfigureAwait(false)) + { + yield return update.ToByteArray(); + } + } } diff --git a/dotnet/src/Connectors/Connectors.AI.OpenAI/TextCompletion/OpenAITextCompletion.cs b/dotnet/src/Connectors/Connectors.AI.OpenAI/TextCompletion/OpenAITextCompletion.cs index a49f3128b958..dc66f2026c64 100644 --- a/dotnet/src/Connectors/Connectors.AI.OpenAI/TextCompletion/OpenAITextCompletion.cs +++ b/dotnet/src/Connectors/Connectors.AI.OpenAI/TextCompletion/OpenAITextCompletion.cs @@ -2,6 +2,7 @@ using System.Collections.Generic; using System.Net.Http; +using System.Runtime.CompilerServices; using System.Threading; using System.Threading.Tasks; using Microsoft.Extensions.Logging; @@ -60,4 +61,28 @@ public Task> GetCompletionsAsync( this.LogActionDetails(); return this.InternalGetTextResultsAsync(text, requestSettings, cancellationToken); } + + /// + public IAsyncEnumerable GetStreamingUpdatesAsync(string input, AIRequestSettings? requestSettings = null, CancellationToken cancellationToken = default) + { + return this.InternalGetTextStreamingUpdatesAsync(input, requestSettings, cancellationToken); + } + + /// + public async IAsyncEnumerable GetStringStreamingUpdatesAsync(string input, AIRequestSettings? requestSettings = null, [EnumeratorCancellation] CancellationToken cancellationToken = default) + { + await foreach (var update in this.GetStreamingUpdatesAsync(input, requestSettings, cancellationToken).ConfigureAwait(false)) + { + yield return update.ToString(); + } + } + + /// + public async IAsyncEnumerable GetByteStreamingUpdatesAsync(string input, AIRequestSettings? requestSettings = null, [EnumeratorCancellation] CancellationToken cancellationToken = default) + { + await foreach (var update in this.GetStreamingUpdatesAsync(input, requestSettings, cancellationToken).ConfigureAwait(false)) + { + yield return update.ToByteArray(); + } + } } diff --git a/dotnet/src/IntegrationTests/Extensions/KernelSemanticFunctionExtensionsTests.cs b/dotnet/src/IntegrationTests/Extensions/KernelSemanticFunctionExtensionsTests.cs index 50c4b2568fdc..a37a4b2f12c8 100644 --- a/dotnet/src/IntegrationTests/Extensions/KernelSemanticFunctionExtensionsTests.cs +++ b/dotnet/src/IntegrationTests/Extensions/KernelSemanticFunctionExtensionsTests.cs @@ -82,6 +82,20 @@ IAsyncEnumerable ITextCompletion.GetStreamingCompletionsAs { throw new NotImplementedException(); // TODO } + public IAsyncEnumerable GetByteStreamingUpdatesAsync(string input, AIRequestSettings? requestSettings = null, CancellationToken cancellationToken = default) + { + throw new NotImplementedException(); + } + + public IAsyncEnumerable GetStreamingUpdatesAsync(string input, AIRequestSettings? requestSettings = null, CancellationToken cancellationToken = default) + { + throw new NotImplementedException(); + } + + public IAsyncEnumerable GetStringStreamingUpdatesAsync(string input, AIRequestSettings? requestSettings = null, CancellationToken cancellationToken = default) + { + throw new NotImplementedException(); + } } internal sealed class RedirectTextCompletionResult : ITextResult diff --git a/dotnet/src/SemanticKernel.Abstractions/AI/TextCompletion/ITextCompletion.cs b/dotnet/src/SemanticKernel.Abstractions/AI/TextCompletion/ITextCompletion.cs index 6de89e76e8d1..5dd3e0fb12db 100644 --- a/dotnet/src/SemanticKernel.Abstractions/AI/TextCompletion/ITextCompletion.cs +++ b/dotnet/src/SemanticKernel.Abstractions/AI/TextCompletion/ITextCompletion.cs @@ -39,36 +39,36 @@ IAsyncEnumerable GetStreamingCompletionsAsync( /// /// Get streaming completion results for the prompt and settings. /// - /// The prompt to complete. + /// The prompt to complete. /// Request settings for the completion API /// The to monitor for cancellation requests. The default is . /// Streaming list of different completion streaming result updates generated by the remote model IAsyncEnumerable GetStreamingUpdatesAsync( - string text, + string input, AIRequestSettings? requestSettings = null, CancellationToken cancellationToken = default); /// /// Get streaming results for the prompt and settings. /// - /// The prompt to complete. + /// The prompt to complete. /// Request settings for the completion API /// The to monitor for cancellation requests. The default is . /// Streaming list of different completion streaming string updates generated by the remote model IAsyncEnumerable GetStringStreamingUpdatesAsync( - string text, + string input, AIRequestSettings? requestSettings = null, CancellationToken cancellationToken = default); /// /// Get streaming results for the prompt and settings. /// - /// The prompt to complete. + /// The prompt to complete. /// Request settings for the completion API /// The to monitor for cancellation requests. The default is . /// Streaming list of different completion streaming byte array updates generated by the remote model IAsyncEnumerable GetByteStreamingUpdatesAsync( - string text, + string input, AIRequestSettings? requestSettings = null, CancellationToken cancellationToken = default); } diff --git a/dotnet/src/SemanticKernel.Core/Orchestration/FunctionRunner.cs b/dotnet/src/SemanticKernel.Core/Orchestration/FunctionRunner.cs index f1bfe50a10f2..86d03955d2ec 100644 --- a/dotnet/src/SemanticKernel.Core/Orchestration/FunctionRunner.cs +++ b/dotnet/src/SemanticKernel.Core/Orchestration/FunctionRunner.cs @@ -1,8 +1,10 @@ // Copyright (c) Microsoft. All rights reserved. +using System.Collections.Generic; using System.Linq; using System.Threading; using System.Threading.Tasks; +using Microsoft.SemanticKernel.AI; namespace Microsoft.SemanticKernel.Orchestration; @@ -35,4 +37,14 @@ public FunctionRunner(IKernel kernel) var function = this._kernel.Functions.GetFunction(pluginName, functionName); return this.RunAsync(function, variables, cancellationToken); } + + public IAsyncEnumerable StreamingRunAsync(ISKFunction skFunction, ContextVariables? variables = null, CancellationToken cancellationToken = default) + { + return this._kernel.StreamingRunAsync(skFunction, variables, cancellationToken); + } + + public IAsyncEnumerable StreamingRunAsync(string pluginName, string functionName, ContextVariables? variables = null, CancellationToken cancellationToken = default) + { + throw new System.NotImplementedException(); + } } diff --git a/dotnet/src/SemanticKernel.UnitTests/Functions/OrderedIAIServiceConfigurationProviderTests.cs b/dotnet/src/SemanticKernel.UnitTests/Functions/OrderedIAIServiceConfigurationProviderTests.cs index a18adcf5cb37..9845a2561d80 100644 --- a/dotnet/src/SemanticKernel.UnitTests/Functions/OrderedIAIServiceConfigurationProviderTests.cs +++ b/dotnet/src/SemanticKernel.UnitTests/Functions/OrderedIAIServiceConfigurationProviderTests.cs @@ -209,6 +209,11 @@ private sealed class TextCompletion : ITextCompletion public string? ModelId { get; } + public IAsyncEnumerable GetByteStreamingUpdatesAsync(string input, AIRequestSettings? requestSettings = null, CancellationToken cancellationToken = default) + { + throw new NotImplementedException(); + } + public Task> GetCompletionsAsync(string text, AIRequestSettings? requestSettings = null, CancellationToken cancellationToken = default) { throw new NotImplementedException(); @@ -218,6 +223,16 @@ public IAsyncEnumerable GetStreamingCompletionsAsync(strin { throw new NotImplementedException(); } + + public IAsyncEnumerable GetStreamingUpdatesAsync(string input, AIRequestSettings? requestSettings = null, CancellationToken cancellationToken = default) + { + throw new NotImplementedException(); + } + + public IAsyncEnumerable GetStringStreamingUpdatesAsync(string input, AIRequestSettings? requestSettings = null, CancellationToken cancellationToken = default) + { + throw new NotImplementedException(); + } } #endregion } From 00934cb1508b649da4fa663d660f2f1ec65fbe0d Mon Sep 17 00:00:00 2001 From: Roger Barreto <19890735+RogerBarreto@users.noreply.github.com> Date: Tue, 14 Nov 2023 19:32:10 +0000 Subject: [PATCH 11/46] Added streaming examples + Fix on Streaming native functions --- .../Example65_KernelStreaming..cs | 251 ++++++++++++++++++ .../AzureOpenAIChatCompletion.cs | 2 +- .../ChatCompletion/OpenAIChatCompletion.cs | 2 +- .../Functions/NativeFunction.cs | 24 +- .../Functions/StreamingNativeResultUpdate.cs | 14 +- 5 files changed, 277 insertions(+), 16 deletions(-) create mode 100644 dotnet/samples/KernelSyntaxExamples/Example65_KernelStreaming..cs diff --git a/dotnet/samples/KernelSyntaxExamples/Example65_KernelStreaming..cs b/dotnet/samples/KernelSyntaxExamples/Example65_KernelStreaming..cs new file mode 100644 index 000000000000..e6629e9b38f5 --- /dev/null +++ b/dotnet/samples/KernelSyntaxExamples/Example65_KernelStreaming..cs @@ -0,0 +1,251 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; +using System.ComponentModel; +using System.Globalization; +using System.Text; +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.SemanticKernel; +using Microsoft.SemanticKernel.Connectors.AI.OpenAI; +using Microsoft.SemanticKernel.Connectors.AI.OpenAI.AzureSdk; +using Microsoft.SemanticKernel.Orchestration; +using RepoUtils; + +#pragma warning disable RCS1110 // Declare type inside namespace. +#pragma warning disable CA1819 // Properties should not return arrays + +/** + * This example shows how to use multiple prompt template formats. + */ +// ReSharper disable once InconsistentNaming +public static class Example65_KernelStreaming +{ + /// + /// Show how to combine multiple prompt template factories. + /// + public static async Task RunAsync(CancellationToken cancellationToken) + { + Console.WriteLine("======== Example65_KernelStreaming ========"); + + string apiKey = TestConfiguration.AzureOpenAI.ApiKey; + string chatDeploymentName = TestConfiguration.AzureOpenAI.ChatDeploymentName; + string endpoint = TestConfiguration.AzureOpenAI.Endpoint; + + if (apiKey == null || chatDeploymentName == null || endpoint == null) + { + Console.WriteLine("Azure endpoint, apiKey, or deploymentName not found. Skipping example."); + return; + } + + IKernel kernel = new KernelBuilder() + .WithLoggerFactory(ConsoleLogger.LoggerFactory) + .WithAzureOpenAIChatCompletionService( + deploymentName: chatDeploymentName, + endpoint: endpoint, + serviceId: "AzureOpenAIChat", + apiKey: apiKey) + .Build(); + + // Function defined using few-shot design pattern + string promptTemplate = @" +Generate a creative reason or excuse for the given event. +Be creative and be funny. Let your imagination run wild. + +Event: I am running late. +Excuse: I was being held ransom by giraffe gangsters. + +Event: I haven't been to the gym for a year +Excuse: I've been too busy training my pet dragon. + +Event: {{$input}} +"; + + var excuseFunction = kernel.CreateSemanticFunction(promptTemplate, new OpenAIRequestSettings() { MaxTokens = 100, Temperature = 0.4, TopP = 1 }); + + var roleDisplayed = false; + + Console.WriteLine("=== Semantic Function - Streaming ===\n"); + + // Streaming can be of any type depending on the underlying service the function is using. + await foreach (var update in kernel.StreamingRunAsync(excuseFunction, new ContextVariables("I missed the F1 final race"), cancellationToken)) + { + // You will be aways able to know the type of the update by checking the Type property. + if (update.Type == "openai_chat_message_update" && update is StreamingChatResultUpdate chatUpdate) + { + if (!roleDisplayed && chatUpdate.Role.HasValue) + { + Console.WriteLine($"Role: {chatUpdate.Role}"); + roleDisplayed = true; + } + + if (chatUpdate.Content is { Length: > 0 }) + { + Console.Write(chatUpdate.Content); + } + } + }; + + var functions = kernel.ImportFunctions(new MyNativePlugin(), "MyNativePlugin"); + + await NativeStreamingValueTypeAsync(kernel, functions, cancellationToken); + + await NativeStreamingComplexTypeAsync(kernel, functions, cancellationToken); + + await NativeValueTypeAsync(kernel, functions, cancellationToken); + + await NativeComplexTypeAsync(kernel, functions, cancellationToken); + } + + private static async Task NativeStreamingValueTypeAsync(IKernel kernel, IDictionary functions, CancellationToken cancellationToken) + { + Console.WriteLine("\n=== Native Streaming Functions - Streaming (Value Type) ===\n"); + + // Native string value type streaming function + await foreach (var update in kernel.StreamingRunAsync(functions["MyValueTypeStreamingNativeFunction"], new ContextVariables("My Value Type Streaming Function Input"), cancellationToken)) + { + if (update.Type == "native_result_update" && update is StreamingNativeResultUpdate nativeUpdate) + { + Console.Write(nativeUpdate.Value); + } + } + } + + private static async Task NativeStreamingComplexTypeAsync(IKernel kernel, IDictionary functions, CancellationToken cancellationToken) + { + Console.WriteLine("\n=== Native Streaming Functions - Streaming (Complex Type) ===\n"); + + // Native complex type streaming function + await foreach (var update in kernel.StreamingRunAsync(functions["MyComplexTypeStreamingNativeFunction"], new ContextVariables("My Complex Type Streaming Function Input"), cancellationToken)) + { + // the complex type will be available thru the Value property of the native update abstraction. + if (update.Type == "native_result_update" && update is StreamingNativeResultUpdate nativeUpdate && nativeUpdate.Value is MyStreamingBlock myComplexType) + { + Console.WriteLine(Encoding.UTF8.GetString(myComplexType.Content)); + } + } + } + + private static async Task NativeValueTypeAsync(IKernel kernel, IDictionary functions, CancellationToken cancellationToken) + { + Console.WriteLine("\n=== Native Non-Streaming Functions - Streaming (Value Type) ===\n"); + + // Native functions that don't support streaming and return value types can be executed with the streaming API's but the behavior will not be streamlike. + await foreach (var update in kernel.StreamingRunAsync(functions["MyValueTypeNativeFunction"], new ContextVariables("My Value Type Non Streaming Function Input"), cancellationToken)) + { + // the complex type will be available thru the Value property of the native update abstraction. + if (update.Type == "native_result_update" && update is StreamingNativeResultUpdate nativeUpdate) + { + Console.WriteLine(nativeUpdate.Value); + } + } + } + + private static async Task NativeComplexTypeAsync(IKernel kernel, IDictionary functions, CancellationToken cancellationToken) + { + Console.WriteLine("\n=== Native Non-Streaming Functions - Streaming (Complex Type) ===\n"); + + // Native functions that don't support streaming and return complex types can be executed with the streaming API's but the behavior will not be streamlike. + await foreach (var update in kernel.StreamingRunAsync(functions["MyComplexTypeNativeFunction"], new ContextVariables("My Complex Type Non Streaming Function Input"), cancellationToken)) + { + // the complex type will be available thru the Value property of the native update abstraction. + if (update.Type == "native_result_update" && update is StreamingNativeResultUpdate nativeUpdate && nativeUpdate.Value is MyCustomType myComplexType) + { + Console.WriteLine($"Text: {myComplexType.Text}, Number: {myComplexType.Number}"); + } + } + } + + private class MyNativePlugin + { + [SKFunction] + public async IAsyncEnumerable MyValueTypeStreamingNativeFunctionAsync(string input) + { + var result = "My streaming example, should load word by word in 0.5sec".Split(' '); + foreach (var item in result) + { + yield return $"{item} "; + await Task.Delay(500); + } + } + + [SKFunction] + public async IAsyncEnumerable MyComplexTypeStreamingNativeFunctionAsync(string input) + { + // Base64 encoded string "Base64VideoPacket1 Base64VideoPacket2 Base64VideoPacket3" + var result = $"QmFzZTY0VmlkZW9QYWNrZXQx QmFzZTY0VmlkZW9QYWNrZXQy QmFzZTY0VmlkZW9QYWNrZXQz".Split(' '); + + foreach (var item in result) + { + yield return new MyStreamingBlock(Convert.FromBase64String(item)); + + await Task.Delay(500); + } + } + + [SKFunction] + public async Task MyValueTypeNativeFunctionAsync(string input) + { + await Task.Delay(1000); + return input; + } + + [SKFunction] + public MyCustomType MyComplexTypeNativeFunction(string input) + { + Thread.Sleep(1000); + return new MyCustomType() { Number = 1, Text = "My Specific Text" }; + } + } + + private record MyStreamingBlock(byte[] Content); + + /// + /// In order to use custom types, should be specified, + /// that will convert object instance to string representation. + /// + /// + /// is used to represent complex object as meaningful string, so + /// it can be passed to AI for further processing using semantic functions. + /// It's possible to choose any format (e.g. XML, JSON, YAML) to represent your object. + /// + [TypeConverter(typeof(MyCustomTypeConverter))] + private sealed class MyCustomType + { + public int Number { get; set; } + + public string? Text { get; set; } + } + + /// + /// Implementation of for . + /// In this example, object instance is serialized with from System.Text.Json, + /// but it's possible to convert object to string using any other serialization logic. + /// +#pragma warning disable CA1812 // instantiated by Kernel + private sealed class MyCustomTypeConverter : TypeConverter +#pragma warning restore CA1812 + { + public override bool CanConvertFrom(ITypeDescriptorContext? context, Type sourceType) => true; + + /// + /// This method is used to convert object from string to actual type. This will allow to pass object to + /// native function which requires it. + /// + public override object? ConvertFrom(ITypeDescriptorContext? context, CultureInfo? culture, object value) + { + return JsonSerializer.Deserialize((string)value); + } + + /// + /// This method is used to convert actual type to string representation, so it can be passed to AI + /// for further processing. + /// + public override object? ConvertTo(ITypeDescriptorContext? context, CultureInfo? culture, object? value, Type destinationType) + { + return JsonSerializer.Serialize(value); + } + } +} diff --git a/dotnet/src/Connectors/Connectors.AI.OpenAI/ChatCompletion/AzureOpenAIChatCompletion.cs b/dotnet/src/Connectors/Connectors.AI.OpenAI/ChatCompletion/AzureOpenAIChatCompletion.cs index 5cb33b4ade8d..b05db17ae1dc 100644 --- a/dotnet/src/Connectors/Connectors.AI.OpenAI/ChatCompletion/AzureOpenAIChatCompletion.cs +++ b/dotnet/src/Connectors/Connectors.AI.OpenAI/ChatCompletion/AzureOpenAIChatCompletion.cs @@ -131,7 +131,7 @@ public Task> GetCompletionsAsync( /// public IAsyncEnumerable GetStreamingUpdatesAsync(string input, AIRequestSettings? requestSettings = null, CancellationToken cancellationToken = default) { - var chatHistory = JsonSerializer.Deserialize(input) ?? new(); + var chatHistory = this.CreateNewChat(input); return this.InternalGetChatStreamingUpdatesAsync(chatHistory, requestSettings, cancellationToken); } diff --git a/dotnet/src/Connectors/Connectors.AI.OpenAI/ChatCompletion/OpenAIChatCompletion.cs b/dotnet/src/Connectors/Connectors.AI.OpenAI/ChatCompletion/OpenAIChatCompletion.cs index b50e8cdecf27..8c24b5d6528f 100644 --- a/dotnet/src/Connectors/Connectors.AI.OpenAI/ChatCompletion/OpenAIChatCompletion.cs +++ b/dotnet/src/Connectors/Connectors.AI.OpenAI/ChatCompletion/OpenAIChatCompletion.cs @@ -107,7 +107,7 @@ public Task> GetCompletionsAsync( /// public IAsyncEnumerable GetStreamingUpdatesAsync(string input, AIRequestSettings? requestSettings = null, CancellationToken cancellationToken = default) { - var chatHistory = JsonSerializer.Deserialize(input) ?? new(); + var chatHistory = this.CreateNewChat(input); return this.InternalGetChatStreamingUpdatesAsync(chatHistory, requestSettings, cancellationToken); } diff --git a/dotnet/src/SemanticKernel.Core/Functions/NativeFunction.cs b/dotnet/src/SemanticKernel.Core/Functions/NativeFunction.cs index 1a1a28c9b07a..495fc7983cec 100644 --- a/dotnet/src/SemanticKernel.Core/Functions/NativeFunction.cs +++ b/dotnet/src/SemanticKernel.Core/Functions/NativeFunction.cs @@ -365,7 +365,7 @@ async IAsyncEnumerable StreamingFunction( if (IsAsyncEnumerable(method, out var enumeratedTypes)) { // Invoke the method to get the IAsyncEnumerable instance - object asyncEnumerable = method.Invoke(target, null); + object asyncEnumerable = method.Invoke(target, args); if (asyncEnumerable == null) { yield break; @@ -376,11 +376,15 @@ async IAsyncEnumerable StreamingFunction( .MakeGenericType(enumeratedTypes) .GetMethod("GetAsyncEnumerator"); - object asyncEnumerator = getAsyncEnumeratorMethod.Invoke(asyncEnumerable, null); + // Get the IAsyncEnumerator type + Type asyncEnumeratorType = typeof(IAsyncEnumerator<>).MakeGenericType(enumeratedTypes); + + // Invoke GetAsyncEnumerator() to get the IAsyncEnumerator instance + object asyncEnumerator = getAsyncEnumeratorMethod.Invoke(asyncEnumerable, new object[] { cancellationToken }); // Get the MoveNextAsync() and Current properties - MethodInfo moveNextAsyncMethod = asyncEnumerator.GetType().GetMethod("MoveNextAsync"); - PropertyInfo currentProperty = asyncEnumerator.GetType().GetProperty("Current"); + MethodInfo moveNextAsyncMethod = asyncEnumeratorType.GetMethod("MoveNextAsync"); + PropertyInfo currentProperty = asyncEnumeratorType.GetProperty("Current"); // Iterate over the items while (await ((ValueTask)moveNextAsyncMethod.Invoke(asyncEnumerator, null)).ConfigureAwait(false)) @@ -392,12 +396,18 @@ async IAsyncEnumerable StreamingFunction( } else { - // Invoke the method. + // When streaming is requested for a non-streaming native method, it returns just one result object? result = method.Invoke(target, args); if (result is not null) { - yield return new StreamingNativeResultUpdate(result); + var functionResult = await returnFunc(functionName!, pluginName, result, context).ConfigureAwait(false); + + // The enumeration will only return if there's actually a result. + if (functionResult.Value is not null) + { + yield return new StreamingNativeResultUpdate(functionResult.Value); + } } } } @@ -427,7 +437,7 @@ private static bool IsAsyncMethod(MethodInfo method) if (t.IsGenericType) { t = t.GetGenericTypeDefinition(); - if (t == typeof(Task<>) || t == typeof(ValueTask<>)) + if (t == typeof(Task<>) || t == typeof(ValueTask<>) || t == typeof(IAsyncEnumerable<>)) { return true; } diff --git a/dotnet/src/SemanticKernel.Core/Functions/StreamingNativeResultUpdate.cs b/dotnet/src/SemanticKernel.Core/Functions/StreamingNativeResultUpdate.cs index ae9d64aeb89d..87d0af5c9587 100644 --- a/dotnet/src/SemanticKernel.Core/Functions/StreamingNativeResultUpdate.cs +++ b/dotnet/src/SemanticKernel.Core/Functions/StreamingNativeResultUpdate.cs @@ -20,25 +20,25 @@ public sealed class StreamingNativeResultUpdate : StreamingResultUpdate public override int ResultIndex => 0; /// - /// Native object that represents the update + /// Native object value that represents the update /// - public object Update { get; } + public object Value { get; } /// public override byte[] ToByteArray() { - if (this.Update is byte[]) + if (this.Value is byte[]) { - return (byte[])this.Update; + return (byte[])this.Value; } - return Encoding.UTF8.GetBytes(this.Update?.ToString()); + return Encoding.UTF8.GetBytes(this.Value?.ToString()); } /// public override string ToString() { - return this.Update.ToString(); + return this.Value.ToString(); } /// @@ -47,6 +47,6 @@ public override string ToString() /// Underlying object that represents the update public StreamingNativeResultUpdate(object update) { - this.Update = update; + this.Value = update; } } From a1f12e97c776d5157b591fe969e8999cba9cc889 Mon Sep 17 00:00:00 2001 From: Roger Barreto <19890735+RogerBarreto@users.noreply.github.com> Date: Tue, 14 Nov 2023 19:47:27 +0000 Subject: [PATCH 12/46] Warning fix --- .../KernelSyntaxExamples/Example65_KernelStreaming..cs | 6 +++--- .../ChatCompletion/AzureOpenAIChatCompletion.cs | 1 - .../ChatCompletion/OpenAIChatCompletion.cs | 1 - 3 files changed, 3 insertions(+), 5 deletions(-) diff --git a/dotnet/samples/KernelSyntaxExamples/Example65_KernelStreaming..cs b/dotnet/samples/KernelSyntaxExamples/Example65_KernelStreaming..cs index e6629e9b38f5..b19c4538f84b 100644 --- a/dotnet/samples/KernelSyntaxExamples/Example65_KernelStreaming..cs +++ b/dotnet/samples/KernelSyntaxExamples/Example65_KernelStreaming..cs @@ -158,7 +158,7 @@ private static async Task NativeComplexTypeAsync(IKernel kernel, IDictionary MyValueTypeStreamingNativeFunctionAsync(string input) @@ -175,7 +175,7 @@ public async IAsyncEnumerable MyValueTypeStreamingNativeFunctionAsync(st public async IAsyncEnumerable MyComplexTypeStreamingNativeFunctionAsync(string input) { // Base64 encoded string "Base64VideoPacket1 Base64VideoPacket2 Base64VideoPacket3" - var result = $"QmFzZTY0VmlkZW9QYWNrZXQx QmFzZTY0VmlkZW9QYWNrZXQy QmFzZTY0VmlkZW9QYWNrZXQz".Split(' '); + var result = "QmFzZTY0VmlkZW9QYWNrZXQx QmFzZTY0VmlkZW9QYWNrZXQy QmFzZTY0VmlkZW9QYWNrZXQz".Split(' '); foreach (var item in result) { @@ -200,7 +200,7 @@ public MyCustomType MyComplexTypeNativeFunction(string input) } } - private record MyStreamingBlock(byte[] Content); + private sealed record MyStreamingBlock(byte[] Content); /// /// In order to use custom types, should be specified, diff --git a/dotnet/src/Connectors/Connectors.AI.OpenAI/ChatCompletion/AzureOpenAIChatCompletion.cs b/dotnet/src/Connectors/Connectors.AI.OpenAI/ChatCompletion/AzureOpenAIChatCompletion.cs index b05db17ae1dc..66f165744cb7 100644 --- a/dotnet/src/Connectors/Connectors.AI.OpenAI/ChatCompletion/AzureOpenAIChatCompletion.cs +++ b/dotnet/src/Connectors/Connectors.AI.OpenAI/ChatCompletion/AzureOpenAIChatCompletion.cs @@ -3,7 +3,6 @@ using System.Collections.Generic; using System.Net.Http; using System.Runtime.CompilerServices; -using System.Text.Json; using System.Threading; using System.Threading.Tasks; using Azure.AI.OpenAI; diff --git a/dotnet/src/Connectors/Connectors.AI.OpenAI/ChatCompletion/OpenAIChatCompletion.cs b/dotnet/src/Connectors/Connectors.AI.OpenAI/ChatCompletion/OpenAIChatCompletion.cs index 8c24b5d6528f..b8b47b6b68d1 100644 --- a/dotnet/src/Connectors/Connectors.AI.OpenAI/ChatCompletion/OpenAIChatCompletion.cs +++ b/dotnet/src/Connectors/Connectors.AI.OpenAI/ChatCompletion/OpenAIChatCompletion.cs @@ -3,7 +3,6 @@ using System.Collections.Generic; using System.Net.Http; using System.Runtime.CompilerServices; -using System.Text.Json; using System.Threading; using System.Threading.Tasks; using Azure.AI.OpenAI; From 9938c3b7b39d300b5e2fde405d73b1356ad89679 Mon Sep 17 00:00:00 2001 From: Roger Barreto <19890735+RogerBarreto@users.noreply.github.com> Date: Tue, 14 Nov 2023 19:55:13 +0000 Subject: [PATCH 13/46] Removing double dots in filename --- ...Example65_KernelStreaming..cs => Example65_KernelStreaming.cs} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename dotnet/samples/KernelSyntaxExamples/{Example65_KernelStreaming..cs => Example65_KernelStreaming.cs} (100%) diff --git a/dotnet/samples/KernelSyntaxExamples/Example65_KernelStreaming..cs b/dotnet/samples/KernelSyntaxExamples/Example65_KernelStreaming.cs similarity index 100% rename from dotnet/samples/KernelSyntaxExamples/Example65_KernelStreaming..cs rename to dotnet/samples/KernelSyntaxExamples/Example65_KernelStreaming.cs From 9c650c1ec888c6516eabd54ae823c6966fac7395 Mon Sep 17 00:00:00 2001 From: Roger Barreto <19890735+RogerBarreto@users.noreply.github.com> Date: Tue, 14 Nov 2023 19:57:44 +0000 Subject: [PATCH 14/46] Typo fix --- .../samples/KernelSyntaxExamples/Example65_KernelStreaming.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dotnet/samples/KernelSyntaxExamples/Example65_KernelStreaming.cs b/dotnet/samples/KernelSyntaxExamples/Example65_KernelStreaming.cs index 94901ce17eec..ea6d1e1f040a 100644 --- a/dotnet/samples/KernelSyntaxExamples/Example65_KernelStreaming.cs +++ b/dotnet/samples/KernelSyntaxExamples/Example65_KernelStreaming.cs @@ -70,7 +70,7 @@ Be creative and be funny. Let your imagination run wild. // Streaming can be of any type depending on the underlying service the function is using. await foreach (var update in kernel.StreamingRunAsync(excuseFunction, new ContextVariables("I missed the F1 final race"), cancellationToken)) { - // You will be aways able to know the type of the update by checking the Type property. + // You will be always able to know the type of the update by checking the Type property. if (update.Type == "openai_chat_message_update" && update is StreamingChatResultUpdate chatUpdate) { if (!roleDisplayed && chatUpdate.Role.HasValue) From 699938f1debeff85dc010299adfd6c557127ca95 Mon Sep 17 00:00:00 2001 From: Roger Barreto <19890735+RogerBarreto@users.noreply.github.com> Date: Tue, 14 Nov 2023 21:53:37 +0000 Subject: [PATCH 15/46] Small update to ADR --- docs/decisions/0023-kernel-streaming.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/decisions/0023-kernel-streaming.md b/docs/decisions/0023-kernel-streaming.md index 163769850c4c..cad92738d2f3 100644 --- a/docs/decisions/0023-kernel-streaming.md +++ b/docs/decisions/0023-kernel-streaming.md @@ -31,15 +31,15 @@ Needs to be clear for the sk developer when he is attempting to get streaming da //(providing the type at as generic parameter) // Getting a Raw Streaming data from Kernel -await foreach(string update in kernel.StreamingRunAsync(variables, function)) +await foreach(string update in kernel.StreamingRunAsync(function, variables)) // Getting a String as Streaming data from Kernel -await foreach(string update in kernel.StreamingRunAsync(variables, function)) +await foreach(string update in kernel.StreamingRunAsync(function, variables)) // Getting a StreamingResultUpdate as Streaming data from Kernel await foreach(StreamingResultUpdate update in kernel.StreamingRunAsync(variables, function)) // OR -await foreach(StreamingResultUpdate update in kernel.StreamingRunAsync(variables, function)) // defaults to Generic above) +await foreach(StreamingResultUpdate update in kernel.StreamingRunAsync(function, variables)) // defaults to Generic above) { Console.WriteLine(update); } From c2c67127d039cde849abe298edeafb1570effead Mon Sep 17 00:00:00 2001 From: Roger Barreto <19890735+RogerBarreto@users.noreply.github.com> Date: Thu, 16 Nov 2023 10:17:59 +0000 Subject: [PATCH 16/46] Wip --- docs/decisions/0023-kernel-streaming.md | 86 ++++++++++++++++--------- 1 file changed, 57 insertions(+), 29 deletions(-) diff --git a/docs/decisions/0023-kernel-streaming.md b/docs/decisions/0023-kernel-streaming.md index cad92738d2f3..c41428d81d51 100644 --- a/docs/decisions/0023-kernel-streaming.md +++ b/docs/decisions/0023-kernel-streaming.md @@ -36,10 +36,10 @@ await foreach(string update in kernel.StreamingRunAsync(function, variab // Getting a String as Streaming data from Kernel await foreach(string update in kernel.StreamingRunAsync(function, variables)) -// Getting a StreamingResultUpdate as Streaming data from Kernel -await foreach(StreamingResultUpdate update in kernel.StreamingRunAsync(variables, function)) +// Getting a StreamingResultChunk as Streaming data from Kernel +await foreach(StreamingResultChunk update in kernel.StreamingRunAsync(variables, function)) // OR -await foreach(StreamingResultUpdate update in kernel.StreamingRunAsync(function, variables)) // defaults to Generic above) +await foreach(StreamingResultChunk update in kernel.StreamingRunAsync(function, variables)) // defaults to Generic above) { Console.WriteLine(update); } @@ -49,6 +49,7 @@ await foreach(StreamingResultUpdate update in kernel.StreamingRunAsync(function, - Streaming with plans will not be supported in this phase. Attempting to do so will throw an exception. - Kernel streaming will not support multiple functions (pipeline). +- Input streaming will not be supported in this phase. ## Considered Options @@ -58,40 +59,43 @@ Using dedicated streaming interfaces that allow the sk developer to get the stre This approach also exposes dedicated interfaces in the kernel and functions to use streaming making it clear to the sk developer what is the type of data being returned in IAsyncEnumerable format. -`ITextCompletion` and `IChatCompletion` will have new APIs to get `byte[]` and `string` streaming data directly as well as the specialized `StreamingResultUpdate` return. +`ITextCompletion` and `IChatCompletion` will have new APIs to get `byte[]` and `string` streaming data directly as well as the specialized `StreamingResultChunk` return. -The sk developer will be able to specify a generic type to the `Kernel.StreamingRunAsync()` and `ISKFunction.StreamingInvokeAsync` to get the streaming data. If the type is not specified, the Kernel and Functions will return the data as StreamingResultUpdate. +The sk developer will be able to specify a generic type to the `Kernel.StreamingRunAsync()` and `ISKFunction.StreamingInvokeAsync` to get the streaming data. If the type is not specified, the Kernel and Functions will return the data as StreamingResultChunk. If the type is not specified or if the string representation cannot be cast, an exception will be thrown. -If the type specified is `StreamingResultUpdate`, `string` or `byte[]` no error will be thrown as the connectors will have interface methods that guarantee the data to be given in at least those types. +If the type specified is `StreamingResultChunk`, `string` or `byte[]` no error will be thrown as the connectors will have interface methods that guarantee the data to be given in at least those types. ```csharp // Depending on the underlying function model and connector used the streaming -// data will be of a specialization of StreamingResultUpdate exposing useful +// data will be of a specialization of StreamingResultChunk exposing useful // properties including Type, the raw data in byte[] and string representation. -abstract class StreamingResultUpdate +abstract class StreamingResultChunk { public abstract string Type { get; } + // In a scenario of multiple results, this represents zero-based index of the result in the streaming sequence public abstract int ResultIndex { get; } public abstract override string ToString(); public abstract byte[] ToByteArray(); + + public abstract object Contents { get; } } -// Specialization example of a ChatMessageUpdate -public class ChatMessageUpdate : StreamingResultUpdate +// Specialization example of a ChatMessageChunk +public class ChatMessageChunk : StreamingResultChunk { - public override string Type => "ChatMessage"; - public override string ToString() => JsonSerializer.Serialize(this); + public override string Type => "openai_chatmessage_chunk"; + public override string ToString() => Value.ToString(); // Value to be appended or the whole object? public override byte[] ToByteArray() => Encoding.UTF8.GetBytes(Value); public string Message { get; } public string Role { get; } - public ChatMessageUpdate(string message, string role, int resultIndex = 0) + public ChatMessageChunk(string message, string role, int resultIndex = 0) { Message = message; Role = role; @@ -99,36 +103,40 @@ public class ChatMessageUpdate : StreamingResultUpdate } } -interface IChatCompletion +interface ITextCompletion + IChatCompletion { - IAsyncEnumerable GetStreamingUpdatesAsync(); - IAsyncEnumerable GetStringStreamingUpdatesAsync(); - IAsyncEnumerable GetByteStreamingUpdatesAsync(); -} + IAsyncEnumerable GetStreamingUpdatesAsync(); + //IAsyncEnumerable GetStringStreamingUpdatesAsync(); + //IAsyncEnumerable GetByteStreamingUpdatesAsync(); -interface ITextCompletion -{ - IAsyncEnumerable GetStreamingUpdatesAsync(); - IAsyncEnumerable GetStringStreamingUpdatesAsync(); - IAsyncEnumerable GetByteStreamingUpdatesAsync(); + // OR + IAsyncEnumerable GetStreamingUpdatesAsync() where T : StreamingResultChunk or string or byte[]; + // Throw exception if T is not supported } interface IKernel { // When the developer provides a T, the Kernel will try to get the streaming data as T - IAsyncEnumerable StreamingRunAsync(ContextVariables variables, ISKFunction function); + IAsyncEnumerable StreamingRunAsync(ContextVariables variables, ISKFunction function); + + // Extension generic method to get from type + IAsyncEnumerable RunStreamingAsync(ContextVariables variables, ISKFunction function); } + interface ISKFunction { // When the developer provides a T, the Kernel will try to get the streaming data as T - IAsyncEnumerable StreamingInvokeAsync(SKContext context); + IAsyncEnumerable InvokeStreamingAsync(SKContext context); + + // Extension generic method to get from type + IAsyncEnumerable InvokeStreamingAsync(SKContext context); } ``` ## Semantic Functions Behavior -When Semantic Functions are invoked using the Streaming API, they will attempt to use the Connectors streaming implementation. The connector will be responsible to provide the specialized type of `StreamingResultUpdate` as well as to keep the API streaming working (streaming the single complete result) even when the backend don't support it. +When Semantic Functions are invoked using the Streaming API, they will attempt to use the Connectors streaming implementation. The connector will be responsible to provide the specialized type of `StreamingResultChunk` as well as to keep the API streaming working (streaming the single complete result) even when the backend don't support it. ## Native Functions Behavior @@ -143,15 +151,35 @@ If NativeFunctions don't return IAsyncEnumerable, the result will be wrapped in 1. All the User Experience Goal section options will be possible. 2. Kernel and Functions implementations will be able to stream data of any type, not limited to text 3. The sk developer will be able to provide the type it expects from the `GetStreamingResultAsync` method. -4. The above will allow the sk developer to get the streaming data in a generic way, including `string`, `byte array`, or the `StreamingResultUpdate` abstraction directly from the connector. +4. The above will allow the sk developer to get the streaming data in a generic way, including `string`, `byte array`, or the `StreamingResultChunk` abstraction directly from the connector. 5. IChatCompletion, IKernel and ISKFunction ## Cons -1. If the sk developer wants to use the specialized type of `StreamingResultUpdate` he will need to know what the connector is being used to use the correct **StreamingResultUpdate extension method** or to provide directly type in ``. -2. Connectors will have greater responsibility to provide the correct type of `StreamingResultUpdate` for the connector being used and implementations for both byte[] and string streaming data. +1. If the sk developer wants to use the specialized type of `StreamingResultChunk` he will need to know what the connector is being used to use the correct **StreamingResultChunk extension method** or to provide directly type in ``. +2. Connectors will have greater responsibility to provide the correct type of `StreamingResultChunk` for the connector being used and implementations for both byte[] and string streaming data. ## Decision Outcome As for now, the only proposed solution is the #1. + +## More Information + +Think about generic for the interfaces... + +StreamingFunctionResult ... + +Look into Hugging Face Streaming APIs + +Create a second option after talkin with Stephen. + +Run by Matthew +StreamingAsync vs StreamingRunAsync vs RunStreamingAsync vs StreamAsync +First thing should be a verb + +Add more examples of the API usage for streaming with chunks, for complex scenarios like multiple modals. + +KDifferent impleme tations + differtnt consumers + +I should be able to handle stuff that i don't know. From 752e4d17cfa76e060f6ac765866b300fe5e36980 Mon Sep 17 00:00:00 2001 From: Roger Barreto <19890735+RogerBarreto@users.noreply.github.com> Date: Thu, 16 Nov 2023 22:35:13 +0000 Subject: [PATCH 17/46] WIP chaging Connectors interfaces to support --- docs/decisions/0023-kernel-streaming.md | 27 +++--- .../Example16_CustomLLM.cs | 10 +- .../Example65_KernelStreaming.cs | 2 +- .../HuggingFaceTextCompletion.cs | 6 +- .../StreamingTextResultUpdate.cs | 2 +- .../AzureSdk/ClientBase.cs | 56 +++++++++++- ...tUpdate.cs => StreamingChatResultChunk.cs} | 14 +-- ...tUpdate.cs => StreamingTextResultChunk.cs} | 6 +- .../AzureOpenAIChatCompletion.cs | 6 +- .../ChatCompletion/OpenAIChatCompletion.cs | 6 +- .../AzureOpenAIChatCompletionWithData.cs | 8 +- .../TextCompletion/AzureTextCompletion.cs | 6 +- .../TextCompletion/OpenAITextCompletion.cs | 12 ++- .../KernelSemanticFunctionExtensionsTests.cs | 2 +- .../AI/ChatCompletion/IChatCompletion.cs | 2 +- ...esultUpdate.cs => StreamingResultChunk.cs} | 2 +- .../AI/TextCompletion/ITextCompletion.cs | 24 ++--- .../Functions/ISKFunction.cs | 2 +- .../SemanticKernel.Abstractions/IKernel.cs | 2 +- .../Orchestration/IFunctionRunner.cs | 4 +- .../Orchestration/StreamingFunctionResult.cs | 91 +++++++++++++++++++ .../Functions/InstrumentedSKFunction.cs | 4 +- .../Functions/NativeFunction.cs | 2 +- .../Functions/SemanticFunction.cs | 6 +- .../Functions/StreamingNativeResultUpdate.cs | 2 +- dotnet/src/SemanticKernel.Core/Kernel.cs | 4 +- .../Orchestration/FunctionRunner.cs | 4 +- .../Planning/InstrumentedPlan.cs | 2 +- .../src/SemanticKernel.Core/Planning/Plan.cs | 2 +- ...redIAIServiceConfigurationProviderTests.cs | 11 +-- 30 files changed, 231 insertions(+), 96 deletions(-) rename dotnet/src/Connectors/Connectors.AI.OpenAI/AzureSdk/{StreamingChatResultUpdate.cs => StreamingChatResultChunk.cs} (84%) rename dotnet/src/Connectors/Connectors.AI.OpenAI/AzureSdk/{StreamingTextResultUpdate.cs => StreamingTextResultChunk.cs} (87%) rename dotnet/src/SemanticKernel.Abstractions/AI/{StreamingResultUpdate.cs => StreamingResultChunk.cs} (96%) create mode 100644 dotnet/src/SemanticKernel.Abstractions/Orchestration/StreamingFunctionResult.cs diff --git a/docs/decisions/0023-kernel-streaming.md b/docs/decisions/0023-kernel-streaming.md index c41428d81d51..c48dcf16e383 100644 --- a/docs/decisions/0023-kernel-streaming.md +++ b/docs/decisions/0023-kernel-streaming.md @@ -31,15 +31,15 @@ Needs to be clear for the sk developer when he is attempting to get streaming da //(providing the type at as generic parameter) // Getting a Raw Streaming data from Kernel -await foreach(string update in kernel.StreamingRunAsync(function, variables)) +await foreach(string update in kernel.RunStreamingAsync(function, variables)) // Getting a String as Streaming data from Kernel -await foreach(string update in kernel.StreamingRunAsync(function, variables)) +await foreach(string update in kernel.RunStreamingAsync(function, variables)) // Getting a StreamingResultChunk as Streaming data from Kernel -await foreach(StreamingResultChunk update in kernel.StreamingRunAsync(variables, function)) +await foreach(StreamingResultChunk update in kernel.RunStreamingAsync(variables, function)) // OR -await foreach(StreamingResultChunk update in kernel.StreamingRunAsync(function, variables)) // defaults to Generic above) +await foreach(StreamingResultChunk update in kernel.RunStreamingAsync(function, variables)) // defaults to Generic above) { Console.WriteLine(update); } @@ -61,7 +61,7 @@ This approach also exposes dedicated interfaces in the kernel and functions to u `ITextCompletion` and `IChatCompletion` will have new APIs to get `byte[]` and `string` streaming data directly as well as the specialized `StreamingResultChunk` return. -The sk developer will be able to specify a generic type to the `Kernel.StreamingRunAsync()` and `ISKFunction.StreamingInvokeAsync` to get the streaming data. If the type is not specified, the Kernel and Functions will return the data as StreamingResultChunk. +The sk developer will be able to specify a generic type to the `Kernel.RunStreamingAsync()` and `ISKFunction.StreamingInvokeAsync` to get the streaming data. If the type is not specified, the Kernel and Functions will return the data as StreamingResultChunk. If the type is not specified or if the string representation cannot be cast, an exception will be thrown. @@ -105,24 +105,27 @@ public class ChatMessageChunk : StreamingResultChunk interface ITextCompletion + IChatCompletion { - IAsyncEnumerable GetStreamingUpdatesAsync(); - //IAsyncEnumerable GetStringStreamingUpdatesAsync(); - //IAsyncEnumerable GetByteStreamingUpdatesAsync(); + IAsyncEnumerable GetStreamingChunksAsync(); + // Garanteed abstraction to be used by the ISKFunction.RunStreamingAsync() - // OR - IAsyncEnumerable GetStreamingUpdatesAsync() where T : StreamingResultChunk or string or byte[]; + IAsyncEnumerable GetStreamingChunksAsync(); // Throw exception if T is not supported + // Initially connectors } interface IKernel { // When the developer provides a T, the Kernel will try to get the streaming data as T - IAsyncEnumerable StreamingRunAsync(ContextVariables variables, ISKFunction function); + IAsyncEnumerable RunStreamingAsync(ContextVariables variables, ISKFunction function); // Extension generic method to get from type IAsyncEnumerable RunStreamingAsync(ContextVariables variables, ISKFunction function); } +public class StreamingFunctionResult : IAsyncEnumerable +{ + +} interface ISKFunction { @@ -175,7 +178,7 @@ Look into Hugging Face Streaming APIs Create a second option after talkin with Stephen. Run by Matthew -StreamingAsync vs StreamingRunAsync vs RunStreamingAsync vs StreamAsync +StreamingAsync vs RunStreamingAsync vs RunStreamingAsync vs StreamAsync First thing should be a verb Add more examples of the API usage for streaming with chunks, for complex scenarios like multiple modals. diff --git a/dotnet/samples/KernelSyntaxExamples/Example16_CustomLLM.cs b/dotnet/samples/KernelSyntaxExamples/Example16_CustomLLM.cs index 6eb4fed7074b..99f63ac4314e 100644 --- a/dotnet/samples/KernelSyntaxExamples/Example16_CustomLLM.cs +++ b/dotnet/samples/KernelSyntaxExamples/Example16_CustomLLM.cs @@ -52,15 +52,15 @@ public async IAsyncEnumerable GetStreamingCompletionsAsync public async IAsyncEnumerable GetByteStreamingUpdatesAsync(string input, AIRequestSettings? requestSettings = null, [EnumeratorCancellation] CancellationToken cancellationToken = default) { - await foreach (var update in this.GetStreamingUpdatesAsync(input, requestSettings, cancellationToken)) + await foreach (var update in this.GetStreamingChunksAsync(input, requestSettings, cancellationToken)) { yield return update.ToByteArray(); } } - public IAsyncEnumerable GetStreamingUpdatesAsync(string input, AIRequestSettings? requestSettings = null, CancellationToken cancellationToken = default) + public IAsyncEnumerable GetStreamingChunksAsync(string input, AIRequestSettings? requestSettings = null, CancellationToken cancellationToken = default) { - var list = new List() + var list = new List() { new MyStreamingResultUpdate("llm content update 1"), new MyStreamingResultUpdate("llm content update 2") @@ -71,14 +71,14 @@ public IAsyncEnumerable GetStreamingUpdatesAsync(string i public async IAsyncEnumerable GetStringStreamingUpdatesAsync(string input, AIRequestSettings? requestSettings = null, [EnumeratorCancellation] CancellationToken cancellationToken = default) { - await foreach (var update in this.GetStreamingUpdatesAsync(input, requestSettings, cancellationToken)) + await foreach (var update in this.GetStreamingChunksAsync(input, requestSettings, cancellationToken)) { yield return update.ToString(); } } } -public class MyStreamingResultUpdate : StreamingResultUpdate +public class MyStreamingResultUpdate : StreamingResultChunk { public override string Type => "my_text_type"; diff --git a/dotnet/samples/KernelSyntaxExamples/Example65_KernelStreaming.cs b/dotnet/samples/KernelSyntaxExamples/Example65_KernelStreaming.cs index ea6d1e1f040a..ab585e86ba63 100644 --- a/dotnet/samples/KernelSyntaxExamples/Example65_KernelStreaming.cs +++ b/dotnet/samples/KernelSyntaxExamples/Example65_KernelStreaming.cs @@ -71,7 +71,7 @@ Be creative and be funny. Let your imagination run wild. await foreach (var update in kernel.StreamingRunAsync(excuseFunction, new ContextVariables("I missed the F1 final race"), cancellationToken)) { // You will be always able to know the type of the update by checking the Type property. - if (update.Type == "openai_chat_message_update" && update is StreamingChatResultUpdate chatUpdate) + if (update.Type == "openai_chat_message_update" && update is StreamingChatResultChunk chatUpdate) { if (!roleDisplayed && chatUpdate.Role.HasValue) { diff --git a/dotnet/src/Connectors/Connectors.AI.HuggingFace/TextCompletion/HuggingFaceTextCompletion.cs b/dotnet/src/Connectors/Connectors.AI.HuggingFace/TextCompletion/HuggingFaceTextCompletion.cs index cb23fe868c5d..fec84d847814 100644 --- a/dotnet/src/Connectors/Connectors.AI.HuggingFace/TextCompletion/HuggingFaceTextCompletion.cs +++ b/dotnet/src/Connectors/Connectors.AI.HuggingFace/TextCompletion/HuggingFaceTextCompletion.cs @@ -92,7 +92,7 @@ public async Task> GetCompletionsAsync( } /// - public async IAsyncEnumerable GetStreamingUpdatesAsync( + public async IAsyncEnumerable GetStreamingChunksAsync( string input, AIRequestSettings? requestSettings = null, [EnumeratorCancellation] CancellationToken cancellationToken = default) @@ -112,7 +112,7 @@ public async IAsyncEnumerable GetStreamingUpdatesAsync( /// public async IAsyncEnumerable GetStringStreamingUpdatesAsync(string input, AIRequestSettings? requestSettings = null, [EnumeratorCancellation] CancellationToken cancellationToken = default) { - await foreach (var update in this.GetStreamingUpdatesAsync(input, requestSettings, cancellationToken).ConfigureAwait(false)) + await foreach (var update in this.GetStreamingChunksAsync(input, requestSettings, cancellationToken).ConfigureAwait(false)) { yield return update.ToString(); } @@ -121,7 +121,7 @@ public async IAsyncEnumerable GetStringStreamingUpdatesAsync(string inpu /// public async IAsyncEnumerable GetByteStreamingUpdatesAsync(string input, AIRequestSettings? requestSettings = null, [EnumeratorCancellation] CancellationToken cancellationToken = default) { - await foreach (var update in this.GetStreamingUpdatesAsync(input, requestSettings, cancellationToken).ConfigureAwait(false)) + await foreach (var update in this.GetStreamingChunksAsync(input, requestSettings, cancellationToken).ConfigureAwait(false)) { yield return update.ToByteArray(); } diff --git a/dotnet/src/Connectors/Connectors.AI.HuggingFace/TextCompletion/StreamingTextResultUpdate.cs b/dotnet/src/Connectors/Connectors.AI.HuggingFace/TextCompletion/StreamingTextResultUpdate.cs index 45b53b648784..7797fcf9b9c3 100644 --- a/dotnet/src/Connectors/Connectors.AI.HuggingFace/TextCompletion/StreamingTextResultUpdate.cs +++ b/dotnet/src/Connectors/Connectors.AI.HuggingFace/TextCompletion/StreamingTextResultUpdate.cs @@ -8,7 +8,7 @@ namespace Microsoft.SemanticKernel.Connectors.AI.HuggingFace.TextCompletion; /// /// Streaming text result update. /// -public class StreamingTextResultUpdate : StreamingResultUpdate +public class StreamingTextResultUpdate : StreamingResultChunk { /// public override string Type => "huggingface_text_update"; diff --git a/dotnet/src/Connectors/Connectors.AI.OpenAI/AzureSdk/ClientBase.cs b/dotnet/src/Connectors/Connectors.AI.OpenAI/AzureSdk/ClientBase.cs index 374b54a2a644..9a514386dd34 100644 --- a/dotnet/src/Connectors/Connectors.AI.OpenAI/AzureSdk/ClientBase.cs +++ b/dotnet/src/Connectors/Connectors.AI.OpenAI/AzureSdk/ClientBase.cs @@ -17,6 +17,7 @@ using Microsoft.SemanticKernel.Connectors.AI.OpenAI.ChatCompletion; using Microsoft.SemanticKernel.Diagnostics; using Microsoft.SemanticKernel.Prompt; +using ChatMessage = Azure.AI.OpenAI.ChatMessage; namespace Microsoft.SemanticKernel.Connectors.AI.OpenAI.AzureSdk; @@ -151,9 +152,17 @@ private protected async IAsyncEnumerable InternalGetTextStr } } - private protected async IAsyncEnumerable InternalGetTextStreamingUpdatesAsync( + private protected IAsyncEnumerable InternalGetTextStreamingUpdatesAsync( string prompt, AIRequestSettings? requestSettings, + CancellationToken cancellationToken = default) + { + return this.InternalGetTextStreamingUpdatesAsync(prompt, requestSettings, cancellationToken); + } + + private protected async IAsyncEnumerable InternalGetTextStreamingUpdatesAsync( + string prompt, + AIRequestSettings? requestSettings, [EnumeratorCancellation] CancellationToken cancellationToken = default) { OpenAIRequestSettings textRequestSettings = OpenAIRequestSettings.FromRequestSettings(requestSettings, OpenAIRequestSettings.DefaultTextMaxTokens); @@ -172,7 +181,21 @@ private protected async IAsyncEnumerable InternalGetT { await foreach (string textUpdate in choice.GetTextStreaming(cancellationToken).ConfigureAwait(false)) { - yield return new StreamingTextResultUpdate(textUpdate, choiceIndex); + if (typeof(T) == typeof(string)) + { + yield return (T)(object)textUpdate; + continue; + } + + // If the provided T is an specialized class of StreamingResultChunk interface works + if (typeof(T) == typeof(StreamingTextResultChunk) || + typeof(T) == typeof(StreamingResultChunk)) + { + yield return (T)(object)new StreamingTextResultChunk(textUpdate, choiceIndex); + continue; + } + + throw new NotSupportedException($"Type {typeof(T)} is not supported"); } choiceIndex++; } @@ -287,8 +310,8 @@ private protected async IAsyncEnumerable InternalGetChatSt } } - private protected async IAsyncEnumerable InternalGetChatStreamingUpdatesAsync( - IEnumerable chat, + private protected async IAsyncEnumerable InternalGetChatStreamingUpdatesAsync( + IEnumerable chat, AIRequestSettings? requestSettings, [EnumeratorCancellation] CancellationToken cancellationToken = default) { @@ -309,17 +332,40 @@ private protected async IAsyncEnumerable InternalGetC } using StreamingChatCompletions streamingChatCompletions = response.Value; + int choiceIndex = 0; await foreach (StreamingChatChoice choice in streamingChatCompletions.GetChoicesStreaming(cancellationToken).ConfigureAwait(false)) { await foreach (ChatMessage chatMessage in choice.GetMessageStreaming(cancellationToken).ConfigureAwait(false)) { - yield return new StreamingChatResultUpdate(chatMessage, choiceIndex); + if (typeof(T) == typeof(string)) + { + yield return (T)(object)chatMessage.Content; + continue; + } + + // If the provided T is an specialized class of StreamingResultChunk interface works + if (typeof(T) == typeof(StreamingChatResultChunk) || + typeof(T) == typeof(StreamingResultChunk)) + { + yield return (T)(object)new StreamingChatResultChunk(chatMessage, choiceIndex); + continue; + } + + throw new NotSupportedException($"Type {typeof(T)} is not supported"); } choiceIndex++; } } + private protected IAsyncEnumerable InternalGetChatStreamingUpdatesAsync( + IEnumerable chat, + AIRequestSettings? requestSettings, + CancellationToken cancellationToken = default) + { + return this.InternalGetChatStreamingUpdatesAsync(chat, requestSettings, cancellationToken); + } + /// /// Create a new empty chat instance /// diff --git a/dotnet/src/Connectors/Connectors.AI.OpenAI/AzureSdk/StreamingChatResultUpdate.cs b/dotnet/src/Connectors/Connectors.AI.OpenAI/AzureSdk/StreamingChatResultChunk.cs similarity index 84% rename from dotnet/src/Connectors/Connectors.AI.OpenAI/AzureSdk/StreamingChatResultUpdate.cs rename to dotnet/src/Connectors/Connectors.AI.OpenAI/AzureSdk/StreamingChatResultChunk.cs index b07452edaaa2..9dc5251e8dc1 100644 --- a/dotnet/src/Connectors/Connectors.AI.OpenAI/AzureSdk/StreamingChatResultUpdate.cs +++ b/dotnet/src/Connectors/Connectors.AI.OpenAI/AzureSdk/StreamingChatResultChunk.cs @@ -11,7 +11,7 @@ namespace Microsoft.SemanticKernel.Connectors.AI.OpenAI.AzureSdk; /// /// Streaming chat result update. /// -public class StreamingChatResultUpdate : StreamingResultUpdate +public class StreamingChatResultChunk : StreamingResultChunk { /// public override string Type => "openai_chat_message_update"; @@ -40,25 +40,25 @@ public class StreamingChatResultUpdate : StreamingResultUpdate public string? Name { get; } /// - /// Create a new instance of the class. + /// Create a new instance of the class. /// /// Original Azure SDK Message update representation /// Index of the choice - public StreamingChatResultUpdate(ChatMessage chatMessage, int resultIndex) + public StreamingChatResultChunk(AzureOpenAIChatMessage chatMessage, int resultIndex) { this.ResultIndex = resultIndex; - this.FunctionCall = chatMessage.FunctionCall; + this.FunctionCall = chatMessage.InnerChatMessage?.FunctionCall; this.Content = chatMessage.Content; this.Role = new AuthorRole(chatMessage.Role.ToString()); - this.Name = chatMessage.Name; + this.Name = chatMessage.InnerChatMessage?.Name; } /// - /// Create a new instance of the class. + /// Create a new instance of the class. /// /// Original Azure SDK Message update representation /// Index of the choice - public StreamingChatResultUpdate(SKChatMessage chatMessage, int resultIndex) + public StreamingChatResultChunk(Azure.AI.OpenAI.ChatMessage chatMessage, int resultIndex) { this.ResultIndex = resultIndex; this.FunctionCall = chatMessage.FunctionCall; diff --git a/dotnet/src/Connectors/Connectors.AI.OpenAI/AzureSdk/StreamingTextResultUpdate.cs b/dotnet/src/Connectors/Connectors.AI.OpenAI/AzureSdk/StreamingTextResultChunk.cs similarity index 87% rename from dotnet/src/Connectors/Connectors.AI.OpenAI/AzureSdk/StreamingTextResultUpdate.cs rename to dotnet/src/Connectors/Connectors.AI.OpenAI/AzureSdk/StreamingTextResultChunk.cs index d73fb3043e8d..d688f20c91f9 100644 --- a/dotnet/src/Connectors/Connectors.AI.OpenAI/AzureSdk/StreamingTextResultUpdate.cs +++ b/dotnet/src/Connectors/Connectors.AI.OpenAI/AzureSdk/StreamingTextResultChunk.cs @@ -8,7 +8,7 @@ namespace Microsoft.SemanticKernel.Connectors.AI.OpenAI.AzureSdk; /// /// Streaming text result update. /// -public class StreamingTextResultUpdate : StreamingResultUpdate +public class StreamingTextResultChunk : StreamingResultChunk { /// public override string Type => "openai_text_update"; @@ -22,11 +22,11 @@ public class StreamingTextResultUpdate : StreamingResultUpdate public string Content { get; } /// - /// Create a new instance of the class. + /// Create a new instance of the class. /// /// Text update /// Index of the choice - public StreamingTextResultUpdate(string text, int resultIndex) + public StreamingTextResultChunk(string text, int resultIndex) { this.ResultIndex = resultIndex; this.Content = text; diff --git a/dotnet/src/Connectors/Connectors.AI.OpenAI/ChatCompletion/AzureOpenAIChatCompletion.cs b/dotnet/src/Connectors/Connectors.AI.OpenAI/ChatCompletion/AzureOpenAIChatCompletion.cs index 66f165744cb7..189fec6d810b 100644 --- a/dotnet/src/Connectors/Connectors.AI.OpenAI/ChatCompletion/AzureOpenAIChatCompletion.cs +++ b/dotnet/src/Connectors/Connectors.AI.OpenAI/ChatCompletion/AzureOpenAIChatCompletion.cs @@ -128,7 +128,7 @@ public Task> GetCompletionsAsync( } /// - public IAsyncEnumerable GetStreamingUpdatesAsync(string input, AIRequestSettings? requestSettings = null, CancellationToken cancellationToken = default) + public IAsyncEnumerable GetStreamingChunksAsync(string input, AIRequestSettings? requestSettings = null, CancellationToken cancellationToken = default) { var chatHistory = this.CreateNewChat(input); return this.InternalGetChatStreamingUpdatesAsync(chatHistory, requestSettings, cancellationToken); @@ -137,7 +137,7 @@ public IAsyncEnumerable GetStreamingUpdatesAsync(string i /// public async IAsyncEnumerable GetStringStreamingUpdatesAsync(string input, AIRequestSettings? requestSettings = null, [EnumeratorCancellation] CancellationToken cancellationToken = default) { - await foreach (var update in this.GetStreamingUpdatesAsync(input, requestSettings, cancellationToken).ConfigureAwait(false)) + await foreach (var update in this.GetStreamingChunksAsync(input, requestSettings, cancellationToken).ConfigureAwait(false)) { yield return update.ToString(); } @@ -146,7 +146,7 @@ public async IAsyncEnumerable GetStringStreamingUpdatesAsync(string inpu /// public async IAsyncEnumerable GetByteStreamingUpdatesAsync(string input, AIRequestSettings? requestSettings = null, [EnumeratorCancellation] CancellationToken cancellationToken = default) { - await foreach (var update in this.GetStreamingUpdatesAsync(input, requestSettings, cancellationToken).ConfigureAwait(false)) + await foreach (var update in this.GetStreamingChunksAsync(input, requestSettings, cancellationToken).ConfigureAwait(false)) { yield return update.ToByteArray(); } diff --git a/dotnet/src/Connectors/Connectors.AI.OpenAI/ChatCompletion/OpenAIChatCompletion.cs b/dotnet/src/Connectors/Connectors.AI.OpenAI/ChatCompletion/OpenAIChatCompletion.cs index b8b47b6b68d1..f4df02d5d008 100644 --- a/dotnet/src/Connectors/Connectors.AI.OpenAI/ChatCompletion/OpenAIChatCompletion.cs +++ b/dotnet/src/Connectors/Connectors.AI.OpenAI/ChatCompletion/OpenAIChatCompletion.cs @@ -104,7 +104,7 @@ public Task> GetCompletionsAsync( } /// - public IAsyncEnumerable GetStreamingUpdatesAsync(string input, AIRequestSettings? requestSettings = null, CancellationToken cancellationToken = default) + public IAsyncEnumerable GetStreamingChunksAsync(string input, AIRequestSettings? requestSettings = null, CancellationToken cancellationToken = default) { var chatHistory = this.CreateNewChat(input); return this.InternalGetChatStreamingUpdatesAsync(chatHistory, requestSettings, cancellationToken); @@ -113,7 +113,7 @@ public IAsyncEnumerable GetStreamingUpdatesAsync(string i /// public async IAsyncEnumerable GetStringStreamingUpdatesAsync(string input, AIRequestSettings? requestSettings = null, [EnumeratorCancellation] CancellationToken cancellationToken = default) { - await foreach (var update in this.GetStreamingUpdatesAsync(input, requestSettings, cancellationToken).ConfigureAwait(false)) + await foreach (var update in this.GetStreamingChunksAsync(input, requestSettings, cancellationToken).ConfigureAwait(false)) { yield return update.ToString(); } @@ -122,7 +122,7 @@ public async IAsyncEnumerable GetStringStreamingUpdatesAsync(string inpu /// public async IAsyncEnumerable GetByteStreamingUpdatesAsync(string input, AIRequestSettings? requestSettings = null, [EnumeratorCancellation] CancellationToken cancellationToken = default) { - await foreach (var update in this.GetStreamingUpdatesAsync(input, requestSettings, cancellationToken).ConfigureAwait(false)) + await foreach (var update in this.GetStreamingChunksAsync(input, requestSettings, cancellationToken).ConfigureAwait(false)) { yield return update.ToByteArray(); } diff --git a/dotnet/src/Connectors/Connectors.AI.OpenAI/ChatCompletionWithData/AzureOpenAIChatCompletionWithData.cs b/dotnet/src/Connectors/Connectors.AI.OpenAI/ChatCompletionWithData/AzureOpenAIChatCompletionWithData.cs index 6979127ff183..64a63cbddbc7 100644 --- a/dotnet/src/Connectors/Connectors.AI.OpenAI/ChatCompletionWithData/AzureOpenAIChatCompletionWithData.cs +++ b/dotnet/src/Connectors/Connectors.AI.OpenAI/ChatCompletionWithData/AzureOpenAIChatCompletionWithData.cs @@ -119,7 +119,7 @@ public async IAsyncEnumerable GetStreamingCompletionsAsync } /// - public async IAsyncEnumerable GetStreamingUpdatesAsync(string input, AIRequestSettings? requestSettings = null, [EnumeratorCancellation] CancellationToken cancellationToken = default) + public async IAsyncEnumerable GetStreamingChunksAsync(string input, AIRequestSettings? requestSettings = null, [EnumeratorCancellation] CancellationToken cancellationToken = default) { OpenAIRequestSettings chatRequestSettings = OpenAIRequestSettings.FromRequestSettings(requestSettings); @@ -130,7 +130,7 @@ public async IAsyncEnumerable GetStreamingUpdatesAsync(st { await foreach (var message in result.GetStreamingChatMessageAsync(cancellationToken).ConfigureAwait(false)) { - yield return new StreamingChatResultUpdate((SKChatMessage)message, resultIndex); + yield return new StreamingChatResultChunk((AzureOpenAIChatMessage)message, resultIndex); } resultIndex++; } @@ -139,7 +139,7 @@ public async IAsyncEnumerable GetStreamingUpdatesAsync(st /// public async IAsyncEnumerable GetStringStreamingUpdatesAsync(string input, AIRequestSettings? requestSettings = null, [EnumeratorCancellation] CancellationToken cancellationToken = default) { - await foreach (var update in this.GetStreamingUpdatesAsync(input, requestSettings, cancellationToken).ConfigureAwait(false)) + await foreach (var update in this.GetStreamingChunksAsync(input, requestSettings, cancellationToken).ConfigureAwait(false)) { yield return update.ToString(); } @@ -148,7 +148,7 @@ public async IAsyncEnumerable GetStringStreamingUpdatesAsync(string inpu /// public async IAsyncEnumerable GetByteStreamingUpdatesAsync(string input, AIRequestSettings? requestSettings = null, [EnumeratorCancellation] CancellationToken cancellationToken = default) { - await foreach (var update in this.GetStreamingUpdatesAsync(input, requestSettings, cancellationToken).ConfigureAwait(false)) + await foreach (var update in this.GetStreamingChunksAsync(input, requestSettings, cancellationToken).ConfigureAwait(false)) { yield return update.ToByteArray(); } diff --git a/dotnet/src/Connectors/Connectors.AI.OpenAI/TextCompletion/AzureTextCompletion.cs b/dotnet/src/Connectors/Connectors.AI.OpenAI/TextCompletion/AzureTextCompletion.cs index 18334d5bd8e2..710e709eef8d 100644 --- a/dotnet/src/Connectors/Connectors.AI.OpenAI/TextCompletion/AzureTextCompletion.cs +++ b/dotnet/src/Connectors/Connectors.AI.OpenAI/TextCompletion/AzureTextCompletion.cs @@ -101,7 +101,7 @@ public Task> GetCompletionsAsync( } /// - public IAsyncEnumerable GetStreamingUpdatesAsync(string input, AIRequestSettings? requestSettings = null, CancellationToken cancellationToken = default) + public IAsyncEnumerable GetStreamingChunksAsync(string input, AIRequestSettings? requestSettings = null, CancellationToken cancellationToken = default) { return this.InternalGetTextStreamingUpdatesAsync(input, requestSettings, cancellationToken); } @@ -109,7 +109,7 @@ public IAsyncEnumerable GetStreamingUpdatesAsync(string i /// public async IAsyncEnumerable GetStringStreamingUpdatesAsync(string input, AIRequestSettings? requestSettings = null, [EnumeratorCancellation] CancellationToken cancellationToken = default) { - await foreach (var update in this.GetStreamingUpdatesAsync(input, requestSettings, cancellationToken).ConfigureAwait(false)) + await foreach (var update in this.GetStreamingChunksAsync(input, requestSettings, cancellationToken).ConfigureAwait(false)) { yield return update.ToString(); } @@ -118,7 +118,7 @@ public async IAsyncEnumerable GetStringStreamingUpdatesAsync(string inpu /// public async IAsyncEnumerable GetByteStreamingUpdatesAsync(string input, AIRequestSettings? requestSettings = null, [EnumeratorCancellation] CancellationToken cancellationToken = default) { - await foreach (var update in this.GetStreamingUpdatesAsync(input, requestSettings, cancellationToken).ConfigureAwait(false)) + await foreach (var update in this.GetStreamingChunksAsync(input, requestSettings, cancellationToken).ConfigureAwait(false)) { yield return update.ToByteArray(); } diff --git a/dotnet/src/Connectors/Connectors.AI.OpenAI/TextCompletion/OpenAITextCompletion.cs b/dotnet/src/Connectors/Connectors.AI.OpenAI/TextCompletion/OpenAITextCompletion.cs index dc66f2026c64..dedf8fa0b65a 100644 --- a/dotnet/src/Connectors/Connectors.AI.OpenAI/TextCompletion/OpenAITextCompletion.cs +++ b/dotnet/src/Connectors/Connectors.AI.OpenAI/TextCompletion/OpenAITextCompletion.cs @@ -63,7 +63,7 @@ public Task> GetCompletionsAsync( } /// - public IAsyncEnumerable GetStreamingUpdatesAsync(string input, AIRequestSettings? requestSettings = null, CancellationToken cancellationToken = default) + public IAsyncEnumerable GetStreamingChunksAsync(string input, AIRequestSettings? requestSettings = null, CancellationToken cancellationToken = default) { return this.InternalGetTextStreamingUpdatesAsync(input, requestSettings, cancellationToken); } @@ -71,7 +71,7 @@ public IAsyncEnumerable GetStreamingUpdatesAsync(string i /// public async IAsyncEnumerable GetStringStreamingUpdatesAsync(string input, AIRequestSettings? requestSettings = null, [EnumeratorCancellation] CancellationToken cancellationToken = default) { - await foreach (var update in this.GetStreamingUpdatesAsync(input, requestSettings, cancellationToken).ConfigureAwait(false)) + await foreach (var update in this.GetStreamingChunksAsync(input, requestSettings, cancellationToken).ConfigureAwait(false)) { yield return update.ToString(); } @@ -80,9 +80,15 @@ public async IAsyncEnumerable GetStringStreamingUpdatesAsync(string inpu /// public async IAsyncEnumerable GetByteStreamingUpdatesAsync(string input, AIRequestSettings? requestSettings = null, [EnumeratorCancellation] CancellationToken cancellationToken = default) { - await foreach (var update in this.GetStreamingUpdatesAsync(input, requestSettings, cancellationToken).ConfigureAwait(false)) + await foreach (var update in this.GetStreamingChunksAsync(input, requestSettings, cancellationToken).ConfigureAwait(false)) { yield return update.ToByteArray(); } } + + /// + public IAsyncEnumerable GetStreamingChunksAsync(string input, AIRequestSettings? requestSettings = null, CancellationToken cancellationToken = default) + { + return this.InternalGetTextStreamingUpdatesAsync(input, requestSettings, cancellationToken); + } } diff --git a/dotnet/src/IntegrationTests/Extensions/KernelSemanticFunctionExtensionsTests.cs b/dotnet/src/IntegrationTests/Extensions/KernelSemanticFunctionExtensionsTests.cs index a37a4b2f12c8..c4a40e65de9d 100644 --- a/dotnet/src/IntegrationTests/Extensions/KernelSemanticFunctionExtensionsTests.cs +++ b/dotnet/src/IntegrationTests/Extensions/KernelSemanticFunctionExtensionsTests.cs @@ -87,7 +87,7 @@ public IAsyncEnumerable GetByteStreamingUpdatesAsync(string input, AIReq throw new NotImplementedException(); } - public IAsyncEnumerable GetStreamingUpdatesAsync(string input, AIRequestSettings? requestSettings = null, CancellationToken cancellationToken = default) + public IAsyncEnumerable GetStreamingChunksAsync(string input, AIRequestSettings? requestSettings = null, CancellationToken cancellationToken = default) { throw new NotImplementedException(); } diff --git a/dotnet/src/SemanticKernel.Abstractions/AI/ChatCompletion/IChatCompletion.cs b/dotnet/src/SemanticKernel.Abstractions/AI/ChatCompletion/IChatCompletion.cs index 88bb835796f8..e523aa68c0cd 100644 --- a/dotnet/src/SemanticKernel.Abstractions/AI/ChatCompletion/IChatCompletion.cs +++ b/dotnet/src/SemanticKernel.Abstractions/AI/ChatCompletion/IChatCompletion.cs @@ -50,7 +50,7 @@ IAsyncEnumerable GetStreamingChatCompletionsAsync( /// Request settings for the completion API /// The to monitor for cancellation requests. The default is . /// Streaming list of different completion streaming result updates generated by the remote model - IAsyncEnumerable GetStreamingUpdatesAsync( + IAsyncEnumerable GetStreamingChunksAsync( string input, AIRequestSettings? requestSettings = null, CancellationToken cancellationToken = default); diff --git a/dotnet/src/SemanticKernel.Abstractions/AI/StreamingResultUpdate.cs b/dotnet/src/SemanticKernel.Abstractions/AI/StreamingResultChunk.cs similarity index 96% rename from dotnet/src/SemanticKernel.Abstractions/AI/StreamingResultUpdate.cs rename to dotnet/src/SemanticKernel.Abstractions/AI/StreamingResultChunk.cs index ce3010c47fae..6dd5305d30b3 100644 --- a/dotnet/src/SemanticKernel.Abstractions/AI/StreamingResultUpdate.cs +++ b/dotnet/src/SemanticKernel.Abstractions/AI/StreamingResultChunk.cs @@ -8,7 +8,7 @@ namespace Microsoft.SemanticKernel.AI; /// /// Represents a single update to a streaming result. /// -public abstract class StreamingResultUpdate +public abstract class StreamingResultChunk { /// /// Type of the update. diff --git a/dotnet/src/SemanticKernel.Abstractions/AI/TextCompletion/ITextCompletion.cs b/dotnet/src/SemanticKernel.Abstractions/AI/TextCompletion/ITextCompletion.cs index 5dd3e0fb12db..87c3dbaf0616 100644 --- a/dotnet/src/SemanticKernel.Abstractions/AI/TextCompletion/ITextCompletion.cs +++ b/dotnet/src/SemanticKernel.Abstractions/AI/TextCompletion/ITextCompletion.cs @@ -1,5 +1,6 @@ // Copyright (c) Microsoft. All rights reserved. +using System; using System.Collections.Generic; using System.Threading; using System.Threading.Tasks; @@ -43,31 +44,24 @@ IAsyncEnumerable GetStreamingCompletionsAsync( /// Request settings for the completion API /// The to monitor for cancellation requests. The default is . /// Streaming list of different completion streaming result updates generated by the remote model - IAsyncEnumerable GetStreamingUpdatesAsync( + IAsyncEnumerable GetStreamingChunksAsync( string input, AIRequestSettings? requestSettings = null, CancellationToken cancellationToken = default); /// - /// Get streaming results for the prompt and settings. + /// Get streaming results for the prompt and settings of a specific type. + /// Each modality may support for different types of streaming result. /// + /// + /// Usage of this method may be more efficient if the connector has a dedicated API to return this result without extra allocations for StreamingResultChunk abstraction. + /// + /// Throws if the specified type is not the same or fail to cast /// The prompt to complete. /// Request settings for the completion API /// The to monitor for cancellation requests. The default is . /// Streaming list of different completion streaming string updates generated by the remote model - IAsyncEnumerable GetStringStreamingUpdatesAsync( - string input, - AIRequestSettings? requestSettings = null, - CancellationToken cancellationToken = default); - - /// - /// Get streaming results for the prompt and settings. - /// - /// The prompt to complete. - /// Request settings for the completion API - /// The to monitor for cancellation requests. The default is . - /// Streaming list of different completion streaming byte array updates generated by the remote model - IAsyncEnumerable GetByteStreamingUpdatesAsync( + IAsyncEnumerable GetStreamingChunksAsync( string input, AIRequestSettings? requestSettings = null, CancellationToken cancellationToken = default); diff --git a/dotnet/src/SemanticKernel.Abstractions/Functions/ISKFunction.cs b/dotnet/src/SemanticKernel.Abstractions/Functions/ISKFunction.cs index 8023ee254e03..bd8bee0be74c 100644 --- a/dotnet/src/SemanticKernel.Abstractions/Functions/ISKFunction.cs +++ b/dotnet/src/SemanticKernel.Abstractions/Functions/ISKFunction.cs @@ -64,7 +64,7 @@ Task InvokeAsync( /// LLM completion settings (for semantic functions only) /// The updated context, potentially a new one if context switching is implemented. /// The to monitor for cancellation requests. The default is . - IAsyncEnumerable StreamingInvokeAsync( + IAsyncEnumerable StreamingInvokeAsync( SKContext context, AIRequestSettings? requestSettings = null, CancellationToken cancellationToken = default); diff --git a/dotnet/src/SemanticKernel.Abstractions/IKernel.cs b/dotnet/src/SemanticKernel.Abstractions/IKernel.cs index 5c849cf46800..b5930fa7fb7c 100644 --- a/dotnet/src/SemanticKernel.Abstractions/IKernel.cs +++ b/dotnet/src/SemanticKernel.Abstractions/IKernel.cs @@ -63,7 +63,7 @@ Task RunAsync( /// Input to process /// The to monitor for cancellation requests. /// Result of the function composition - public IAsyncEnumerable StreamingRunAsync(ISKFunction skFunction, ContextVariables? variables, CancellationToken cancellationToken); + public IAsyncEnumerable RunStreamingAsync(ISKFunction skFunction, ContextVariables? variables, CancellationToken cancellationToken); /// /// Create a new instance of a context, linked to the kernel internal state. diff --git a/dotnet/src/SemanticKernel.Abstractions/Orchestration/IFunctionRunner.cs b/dotnet/src/SemanticKernel.Abstractions/Orchestration/IFunctionRunner.cs index c1391d321e96..4a9e28760b5a 100644 --- a/dotnet/src/SemanticKernel.Abstractions/Orchestration/IFunctionRunner.cs +++ b/dotnet/src/SemanticKernel.Abstractions/Orchestration/IFunctionRunner.cs @@ -54,7 +54,7 @@ public interface IFunctionRunner /// Input to process /// The to monitor for cancellation requests. The default is . /// Result of the function composition - IAsyncEnumerable StreamingRunAsync( + IAsyncEnumerable StreamingRunAsync( ISKFunction skFunction, ContextVariables? variables = null, CancellationToken cancellationToken = default); @@ -70,7 +70,7 @@ IAsyncEnumerable StreamingRunAsync( /// Input to process /// The to monitor for cancellation requests. The default is . /// Result of the function composition - IAsyncEnumerable StreamingRunAsync( + IAsyncEnumerable StreamingRunAsync( string pluginName, string functionName, ContextVariables? variables = null, diff --git a/dotnet/src/SemanticKernel.Abstractions/Orchestration/StreamingFunctionResult.cs b/dotnet/src/SemanticKernel.Abstractions/Orchestration/StreamingFunctionResult.cs new file mode 100644 index 000000000000..b2fd02a46348 --- /dev/null +++ b/dotnet/src/SemanticKernel.Abstractions/Orchestration/StreamingFunctionResult.cs @@ -0,0 +1,91 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Collections.Generic; +using System.Threading; +using Microsoft.SemanticKernel.AI; + +namespace Microsoft.SemanticKernel.Orchestration; + +/// +/// Function result after execution. +/// +public sealed class StreamingFunctionResult : IAsyncEnumerable +{ + internal Dictionary? _metadata; + private readonly IAsyncEnumerable _streamingResult; + + /// + /// Name of executed function. + /// + public string FunctionName { get; internal set; } + + /// + /// Name of the plugin containing the function. + /// + public string PluginName { get; internal set; } + + /// + /// Metadata for storing additional information about function execution result. + /// + public Dictionary Metadata + { + get => this._metadata ??= new(); + internal set => this._metadata = value; + } + + /// + /// Internal object reference. (Breaking glass). + /// Each connector will have its own internal object representing the result. + /// + /// + /// The usage of this property is considered "unsafe". Use it only if strictly necessary. + /// + public object? InnerFunctionResult { get; private set; } = null; + + /// + /// Instance of used by the function. + /// + internal SKContext Context { get; private set; } + + /// + /// Initializes a new instance of the class. + /// + /// Name of executed function. + /// Name of the plugin containing the function. + /// Instance of to pass in function pipeline. + /// IAsyncEnumerable reference to iterate + /// Inner function result object reference + public StreamingFunctionResult(string functionName, string pluginName, SKContext context, IAsyncEnumerable streamingResult, object? innerFunctionResult) + { + this.FunctionName = functionName; + this.PluginName = pluginName; + this.Context = context; + this._streamingResult = streamingResult; + this.InnerFunctionResult = innerFunctionResult; + } + + /// + /// Get typed value from metadata. + /// + public bool TryGetMetadataValue(string key, out T value) + { + if (this._metadata is { } metadata && + metadata.TryGetValue(key, out object? valueObject) && + valueObject is T typedValue) + { + value = typedValue; + return true; + } + + value = default!; + return false; + } + + /// + /// Get the enumerator for the streaming result. + /// + /// Cancellation token + /// Streaming chunk asynchronous enumerator + public IAsyncEnumerator GetAsyncEnumerator(CancellationToken cancellationToken = default) + => this._streamingResult.GetAsyncEnumerator(cancellationToken); +} diff --git a/dotnet/src/SemanticKernel.Core/Functions/InstrumentedSKFunction.cs b/dotnet/src/SemanticKernel.Core/Functions/InstrumentedSKFunction.cs index 7ccc623283b9..b87ac0568653 100644 --- a/dotnet/src/SemanticKernel.Core/Functions/InstrumentedSKFunction.cs +++ b/dotnet/src/SemanticKernel.Core/Functions/InstrumentedSKFunction.cs @@ -80,7 +80,7 @@ public async Task InvokeAsync( } /// - public IAsyncEnumerable StreamingInvokeAsync( + public IAsyncEnumerable StreamingInvokeAsync( SKContext context, AIRequestSettings? requestSettings = null, CancellationToken cancellationToken = default) @@ -175,7 +175,7 @@ private async Task InvokeWithInstrumentationAsync(Func /// Delegate to instrument. - private async IAsyncEnumerable StreamingInvokeWithInstrumentationAsync(Func> func) + private async IAsyncEnumerable StreamingInvokeWithInstrumentationAsync(Func> func) { using var activity = s_activitySource.StartActivity($"{this.PluginName}.{this.Name}"); diff --git a/dotnet/src/SemanticKernel.Core/Functions/NativeFunction.cs b/dotnet/src/SemanticKernel.Core/Functions/NativeFunction.cs index 295fa6805436..736de672f3ac 100644 --- a/dotnet/src/SemanticKernel.Core/Functions/NativeFunction.cs +++ b/dotnet/src/SemanticKernel.Core/Functions/NativeFunction.cs @@ -154,7 +154,7 @@ public async Task InvokeAsync( } } - public async IAsyncEnumerable StreamingInvokeAsync(SKContext context, + public async IAsyncEnumerable StreamingInvokeAsync(SKContext context, AIRequestSettings? requestSettings = null, [EnumeratorCancellation] CancellationToken cancellationToken = default) { diff --git a/dotnet/src/SemanticKernel.Core/Functions/SemanticFunction.cs b/dotnet/src/SemanticKernel.Core/Functions/SemanticFunction.cs index 35f6760a7d54..f3ab3d47c984 100644 --- a/dotnet/src/SemanticKernel.Core/Functions/SemanticFunction.cs +++ b/dotnet/src/SemanticKernel.Core/Functions/SemanticFunction.cs @@ -98,7 +98,7 @@ public Task InvokeAsync( } /// - public IAsyncEnumerable StreamingInvokeAsync( + public IAsyncEnumerable StreamingInvokeAsync( SKContext context, AIRequestSettings? requestSettings = null, CancellationToken cancellationToken = default) @@ -181,7 +181,7 @@ private void AddDefaultValues(ContextVariables variables) } } - private async IAsyncEnumerable RunStreamingPromptAsync( + private async IAsyncEnumerable RunStreamingPromptAsync( AIRequestSettings? requestSettings, SKContext context, [EnumeratorCancellation] CancellationToken cancellationToken = default) @@ -201,7 +201,7 @@ private async IAsyncEnumerable RunStreamingPromptAsync( renderedPrompt = this.GetPromptFromEventArgsMetadataOrDefault(context, renderedPrompt); StringBuilder fullCompletion = new(); - await foreach (StreamingResultUpdate update in textCompletion.GetStreamingUpdatesAsync(renderedPrompt, requestSettings ?? defaultRequestSettings, cancellationToken).ConfigureAwait(false)) + await foreach (StreamingResultChunk update in textCompletion.GetStreamingChunksAsync(renderedPrompt, requestSettings ?? defaultRequestSettings, cancellationToken).ConfigureAwait(false)) { fullCompletion.Append(update); diff --git a/dotnet/src/SemanticKernel.Core/Functions/StreamingNativeResultUpdate.cs b/dotnet/src/SemanticKernel.Core/Functions/StreamingNativeResultUpdate.cs index 87d0af5c9587..6eb7ee2c24c7 100644 --- a/dotnet/src/SemanticKernel.Core/Functions/StreamingNativeResultUpdate.cs +++ b/dotnet/src/SemanticKernel.Core/Functions/StreamingNativeResultUpdate.cs @@ -11,7 +11,7 @@ namespace Microsoft.SemanticKernel; /// /// Native function streaming result update. /// -public sealed class StreamingNativeResultUpdate : StreamingResultUpdate +public sealed class StreamingNativeResultUpdate : StreamingResultChunk { /// public override string Type => "native_result_update"; diff --git a/dotnet/src/SemanticKernel.Core/Kernel.cs b/dotnet/src/SemanticKernel.Core/Kernel.cs index 30948b0475f8..b155388e7020 100644 --- a/dotnet/src/SemanticKernel.Core/Kernel.cs +++ b/dotnet/src/SemanticKernel.Core/Kernel.cs @@ -177,7 +177,7 @@ public async Task RunAsync(ContextVariables variables, Cancellatio /// Input to process /// The to monitor for cancellation requests. /// Result of the function composition - public async IAsyncEnumerable StreamingRunAsync(ISKFunction skFunction, ContextVariables? variables, [EnumeratorCancellation] CancellationToken cancellationToken) + public async IAsyncEnumerable RunStreamingAsync(ISKFunction skFunction, ContextVariables? variables, [EnumeratorCancellation] CancellationToken cancellationToken) { var context = this.CreateNewContext(variables); @@ -188,7 +188,7 @@ public async IAsyncEnumerable StreamingRunAsync(ISKFuncti repeatRequested = false; var functionDetails = skFunction.Describe(); - await foreach (StreamingResultUpdate update in skFunction.StreamingInvokeAsync(context, null, cancellationToken).ConfigureAwait(false)) + await foreach (StreamingResultChunk update in skFunction.StreamingInvokeAsync(context, null, cancellationToken).ConfigureAwait(false)) { cancellationToken.ThrowIfCancellationRequested(); diff --git a/dotnet/src/SemanticKernel.Core/Orchestration/FunctionRunner.cs b/dotnet/src/SemanticKernel.Core/Orchestration/FunctionRunner.cs index 86d03955d2ec..6f1b8e0250c7 100644 --- a/dotnet/src/SemanticKernel.Core/Orchestration/FunctionRunner.cs +++ b/dotnet/src/SemanticKernel.Core/Orchestration/FunctionRunner.cs @@ -38,12 +38,12 @@ public FunctionRunner(IKernel kernel) return this.RunAsync(function, variables, cancellationToken); } - public IAsyncEnumerable StreamingRunAsync(ISKFunction skFunction, ContextVariables? variables = null, CancellationToken cancellationToken = default) + public IAsyncEnumerable StreamingRunAsync(ISKFunction skFunction, ContextVariables? variables = null, CancellationToken cancellationToken = default) { return this._kernel.StreamingRunAsync(skFunction, variables, cancellationToken); } - public IAsyncEnumerable StreamingRunAsync(string pluginName, string functionName, ContextVariables? variables = null, CancellationToken cancellationToken = default) + public IAsyncEnumerable StreamingRunAsync(string pluginName, string functionName, ContextVariables? variables = null, CancellationToken cancellationToken = default) { throw new System.NotImplementedException(); } diff --git a/dotnet/src/SemanticKernel.Core/Planning/InstrumentedPlan.cs b/dotnet/src/SemanticKernel.Core/Planning/InstrumentedPlan.cs index fe7060e0ed2e..e25fbde3846a 100644 --- a/dotnet/src/SemanticKernel.Core/Planning/InstrumentedPlan.cs +++ b/dotnet/src/SemanticKernel.Core/Planning/InstrumentedPlan.cs @@ -62,7 +62,7 @@ public async Task InvokeAsync( } /// - public IAsyncEnumerable StreamingInvokeAsync(SKContext context, + public IAsyncEnumerable StreamingInvokeAsync(SKContext context, AIRequestSettings? requestSettings = null, CancellationToken cancellationToken = default) { diff --git a/dotnet/src/SemanticKernel.Core/Planning/Plan.cs b/dotnet/src/SemanticKernel.Core/Planning/Plan.cs index 2711e5886c1b..a966268704cc 100644 --- a/dotnet/src/SemanticKernel.Core/Planning/Plan.cs +++ b/dotnet/src/SemanticKernel.Core/Planning/Plan.cs @@ -345,7 +345,7 @@ public async Task InvokeAsync( } /// - public IAsyncEnumerable StreamingInvokeAsync( + public IAsyncEnumerable StreamingInvokeAsync( SKContext context, AIRequestSettings? requestSettings = null, CancellationToken cancellationToken = default) diff --git a/dotnet/src/SemanticKernel.UnitTests/Functions/OrderedIAIServiceConfigurationProviderTests.cs b/dotnet/src/SemanticKernel.UnitTests/Functions/OrderedIAIServiceConfigurationProviderTests.cs index 9845a2561d80..bbba43846e52 100644 --- a/dotnet/src/SemanticKernel.UnitTests/Functions/OrderedIAIServiceConfigurationProviderTests.cs +++ b/dotnet/src/SemanticKernel.UnitTests/Functions/OrderedIAIServiceConfigurationProviderTests.cs @@ -209,27 +209,22 @@ private sealed class TextCompletion : ITextCompletion public string? ModelId { get; } - public IAsyncEnumerable GetByteStreamingUpdatesAsync(string input, AIRequestSettings? requestSettings = null, CancellationToken cancellationToken = default) - { - throw new NotImplementedException(); - } - public Task> GetCompletionsAsync(string text, AIRequestSettings? requestSettings = null, CancellationToken cancellationToken = default) { throw new NotImplementedException(); } - public IAsyncEnumerable GetStreamingCompletionsAsync(string text, AIRequestSettings? requestSettings = null, CancellationToken cancellationToken = default) + public IAsyncEnumerable GetStreamingChunksAsync(string input, AIRequestSettings? requestSettings = null, CancellationToken cancellationToken = default) { throw new NotImplementedException(); } - public IAsyncEnumerable GetStreamingUpdatesAsync(string input, AIRequestSettings? requestSettings = null, CancellationToken cancellationToken = default) + public IAsyncEnumerable GetStreamingCompletionsAsync(string text, AIRequestSettings? requestSettings = null, CancellationToken cancellationToken = default) { throw new NotImplementedException(); } - public IAsyncEnumerable GetStringStreamingUpdatesAsync(string input, AIRequestSettings? requestSettings = null, CancellationToken cancellationToken = default) + public IAsyncEnumerable GetStreamingChunksAsync(string input, AIRequestSettings? requestSettings = null, CancellationToken cancellationToken = default) { throw new NotImplementedException(); } From 83b6aecec68ce88fe318d5bcaf7d8e88ccb8cb2b Mon Sep 17 00:00:00 2001 From: Roger Barreto <19890735+RogerBarreto@users.noreply.github.com> Date: Fri, 17 Nov 2023 15:31:44 +0000 Subject: [PATCH 18/46] Connector Updates WIP --- .../Example16_CustomLLM.cs | 24 +++--- .../Example65_KernelStreaming.cs | 26 +++--- .../HuggingFaceTextCompletion.cs | 45 +++++----- ...tUpdate.cs => StreamingTextResultChunk.cs} | 12 +-- .../AzureSdk/ClientBase.cs | 11 +-- .../AzureSdk/StreamingChatResultChunk.cs | 14 ++-- .../StreamingChatWithDataResultChunk.cs | 59 +++++++++++++ .../AzureSdk/StreamingTextResultChunk.cs | 7 +- .../AzureOpenAIChatCompletion.cs | 23 +----- .../ChatCompletion/OpenAIChatCompletion.cs | 21 +---- .../AzureOpenAIChatCompletionWithData.cs | 82 +++++++++++++++++++ .../ChatWithDataStreamingChoice.cs | 3 + .../TextCompletion/AzureTextCompletion.cs | 17 +--- .../KernelSemanticFunctionExtensionsTests.cs | 5 ++ .../AI/ChatCompletion/IChatCompletion.cs | 26 +++--- .../AI/StreamingResultChunk.cs | 20 ++++- .../Functions/StreamingNativeResultUpdate.cs | 2 +- .../Orchestration/FunctionRunner.cs | 5 +- 18 files changed, 266 insertions(+), 136 deletions(-) rename dotnet/src/Connectors/Connectors.AI.HuggingFace/TextCompletion/{StreamingTextResultUpdate.cs => StreamingTextResultChunk.cs} (75%) create mode 100644 dotnet/src/Connectors/Connectors.AI.OpenAI/AzureSdk/StreamingChatWithDataResultChunk.cs diff --git a/dotnet/samples/KernelSyntaxExamples/Example16_CustomLLM.cs b/dotnet/samples/KernelSyntaxExamples/Example16_CustomLLM.cs index 99f63ac4314e..4f67b046a4c1 100644 --- a/dotnet/samples/KernelSyntaxExamples/Example16_CustomLLM.cs +++ b/dotnet/samples/KernelSyntaxExamples/Example16_CustomLLM.cs @@ -2,7 +2,6 @@ using System; using System.Collections.Generic; -using System.Linq; using System.Runtime.CompilerServices; using System.Text; using System.Text.Json; @@ -60,13 +59,7 @@ public async IAsyncEnumerable GetByteStreamingUpdatesAsync(string input, public IAsyncEnumerable GetStreamingChunksAsync(string input, AIRequestSettings? requestSettings = null, CancellationToken cancellationToken = default) { - var list = new List() - { - new MyStreamingResultUpdate("llm content update 1"), - new MyStreamingResultUpdate("llm content update 2") - }; - - return list.ToAsyncEnumerable(); + return this.GetStreamingChunksAsync(input, requestSettings, cancellationToken); } public async IAsyncEnumerable GetStringStreamingUpdatesAsync(string input, AIRequestSettings? requestSettings = null, [EnumeratorCancellation] CancellationToken cancellationToken = default) @@ -76,17 +69,26 @@ public async IAsyncEnumerable GetStringStreamingUpdatesAsync(string inpu yield return update.ToString(); } } + + public async IAsyncEnumerable GetStreamingChunksAsync(string input, AIRequestSettings? requestSettings = null, [EnumeratorCancellation] CancellationToken cancellationToken = default) + { + if (typeof(T) == typeof(MyStreamingResultChunk)) + { + yield return (T)(object)new MyStreamingResultChunk("llm content update 1"); + yield return (T)(object)new MyStreamingResultChunk("llm content update 2"); + } + } } -public class MyStreamingResultUpdate : StreamingResultChunk +public class MyStreamingResultChunk : StreamingResultChunk { public override string Type => "my_text_type"; - public override int ResultIndex => 0; + public override int ChoiceIndex => 0; public string Content { get; } - public MyStreamingResultUpdate(string content) + public MyStreamingResultChunk(string content) { this.Content = content; } diff --git a/dotnet/samples/KernelSyntaxExamples/Example65_KernelStreaming.cs b/dotnet/samples/KernelSyntaxExamples/Example65_KernelStreaming.cs index ab585e86ba63..842a1c7f499f 100644 --- a/dotnet/samples/KernelSyntaxExamples/Example65_KernelStreaming.cs +++ b/dotnet/samples/KernelSyntaxExamples/Example65_KernelStreaming.cs @@ -68,7 +68,7 @@ Be creative and be funny. Let your imagination run wild. Console.WriteLine("\n=== Semantic Function - Streaming ===\n"); // Streaming can be of any type depending on the underlying service the function is using. - await foreach (var update in kernel.StreamingRunAsync(excuseFunction, new ContextVariables("I missed the F1 final race"), cancellationToken)) + await foreach (var update in kernel.RunStreamingAsync(excuseFunction, new ContextVariables("I missed the F1 final race"), cancellationToken)) { // You will be always able to know the type of the update by checking the Type property. if (update.Type == "openai_chat_message_update" && update is StreamingChatResultChunk chatUpdate) @@ -88,21 +88,21 @@ Be creative and be funny. Let your imagination run wild. var functions = kernel.ImportFunctions(new MyNativePlugin(), "MyNativePlugin"); - await NativeStreamingValueTypeAsync(kernel, functions, cancellationToken); + await NativeFunctionStreamingValueTypeAsync(kernel, functions, cancellationToken); - await NativeStreamingComplexTypeAsync(kernel, functions, cancellationToken); + await NativeFunctionStreamingComplexTypeAsync(kernel, functions, cancellationToken); - await NativeValueTypeAsync(kernel, functions, cancellationToken); + await NativeFunctionValueTypeAsync(kernel, functions, cancellationToken); - await NativeComplexTypeAsync(kernel, functions, cancellationToken); + await NativeFunctionComplexTypeAsync(kernel, functions, cancellationToken); } - private static async Task NativeStreamingValueTypeAsync(IKernel kernel, IDictionary functions, CancellationToken cancellationToken) + private static async Task NativeFunctionStreamingValueTypeAsync(IKernel kernel, IDictionary functions, CancellationToken cancellationToken) { Console.WriteLine("\n\n=== Native Streaming Functions - Streaming (Value Type) ===\n"); // Native string value type streaming function - await foreach (var update in kernel.StreamingRunAsync(functions["MyValueTypeStreamingNativeFunction"], new ContextVariables("My Value Type Streaming Function Input"), cancellationToken)) + await foreach (var update in kernel.RunStreamingAsync(functions["MyValueTypeStreamingNativeFunction"], new ContextVariables("My Value Type Streaming Function Input"), cancellationToken)) { if (update.Type == "native_result_update" && update is StreamingNativeResultUpdate nativeUpdate) { @@ -111,12 +111,12 @@ private static async Task NativeStreamingValueTypeAsync(IKernel kernel, IDiction } } - private static async Task NativeStreamingComplexTypeAsync(IKernel kernel, IDictionary functions, CancellationToken cancellationToken) + private static async Task NativeFunctionStreamingComplexTypeAsync(IKernel kernel, IDictionary functions, CancellationToken cancellationToken) { Console.WriteLine("\n\n=== Native Streaming Functions - Streaming (Complex Type) ===\n"); // Native complex type streaming function - await foreach (var update in kernel.StreamingRunAsync(functions["MyComplexTypeStreamingNativeFunction"], new ContextVariables("My Complex Type Streaming Function Input"), cancellationToken)) + await foreach (var update in kernel.RunStreamingAsync(functions["MyComplexTypeStreamingNativeFunction"], new ContextVariables("My Complex Type Streaming Function Input"), cancellationToken)) { // the complex type will be available thru the Value property of the native update abstraction. if (update.Type == "native_result_update" && update is StreamingNativeResultUpdate nativeUpdate && nativeUpdate.Value is MyStreamingBlock myComplexType) @@ -126,12 +126,12 @@ private static async Task NativeStreamingComplexTypeAsync(IKernel kernel, IDicti } } - private static async Task NativeValueTypeAsync(IKernel kernel, IDictionary functions, CancellationToken cancellationToken) + private static async Task NativeFunctionValueTypeAsync(IKernel kernel, IDictionary functions, CancellationToken cancellationToken) { Console.WriteLine("\n=== Native Non-Streaming Functions - Streaming (Value Type) ===\n"); // Native functions that don't support streaming and return value types can be executed with the streaming API's but the behavior will not be streamlike. - await foreach (var update in kernel.StreamingRunAsync(functions["MyValueTypeNativeFunction"], new ContextVariables("My Value Type Non Streaming Function Input"), cancellationToken)) + await foreach (var update in kernel.RunStreamingAsync(functions["MyValueTypeNativeFunction"], new ContextVariables("My Value Type Non Streaming Function Input"), cancellationToken)) { // the complex type will be available thru the Value property of the native update abstraction. if (update.Type == "native_result_update" && update is StreamingNativeResultUpdate nativeUpdate) @@ -141,12 +141,12 @@ private static async Task NativeValueTypeAsync(IKernel kernel, IDictionary functions, CancellationToken cancellationToken) + private static async Task NativeFunctionComplexTypeAsync(IKernel kernel, IDictionary functions, CancellationToken cancellationToken) { Console.WriteLine("\n=== Native Non-Streaming Functions - Streaming (Complex Type) ===\n"); // Native functions that don't support streaming and return complex types can be executed with the streaming API's but the behavior will not be streamlike. - await foreach (var update in kernel.StreamingRunAsync(functions["MyComplexTypeNativeFunction"], new ContextVariables("My Complex Type Non Streaming Function Input"), cancellationToken)) + await foreach (var update in kernel.RunStreamingAsync(functions["MyComplexTypeNativeFunction"], new ContextVariables("My Complex Type Non Streaming Function Input"), cancellationToken)) { // the complex type will be available thru the Value property of the native update abstraction. if (update.Type == "native_result_update" && update is StreamingNativeResultUpdate nativeUpdate && nativeUpdate.Value is MyCustomType myComplexType) diff --git a/dotnet/src/Connectors/Connectors.AI.HuggingFace/TextCompletion/HuggingFaceTextCompletion.cs b/dotnet/src/Connectors/Connectors.AI.HuggingFace/TextCompletion/HuggingFaceTextCompletion.cs index fec84d847814..84f6576b16b4 100644 --- a/dotnet/src/Connectors/Connectors.AI.HuggingFace/TextCompletion/HuggingFaceTextCompletion.cs +++ b/dotnet/src/Connectors/Connectors.AI.HuggingFace/TextCompletion/HuggingFaceTextCompletion.cs @@ -92,38 +92,43 @@ public async Task> GetCompletionsAsync( } /// - public async IAsyncEnumerable GetStreamingChunksAsync( + public IAsyncEnumerable GetStreamingChunksAsync( + string input, + AIRequestSettings? requestSettings = null, + CancellationToken cancellationToken = default) + { + return this.GetStreamingChunksAsync(input, requestSettings, cancellationToken); + } + + /// + public async IAsyncEnumerable GetStreamingChunksAsync( string input, AIRequestSettings? requestSettings = null, [EnumeratorCancellation] CancellationToken cancellationToken = default) { - var resultIndex = 0; + var choiceIndex = 0; foreach (var result in await this.ExecuteGetCompletionsAsync(input, cancellationToken).ConfigureAwait(false)) { cancellationToken.ThrowIfCancellationRequested(); var completion = await result.GetCompletionAsync(cancellationToken).ConfigureAwait(false); - yield return new StreamingTextResultUpdate(completion, resultIndex); - resultIndex++; - } - } + // If the provided T is a string, return the completion as is + if (typeof(T) == typeof(string)) + { + yield return (T)(object)completion; + continue; + } - /// - public async IAsyncEnumerable GetStringStreamingUpdatesAsync(string input, AIRequestSettings? requestSettings = null, [EnumeratorCancellation] CancellationToken cancellationToken = default) - { - await foreach (var update in this.GetStreamingChunksAsync(input, requestSettings, cancellationToken).ConfigureAwait(false)) - { - yield return update.ToString(); - } - } + // If the provided T is an specialized class of StreamingResultChunk interface + if (typeof(T) == typeof(StreamingTextResultChunk) || + typeof(T) == typeof(StreamingResultChunk)) + { + yield return (T)(object)new StreamingTextResultChunk(completion, choiceIndex); + continue; + } - /// - public async IAsyncEnumerable GetByteStreamingUpdatesAsync(string input, AIRequestSettings? requestSettings = null, [EnumeratorCancellation] CancellationToken cancellationToken = default) - { - await foreach (var update in this.GetStreamingChunksAsync(input, requestSettings, cancellationToken).ConfigureAwait(false)) - { - yield return update.ToByteArray(); + choiceIndex++; } } diff --git a/dotnet/src/Connectors/Connectors.AI.HuggingFace/TextCompletion/StreamingTextResultUpdate.cs b/dotnet/src/Connectors/Connectors.AI.HuggingFace/TextCompletion/StreamingTextResultChunk.cs similarity index 75% rename from dotnet/src/Connectors/Connectors.AI.HuggingFace/TextCompletion/StreamingTextResultUpdate.cs rename to dotnet/src/Connectors/Connectors.AI.HuggingFace/TextCompletion/StreamingTextResultChunk.cs index 7797fcf9b9c3..47f4d7b4c882 100644 --- a/dotnet/src/Connectors/Connectors.AI.HuggingFace/TextCompletion/StreamingTextResultUpdate.cs +++ b/dotnet/src/Connectors/Connectors.AI.HuggingFace/TextCompletion/StreamingTextResultChunk.cs @@ -8,13 +8,13 @@ namespace Microsoft.SemanticKernel.Connectors.AI.HuggingFace.TextCompletion; /// /// Streaming text result update. /// -public class StreamingTextResultUpdate : StreamingResultChunk +public class StreamingTextResultChunk : StreamingResultChunk { /// public override string Type => "huggingface_text_update"; /// - public override int ResultIndex { get; } + public override int ChoiceIndex { get; } /// /// Text associated to the update @@ -22,13 +22,13 @@ public class StreamingTextResultUpdate : StreamingResultChunk public string Content { get; } /// - /// Create a new instance of the class. + /// Create a new instance of the class. /// /// Text update - /// Index of the choice - public StreamingTextResultUpdate(string text, int resultIndex) + /// Index of the choice + public StreamingTextResultChunk(string text, int choiceIndex) { - this.ResultIndex = resultIndex; + this.ChoiceIndex = choiceIndex; this.Content = text; } diff --git a/dotnet/src/Connectors/Connectors.AI.OpenAI/AzureSdk/ClientBase.cs b/dotnet/src/Connectors/Connectors.AI.OpenAI/AzureSdk/ClientBase.cs index 9a514386dd34..46ee881aa5ca 100644 --- a/dotnet/src/Connectors/Connectors.AI.OpenAI/AzureSdk/ClientBase.cs +++ b/dotnet/src/Connectors/Connectors.AI.OpenAI/AzureSdk/ClientBase.cs @@ -179,19 +179,20 @@ private protected async IAsyncEnumerable InternalGetTextStreamingUpdatesAsync int choiceIndex = 0; await foreach (StreamingChoice choice in streamingChatCompletions.GetChoicesStreaming(cancellationToken).ConfigureAwait(false)) { - await foreach (string textUpdate in choice.GetTextStreaming(cancellationToken).ConfigureAwait(false)) + await foreach (string update in choice.GetTextStreaming(cancellationToken).ConfigureAwait(false)) { + // If the provided T is a string, return the completion as is if (typeof(T) == typeof(string)) { - yield return (T)(object)textUpdate; + yield return (T)(object)update; continue; } - // If the provided T is an specialized class of StreamingResultChunk interface works + // If the provided T is an specialized class of StreamingResultChunk interface if (typeof(T) == typeof(StreamingTextResultChunk) || typeof(T) == typeof(StreamingResultChunk)) { - yield return (T)(object)new StreamingTextResultChunk(textUpdate, choiceIndex); + yield return (T)(object)new StreamingTextResultChunk(update, choiceIndex, update); continue; } @@ -344,7 +345,7 @@ private protected async IAsyncEnumerable InternalGetChatStreamingUpdatesAsync continue; } - // If the provided T is an specialized class of StreamingResultChunk interface works + // If the provided T is an specialized class of StreamingResultChunk interface if (typeof(T) == typeof(StreamingChatResultChunk) || typeof(T) == typeof(StreamingResultChunk)) { diff --git a/dotnet/src/Connectors/Connectors.AI.OpenAI/AzureSdk/StreamingChatResultChunk.cs b/dotnet/src/Connectors/Connectors.AI.OpenAI/AzureSdk/StreamingChatResultChunk.cs index 9dc5251e8dc1..f1d2065b15df 100644 --- a/dotnet/src/Connectors/Connectors.AI.OpenAI/AzureSdk/StreamingChatResultChunk.cs +++ b/dotnet/src/Connectors/Connectors.AI.OpenAI/AzureSdk/StreamingChatResultChunk.cs @@ -17,7 +17,7 @@ public class StreamingChatResultChunk : StreamingResultChunk public override string Type => "openai_chat_message_update"; /// - public override int ResultIndex { get; } + public override int ChoiceIndex { get; } /// /// Function call associated to the message payload @@ -42,11 +42,11 @@ public class StreamingChatResultChunk : StreamingResultChunk /// /// Create a new instance of the class. /// - /// Original Azure SDK Message update representation + /// Original Connector Azure Message update representation /// Index of the choice - public StreamingChatResultChunk(AzureOpenAIChatMessage chatMessage, int resultIndex) + public StreamingChatResultChunk(AzureOpenAIChatMessage chatMessage, int resultIndex) : base(chatMessage) { - this.ResultIndex = resultIndex; + this.ChoiceIndex = resultIndex; this.FunctionCall = chatMessage.InnerChatMessage?.FunctionCall; this.Content = chatMessage.Content; this.Role = new AuthorRole(chatMessage.Role.ToString()); @@ -56,11 +56,11 @@ public StreamingChatResultChunk(AzureOpenAIChatMessage chatMessage, int resultIn /// /// Create a new instance of the class. /// - /// Original Azure SDK Message update representation + /// Internal Azure SDK Message update representation /// Index of the choice - public StreamingChatResultChunk(Azure.AI.OpenAI.ChatMessage chatMessage, int resultIndex) + public StreamingChatResultChunk(Azure.AI.OpenAI.ChatMessage chatMessage, int resultIndex) : base(chatMessage) { - this.ResultIndex = resultIndex; + this.ChoiceIndex = resultIndex; this.FunctionCall = chatMessage.FunctionCall; this.Content = chatMessage.Content; this.Role = new AuthorRole(chatMessage.Role.ToString()); diff --git a/dotnet/src/Connectors/Connectors.AI.OpenAI/AzureSdk/StreamingChatWithDataResultChunk.cs b/dotnet/src/Connectors/Connectors.AI.OpenAI/AzureSdk/StreamingChatWithDataResultChunk.cs new file mode 100644 index 000000000000..038e0887b6ae --- /dev/null +++ b/dotnet/src/Connectors/Connectors.AI.OpenAI/AzureSdk/StreamingChatWithDataResultChunk.cs @@ -0,0 +1,59 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Linq; +using System; +using System.Text; +using System.Text.Json; +using Microsoft.SemanticKernel.AI; +using Microsoft.SemanticKernel.AI.ChatCompletion; +using Microsoft.SemanticKernel.Connectors.AI.OpenAI.ChatCompletionWithData; + +namespace Microsoft.SemanticKernel.Connectors.AI.OpenAI.AzureSdk; + +/// +/// Streaming chat result update. +/// +public sealed class StreamingChatWithDataResultChunk : StreamingResultChunk +{ + /// + public override string Type => "openai_chat_message_update"; + + /// + public override int ChoiceIndex { get; } + + /// + /// Chat message abstraction + /// + public SemanticKernel.AI.ChatCompletion.ChatMessage ChatMessage { get; } + + /// + /// Create a new instance of the class. + /// + /// Azure message update representation from WithData apis + /// Index of the choice + internal StreamingChatWithDataResultChunk(ChatWithDataStreamingChoice choice, int resultIndex) : base(choice) + { + this.ChoiceIndex = resultIndex; + var message = choice.Messages.FirstOrDefault(this.IsValidMessage); + + this.ChatMessage = new AzureOpenAIChatMessage(AuthorRole.Assistant.Label, message?.Delta?.Content ?? string.Empty); + } + + /// + public override byte[] ToByteArray() + { + return Encoding.UTF8.GetBytes(this.ToString()); + } + + /// + public override string ToString() + { + return JsonSerializer.Serialize(this); + } + + private bool IsValidMessage(ChatWithDataStreamingMessage message) + { + return !message.EndTurn && + (message.Delta.Role is null || !message.Delta.Role.Equals(AuthorRole.Tool.Label, StringComparison.Ordinal)); + } +} diff --git a/dotnet/src/Connectors/Connectors.AI.OpenAI/AzureSdk/StreamingTextResultChunk.cs b/dotnet/src/Connectors/Connectors.AI.OpenAI/AzureSdk/StreamingTextResultChunk.cs index d688f20c91f9..8a42bde0cd4c 100644 --- a/dotnet/src/Connectors/Connectors.AI.OpenAI/AzureSdk/StreamingTextResultChunk.cs +++ b/dotnet/src/Connectors/Connectors.AI.OpenAI/AzureSdk/StreamingTextResultChunk.cs @@ -14,7 +14,7 @@ public class StreamingTextResultChunk : StreamingResultChunk public override string Type => "openai_text_update"; /// - public override int ResultIndex { get; } + public override int ChoiceIndex { get; } /// /// Text associated to the update @@ -26,9 +26,10 @@ public class StreamingTextResultChunk : StreamingResultChunk /// /// Text update /// Index of the choice - public StreamingTextResultChunk(string text, int resultIndex) + /// Inner chunk object + public StreamingTextResultChunk(string text, int resultIndex, object? innerChunkObject = null) : base(innerChunkObject) { - this.ResultIndex = resultIndex; + this.ChoiceIndex = resultIndex; this.Content = text; } diff --git a/dotnet/src/Connectors/Connectors.AI.OpenAI/ChatCompletion/AzureOpenAIChatCompletion.cs b/dotnet/src/Connectors/Connectors.AI.OpenAI/ChatCompletion/AzureOpenAIChatCompletion.cs index 189fec6d810b..5af8e8e637eb 100644 --- a/dotnet/src/Connectors/Connectors.AI.OpenAI/ChatCompletion/AzureOpenAIChatCompletion.cs +++ b/dotnet/src/Connectors/Connectors.AI.OpenAI/ChatCompletion/AzureOpenAIChatCompletion.cs @@ -2,7 +2,6 @@ using System.Collections.Generic; using System.Net.Http; -using System.Runtime.CompilerServices; using System.Threading; using System.Threading.Tasks; using Azure.AI.OpenAI; @@ -129,26 +128,12 @@ public Task> GetCompletionsAsync( /// public IAsyncEnumerable GetStreamingChunksAsync(string input, AIRequestSettings? requestSettings = null, CancellationToken cancellationToken = default) - { - var chatHistory = this.CreateNewChat(input); - return this.InternalGetChatStreamingUpdatesAsync(chatHistory, requestSettings, cancellationToken); - } - - /// - public async IAsyncEnumerable GetStringStreamingUpdatesAsync(string input, AIRequestSettings? requestSettings = null, [EnumeratorCancellation] CancellationToken cancellationToken = default) - { - await foreach (var update in this.GetStreamingChunksAsync(input, requestSettings, cancellationToken).ConfigureAwait(false)) - { - yield return update.ToString(); - } - } + => this.GetStreamingChunksAsync(input, requestSettings, cancellationToken); /// - public async IAsyncEnumerable GetByteStreamingUpdatesAsync(string input, AIRequestSettings? requestSettings = null, [EnumeratorCancellation] CancellationToken cancellationToken = default) + public IAsyncEnumerable GetStreamingChunksAsync(string input, AIRequestSettings? requestSettings = null, CancellationToken cancellationToken = default) { - await foreach (var update in this.GetStreamingChunksAsync(input, requestSettings, cancellationToken).ConfigureAwait(false)) - { - yield return update.ToByteArray(); - } + var chatHistory = this.CreateNewChat(input); + return this.InternalGetChatStreamingUpdatesAsync(chatHistory, requestSettings, cancellationToken); } } diff --git a/dotnet/src/Connectors/Connectors.AI.OpenAI/ChatCompletion/OpenAIChatCompletion.cs b/dotnet/src/Connectors/Connectors.AI.OpenAI/ChatCompletion/OpenAIChatCompletion.cs index f4df02d5d008..85647b0ac164 100644 --- a/dotnet/src/Connectors/Connectors.AI.OpenAI/ChatCompletion/OpenAIChatCompletion.cs +++ b/dotnet/src/Connectors/Connectors.AI.OpenAI/ChatCompletion/OpenAIChatCompletion.cs @@ -2,7 +2,6 @@ using System.Collections.Generic; using System.Net.Http; -using System.Runtime.CompilerServices; using System.Threading; using System.Threading.Tasks; using Azure.AI.OpenAI; @@ -106,25 +105,13 @@ public Task> GetCompletionsAsync( /// public IAsyncEnumerable GetStreamingChunksAsync(string input, AIRequestSettings? requestSettings = null, CancellationToken cancellationToken = default) { - var chatHistory = this.CreateNewChat(input); - return this.InternalGetChatStreamingUpdatesAsync(chatHistory, requestSettings, cancellationToken); - } - - /// - public async IAsyncEnumerable GetStringStreamingUpdatesAsync(string input, AIRequestSettings? requestSettings = null, [EnumeratorCancellation] CancellationToken cancellationToken = default) - { - await foreach (var update in this.GetStreamingChunksAsync(input, requestSettings, cancellationToken).ConfigureAwait(false)) - { - yield return update.ToString(); - } + return this.GetStreamingChunksAsync(input, requestSettings, cancellationToken); } /// - public async IAsyncEnumerable GetByteStreamingUpdatesAsync(string input, AIRequestSettings? requestSettings = null, [EnumeratorCancellation] CancellationToken cancellationToken = default) + public IAsyncEnumerable GetStreamingChunksAsync(string input, AIRequestSettings? requestSettings = null, CancellationToken cancellationToken = default) { - await foreach (var update in this.GetStreamingChunksAsync(input, requestSettings, cancellationToken).ConfigureAwait(false)) - { - yield return update.ToByteArray(); - } + var chatHistory = this.CreateNewChat(input); + return this.InternalGetChatStreamingUpdatesAsync(chatHistory, requestSettings, cancellationToken); } } diff --git a/dotnet/src/Connectors/Connectors.AI.OpenAI/ChatCompletionWithData/AzureOpenAIChatCompletionWithData.cs b/dotnet/src/Connectors/Connectors.AI.OpenAI/ChatCompletionWithData/AzureOpenAIChatCompletionWithData.cs index 64a63cbddbc7..3b6d3b2e9952 100644 --- a/dotnet/src/Connectors/Connectors.AI.OpenAI/ChatCompletionWithData/AzureOpenAIChatCompletionWithData.cs +++ b/dotnet/src/Connectors/Connectors.AI.OpenAI/ChatCompletionWithData/AzureOpenAIChatCompletionWithData.cs @@ -154,6 +154,25 @@ public async IAsyncEnumerable GetByteStreamingUpdatesAsync(string input, } } + /// + public async IAsyncEnumerable GetStreamingChunksAsync( + string input, + AIRequestSettings? requestSettings = null, + [EnumeratorCancellation] CancellationToken cancellationToken = default) + { + OpenAIRequestSettings chatRequestSettings = OpenAIRequestSettings.FromRequestSettings(requestSettings); + + var chat = this.PrepareChatHistory(input, chatRequestSettings); + + using var request = this.GetRequest(chat, chatRequestSettings, isStreamEnabled: true); + using var response = await this.SendRequestAsync(request, cancellationToken).ConfigureAwait(false); + + await foreach (var result in this.GetChatStreamingUpdatesAsync(response)) + { + yield return result; + } + } + #region private ================================================================================ private const string DefaultApiVersion = "2023-06-01-preview"; @@ -262,6 +281,69 @@ private async IAsyncEnumerable GetStreamingResultsAsync(Ht } } + private async IAsyncEnumerable GetChatStreamingUpdatesAsync(HttpResponseMessage response) + { + const string ServerEventPayloadPrefix = "data:"; + + using var stream = await response.Content.ReadAsStreamAndTranslateExceptionAsync().ConfigureAwait(false); + using var reader = new StreamReader(stream); + + while (!reader.EndOfStream) + { + var body = await reader.ReadLineAsync().ConfigureAwait(false); + + if (string.IsNullOrWhiteSpace(body)) + { + continue; + } + + if (body.StartsWith(ServerEventPayloadPrefix, StringComparison.Ordinal)) + { + body = body.Substring(ServerEventPayloadPrefix.Length); + } + + var chatWithDataResponse = this.DeserializeResponse(body); + + // If the provided T is the response type, return the response as is (Breaking Glass) option 1 + if (typeof(T) == chatWithDataResponse.GetType()) + { + yield return (T)(object)chatWithDataResponse; + continue; + } + + foreach (var choice in chatWithDataResponse.Choices) + { + // If the provided T is an specialized class of StreamingResultChunk interface + if (typeof(T) == typeof(StreamingChatResultChunk) || + typeof(T) == typeof(StreamingResultChunk)) + { + yield return (T)(object)new StreamingChatWithDataResultChunk(choice, choice.Index); + continue; + } + + if (typeof(T) == choice.GetType()) + { + yield return (T)(object)choice; + continue; + } + + var result = new ChatWithDataStreamingResult(chatWithDataResponse, choice); + if (typeof(T) == typeof(string)) + { + await foreach (SemanticKernel.AI.ChatCompletion.ChatMessage message in result.GetStreamingChatMessageAsync().ConfigureAwait(false)) + { + yield return (T)(object)message.Content; + } + } + + if (typeof(T) == typeof(ChatWithDataStreamingResult)) + { + yield return (T)(object)result; + } + } + } + } + private T DeserializeResponse(string body) { var response = Microsoft.SemanticKernel.Text.Json.Deserialize(body); diff --git a/dotnet/src/Connectors/Connectors.AI.OpenAI/ChatCompletionWithData/ChatWithDataStreamingChoice.cs b/dotnet/src/Connectors/Connectors.AI.OpenAI/ChatCompletionWithData/ChatWithDataStreamingChoice.cs index 1718e386279a..2fa6572a73c1 100644 --- a/dotnet/src/Connectors/Connectors.AI.OpenAI/ChatCompletionWithData/ChatWithDataStreamingChoice.cs +++ b/dotnet/src/Connectors/Connectors.AI.OpenAI/ChatCompletionWithData/ChatWithDataStreamingChoice.cs @@ -13,4 +13,7 @@ internal sealed class ChatWithDataStreamingChoice { [JsonPropertyName("messages")] public IList Messages { get; set; } = Array.Empty(); + + [JsonPropertyName("index")] + public int Index { get; set; } = 0; } diff --git a/dotnet/src/Connectors/Connectors.AI.OpenAI/TextCompletion/AzureTextCompletion.cs b/dotnet/src/Connectors/Connectors.AI.OpenAI/TextCompletion/AzureTextCompletion.cs index 710e709eef8d..49ea409ab0e5 100644 --- a/dotnet/src/Connectors/Connectors.AI.OpenAI/TextCompletion/AzureTextCompletion.cs +++ b/dotnet/src/Connectors/Connectors.AI.OpenAI/TextCompletion/AzureTextCompletion.cs @@ -2,7 +2,6 @@ using System.Collections.Generic; using System.Net.Http; -using System.Runtime.CompilerServices; using System.Threading; using System.Threading.Tasks; using Azure.AI.OpenAI; @@ -107,20 +106,8 @@ public IAsyncEnumerable GetStreamingChunksAsync(string inp } /// - public async IAsyncEnumerable GetStringStreamingUpdatesAsync(string input, AIRequestSettings? requestSettings = null, [EnumeratorCancellation] CancellationToken cancellationToken = default) + public IAsyncEnumerable GetStreamingChunksAsync(string input, AIRequestSettings? requestSettings = null, CancellationToken cancellationToken = default) { - await foreach (var update in this.GetStreamingChunksAsync(input, requestSettings, cancellationToken).ConfigureAwait(false)) - { - yield return update.ToString(); - } - } - - /// - public async IAsyncEnumerable GetByteStreamingUpdatesAsync(string input, AIRequestSettings? requestSettings = null, [EnumeratorCancellation] CancellationToken cancellationToken = default) - { - await foreach (var update in this.GetStreamingChunksAsync(input, requestSettings, cancellationToken).ConfigureAwait(false)) - { - yield return update.ToByteArray(); - } + return this.InternalGetTextStreamingUpdatesAsync(input, requestSettings, cancellationToken); } } diff --git a/dotnet/src/IntegrationTests/Extensions/KernelSemanticFunctionExtensionsTests.cs b/dotnet/src/IntegrationTests/Extensions/KernelSemanticFunctionExtensionsTests.cs index c4a40e65de9d..e16b77e190be 100644 --- a/dotnet/src/IntegrationTests/Extensions/KernelSemanticFunctionExtensionsTests.cs +++ b/dotnet/src/IntegrationTests/Extensions/KernelSemanticFunctionExtensionsTests.cs @@ -96,6 +96,11 @@ public IAsyncEnumerable GetStringStreamingUpdatesAsync(string input, AIR { throw new NotImplementedException(); } + + public IAsyncEnumerable GetStreamingChunksAsync(string input, AIRequestSettings? requestSettings = null, CancellationToken cancellationToken = default) + { + throw new NotImplementedException(); + } } internal sealed class RedirectTextCompletionResult : ITextResult diff --git a/dotnet/src/SemanticKernel.Abstractions/AI/ChatCompletion/IChatCompletion.cs b/dotnet/src/SemanticKernel.Abstractions/AI/ChatCompletion/IChatCompletion.cs index e523aa68c0cd..8c702626bd16 100644 --- a/dotnet/src/SemanticKernel.Abstractions/AI/ChatCompletion/IChatCompletion.cs +++ b/dotnet/src/SemanticKernel.Abstractions/AI/ChatCompletion/IChatCompletion.cs @@ -1,5 +1,6 @@ // Copyright (c) Microsoft. All rights reserved. +using System; using System.Collections.Generic; using System.Threading; using System.Threading.Tasks; @@ -44,7 +45,7 @@ IAsyncEnumerable GetStreamingChatCompletionsAsync( CancellationToken cancellationToken = default); /// - /// Get streaming completion results for the prompt and settings. + /// Get streaming completion StreamingResultChunk results for the prompt and settings. /// /// The input string. (May be a JSON for complex objects, Byte64 for binary, will depend on the connector spec). /// Request settings for the completion API @@ -56,25 +57,18 @@ IAsyncEnumerable GetStreamingChunksAsync( CancellationToken cancellationToken = default); /// - /// Get streaming results for the prompt and settings. + /// Get streaming results for the prompt and settings of a specific type. + /// Each modality may support for different types of streaming result. /// - /// The input string. (May be a JSON for complex objects, Byte64 for binary, will depend on the connector spec). + /// + /// Usage of this method may be more efficient if the connector has a dedicated API to return this result without extra allocations for StreamingResultChunk abstraction. + /// + /// Throws if the specified type is not the same or fail to cast + /// The prompt to complete. /// Request settings for the completion API /// The to monitor for cancellation requests. The default is . /// Streaming list of different completion streaming string updates generated by the remote model - IAsyncEnumerable GetStringStreamingUpdatesAsync( - string input, - AIRequestSettings? requestSettings = null, - CancellationToken cancellationToken = default); - - /// - /// Get streaming results for the prompt and settings. - /// - /// The input string. (May be a JSON for complex objects, Byte64 for binary, will depend on the connector spec). - /// Request settings for the completion API - /// The to monitor for cancellation requests. The default is . - /// Streaming list of different completion streaming byte array updates generated by the remote model - IAsyncEnumerable GetByteStreamingUpdatesAsync( + IAsyncEnumerable GetStreamingChunksAsync( string input, AIRequestSettings? requestSettings = null, CancellationToken cancellationToken = default); diff --git a/dotnet/src/SemanticKernel.Abstractions/AI/StreamingResultChunk.cs b/dotnet/src/SemanticKernel.Abstractions/AI/StreamingResultChunk.cs index 6dd5305d30b3..d39cb5887cfa 100644 --- a/dotnet/src/SemanticKernel.Abstractions/AI/StreamingResultChunk.cs +++ b/dotnet/src/SemanticKernel.Abstractions/AI/StreamingResultChunk.cs @@ -18,7 +18,7 @@ public abstract class StreamingResultChunk /// /// In a scenario of multiple results, this represents zero-based index of the result in the streaming sequence /// - public abstract int ResultIndex { get; } + public abstract int ChoiceIndex { get; } /// /// Converts the update class to string. @@ -32,9 +32,27 @@ public abstract class StreamingResultChunk /// Byte array representation of the update public abstract byte[] ToByteArray(); + /// + /// Internal chunk object reference. (Breaking glass). + /// Each connector will have its own internal object representing the result chunk. + /// + /// + /// The usage of this property is considered "unsafe". Use it only if strictly necessary. + /// + public object? InnerResultChunk { get; } + /// /// The current context associated the function call. /// [JsonIgnore] internal SKContext? Context { get; set; } + + /// + /// Initializes a new instance of the class. + /// + /// Inner result chunk object reference + protected StreamingResultChunk(object? innerResultChunk = null) + { + this.InnerResultChunk = innerResultChunk; + } } diff --git a/dotnet/src/SemanticKernel.Core/Functions/StreamingNativeResultUpdate.cs b/dotnet/src/SemanticKernel.Core/Functions/StreamingNativeResultUpdate.cs index 6eb7ee2c24c7..ce292d60c672 100644 --- a/dotnet/src/SemanticKernel.Core/Functions/StreamingNativeResultUpdate.cs +++ b/dotnet/src/SemanticKernel.Core/Functions/StreamingNativeResultUpdate.cs @@ -17,7 +17,7 @@ public sealed class StreamingNativeResultUpdate : StreamingResultChunk public override string Type => "native_result_update"; /// - public override int ResultIndex => 0; + public override int ChoiceIndex => 0; /// /// Native object value that represents the update diff --git a/dotnet/src/SemanticKernel.Core/Orchestration/FunctionRunner.cs b/dotnet/src/SemanticKernel.Core/Orchestration/FunctionRunner.cs index 6f1b8e0250c7..6269807b1593 100644 --- a/dotnet/src/SemanticKernel.Core/Orchestration/FunctionRunner.cs +++ b/dotnet/src/SemanticKernel.Core/Orchestration/FunctionRunner.cs @@ -40,11 +40,12 @@ public FunctionRunner(IKernel kernel) public IAsyncEnumerable StreamingRunAsync(ISKFunction skFunction, ContextVariables? variables = null, CancellationToken cancellationToken = default) { - return this._kernel.StreamingRunAsync(skFunction, variables, cancellationToken); + return this._kernel.RunStreamingAsync(skFunction, variables, cancellationToken); } public IAsyncEnumerable StreamingRunAsync(string pluginName, string functionName, ContextVariables? variables = null, CancellationToken cancellationToken = default) { - throw new System.NotImplementedException(); + var function = this._kernel.Functions.GetFunction(pluginName, functionName); + return this.StreamingRunAsync(function, variables, cancellationToken); } } From 59da387acd143eb8afbd1bd3e561a608d24108d3 Mon Sep 17 00:00:00 2001 From: Roger Barreto <19890735+RogerBarreto@users.noreply.github.com> Date: Sun, 19 Nov 2023 08:53:29 +0000 Subject: [PATCH 19/46] XMl updates --- .../AI/StreamingResultChunk.cs | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/dotnet/src/SemanticKernel.Abstractions/AI/StreamingResultChunk.cs b/dotnet/src/SemanticKernel.Abstractions/AI/StreamingResultChunk.cs index d39cb5887cfa..0fc6ec72d920 100644 --- a/dotnet/src/SemanticKernel.Abstractions/AI/StreamingResultChunk.cs +++ b/dotnet/src/SemanticKernel.Abstractions/AI/StreamingResultChunk.cs @@ -11,25 +11,31 @@ namespace Microsoft.SemanticKernel.AI; public abstract class StreamingResultChunk { /// - /// Type of the update. + /// Type of the chunk. /// public abstract string Type { get; } /// - /// In a scenario of multiple results, this represents zero-based index of the result in the streaming sequence + /// In a scenario of multiple choices per request, this represents zero-based index of the choice in the streaming sequence /// public abstract int ChoiceIndex { get; } /// - /// Converts the update class to string. + /// Abstract string representation of the chunk in a way it could compose/append with previous chunks. /// - /// String representation of the update + /// + /// Depending on the nature of the underlying type, this method may be more efficient than . + /// + /// String representation of the chunk public abstract override string ToString(); /// - /// Converts the update class to byte array. + /// Abstract byte[] representation of the chunk in a way it could be composed/appended with previous chunks. /// - /// Byte array representation of the update + /// + /// Depending on the nature of the underlying type, this method may be more efficient than . + /// + /// Byte array representation of the chunk public abstract byte[] ToByteArray(); /// From 42d11b0eb3d915be6c980992b25b68cd6901f085 Mon Sep 17 00:00:00 2001 From: Roger Barreto <19890735+RogerBarreto@users.noreply.github.com> Date: Sun, 19 Nov 2023 08:57:15 +0000 Subject: [PATCH 20/46] Warning fixes --- .../AzureSdk/StreamingChatWithDataResultChunk.cs | 2 +- dotnet/src/SemanticKernel.Core/Functions/NativeFunction.cs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dotnet/src/Connectors/Connectors.AI.OpenAI/AzureSdk/StreamingChatWithDataResultChunk.cs b/dotnet/src/Connectors/Connectors.AI.OpenAI/AzureSdk/StreamingChatWithDataResultChunk.cs index 038e0887b6ae..2705855764a5 100644 --- a/dotnet/src/Connectors/Connectors.AI.OpenAI/AzureSdk/StreamingChatWithDataResultChunk.cs +++ b/dotnet/src/Connectors/Connectors.AI.OpenAI/AzureSdk/StreamingChatWithDataResultChunk.cs @@ -1,7 +1,7 @@ // Copyright (c) Microsoft. All rights reserved. -using System.Linq; using System; +using System.Linq; using System.Text; using System.Text.Json; using Microsoft.SemanticKernel.AI; diff --git a/dotnet/src/SemanticKernel.Core/Functions/NativeFunction.cs b/dotnet/src/SemanticKernel.Core/Functions/NativeFunction.cs index 437ecc7ace39..383b7a5027d1 100644 --- a/dotnet/src/SemanticKernel.Core/Functions/NativeFunction.cs +++ b/dotnet/src/SemanticKernel.Core/Functions/NativeFunction.cs @@ -9,8 +9,8 @@ using System.Globalization; using System.Linq; using System.Reflection; -using System.Runtime.ExceptionServices; using System.Runtime.CompilerServices; +using System.Runtime.ExceptionServices; using System.Text.Json; using System.Text.RegularExpressions; using System.Threading; From 150dc4ddcdadd7c48f387c2407054eccfc125ff8 Mon Sep 17 00:00:00 2001 From: Roger Barreto <19890735+RogerBarreto@users.noreply.github.com> Date: Mon, 20 Nov 2023 18:19:05 +0000 Subject: [PATCH 21/46] Add Hugging face streaming support and examples --- .vscode/settings.json | 12 +++- .../Example16_CustomLLM.cs | 2 +- .../Example20_HuggingFace.cs | 30 ++++++++ ...eaming.cs => Example72_KernelStreaming.cs} | 10 +-- .../KernelSyntaxExamples/TestConfiguration.cs | 1 + .../HuggingFaceTextCompletion.cs | 56 ++++++++++++--- .../StreamingTextResultChunk.cs | 68 ++++++++++++++++--- .../TextCompletion/TextCompletionRequest.cs | 6 ++ .../src/Http/HttpClientExtensions.cs | 1 + .../InternalUtilities/src/Http/HttpRequest.cs | 6 ++ .../AI/StreamingResultChunk.cs | 2 +- .../Functions/NativeFunction.cs | 8 +-- .../Functions/StreamingNativeResultUpdate.cs | 11 +-- .../SemanticKernel.Core/KernelExtensions.cs | 31 +++++++++ 14 files changed, 206 insertions(+), 38 deletions(-) rename dotnet/samples/KernelSyntaxExamples/{Example65_KernelStreaming.cs => Example72_KernelStreaming.cs} (96%) diff --git a/.vscode/settings.json b/.vscode/settings.json index dae66ae37dc7..837d05d0ab42 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -9,7 +9,7 @@ "[csharp]": { "editor.defaultFormatter": "ms-dotnettools.csharp", "editor.codeActionsOnSave": { - "source.fixAll": true + "source.fixAll": "explicit" } }, "editor.bracketPairColorization.enabled": true, @@ -77,7 +77,7 @@ "editor.formatOnSave": false, "editor.tabSize": 4, "editor.codeActionsOnSave": { - "source.fixAll": false + "source.fixAll": "never" }, }, "emeraldwalk.runonsave": { @@ -90,5 +90,11 @@ }, "java.debug.settings.onBuildFailureProceed": true, "java.compile.nullAnalysis.mode": "disabled", - "dotnet.defaultSolution": "dotnet\\SK-dotnet.sln" + "dotnet.defaultSolution": "dotnet\\SK-dotnet.sln", + "[typescript][typescriptreact]": { + "editor.codeActionsOnSave": { + "source.organizeImports": "explicit", + "source.fixAll": "explicit" + } + } } \ No newline at end of file diff --git a/dotnet/samples/KernelSyntaxExamples/Example16_CustomLLM.cs b/dotnet/samples/KernelSyntaxExamples/Example16_CustomLLM.cs index 4f67b046a4c1..28ba17fa8933 100644 --- a/dotnet/samples/KernelSyntaxExamples/Example16_CustomLLM.cs +++ b/dotnet/samples/KernelSyntaxExamples/Example16_CustomLLM.cs @@ -88,7 +88,7 @@ public class MyStreamingResultChunk : StreamingResultChunk public string Content { get; } - public MyStreamingResultChunk(string content) + public MyStreamingResultChunk(string content) : base(content) { this.Content = content; } diff --git a/dotnet/samples/KernelSyntaxExamples/Example20_HuggingFace.cs b/dotnet/samples/KernelSyntaxExamples/Example20_HuggingFace.cs index 124f68c3f639..45726f5f71b1 100644 --- a/dotnet/samples/KernelSyntaxExamples/Example20_HuggingFace.cs +++ b/dotnet/samples/KernelSyntaxExamples/Example20_HuggingFace.cs @@ -13,6 +13,7 @@ public static class Example20_HuggingFace { public static async Task RunAsync() { + await RunInferenceApiStreamingExampleAsync(); await RunInferenceApiExampleAsync(); await RunLlamaExampleAsync(); } @@ -21,6 +22,9 @@ public static async Task RunAsync() /// This example uses HuggingFace Inference API to access hosted models. /// More information here: /// + /// + /// Tested models: google/flan-t5-xxl + /// private static async Task RunInferenceApiExampleAsync() { Console.WriteLine("\n======== HuggingFace Inference API example ========\n"); @@ -39,6 +43,32 @@ private static async Task RunInferenceApiExampleAsync() Console.WriteLine(result.GetValue()); } + /// + /// This example uses HuggingFace Inference API to access hosted models. + /// More information here: + /// + /// + /// Tested models: HuggingFaceH4/zephyr-7b-beta + /// + private static async Task RunInferenceApiStreamingExampleAsync() + { + Console.WriteLine("\n======== HuggingFace Inference API Streaming example ========\n"); + + IKernel kernel = new KernelBuilder() + .WithLoggerFactory(ConsoleLogger.LoggerFactory) + .WithHuggingFaceTextCompletionService( + model: TestConfiguration.HuggingFace.StreamingModelId, + apiKey: TestConfiguration.HuggingFace.ApiKey) + .Build(); + + var questionAnswerFunction = kernel.CreateSemanticFunction("Why is streaming important?"); + + await foreach (var chunk in kernel.RunStreamingAsync(questionAnswerFunction)) + { + Console.Write(chunk); + } + } + /// /// This example uses HuggingFace Llama 2 model and local HTTP server from Semantic Kernel repository. /// How to setup local HTTP server: . diff --git a/dotnet/samples/KernelSyntaxExamples/Example65_KernelStreaming.cs b/dotnet/samples/KernelSyntaxExamples/Example72_KernelStreaming.cs similarity index 96% rename from dotnet/samples/KernelSyntaxExamples/Example65_KernelStreaming.cs rename to dotnet/samples/KernelSyntaxExamples/Example72_KernelStreaming.cs index 842a1c7f499f..bb26cb091b57 100644 --- a/dotnet/samples/KernelSyntaxExamples/Example65_KernelStreaming.cs +++ b/dotnet/samples/KernelSyntaxExamples/Example72_KernelStreaming.cs @@ -21,7 +21,7 @@ * This example shows how to use multiple prompt template formats. */ // ReSharper disable once InconsistentNaming -public static class Example65_KernelStreaming +public static class Example72_KernelStreaming { /// /// Show how to combine multiple prompt template factories. @@ -104,7 +104,7 @@ private static async Task NativeFunctionStreamingValueTypeAsync(IKernel kernel, // Native string value type streaming function await foreach (var update in kernel.RunStreamingAsync(functions["MyValueTypeStreamingNativeFunction"], new ContextVariables("My Value Type Streaming Function Input"), cancellationToken)) { - if (update.Type == "native_result_update" && update is StreamingNativeResultUpdate nativeUpdate) + if (update.Type == "native_result_update" && update is StreamingNativeResultChunk nativeUpdate) { Console.Write(nativeUpdate.Value); } @@ -119,7 +119,7 @@ private static async Task NativeFunctionStreamingComplexTypeAsync(IKernel kernel await foreach (var update in kernel.RunStreamingAsync(functions["MyComplexTypeStreamingNativeFunction"], new ContextVariables("My Complex Type Streaming Function Input"), cancellationToken)) { // the complex type will be available thru the Value property of the native update abstraction. - if (update.Type == "native_result_update" && update is StreamingNativeResultUpdate nativeUpdate && nativeUpdate.Value is MyStreamingBlock myComplexType) + if (update.Type == "native_result_update" && update is StreamingNativeResultChunk nativeUpdate && nativeUpdate.Value is MyStreamingBlock myComplexType) { Console.WriteLine(Encoding.UTF8.GetString(myComplexType.Content)); } @@ -134,7 +134,7 @@ private static async Task NativeFunctionValueTypeAsync(IKernel kernel, IDictiona await foreach (var update in kernel.RunStreamingAsync(functions["MyValueTypeNativeFunction"], new ContextVariables("My Value Type Non Streaming Function Input"), cancellationToken)) { // the complex type will be available thru the Value property of the native update abstraction. - if (update.Type == "native_result_update" && update is StreamingNativeResultUpdate nativeUpdate) + if (update.Type == "native_result_update" && update is StreamingNativeResultChunk nativeUpdate) { Console.WriteLine(nativeUpdate.Value); } @@ -149,7 +149,7 @@ private static async Task NativeFunctionComplexTypeAsync(IKernel kernel, IDictio await foreach (var update in kernel.RunStreamingAsync(functions["MyComplexTypeNativeFunction"], new ContextVariables("My Complex Type Non Streaming Function Input"), cancellationToken)) { // the complex type will be available thru the Value property of the native update abstraction. - if (update.Type == "native_result_update" && update is StreamingNativeResultUpdate nativeUpdate && nativeUpdate.Value is MyCustomType myComplexType) + if (update.Type == "native_result_update" && update is StreamingNativeResultChunk nativeUpdate && nativeUpdate.Value is MyCustomType myComplexType) { Console.WriteLine($"Text: {myComplexType.Text}, Number: {myComplexType.Number}"); } diff --git a/dotnet/samples/KernelSyntaxExamples/TestConfiguration.cs b/dotnet/samples/KernelSyntaxExamples/TestConfiguration.cs index 9b7e62af9d43..00b6e52638e5 100644 --- a/dotnet/samples/KernelSyntaxExamples/TestConfiguration.cs +++ b/dotnet/samples/KernelSyntaxExamples/TestConfiguration.cs @@ -114,6 +114,7 @@ public class HuggingFaceConfig { public string ApiKey { get; set; } public string ModelId { get; set; } + public string StreamingModelId { get; set; } } public class PineconeConfig diff --git a/dotnet/src/Connectors/Connectors.AI.HuggingFace/TextCompletion/HuggingFaceTextCompletion.cs b/dotnet/src/Connectors/Connectors.AI.HuggingFace/TextCompletion/HuggingFaceTextCompletion.cs index 84f6576b16b4..22075fdfa581 100644 --- a/dotnet/src/Connectors/Connectors.AI.HuggingFace/TextCompletion/HuggingFaceTextCompletion.cs +++ b/dotnet/src/Connectors/Connectors.AI.HuggingFace/TextCompletion/HuggingFaceTextCompletion.cs @@ -2,12 +2,15 @@ using System; using System.Collections.Generic; +using System.IO; using System.Net.Http; using System.Runtime.CompilerServices; +using System.Text; using System.Text.Json; using System.Threading; using System.Threading.Tasks; using Microsoft.SemanticKernel.AI; +using Microsoft.SemanticKernel.AI.ChatCompletion; using Microsoft.SemanticKernel.AI.TextCompletion; using Microsoft.SemanticKernel.Diagnostics; using Microsoft.SemanticKernel.Services; @@ -106,17 +109,14 @@ public async IAsyncEnumerable GetStreamingChunksAsync( AIRequestSettings? requestSettings = null, [EnumeratorCancellation] CancellationToken cancellationToken = default) { - var choiceIndex = 0; - foreach (var result in await this.ExecuteGetCompletionsAsync(input, cancellationToken).ConfigureAwait(false)) + await foreach(var result in this.InternalGetStreamingChunksAsync(input, cancellationToken).ConfigureAwait(false)) { cancellationToken.ThrowIfCancellationRequested(); - var completion = await result.GetCompletionAsync(cancellationToken).ConfigureAwait(false); - // If the provided T is a string, return the completion as is if (typeof(T) == typeof(string)) { - yield return (T)(object)completion; + yield return (T)(object)result.Token.Text; continue; } @@ -124,11 +124,8 @@ public async IAsyncEnumerable GetStreamingChunksAsync( if (typeof(T) == typeof(StreamingTextResultChunk) || typeof(T) == typeof(StreamingResultChunk)) { - yield return (T)(object)new StreamingTextResultChunk(completion, choiceIndex); - continue; + yield return (T)(object)result; } - - choiceIndex++; } } @@ -166,6 +163,47 @@ private async Task> ExecuteGetCompletionsAsync(string return completionResponse.ConvertAll(c => new TextCompletionResult(c)); } + private async IAsyncEnumerable InternalGetStreamingChunksAsync(string text, [EnumeratorCancellation] CancellationToken cancellationToken) + { + var completionRequest = new TextCompletionRequest + { + Input = text, + Stream = true + }; + + using var httpRequestMessage = HttpRequest.CreatePostRequest(this.GetRequestUri(), completionRequest); + + httpRequestMessage.Headers.Add("User-Agent", Telemetry.HttpUserAgent); + if (!string.IsNullOrEmpty(this._apiKey)) + { + httpRequestMessage.Headers.Add("Authorization", $"Bearer {this._apiKey}"); + } + + using var response = await this._httpClient.SendWithSuccessCheckAsync(httpRequestMessage, cancellationToken).ConfigureAwait(false); + using var stream = await response.Content.ReadAsStreamAndTranslateExceptionAsync().ConfigureAwait(false); + using var reader = new StreamReader(stream); + + const string ServerEventPayloadPrefix = "data:"; + while (!reader.EndOfStream) + { + var body = await reader.ReadLineAsync().ConfigureAwait(false); + + if (body.StartsWith(ServerEventPayloadPrefix, StringComparison.Ordinal)) + { + body = body.Substring(ServerEventPayloadPrefix.Length); + } + + if (string.IsNullOrWhiteSpace(body)) + { + continue; + } + + JsonElement chunkObject = JsonSerializer.Deserialize(body); + + yield return new StreamingTextResultChunk("hugging-text-stream=chunk", chunkObject); + } + } + /// /// Retrieves the request URI based on the provided endpoint and model information. /// diff --git a/dotnet/src/Connectors/Connectors.AI.HuggingFace/TextCompletion/StreamingTextResultChunk.cs b/dotnet/src/Connectors/Connectors.AI.HuggingFace/TextCompletion/StreamingTextResultChunk.cs index 47f4d7b4c882..cf2278367eed 100644 --- a/dotnet/src/Connectors/Connectors.AI.HuggingFace/TextCompletion/StreamingTextResultChunk.cs +++ b/dotnet/src/Connectors/Connectors.AI.HuggingFace/TextCompletion/StreamingTextResultChunk.cs @@ -1,35 +1,52 @@ // Copyright (c) Microsoft. All rights reserved. using System.Text; +using System.Text.Json; +using System.Text.Json.Serialization; using Microsoft.SemanticKernel.AI; namespace Microsoft.SemanticKernel.Connectors.AI.HuggingFace.TextCompletion; /// -/// Streaming text result update. +/// StreamResponse class in /// public class StreamingTextResultChunk : StreamingResultChunk { /// - public override string Type => "huggingface_text_update"; + public override string Type { get; } /// public override int ChoiceIndex { get; } /// - /// Text associated to the update + /// Complete generated text + /// Only available when the generation is finished /// - public string Content { get; } + public string? GeneratedText { get; set; } + + /// + /// Optional Generation details + /// Only available when the generation is finished + /// + public string? Details { get; set; } + + /// + /// Token details + /// + public TokenChunkModel Token { get; set; } /// /// Create a new instance of the class. /// - /// Text update - /// Index of the choice - public StreamingTextResultChunk(string text, int choiceIndex) + /// Type of the chunk + /// JsonElement representation of the chunk + public StreamingTextResultChunk(string type, JsonElement jsonChunk) : base(jsonChunk) { - this.ChoiceIndex = choiceIndex; - this.Content = text; + this.Type = type; + this.ChoiceIndex = 0; + this.GeneratedText = jsonChunk.GetProperty("generated_text").GetString(); + this.Details = jsonChunk.GetProperty("details").GetString(); + this.Token = JsonSerializer.Deserialize(jsonChunk.GetProperty("token").GetRawText())!; } /// @@ -41,6 +58,37 @@ public override byte[] ToByteArray() /// public override string ToString() { - return this.Content; + return this.Token.Text; + } + + /// + /// Token class in + /// + public record TokenChunkModel + { + /// + /// Id of the token + /// + [JsonPropertyName("id")] + public int Id { get; set; } + + /// + /// Text associated to the Token + /// + [JsonPropertyName("text")] + public string Text { get; set; } + + /// + /// Log probability of the token + /// + [JsonPropertyName("logprob")] + public decimal LogProb { get; set; } + + /// + /// Is the token a special token? + /// Can be used to ignore tokens when concatenating + /// + [JsonPropertyName("special")] + public bool Special { get; set; } } } diff --git a/dotnet/src/Connectors/Connectors.AI.HuggingFace/TextCompletion/TextCompletionRequest.cs b/dotnet/src/Connectors/Connectors.AI.HuggingFace/TextCompletion/TextCompletionRequest.cs index a57ca56bb037..cdc6dd73535a 100644 --- a/dotnet/src/Connectors/Connectors.AI.HuggingFace/TextCompletion/TextCompletionRequest.cs +++ b/dotnet/src/Connectors/Connectors.AI.HuggingFace/TextCompletion/TextCompletionRequest.cs @@ -16,4 +16,10 @@ public sealed class TextCompletionRequest /// [JsonPropertyName("inputs")] public string Input { get; set; } = string.Empty; + + /// + /// Enable streaming + /// + [JsonPropertyName("stream")] + public bool Stream { get; set; } = false; } diff --git a/dotnet/src/InternalUtilities/src/Http/HttpClientExtensions.cs b/dotnet/src/InternalUtilities/src/Http/HttpClientExtensions.cs index 964bef838399..95eea0fb295a 100644 --- a/dotnet/src/InternalUtilities/src/Http/HttpClientExtensions.cs +++ b/dotnet/src/InternalUtilities/src/Http/HttpClientExtensions.cs @@ -2,6 +2,7 @@ using System.Net; using System.Net.Http; +using System.Text; using System.Threading; using System.Threading.Tasks; using Microsoft.SemanticKernel.Diagnostics; diff --git a/dotnet/src/InternalUtilities/src/Http/HttpRequest.cs b/dotnet/src/InternalUtilities/src/Http/HttpRequest.cs index 206c6e18ae77..f190aad49c86 100644 --- a/dotnet/src/InternalUtilities/src/Http/HttpRequest.cs +++ b/dotnet/src/InternalUtilities/src/Http/HttpRequest.cs @@ -40,6 +40,7 @@ private static HttpRequestMessage CreateRequest(HttpMethod method, Uri url, obje private static HttpContent? CreateJsonContent(object? payload) { HttpContent? content = null; + /* if (payload is not null) { byte[] utf8Bytes = payload is string s ? @@ -48,6 +49,11 @@ private static HttpRequestMessage CreateRequest(HttpMethod method, Uri url, obje content = new ByteArrayContent(utf8Bytes); content.Headers.ContentType = new MediaTypeHeaderValue("application/json") { CharSet = "utf-8" }; + }*/ + + if (payload is not null) + { + return new StringContent(JsonSerializer.Serialize(payload, s_jsonSerializerOptions), Encoding.UTF8, "application/json"); } return content; diff --git a/dotnet/src/SemanticKernel.Abstractions/AI/StreamingResultChunk.cs b/dotnet/src/SemanticKernel.Abstractions/AI/StreamingResultChunk.cs index 0fc6ec72d920..9b7de021fd32 100644 --- a/dotnet/src/SemanticKernel.Abstractions/AI/StreamingResultChunk.cs +++ b/dotnet/src/SemanticKernel.Abstractions/AI/StreamingResultChunk.cs @@ -57,7 +57,7 @@ public abstract class StreamingResultChunk /// Initializes a new instance of the class. /// /// Inner result chunk object reference - protected StreamingResultChunk(object? innerResultChunk = null) + protected StreamingResultChunk(object? innerResultChunk) { this.InnerResultChunk = innerResultChunk; } diff --git a/dotnet/src/SemanticKernel.Core/Functions/NativeFunction.cs b/dotnet/src/SemanticKernel.Core/Functions/NativeFunction.cs index 383b7a5027d1..a21b635f121b 100644 --- a/dotnet/src/SemanticKernel.Core/Functions/NativeFunction.cs +++ b/dotnet/src/SemanticKernel.Core/Functions/NativeFunction.cs @@ -245,7 +245,7 @@ private delegate ValueTask ImplementationFunc( SKContext context, CancellationToken cancellationToken); - private delegate IAsyncEnumerable ImplementationStreamingFunc( + private delegate IAsyncEnumerable ImplementationStreamingFunc( ITextCompletion? textCompletion, AIRequestSettings? requestSettingsk, SKContext context, @@ -351,7 +351,7 @@ ValueTask Function(ITextCompletion? text, AIRequestSettings? req } // Create the streaming func - async IAsyncEnumerable StreamingFunction( + async IAsyncEnumerable StreamingFunction( ITextCompletion? text, AIRequestSettings? requestSettings, SKContext context, @@ -393,7 +393,7 @@ async IAsyncEnumerable StreamingFunction( { object currentItem = currentProperty.GetValue(asyncEnumerator); - yield return new StreamingNativeResultUpdate(currentItem); + yield return new StreamingNativeResultChunk(currentItem); } } else @@ -408,7 +408,7 @@ async IAsyncEnumerable StreamingFunction( // The enumeration will only return if there's actually a result. if (functionResult.Value is not null) { - yield return new StreamingNativeResultUpdate(functionResult.Value); + yield return new StreamingNativeResultChunk(functionResult.Value); } } } diff --git a/dotnet/src/SemanticKernel.Core/Functions/StreamingNativeResultUpdate.cs b/dotnet/src/SemanticKernel.Core/Functions/StreamingNativeResultUpdate.cs index ce292d60c672..32f1ce6b4ccc 100644 --- a/dotnet/src/SemanticKernel.Core/Functions/StreamingNativeResultUpdate.cs +++ b/dotnet/src/SemanticKernel.Core/Functions/StreamingNativeResultUpdate.cs @@ -11,7 +11,7 @@ namespace Microsoft.SemanticKernel; /// /// Native function streaming result update. /// -public sealed class StreamingNativeResultUpdate : StreamingResultChunk +public sealed class StreamingNativeResultChunk : StreamingResultChunk { /// public override string Type => "native_result_update"; @@ -32,6 +32,7 @@ public override byte[] ToByteArray() return (byte[])this.Value; } + // By default if a native value is not Byte[] we output the UTF8 string representation of the value return Encoding.UTF8.GetBytes(this.Value?.ToString()); } @@ -42,11 +43,11 @@ public override string ToString() } /// - /// Initializes a new instance of the class. + /// Initializes a new instance of the class. /// - /// Underlying object that represents the update - public StreamingNativeResultUpdate(object update) + /// Underlying object that represents the chunk + public StreamingNativeResultChunk(object innerResultChunk) : base(innerResultChunk) { - this.Value = update; + this.Value = innerResultChunk; } } diff --git a/dotnet/src/SemanticKernel.Core/KernelExtensions.cs b/dotnet/src/SemanticKernel.Core/KernelExtensions.cs index a4040eb20051..fa3e9c6b4a4e 100644 --- a/dotnet/src/SemanticKernel.Core/KernelExtensions.cs +++ b/dotnet/src/SemanticKernel.Core/KernelExtensions.cs @@ -6,6 +6,7 @@ using System.Threading; using System.Threading.Tasks; using Microsoft.Extensions.Logging; +using Microsoft.SemanticKernel.AI; using Microsoft.SemanticKernel.Diagnostics; using Microsoft.SemanticKernel.Orchestration; @@ -167,4 +168,34 @@ public static Task RunAsync( Verify.NotNull(kernel); return kernel.RunAsync(new ContextVariables(input), cancellationToken, pipeline); } + + /// + /// Run a function in streaming mode. + /// + /// Target kernel + /// Target function to run + /// Input to process + /// The to monitor for cancellation requests. + /// Result of the function composition + public static IAsyncEnumerable RunStreamingAsync(this IKernel kernel, ISKFunction skFunction, string input, CancellationToken cancellationToken) + => kernel.RunStreamingAsync(skFunction, new ContextVariables(input), cancellationToken); + + /// + /// Run a function in streaming mode. + /// + /// Target kernel + /// Target function to run + /// Input to process + /// Result of the function composition + public static IAsyncEnumerable RunStreamingAsync(this IKernel kernel, ISKFunction skFunction, string input) + => kernel.RunStreamingAsync(skFunction, new ContextVariables(input), CancellationToken.None); + + /// + /// Run a function in streaming mode. + /// + /// Target kernel + /// Target function to run + /// Result of the function composition + public static IAsyncEnumerable RunStreamingAsync(this IKernel kernel, ISKFunction skFunction) + => kernel.RunStreamingAsync(skFunction, null, CancellationToken.None); } From 66a8a85bc6c61fd21b011b7a0466f425b55f4402 Mon Sep 17 00:00:00 2001 From: Roger Barreto <19890735+RogerBarreto@users.noreply.github.com> Date: Mon, 20 Nov 2023 18:21:58 +0000 Subject: [PATCH 22/46] Warning fix --- .../TextCompletion/HuggingFaceTextCompletion.cs | 4 +--- dotnet/src/InternalUtilities/src/Http/HttpClientExtensions.cs | 1 - dotnet/src/InternalUtilities/src/Http/HttpRequest.cs | 1 - 3 files changed, 1 insertion(+), 5 deletions(-) diff --git a/dotnet/src/Connectors/Connectors.AI.HuggingFace/TextCompletion/HuggingFaceTextCompletion.cs b/dotnet/src/Connectors/Connectors.AI.HuggingFace/TextCompletion/HuggingFaceTextCompletion.cs index 22075fdfa581..a833b1565429 100644 --- a/dotnet/src/Connectors/Connectors.AI.HuggingFace/TextCompletion/HuggingFaceTextCompletion.cs +++ b/dotnet/src/Connectors/Connectors.AI.HuggingFace/TextCompletion/HuggingFaceTextCompletion.cs @@ -5,12 +5,10 @@ using System.IO; using System.Net.Http; using System.Runtime.CompilerServices; -using System.Text; using System.Text.Json; using System.Threading; using System.Threading.Tasks; using Microsoft.SemanticKernel.AI; -using Microsoft.SemanticKernel.AI.ChatCompletion; using Microsoft.SemanticKernel.AI.TextCompletion; using Microsoft.SemanticKernel.Diagnostics; using Microsoft.SemanticKernel.Services; @@ -109,7 +107,7 @@ public async IAsyncEnumerable GetStreamingChunksAsync( AIRequestSettings? requestSettings = null, [EnumeratorCancellation] CancellationToken cancellationToken = default) { - await foreach(var result in this.InternalGetStreamingChunksAsync(input, cancellationToken).ConfigureAwait(false)) + await foreach (var result in this.InternalGetStreamingChunksAsync(input, cancellationToken).ConfigureAwait(false)) { cancellationToken.ThrowIfCancellationRequested(); diff --git a/dotnet/src/InternalUtilities/src/Http/HttpClientExtensions.cs b/dotnet/src/InternalUtilities/src/Http/HttpClientExtensions.cs index 95eea0fb295a..964bef838399 100644 --- a/dotnet/src/InternalUtilities/src/Http/HttpClientExtensions.cs +++ b/dotnet/src/InternalUtilities/src/Http/HttpClientExtensions.cs @@ -2,7 +2,6 @@ using System.Net; using System.Net.Http; -using System.Text; using System.Threading; using System.Threading.Tasks; using Microsoft.SemanticKernel.Diagnostics; diff --git a/dotnet/src/InternalUtilities/src/Http/HttpRequest.cs b/dotnet/src/InternalUtilities/src/Http/HttpRequest.cs index f190aad49c86..daa4b778d861 100644 --- a/dotnet/src/InternalUtilities/src/Http/HttpRequest.cs +++ b/dotnet/src/InternalUtilities/src/Http/HttpRequest.cs @@ -2,7 +2,6 @@ using System; using System.Net.Http; -using System.Net.Http.Headers; using System.Text; using System.Text.Json; using Microsoft.SemanticKernel.Text; From 89a7f584c5156b4acb7f05d9b15e6e13ef435d99 Mon Sep 17 00:00:00 2001 From: Roger Barreto <19890735+RogerBarreto@users.noreply.github.com> Date: Mon, 20 Nov 2023 18:22:42 +0000 Subject: [PATCH 23/46] Remove commented code --- dotnet/src/InternalUtilities/src/Http/HttpRequest.cs | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/dotnet/src/InternalUtilities/src/Http/HttpRequest.cs b/dotnet/src/InternalUtilities/src/Http/HttpRequest.cs index daa4b778d861..1743e5054de5 100644 --- a/dotnet/src/InternalUtilities/src/Http/HttpRequest.cs +++ b/dotnet/src/InternalUtilities/src/Http/HttpRequest.cs @@ -39,17 +39,6 @@ private static HttpRequestMessage CreateRequest(HttpMethod method, Uri url, obje private static HttpContent? CreateJsonContent(object? payload) { HttpContent? content = null; - /* - if (payload is not null) - { - byte[] utf8Bytes = payload is string s ? - Encoding.UTF8.GetBytes(s) : - JsonSerializer.SerializeToUtf8Bytes(payload, s_jsonSerializerOptions); - - content = new ByteArrayContent(utf8Bytes); - content.Headers.ContentType = new MediaTypeHeaderValue("application/json") { CharSet = "utf-8" }; - }*/ - if (payload is not null) { return new StringContent(JsonSerializer.Serialize(payload, s_jsonSerializerOptions), Encoding.UTF8, "application/json"); From aa76bb6e8d25b8c6dbdd989086271741def7a922 Mon Sep 17 00:00:00 2001 From: Roger Barreto <19890735+RogerBarreto@users.noreply.github.com> Date: Mon, 20 Nov 2023 18:25:52 +0000 Subject: [PATCH 24/46] Nullable Warning fix --- .../TextCompletion/HuggingFaceTextCompletion.cs | 2 +- .../TextCompletion/StreamingTextResultChunk.cs | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/dotnet/src/Connectors/Connectors.AI.HuggingFace/TextCompletion/HuggingFaceTextCompletion.cs b/dotnet/src/Connectors/Connectors.AI.HuggingFace/TextCompletion/HuggingFaceTextCompletion.cs index a833b1565429..981d238aca93 100644 --- a/dotnet/src/Connectors/Connectors.AI.HuggingFace/TextCompletion/HuggingFaceTextCompletion.cs +++ b/dotnet/src/Connectors/Connectors.AI.HuggingFace/TextCompletion/HuggingFaceTextCompletion.cs @@ -114,7 +114,7 @@ public async IAsyncEnumerable GetStreamingChunksAsync( // If the provided T is a string, return the completion as is if (typeof(T) == typeof(string)) { - yield return (T)(object)result.Token.Text; + yield return (T)(object)(result.Token?.Text ?? string.Empty); continue; } diff --git a/dotnet/src/Connectors/Connectors.AI.HuggingFace/TextCompletion/StreamingTextResultChunk.cs b/dotnet/src/Connectors/Connectors.AI.HuggingFace/TextCompletion/StreamingTextResultChunk.cs index cf2278367eed..af13c3821b82 100644 --- a/dotnet/src/Connectors/Connectors.AI.HuggingFace/TextCompletion/StreamingTextResultChunk.cs +++ b/dotnet/src/Connectors/Connectors.AI.HuggingFace/TextCompletion/StreamingTextResultChunk.cs @@ -58,7 +58,7 @@ public override byte[] ToByteArray() /// public override string ToString() { - return this.Token.Text; + return this.Token?.Text ?? string.Empty; } /// @@ -76,7 +76,7 @@ public record TokenChunkModel /// Text associated to the Token /// [JsonPropertyName("text")] - public string Text { get; set; } + public string? Text { get; set; } /// /// Log probability of the token From d2e9d19d263bf48140e1cbe83a604bb1076e18dd Mon Sep 17 00:00:00 2001 From: Roger Barreto <19890735+RogerBarreto@users.noreply.github.com> Date: Tue, 21 Nov 2023 12:05:35 +0000 Subject: [PATCH 25/46] Adding generic support and moving all the non generic to extension --- docs/decisions/0023-kernel-streaming.md | 2 +- .../Example72_KernelStreaming.cs | 11 ++--- .../Extensions/ISKFunctionExtensions.cs | 22 ++++++++++ .../Functions/ISKFunction.cs | 2 +- .../Functions/InstrumentedSKFunction.cs | 6 +-- .../Functions/SKFunctionFromMethod.cs | 33 ++++++++++++++- .../Functions/SKFunctionFromPrompt.cs | 16 +++++--- .../SemanticKernel.Core/KernelExtensions.cs | 40 ++++++++++++++++--- .../src/SemanticKernel.Core/Planning/Plan.cs | 2 +- 9 files changed, 111 insertions(+), 23 deletions(-) diff --git a/docs/decisions/0023-kernel-streaming.md b/docs/decisions/0023-kernel-streaming.md index c48dcf16e383..61528a942ee7 100644 --- a/docs/decisions/0023-kernel-streaming.md +++ b/docs/decisions/0023-kernel-streaming.md @@ -106,7 +106,7 @@ public class ChatMessageChunk : StreamingResultChunk interface ITextCompletion + IChatCompletion { IAsyncEnumerable GetStreamingChunksAsync(); - // Garanteed abstraction to be used by the ISKFunction.RunStreamingAsync() + // Guaranteed abstraction to be used by the ISKFunction.RunStreamingAsync() IAsyncEnumerable GetStreamingChunksAsync(); // Throw exception if T is not supported diff --git a/dotnet/samples/KernelSyntaxExamples/Example72_KernelStreaming.cs b/dotnet/samples/KernelSyntaxExamples/Example72_KernelStreaming.cs index 7e857d22401a..7b2451d498c3 100644 --- a/dotnet/samples/KernelSyntaxExamples/Example72_KernelStreaming.cs +++ b/dotnet/samples/KernelSyntaxExamples/Example72_KernelStreaming.cs @@ -9,6 +9,7 @@ using System.Threading; using System.Threading.Tasks; using Microsoft.SemanticKernel; +using Microsoft.SemanticKernel.AI; using Microsoft.SemanticKernel.Connectors.AI.OpenAI; using Microsoft.SemanticKernel.Connectors.AI.OpenAI.AzureSdk; using Microsoft.SemanticKernel.Orchestration; @@ -68,7 +69,7 @@ Be creative and be funny. Let your imagination run wild. Console.WriteLine("\n=== Semantic Function - Streaming ===\n"); // Streaming can be of any type depending on the underlying service the function is using. - await foreach (var update in kernel.RunStreamingAsync(excuseFunction, new ContextVariables("I missed the F1 final race"), cancellationToken)) + await foreach (var update in kernel.RunStreamingAsync(excuseFunction, "I missed the F1 final race", cancellationToken)) { // You will be always able to know the type of the update by checking the Type property. if (update.Type == "openai_chat_message_update" && update is StreamingChatResultChunk chatUpdate) @@ -102,7 +103,7 @@ private static async Task NativeFunctionStreamingValueTypeAsync(Kernel kernel, I Console.WriteLine("\n\n=== Native Streaming Functions - Streaming (Value Type) ===\n"); // Native string value type streaming function - await foreach (var update in kernel.RunStreamingAsync(plugin["MyValueTypeStreamingNativeFunction"], new ContextVariables("My Value Type Streaming Function Input"), cancellationToken)) + await foreach (var update in kernel.RunStreamingAsync(plugin["MyValueTypeStreamingNativeFunction"], "My Value Type Streaming Function Input", cancellationToken)) { if (update.Type == "native_result_update" && update is StreamingNativeResultChunk nativeUpdate) { @@ -116,7 +117,7 @@ private static async Task NativeFunctionStreamingComplexTypeAsync(Kernel kernel, Console.WriteLine("\n\n=== Native Streaming Functions - Streaming (Complex Type) ===\n"); // Native complex type streaming function - await foreach (var update in kernel.RunStreamingAsync(plugin["MyComplexTypeStreamingNativeFunction"], new ContextVariables("My Complex Type Streaming Function Input"), cancellationToken)) + await foreach (var update in kernel.RunStreamingAsync(plugin["MyComplexTypeStreamingNativeFunction"], "My Complex Type Streaming Function Input", cancellationToken)) { // the complex type will be available thru the Value property of the native update abstraction. if (update.Type == "native_result_update" && update is StreamingNativeResultChunk nativeUpdate && nativeUpdate.Value is MyStreamingBlock myComplexType) @@ -131,7 +132,7 @@ private static async Task NativeFunctionValueTypeAsync(Kernel kernel, ISKPlugin Console.WriteLine("\n=== Native Non-Streaming Functions - Streaming (Value Type) ===\n"); // Native functions that don't support streaming and return value types can be executed with the streaming API's but the behavior will not be streamlike. - await foreach (var update in kernel.RunStreamingAsync(plugin["MyValueTypeNativeFunction"], new ContextVariables("My Value Type Non Streaming Function Input"), cancellationToken)) + await foreach (var update in kernel.RunStreamingAsync(plugin["MyValueTypeNativeFunction"], "My Value Type Non Streaming Function Input", cancellationToken)) { // the complex type will be available thru the Value property of the native update abstraction. if (update.Type == "native_result_update" && update is StreamingNativeResultChunk nativeUpdate) @@ -146,7 +147,7 @@ private static async Task NativeFunctionComplexTypeAsync(Kernel kernel, ISKPlugi Console.WriteLine("\n=== Native Non-Streaming Functions - Streaming (Complex Type) ===\n"); // Native functions that don't support streaming and return complex types can be executed with the streaming API's but the behavior will not be streamlike. - await foreach (var update in kernel.RunStreamingAsync(plugin["MyComplexTypeNativeFunction"], new ContextVariables("My Complex Type Non Streaming Function Input"), cancellationToken)) + await foreach (var update in kernel.RunStreamingAsync(plugin["MyComplexTypeNativeFunction"], "My Complex Type Non Streaming Function Input", cancellationToken)) { // the complex type will be available thru the Value property of the native update abstraction. if (update.Type == "native_result_update" && update is StreamingNativeResultChunk nativeUpdate && nativeUpdate.Value is MyCustomType myComplexType) diff --git a/dotnet/src/Experimental/Assistants/Extensions/ISKFunctionExtensions.cs b/dotnet/src/Experimental/Assistants/Extensions/ISKFunctionExtensions.cs index 31cddf5986b5..356e772cfaaa 100644 --- a/dotnet/src/Experimental/Assistants/Extensions/ISKFunctionExtensions.cs +++ b/dotnet/src/Experimental/Assistants/Extensions/ISKFunctionExtensions.cs @@ -3,8 +3,11 @@ using System; using System.Collections.Generic; using System.Linq; +using System.Threading; using Json.More; +using Microsoft.SemanticKernel.AI; using Microsoft.SemanticKernel.Experimental.Assistants.Models; +using Microsoft.SemanticKernel.Orchestration; namespace Microsoft.SemanticKernel.Experimental.Assistants.Extensions; @@ -67,6 +70,25 @@ public static ToolModel ToToolModel(this ISKFunction function, string pluginName return payload; } + /// + /// Invoke the in streaming mode. + /// + /// Target function + /// The kernel + /// SK context + /// LLM completion settings (for semantic functions only) + /// The to monitor for cancellation requests. The default is . + /// A asynchronous list of streaming result chunks + public static IAsyncEnumerable InvokeStreamingAsync( + this ISKFunction function, + Kernel kernel, + SKContext context, + AIRequestSettings? requestSettings = null, + CancellationToken cancellationToken = default) + { + return function.InvokeStreamingAsync(kernel, context, requestSettings, cancellationToken); + } + private static string ConvertType(Type? type) { if (type == null || type == typeof(string)) diff --git a/dotnet/src/SemanticKernel.Abstractions/Functions/ISKFunction.cs b/dotnet/src/SemanticKernel.Abstractions/Functions/ISKFunction.cs index e06d683bca74..1b1e90d4070c 100644 --- a/dotnet/src/SemanticKernel.Abstractions/Functions/ISKFunction.cs +++ b/dotnet/src/SemanticKernel.Abstractions/Functions/ISKFunction.cs @@ -59,7 +59,7 @@ Task InvokeAsync( /// LLM completion settings (for semantic functions only) /// The to monitor for cancellation requests. The default is . /// A asynchronous list of streaming result chunks - IAsyncEnumerable InvokeStreamingAsync( + IAsyncEnumerable InvokeStreamingAsync( Kernel kernel, SKContext context, AIRequestSettings? requestSettings = null, diff --git a/dotnet/src/SemanticKernel.Core/Functions/InstrumentedSKFunction.cs b/dotnet/src/SemanticKernel.Core/Functions/InstrumentedSKFunction.cs index 172f4f3eb441..3153fd3f6171 100644 --- a/dotnet/src/SemanticKernel.Core/Functions/InstrumentedSKFunction.cs +++ b/dotnet/src/SemanticKernel.Core/Functions/InstrumentedSKFunction.cs @@ -59,13 +59,13 @@ public InstrumentedSKFunction(ISKFunction function, ILoggerFactory? loggerFactor public IEnumerable ModelSettings => this._function.ModelSettings; /// - public IAsyncEnumerable InvokeStreamingAsync( + public IAsyncEnumerable InvokeStreamingAsync( Kernel kernel, SKContext context, AIRequestSettings? requestSettings = null, CancellationToken cancellationToken = default) => this.StreamingInvokeWithInstrumentationAsync( - () => this._function.InvokeStreamingAsync(kernel, context, requestSettings, cancellationToken)); + () => this._function.InvokeStreamingAsync(kernel, context, requestSettings, cancellationToken)); #region private ================================================================================ @@ -73,7 +73,7 @@ public IAsyncEnumerable InvokeStreamingAsync( /// Wrapper for instrumentation to be used in multiple invocation places. /// /// Delegate to instrument. - private async IAsyncEnumerable StreamingInvokeWithInstrumentationAsync(Func> func) + private async IAsyncEnumerable StreamingInvokeWithInstrumentationAsync(Func> func) { using var activity = s_activitySource.StartActivity(this.Name); diff --git a/dotnet/src/SemanticKernel.Core/Functions/SKFunctionFromMethod.cs b/dotnet/src/SemanticKernel.Core/Functions/SKFunctionFromMethod.cs index 5499321321d9..6b4e191ac915 100644 --- a/dotnet/src/SemanticKernel.Core/Functions/SKFunctionFromMethod.cs +++ b/dotnet/src/SemanticKernel.Core/Functions/SKFunctionFromMethod.cs @@ -151,11 +151,42 @@ async Task ISKFunction.InvokeAsync( } /// - public async IAsyncEnumerable InvokeStreamingAsync( + public async IAsyncEnumerable InvokeStreamingAsync( Kernel kernel, SKContext context, AIRequestSettings? requestSettings = null, [EnumeratorCancellation] CancellationToken cancellationToken = default) + { + await foreach (var chunk in this.InvokeStreamingAsync(kernel, context, requestSettings, cancellationToken).ConfigureAwait(false)) + { + if (typeof(T).IsSubclassOf(typeof(StreamingResultChunk)) || typeof(T) == typeof(StreamingResultChunk)) + { + yield return (T)(object)chunk; + continue; + } + + if (chunk is StreamingNativeResultChunk nativeChunk) + { + yield return (T)nativeChunk.Value; + } + + throw new NotSupportedException($"Streaming result chunk of type {typeof(T)} is not supported."); + } + } + + /// + /// Invoke the in streaming mode. + /// + /// The kernel + /// SK context + /// LLM completion settings (for semantic functions only) + /// The to monitor for cancellation requests. The default is . + /// A asynchronous list of streaming result chunks + public async IAsyncEnumerable InvokeStreamingAsync( + Kernel kernel, + SKContext context, + AIRequestSettings? requestSettings = null, + [EnumeratorCancellation] CancellationToken cancellationToken = default) { // Invoke pre hook, and stop if skipping requested. this.CallFunctionInvoking(context); diff --git a/dotnet/src/SemanticKernel.Core/Functions/SKFunctionFromPrompt.cs b/dotnet/src/SemanticKernel.Core/Functions/SKFunctionFromPrompt.cs index 8a6a1433d165..43f0665599e5 100644 --- a/dotnet/src/SemanticKernel.Core/Functions/SKFunctionFromPrompt.cs +++ b/dotnet/src/SemanticKernel.Core/Functions/SKFunctionFromPrompt.cs @@ -187,7 +187,7 @@ public async Task InvokeAsync( } } - public async IAsyncEnumerable InvokeStreamingAsync( + public async IAsyncEnumerable InvokeStreamingAsync( Kernel kernel, SKContext context, AIRequestSettings? requestSettings = null, @@ -200,14 +200,18 @@ public async IAsyncEnumerable InvokeStreamingAsync( } StringBuilder fullCompletion = new(); - await foreach (StreamingResultChunk update in textCompletion.GetStreamingChunksAsync(renderedPrompt, requestSettings ?? defaultRequestSettings, cancellationToken).ConfigureAwait(false)) + await foreach (T genericChunk in textCompletion.GetStreamingChunksAsync(renderedPrompt, requestSettings ?? defaultRequestSettings, cancellationToken).ConfigureAwait(false)) { - fullCompletion.Append(update); + fullCompletion.Append(genericChunk); - // This currently is needed as plans need use it to update the variables created after the stream ends. - update.Context = context; + // Check if genericChunk is a StreamingResultChunk and update the context + if (genericChunk is StreamingResultChunk resultChunk) + { + // This currently is needed so plans can get the context from the chunks to update the variables generated when the stream ends. + resultChunk.Context = context; + } - yield return update; + yield return genericChunk; } // Update the result with the completion diff --git a/dotnet/src/SemanticKernel.Core/KernelExtensions.cs b/dotnet/src/SemanticKernel.Core/KernelExtensions.cs index 1872bd336d09..bf71f3ac5546 100644 --- a/dotnet/src/SemanticKernel.Core/KernelExtensions.cs +++ b/dotnet/src/SemanticKernel.Core/KernelExtensions.cs @@ -622,7 +622,7 @@ private static bool IsRepeatRequested(ISKFunction skFunction, SKContext context, /// Input to process /// The to monitor for cancellation requests. /// Result of the function composition - public static async IAsyncEnumerable RunStreamingAsync(this Kernel kernel, ISKFunction skFunction, ContextVariables? variables, [EnumeratorCancellation] CancellationToken cancellationToken) + public static async IAsyncEnumerable RunStreamingAsync(this Kernel kernel, ISKFunction skFunction, ContextVariables? variables, [EnumeratorCancellation] CancellationToken cancellationToken) { var context = kernel.CreateNewContext(variables); var repeatRequested = false; @@ -634,7 +634,7 @@ public static async IAsyncEnumerable RunStreamingAsync(thi repeatRequested = false; var functionDetails = skFunction.GetMetadata(); - await foreach (StreamingResultChunk update in skFunction.InvokeStreamingAsync(kernel, context, null, cancellationToken).ConfigureAwait(false)) + await foreach (T update in skFunction.InvokeStreamingAsync(kernel, context, null, cancellationToken).ConfigureAwait(false)) { cancellationToken.ThrowIfCancellationRequested(); @@ -668,7 +668,7 @@ public static async IAsyncEnumerable RunStreamingAsync(thi /// The to monitor for cancellation requests. /// Result of the function composition public static IAsyncEnumerable RunStreamingAsync(this Kernel kernel, ISKFunction skFunction, string input, CancellationToken cancellationToken) - => kernel.RunStreamingAsync(skFunction, new ContextVariables(input), cancellationToken); + => kernel.RunStreamingAsync(skFunction, new ContextVariables(input), cancellationToken); /// /// Run a function in streaming mode. @@ -678,7 +678,7 @@ public static IAsyncEnumerable RunStreamingAsync(this Kern /// Input to process /// Result of the function composition public static IAsyncEnumerable RunStreamingAsync(this Kernel kernel, ISKFunction skFunction, string input) - => kernel.RunStreamingAsync(skFunction, new ContextVariables(input), CancellationToken.None); + => kernel.RunStreamingAsync(skFunction, new ContextVariables(input), CancellationToken.None); /// /// Run a function in streaming mode. @@ -687,5 +687,35 @@ public static IAsyncEnumerable RunStreamingAsync(this Kern /// Target function to run /// Result of the function composition public static IAsyncEnumerable RunStreamingAsync(this Kernel kernel, ISKFunction skFunction) - => kernel.RunStreamingAsync(skFunction, (ContextVariables?)null, CancellationToken.None); + => kernel.RunStreamingAsync(skFunction, (ContextVariables?)null, CancellationToken.None); + + /// + /// Run a function in streaming mode. + /// + /// Target kernel + /// Target function to run + /// Input to process + /// The to monitor for cancellation requests. + /// Result of the function composition + public static IAsyncEnumerable RunStreamingAsync(this Kernel kernel, ISKFunction skFunction, string input, CancellationToken cancellationToken) + => kernel.RunStreamingAsync(skFunction, new ContextVariables(input), cancellationToken); + + /// + /// Run a function in streaming mode. + /// + /// Target kernel + /// Target function to run + /// Input to process + /// Result of the function composition + public static IAsyncEnumerable RunStreamingAsync(this Kernel kernel, ISKFunction skFunction, string input) + => kernel.RunStreamingAsync(skFunction, new ContextVariables(input), CancellationToken.None); + + /// + /// Run a function in streaming mode. + /// + /// Target kernel + /// Target function to run + /// Result of the function composition + public static IAsyncEnumerable RunStreamingAsync(this Kernel kernel, ISKFunction skFunction) + => kernel.RunStreamingAsync(skFunction, (ContextVariables?)null, CancellationToken.None); } diff --git a/dotnet/src/SemanticKernel.Core/Planning/Plan.cs b/dotnet/src/SemanticKernel.Core/Planning/Plan.cs index 599cea609d3e..1b8d56a31edd 100644 --- a/dotnet/src/SemanticKernel.Core/Planning/Plan.cs +++ b/dotnet/src/SemanticKernel.Core/Planning/Plan.cs @@ -356,7 +356,7 @@ public async Task InvokeAsync( } /// - public IAsyncEnumerable InvokeStreamingAsync( + public IAsyncEnumerable InvokeStreamingAsync( Kernel kernel, SKContext context, AIRequestSettings? requestSettings = null, From e5cc5043e2c6346b90f9dbe03c5f049dbd41e761 Mon Sep 17 00:00:00 2001 From: Roger Barreto <19890735+RogerBarreto@users.noreply.github.com> Date: Tue, 21 Nov 2023 13:17:49 +0000 Subject: [PATCH 26/46] Waring fixes --- .../samples/KernelSyntaxExamples/Example72_KernelStreaming.cs | 2 -- 1 file changed, 2 deletions(-) diff --git a/dotnet/samples/KernelSyntaxExamples/Example72_KernelStreaming.cs b/dotnet/samples/KernelSyntaxExamples/Example72_KernelStreaming.cs index 7b2451d498c3..14acf1be322c 100644 --- a/dotnet/samples/KernelSyntaxExamples/Example72_KernelStreaming.cs +++ b/dotnet/samples/KernelSyntaxExamples/Example72_KernelStreaming.cs @@ -9,10 +9,8 @@ using System.Threading; using System.Threading.Tasks; using Microsoft.SemanticKernel; -using Microsoft.SemanticKernel.AI; using Microsoft.SemanticKernel.Connectors.AI.OpenAI; using Microsoft.SemanticKernel.Connectors.AI.OpenAI.AzureSdk; -using Microsoft.SemanticKernel.Orchestration; using RepoUtils; #pragma warning disable RCS1110 // Declare type inside namespace. From c0dd34fbedf8f4967c7edf1ce579e1fda27977f5 Mon Sep 17 00:00:00 2001 From: Roger Barreto <19890735+RogerBarreto@users.noreply.github.com> Date: Tue, 21 Nov 2023 16:46:35 +0000 Subject: [PATCH 27/46] Adding exception handling for streaming errors --- .../Functions/InstrumentedSKFunction.cs | 35 ++++++++++++-- .../Functions/SKFunctionFromMethod.cs | 46 +++++++++++++++++++ .../Functions/SKFunctionFromPrompt.cs | 39 ++++++++++++---- 3 files changed, 106 insertions(+), 14 deletions(-) diff --git a/dotnet/src/SemanticKernel.Core/Functions/InstrumentedSKFunction.cs b/dotnet/src/SemanticKernel.Core/Functions/InstrumentedSKFunction.cs index 3153fd3f6171..6b299a45a4bd 100644 --- a/dotnet/src/SemanticKernel.Core/Functions/InstrumentedSKFunction.cs +++ b/dotnet/src/SemanticKernel.Core/Functions/InstrumentedSKFunction.cs @@ -76,17 +76,42 @@ public IAsyncEnumerable InvokeStreamingAsync( private async IAsyncEnumerable StreamingInvokeWithInstrumentationAsync(Func> func) { using var activity = s_activitySource.StartActivity(this.Name); - this._logger.LogInformation("Function invoking streaming"); TagList tags = new() { { "sk.function.name", this.Name } }; long startingTimestamp = Stopwatch.GetTimestamp(); StringBuilder fullResult = new(); - await foreach (var update in func().ConfigureAwait(false)) + + IAsyncEnumerator enumerator = func().GetAsyncEnumerator(); + bool moreChunks; + do { - fullResult.Append(update); - yield return update; - } + T? genericChunk = default; + + try + { + moreChunks = await enumerator.MoveNextAsync().ConfigureAwait(false); + if (moreChunks) + { + genericChunk = enumerator.Current; + fullResult.Append(genericChunk); + } + } + catch (Exception ex) + { + if (this._logger.IsEnabled(LogLevel.Error)) + { + this._logger.LogError(ex, "Function invocation failed: {Message}", ex.Message); + tags.Add("error.type", ex.GetType().FullName); + } + throw; + } + + if (moreChunks && genericChunk is not null) + { + yield return genericChunk; + } + } while (moreChunks); if (this._logger.IsEnabled(LogLevel.Trace)) { diff --git a/dotnet/src/SemanticKernel.Core/Functions/SKFunctionFromMethod.cs b/dotnet/src/SemanticKernel.Core/Functions/SKFunctionFromMethod.cs index 069957425698..712c8aa15c01 100644 --- a/dotnet/src/SemanticKernel.Core/Functions/SKFunctionFromMethod.cs +++ b/dotnet/src/SemanticKernel.Core/Functions/SKFunctionFromMethod.cs @@ -172,6 +172,52 @@ public async IAsyncEnumerable InvokeStreamingAsync( throw new NotSupportedException($"Streaming result chunk of type {typeof(T)} is not supported."); } + + IAsyncEnumerator enumerator = this.InvokeStreamingAsync(kernel, context, requestSettings, cancellationToken).GetAsyncEnumerator(cancellationToken); + + T? genericChunk = default; + bool moreItems; + + // Manually handling the enumeration to properly log any exception + do + { + try + { + moreItems = await enumerator.MoveNextAsync().ConfigureAwait(false); + + if (moreItems) + { + var chunk = enumerator.Current; + + if (typeof(T).IsSubclassOf(typeof(StreamingResultChunk)) || typeof(T) == typeof(StreamingResultChunk)) + { + genericChunk = (T)(object)chunk; + continue; + } + + if (chunk is StreamingNativeResultChunk nativeChunk) + { + genericChunk = (T)nativeChunk.Value; + continue; + } + + throw new NotSupportedException($"Streaming result chunk of type {typeof(T)} is not supported."); + } + } + catch (Exception e) + { + if (this._logger.IsEnabled(LogLevel.Error)) + { + this._logger.LogError(e, "Function {Name} execution failed: {Error}", this.Name, e.Message); + } + throw; + } + + if (moreItems && genericChunk is not null) + { + yield return genericChunk; + } + } while (moreItems); } /// diff --git a/dotnet/src/SemanticKernel.Core/Functions/SKFunctionFromPrompt.cs b/dotnet/src/SemanticKernel.Core/Functions/SKFunctionFromPrompt.cs index 4b754e3a3490..5ef313d922bd 100644 --- a/dotnet/src/SemanticKernel.Core/Functions/SKFunctionFromPrompt.cs +++ b/dotnet/src/SemanticKernel.Core/Functions/SKFunctionFromPrompt.cs @@ -200,19 +200,40 @@ public async IAsyncEnumerable InvokeStreamingAsync( } StringBuilder fullCompletion = new(); - await foreach (T genericChunk in textCompletion.GetStreamingChunksAsync(renderedPrompt, requestSettings ?? defaultRequestSettings, cancellationToken).ConfigureAwait(false)) - { - fullCompletion.Append(genericChunk); + IAsyncEnumerator enumerator = textCompletion.GetStreamingChunksAsync(renderedPrompt, requestSettings ?? defaultRequestSettings, cancellationToken).GetAsyncEnumerator(cancellationToken); - // Check if genericChunk is a StreamingResultChunk and update the context - if (genericChunk is StreamingResultChunk resultChunk) + // Manually handling the enumeration to properly log any exception + bool moreItems; + do + { + T? genericChunk = default; + try + { + moreItems = await enumerator.MoveNextAsync().ConfigureAwait(false); + if (moreItems) + { + genericChunk = enumerator.Current; + fullCompletion.Append(genericChunk); + + // Check if genericChunk is a StreamingResultChunk and update the context + if (genericChunk is StreamingResultChunk resultChunk) + { + // This currently is needed so plans can get the context from the chunks to update the variables generated when the stream ends. + resultChunk.Context = context; + } + } + } + catch (Exception ex) when (!ex.IsCriticalException()) { - // This currently is needed so plans can get the context from the chunks to update the variables generated when the stream ends. - resultChunk.Context = context; + this._logger?.LogError(ex, "Semantic function {Name} execution failed with error {Error}", this.Name, ex.Message); + throw; } - yield return genericChunk; - } + if (moreItems && genericChunk is not null) + { + yield return genericChunk; + } + } while (moreItems); // Update the result with the completion context.Variables.Update(fullCompletion.ToString()); From e5c8a9fc1e5c15c5da9169df4da3d612389124e0 Mon Sep 17 00:00:00 2001 From: Roger Barreto <19890735+RogerBarreto@users.noreply.github.com> Date: Wed, 22 Nov 2023 09:29:15 +0000 Subject: [PATCH 28/46] Removed unused code + Renamed native to method + Example simplification --- .../Example72_KernelStreaming.cs | 52 ++++------- .../AzureOpenAIChatCompletionWithData.cs | 18 ---- .../TextCompletion/OpenAITextCompletion.cs | 19 ---- .../Orchestration/StreamingFunctionResult.cs | 91 ------------------- .../Functions/SKFunctionFromMethod.cs | 12 +-- ...pdate.cs => StreamingMethodResultChunk.cs} | 12 +-- 6 files changed, 31 insertions(+), 173 deletions(-) delete mode 100644 dotnet/src/SemanticKernel.Abstractions/Orchestration/StreamingFunctionResult.cs rename dotnet/src/SemanticKernel.Core/Functions/{StreamingNativeResultUpdate.cs => StreamingMethodResultChunk.cs} (74%) diff --git a/dotnet/samples/KernelSyntaxExamples/Example72_KernelStreaming.cs b/dotnet/samples/KernelSyntaxExamples/Example72_KernelStreaming.cs index 14acf1be322c..8c5656ace38a 100644 --- a/dotnet/samples/KernelSyntaxExamples/Example72_KernelStreaming.cs +++ b/dotnet/samples/KernelSyntaxExamples/Example72_KernelStreaming.cs @@ -25,7 +25,7 @@ public static class Example72_KernelStreaming /// /// Show how to combine multiple prompt template factories. /// - public static async Task RunAsync(CancellationToken cancellationToken) + public static async Task RunAsync() { string apiKey = TestConfiguration.AzureOpenAI.ApiKey; string chatDeploymentName = TestConfiguration.AzureOpenAI.ChatDeploymentName; @@ -46,28 +46,14 @@ public static async Task RunAsync(CancellationToken cancellationToken) apiKey: apiKey) .Build(); - // Function defined using few-shot design pattern - string promptTemplate = @" -Generate a creative reason or excuse for the given event. -Be creative and be funny. Let your imagination run wild. - -Event: I am running late. -Excuse: I was being held ransom by giraffe gangsters. - -Event: I haven't been to the gym for a year -Excuse: I've been too busy training my pet dragon. - -Event: {{$input}} -"; - - var excuseFunction = kernel.CreateFunctionFromPrompt(promptTemplate, new OpenAIRequestSettings() { MaxTokens = 100, Temperature = 0.4, TopP = 1 }); + var funyParagraphFunction = kernel.CreateFunctionFromPrompt("Write a funny paragraph about streaming", new OpenAIRequestSettings() { MaxTokens = 100, Temperature = 0.4, TopP = 1 }); var roleDisplayed = false; Console.WriteLine("\n=== Semantic Function - Streaming ===\n"); // Streaming can be of any type depending on the underlying service the function is using. - await foreach (var update in kernel.RunStreamingAsync(excuseFunction, "I missed the F1 final race", cancellationToken)) + await foreach (var update in kernel.RunStreamingAsync(funyParagraphFunction)) { // You will be always able to know the type of the update by checking the Type property. if (update.Type == "openai_chat_message_update" && update is StreamingChatResultChunk chatUpdate) @@ -87,68 +73,68 @@ Be creative and be funny. Let your imagination run wild. var plugin = kernel.ImportPluginFromObject(new MyNativePlugin(), "MyNativePlugin"); - await NativeFunctionStreamingValueTypeAsync(kernel, plugin, cancellationToken); + await NativeFunctionStreamingValueTypeAsync(kernel, plugin); - await NativeFunctionStreamingComplexTypeAsync(kernel, plugin, cancellationToken); + await NativeFunctionStreamingComplexTypeAsync(kernel, plugin); - await NativeFunctionValueTypeAsync(kernel, plugin, cancellationToken); + await NativeFunctionValueTypeAsync(kernel, plugin); - await NativeFunctionComplexTypeAsync(kernel, plugin, cancellationToken); + await NativeFunctionComplexTypeAsync(kernel, plugin); } - private static async Task NativeFunctionStreamingValueTypeAsync(Kernel kernel, ISKPlugin plugin, CancellationToken cancellationToken) + private static async Task NativeFunctionStreamingValueTypeAsync(Kernel kernel, ISKPlugin plugin) { Console.WriteLine("\n\n=== Native Streaming Functions - Streaming (Value Type) ===\n"); // Native string value type streaming function - await foreach (var update in kernel.RunStreamingAsync(plugin["MyValueTypeStreamingNativeFunction"], "My Value Type Streaming Function Input", cancellationToken)) + await foreach (var update in kernel.RunStreamingAsync(plugin["MyValueTypeStreamingNativeFunction"], "My Value Type Streaming Function Input")) { - if (update.Type == "native_result_update" && update is StreamingNativeResultChunk nativeUpdate) + if (update.Type == "method_result_chunk" && update is StreamingMethodResultChunk nativeUpdate) { Console.Write(nativeUpdate.Value); } } } - private static async Task NativeFunctionStreamingComplexTypeAsync(Kernel kernel, ISKPlugin plugin, CancellationToken cancellationToken) + private static async Task NativeFunctionStreamingComplexTypeAsync(Kernel kernel, ISKPlugin plugin) { Console.WriteLine("\n\n=== Native Streaming Functions - Streaming (Complex Type) ===\n"); // Native complex type streaming function - await foreach (var update in kernel.RunStreamingAsync(plugin["MyComplexTypeStreamingNativeFunction"], "My Complex Type Streaming Function Input", cancellationToken)) + await foreach (var update in kernel.RunStreamingAsync(plugin["MyComplexTypeStreamingNativeFunction"], "My Complex Type Streaming Function Input")) { // the complex type will be available thru the Value property of the native update abstraction. - if (update.Type == "native_result_update" && update is StreamingNativeResultChunk nativeUpdate && nativeUpdate.Value is MyStreamingBlock myComplexType) + if (update.Type == "method_result_chunk" && update is StreamingMethodResultChunk nativeUpdate && nativeUpdate.Value is MyStreamingBlock myComplexType) { Console.WriteLine(Encoding.UTF8.GetString(myComplexType.Content)); } } } - private static async Task NativeFunctionValueTypeAsync(Kernel kernel, ISKPlugin plugin, CancellationToken cancellationToken) + private static async Task NativeFunctionValueTypeAsync(Kernel kernel, ISKPlugin plugin) { Console.WriteLine("\n=== Native Non-Streaming Functions - Streaming (Value Type) ===\n"); // Native functions that don't support streaming and return value types can be executed with the streaming API's but the behavior will not be streamlike. - await foreach (var update in kernel.RunStreamingAsync(plugin["MyValueTypeNativeFunction"], "My Value Type Non Streaming Function Input", cancellationToken)) + await foreach (var update in kernel.RunStreamingAsync(plugin["MyValueTypeNativeFunction"], "My Value Type Non Streaming Function Input")) { // the complex type will be available thru the Value property of the native update abstraction. - if (update.Type == "native_result_update" && update is StreamingNativeResultChunk nativeUpdate) + if (update.Type == "method_result_chunk" && update is StreamingMethodResultChunk nativeUpdate) { Console.WriteLine(nativeUpdate.Value); } } } - private static async Task NativeFunctionComplexTypeAsync(Kernel kernel, ISKPlugin plugin, CancellationToken cancellationToken) + private static async Task NativeFunctionComplexTypeAsync(Kernel kernel, ISKPlugin plugin) { Console.WriteLine("\n=== Native Non-Streaming Functions - Streaming (Complex Type) ===\n"); // Native functions that don't support streaming and return complex types can be executed with the streaming API's but the behavior will not be streamlike. - await foreach (var update in kernel.RunStreamingAsync(plugin["MyComplexTypeNativeFunction"], "My Complex Type Non Streaming Function Input", cancellationToken)) + await foreach (var update in kernel.RunStreamingAsync(plugin["MyComplexTypeNativeFunction"], "My Complex Type Non Streaming Function Input")) { // the complex type will be available thru the Value property of the native update abstraction. - if (update.Type == "native_result_update" && update is StreamingNativeResultChunk nativeUpdate && nativeUpdate.Value is MyCustomType myComplexType) + if (update.Type == "method_result_chunk" && update is StreamingMethodResultChunk nativeUpdate && nativeUpdate.Value is MyCustomType myComplexType) { Console.WriteLine($"Text: {myComplexType.Text}, Number: {myComplexType.Number}"); } diff --git a/dotnet/src/Connectors/Connectors.AI.OpenAI/ChatCompletionWithData/AzureOpenAIChatCompletionWithData.cs b/dotnet/src/Connectors/Connectors.AI.OpenAI/ChatCompletionWithData/AzureOpenAIChatCompletionWithData.cs index 755c7e66fd5e..0cbd4e050ea4 100644 --- a/dotnet/src/Connectors/Connectors.AI.OpenAI/ChatCompletionWithData/AzureOpenAIChatCompletionWithData.cs +++ b/dotnet/src/Connectors/Connectors.AI.OpenAI/ChatCompletionWithData/AzureOpenAIChatCompletionWithData.cs @@ -136,24 +136,6 @@ public async IAsyncEnumerable GetStreamingChunksAsync(stri } } - /// - public async IAsyncEnumerable GetStringStreamingUpdatesAsync(string input, AIRequestSettings? requestSettings = null, [EnumeratorCancellation] CancellationToken cancellationToken = default) - { - await foreach (var update in this.GetStreamingChunksAsync(input, requestSettings, cancellationToken).ConfigureAwait(false)) - { - yield return update.ToString(); - } - } - - /// - public async IAsyncEnumerable GetByteStreamingUpdatesAsync(string input, AIRequestSettings? requestSettings = null, [EnumeratorCancellation] CancellationToken cancellationToken = default) - { - await foreach (var update in this.GetStreamingChunksAsync(input, requestSettings, cancellationToken).ConfigureAwait(false)) - { - yield return update.ToByteArray(); - } - } - /// public async IAsyncEnumerable GetStreamingChunksAsync( string input, diff --git a/dotnet/src/Connectors/Connectors.AI.OpenAI/TextCompletion/OpenAITextCompletion.cs b/dotnet/src/Connectors/Connectors.AI.OpenAI/TextCompletion/OpenAITextCompletion.cs index dedf8fa0b65a..b21c5581ecbd 100644 --- a/dotnet/src/Connectors/Connectors.AI.OpenAI/TextCompletion/OpenAITextCompletion.cs +++ b/dotnet/src/Connectors/Connectors.AI.OpenAI/TextCompletion/OpenAITextCompletion.cs @@ -2,7 +2,6 @@ using System.Collections.Generic; using System.Net.Http; -using System.Runtime.CompilerServices; using System.Threading; using System.Threading.Tasks; using Microsoft.Extensions.Logging; @@ -68,24 +67,6 @@ public IAsyncEnumerable GetStreamingChunksAsync(string inp return this.InternalGetTextStreamingUpdatesAsync(input, requestSettings, cancellationToken); } - /// - public async IAsyncEnumerable GetStringStreamingUpdatesAsync(string input, AIRequestSettings? requestSettings = null, [EnumeratorCancellation] CancellationToken cancellationToken = default) - { - await foreach (var update in this.GetStreamingChunksAsync(input, requestSettings, cancellationToken).ConfigureAwait(false)) - { - yield return update.ToString(); - } - } - - /// - public async IAsyncEnumerable GetByteStreamingUpdatesAsync(string input, AIRequestSettings? requestSettings = null, [EnumeratorCancellation] CancellationToken cancellationToken = default) - { - await foreach (var update in this.GetStreamingChunksAsync(input, requestSettings, cancellationToken).ConfigureAwait(false)) - { - yield return update.ToByteArray(); - } - } - /// public IAsyncEnumerable GetStreamingChunksAsync(string input, AIRequestSettings? requestSettings = null, CancellationToken cancellationToken = default) { diff --git a/dotnet/src/SemanticKernel.Abstractions/Orchestration/StreamingFunctionResult.cs b/dotnet/src/SemanticKernel.Abstractions/Orchestration/StreamingFunctionResult.cs deleted file mode 100644 index b2fd02a46348..000000000000 --- a/dotnet/src/SemanticKernel.Abstractions/Orchestration/StreamingFunctionResult.cs +++ /dev/null @@ -1,91 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System.Collections.Generic; -using System.Threading; -using Microsoft.SemanticKernel.AI; - -namespace Microsoft.SemanticKernel.Orchestration; - -/// -/// Function result after execution. -/// -public sealed class StreamingFunctionResult : IAsyncEnumerable -{ - internal Dictionary? _metadata; - private readonly IAsyncEnumerable _streamingResult; - - /// - /// Name of executed function. - /// - public string FunctionName { get; internal set; } - - /// - /// Name of the plugin containing the function. - /// - public string PluginName { get; internal set; } - - /// - /// Metadata for storing additional information about function execution result. - /// - public Dictionary Metadata - { - get => this._metadata ??= new(); - internal set => this._metadata = value; - } - - /// - /// Internal object reference. (Breaking glass). - /// Each connector will have its own internal object representing the result. - /// - /// - /// The usage of this property is considered "unsafe". Use it only if strictly necessary. - /// - public object? InnerFunctionResult { get; private set; } = null; - - /// - /// Instance of used by the function. - /// - internal SKContext Context { get; private set; } - - /// - /// Initializes a new instance of the class. - /// - /// Name of executed function. - /// Name of the plugin containing the function. - /// Instance of to pass in function pipeline. - /// IAsyncEnumerable reference to iterate - /// Inner function result object reference - public StreamingFunctionResult(string functionName, string pluginName, SKContext context, IAsyncEnumerable streamingResult, object? innerFunctionResult) - { - this.FunctionName = functionName; - this.PluginName = pluginName; - this.Context = context; - this._streamingResult = streamingResult; - this.InnerFunctionResult = innerFunctionResult; - } - - /// - /// Get typed value from metadata. - /// - public bool TryGetMetadataValue(string key, out T value) - { - if (this._metadata is { } metadata && - metadata.TryGetValue(key, out object? valueObject) && - valueObject is T typedValue) - { - value = typedValue; - return true; - } - - value = default!; - return false; - } - - /// - /// Get the enumerator for the streaming result. - /// - /// Cancellation token - /// Streaming chunk asynchronous enumerator - public IAsyncEnumerator GetAsyncEnumerator(CancellationToken cancellationToken = default) - => this._streamingResult.GetAsyncEnumerator(cancellationToken); -} diff --git a/dotnet/src/SemanticKernel.Core/Functions/SKFunctionFromMethod.cs b/dotnet/src/SemanticKernel.Core/Functions/SKFunctionFromMethod.cs index 712c8aa15c01..b58ac4261888 100644 --- a/dotnet/src/SemanticKernel.Core/Functions/SKFunctionFromMethod.cs +++ b/dotnet/src/SemanticKernel.Core/Functions/SKFunctionFromMethod.cs @@ -165,7 +165,7 @@ public async IAsyncEnumerable InvokeStreamingAsync( continue; } - if (chunk is StreamingNativeResultChunk nativeChunk) + if (chunk is StreamingMethodResultChunk nativeChunk) { yield return (T)nativeChunk.Value; } @@ -195,7 +195,7 @@ public async IAsyncEnumerable InvokeStreamingAsync( continue; } - if (chunk is StreamingNativeResultChunk nativeChunk) + if (chunk is StreamingMethodResultChunk nativeChunk) { genericChunk = (T)nativeChunk.Value; continue; @@ -321,7 +321,7 @@ private delegate ValueTask ImplementationFunc( SKContext context, CancellationToken cancellationToken); - private delegate IAsyncEnumerable ImplementationStreamingFunc( + private delegate IAsyncEnumerable ImplementationStreamingFunc( ITextCompletion? textCompletion, AIRequestSettings? requestSettingsk, Kernel kernel, @@ -429,7 +429,7 @@ ValueTask Function(ITextCompletion? text, AIRequestSettings? req } // Create the streaming func - async IAsyncEnumerable StreamingFunction( + async IAsyncEnumerable StreamingFunction( ITextCompletion? text, AIRequestSettings? requestSettings, Kernel kernel, @@ -472,7 +472,7 @@ async IAsyncEnumerable StreamingFunction( { object currentItem = currentProperty.GetValue(asyncEnumerator); - yield return new StreamingNativeResultChunk(currentItem); + yield return new StreamingMethodResultChunk(currentItem); } } else @@ -487,7 +487,7 @@ async IAsyncEnumerable StreamingFunction( // The enumeration will only return if there's actually a result. if (functionResult.Value is not null) { - yield return new StreamingNativeResultChunk(functionResult.Value); + yield return new StreamingMethodResultChunk(functionResult.Value); } } } diff --git a/dotnet/src/SemanticKernel.Core/Functions/StreamingNativeResultUpdate.cs b/dotnet/src/SemanticKernel.Core/Functions/StreamingMethodResultChunk.cs similarity index 74% rename from dotnet/src/SemanticKernel.Core/Functions/StreamingNativeResultUpdate.cs rename to dotnet/src/SemanticKernel.Core/Functions/StreamingMethodResultChunk.cs index 32f1ce6b4ccc..f1cbe790d671 100644 --- a/dotnet/src/SemanticKernel.Core/Functions/StreamingNativeResultUpdate.cs +++ b/dotnet/src/SemanticKernel.Core/Functions/StreamingMethodResultChunk.cs @@ -9,18 +9,18 @@ namespace Microsoft.SemanticKernel; #pragma warning restore IDE0130 /// -/// Native function streaming result update. +/// Method function streaming result chunk. /// -public sealed class StreamingNativeResultChunk : StreamingResultChunk +public sealed class StreamingMethodResultChunk : StreamingResultChunk { /// - public override string Type => "native_result_update"; + public override string Type => "method_result_chunk"; /// public override int ChoiceIndex => 0; /// - /// Native object value that represents the update + /// Method object value that represents the chunk /// public object Value { get; } @@ -43,10 +43,10 @@ public override string ToString() } /// - /// Initializes a new instance of the class. + /// Initializes a new instance of the class. /// /// Underlying object that represents the chunk - public StreamingNativeResultChunk(object innerResultChunk) : base(innerResultChunk) + public StreamingMethodResultChunk(object innerResultChunk) : base(innerResultChunk) { this.Value = innerResultChunk; } From b8570ad17e584bfb8b5be1d3cf468f271ce41b4e Mon Sep 17 00:00:00 2001 From: Roger Barreto <19890735+RogerBarreto@users.noreply.github.com> Date: Wed, 22 Nov 2023 16:39:01 +0000 Subject: [PATCH 29/46] WIP option 1+2 --- docs/decisions/0023-kernel-streaming.md | 189 +++++++++++++++--- .../Example16_CustomLLM.cs | 10 +- .../Example72_KernelStreaming.cs | 10 +- .../HuggingFaceTextCompletion.cs | 8 +- .../StreamingTextResultChunk.cs | 2 +- .../AzureSdk/ClientBase.cs | 20 +- .../AzureSdk/StreamingChatResultChunk.cs | 5 +- .../StreamingChatWithDataResultChunk.cs | 2 +- .../AzureSdk/StreamingTextResultChunk.cs | 2 +- .../AzureOpenAIChatCompletion.cs | 6 +- .../ChatCompletion/OpenAIChatCompletion.cs | 8 +- .../AzureOpenAIChatCompletionWithData.cs | 6 +- .../TextCompletion/AzureTextCompletion.cs | 8 +- .../TextCompletion/OpenAITextCompletion.cs | 8 +- .../Extensions/ISKFunctionExtensions.cs | 4 +- .../KernelSemanticFunctionExtensionsTests.cs | 4 +- .../ChatCompletionExtensions.cs | 15 ++ .../AI/ChatCompletion/IChatCompletion.cs | 14 +- ...mingResultChunk.cs => StreamingContent.cs} | 29 ++- .../AI/TextCompletion/ITextCompletion.cs | 14 +- .../TextCompletionExtensions.cs | 15 ++ .../Functions/SKFunctionFromMethod.cs | 6 +- .../Functions/SKFunctionFromPrompt.cs | 2 +- .../Functions/StreamingMethodResultChunk.cs | 5 +- .../SemanticKernel.Core/KernelExtensions.cs | 12 +- ...redIAIServiceConfigurationProviderTests.cs | 4 +- 26 files changed, 245 insertions(+), 163 deletions(-) rename dotnet/src/SemanticKernel.Abstractions/AI/{StreamingResultChunk.cs => StreamingContent.cs} (75%) diff --git a/docs/decisions/0023-kernel-streaming.md b/docs/decisions/0023-kernel-streaming.md index 61528a942ee7..ab4e130a9312 100644 --- a/docs/decisions/0023-kernel-streaming.md +++ b/docs/decisions/0023-kernel-streaming.md @@ -25,26 +25,6 @@ Needs to be clear for the sk developer when he is attempting to get streaming da 3. The sk developer when using streaming from a model that does not support streaming should still be able to use it with only one streaming update representing the whole data. -## User Experience Goal - -```csharp -//(providing the type at as generic parameter) - -// Getting a Raw Streaming data from Kernel -await foreach(string update in kernel.RunStreamingAsync(function, variables)) - -// Getting a String as Streaming data from Kernel -await foreach(string update in kernel.RunStreamingAsync(function, variables)) - -// Getting a StreamingResultChunk as Streaming data from Kernel -await foreach(StreamingResultChunk update in kernel.RunStreamingAsync(variables, function)) -// OR -await foreach(StreamingResultChunk update in kernel.RunStreamingAsync(function, variables)) // defaults to Generic above) -{ - Console.WriteLine(update); -} -``` - ## Out of Scope - Streaming with plans will not be supported in this phase. Attempting to do so will throw an exception. @@ -67,6 +47,26 @@ If the type is not specified or if the string representation cannot be cast, an If the type specified is `StreamingResultChunk`, `string` or `byte[]` no error will be thrown as the connectors will have interface methods that guarantee the data to be given in at least those types. +## User Experience Goal + +```csharp +//(providing the type at as generic parameter) + +// Getting a Raw Streaming data from Kernel +await foreach(string update in kernel.RunStreamingAsync(function, variables)) + +// Getting a String as Streaming data from Kernel +await foreach(string update in kernel.RunStreamingAsync(function, variables)) + +// Getting a StreamingResultChunk as Streaming data from Kernel +await foreach(StreamingResultChunk update in kernel.RunStreamingAsync(variables, function)) +// OR +await foreach(StreamingResultChunk update in kernel.RunStreamingAsync(function, variables)) // defaults to Generic above) +{ + Console.WriteLine(update); +} +``` + ```csharp // Depending on the underlying function model and connector used the streaming @@ -163,26 +163,149 @@ If NativeFunctions don't return IAsyncEnumerable, the result will be wrapped in 1. If the sk developer wants to use the specialized type of `StreamingResultChunk` he will need to know what the connector is being used to use the correct **StreamingResultChunk extension method** or to provide directly type in ``. 2. Connectors will have greater responsibility to provide the correct type of `StreamingResultChunk` for the connector being used and implementations for both byte[] and string streaming data. -## Decision Outcome +### Option 2 - Dedicated Streaming Interfaces (Returning a Class) -As for now, the only proposed solution is the #1. +This option includes all the suggestions from Option 1 with the only change being that the interfaces now instead of returning an `IAsyncEnumerable` will return `StreamingFunctionResult` which also implements `IAsyncEnumerable` -## More Information +Connectors will have a streaming connector class representation that will have any data related to the request as well as a breaking glass reference to the underlying connector object (response) if needed. + +This abstraction is necessary to provide the extra connector information to the StreamingFunctionResult returned by KernelPromptFunctions. + +## User Experience Goal + +Option 2 Biggest benefit: + +```csharp +// When the caller needs to know more about the streaming he can get the result reference before starting the streaming. +var streamingResult = await kernel.RunStreamingAsync(function); +// Do something with streamingResult properties + +// Consuming the streamingResult: +await foreach(StreamingResultChunk chunk in await streamingResult) +``` + +Using the other operations will be quite similar (extra `await` to get the iteratable streaming reference) if the sk developer don't want to check for the result properties + +````csharp +// Getting a Raw Streaming data from Kernel +await foreach(string update in await kernel.RunStreamingAsync(function, variables)) -Think about generic for the interfaces... +// Getting a String as Streaming data from Kernel +await foreach(string update in await kernel.RunStreamingAsync(function, variables)) -StreamingFunctionResult ... +// Getting a StreamingResultChunk as Streaming data from Kernel +await foreach(StreamingResultChunk update in await kernel.RunStreamingAsync(variables, function)) +// OR +await foreach(StreamingResultChunk update in await kernel.RunStreamingAsync(function, variables)) // defaults to Generic above) +{ + Console.WriteLine(update); +} +``` -Look into Hugging Face Streaming APIs +```csharp -Create a second option after talkin with Stephen. +public sealed class StreamingConnectorResult : IAsyncEnumerable +{ + private readonly IAsyncEnumerable _streamingResultSource; + + /// + /// Internal object reference. (Breaking glass). + /// Each connector will have its own internal object representing the result. + /// + public object? InnerResult { get; private set; } = null; + + /// + /// Initializes a new instance of the class. + /// + /// + /// + public ConnectorAsyncEnumerable(Func> streamingReference, object? innerConnectorResult) + { + this._streamingResultSource = streamingReference.Invoke(); + this.InnerResult = innerConnectorResult; + } +} -Run by Matthew -StreamingAsync vs RunStreamingAsync vs RunStreamingAsync vs StreamAsync -First thing should be a verb +interface ITextCompletion + IChatCompletion +{ + Task> GetStreamingChunksAsync(); + // Guaranteed abstraction to be used by the ISKFunction.RunStreamingAsync() -Add more examples of the API usage for streaming with chunks, for complex scenarios like multiple modals. + Task> GetStreamingChunksAsync(); + // Throw exception if T is not supported + // Initially connectors +} +``` -KDifferent impleme tations + differtnt consumers +StreamingFunctionResult is a class that can store information regarding the result before the stream is consumed as well as any underlying object (breaking glass) that the stream consumes. -I should be able to handle stuff that i don't know. +```csharp +public sealed class StreamingFunctionResult : IAsyncEnumerable +{ + internal Dictionary? _metadata; + private readonly IAsyncEnumerable _streamingResult; + + public string FunctionName { get; internal set; } + public Dictionary Metadata { get; internal set; } + + /// + /// Internal object reference. (Breaking glass). + /// Each connector will have its own internal object representing the result. + /// + public object? InnerResult { get; private set; } = null; + + /// + /// Instance of used by the function. + /// + internal SKContext Context { get; private set; } + + public StreamingFunctionResult(string functionName, SKContext context, Func> streamingResult, object? innerFunctionResult) + { + this.FunctionName = functionName; + this.Context = context; + this._streamingResult = streamingResult.Invoke(); + this.InnerResult = innerFunctionResult; + } +} + +interface ISKFunction +{ + // When the developer provides a T, the Kernel will try to get the streaming data as T + Task> InvokeStreamingAsync(...); + + // Extension generic method to get from type + Task> InvokeStreamingAsync(...); +} + +static class KernelExtensions +{ + public static async Task> RunStreamingAsync(this Kernel kernel, ISKFunction skFunction, ContextVariables? variables, CancellationToken cancellationToken) + { + ... + } +} +``` + +## Pros + +1. All benefits from Option 1 + +2. Having StreamingFunctionResults allow sk developer to know more details about the result before consuming the stream, like: + - Any metadata provided by the underlying API, + - SKContext + - Function Name and Details +3. Experience using the Streaming is quite similar (need an extra await to get the result) to option 1 +4. APIs behave similarly to the non-streaming API (returning a result representation to get the value) + +## Cons + +1. All cons from Option 1 + +2. Added complexity as the IAsyncEnumerable cannot be passed directly in the method result demanding a delegate approach to be adapted inside of the Results that implements the IAsyncEnumerator. + + +## Decision Outcome + +Option 3 +Try to use the Task interface with a dumb StreamingFunctionResult which wraps all the logic + +## More Information +```` diff --git a/dotnet/samples/KernelSyntaxExamples/Example16_CustomLLM.cs b/dotnet/samples/KernelSyntaxExamples/Example16_CustomLLM.cs index 2a48a488bb31..784322e67b3c 100644 --- a/dotnet/samples/KernelSyntaxExamples/Example16_CustomLLM.cs +++ b/dotnet/samples/KernelSyntaxExamples/Example16_CustomLLM.cs @@ -57,9 +57,9 @@ public async IAsyncEnumerable GetByteStreamingUpdatesAsync(string input, } } - public IAsyncEnumerable GetStreamingChunksAsync(string input, AIRequestSettings? requestSettings = null, CancellationToken cancellationToken = default) + public IAsyncEnumerable GetStreamingChunksAsync(string input, AIRequestSettings? requestSettings = null, CancellationToken cancellationToken = default) { - return this.GetStreamingChunksAsync(input, requestSettings, cancellationToken); + return this.GetStreamingContentAsync(input, requestSettings, cancellationToken); } public async IAsyncEnumerable GetStringStreamingUpdatesAsync(string input, AIRequestSettings? requestSettings = null, [EnumeratorCancellation] CancellationToken cancellationToken = default) @@ -70,7 +70,7 @@ public async IAsyncEnumerable GetStringStreamingUpdatesAsync(string inpu } } - public async IAsyncEnumerable GetStreamingChunksAsync(string input, AIRequestSettings? requestSettings = null, [EnumeratorCancellation] CancellationToken cancellationToken = default) + public async IAsyncEnumerable GetStreamingContentAsync(string input, AIRequestSettings? requestSettings = null, [EnumeratorCancellation] CancellationToken cancellationToken = default) { if (typeof(T) == typeof(MyStreamingResultChunk)) { @@ -80,10 +80,8 @@ public async IAsyncEnumerable GetStreamingChunksAsync(string input, AIRequ } } -public class MyStreamingResultChunk : StreamingResultChunk +public class MyStreamingResultChunk : StreamingContent { - public override string Type => "my_text_type"; - public override int ChoiceIndex => 0; public string Content { get; } diff --git a/dotnet/samples/KernelSyntaxExamples/Example72_KernelStreaming.cs b/dotnet/samples/KernelSyntaxExamples/Example72_KernelStreaming.cs index 8c5656ace38a..643895978a68 100644 --- a/dotnet/samples/KernelSyntaxExamples/Example72_KernelStreaming.cs +++ b/dotnet/samples/KernelSyntaxExamples/Example72_KernelStreaming.cs @@ -56,7 +56,7 @@ public static async Task RunAsync() await foreach (var update in kernel.RunStreamingAsync(funyParagraphFunction)) { // You will be always able to know the type of the update by checking the Type property. - if (update.Type == "openai_chat_message_update" && update is StreamingChatResultChunk chatUpdate) + if (update is StreamingChatResultChunk chatUpdate) { if (!roleDisplayed && chatUpdate.Role.HasValue) { @@ -89,7 +89,7 @@ private static async Task NativeFunctionStreamingValueTypeAsync(Kernel kernel, I // Native string value type streaming function await foreach (var update in kernel.RunStreamingAsync(plugin["MyValueTypeStreamingNativeFunction"], "My Value Type Streaming Function Input")) { - if (update.Type == "method_result_chunk" && update is StreamingMethodResultChunk nativeUpdate) + if (update is StreamingMethodResultChunk nativeUpdate) { Console.Write(nativeUpdate.Value); } @@ -104,7 +104,7 @@ private static async Task NativeFunctionStreamingComplexTypeAsync(Kernel kernel, await foreach (var update in kernel.RunStreamingAsync(plugin["MyComplexTypeStreamingNativeFunction"], "My Complex Type Streaming Function Input")) { // the complex type will be available thru the Value property of the native update abstraction. - if (update.Type == "method_result_chunk" && update is StreamingMethodResultChunk nativeUpdate && nativeUpdate.Value is MyStreamingBlock myComplexType) + if (update is StreamingMethodResultChunk nativeUpdate && nativeUpdate.Value is MyStreamingBlock myComplexType) { Console.WriteLine(Encoding.UTF8.GetString(myComplexType.Content)); } @@ -119,7 +119,7 @@ private static async Task NativeFunctionValueTypeAsync(Kernel kernel, ISKPlugin await foreach (var update in kernel.RunStreamingAsync(plugin["MyValueTypeNativeFunction"], "My Value Type Non Streaming Function Input")) { // the complex type will be available thru the Value property of the native update abstraction. - if (update.Type == "method_result_chunk" && update is StreamingMethodResultChunk nativeUpdate) + if (update is StreamingMethodResultChunk nativeUpdate) { Console.WriteLine(nativeUpdate.Value); } @@ -134,7 +134,7 @@ private static async Task NativeFunctionComplexTypeAsync(Kernel kernel, ISKPlugi await foreach (var update in kernel.RunStreamingAsync(plugin["MyComplexTypeNativeFunction"], "My Complex Type Non Streaming Function Input")) { // the complex type will be available thru the Value property of the native update abstraction. - if (update.Type == "method_result_chunk" && update is StreamingMethodResultChunk nativeUpdate && nativeUpdate.Value is MyCustomType myComplexType) + if (update is StreamingMethodResultChunk nativeUpdate && nativeUpdate.Value is MyCustomType myComplexType) { Console.WriteLine($"Text: {myComplexType.Text}, Number: {myComplexType.Number}"); } diff --git a/dotnet/src/Connectors/Connectors.AI.HuggingFace/TextCompletion/HuggingFaceTextCompletion.cs b/dotnet/src/Connectors/Connectors.AI.HuggingFace/TextCompletion/HuggingFaceTextCompletion.cs index 70226dbca70e..f757ceec6d3a 100644 --- a/dotnet/src/Connectors/Connectors.AI.HuggingFace/TextCompletion/HuggingFaceTextCompletion.cs +++ b/dotnet/src/Connectors/Connectors.AI.HuggingFace/TextCompletion/HuggingFaceTextCompletion.cs @@ -93,16 +93,16 @@ public async Task> GetCompletionsAsync( } /// - public IAsyncEnumerable GetStreamingChunksAsync( + public IAsyncEnumerable GetStreamingChunksAsync( string input, AIRequestSettings? requestSettings = null, CancellationToken cancellationToken = default) { - return this.GetStreamingChunksAsync(input, requestSettings, cancellationToken); + return this.GetStreamingContentAsync(input, requestSettings, cancellationToken); } /// - public async IAsyncEnumerable GetStreamingChunksAsync( + public async IAsyncEnumerable GetStreamingContentAsync( string input, AIRequestSettings? requestSettings = null, [EnumeratorCancellation] CancellationToken cancellationToken = default) @@ -120,7 +120,7 @@ public async IAsyncEnumerable GetStreamingChunksAsync( // If the provided T is an specialized class of StreamingResultChunk interface if (typeof(T) == typeof(StreamingTextResultChunk) || - typeof(T) == typeof(StreamingResultChunk)) + typeof(T) == typeof(StreamingContent)) { yield return (T)(object)result; } diff --git a/dotnet/src/Connectors/Connectors.AI.HuggingFace/TextCompletion/StreamingTextResultChunk.cs b/dotnet/src/Connectors/Connectors.AI.HuggingFace/TextCompletion/StreamingTextResultChunk.cs index af13c3821b82..a466c1ccee11 100644 --- a/dotnet/src/Connectors/Connectors.AI.HuggingFace/TextCompletion/StreamingTextResultChunk.cs +++ b/dotnet/src/Connectors/Connectors.AI.HuggingFace/TextCompletion/StreamingTextResultChunk.cs @@ -10,7 +10,7 @@ namespace Microsoft.SemanticKernel.Connectors.AI.HuggingFace.TextCompletion; /// /// StreamResponse class in /// -public class StreamingTextResultChunk : StreamingResultChunk +public class StreamingTextResultChunk : StreamingContent { /// public override string Type { get; } diff --git a/dotnet/src/Connectors/Connectors.AI.OpenAI/AzureSdk/ClientBase.cs b/dotnet/src/Connectors/Connectors.AI.OpenAI/AzureSdk/ClientBase.cs index 4c56f8f21880..249beb69404f 100644 --- a/dotnet/src/Connectors/Connectors.AI.OpenAI/AzureSdk/ClientBase.cs +++ b/dotnet/src/Connectors/Connectors.AI.OpenAI/AzureSdk/ClientBase.cs @@ -154,14 +154,6 @@ private protected async IAsyncEnumerable InternalGetTextStr } } - private protected IAsyncEnumerable InternalGetTextStreamingUpdatesAsync( - string prompt, - AIRequestSettings? requestSettings, - CancellationToken cancellationToken = default) - { - return this.InternalGetTextStreamingUpdatesAsync(prompt, requestSettings, cancellationToken); - } - private protected async IAsyncEnumerable InternalGetTextStreamingUpdatesAsync( string prompt, AIRequestSettings? requestSettings, @@ -192,7 +184,7 @@ private protected async IAsyncEnumerable InternalGetTextStreamingUpdatesAsync // If the provided T is an specialized class of StreamingResultChunk interface if (typeof(T) == typeof(StreamingTextResultChunk) || - typeof(T) == typeof(StreamingResultChunk)) + typeof(T) == typeof(StreamingContent)) { yield return (T)(object)new StreamingTextResultChunk(update, choiceIndex, update); continue; @@ -349,7 +341,7 @@ private protected async IAsyncEnumerable InternalGetChatStreamingUpdatesAsync // If the provided T is an specialized class of StreamingResultChunk interface if (typeof(T) == typeof(StreamingChatResultChunk) || - typeof(T) == typeof(StreamingResultChunk)) + typeof(T) == typeof(StreamingContent)) { yield return (T)(object)new StreamingChatResultChunk(chatMessage, choiceIndex); continue; @@ -361,14 +353,6 @@ private protected async IAsyncEnumerable InternalGetChatStreamingUpdatesAsync } } - private protected IAsyncEnumerable InternalGetChatStreamingUpdatesAsync( - IEnumerable chat, - AIRequestSettings? requestSettings, - CancellationToken cancellationToken = default) - { - return this.InternalGetChatStreamingUpdatesAsync(chat, requestSettings, cancellationToken); - } - /// /// Create a new empty chat instance /// diff --git a/dotnet/src/Connectors/Connectors.AI.OpenAI/AzureSdk/StreamingChatResultChunk.cs b/dotnet/src/Connectors/Connectors.AI.OpenAI/AzureSdk/StreamingChatResultChunk.cs index f1d2065b15df..0699f9334a11 100644 --- a/dotnet/src/Connectors/Connectors.AI.OpenAI/AzureSdk/StreamingChatResultChunk.cs +++ b/dotnet/src/Connectors/Connectors.AI.OpenAI/AzureSdk/StreamingChatResultChunk.cs @@ -11,11 +11,8 @@ namespace Microsoft.SemanticKernel.Connectors.AI.OpenAI.AzureSdk; /// /// Streaming chat result update. /// -public class StreamingChatResultChunk : StreamingResultChunk +public class StreamingChatResultChunk : StreamingContent { - /// - public override string Type => "openai_chat_message_update"; - /// public override int ChoiceIndex { get; } diff --git a/dotnet/src/Connectors/Connectors.AI.OpenAI/AzureSdk/StreamingChatWithDataResultChunk.cs b/dotnet/src/Connectors/Connectors.AI.OpenAI/AzureSdk/StreamingChatWithDataResultChunk.cs index 2705855764a5..4a10db68ad33 100644 --- a/dotnet/src/Connectors/Connectors.AI.OpenAI/AzureSdk/StreamingChatWithDataResultChunk.cs +++ b/dotnet/src/Connectors/Connectors.AI.OpenAI/AzureSdk/StreamingChatWithDataResultChunk.cs @@ -13,7 +13,7 @@ namespace Microsoft.SemanticKernel.Connectors.AI.OpenAI.AzureSdk; /// /// Streaming chat result update. /// -public sealed class StreamingChatWithDataResultChunk : StreamingResultChunk +public sealed class StreamingChatWithDataResultChunk : StreamingContent { /// public override string Type => "openai_chat_message_update"; diff --git a/dotnet/src/Connectors/Connectors.AI.OpenAI/AzureSdk/StreamingTextResultChunk.cs b/dotnet/src/Connectors/Connectors.AI.OpenAI/AzureSdk/StreamingTextResultChunk.cs index 8a42bde0cd4c..3281a2b7fc92 100644 --- a/dotnet/src/Connectors/Connectors.AI.OpenAI/AzureSdk/StreamingTextResultChunk.cs +++ b/dotnet/src/Connectors/Connectors.AI.OpenAI/AzureSdk/StreamingTextResultChunk.cs @@ -8,7 +8,7 @@ namespace Microsoft.SemanticKernel.Connectors.AI.OpenAI.AzureSdk; /// /// Streaming text result update. /// -public class StreamingTextResultChunk : StreamingResultChunk +public class StreamingTextResultChunk : StreamingContent { /// public override string Type => "openai_text_update"; diff --git a/dotnet/src/Connectors/Connectors.AI.OpenAI/ChatCompletion/AzureOpenAIChatCompletion.cs b/dotnet/src/Connectors/Connectors.AI.OpenAI/ChatCompletion/AzureOpenAIChatCompletion.cs index 5af8e8e637eb..98dd3ede77a2 100644 --- a/dotnet/src/Connectors/Connectors.AI.OpenAI/ChatCompletion/AzureOpenAIChatCompletion.cs +++ b/dotnet/src/Connectors/Connectors.AI.OpenAI/ChatCompletion/AzureOpenAIChatCompletion.cs @@ -127,11 +127,7 @@ public Task> GetCompletionsAsync( } /// - public IAsyncEnumerable GetStreamingChunksAsync(string input, AIRequestSettings? requestSettings = null, CancellationToken cancellationToken = default) - => this.GetStreamingChunksAsync(input, requestSettings, cancellationToken); - - /// - public IAsyncEnumerable GetStreamingChunksAsync(string input, AIRequestSettings? requestSettings = null, CancellationToken cancellationToken = default) + public IAsyncEnumerable GetStreamingContentAsync(string input, AIRequestSettings? requestSettings = null, CancellationToken cancellationToken = default) { var chatHistory = this.CreateNewChat(input); return this.InternalGetChatStreamingUpdatesAsync(chatHistory, requestSettings, cancellationToken); diff --git a/dotnet/src/Connectors/Connectors.AI.OpenAI/ChatCompletion/OpenAIChatCompletion.cs b/dotnet/src/Connectors/Connectors.AI.OpenAI/ChatCompletion/OpenAIChatCompletion.cs index 85647b0ac164..db976ea8f1d4 100644 --- a/dotnet/src/Connectors/Connectors.AI.OpenAI/ChatCompletion/OpenAIChatCompletion.cs +++ b/dotnet/src/Connectors/Connectors.AI.OpenAI/ChatCompletion/OpenAIChatCompletion.cs @@ -103,13 +103,7 @@ public Task> GetCompletionsAsync( } /// - public IAsyncEnumerable GetStreamingChunksAsync(string input, AIRequestSettings? requestSettings = null, CancellationToken cancellationToken = default) - { - return this.GetStreamingChunksAsync(input, requestSettings, cancellationToken); - } - - /// - public IAsyncEnumerable GetStreamingChunksAsync(string input, AIRequestSettings? requestSettings = null, CancellationToken cancellationToken = default) + public IAsyncEnumerable GetStreamingContentAsync(string input, AIRequestSettings? requestSettings = null, CancellationToken cancellationToken = default) { var chatHistory = this.CreateNewChat(input); return this.InternalGetChatStreamingUpdatesAsync(chatHistory, requestSettings, cancellationToken); diff --git a/dotnet/src/Connectors/Connectors.AI.OpenAI/ChatCompletionWithData/AzureOpenAIChatCompletionWithData.cs b/dotnet/src/Connectors/Connectors.AI.OpenAI/ChatCompletionWithData/AzureOpenAIChatCompletionWithData.cs index 0cbd4e050ea4..e4cd2e8ef317 100644 --- a/dotnet/src/Connectors/Connectors.AI.OpenAI/ChatCompletionWithData/AzureOpenAIChatCompletionWithData.cs +++ b/dotnet/src/Connectors/Connectors.AI.OpenAI/ChatCompletionWithData/AzureOpenAIChatCompletionWithData.cs @@ -119,7 +119,7 @@ public async IAsyncEnumerable GetStreamingCompletionsAsync } /// - public async IAsyncEnumerable GetStreamingChunksAsync(string input, AIRequestSettings? requestSettings = null, [EnumeratorCancellation] CancellationToken cancellationToken = default) + public async IAsyncEnumerable GetStreamingChunksAsync(string input, AIRequestSettings? requestSettings = null, [EnumeratorCancellation] CancellationToken cancellationToken = default) { OpenAIRequestSettings chatRequestSettings = OpenAIRequestSettings.FromRequestSettings(requestSettings); @@ -137,7 +137,7 @@ public async IAsyncEnumerable GetStreamingChunksAsync(stri } /// - public async IAsyncEnumerable GetStreamingChunksAsync( + public async IAsyncEnumerable GetStreamingContentAsync( string input, AIRequestSettings? requestSettings = null, [EnumeratorCancellation] CancellationToken cancellationToken = default) @@ -297,7 +297,7 @@ private async IAsyncEnumerable GetChatStreamingUpdatesAsync(HttpResponseMe { // If the provided T is an specialized class of StreamingResultChunk interface if (typeof(T) == typeof(StreamingChatResultChunk) || - typeof(T) == typeof(StreamingResultChunk)) + typeof(T) == typeof(StreamingContent)) { yield return (T)(object)new StreamingChatWithDataResultChunk(choice, choice.Index); continue; diff --git a/dotnet/src/Connectors/Connectors.AI.OpenAI/TextCompletion/AzureTextCompletion.cs b/dotnet/src/Connectors/Connectors.AI.OpenAI/TextCompletion/AzureTextCompletion.cs index 49ea409ab0e5..e09ba66bc411 100644 --- a/dotnet/src/Connectors/Connectors.AI.OpenAI/TextCompletion/AzureTextCompletion.cs +++ b/dotnet/src/Connectors/Connectors.AI.OpenAI/TextCompletion/AzureTextCompletion.cs @@ -100,13 +100,7 @@ public Task> GetCompletionsAsync( } /// - public IAsyncEnumerable GetStreamingChunksAsync(string input, AIRequestSettings? requestSettings = null, CancellationToken cancellationToken = default) - { - return this.InternalGetTextStreamingUpdatesAsync(input, requestSettings, cancellationToken); - } - - /// - public IAsyncEnumerable GetStreamingChunksAsync(string input, AIRequestSettings? requestSettings = null, CancellationToken cancellationToken = default) + public IAsyncEnumerable GetStreamingContentAsync(string input, AIRequestSettings? requestSettings = null, CancellationToken cancellationToken = default) { return this.InternalGetTextStreamingUpdatesAsync(input, requestSettings, cancellationToken); } diff --git a/dotnet/src/Connectors/Connectors.AI.OpenAI/TextCompletion/OpenAITextCompletion.cs b/dotnet/src/Connectors/Connectors.AI.OpenAI/TextCompletion/OpenAITextCompletion.cs index b21c5581ecbd..bd098bc50a90 100644 --- a/dotnet/src/Connectors/Connectors.AI.OpenAI/TextCompletion/OpenAITextCompletion.cs +++ b/dotnet/src/Connectors/Connectors.AI.OpenAI/TextCompletion/OpenAITextCompletion.cs @@ -62,13 +62,7 @@ public Task> GetCompletionsAsync( } /// - public IAsyncEnumerable GetStreamingChunksAsync(string input, AIRequestSettings? requestSettings = null, CancellationToken cancellationToken = default) - { - return this.InternalGetTextStreamingUpdatesAsync(input, requestSettings, cancellationToken); - } - - /// - public IAsyncEnumerable GetStreamingChunksAsync(string input, AIRequestSettings? requestSettings = null, CancellationToken cancellationToken = default) + public IAsyncEnumerable GetStreamingContentAsync(string input, AIRequestSettings? requestSettings = null, CancellationToken cancellationToken = default) { return this.InternalGetTextStreamingUpdatesAsync(input, requestSettings, cancellationToken); } diff --git a/dotnet/src/Experimental/Assistants/Extensions/ISKFunctionExtensions.cs b/dotnet/src/Experimental/Assistants/Extensions/ISKFunctionExtensions.cs index 356e772cfaaa..5654bedec177 100644 --- a/dotnet/src/Experimental/Assistants/Extensions/ISKFunctionExtensions.cs +++ b/dotnet/src/Experimental/Assistants/Extensions/ISKFunctionExtensions.cs @@ -79,14 +79,14 @@ public static ToolModel ToToolModel(this ISKFunction function, string pluginName /// LLM completion settings (for semantic functions only) /// The to monitor for cancellation requests. The default is . /// A asynchronous list of streaming result chunks - public static IAsyncEnumerable InvokeStreamingAsync( + public static IAsyncEnumerable InvokeStreamingAsync( this ISKFunction function, Kernel kernel, SKContext context, AIRequestSettings? requestSettings = null, CancellationToken cancellationToken = default) { - return function.InvokeStreamingAsync(kernel, context, requestSettings, cancellationToken); + return function.InvokeStreamingAsync(kernel, context, requestSettings, cancellationToken); } private static string ConvertType(Type? type) diff --git a/dotnet/src/IntegrationTests/Extensions/KernelSemanticFunctionExtensionsTests.cs b/dotnet/src/IntegrationTests/Extensions/KernelSemanticFunctionExtensionsTests.cs index 45a20bad368a..8875e323f4bd 100644 --- a/dotnet/src/IntegrationTests/Extensions/KernelSemanticFunctionExtensionsTests.cs +++ b/dotnet/src/IntegrationTests/Extensions/KernelSemanticFunctionExtensionsTests.cs @@ -87,7 +87,7 @@ public IAsyncEnumerable GetByteStreamingUpdatesAsync(string input, AIReq throw new NotImplementedException(); } - public IAsyncEnumerable GetStreamingChunksAsync(string input, AIRequestSettings? requestSettings = null, CancellationToken cancellationToken = default) + public IAsyncEnumerable GetStreamingChunksAsync(string input, AIRequestSettings? requestSettings = null, CancellationToken cancellationToken = default) { throw new NotImplementedException(); } @@ -97,7 +97,7 @@ public IAsyncEnumerable GetStringStreamingUpdatesAsync(string input, AIR throw new NotImplementedException(); } - public IAsyncEnumerable GetStreamingChunksAsync(string input, AIRequestSettings? requestSettings = null, CancellationToken cancellationToken = default) + public IAsyncEnumerable GetStreamingContentAsync(string input, AIRequestSettings? requestSettings = null, CancellationToken cancellationToken = default) { throw new NotImplementedException(); } diff --git a/dotnet/src/SemanticKernel.Abstractions/AI/ChatCompletion/ChatCompletionExtensions.cs b/dotnet/src/SemanticKernel.Abstractions/AI/ChatCompletion/ChatCompletionExtensions.cs index 94dcdd9c39d9..cdb2ec941874 100644 --- a/dotnet/src/SemanticKernel.Abstractions/AI/ChatCompletion/ChatCompletionExtensions.cs +++ b/dotnet/src/SemanticKernel.Abstractions/AI/ChatCompletion/ChatCompletionExtensions.cs @@ -59,4 +59,19 @@ public static async Task GenerateMessageAsync( var firstChatMessage = await chatResults[0].GetChatMessageAsync(cancellationToken).ConfigureAwait(false); return firstChatMessage.Content; } + + /// + /// Get streaming completion StreamingResultChunk results for the prompt and settings. + /// + /// Chat completion target + /// The input string. (May be a JSON for complex objects, Byte64 for binary, will depend on the connector spec). + /// Request settings for the completion API + /// The to monitor for cancellation requests. The default is . + /// Streaming list of different completion streaming result updates generated by the remote model + public static IAsyncEnumerable GetStreamingChunksAsync( + this IChatCompletion chatCompletion, + string input, + AIRequestSettings? requestSettings = null, + CancellationToken cancellationToken = default) + => chatCompletion.GetStreamingContentAsync(input, requestSettings, cancellationToken); } diff --git a/dotnet/src/SemanticKernel.Abstractions/AI/ChatCompletion/IChatCompletion.cs b/dotnet/src/SemanticKernel.Abstractions/AI/ChatCompletion/IChatCompletion.cs index 8c702626bd16..8fde955fd376 100644 --- a/dotnet/src/SemanticKernel.Abstractions/AI/ChatCompletion/IChatCompletion.cs +++ b/dotnet/src/SemanticKernel.Abstractions/AI/ChatCompletion/IChatCompletion.cs @@ -44,18 +44,6 @@ IAsyncEnumerable GetStreamingChatCompletionsAsync( AIRequestSettings? requestSettings = null, CancellationToken cancellationToken = default); - /// - /// Get streaming completion StreamingResultChunk results for the prompt and settings. - /// - /// The input string. (May be a JSON for complex objects, Byte64 for binary, will depend on the connector spec). - /// Request settings for the completion API - /// The to monitor for cancellation requests. The default is . - /// Streaming list of different completion streaming result updates generated by the remote model - IAsyncEnumerable GetStreamingChunksAsync( - string input, - AIRequestSettings? requestSettings = null, - CancellationToken cancellationToken = default); - /// /// Get streaming results for the prompt and settings of a specific type. /// Each modality may support for different types of streaming result. @@ -68,7 +56,7 @@ IAsyncEnumerable GetStreamingChunksAsync( /// Request settings for the completion API /// The to monitor for cancellation requests. The default is . /// Streaming list of different completion streaming string updates generated by the remote model - IAsyncEnumerable GetStreamingChunksAsync( + IAsyncEnumerable GetStreamingContentAsync( string input, AIRequestSettings? requestSettings = null, CancellationToken cancellationToken = default); diff --git a/dotnet/src/SemanticKernel.Abstractions/AI/StreamingResultChunk.cs b/dotnet/src/SemanticKernel.Abstractions/AI/StreamingContent.cs similarity index 75% rename from dotnet/src/SemanticKernel.Abstractions/AI/StreamingResultChunk.cs rename to dotnet/src/SemanticKernel.Abstractions/AI/StreamingContent.cs index 9b7de021fd32..90926c2ca3b5 100644 --- a/dotnet/src/SemanticKernel.Abstractions/AI/StreamingResultChunk.cs +++ b/dotnet/src/SemanticKernel.Abstractions/AI/StreamingContent.cs @@ -1,20 +1,15 @@ // Copyright (c) Microsoft. All rights reserved. -using System.Text.Json.Serialization; +using System.Collections.Generic; using Microsoft.SemanticKernel.Orchestration; namespace Microsoft.SemanticKernel.AI; /// -/// Represents a single update to a streaming result. +/// Represents a single update to a streaming content. /// -public abstract class StreamingResultChunk +public abstract class StreamingContent { - /// - /// Type of the chunk. - /// - public abstract string Type { get; } - /// /// In a scenario of multiple choices per request, this represents zero-based index of the choice in the streaming sequence /// @@ -40,25 +35,29 @@ public abstract class StreamingResultChunk /// /// Internal chunk object reference. (Breaking glass). - /// Each connector will have its own internal object representing the result chunk. + /// Each connector will have its own internal object representing the content chunk. /// /// /// The usage of this property is considered "unsafe". Use it only if strictly necessary. /// - public object? InnerResultChunk { get; } + public object? InnerContent { get; } /// /// The current context associated the function call. /// - [JsonIgnore] internal SKContext? Context { get; set; } /// - /// Initializes a new instance of the class. + /// The metadata associated the content. + /// + internal Dictionary? Metadata { get; set; } + + /// + /// Initializes a new instance of the class. /// - /// Inner result chunk object reference - protected StreamingResultChunk(object? innerResultChunk) + /// Inner content object reference + protected StreamingContent(object? innerContent) { - this.InnerResultChunk = innerResultChunk; + this.InnerContent = innerContent; } } diff --git a/dotnet/src/SemanticKernel.Abstractions/AI/TextCompletion/ITextCompletion.cs b/dotnet/src/SemanticKernel.Abstractions/AI/TextCompletion/ITextCompletion.cs index 87c3dbaf0616..fdd1e34691a2 100644 --- a/dotnet/src/SemanticKernel.Abstractions/AI/TextCompletion/ITextCompletion.cs +++ b/dotnet/src/SemanticKernel.Abstractions/AI/TextCompletion/ITextCompletion.cs @@ -37,18 +37,6 @@ IAsyncEnumerable GetStreamingCompletionsAsync( AIRequestSettings? requestSettings = null, CancellationToken cancellationToken = default); - /// - /// Get streaming completion results for the prompt and settings. - /// - /// The prompt to complete. - /// Request settings for the completion API - /// The to monitor for cancellation requests. The default is . - /// Streaming list of different completion streaming result updates generated by the remote model - IAsyncEnumerable GetStreamingChunksAsync( - string input, - AIRequestSettings? requestSettings = null, - CancellationToken cancellationToken = default); - /// /// Get streaming results for the prompt and settings of a specific type. /// Each modality may support for different types of streaming result. @@ -61,7 +49,7 @@ IAsyncEnumerable GetStreamingChunksAsync( /// Request settings for the completion API /// The to monitor for cancellation requests. The default is . /// Streaming list of different completion streaming string updates generated by the remote model - IAsyncEnumerable GetStreamingChunksAsync( + IAsyncEnumerable GetStreamingContentAsync( string input, AIRequestSettings? requestSettings = null, CancellationToken cancellationToken = default); diff --git a/dotnet/src/SemanticKernel.Abstractions/AI/TextCompletion/TextCompletionExtensions.cs b/dotnet/src/SemanticKernel.Abstractions/AI/TextCompletion/TextCompletionExtensions.cs index 4350f709d287..54e713614506 100644 --- a/dotnet/src/SemanticKernel.Abstractions/AI/TextCompletion/TextCompletionExtensions.cs +++ b/dotnet/src/SemanticKernel.Abstractions/AI/TextCompletion/TextCompletionExtensions.cs @@ -82,4 +82,19 @@ public static async IAsyncEnumerable CompleteStreamsAsync(this ITextComp } } } + + /// + /// Get streaming completion results for the prompt and settings. + /// + /// Target text completion + /// The prompt to complete. + /// Request settings for the completion API + /// The to monitor for cancellation requests. The default is . + /// Streaming list of different completion streaming result updates generated by the remote model + public static IAsyncEnumerable GetStreamingContentAsync( + this ITextCompletion textCompletion, + string input, + AIRequestSettings? requestSettings = null, + CancellationToken cancellationToken = default) + => textCompletion.GetStreamingContentAsync(input, requestSettings, cancellationToken); } diff --git a/dotnet/src/SemanticKernel.Core/Functions/SKFunctionFromMethod.cs b/dotnet/src/SemanticKernel.Core/Functions/SKFunctionFromMethod.cs index b58ac4261888..8b1824b4dd69 100644 --- a/dotnet/src/SemanticKernel.Core/Functions/SKFunctionFromMethod.cs +++ b/dotnet/src/SemanticKernel.Core/Functions/SKFunctionFromMethod.cs @@ -159,7 +159,7 @@ public async IAsyncEnumerable InvokeStreamingAsync( { await foreach (var chunk in this.InvokeStreamingAsync(kernel, context, requestSettings, cancellationToken).ConfigureAwait(false)) { - if (typeof(T).IsSubclassOf(typeof(StreamingResultChunk)) || typeof(T) == typeof(StreamingResultChunk)) + if (typeof(T).IsSubclassOf(typeof(StreamingContent)) || typeof(T) == typeof(StreamingContent)) { yield return (T)(object)chunk; continue; @@ -189,7 +189,7 @@ public async IAsyncEnumerable InvokeStreamingAsync( { var chunk = enumerator.Current; - if (typeof(T).IsSubclassOf(typeof(StreamingResultChunk)) || typeof(T) == typeof(StreamingResultChunk)) + if (typeof(T).IsSubclassOf(typeof(StreamingContent)) || typeof(T) == typeof(StreamingContent)) { genericChunk = (T)(object)chunk; continue; @@ -228,7 +228,7 @@ public async IAsyncEnumerable InvokeStreamingAsync( /// LLM completion settings (for semantic functions only) /// The to monitor for cancellation requests. The default is . /// A asynchronous list of streaming result chunks - public async IAsyncEnumerable InvokeStreamingAsync( + public async IAsyncEnumerable InvokeStreamingAsync( Kernel kernel, SKContext context, AIRequestSettings? requestSettings = null, diff --git a/dotnet/src/SemanticKernel.Core/Functions/SKFunctionFromPrompt.cs b/dotnet/src/SemanticKernel.Core/Functions/SKFunctionFromPrompt.cs index 5ef313d922bd..c967d38a37ae 100644 --- a/dotnet/src/SemanticKernel.Core/Functions/SKFunctionFromPrompt.cs +++ b/dotnet/src/SemanticKernel.Core/Functions/SKFunctionFromPrompt.cs @@ -216,7 +216,7 @@ public async IAsyncEnumerable InvokeStreamingAsync( fullCompletion.Append(genericChunk); // Check if genericChunk is a StreamingResultChunk and update the context - if (genericChunk is StreamingResultChunk resultChunk) + if (genericChunk is StreamingContent resultChunk) { // This currently is needed so plans can get the context from the chunks to update the variables generated when the stream ends. resultChunk.Context = context; diff --git a/dotnet/src/SemanticKernel.Core/Functions/StreamingMethodResultChunk.cs b/dotnet/src/SemanticKernel.Core/Functions/StreamingMethodResultChunk.cs index f1cbe790d671..a7a04508f0ee 100644 --- a/dotnet/src/SemanticKernel.Core/Functions/StreamingMethodResultChunk.cs +++ b/dotnet/src/SemanticKernel.Core/Functions/StreamingMethodResultChunk.cs @@ -11,11 +11,8 @@ namespace Microsoft.SemanticKernel; /// /// Method function streaming result chunk. /// -public sealed class StreamingMethodResultChunk : StreamingResultChunk +public sealed class StreamingMethodResultChunk : StreamingContent { - /// - public override string Type => "method_result_chunk"; - /// public override int ChoiceIndex => 0; diff --git a/dotnet/src/SemanticKernel.Core/KernelExtensions.cs b/dotnet/src/SemanticKernel.Core/KernelExtensions.cs index bf71f3ac5546..84ff568b447a 100644 --- a/dotnet/src/SemanticKernel.Core/KernelExtensions.cs +++ b/dotnet/src/SemanticKernel.Core/KernelExtensions.cs @@ -667,8 +667,8 @@ public static async IAsyncEnumerable RunStreamingAsync(this Kernel kernel, /// Input to process /// The to monitor for cancellation requests. /// Result of the function composition - public static IAsyncEnumerable RunStreamingAsync(this Kernel kernel, ISKFunction skFunction, string input, CancellationToken cancellationToken) - => kernel.RunStreamingAsync(skFunction, new ContextVariables(input), cancellationToken); + public static IAsyncEnumerable RunStreamingAsync(this Kernel kernel, ISKFunction skFunction, string input, CancellationToken cancellationToken) + => kernel.RunStreamingAsync(skFunction, new ContextVariables(input), cancellationToken); /// /// Run a function in streaming mode. @@ -677,8 +677,8 @@ public static IAsyncEnumerable RunStreamingAsync(this Kern /// Target function to run /// Input to process /// Result of the function composition - public static IAsyncEnumerable RunStreamingAsync(this Kernel kernel, ISKFunction skFunction, string input) - => kernel.RunStreamingAsync(skFunction, new ContextVariables(input), CancellationToken.None); + public static IAsyncEnumerable RunStreamingAsync(this Kernel kernel, ISKFunction skFunction, string input) + => kernel.RunStreamingAsync(skFunction, new ContextVariables(input), CancellationToken.None); /// /// Run a function in streaming mode. @@ -686,8 +686,8 @@ public static IAsyncEnumerable RunStreamingAsync(this Kern /// Target kernel /// Target function to run /// Result of the function composition - public static IAsyncEnumerable RunStreamingAsync(this Kernel kernel, ISKFunction skFunction) - => kernel.RunStreamingAsync(skFunction, (ContextVariables?)null, CancellationToken.None); + public static IAsyncEnumerable RunStreamingAsync(this Kernel kernel, ISKFunction skFunction) + => kernel.RunStreamingAsync(skFunction, (ContextVariables?)null, CancellationToken.None); /// /// Run a function in streaming mode. diff --git a/dotnet/src/SemanticKernel.UnitTests/Functions/OrderedIAIServiceConfigurationProviderTests.cs b/dotnet/src/SemanticKernel.UnitTests/Functions/OrderedIAIServiceConfigurationProviderTests.cs index 9b9d373738c7..b3b115539fc9 100644 --- a/dotnet/src/SemanticKernel.UnitTests/Functions/OrderedIAIServiceConfigurationProviderTests.cs +++ b/dotnet/src/SemanticKernel.UnitTests/Functions/OrderedIAIServiceConfigurationProviderTests.cs @@ -212,7 +212,7 @@ public Task> GetCompletionsAsync(string text, AIReque throw new NotImplementedException(); } - public IAsyncEnumerable GetStreamingChunksAsync(string input, AIRequestSettings? requestSettings = null, CancellationToken cancellationToken = default) + public IAsyncEnumerable GetStreamingContentAsync(string input, AIRequestSettings? requestSettings = null, CancellationToken cancellationToken = default) { throw new NotImplementedException(); } @@ -222,7 +222,7 @@ public IAsyncEnumerable GetStreamingCompletionsAsync(strin throw new NotImplementedException(); } - public IAsyncEnumerable GetStreamingChunksAsync(string input, AIRequestSettings? requestSettings = null, CancellationToken cancellationToken = default) + public IAsyncEnumerable GetStreamingChunksAsync(string input, AIRequestSettings? requestSettings = null, CancellationToken cancellationToken = default) { throw new NotImplementedException(); } From ae1985d6b02d7d0beb9e3a0c83c4f80291ef6365 Mon Sep 17 00:00:00 2001 From: Roger Barreto <19890735+RogerBarreto@users.noreply.github.com> Date: Thu, 23 Nov 2023 00:01:25 +0000 Subject: [PATCH 30/46] Latest changes to Streaming addressing ADR feedback --- docs/decisions/0023-kernel-streaming.md | 220 ++++++++++-------- .../Example16_CustomLLM.cs | 31 +-- .../Example72_KernelStreaming.cs | 10 +- .../HuggingFaceTextCompletion.cs | 21 +- ...ResultChunk.cs => StreamingTextContent.cs} | 17 +- .../AzureSdk/ClientBase.cs | 17 +- ...ResultChunk.cs => StreamingChatContent.cs} | 20 +- ...unk.cs => StreamingChatWithDataContent.cs} | 11 +- ...ResultChunk.cs => StreamingTextContent.cs} | 13 +- .../AzureOpenAIChatCompletionWithData.cs | 24 +- .../KernelSemanticFunctionExtensionsTests.cs | 14 -- .../ChatCompletionExtensions.cs | 4 +- .../AI/StreamingContent.cs | 32 +-- .../AI/TextCompletion/ITextCompletion.cs | 6 +- .../Functions/SKFunctionFromMethod.cs | 12 +- .../Functions/SKFunctionFromPrompt.cs | 2 +- ...sultChunk.cs => StreamingMethodContent.cs} | 10 +- ...redIAIServiceConfigurationProviderTests.cs | 5 - 18 files changed, 217 insertions(+), 252 deletions(-) rename dotnet/src/Connectors/Connectors.AI.HuggingFace/TextCompletion/{StreamingTextResultChunk.cs => StreamingTextContent.cs} (80%) rename dotnet/src/Connectors/Connectors.AI.OpenAI/AzureSdk/{StreamingChatResultChunk.cs => StreamingChatContent.cs} (74%) rename dotnet/src/Connectors/Connectors.AI.OpenAI/AzureSdk/{StreamingChatWithDataResultChunk.cs => StreamingChatWithDataContent.cs} (77%) rename dotnet/src/Connectors/Connectors.AI.OpenAI/AzureSdk/{StreamingTextResultChunk.cs => StreamingTextContent.cs} (65%) rename dotnet/src/SemanticKernel.Core/Functions/{StreamingMethodResultChunk.cs => StreamingMethodContent.cs} (78%) diff --git a/docs/decisions/0023-kernel-streaming.md b/docs/decisions/0023-kernel-streaming.md index ab4e130a9312..4395d144980d 100644 --- a/docs/decisions/0023-kernel-streaming.md +++ b/docs/decisions/0023-kernel-streaming.md @@ -39,13 +39,13 @@ Using dedicated streaming interfaces that allow the sk developer to get the stre This approach also exposes dedicated interfaces in the kernel and functions to use streaming making it clear to the sk developer what is the type of data being returned in IAsyncEnumerable format. -`ITextCompletion` and `IChatCompletion` will have new APIs to get `byte[]` and `string` streaming data directly as well as the specialized `StreamingResultChunk` return. +`ITextCompletion` and `IChatCompletion` will have new APIs to get `byte[]` and `string` streaming data directly as well as the specialized `StreamingContent` return. -The sk developer will be able to specify a generic type to the `Kernel.RunStreamingAsync()` and `ISKFunction.StreamingInvokeAsync` to get the streaming data. If the type is not specified, the Kernel and Functions will return the data as StreamingResultChunk. +The sk developer will be able to specify a generic type to the `Kernel.RunStreamingAsync()` and `ISKFunction.InvokeStreamingAsync` to get the streaming data. If the type is not specified, the Kernel and Functions will return the data as StreamingContent. If the type is not specified or if the string representation cannot be cast, an exception will be thrown. -If the type specified is `StreamingResultChunk`, `string` or `byte[]` no error will be thrown as the connectors will have interface methods that guarantee the data to be given in at least those types. +If the type specified is `StreamingContent` or another any type supported by the connector no error will be thrown. ## User Experience Goal @@ -58,118 +58,169 @@ await foreach(string update in kernel.RunStreamingAsync(function, variab // Getting a String as Streaming data from Kernel await foreach(string update in kernel.RunStreamingAsync(function, variables)) -// Getting a StreamingResultChunk as Streaming data from Kernel -await foreach(StreamingResultChunk update in kernel.RunStreamingAsync(variables, function)) +// Getting a StreamingContent as Streaming data from Kernel +await foreach(StreamingContent update in kernel.RunStreamingAsync(variables, function)) // OR -await foreach(StreamingResultChunk update in kernel.RunStreamingAsync(function, variables)) // defaults to Generic above) +await foreach(StreamingContent update in kernel.RunStreamingAsync(function, variables)) // defaults to Generic above) { Console.WriteLine(update); } ``` +Abstraction class for any stream content, connectors will be responsible to provide the specialized type of `StreamingContent` which will contain the data as well as any metadata related to the streaming result. + ```csharp -// Depending on the underlying function model and connector used the streaming -// data will be of a specialization of StreamingResultChunk exposing useful -// properties including Type, the raw data in byte[] and string representation. -abstract class StreamingResultChunk +public abstract class StreamingContent { - public abstract string Type { get; } - - // In a scenario of multiple results, this represents zero-based index of the result in the streaming sequence - public abstract int ResultIndex { get; } + public abstract int ChoiceIndex { get; } + /// Returns a string representation of the chunk content public abstract override string ToString(); + + /// Abstract byte[] representation of the chunk content in a way it could be composed/appended with previous chunk contents. + /// Depending on the nature of the underlying type, this method may be more efficient than . public abstract byte[] ToByteArray(); - public abstract object Contents { get; } + /// Internal chunk content object reference. (Breaking glass). + /// Each connector will have its own internal object representing the content chunk content. + /// The usage of this property is considered "unsafe". Use it only if strictly necessary. + public object? InnerContent { get; } + + /// The metadata associated with the content. + public Dictionary? Metadata { get; set; } + + /// The current context associated the function call. + internal SKContext? Context { get; set; } + + /// Inner content object reference + protected StreamingContent(object? innerContent) + { + this.InnerContent = innerContent; + } } +``` -// Specialization example of a ChatMessageChunk -public class ChatMessageChunk : StreamingResultChunk -{ - public override string Type => "openai_chatmessage_chunk"; - public override string ToString() => Value.ToString(); // Value to be appended or the whole object? - public override byte[] ToByteArray() => Encoding.UTF8.GetBytes(Value); +Specialization example of a StreamingChatContent - public string Message { get; } - public string Role { get; } +```csharp +// +public class StreamingChatContent : StreamingContent +{ + public override int ChoiceIndex { get; } + public FunctionCall? FunctionCall { get; } + public string? Content { get; } + public AuthorRole? Role { get; } + public string? Name { get; } - public ChatMessageChunk(string message, string role, int resultIndex = 0) + public StreamingChatContent(AzureOpenAIChatMessage chatMessage, int resultIndex) : base(chatMessage) { - Message = message; - Role = role; - ResultIndex = resultIndex; + this.ChoiceIndex = resultIndex; + this.FunctionCall = chatMessage.InnerChatMessage?.FunctionCall; + this.Content = chatMessage.Content; + this.Role = new AuthorRole(chatMessage.Role.ToString()); + this.Name = chatMessage.InnerChatMessage?.Name; } + + public override byte[] ToByteArray() => Encoding.UTF8.GetBytes(this.ToString()); + public override string ToString() => this.Content ?? string.Empty; } +``` +`IChatCompletion` and `ITextCompletion` interfaces will have new APIs to get a generic streaming content data. + +```csharp interface ITextCompletion + IChatCompletion { - IAsyncEnumerable GetStreamingChunksAsync(); - // Guaranteed abstraction to be used by the ISKFunction.RunStreamingAsync() + IAsyncEnumerable GetStreamingContentAsync(...); - IAsyncEnumerable GetStreamingChunksAsync(); // Throw exception if T is not supported - // Initially connectors } interface IKernel { - // When the developer provides a T, the Kernel will try to get the streaming data as T - IAsyncEnumerable RunStreamingAsync(ContextVariables variables, ISKFunction function); - - // Extension generic method to get from type + // Get streaming function content of T IAsyncEnumerable RunStreamingAsync(ContextVariables variables, ISKFunction function); } -public class StreamingFunctionResult : IAsyncEnumerable -{ - -} - interface ISKFunction { - // When the developer provides a T, the Kernel will try to get the streaming data as T - IAsyncEnumerable InvokeStreamingAsync(SKContext context); - - // Extension generic method to get from type + // Get streaming function content of T IAsyncEnumerable InvokeStreamingAsync(SKContext context); } ``` -## Semantic Functions Behavior +## Prompt/Semantic Functions Behavior -When Semantic Functions are invoked using the Streaming API, they will attempt to use the Connectors streaming implementation. The connector will be responsible to provide the specialized type of `StreamingResultChunk` as well as to keep the API streaming working (streaming the single complete result) even when the backend don't support it. +When Prompt Functions are invoked using the Streaming API, they will attempt to use the Connectors streaming implementation. +The connector will be responsible to provide the specialized type of `StreamingContent` and even if the underlying backend API don't support streaming the output will be one streamingcontent with the whole data. -## Native Functions Behavior +## Method/Native Functions Behavior -NativeFunctions will support StreamingResults automatically with a StreamingNativeResultUpdate wrapping the object returned in the iterator. +Method Functions will support `StreamingContent` automatically with as a `StreamingMethodContent` wrapping the object returned in the iterator. -If NativeFunctions are already IAsyncEnumerable methods, the result will be automatically wrapped in the StreamingNativeResultUpdate keeping the streaming behavior and the overall abstraction consistent. +```csharp +public sealed class StreamingMethodContent : StreamingContent +{ + public override int ChoiceIndex => 0; -If NativeFunctions don't return IAsyncEnumerable, the result will be wrapped in a StreamingNativeResultUpdate and the result will be returned as a single result. + /// Method object value that represents the content chunk + public object Value { get; } + + /// Default implementation + public override byte[] ToByteArray() + { + if (this.Value is byte[]) + { + // If the method value is byte[] we return it directly + return (byte[])this.Value; + } + + // By default if a native value is not byte[] we output the UTF8 string representation of the value + return Encoding.UTF8.GetBytes(this.Value?.ToString()); + } + + /// + public override string ToString() + { + return this.Value.ToString(); + } + + /// + /// Initializes a new instance of the class. + /// + /// Underlying object that represents the chunk + public StreamingMethodContent(object innerContent) : base(innerContent) + { + this.Value = innerContent; + } +} +``` + +If a MethodFunction is returning an `IAsyncEnumerable` each enumerable result will be automatically wrapped in the `StreamingMethodContent` keeping the streaming behavior and the overall abstraction consistent. + +When a MethodFunction is not an `IAsyncEnumerable`, the complete result will be wrapped in a `StreamingMethodContent` and will be returned as a single item. ## Pros 1. All the User Experience Goal section options will be possible. 2. Kernel and Functions implementations will be able to stream data of any type, not limited to text -3. The sk developer will be able to provide the type it expects from the `GetStreamingResultAsync` method. -4. The above will allow the sk developer to get the streaming data in a generic way, including `string`, `byte array`, or the `StreamingResultChunk` abstraction directly from the connector. - -5. IChatCompletion, IKernel and ISKFunction +3. The sk developer will be able to provide the streaming content type it expects from the `GetStreamingContentAsync` method. +4. Sk developer will be able to get streaming from the Kernel, Functions and Connectors with the same result type. ## Cons -1. If the sk developer wants to use the specialized type of `StreamingResultChunk` he will need to know what the connector is being used to use the correct **StreamingResultChunk extension method** or to provide directly type in ``. -2. Connectors will have greater responsibility to provide the correct type of `StreamingResultChunk` for the connector being used and implementations for both byte[] and string streaming data. +1. If the sk developer wants to use the specialized type of `StreamingContent` he will need to know what the connector is being used to use the correct **StreamingContent extension method** or to provide directly type in ``. +2. Connectors will have greater responsibility to support the correct special types of `StreamingContent`. ### Option 2 - Dedicated Streaming Interfaces (Returning a Class) -This option includes all the suggestions from Option 1 with the only change being that the interfaces now instead of returning an `IAsyncEnumerable` will return `StreamingFunctionResult` which also implements `IAsyncEnumerable` +All changes from option 1 with the small difference below: -Connectors will have a streaming connector class representation that will have any data related to the request as well as a breaking glass reference to the underlying connector object (response) if needed. +- The Kernel and SKFunction streaming APIs interfaces will return `StreamingFunctionResult` which also implements `IAsyncEnumerable` +- Connectors streaming APIs interfaces will return `StreamingConnectorContent` which also implements `IAsyncEnumerable` -This abstraction is necessary to provide the extra connector information to the StreamingFunctionResult returned by KernelPromptFunctions. +The `StreamingConnectorContent` class is needed for connectors as one way to pass any information relative to the request and not the chunk that can be used by the functions to fill `StreamingFunctionResult` metadata. ## User Experience Goal @@ -180,64 +231,55 @@ Option 2 Biggest benefit: var streamingResult = await kernel.RunStreamingAsync(function); // Do something with streamingResult properties -// Consuming the streamingResult: -await foreach(StreamingResultChunk chunk in await streamingResult) +// Consuming the streamingResult requires an extra await: +await foreach(StreamingContent chunk content in await streamingResult) ``` -Using the other operations will be quite similar (extra `await` to get the iteratable streaming reference) if the sk developer don't want to check for the result properties +Using the other operations will be quite similar (only needing an extra `await` to get the iterator) -````csharp +```csharp // Getting a Raw Streaming data from Kernel await foreach(string update in await kernel.RunStreamingAsync(function, variables)) // Getting a String as Streaming data from Kernel await foreach(string update in await kernel.RunStreamingAsync(function, variables)) -// Getting a StreamingResultChunk as Streaming data from Kernel -await foreach(StreamingResultChunk update in await kernel.RunStreamingAsync(variables, function)) +// Getting a StreamingContent as Streaming data from Kernel +await foreach(StreamingContent update in await kernel.RunStreamingAsync(variables, function)) // OR -await foreach(StreamingResultChunk update in await kernel.RunStreamingAsync(function, variables)) // defaults to Generic above) +await foreach(StreamingContent update in await kernel.RunStreamingAsync(function, variables)) // defaults to Generic above) { Console.WriteLine(update); } + ``` +StreamingConnectorResult is a class that can store information regarding the result before the stream is consumed as well as any underlying object (breaking glass) that the stream consumes at the connector level. + ```csharp public sealed class StreamingConnectorResult : IAsyncEnumerable { - private readonly IAsyncEnumerable _streamingResultSource; + private readonly IAsyncEnumerable _StreamingContentource; - /// - /// Internal object reference. (Breaking glass). - /// Each connector will have its own internal object representing the result. - /// public object? InnerResult { get; private set; } = null; - /// - /// Initializes a new instance of the class. - /// - /// - /// - public ConnectorAsyncEnumerable(Func> streamingReference, object? innerConnectorResult) + public StreamingConnectorResult(Func> streamingReference, object? innerConnectorResult) { - this._streamingResultSource = streamingReference.Invoke(); + this._StreamingContentource = streamingReference.Invoke(); this.InnerResult = innerConnectorResult; } } interface ITextCompletion + IChatCompletion { - Task> GetStreamingChunksAsync(); - // Guaranteed abstraction to be used by the ISKFunction.RunStreamingAsync() - - Task> GetStreamingChunksAsync(); + Task> GetStreamingContentAsync(); // Throw exception if T is not supported // Initially connectors } ``` -StreamingFunctionResult is a class that can store information regarding the result before the stream is consumed as well as any underlying object (breaking glass) that the stream consumes. +StreamingFunctionResult is a class that can store information regarding the result before the stream is consumed as well as any underlying object (breaking glass) that the stream consumes from Kernel and SKFunctions. ```csharp public sealed class StreamingFunctionResult : IAsyncEnumerable @@ -270,9 +312,6 @@ public sealed class StreamingFunctionResult : IAsyncEnumerable interface ISKFunction { - // When the developer provides a T, the Kernel will try to get the streaming data as T - Task> InvokeStreamingAsync(...); - // Extension generic method to get from type Task> InvokeStreamingAsync(...); } @@ -300,12 +339,11 @@ static class KernelExtensions 1. All cons from Option 1 + 2. Added complexity as the IAsyncEnumerable cannot be passed directly in the method result demanding a delegate approach to be adapted inside of the Results that implements the IAsyncEnumerator. - +3. Added complexity where IDisposable is needed to be implemented in the Results to dispose the response object and the caller would need to handle the disposal of the result. +4. As soon the caller gets a `StreamingFunctionResult` a network connection will be kept open until the caller implementation consume it (Enumerate over the `IAsyncEnumerable`). ## Decision Outcome -Option 3 -Try to use the Task interface with a dumb StreamingFunctionResult which wraps all the logic +Option 1 was chosen as the best option as small benefit of the Option 2 don't justify the complexity involved described in the Cons. -## More Information -```` +Was also decided that the Metadata related to a connector backend response can be added to the `StreamingContent.Metadata` property. This will allow the sk developer to get the metadata even without a `StreamingConnectorResult` or `StreamingFunctionResult`. diff --git a/dotnet/samples/KernelSyntaxExamples/Example16_CustomLLM.cs b/dotnet/samples/KernelSyntaxExamples/Example16_CustomLLM.cs index 784322e67b3c..b369628eb4e5 100644 --- a/dotnet/samples/KernelSyntaxExamples/Example16_CustomLLM.cs +++ b/dotnet/samples/KernelSyntaxExamples/Example16_CustomLLM.cs @@ -49,44 +49,23 @@ public async IAsyncEnumerable GetStreamingCompletionsAsync yield return new MyTextCompletionStreamingResult(); } - public async IAsyncEnumerable GetByteStreamingUpdatesAsync(string input, AIRequestSettings? requestSettings = null, [EnumeratorCancellation] CancellationToken cancellationToken = default) - { - await foreach (var update in this.GetStreamingChunksAsync(input, requestSettings, cancellationToken)) - { - yield return update.ToByteArray(); - } - } - - public IAsyncEnumerable GetStreamingChunksAsync(string input, AIRequestSettings? requestSettings = null, CancellationToken cancellationToken = default) - { - return this.GetStreamingContentAsync(input, requestSettings, cancellationToken); - } - - public async IAsyncEnumerable GetStringStreamingUpdatesAsync(string input, AIRequestSettings? requestSettings = null, [EnumeratorCancellation] CancellationToken cancellationToken = default) - { - await foreach (var update in this.GetStreamingChunksAsync(input, requestSettings, cancellationToken)) - { - yield return update.ToString(); - } - } - public async IAsyncEnumerable GetStreamingContentAsync(string input, AIRequestSettings? requestSettings = null, [EnumeratorCancellation] CancellationToken cancellationToken = default) { - if (typeof(T) == typeof(MyStreamingResultChunk)) + if (typeof(T) == typeof(MyStreamingContent)) { - yield return (T)(object)new MyStreamingResultChunk("llm content update 1"); - yield return (T)(object)new MyStreamingResultChunk("llm content update 2"); + yield return (T)(object)new MyStreamingContent("llm content update 1"); + yield return (T)(object)new MyStreamingContent("llm content update 2"); } } } -public class MyStreamingResultChunk : StreamingContent +public class MyStreamingContent : StreamingContent { public override int ChoiceIndex => 0; public string Content { get; } - public MyStreamingResultChunk(string content) : base(content) + public MyStreamingContent(string content) : base(content) { this.Content = content; } diff --git a/dotnet/samples/KernelSyntaxExamples/Example72_KernelStreaming.cs b/dotnet/samples/KernelSyntaxExamples/Example72_KernelStreaming.cs index 643895978a68..dd5a5097c4dc 100644 --- a/dotnet/samples/KernelSyntaxExamples/Example72_KernelStreaming.cs +++ b/dotnet/samples/KernelSyntaxExamples/Example72_KernelStreaming.cs @@ -56,7 +56,7 @@ public static async Task RunAsync() await foreach (var update in kernel.RunStreamingAsync(funyParagraphFunction)) { // You will be always able to know the type of the update by checking the Type property. - if (update is StreamingChatResultChunk chatUpdate) + if (update is StreamingChatContent chatUpdate) { if (!roleDisplayed && chatUpdate.Role.HasValue) { @@ -89,7 +89,7 @@ private static async Task NativeFunctionStreamingValueTypeAsync(Kernel kernel, I // Native string value type streaming function await foreach (var update in kernel.RunStreamingAsync(plugin["MyValueTypeStreamingNativeFunction"], "My Value Type Streaming Function Input")) { - if (update is StreamingMethodResultChunk nativeUpdate) + if (update is StreamingMethodContent nativeUpdate) { Console.Write(nativeUpdate.Value); } @@ -104,7 +104,7 @@ private static async Task NativeFunctionStreamingComplexTypeAsync(Kernel kernel, await foreach (var update in kernel.RunStreamingAsync(plugin["MyComplexTypeStreamingNativeFunction"], "My Complex Type Streaming Function Input")) { // the complex type will be available thru the Value property of the native update abstraction. - if (update is StreamingMethodResultChunk nativeUpdate && nativeUpdate.Value is MyStreamingBlock myComplexType) + if (update is StreamingMethodContent nativeUpdate && nativeUpdate.Value is MyStreamingBlock myComplexType) { Console.WriteLine(Encoding.UTF8.GetString(myComplexType.Content)); } @@ -119,7 +119,7 @@ private static async Task NativeFunctionValueTypeAsync(Kernel kernel, ISKPlugin await foreach (var update in kernel.RunStreamingAsync(plugin["MyValueTypeNativeFunction"], "My Value Type Non Streaming Function Input")) { // the complex type will be available thru the Value property of the native update abstraction. - if (update is StreamingMethodResultChunk nativeUpdate) + if (update is StreamingMethodContent nativeUpdate) { Console.WriteLine(nativeUpdate.Value); } @@ -134,7 +134,7 @@ private static async Task NativeFunctionComplexTypeAsync(Kernel kernel, ISKPlugi await foreach (var update in kernel.RunStreamingAsync(plugin["MyComplexTypeNativeFunction"], "My Complex Type Non Streaming Function Input")) { // the complex type will be available thru the Value property of the native update abstraction. - if (update is StreamingMethodResultChunk nativeUpdate && nativeUpdate.Value is MyCustomType myComplexType) + if (update is StreamingMethodContent nativeUpdate && nativeUpdate.Value is MyCustomType myComplexType) { Console.WriteLine($"Text: {myComplexType.Text}, Number: {myComplexType.Number}"); } diff --git a/dotnet/src/Connectors/Connectors.AI.HuggingFace/TextCompletion/HuggingFaceTextCompletion.cs b/dotnet/src/Connectors/Connectors.AI.HuggingFace/TextCompletion/HuggingFaceTextCompletion.cs index f757ceec6d3a..881e6c892cd2 100644 --- a/dotnet/src/Connectors/Connectors.AI.HuggingFace/TextCompletion/HuggingFaceTextCompletion.cs +++ b/dotnet/src/Connectors/Connectors.AI.HuggingFace/TextCompletion/HuggingFaceTextCompletion.cs @@ -92,22 +92,13 @@ public async Task> GetCompletionsAsync( return await this.ExecuteGetCompletionsAsync(text, cancellationToken).ConfigureAwait(false); } - /// - public IAsyncEnumerable GetStreamingChunksAsync( - string input, - AIRequestSettings? requestSettings = null, - CancellationToken cancellationToken = default) - { - return this.GetStreamingContentAsync(input, requestSettings, cancellationToken); - } - /// public async IAsyncEnumerable GetStreamingContentAsync( string input, AIRequestSettings? requestSettings = null, [EnumeratorCancellation] CancellationToken cancellationToken = default) { - await foreach (var result in this.InternalGetStreamingChunksAsync(input, cancellationToken).ConfigureAwait(false)) + await foreach (var result in this.InternalGetStreamingContentAsync(input, cancellationToken).ConfigureAwait(false)) { cancellationToken.ThrowIfCancellationRequested(); @@ -118,8 +109,8 @@ public async IAsyncEnumerable GetStreamingContentAsync( continue; } - // If the provided T is an specialized class of StreamingResultChunk interface - if (typeof(T) == typeof(StreamingTextResultChunk) || + // If the provided T is an specialized class of StreamingContent interface + if (typeof(T) == typeof(StreamingTextContent) || typeof(T) == typeof(StreamingContent)) { yield return (T)(object)result; @@ -161,7 +152,7 @@ private async Task> ExecuteGetCompletionsAsync(string return completionResponse.ConvertAll(c => new TextCompletionResult(c)); } - private async IAsyncEnumerable InternalGetStreamingChunksAsync(string text, [EnumeratorCancellation] CancellationToken cancellationToken) + private async IAsyncEnumerable InternalGetStreamingContentAsync(string text, [EnumeratorCancellation] CancellationToken cancellationToken) { var completionRequest = new TextCompletionRequest { @@ -196,9 +187,9 @@ private async IAsyncEnumerable InternalGetStreamingChu continue; } - JsonElement chunkObject = JsonSerializer.Deserialize(body); + JsonElement contentObject = JsonSerializer.Deserialize(body); - yield return new StreamingTextResultChunk("hugging-text-stream=chunk", chunkObject); + yield return new StreamingTextContent(contentObject); } } diff --git a/dotnet/src/Connectors/Connectors.AI.HuggingFace/TextCompletion/StreamingTextResultChunk.cs b/dotnet/src/Connectors/Connectors.AI.HuggingFace/TextCompletion/StreamingTextContent.cs similarity index 80% rename from dotnet/src/Connectors/Connectors.AI.HuggingFace/TextCompletion/StreamingTextResultChunk.cs rename to dotnet/src/Connectors/Connectors.AI.HuggingFace/TextCompletion/StreamingTextContent.cs index a466c1ccee11..2ce13b7c8865 100644 --- a/dotnet/src/Connectors/Connectors.AI.HuggingFace/TextCompletion/StreamingTextResultChunk.cs +++ b/dotnet/src/Connectors/Connectors.AI.HuggingFace/TextCompletion/StreamingTextContent.cs @@ -10,11 +10,8 @@ namespace Microsoft.SemanticKernel.Connectors.AI.HuggingFace.TextCompletion; /// /// StreamResponse class in /// -public class StreamingTextResultChunk : StreamingContent +public class StreamingTextContent : StreamingContent { - /// - public override string Type { get; } - /// public override int ChoiceIndex { get; } @@ -33,20 +30,18 @@ public class StreamingTextResultChunk : StreamingContent /// /// Token details /// - public TokenChunkModel Token { get; set; } + public TokenContentModel Token { get; set; } /// - /// Create a new instance of the class. + /// Create a new instance of the class. /// - /// Type of the chunk /// JsonElement representation of the chunk - public StreamingTextResultChunk(string type, JsonElement jsonChunk) : base(jsonChunk) + public StreamingTextContent(JsonElement jsonChunk) : base(jsonChunk) { - this.Type = type; this.ChoiceIndex = 0; this.GeneratedText = jsonChunk.GetProperty("generated_text").GetString(); this.Details = jsonChunk.GetProperty("details").GetString(); - this.Token = JsonSerializer.Deserialize(jsonChunk.GetProperty("token").GetRawText())!; + this.Token = JsonSerializer.Deserialize(jsonChunk.GetProperty("token").GetRawText())!; } /// @@ -64,7 +59,7 @@ public override string ToString() /// /// Token class in /// - public record TokenChunkModel + public record TokenContentModel { /// /// Id of the token diff --git a/dotnet/src/Connectors/Connectors.AI.OpenAI/AzureSdk/ClientBase.cs b/dotnet/src/Connectors/Connectors.AI.OpenAI/AzureSdk/ClientBase.cs index 249beb69404f..bcc902637185 100644 --- a/dotnet/src/Connectors/Connectors.AI.OpenAI/AzureSdk/ClientBase.cs +++ b/dotnet/src/Connectors/Connectors.AI.OpenAI/AzureSdk/ClientBase.cs @@ -170,6 +170,13 @@ private protected async IAsyncEnumerable InternalGetTextStreamingUpdatesAsync using StreamingCompletions streamingChatCompletions = response.Value; + var metadata = new Dictionary() + { + { $"{nameof(StreamingCompletions)}.{nameof(streamingChatCompletions.Id)}", streamingChatCompletions.Id }, + { $"{nameof(StreamingCompletions)}.{nameof(streamingChatCompletions.Created)}", streamingChatCompletions.Created }, + { $"{nameof(StreamingCompletions)}.{nameof(streamingChatCompletions.PromptFilterResults)}", streamingChatCompletions.PromptFilterResults }, + }; + int choiceIndex = 0; await foreach (StreamingChoice choice in streamingChatCompletions.GetChoicesStreaming(cancellationToken).ConfigureAwait(false)) { @@ -182,11 +189,11 @@ private protected async IAsyncEnumerable InternalGetTextStreamingUpdatesAsync continue; } - // If the provided T is an specialized class of StreamingResultChunk interface - if (typeof(T) == typeof(StreamingTextResultChunk) || + // If the provided T is an specialized class of StreamingContent interface + if (typeof(T) == typeof(StreamingTextContent) || typeof(T) == typeof(StreamingContent)) { - yield return (T)(object)new StreamingTextResultChunk(update, choiceIndex, update); + yield return (T)(object)new StreamingTextContent(update, choiceIndex, update, metadata); continue; } @@ -340,10 +347,10 @@ private protected async IAsyncEnumerable InternalGetChatStreamingUpdatesAsync } // If the provided T is an specialized class of StreamingResultChunk interface - if (typeof(T) == typeof(StreamingChatResultChunk) || + if (typeof(T) == typeof(StreamingChatContent) || typeof(T) == typeof(StreamingContent)) { - yield return (T)(object)new StreamingChatResultChunk(chatMessage, choiceIndex); + yield return (T)(object)new StreamingChatContent(chatMessage, choiceIndex); continue; } diff --git a/dotnet/src/Connectors/Connectors.AI.OpenAI/AzureSdk/StreamingChatResultChunk.cs b/dotnet/src/Connectors/Connectors.AI.OpenAI/AzureSdk/StreamingChatContent.cs similarity index 74% rename from dotnet/src/Connectors/Connectors.AI.OpenAI/AzureSdk/StreamingChatResultChunk.cs rename to dotnet/src/Connectors/Connectors.AI.OpenAI/AzureSdk/StreamingChatContent.cs index 0699f9334a11..3038be7565fb 100644 --- a/dotnet/src/Connectors/Connectors.AI.OpenAI/AzureSdk/StreamingChatResultChunk.cs +++ b/dotnet/src/Connectors/Connectors.AI.OpenAI/AzureSdk/StreamingChatContent.cs @@ -11,7 +11,7 @@ namespace Microsoft.SemanticKernel.Connectors.AI.OpenAI.AzureSdk; /// /// Streaming chat result update. /// -public class StreamingChatResultChunk : StreamingContent +public class StreamingChatContent : StreamingContent { /// public override int ChoiceIndex { get; } @@ -37,11 +37,11 @@ public class StreamingChatResultChunk : StreamingContent public string? Name { get; } /// - /// Create a new instance of the class. + /// Create a new instance of the class. /// /// Original Connector Azure Message update representation /// Index of the choice - public StreamingChatResultChunk(AzureOpenAIChatMessage chatMessage, int resultIndex) : base(chatMessage) + public StreamingChatContent(AzureOpenAIChatMessage chatMessage, int resultIndex) : base(chatMessage) { this.ChoiceIndex = resultIndex; this.FunctionCall = chatMessage.InnerChatMessage?.FunctionCall; @@ -51,11 +51,11 @@ public StreamingChatResultChunk(AzureOpenAIChatMessage chatMessage, int resultIn } /// - /// Create a new instance of the class. + /// Create a new instance of the class. /// /// Internal Azure SDK Message update representation /// Index of the choice - public StreamingChatResultChunk(Azure.AI.OpenAI.ChatMessage chatMessage, int resultIndex) : base(chatMessage) + public StreamingChatContent(Azure.AI.OpenAI.ChatMessage chatMessage, int resultIndex) : base(chatMessage) { this.ChoiceIndex = resultIndex; this.FunctionCall = chatMessage.FunctionCall; @@ -65,14 +65,8 @@ public StreamingChatResultChunk(Azure.AI.OpenAI.ChatMessage chatMessage, int res } /// - public override byte[] ToByteArray() - { - return Encoding.UTF8.GetBytes(this.ToString()); - } + public override byte[] ToByteArray() => Encoding.UTF8.GetBytes(this.ToString()); /// - public override string ToString() - { - return JsonSerializer.Serialize(this); - } + public override string ToString() => this.Content ?? string.Empty; } diff --git a/dotnet/src/Connectors/Connectors.AI.OpenAI/AzureSdk/StreamingChatWithDataResultChunk.cs b/dotnet/src/Connectors/Connectors.AI.OpenAI/AzureSdk/StreamingChatWithDataContent.cs similarity index 77% rename from dotnet/src/Connectors/Connectors.AI.OpenAI/AzureSdk/StreamingChatWithDataResultChunk.cs rename to dotnet/src/Connectors/Connectors.AI.OpenAI/AzureSdk/StreamingChatWithDataContent.cs index 4a10db68ad33..116cdb0506ef 100644 --- a/dotnet/src/Connectors/Connectors.AI.OpenAI/AzureSdk/StreamingChatWithDataResultChunk.cs +++ b/dotnet/src/Connectors/Connectors.AI.OpenAI/AzureSdk/StreamingChatWithDataContent.cs @@ -13,25 +13,22 @@ namespace Microsoft.SemanticKernel.Connectors.AI.OpenAI.AzureSdk; /// /// Streaming chat result update. /// -public sealed class StreamingChatWithDataResultChunk : StreamingContent +public sealed class StreamingChatWithDataContent : StreamingContent { - /// - public override string Type => "openai_chat_message_update"; - /// public override int ChoiceIndex { get; } /// /// Chat message abstraction /// - public SemanticKernel.AI.ChatCompletion.ChatMessage ChatMessage { get; } + public ChatMessage ChatMessage { get; } /// - /// Create a new instance of the class. + /// Create a new instance of the class. /// /// Azure message update representation from WithData apis /// Index of the choice - internal StreamingChatWithDataResultChunk(ChatWithDataStreamingChoice choice, int resultIndex) : base(choice) + internal StreamingChatWithDataContent(ChatWithDataStreamingChoice choice, int resultIndex) : base(choice) { this.ChoiceIndex = resultIndex; var message = choice.Messages.FirstOrDefault(this.IsValidMessage); diff --git a/dotnet/src/Connectors/Connectors.AI.OpenAI/AzureSdk/StreamingTextResultChunk.cs b/dotnet/src/Connectors/Connectors.AI.OpenAI/AzureSdk/StreamingTextContent.cs similarity index 65% rename from dotnet/src/Connectors/Connectors.AI.OpenAI/AzureSdk/StreamingTextResultChunk.cs rename to dotnet/src/Connectors/Connectors.AI.OpenAI/AzureSdk/StreamingTextContent.cs index 3281a2b7fc92..a7497d131b7f 100644 --- a/dotnet/src/Connectors/Connectors.AI.OpenAI/AzureSdk/StreamingTextResultChunk.cs +++ b/dotnet/src/Connectors/Connectors.AI.OpenAI/AzureSdk/StreamingTextContent.cs @@ -1,5 +1,6 @@ // Copyright (c) Microsoft. All rights reserved. +using System.Collections.Generic; using System.Text; using Microsoft.SemanticKernel.AI; @@ -8,11 +9,8 @@ namespace Microsoft.SemanticKernel.Connectors.AI.OpenAI.AzureSdk; /// /// Streaming text result update. /// -public class StreamingTextResultChunk : StreamingContent +public class StreamingTextContent : StreamingContent { - /// - public override string Type => "openai_text_update"; - /// public override int ChoiceIndex { get; } @@ -22,12 +20,13 @@ public class StreamingTextResultChunk : StreamingContent public string Content { get; } /// - /// Create a new instance of the class. + /// Create a new instance of the class. /// /// Text update /// Index of the choice - /// Inner chunk object - public StreamingTextResultChunk(string text, int resultIndex, object? innerChunkObject = null) : base(innerChunkObject) + /// Inner chunk object + /// Metadata information + public StreamingTextContent(string text, int resultIndex, object? innerContentObject = null, Dictionary? metadata = null) : base(innerContentObject, metadata) { this.ChoiceIndex = resultIndex; this.Content = text; diff --git a/dotnet/src/Connectors/Connectors.AI.OpenAI/ChatCompletionWithData/AzureOpenAIChatCompletionWithData.cs b/dotnet/src/Connectors/Connectors.AI.OpenAI/ChatCompletionWithData/AzureOpenAIChatCompletionWithData.cs index e4cd2e8ef317..2dcd694c0355 100644 --- a/dotnet/src/Connectors/Connectors.AI.OpenAI/ChatCompletionWithData/AzureOpenAIChatCompletionWithData.cs +++ b/dotnet/src/Connectors/Connectors.AI.OpenAI/ChatCompletionWithData/AzureOpenAIChatCompletionWithData.cs @@ -118,24 +118,6 @@ public async IAsyncEnumerable GetStreamingCompletionsAsync } } - /// - public async IAsyncEnumerable GetStreamingChunksAsync(string input, AIRequestSettings? requestSettings = null, [EnumeratorCancellation] CancellationToken cancellationToken = default) - { - OpenAIRequestSettings chatRequestSettings = OpenAIRequestSettings.FromRequestSettings(requestSettings); - - var chat = this.PrepareChatHistory(input, chatRequestSettings); - - var resultIndex = 0; - await foreach (var result in this.GetStreamingChatCompletionsAsync(chat, requestSettings, cancellationToken).ConfigureAwait(false)) - { - await foreach (var message in result.GetStreamingChatMessageAsync(cancellationToken).ConfigureAwait(false)) - { - yield return new StreamingChatResultChunk((AzureOpenAIChatMessage)message, resultIndex); - } - resultIndex++; - } - } - /// public async IAsyncEnumerable GetStreamingContentAsync( string input, @@ -295,11 +277,11 @@ private async IAsyncEnumerable GetChatStreamingUpdatesAsync(HttpResponseMe foreach (var choice in chatWithDataResponse.Choices) { - // If the provided T is an specialized class of StreamingResultChunk interface - if (typeof(T) == typeof(StreamingChatResultChunk) || + // If the provided T is an specialized class of StreamingContent interface + if (typeof(T) == typeof(StreamingChatContent) || typeof(T) == typeof(StreamingContent)) { - yield return (T)(object)new StreamingChatWithDataResultChunk(choice, choice.Index); + yield return (T)(object)new StreamingChatWithDataContent(choice, choice.Index); continue; } diff --git a/dotnet/src/IntegrationTests/Extensions/KernelSemanticFunctionExtensionsTests.cs b/dotnet/src/IntegrationTests/Extensions/KernelSemanticFunctionExtensionsTests.cs index 8875e323f4bd..87a2198bcd65 100644 --- a/dotnet/src/IntegrationTests/Extensions/KernelSemanticFunctionExtensionsTests.cs +++ b/dotnet/src/IntegrationTests/Extensions/KernelSemanticFunctionExtensionsTests.cs @@ -82,20 +82,6 @@ IAsyncEnumerable ITextCompletion.GetStreamingCompletionsAs { throw new NotImplementedException(); // TODO } - public IAsyncEnumerable GetByteStreamingUpdatesAsync(string input, AIRequestSettings? requestSettings = null, CancellationToken cancellationToken = default) - { - throw new NotImplementedException(); - } - - public IAsyncEnumerable GetStreamingChunksAsync(string input, AIRequestSettings? requestSettings = null, CancellationToken cancellationToken = default) - { - throw new NotImplementedException(); - } - - public IAsyncEnumerable GetStringStreamingUpdatesAsync(string input, AIRequestSettings? requestSettings = null, CancellationToken cancellationToken = default) - { - throw new NotImplementedException(); - } public IAsyncEnumerable GetStreamingContentAsync(string input, AIRequestSettings? requestSettings = null, CancellationToken cancellationToken = default) { diff --git a/dotnet/src/SemanticKernel.Abstractions/AI/ChatCompletion/ChatCompletionExtensions.cs b/dotnet/src/SemanticKernel.Abstractions/AI/ChatCompletion/ChatCompletionExtensions.cs index cdb2ec941874..a60e255ef845 100644 --- a/dotnet/src/SemanticKernel.Abstractions/AI/ChatCompletion/ChatCompletionExtensions.cs +++ b/dotnet/src/SemanticKernel.Abstractions/AI/ChatCompletion/ChatCompletionExtensions.cs @@ -61,14 +61,14 @@ public static async Task GenerateMessageAsync( } /// - /// Get streaming completion StreamingResultChunk results for the prompt and settings. + /// Get asynchronous stream.of . /// /// Chat completion target /// The input string. (May be a JSON for complex objects, Byte64 for binary, will depend on the connector spec). /// Request settings for the completion API /// The to monitor for cancellation requests. The default is . /// Streaming list of different completion streaming result updates generated by the remote model - public static IAsyncEnumerable GetStreamingChunksAsync( + public static IAsyncEnumerable GetStreamingContentAsync( this IChatCompletion chatCompletion, string input, AIRequestSettings? requestSettings = null, diff --git a/dotnet/src/SemanticKernel.Abstractions/AI/StreamingContent.cs b/dotnet/src/SemanticKernel.Abstractions/AI/StreamingContent.cs index 90926c2ca3b5..56474cb5d704 100644 --- a/dotnet/src/SemanticKernel.Abstractions/AI/StreamingContent.cs +++ b/dotnet/src/SemanticKernel.Abstractions/AI/StreamingContent.cs @@ -15,6 +15,20 @@ public abstract class StreamingContent /// public abstract int ChoiceIndex { get; } + /// + /// Internal chunk object reference. (Breaking glass). + /// Each connector will have its own internal object representing the content chunk. + /// + /// + /// The usage of this property is considered "unsafe". Use it only if strictly necessary. + /// + public object? InnerContent { get; } + + /// + /// The metadata associated with the content. + /// + public Dictionary? Metadata { get; } + /// /// Abstract string representation of the chunk in a way it could compose/append with previous chunks. /// @@ -33,31 +47,19 @@ public abstract class StreamingContent /// Byte array representation of the chunk public abstract byte[] ToByteArray(); - /// - /// Internal chunk object reference. (Breaking glass). - /// Each connector will have its own internal object representing the content chunk. - /// - /// - /// The usage of this property is considered "unsafe". Use it only if strictly necessary. - /// - public object? InnerContent { get; } - /// /// The current context associated the function call. /// internal SKContext? Context { get; set; } - /// - /// The metadata associated the content. - /// - internal Dictionary? Metadata { get; set; } - /// /// Initializes a new instance of the class. /// /// Inner content object reference - protected StreamingContent(object? innerContent) + /// + protected StreamingContent(object? innerContent, Dictionary? metadata = null) { this.InnerContent = innerContent; + this.Metadata = metadata ?? new(); } } diff --git a/dotnet/src/SemanticKernel.Abstractions/AI/TextCompletion/ITextCompletion.cs b/dotnet/src/SemanticKernel.Abstractions/AI/TextCompletion/ITextCompletion.cs index fdd1e34691a2..49ef12704c99 100644 --- a/dotnet/src/SemanticKernel.Abstractions/AI/TextCompletion/ITextCompletion.cs +++ b/dotnet/src/SemanticKernel.Abstractions/AI/TextCompletion/ITextCompletion.cs @@ -38,11 +38,11 @@ IAsyncEnumerable GetStreamingCompletionsAsync( CancellationToken cancellationToken = default); /// - /// Get streaming results for the prompt and settings of a specific type. - /// Each modality may support for different types of streaming result. + /// Get asynchronous stream of the provided generic type. + /// Each modality may support for different types of streaming contents. /// /// - /// Usage of this method may be more efficient if the connector has a dedicated API to return this result without extra allocations for StreamingResultChunk abstraction. + /// Usage of this method with value types may be more efficient if the connector supports it. /// /// Throws if the specified type is not the same or fail to cast /// The prompt to complete. diff --git a/dotnet/src/SemanticKernel.Core/Functions/SKFunctionFromMethod.cs b/dotnet/src/SemanticKernel.Core/Functions/SKFunctionFromMethod.cs index 8b1824b4dd69..700319e56733 100644 --- a/dotnet/src/SemanticKernel.Core/Functions/SKFunctionFromMethod.cs +++ b/dotnet/src/SemanticKernel.Core/Functions/SKFunctionFromMethod.cs @@ -165,7 +165,7 @@ public async IAsyncEnumerable InvokeStreamingAsync( continue; } - if (chunk is StreamingMethodResultChunk nativeChunk) + if (chunk is StreamingMethodContent nativeChunk) { yield return (T)nativeChunk.Value; } @@ -195,7 +195,7 @@ public async IAsyncEnumerable InvokeStreamingAsync( continue; } - if (chunk is StreamingMethodResultChunk nativeChunk) + if (chunk is StreamingMethodContent nativeChunk) { genericChunk = (T)nativeChunk.Value; continue; @@ -321,7 +321,7 @@ private delegate ValueTask ImplementationFunc( SKContext context, CancellationToken cancellationToken); - private delegate IAsyncEnumerable ImplementationStreamingFunc( + private delegate IAsyncEnumerable ImplementationStreamingFunc( ITextCompletion? textCompletion, AIRequestSettings? requestSettingsk, Kernel kernel, @@ -429,7 +429,7 @@ ValueTask Function(ITextCompletion? text, AIRequestSettings? req } // Create the streaming func - async IAsyncEnumerable StreamingFunction( + async IAsyncEnumerable StreamingFunction( ITextCompletion? text, AIRequestSettings? requestSettings, Kernel kernel, @@ -472,7 +472,7 @@ async IAsyncEnumerable StreamingFunction( { object currentItem = currentProperty.GetValue(asyncEnumerator); - yield return new StreamingMethodResultChunk(currentItem); + yield return new StreamingMethodContent(currentItem); } } else @@ -487,7 +487,7 @@ async IAsyncEnumerable StreamingFunction( // The enumeration will only return if there's actually a result. if (functionResult.Value is not null) { - yield return new StreamingMethodResultChunk(functionResult.Value); + yield return new StreamingMethodContent(functionResult.Value); } } } diff --git a/dotnet/src/SemanticKernel.Core/Functions/SKFunctionFromPrompt.cs b/dotnet/src/SemanticKernel.Core/Functions/SKFunctionFromPrompt.cs index c967d38a37ae..03c304543ce6 100644 --- a/dotnet/src/SemanticKernel.Core/Functions/SKFunctionFromPrompt.cs +++ b/dotnet/src/SemanticKernel.Core/Functions/SKFunctionFromPrompt.cs @@ -200,7 +200,7 @@ public async IAsyncEnumerable InvokeStreamingAsync( } StringBuilder fullCompletion = new(); - IAsyncEnumerator enumerator = textCompletion.GetStreamingChunksAsync(renderedPrompt, requestSettings ?? defaultRequestSettings, cancellationToken).GetAsyncEnumerator(cancellationToken); + IAsyncEnumerator enumerator = textCompletion.GetStreamingContentAsync(renderedPrompt, requestSettings ?? defaultRequestSettings, cancellationToken).GetAsyncEnumerator(cancellationToken); // Manually handling the enumeration to properly log any exception bool moreItems; diff --git a/dotnet/src/SemanticKernel.Core/Functions/StreamingMethodResultChunk.cs b/dotnet/src/SemanticKernel.Core/Functions/StreamingMethodContent.cs similarity index 78% rename from dotnet/src/SemanticKernel.Core/Functions/StreamingMethodResultChunk.cs rename to dotnet/src/SemanticKernel.Core/Functions/StreamingMethodContent.cs index a7a04508f0ee..335a380f5961 100644 --- a/dotnet/src/SemanticKernel.Core/Functions/StreamingMethodResultChunk.cs +++ b/dotnet/src/SemanticKernel.Core/Functions/StreamingMethodContent.cs @@ -11,7 +11,7 @@ namespace Microsoft.SemanticKernel; /// /// Method function streaming result chunk. /// -public sealed class StreamingMethodResultChunk : StreamingContent +public sealed class StreamingMethodContent : StreamingContent { /// public override int ChoiceIndex => 0; @@ -40,11 +40,11 @@ public override string ToString() } /// - /// Initializes a new instance of the class. + /// Initializes a new instance of the class. /// - /// Underlying object that represents the chunk - public StreamingMethodResultChunk(object innerResultChunk) : base(innerResultChunk) + /// Underlying object that represents the chunk + public StreamingMethodContent(object innerContent) : base(innerContent) { - this.Value = innerResultChunk; + this.Value = innerContent; } } diff --git a/dotnet/src/SemanticKernel.UnitTests/Functions/OrderedIAIServiceConfigurationProviderTests.cs b/dotnet/src/SemanticKernel.UnitTests/Functions/OrderedIAIServiceConfigurationProviderTests.cs index b3b115539fc9..6176aad6b3f4 100644 --- a/dotnet/src/SemanticKernel.UnitTests/Functions/OrderedIAIServiceConfigurationProviderTests.cs +++ b/dotnet/src/SemanticKernel.UnitTests/Functions/OrderedIAIServiceConfigurationProviderTests.cs @@ -221,11 +221,6 @@ public IAsyncEnumerable GetStreamingCompletionsAsync(strin { throw new NotImplementedException(); } - - public IAsyncEnumerable GetStreamingChunksAsync(string input, AIRequestSettings? requestSettings = null, CancellationToken cancellationToken = default) - { - throw new NotImplementedException(); - } } #endregion } From ac9dbcf6ca18e36b261efe9f7ec4b66067ef7cbf Mon Sep 17 00:00:00 2001 From: Roger Barreto <19890735+RogerBarreto@users.noreply.github.com> Date: Thu, 23 Nov 2023 08:53:51 +0000 Subject: [PATCH 31/46] Warning fix --- .../Connectors.AI.OpenAI/AzureSdk/StreamingChatContent.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/dotnet/src/Connectors/Connectors.AI.OpenAI/AzureSdk/StreamingChatContent.cs b/dotnet/src/Connectors/Connectors.AI.OpenAI/AzureSdk/StreamingChatContent.cs index 3038be7565fb..1bb4da876f60 100644 --- a/dotnet/src/Connectors/Connectors.AI.OpenAI/AzureSdk/StreamingChatContent.cs +++ b/dotnet/src/Connectors/Connectors.AI.OpenAI/AzureSdk/StreamingChatContent.cs @@ -1,7 +1,6 @@ // Copyright (c) Microsoft. All rights reserved. using System.Text; -using System.Text.Json; using Azure.AI.OpenAI; using Microsoft.SemanticKernel.AI; using Microsoft.SemanticKernel.AI.ChatCompletion; From 5911482a6c1c92f7f44ffc5fa65998c72c697c90 Mon Sep 17 00:00:00 2001 From: Roger Barreto <19890735+RogerBarreto@users.noreply.github.com> Date: Thu, 23 Nov 2023 10:27:14 +0000 Subject: [PATCH 32/46] Warning fix --- dotnet/src/SemanticKernel.Core/KernelExtensions.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dotnet/src/SemanticKernel.Core/KernelExtensions.cs b/dotnet/src/SemanticKernel.Core/KernelExtensions.cs index e5bf7210f8cc..36c231fb5077 100644 --- a/dotnet/src/SemanticKernel.Core/KernelExtensions.cs +++ b/dotnet/src/SemanticKernel.Core/KernelExtensions.cs @@ -6,8 +6,8 @@ using System.IO; using System.Linq; using System.Reflection; -using System.Text.Json; using System.Runtime.CompilerServices; +using System.Text.Json; using System.Threading; using System.Threading.Tasks; using Microsoft.Extensions.Logging; From 7c7f4f07f773b2b6c132cc92e7b22f0b7c104f66 Mon Sep 17 00:00:00 2001 From: Roger Barreto <19890735+RogerBarreto@users.noreply.github.com> Date: Thu, 23 Nov 2023 11:08:46 +0000 Subject: [PATCH 33/46] Fix Method function streaming bug --- .../Functions/SKFunctionFromMethod.cs | 29 ++++++++++--------- .../Functions/SKFunctionFromPrompt.cs | 2 ++ 2 files changed, 18 insertions(+), 13 deletions(-) diff --git a/dotnet/src/SemanticKernel.Core/Functions/SKFunctionFromMethod.cs b/dotnet/src/SemanticKernel.Core/Functions/SKFunctionFromMethod.cs index 815e894500e5..92b092a99db8 100644 --- a/dotnet/src/SemanticKernel.Core/Functions/SKFunctionFromMethod.cs +++ b/dotnet/src/SemanticKernel.Core/Functions/SKFunctionFromMethod.cs @@ -162,49 +162,52 @@ protected override async IAsyncEnumerable InvokeCoreStreamingAsync( IAsyncEnumerator enumerator = this._streamingFunction(null, requestSettings, kernel, context, cancellationToken).GetAsyncEnumerator(cancellationToken); - T? genericChunk = default; - bool moreItems; + T? genericChunk; + bool moreChunks; // Manually handling the enumeration to properly log any exception do { + genericChunk = default; try { - moreItems = await enumerator.MoveNextAsync().ConfigureAwait(false); + cancellationToken.ThrowIfCancellationRequested(); - if (moreItems) + moreChunks = await enumerator.MoveNextAsync().ConfigureAwait(false); + + if (moreChunks) { var chunk = enumerator.Current; if (typeof(T).IsSubclassOf(typeof(StreamingContent)) || typeof(T) == typeof(StreamingContent)) { genericChunk = (T)(object)chunk; - continue; } - - if (chunk is StreamingMethodContent nativeChunk) + else if (chunk is StreamingMethodContent nativeChunk) { + // If the provided T is not a specialization of StreamingContent interface, cast the function value as is to the T genericChunk = (T)nativeChunk.Value; - continue; } - - throw new NotSupportedException($"Streaming result chunk of type {typeof(T)} is not supported."); + else + { + throw new NotSupportedException($"Streaming result chunk of type {typeof(T)} is not supported."); + } } } catch (Exception e) { if (this._logger.IsEnabled(LogLevel.Error)) { - this._logger.LogError(e, "Function {Name} execution failed: {Error}", this.Name, e.Message); + this._logger.LogError(e, "Function {Name} streaming execution failed: {Error}", this.Name, e.Message); } throw; } - if (moreItems && genericChunk is not null) + if (moreChunks && genericChunk is not null) { yield return genericChunk; } - } while (moreItems); + } while (moreChunks); var result = this.CallFunctionInvoked(context); diff --git a/dotnet/src/SemanticKernel.Core/Functions/SKFunctionFromPrompt.cs b/dotnet/src/SemanticKernel.Core/Functions/SKFunctionFromPrompt.cs index 3481fa28d3cd..4d219e105c9e 100644 --- a/dotnet/src/SemanticKernel.Core/Functions/SKFunctionFromPrompt.cs +++ b/dotnet/src/SemanticKernel.Core/Functions/SKFunctionFromPrompt.cs @@ -198,6 +198,8 @@ protected override async IAsyncEnumerable InvokeCoreStreamingAsync( T? genericChunk = default; try { + cancellationToken.ThrowIfCancellationRequested(); + moreChunks = await enumerator.MoveNextAsync().ConfigureAwait(false); if (moreChunks) { From 19de12b12fae0520c0a527e6c4764a76fcc34aac Mon Sep 17 00:00:00 2001 From: Roger Barreto <19890735+RogerBarreto@users.noreply.github.com> Date: Thu, 23 Nov 2023 11:20:47 +0000 Subject: [PATCH 34/46] Removing ISKFunction remaints --- .../Extensions/KernelFunctionExtensions.cs | 22 ------ .../Functions/ISKFunction.cs | 67 ------------------- .../Functions/KernelFunction.cs | 2 +- .../Functions/KernelFunctionExtensions.cs | 33 +++++++++ 4 files changed, 34 insertions(+), 90 deletions(-) delete mode 100644 dotnet/src/SemanticKernel.Abstractions/Functions/ISKFunction.cs create mode 100644 dotnet/src/SemanticKernel.Abstractions/Functions/KernelFunctionExtensions.cs diff --git a/dotnet/src/Experimental/Assistants/Extensions/KernelFunctionExtensions.cs b/dotnet/src/Experimental/Assistants/Extensions/KernelFunctionExtensions.cs index 2b7f11deb16b..71261ee7f005 100644 --- a/dotnet/src/Experimental/Assistants/Extensions/KernelFunctionExtensions.cs +++ b/dotnet/src/Experimental/Assistants/Extensions/KernelFunctionExtensions.cs @@ -3,11 +3,8 @@ using System; using System.Collections.Generic; using System.Linq; -using System.Threading; using Json.More; -using Microsoft.SemanticKernel.AI; using Microsoft.SemanticKernel.Experimental.Assistants.Models; -using Microsoft.SemanticKernel.Orchestration; namespace Microsoft.SemanticKernel.Experimental.Assistants.Extensions; @@ -70,25 +67,6 @@ public static ToolModel ToToolModel(this KernelFunction function, string pluginN return payload; } - /// - /// Invoke the in streaming mode. - /// - /// Target function - /// The kernel - /// SK context - /// LLM completion settings (for semantic functions only) - /// The to monitor for cancellation requests. The default is . - /// A asynchronous list of streaming result chunks - public static IAsyncEnumerable InvokeStreamingAsync( - this ISKFunction function, - Kernel kernel, - SKContext context, - AIRequestSettings? requestSettings = null, - CancellationToken cancellationToken = default) - { - return function.InvokeStreamingAsync(kernel, context, requestSettings, cancellationToken); - } - private static string ConvertType(Type? type) { if (type == null || type == typeof(string)) diff --git a/dotnet/src/SemanticKernel.Abstractions/Functions/ISKFunction.cs b/dotnet/src/SemanticKernel.Abstractions/Functions/ISKFunction.cs deleted file mode 100644 index 1b1e90d4070c..000000000000 --- a/dotnet/src/SemanticKernel.Abstractions/Functions/ISKFunction.cs +++ /dev/null @@ -1,67 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System.Collections.Generic; -using System.Threading; -using System.Threading.Tasks; -using Microsoft.SemanticKernel.AI; -using Microsoft.SemanticKernel.Orchestration; - -#pragma warning disable IDE0130 -// ReSharper disable once CheckNamespace - Using the main namespace -namespace Microsoft.SemanticKernel; -#pragma warning restore IDE0130 - -/// -/// Semantic Kernel callable function interface -/// -public interface ISKFunction -{ - /// - /// Name of the function. The name is used by the function collection and in prompt templates e.g. {{pluginName.functionName}} - /// - string Name { get; } - - /// - /// Function description. The description is used in combination with embeddings when searching relevant functions. - /// - string Description { get; } - - /// - /// Model request settings. - /// - IEnumerable ModelSettings { get; } - - /// - /// Gets the metadata describing the function. - /// - /// An instance of describing the function - SKFunctionMetadata GetMetadata(); - - /// - /// Invoke the . - /// - /// The kernel - /// SK context - /// LLM completion settings (for semantic functions only) - /// The to monitor for cancellation requests. The default is . - /// The updated context, potentially a new one if context switching is implemented. - Task InvokeAsync( - Kernel kernel, - SKContext context, - AIRequestSettings? requestSettings = null, - CancellationToken cancellationToken = default); - - /// - /// Invoke the in streaming mode. - /// - /// The kernel - /// SK context - /// LLM completion settings (for semantic functions only) - /// The to monitor for cancellation requests. The default is . - /// A asynchronous list of streaming result chunks - IAsyncEnumerable InvokeStreamingAsync( - Kernel kernel, - SKContext context, - AIRequestSettings? requestSettings = null, - CancellationToken cancellationToken = default); -} diff --git a/dotnet/src/SemanticKernel.Abstractions/Functions/KernelFunction.cs b/dotnet/src/SemanticKernel.Abstractions/Functions/KernelFunction.cs index b5aee4bc6e96..97ef6b26dcaa 100644 --- a/dotnet/src/SemanticKernel.Abstractions/Functions/KernelFunction.cs +++ b/dotnet/src/SemanticKernel.Abstractions/Functions/KernelFunction.cs @@ -125,7 +125,7 @@ public async Task InvokeAsync( } /// - /// Invoke the in streaming mode. + /// Invoke the in streaming mode. /// /// The kernel /// SK context diff --git a/dotnet/src/SemanticKernel.Abstractions/Functions/KernelFunctionExtensions.cs b/dotnet/src/SemanticKernel.Abstractions/Functions/KernelFunctionExtensions.cs new file mode 100644 index 000000000000..3b12ec018b69 --- /dev/null +++ b/dotnet/src/SemanticKernel.Abstractions/Functions/KernelFunctionExtensions.cs @@ -0,0 +1,33 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Collections.Generic; +using Microsoft.SemanticKernel.AI; +using Microsoft.SemanticKernel.Orchestration; +using System.Threading; + +namespace Microsoft.SemanticKernel.Functions; + +/// +/// Kernel function extensions class. +/// +public static class KernelFunctionExtensions +{ + /// + /// Invoke the in streaming mode. + /// + /// Target function + /// The kernel + /// SK context + /// LLM completion settings (for semantic functions only) + /// The to monitor for cancellation requests. The default is . + /// A asynchronous list of streaming result chunks + public static IAsyncEnumerable InvokeStreamingAsync( + this KernelFunction function, + Kernel kernel, + SKContext context, + AIRequestSettings? requestSettings = null, + CancellationToken cancellationToken = default) + { + return function.InvokeStreamingAsync(kernel, context, requestSettings, cancellationToken); + } +} From 9460a0a47ff22e39294ec4851e6e154964dc70c4 Mon Sep 17 00:00:00 2001 From: Roger Barreto <19890735+RogerBarreto@users.noreply.github.com> Date: Thu, 23 Nov 2023 15:37:53 +0000 Subject: [PATCH 35/46] Update readme + variable legibility --- docs/decisions/0023-kernel-streaming.md | 2 +- .../Functions/KernelFunction.cs | 12 ++++++------ .../Functions/SKFunctionFromMethod.cs | 10 +++++----- .../Functions/SKFunctionFromPrompt.cs | 10 +++++----- 4 files changed, 17 insertions(+), 17 deletions(-) diff --git a/docs/decisions/0023-kernel-streaming.md b/docs/decisions/0023-kernel-streaming.md index 9948c039c7fa..c4116cb5eaa9 100644 --- a/docs/decisions/0023-kernel-streaming.md +++ b/docs/decisions/0023-kernel-streaming.md @@ -30,7 +30,7 @@ Needs to be clear for the sk developer when he is attempting to get streaming da - Streaming with plans will not be supported in this phase. Attempting to do so will throw an exception. - Kernel streaming will not support multiple functions (pipeline). - Input streaming will not be supported in this phase. -- Skipping streaming functions will not be supported in this phase. +- Post Hook Skipping, Repeat and Cancelling of streaming functions are not supported. ## Considered Options diff --git a/dotnet/src/SemanticKernel.Abstractions/Functions/KernelFunction.cs b/dotnet/src/SemanticKernel.Abstractions/Functions/KernelFunction.cs index ef73e59a3089..1d36cc422155 100644 --- a/dotnet/src/SemanticKernel.Abstractions/Functions/KernelFunction.cs +++ b/dotnet/src/SemanticKernel.Abstractions/Functions/KernelFunction.cs @@ -131,7 +131,7 @@ public async Task InvokeAsync( /// SK context /// LLM completion settings (for semantic functions only) /// The to monitor for cancellation requests. The default is . - /// A asynchronous list of streaming result chunks + /// A asynchronous list of streaming content chunks public async IAsyncEnumerable InvokeStreamingAsync( Kernel kernel, SKContext context, @@ -150,15 +150,15 @@ public async IAsyncEnumerable InvokeStreamingAsync( IAsyncEnumerator enumerator = this.InvokeCoreStreamingAsync(kernel, context, requestSettings, cancellationToken).GetAsyncEnumerator(cancellationToken); // Manually handling the enumeration to properly log any exception - bool moreChunks; + bool hasNextChunk; do { T? genericChunk = default; try { cancellationToken.ThrowIfCancellationRequested(); - moreChunks = await enumerator.MoveNextAsync().ConfigureAwait(false); - if (moreChunks) + hasNextChunk = await enumerator.MoveNextAsync().ConfigureAwait(false); + if (hasNextChunk) { genericChunk = enumerator.Current; fullCompletion.Append(genericChunk); @@ -182,11 +182,11 @@ public async IAsyncEnumerable InvokeStreamingAsync( throw; } - if (moreChunks && genericChunk is not null) + if (hasNextChunk && genericChunk is not null) { yield return genericChunk; } - } while (moreChunks); + } while (hasNextChunk); if (logger.IsEnabled(LogLevel.Trace)) { diff --git a/dotnet/src/SemanticKernel.Core/Functions/SKFunctionFromMethod.cs b/dotnet/src/SemanticKernel.Core/Functions/SKFunctionFromMethod.cs index 393189d4a97b..0a4b37f9b499 100644 --- a/dotnet/src/SemanticKernel.Core/Functions/SKFunctionFromMethod.cs +++ b/dotnet/src/SemanticKernel.Core/Functions/SKFunctionFromMethod.cs @@ -170,7 +170,7 @@ protected override async IAsyncEnumerable InvokeCoreStreamingAsync( IAsyncEnumerator enumerator = this._streamingFunction(null, requestSettings, kernel, context, cancellationToken).GetAsyncEnumerator(cancellationToken); T? genericChunk; - bool moreChunks; + bool hasNextChunk; // Manually handling the enumeration to properly log any exception do @@ -180,9 +180,9 @@ protected override async IAsyncEnumerable InvokeCoreStreamingAsync( { cancellationToken.ThrowIfCancellationRequested(); - moreChunks = await enumerator.MoveNextAsync().ConfigureAwait(false); + hasNextChunk = await enumerator.MoveNextAsync().ConfigureAwait(false); - if (moreChunks) + if (hasNextChunk) { var chunk = enumerator.Current; @@ -210,11 +210,11 @@ protected override async IAsyncEnumerable InvokeCoreStreamingAsync( throw; } - if (moreChunks && genericChunk is not null) + if (hasNextChunk && genericChunk is not null) { yield return genericChunk; } - } while (moreChunks); + } while (hasNextChunk); // Invoke the post hook. var (invokedEventArgs, result) = this.CallFunctionInvoked(kernel, context); diff --git a/dotnet/src/SemanticKernel.Core/Functions/SKFunctionFromPrompt.cs b/dotnet/src/SemanticKernel.Core/Functions/SKFunctionFromPrompt.cs index d080ed94ed27..c32035ba83ee 100644 --- a/dotnet/src/SemanticKernel.Core/Functions/SKFunctionFromPrompt.cs +++ b/dotnet/src/SemanticKernel.Core/Functions/SKFunctionFromPrompt.cs @@ -199,7 +199,7 @@ protected override async IAsyncEnumerable InvokeCoreStreamingAsync( IAsyncEnumerator enumerator = textCompletion.GetStreamingContentAsync(renderedPrompt, requestSettings ?? defaultRequestSettings, cancellationToken).GetAsyncEnumerator(cancellationToken); // Manually handling the enumeration to properly log any exception - bool moreChunks; + bool hasNextChunk; do { T? genericChunk = default; @@ -207,8 +207,8 @@ protected override async IAsyncEnumerable InvokeCoreStreamingAsync( { cancellationToken.ThrowIfCancellationRequested(); - moreChunks = await enumerator.MoveNextAsync().ConfigureAwait(false); - if (moreChunks) + hasNextChunk = await enumerator.MoveNextAsync().ConfigureAwait(false); + if (hasNextChunk) { genericChunk = enumerator.Current; fullCompletion.Append(genericChunk); @@ -227,11 +227,11 @@ protected override async IAsyncEnumerable InvokeCoreStreamingAsync( throw; } - if (moreChunks && genericChunk is not null) + if (hasNextChunk && genericChunk is not null) { yield return genericChunk; } - } while (moreChunks); + } while (hasNextChunk); // Update the result with the completion context.Variables.Update(fullCompletion.ToString()); From fdefcc3fdc42e9bc5c50fd506e7491f63b99b80a Mon Sep 17 00:00:00 2001 From: Roger Barreto <19890735+RogerBarreto@users.noreply.github.com> Date: Thu, 23 Nov 2023 19:12:01 +0000 Subject: [PATCH 36/46] Simplifying the CLients, passing metadata where needed --- .../Example72_KernelStreaming.cs | 185 +----------------- .../AzureSdk/ClientBase.cs | 20 +- .../AzureSdk/StreamingChatContent.cs | 18 +- .../AzureSdk/StreamingChatWithDataContent.cs | 4 +- .../AzureOpenAIChatCompletionWithData.cs | 27 ++- 5 files changed, 38 insertions(+), 216 deletions(-) diff --git a/dotnet/samples/KernelSyntaxExamples/Example72_KernelStreaming.cs b/dotnet/samples/KernelSyntaxExamples/Example72_KernelStreaming.cs index dd5a5097c4dc..8bc203aa4b53 100644 --- a/dotnet/samples/KernelSyntaxExamples/Example72_KernelStreaming.cs +++ b/dotnet/samples/KernelSyntaxExamples/Example72_KernelStreaming.cs @@ -1,12 +1,6 @@ // Copyright (c) Microsoft. All rights reserved. using System; -using System.Collections.Generic; -using System.ComponentModel; -using System.Globalization; -using System.Text; -using System.Text.Json; -using System.Threading; using System.Threading.Tasks; using Microsoft.SemanticKernel; using Microsoft.SemanticKernel.Connectors.AI.OpenAI; @@ -34,7 +28,7 @@ public static async Task RunAsync() if (apiKey == null || chatDeploymentName == null || endpoint == null) { Console.WriteLine("Azure endpoint, apiKey, or deploymentName not found. Skipping example."); - return; + return;j } Kernel kernel = new KernelBuilder() @@ -53,182 +47,19 @@ public static async Task RunAsync() Console.WriteLine("\n=== Semantic Function - Streaming ===\n"); // Streaming can be of any type depending on the underlying service the function is using. - await foreach (var update in kernel.RunStreamingAsync(funyParagraphFunction)) + await foreach (var update in kernel.RunStreamingAsync(funyParagraphFunction)) { // You will be always able to know the type of the update by checking the Type property. - if (update is StreamingChatContent chatUpdate) + if (!roleDisplayed && update.Role.HasValue) { - if (!roleDisplayed && chatUpdate.Role.HasValue) - { - Console.WriteLine($"Role: {chatUpdate.Role}"); - roleDisplayed = true; - } - - if (chatUpdate.Content is { Length: > 0 }) - { - Console.Write(chatUpdate.Content); - } - } - }; - - var plugin = kernel.ImportPluginFromObject(new MyNativePlugin(), "MyNativePlugin"); - - await NativeFunctionStreamingValueTypeAsync(kernel, plugin); - - await NativeFunctionStreamingComplexTypeAsync(kernel, plugin); - - await NativeFunctionValueTypeAsync(kernel, plugin); - - await NativeFunctionComplexTypeAsync(kernel, plugin); - } - - private static async Task NativeFunctionStreamingValueTypeAsync(Kernel kernel, ISKPlugin plugin) - { - Console.WriteLine("\n\n=== Native Streaming Functions - Streaming (Value Type) ===\n"); - - // Native string value type streaming function - await foreach (var update in kernel.RunStreamingAsync(plugin["MyValueTypeStreamingNativeFunction"], "My Value Type Streaming Function Input")) - { - if (update is StreamingMethodContent nativeUpdate) - { - Console.Write(nativeUpdate.Value); - } - } - } - - private static async Task NativeFunctionStreamingComplexTypeAsync(Kernel kernel, ISKPlugin plugin) - { - Console.WriteLine("\n\n=== Native Streaming Functions - Streaming (Complex Type) ===\n"); - - // Native complex type streaming function - await foreach (var update in kernel.RunStreamingAsync(plugin["MyComplexTypeStreamingNativeFunction"], "My Complex Type Streaming Function Input")) - { - // the complex type will be available thru the Value property of the native update abstraction. - if (update is StreamingMethodContent nativeUpdate && nativeUpdate.Value is MyStreamingBlock myComplexType) - { - Console.WriteLine(Encoding.UTF8.GetString(myComplexType.Content)); + Console.WriteLine($"Role: {update.Role}"); + roleDisplayed = true; } - } - } - private static async Task NativeFunctionValueTypeAsync(Kernel kernel, ISKPlugin plugin) - { - Console.WriteLine("\n=== Native Non-Streaming Functions - Streaming (Value Type) ===\n"); - - // Native functions that don't support streaming and return value types can be executed with the streaming API's but the behavior will not be streamlike. - await foreach (var update in kernel.RunStreamingAsync(plugin["MyValueTypeNativeFunction"], "My Value Type Non Streaming Function Input")) - { - // the complex type will be available thru the Value property of the native update abstraction. - if (update is StreamingMethodContent nativeUpdate) + if (update.Content is { Length: > 0 }) { - Console.WriteLine(nativeUpdate.Value); + Console.Write(update.Content); } - } - } - - private static async Task NativeFunctionComplexTypeAsync(Kernel kernel, ISKPlugin plugin) - { - Console.WriteLine("\n=== Native Non-Streaming Functions - Streaming (Complex Type) ===\n"); - - // Native functions that don't support streaming and return complex types can be executed with the streaming API's but the behavior will not be streamlike. - await foreach (var update in kernel.RunStreamingAsync(plugin["MyComplexTypeNativeFunction"], "My Complex Type Non Streaming Function Input")) - { - // the complex type will be available thru the Value property of the native update abstraction. - if (update is StreamingMethodContent nativeUpdate && nativeUpdate.Value is MyCustomType myComplexType) - { - Console.WriteLine($"Text: {myComplexType.Text}, Number: {myComplexType.Number}"); - } - } - } - - private sealed class MyNativePlugin - { - [SKFunction] - public async IAsyncEnumerable MyValueTypeStreamingNativeFunctionAsync(string input) - { - var result = "My streaming example, should load word by word in 0.5sec".Split(' '); - foreach (var item in result) - { - yield return $"{item} "; - await Task.Delay(500); - } - } - - [SKFunction] - public async IAsyncEnumerable MyComplexTypeStreamingNativeFunctionAsync(string input) - { - // Base64 encoded string "Base64VideoPacket1 Base64VideoPacket2 Base64VideoPacket3" - var result = "QmFzZTY0VmlkZW9QYWNrZXQx QmFzZTY0VmlkZW9QYWNrZXQy QmFzZTY0VmlkZW9QYWNrZXQz".Split(' '); - - foreach (var item in result) - { - yield return new MyStreamingBlock(Convert.FromBase64String(item)); - - await Task.Delay(500); - } - } - - [SKFunction] - public async Task MyValueTypeNativeFunctionAsync(string input) - { - await Task.Delay(1000); - return input; - } - - [SKFunction] - public MyCustomType MyComplexTypeNativeFunction(string input) - { - Thread.Sleep(1000); - return new MyCustomType() { Number = 1, Text = "My Specific Text" }; - } - } - - private sealed record MyStreamingBlock(byte[] Content); - - /// - /// In order to use custom types, should be specified, - /// that will convert object instance to string representation. - /// - /// - /// is used to represent complex object as meaningful string, so - /// it can be passed to AI for further processing using semantic functions. - /// It's possible to choose any format (e.g. XML, JSON, YAML) to represent your object. - /// - [TypeConverter(typeof(MyCustomTypeConverter))] - private sealed class MyCustomType - { - public int Number { get; set; } - - public string? Text { get; set; } - } - - /// - /// Implementation of for . - /// In this example, object instance is serialized with from System.Text.Json, - /// but it's possible to convert object to string using any other serialization logic. - /// -#pragma warning disable CA1812 // instantiated by Kernel - private sealed class MyCustomTypeConverter : TypeConverter -#pragma warning restore CA1812 - { - public override bool CanConvertFrom(ITypeDescriptorContext? context, Type sourceType) => true; - - /// - /// This method is used to convert object from string to actual type. This will allow to pass object to - /// native function which requires it. - /// - public override object? ConvertFrom(ITypeDescriptorContext? context, CultureInfo? culture, object value) - { - return JsonSerializer.Deserialize((string)value); - } - - /// - /// This method is used to convert actual type to string representation, so it can be passed to AI - /// for further processing. - /// - public override object? ConvertTo(ITypeDescriptorContext? context, CultureInfo? culture, object? value, Type destinationType) - { - return JsonSerializer.Serialize(value); - } + }; } } diff --git a/dotnet/src/Connectors/Connectors.AI.OpenAI/AzureSdk/ClientBase.cs b/dotnet/src/Connectors/Connectors.AI.OpenAI/AzureSdk/ClientBase.cs index bcc902637185..ed661ff79648 100644 --- a/dotnet/src/Connectors/Connectors.AI.OpenAI/AzureSdk/ClientBase.cs +++ b/dotnet/src/Connectors/Connectors.AI.OpenAI/AzureSdk/ClientBase.cs @@ -169,13 +169,7 @@ private protected async IAsyncEnumerable InternalGetTextStreamingUpdatesAsync () => this.Client.GetCompletionsStreamingAsync(this.DeploymentOrModelName, options, cancellationToken)).ConfigureAwait(false); using StreamingCompletions streamingChatCompletions = response.Value; - - var metadata = new Dictionary() - { - { $"{nameof(StreamingCompletions)}.{nameof(streamingChatCompletions.Id)}", streamingChatCompletions.Id }, - { $"{nameof(StreamingCompletions)}.{nameof(streamingChatCompletions.Created)}", streamingChatCompletions.Created }, - { $"{nameof(StreamingCompletions)}.{nameof(streamingChatCompletions.PromptFilterResults)}", streamingChatCompletions.PromptFilterResults }, - }; + var responseMetadata = GetResponseMetadata(streamingChatCompletions); int choiceIndex = 0; await foreach (StreamingChoice choice in streamingChatCompletions.GetChoicesStreaming(cancellationToken).ConfigureAwait(false)) @@ -193,7 +187,7 @@ private protected async IAsyncEnumerable InternalGetTextStreamingUpdatesAsync if (typeof(T) == typeof(StreamingTextContent) || typeof(T) == typeof(StreamingContent)) { - yield return (T)(object)new StreamingTextContent(update, choiceIndex, update, metadata); + yield return (T)(object)new StreamingTextContent(update, choiceIndex, update, responseMetadata); continue; } @@ -203,6 +197,16 @@ private protected async IAsyncEnumerable InternalGetTextStreamingUpdatesAsync } } + private static Dictionary GetResponseMetadata(StreamingCompletions streamingChatCompletions) + { + return new Dictionary() + { + { $"{nameof(StreamingCompletions)}.{nameof(streamingChatCompletions.Id)}", streamingChatCompletions.Id }, + { $"{nameof(StreamingCompletions)}.{nameof(streamingChatCompletions.Created)}", streamingChatCompletions.Created }, + { $"{nameof(StreamingCompletions)}.{nameof(streamingChatCompletions.PromptFilterResults)}", streamingChatCompletions.PromptFilterResults }, + }; + } + /// /// Generates an embedding from the given . /// diff --git a/dotnet/src/Connectors/Connectors.AI.OpenAI/AzureSdk/StreamingChatContent.cs b/dotnet/src/Connectors/Connectors.AI.OpenAI/AzureSdk/StreamingChatContent.cs index 1bb4da876f60..017243f6fd55 100644 --- a/dotnet/src/Connectors/Connectors.AI.OpenAI/AzureSdk/StreamingChatContent.cs +++ b/dotnet/src/Connectors/Connectors.AI.OpenAI/AzureSdk/StreamingChatContent.cs @@ -1,5 +1,6 @@ // Copyright (c) Microsoft. All rights reserved. +using System.Collections.Generic; using System.Text; using Azure.AI.OpenAI; using Microsoft.SemanticKernel.AI; @@ -35,26 +36,13 @@ public class StreamingChatContent : StreamingContent /// public string? Name { get; } - /// - /// Create a new instance of the class. - /// - /// Original Connector Azure Message update representation - /// Index of the choice - public StreamingChatContent(AzureOpenAIChatMessage chatMessage, int resultIndex) : base(chatMessage) - { - this.ChoiceIndex = resultIndex; - this.FunctionCall = chatMessage.InnerChatMessage?.FunctionCall; - this.Content = chatMessage.Content; - this.Role = new AuthorRole(chatMessage.Role.ToString()); - this.Name = chatMessage.InnerChatMessage?.Name; - } - /// /// Create a new instance of the class. /// /// Internal Azure SDK Message update representation /// Index of the choice - public StreamingChatContent(Azure.AI.OpenAI.ChatMessage chatMessage, int resultIndex) : base(chatMessage) + /// Additional metadata + public StreamingChatContent(Azure.AI.OpenAI.ChatMessage chatMessage, int resultIndex, Dictionary metadata) : base(chatMessage, metadata) { this.ChoiceIndex = resultIndex; this.FunctionCall = chatMessage.FunctionCall; diff --git a/dotnet/src/Connectors/Connectors.AI.OpenAI/AzureSdk/StreamingChatWithDataContent.cs b/dotnet/src/Connectors/Connectors.AI.OpenAI/AzureSdk/StreamingChatWithDataContent.cs index 116cdb0506ef..26c3e55c4569 100644 --- a/dotnet/src/Connectors/Connectors.AI.OpenAI/AzureSdk/StreamingChatWithDataContent.cs +++ b/dotnet/src/Connectors/Connectors.AI.OpenAI/AzureSdk/StreamingChatWithDataContent.cs @@ -1,6 +1,7 @@ // Copyright (c) Microsoft. All rights reserved. using System; +using System.Collections.Generic; using System.Linq; using System.Text; using System.Text.Json; @@ -28,7 +29,8 @@ public sealed class StreamingChatWithDataContent : StreamingContent /// /// Azure message update representation from WithData apis /// Index of the choice - internal StreamingChatWithDataContent(ChatWithDataStreamingChoice choice, int resultIndex) : base(choice) + /// Additional metadata + internal StreamingChatWithDataContent(ChatWithDataStreamingChoice choice, int resultIndex, Dictionary metadata) : base(choice, metadata) { this.ChoiceIndex = resultIndex; var message = choice.Messages.FirstOrDefault(this.IsValidMessage); diff --git a/dotnet/src/Connectors/Connectors.AI.OpenAI/ChatCompletionWithData/AzureOpenAIChatCompletionWithData.cs b/dotnet/src/Connectors/Connectors.AI.OpenAI/ChatCompletionWithData/AzureOpenAIChatCompletionWithData.cs index c87c42b19f54..b8cba6d564a6 100644 --- a/dotnet/src/Connectors/Connectors.AI.OpenAI/ChatCompletionWithData/AzureOpenAIChatCompletionWithData.cs +++ b/dotnet/src/Connectors/Connectors.AI.OpenAI/ChatCompletionWithData/AzureOpenAIChatCompletionWithData.cs @@ -269,27 +269,14 @@ private async IAsyncEnumerable GetChatStreamingUpdatesAsync(HttpResponseMe } var chatWithDataResponse = this.DeserializeResponse(body); - - // If the provided T is the response type, return the response as is (Breaking Glass) option 1 - if (typeof(T) == chatWithDataResponse.GetType()) - { - yield return (T)(object)chatWithDataResponse; - continue; - } - + var responseMetadata = this.GetResponseMetadata(response); foreach (var choice in chatWithDataResponse.Choices) { // If the provided T is an specialized class of StreamingContent interface if (typeof(T) == typeof(StreamingChatContent) || typeof(T) == typeof(StreamingContent)) { - yield return (T)(object)new StreamingChatWithDataContent(choice, choice.Index); - continue; - } - - if (typeof(T) == choice.GetType()) - { - yield return (T)(object)choice; + yield return (T)(object)new StreamingChatWithDataContent(choice, choice.Index, responseMetadata); continue; } @@ -306,10 +293,20 @@ private async IAsyncEnumerable GetChatStreamingUpdatesAsync(HttpResponseMe { yield return (T)(object)result; } + + throw new NotSupportedException($"Type {typeof(T)} is not supported"); } } } + private Dictionary GetResponseMetadata(HttpResponseMessage response) + { + return new Dictionary() + { + { nameof(HttpResponseMessage), response }, + }; + } + private T DeserializeResponse(string body) { var response = JsonSerializer.Deserialize(body, JsonOptionsCache.ReadPermissive); From 6d6d0a2fce2404d1a2e30e948ae415e76f89eb30 Mon Sep 17 00:00:00 2001 From: Roger Barreto <19890735+RogerBarreto@users.noreply.github.com> Date: Thu, 23 Nov 2023 20:24:06 +0000 Subject: [PATCH 37/46] Removing complexities + Native Streaming Function Support, Logging from Streaming Functions --- .../Example20_HuggingFace.cs | 30 ---- .../Example72_KernelStreaming.cs | 2 +- .../KernelSyntaxExamples/TestConfiguration.cs | 1 - .../HuggingFaceTextCompletion.cs | 52 +----- .../TextCompletion/StreamingTextContent.cs | 65 ++----- .../AzureSdk/ClientBase.cs | 13 +- .../AI/StreamingContent.cs | 6 - .../Functions/KernelFunction.cs | 55 +----- .../Functions/SKFunctionFromMethod.cs | 164 +----------------- .../Functions/SKFunctionFromPrompt.cs | 61 +------ .../SemanticKernel.Core/KernelExtensions.cs | 91 ++-------- 11 files changed, 60 insertions(+), 480 deletions(-) diff --git a/dotnet/samples/KernelSyntaxExamples/Example20_HuggingFace.cs b/dotnet/samples/KernelSyntaxExamples/Example20_HuggingFace.cs index aff4b64f37e8..7c98c4546afe 100644 --- a/dotnet/samples/KernelSyntaxExamples/Example20_HuggingFace.cs +++ b/dotnet/samples/KernelSyntaxExamples/Example20_HuggingFace.cs @@ -13,7 +13,6 @@ public static class Example20_HuggingFace { public static async Task RunAsync() { - await RunInferenceApiStreamingExampleAsync(); await RunInferenceApiExampleAsync(); await RunLlamaExampleAsync(); } @@ -22,9 +21,6 @@ public static async Task RunAsync() /// This example uses HuggingFace Inference API to access hosted models. /// More information here: /// - /// - /// Tested models: google/flan-t5-xxl - /// private static async Task RunInferenceApiExampleAsync() { Console.WriteLine("\n======== HuggingFace Inference API example ========\n"); @@ -43,32 +39,6 @@ private static async Task RunInferenceApiExampleAsync() Console.WriteLine(result.GetValue()); } - /// - /// This example uses HuggingFace Inference API to access hosted models. - /// More information here: - /// - /// - /// Tested models: HuggingFaceH4/zephyr-7b-beta - /// - private static async Task RunInferenceApiStreamingExampleAsync() - { - Console.WriteLine("\n======== HuggingFace Inference API Streaming example ========\n"); - - Kernel kernel = new KernelBuilder() - .WithLoggerFactory(ConsoleLogger.LoggerFactory) - .WithHuggingFaceTextCompletionService( - model: TestConfiguration.HuggingFace.StreamingModelId, - apiKey: TestConfiguration.HuggingFace.ApiKey) - .Build(); - - var questionAnswerFunction = kernel.CreateFunctionFromPrompt("Why is streaming important?"); - - await foreach (var chunk in kernel.RunStreamingAsync(questionAnswerFunction)) - { - Console.Write(chunk); - } - } - /// /// This example uses HuggingFace Llama 2 model and local HTTP server from Semantic Kernel repository. /// How to setup local HTTP server: . diff --git a/dotnet/samples/KernelSyntaxExamples/Example72_KernelStreaming.cs b/dotnet/samples/KernelSyntaxExamples/Example72_KernelStreaming.cs index 8bc203aa4b53..cfeba26793ee 100644 --- a/dotnet/samples/KernelSyntaxExamples/Example72_KernelStreaming.cs +++ b/dotnet/samples/KernelSyntaxExamples/Example72_KernelStreaming.cs @@ -28,7 +28,7 @@ public static async Task RunAsync() if (apiKey == null || chatDeploymentName == null || endpoint == null) { Console.WriteLine("Azure endpoint, apiKey, or deploymentName not found. Skipping example."); - return;j + return; } Kernel kernel = new KernelBuilder() diff --git a/dotnet/samples/KernelSyntaxExamples/TestConfiguration.cs b/dotnet/samples/KernelSyntaxExamples/TestConfiguration.cs index 00b6e52638e5..9b7e62af9d43 100644 --- a/dotnet/samples/KernelSyntaxExamples/TestConfiguration.cs +++ b/dotnet/samples/KernelSyntaxExamples/TestConfiguration.cs @@ -114,7 +114,6 @@ public class HuggingFaceConfig { public string ApiKey { get; set; } public string ModelId { get; set; } - public string StreamingModelId { get; set; } } public class PineconeConfig diff --git a/dotnet/src/Connectors/Connectors.AI.HuggingFace/TextCompletion/HuggingFaceTextCompletion.cs b/dotnet/src/Connectors/Connectors.AI.HuggingFace/TextCompletion/HuggingFaceTextCompletion.cs index 4f8988f23f1f..44c69f3a0162 100644 --- a/dotnet/src/Connectors/Connectors.AI.HuggingFace/TextCompletion/HuggingFaceTextCompletion.cs +++ b/dotnet/src/Connectors/Connectors.AI.HuggingFace/TextCompletion/HuggingFaceTextCompletion.cs @@ -2,7 +2,6 @@ using System; using System.Collections.Generic; -using System.IO; using System.Net.Http; using System.Runtime.CompilerServices; using System.Text.Json; @@ -100,14 +99,16 @@ public async IAsyncEnumerable GetStreamingContentAsync( AIRequestSettings? requestSettings = null, [EnumeratorCancellation] CancellationToken cancellationToken = default) { - await foreach (var result in this.InternalGetStreamingContentAsync(input, cancellationToken).ConfigureAwait(false)) + foreach (var result in await this.ExecuteGetCompletionsAsync(input, cancellationToken).ConfigureAwait(false)) { cancellationToken.ThrowIfCancellationRequested(); + // Gets the non streaming content and returns as one complete result + var content = await result.GetCompletionAsync(cancellationToken).ConfigureAwait(false); // If the provided T is a string, return the completion as is if (typeof(T) == typeof(string)) { - yield return (T)(object)(result.Token?.Text ?? string.Empty); + yield return (T)(object)content; continue; } @@ -115,8 +116,10 @@ public async IAsyncEnumerable GetStreamingContentAsync( if (typeof(T) == typeof(StreamingTextContent) || typeof(T) == typeof(StreamingContent)) { - yield return (T)(object)result; + yield return (T)(object)new StreamingTextContent(content, 1, result); } + + throw new NotSupportedException($"Type {typeof(T)} is not supported"); } } @@ -154,47 +157,6 @@ private async Task> ExecuteGetCompletionsAsy return completionResponse.ConvertAll(c => new TextCompletionResult(c)); } - private async IAsyncEnumerable InternalGetStreamingContentAsync(string text, [EnumeratorCancellation] CancellationToken cancellationToken) - { - var completionRequest = new TextCompletionRequest - { - Input = text, - Stream = true - }; - - using var httpRequestMessage = HttpRequest.CreatePostRequest(this.GetRequestUri(), completionRequest); - - httpRequestMessage.Headers.Add("User-Agent", HttpHeaderValues.UserAgent); - if (!string.IsNullOrEmpty(this._apiKey)) - { - httpRequestMessage.Headers.Add("Authorization", $"Bearer {this._apiKey}"); - } - - using var response = await this._httpClient.SendWithSuccessCheckAsync(httpRequestMessage, cancellationToken).ConfigureAwait(false); - using var stream = await response.Content.ReadAsStreamAndTranslateExceptionAsync().ConfigureAwait(false); - using var reader = new StreamReader(stream); - - const string ServerEventPayloadPrefix = "data:"; - while (!reader.EndOfStream) - { - var body = await reader.ReadLineAsync().ConfigureAwait(false); - - if (body.StartsWith(ServerEventPayloadPrefix, StringComparison.Ordinal)) - { - body = body.Substring(ServerEventPayloadPrefix.Length); - } - - if (string.IsNullOrWhiteSpace(body)) - { - continue; - } - - JsonElement contentObject = JsonSerializer.Deserialize(body); - - yield return new StreamingTextContent(contentObject); - } - } - /// /// Retrieves the request URI based on the provided endpoint and model information. /// diff --git a/dotnet/src/Connectors/Connectors.AI.HuggingFace/TextCompletion/StreamingTextContent.cs b/dotnet/src/Connectors/Connectors.AI.HuggingFace/TextCompletion/StreamingTextContent.cs index 2ce13b7c8865..3f1a1c516dcc 100644 --- a/dotnet/src/Connectors/Connectors.AI.HuggingFace/TextCompletion/StreamingTextContent.cs +++ b/dotnet/src/Connectors/Connectors.AI.HuggingFace/TextCompletion/StreamingTextContent.cs @@ -1,8 +1,7 @@ // Copyright (c) Microsoft. All rights reserved. +using System.Collections.Generic; using System.Text; -using System.Text.Json; -using System.Text.Json.Serialization; using Microsoft.SemanticKernel.AI; namespace Microsoft.SemanticKernel.Connectors.AI.HuggingFace.TextCompletion; @@ -16,32 +15,21 @@ public class StreamingTextContent : StreamingContent public override int ChoiceIndex { get; } /// - /// Complete generated text - /// Only available when the generation is finished + /// Text associated to the update /// - public string? GeneratedText { get; set; } - - /// - /// Optional Generation details - /// Only available when the generation is finished - /// - public string? Details { get; set; } - - /// - /// Token details - /// - public TokenContentModel Token { get; set; } + public string Content { get; } /// /// Create a new instance of the class. /// - /// JsonElement representation of the chunk - public StreamingTextContent(JsonElement jsonChunk) : base(jsonChunk) + /// Text update + /// Index of the choice + /// Inner chunk object + /// Metadata information + public StreamingTextContent(string text, int resultIndex, object? innerContentObject = null, Dictionary? metadata = null) : base(innerContentObject, metadata) { - this.ChoiceIndex = 0; - this.GeneratedText = jsonChunk.GetProperty("generated_text").GetString(); - this.Details = jsonChunk.GetProperty("details").GetString(); - this.Token = JsonSerializer.Deserialize(jsonChunk.GetProperty("token").GetRawText())!; + this.ChoiceIndex = resultIndex; + this.Content = text; } /// @@ -53,37 +41,6 @@ public override byte[] ToByteArray() /// public override string ToString() { - return this.Token?.Text ?? string.Empty; - } - - /// - /// Token class in - /// - public record TokenContentModel - { - /// - /// Id of the token - /// - [JsonPropertyName("id")] - public int Id { get; set; } - - /// - /// Text associated to the Token - /// - [JsonPropertyName("text")] - public string? Text { get; set; } - - /// - /// Log probability of the token - /// - [JsonPropertyName("logprob")] - public decimal LogProb { get; set; } - - /// - /// Is the token a special token? - /// Can be used to ignore tokens when concatenating - /// - [JsonPropertyName("special")] - public bool Special { get; set; } + return this.Content; } } diff --git a/dotnet/src/Connectors/Connectors.AI.OpenAI/AzureSdk/ClientBase.cs b/dotnet/src/Connectors/Connectors.AI.OpenAI/AzureSdk/ClientBase.cs index ed661ff79648..adc2a7c81577 100644 --- a/dotnet/src/Connectors/Connectors.AI.OpenAI/AzureSdk/ClientBase.cs +++ b/dotnet/src/Connectors/Connectors.AI.OpenAI/AzureSdk/ClientBase.cs @@ -207,6 +207,16 @@ private static Dictionary GetResponseMetadata(StreamingCompletio }; } + private static Dictionary GetResponseMetadata(StreamingChatCompletions streamingChatCompletions) + { + return new Dictionary() + { + { $"{nameof(StreamingChatCompletions)}.{nameof(streamingChatCompletions.Id)}", streamingChatCompletions.Id }, + { $"{nameof(StreamingChatCompletions)}.{nameof(streamingChatCompletions.Created)}", streamingChatCompletions.Created }, + { $"{nameof(StreamingChatCompletions)}.{nameof(streamingChatCompletions.PromptFilterResults)}", streamingChatCompletions.PromptFilterResults }, + }; + } + /// /// Generates an embedding from the given . /// @@ -338,6 +348,7 @@ private protected async IAsyncEnumerable InternalGetChatStreamingUpdatesAsync } using StreamingChatCompletions streamingChatCompletions = response.Value; + var responseMetadata = GetResponseMetadata(streamingChatCompletions); int choiceIndex = 0; await foreach (StreamingChatChoice choice in streamingChatCompletions.GetChoicesStreaming(cancellationToken).ConfigureAwait(false)) @@ -354,7 +365,7 @@ private protected async IAsyncEnumerable InternalGetChatStreamingUpdatesAsync if (typeof(T) == typeof(StreamingChatContent) || typeof(T) == typeof(StreamingContent)) { - yield return (T)(object)new StreamingChatContent(chatMessage, choiceIndex); + yield return (T)(object)new StreamingChatContent(chatMessage, choiceIndex, responseMetadata); continue; } diff --git a/dotnet/src/SemanticKernel.Abstractions/AI/StreamingContent.cs b/dotnet/src/SemanticKernel.Abstractions/AI/StreamingContent.cs index 56474cb5d704..7c3b1d2e3733 100644 --- a/dotnet/src/SemanticKernel.Abstractions/AI/StreamingContent.cs +++ b/dotnet/src/SemanticKernel.Abstractions/AI/StreamingContent.cs @@ -1,7 +1,6 @@ // Copyright (c) Microsoft. All rights reserved. using System.Collections.Generic; -using Microsoft.SemanticKernel.Orchestration; namespace Microsoft.SemanticKernel.AI; @@ -47,11 +46,6 @@ public abstract class StreamingContent /// Byte array representation of the chunk public abstract byte[] ToByteArray(); - /// - /// The current context associated the function call. - /// - internal SKContext? Context { get; set; } - /// /// Initializes a new instance of the class. /// diff --git a/dotnet/src/SemanticKernel.Abstractions/Functions/KernelFunction.cs b/dotnet/src/SemanticKernel.Abstractions/Functions/KernelFunction.cs index 1d36cc422155..4329da8dd8c9 100644 --- a/dotnet/src/SemanticKernel.Abstractions/Functions/KernelFunction.cs +++ b/dotnet/src/SemanticKernel.Abstractions/Functions/KernelFunction.cs @@ -147,62 +147,23 @@ public async IAsyncEnumerable InvokeStreamingAsync( long startingTimestamp = Stopwatch.GetTimestamp(); StringBuilder fullCompletion = new(); - IAsyncEnumerator enumerator = this.InvokeCoreStreamingAsync(kernel, context, requestSettings, cancellationToken).GetAsyncEnumerator(cancellationToken); - // Manually handling the enumeration to properly log any exception - bool hasNextChunk; - do + await foreach (var genericChunk in this.InvokeCoreStreamingAsync(kernel, context, requestSettings, cancellationToken)) { - T? genericChunk = default; - try - { - cancellationToken.ThrowIfCancellationRequested(); - hasNextChunk = await enumerator.MoveNextAsync().ConfigureAwait(false); - if (hasNextChunk) - { - genericChunk = enumerator.Current; - fullCompletion.Append(genericChunk); - // Check if genericChunk is a StreamingResultChunk and update the context - if (genericChunk is StreamingContent resultChunk) - { - // This currently is needed so plans can get the context from the chunks to update the variables generated when the stream ends. - resultChunk.Context = context; - } - } - } - catch (Exception ex) - { - tags.Add("error.type", ex.GetType().FullName); - if (logger.IsEnabled(LogLevel.Error)) - { - logger.LogError(ex, "Function streaming failed. Error: {Message}", ex.Message); - } - - LogCompleted(logger, tags, startingTimestamp); - throw; - } - - if (hasNextChunk && genericChunk is not null) - { - yield return genericChunk; - } - } while (hasNextChunk); + fullCompletion.Append(genericChunk); + yield return genericChunk; + } if (logger.IsEnabled(LogLevel.Trace)) { logger.LogTrace("Function streaming succeeded. Result: {Result}", fullCompletion); // Sensitive data, logging as trace, disabled by default } - LogCompleted(logger, tags, startingTimestamp); - - static void LogCompleted(ILogger logger, TagList tags, long startingTimestamp) + TimeSpan duration = new((long)((Stopwatch.GetTimestamp() - startingTimestamp) * (10_000_000.0 / Stopwatch.Frequency))); + s_invocationDuration.Record(duration.TotalSeconds, in tags); + if (logger.IsEnabled(LogLevel.Information)) { - TimeSpan duration = new((long)((Stopwatch.GetTimestamp() - startingTimestamp) * (10_000_000.0 / Stopwatch.Frequency))); - s_invocationDuration.Record(duration.TotalSeconds, in tags); - if (logger.IsEnabled(LogLevel.Information)) - { - logger.LogInformation("Function streaming completed. Duration: {Duration}ms", duration.TotalMilliseconds); - } + logger.LogInformation("Function streaming completed. Duration: {Duration}ms", duration.TotalMilliseconds); } } diff --git a/dotnet/src/SemanticKernel.Core/Functions/SKFunctionFromMethod.cs b/dotnet/src/SemanticKernel.Core/Functions/SKFunctionFromMethod.cs index 0a4b37f9b499..87cd8d3f605c 100644 --- a/dotnet/src/SemanticKernel.Core/Functions/SKFunctionFromMethod.cs +++ b/dotnet/src/SemanticKernel.Core/Functions/SKFunctionFromMethod.cs @@ -65,7 +65,6 @@ public static KernelFunction Create( MethodDetails methodDetails = GetMethodDetails(functionName, method, target, logger); var result = new KernelFunctionFromMethod( methodDetails.Function, - methodDetails.StreamingFunc, methodDetails.Name, description ?? methodDetails.Description, parameters?.ToList() ?? methodDetails.Parameters, @@ -156,75 +155,14 @@ protected override async IAsyncEnumerable InvokeCoreStreamingAsync( AIRequestSettings? requestSettings = null, [EnumeratorCancellation] CancellationToken cancellationToken = default) { - var invokingEventArgs = this.CallFunctionInvoking(kernel, context); - if (invokingEventArgs.IsSkipRequested || invokingEventArgs.CancelToken.IsCancellationRequested) + var functionResult = await this.InvokeCoreAsync(kernel, context, requestSettings, cancellationToken).ConfigureAwait(false); + if (functionResult.Value is T) { - if (this._logger.IsEnabled(LogLevel.Trace)) - { - this._logger.LogTrace("Function {Name} cancelled or skipped prior to invocation streaming.", this.Name); - } - - yield break; + yield return (T)functionResult.Value; } - - IAsyncEnumerator enumerator = this._streamingFunction(null, requestSettings, kernel, context, cancellationToken).GetAsyncEnumerator(cancellationToken); - - T? genericChunk; - bool hasNextChunk; - - // Manually handling the enumeration to properly log any exception - do + else if (functionResult.Value is not null) { - genericChunk = default; - try - { - cancellationToken.ThrowIfCancellationRequested(); - - hasNextChunk = await enumerator.MoveNextAsync().ConfigureAwait(false); - - if (hasNextChunk) - { - var chunk = enumerator.Current; - - if (typeof(T).IsSubclassOf(typeof(StreamingContent)) || typeof(T) == typeof(StreamingContent)) - { - genericChunk = (T)(object)chunk; - } - else if (chunk is StreamingMethodContent nativeChunk) - { - // If the provided T is not a specialization of StreamingContent interface, cast the function value as is to the T - genericChunk = (T)nativeChunk.Value; - } - else - { - throw new NotSupportedException($"Streaming result chunk of type {typeof(T)} is not supported."); - } - } - } - catch (Exception e) - { - if (this._logger.IsEnabled(LogLevel.Error)) - { - this._logger.LogError(e, "Function {Name} streaming execution failed: {Error}", this.Name, e.Message); - } - throw; - } - - if (hasNextChunk && genericChunk is not null) - { - yield return genericChunk; - } - } while (hasNextChunk); - - // Invoke the post hook. - var (invokedEventArgs, result) = this.CallFunctionInvoked(kernel, context); - - if (this._logger.IsEnabled(LogLevel.Trace)) - { - this._logger.LogTrace("Function {Name} invocation streaming {Completion}: {Result}", - this.Name, - invokedEventArgs.CancelToken.IsCancellationRequested ? "canceled" : "completed", - result.Value); + yield return (T)(object)new StreamingMethodContent(functionResult.Value); } } @@ -269,25 +207,16 @@ private delegate ValueTask ImplementationFunc( SKContext context, CancellationToken cancellationToken); - private delegate IAsyncEnumerable ImplementationStreamingFunc( - ITextCompletion? textCompletion, - AIRequestSettings? requestSettingsk, - Kernel kernel, - SKContext context, - CancellationToken cancellationToken); - private static readonly object[] s_cancellationTokenNoneArray = new object[] { CancellationToken.None }; private readonly ImplementationFunc _function; private readonly IReadOnlyList _parameters; private readonly SKReturnParameterMetadata _returnParameter; - private readonly ImplementationStreamingFunc _streamingFunction; private readonly ILogger _logger; - private record struct MethodDetails(string Name, string Description, ImplementationFunc Function, ImplementationStreamingFunc StreamingFunc, List Parameters, SKReturnParameterMetadata ReturnParameter); + private record struct MethodDetails(string Name, string Description, ImplementationFunc Function, List Parameters, SKReturnParameterMetadata ReturnParameter); private KernelFunctionFromMethod( ImplementationFunc implementationFunc, - ImplementationStreamingFunc implementationStreamingFunc, string functionName, string description, IReadOnlyList parameters, @@ -299,7 +228,6 @@ private KernelFunctionFromMethod( this._logger = logger; this._function = implementationFunc; - this._streamingFunction = implementationStreamingFunc; this._parameters = parameters.ToArray(); Verify.ParametersUniqueness(this._parameters); this._returnParameter = returnParameter; @@ -371,76 +299,10 @@ ValueTask Function(ITextCompletion? text, AIRequestSettings? req return returnFunc(functionName!, result, context, kernel); } - // Create the streaming func - async IAsyncEnumerable StreamingFunction( - ITextCompletion? text, - AIRequestSettings? requestSettings, - Kernel kernel, - SKContext context, - [EnumeratorCancellation] CancellationToken cancellationToken) - { - // Create the arguments. - object?[] args = parameterFuncs!.Length != 0 ? new object?[parameterFuncs.Length] : Array.Empty(); - for (int i = 0; i < args.Length; i++) - { - args[i] = parameterFuncs[i](kernel, context, cancellationToken); - } - - if (IsAsyncEnumerable(method, out var enumeratedTypes)) - { - // Invoke the method to get the IAsyncEnumerable instance - object asyncEnumerable = method.Invoke(target, args); - if (asyncEnumerable == null) - { - yield break; - } - - // Get the method for GetAsyncEnumerator() - MethodInfo getAsyncEnumeratorMethod = typeof(IAsyncEnumerable<>) - .MakeGenericType(enumeratedTypes) - .GetMethod("GetAsyncEnumerator"); - - // Get the IAsyncEnumerator type - Type asyncEnumeratorType = typeof(IAsyncEnumerator<>).MakeGenericType(enumeratedTypes); - - // Invoke GetAsyncEnumerator() to get the IAsyncEnumerator instance - object asyncEnumerator = getAsyncEnumeratorMethod.Invoke(asyncEnumerable, new object[] { cancellationToken }); - - // Get the MoveNextAsync() and Current properties - MethodInfo moveNextAsyncMethod = asyncEnumeratorType.GetMethod("MoveNextAsync"); - PropertyInfo currentProperty = asyncEnumeratorType.GetProperty("Current"); - - // Iterate over the items - while (await ((ValueTask)moveNextAsyncMethod.Invoke(asyncEnumerator, null)).ConfigureAwait(false)) - { - object currentItem = currentProperty.GetValue(asyncEnumerator); - - yield return new StreamingMethodContent(currentItem); - } - } - else - { - // When streaming is requested for a non-streaming native method, it returns just one result - object? result = method.Invoke(target, args); - - if (result is not null) - { - var functionResult = await returnFunc(functionName!, result, context, kernel).ConfigureAwait(false); - - // The enumeration will only return if there's actually a result. - if (functionResult.Value is not null) - { - yield return new StreamingMethodContent(functionResult.Value); - } - } - } - } - // And return the details. return new MethodDetails { Function = Function, - StreamingFunc = StreamingFunction, Name = functionName!, Description = method.GetCustomAttribute(inherit: true)?.Description ?? "", Parameters = stringParameterViews, @@ -474,20 +336,6 @@ private static bool IsAsyncMethod(MethodInfo method) return false; } - private static bool IsAsyncEnumerable(MethodInfo method, out Type[] enumeratedTypes) - { - Type returnType = method.ReturnType; - - if (returnType.IsGenericType && returnType.GetGenericTypeDefinition() == typeof(IAsyncEnumerable<>)) - { - enumeratedTypes = returnType.GetGenericArguments(); - return true; - } - - enumeratedTypes = Array.Empty(); - return false; - } - /// /// Gets a delegate for handling the marshaling of a parameter. /// diff --git a/dotnet/src/SemanticKernel.Core/Functions/SKFunctionFromPrompt.cs b/dotnet/src/SemanticKernel.Core/Functions/SKFunctionFromPrompt.cs index c32035ba83ee..ddc5f657e704 100644 --- a/dotnet/src/SemanticKernel.Core/Functions/SKFunctionFromPrompt.cs +++ b/dotnet/src/SemanticKernel.Core/Functions/SKFunctionFromPrompt.cs @@ -168,7 +168,6 @@ protected override async Task InvokeCoreAsync( var result = new FunctionResult(this.Name, context, completion); result.Metadata.Add(AIFunctionResultExtensions.ModelResultsMetadataKey, modelResults); - result.Metadata.Add(SKEventArgsExtensions.RenderedPromptMetadataKey, renderedPrompt); (var invokedEventArgs, result) = this.CallFunctionInvoked(kernel, context, result, renderedPrompt); result.IsCancellationRequested = invokedEventArgs.CancelToken.IsCancellationRequested; @@ -196,50 +195,15 @@ protected override async IAsyncEnumerable InvokeCoreStreamingAsync( } StringBuilder fullCompletion = new(); - IAsyncEnumerator enumerator = textCompletion.GetStreamingContentAsync(renderedPrompt, requestSettings ?? defaultRequestSettings, cancellationToken).GetAsyncEnumerator(cancellationToken); - - // Manually handling the enumeration to properly log any exception - bool hasNextChunk; - do + await foreach (T genericChunk in textCompletion.GetStreamingContentAsync(renderedPrompt, requestSettings ?? defaultRequestSettings, cancellationToken)) { - T? genericChunk = default; - try - { - cancellationToken.ThrowIfCancellationRequested(); - - hasNextChunk = await enumerator.MoveNextAsync().ConfigureAwait(false); - if (hasNextChunk) - { - genericChunk = enumerator.Current; - fullCompletion.Append(genericChunk); - - // Check if genericChunk is a StreamingResultChunk and update the context - if (genericChunk is StreamingContent resultChunk) - { - // This currently is needed so plans can get the context from the chunks to update the variables generated when the stream ends. - resultChunk.Context = context; - } - } - } - catch (Exception ex) when (!ex.IsCriticalException()) - { - this._logger?.LogError(ex, "Prompt function {Name} execution failed with error {Error}", this.Name, ex.Message); - throw; - } - - if (hasNextChunk && genericChunk is not null) - { - yield return genericChunk; - } - } while (hasNextChunk); + cancellationToken.ThrowIfCancellationRequested(); + fullCompletion.Append(genericChunk); + } // Update the result with the completion context.Variables.Update(fullCompletion.ToString()); - var result = new FunctionResult(this.Name, context, fullCompletion.ToString()); - - result.Metadata.Add(SKEventArgsExtensions.RenderedPromptMetadataKey, renderedPrompt); - this.CallFunctionInvoked(kernel, context, null, renderedPrompt); // There is no post cancellation check to override the result as the stream data was already sent. } @@ -335,23 +299,6 @@ private FunctionInvokingEventArgs CallFunctionInvoking(Kernel kernel, SKContext return (eventArgs, result); } - /// - /// Try to get the prompt from the event args metadata. - /// - /// Function invoking event args - /// Default prompt if none is found in metadata - /// - private string GetPromptFromEventArgsMetadataOrDefault(FunctionInvokingEventArgs eventArgs, string defaultPrompt) - { - if (!eventArgs.Metadata.TryGetValue(SKEventArgsExtensions.RenderedPromptMetadataKey, out var renderedPromptFromMetadata)) - { - return defaultPrompt; - } - - // If prompt key exists and was modified to null default to an empty string - return renderedPromptFromMetadata?.ToString() ?? string.Empty; - } - /// Create a random, valid function name. private static string RandomFunctionName() => $"func{Guid.NewGuid():N}"; diff --git a/dotnet/src/SemanticKernel.Core/KernelExtensions.cs b/dotnet/src/SemanticKernel.Core/KernelExtensions.cs index 945c462d49ca..1cad9800330f 100644 --- a/dotnet/src/SemanticKernel.Core/KernelExtensions.cs +++ b/dotnet/src/SemanticKernel.Core/KernelExtensions.cs @@ -6,7 +6,6 @@ using System.IO; using System.Linq; using System.Reflection; -using System.Runtime.CompilerServices; using System.Text.Json; using System.Threading; using System.Threading.Tasks; @@ -560,7 +559,7 @@ public static async Task RunAsync(this Kernel kernel, ContextVar /// Target function /// Current pipeline step /// The logger. - /// + /// Streaming result of the function private static bool IsCancelRequested(FunctionResult result, KernelFunction function, int pipelineStepCount, ILogger logger) { if (result.IsCancellationRequested) @@ -579,7 +578,7 @@ private static bool IsCancelRequested(FunctionResult result, KernelFunction func /// Target function /// Current pipeline step /// The logger. - /// + /// Streaming result of the function private static bool IsSkipRequested(FunctionResult result, KernelFunction function, int pipelineStepCount, ILogger logger) { if (result.IsSkipRequested) @@ -598,7 +597,7 @@ private static bool IsSkipRequested(FunctionResult result, KernelFunction functi /// Target function /// Current pipeline step /// The logger. - /// + /// Streaming result of the function private static bool IsRepeatRequested(FunctionResult result, KernelFunction function, int pipelineStepCount, ILogger logger) { if (result.IsRepeatRequested) @@ -616,86 +615,18 @@ private static bool IsRepeatRequested(FunctionResult result, KernelFunction func /// Target function to run /// Input to process /// The to monitor for cancellation requests. - /// Result of the function composition - public static async IAsyncEnumerable RunStreamingAsync(this Kernel kernel, KernelFunction function, ContextVariables? variables, [EnumeratorCancellation] CancellationToken cancellationToken) - { - var context = kernel.CreateNewContext(variables); - var repeatRequested = false; - - var logger = kernel.LoggerFactory.CreateLogger(typeof(Kernel)); - - do - { - repeatRequested = false; - - var functionDetails = function.GetMetadata(); - await foreach (T update in function.InvokeStreamingAsync(kernel, context, null, cancellationToken).ConfigureAwait(false)) - { - yield return update; - } - - // Repeat is not supported for streaming functions - } - while (repeatRequested); - } - - /// - /// Run a function in streaming mode. - /// - /// Target kernel - /// Target function to run - /// Input to process - /// The to monitor for cancellation requests. - /// Result of the function composition - public static IAsyncEnumerable RunStreamingAsync(this Kernel kernel, KernelFunction function, string input, CancellationToken cancellationToken) - => kernel.RunStreamingAsync(function, new ContextVariables(input), cancellationToken); - - /// - /// Run a function in streaming mode. - /// - /// Target kernel - /// Target function to run - /// Input to process - /// Result of the function composition - public static IAsyncEnumerable RunStreamingAsync(this Kernel kernel, KernelFunction function, string input) - => kernel.RunStreamingAsync(function, new ContextVariables(input), CancellationToken.None); - - /// - /// Run a function in streaming mode. - /// - /// Target kernel - /// Target function to run - /// Result of the function composition - public static IAsyncEnumerable RunStreamingAsync(this Kernel kernel, KernelFunction function) - => kernel.RunStreamingAsync(function, (ContextVariables?)null, CancellationToken.None); - - /// - /// Run a function in streaming mode. - /// - /// Target kernel - /// Target function to run - /// Input to process - /// The to monitor for cancellation requests. - /// Result of the function composition - public static IAsyncEnumerable RunStreamingAsync(this Kernel kernel, KernelFunction function, string input, CancellationToken cancellationToken) - => kernel.RunStreamingAsync(function, new ContextVariables(input), cancellationToken); - - /// - /// Run a function in streaming mode. - /// - /// Target kernel - /// Target function to run - /// Input to process - /// Result of the function composition - public static IAsyncEnumerable RunStreamingAsync(this Kernel kernel, KernelFunction function, string input) - => kernel.RunStreamingAsync(function, new ContextVariables(input), CancellationToken.None); + /// Streaming result of the function + public static IAsyncEnumerable RunStreamingAsync(this Kernel kernel, KernelFunction function, ContextVariables? variables = null, CancellationToken cancellationToken = default) + => function.InvokeStreamingAsync(kernel, kernel.CreateNewContext(variables), null, cancellationToken); /// /// Run a function in streaming mode. /// /// Target kernel /// Target function to run - /// Result of the function composition - public static IAsyncEnumerable RunStreamingAsync(this Kernel kernel, KernelFunction function) - => kernel.RunStreamingAsync(function, (ContextVariables?)null, CancellationToken.None); + /// Input to process + /// Cancellation token + /// Streaming result of the function + public static IAsyncEnumerable RunStreamingAsync(this Kernel kernel, KernelFunction function, ContextVariables? variables = null, CancellationToken cancellationToken = default) + => kernel.RunStreamingAsync(function, variables ?? new ContextVariables(), CancellationToken.None); } From fbd158d979e5e03331d319d406716491a7f6a94d Mon Sep 17 00:00:00 2001 From: Roger Barreto <19890735+RogerBarreto@users.noreply.github.com> Date: Thu, 23 Nov 2023 22:46:24 +0000 Subject: [PATCH 38/46] Reverting HttpRequest... --- dotnet/src/InternalUtilities/src/Http/HttpRequest.cs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/dotnet/src/InternalUtilities/src/Http/HttpRequest.cs b/dotnet/src/InternalUtilities/src/Http/HttpRequest.cs index a0c1a7a792d7..883eb07d9d11 100644 --- a/dotnet/src/InternalUtilities/src/Http/HttpRequest.cs +++ b/dotnet/src/InternalUtilities/src/Http/HttpRequest.cs @@ -2,6 +2,7 @@ using System; using System.Net.Http; +using System.Net.Http.Headers; using System.Text; using System.Text.Json; using Microsoft.SemanticKernel.Text; @@ -41,7 +42,12 @@ private static HttpRequestMessage CreateRequest(HttpMethod method, Uri url, obje HttpContent? content = null; if (payload is not null) { - return new StringContent(JsonSerializer.Serialize(payload, JsonOptionsCache.Default), Encoding.UTF8, "application/json"); + byte[] utf8Bytes = payload is string s ? + Encoding.UTF8.GetBytes(s) : + JsonSerializer.SerializeToUtf8Bytes(payload, JsonOptionsCache.Default); + + content = new ByteArrayContent(utf8Bytes); + content.Headers.ContentType = new MediaTypeHeaderValue("application/json") { CharSet = "utf-8" }; } return content; From a737deb1af87ef36712b7c01bc5c2357e5722a69 Mon Sep 17 00:00:00 2001 From: Roger Barreto <19890735+RogerBarreto@users.noreply.github.com> Date: Thu, 23 Nov 2023 23:26:49 +0000 Subject: [PATCH 39/46] Addressed PR feedback --- .../HuggingFaceTextCompletion.cs | 4 ++-- .../AzureOpenAIChatCompletionWithData.cs | 4 ++-- .../TextCompletion/AzureTextCompletion.cs | 4 ++-- .../TextCompletion/OpenAITextCompletion.cs | 4 ++-- .../AI/ChatCompletion/IChatCompletion.cs | 6 ++--- .../AI/TextCompletion/ITextCompletion.cs | 6 ++--- .../Functions/KernelFunction.cs | 24 ++++--------------- .../Functions/SKFunctionFromMethod.cs | 12 ++++++++-- .../Functions/StreamingMethodContent.cs | 18 +++++++------- 9 files changed, 37 insertions(+), 45 deletions(-) diff --git a/dotnet/src/Connectors/Connectors.AI.HuggingFace/TextCompletion/HuggingFaceTextCompletion.cs b/dotnet/src/Connectors/Connectors.AI.HuggingFace/TextCompletion/HuggingFaceTextCompletion.cs index 44c69f3a0162..6ad24932f427 100644 --- a/dotnet/src/Connectors/Connectors.AI.HuggingFace/TextCompletion/HuggingFaceTextCompletion.cs +++ b/dotnet/src/Connectors/Connectors.AI.HuggingFace/TextCompletion/HuggingFaceTextCompletion.cs @@ -95,11 +95,11 @@ public async Task> GetCompletionsAsync( /// public async IAsyncEnumerable GetStreamingContentAsync( - string input, + string prompt, AIRequestSettings? requestSettings = null, [EnumeratorCancellation] CancellationToken cancellationToken = default) { - foreach (var result in await this.ExecuteGetCompletionsAsync(input, cancellationToken).ConfigureAwait(false)) + foreach (var result in await this.ExecuteGetCompletionsAsync(prompt, cancellationToken).ConfigureAwait(false)) { cancellationToken.ThrowIfCancellationRequested(); // Gets the non streaming content and returns as one complete result diff --git a/dotnet/src/Connectors/Connectors.AI.OpenAI/ChatCompletionWithData/AzureOpenAIChatCompletionWithData.cs b/dotnet/src/Connectors/Connectors.AI.OpenAI/ChatCompletionWithData/AzureOpenAIChatCompletionWithData.cs index b8cba6d564a6..30a5a17bfb13 100644 --- a/dotnet/src/Connectors/Connectors.AI.OpenAI/ChatCompletionWithData/AzureOpenAIChatCompletionWithData.cs +++ b/dotnet/src/Connectors/Connectors.AI.OpenAI/ChatCompletionWithData/AzureOpenAIChatCompletionWithData.cs @@ -122,13 +122,13 @@ public async IAsyncEnumerable GetStreamingCompletionsAsync /// public async IAsyncEnumerable GetStreamingContentAsync( - string input, + string prompt, AIRequestSettings? requestSettings = null, [EnumeratorCancellation] CancellationToken cancellationToken = default) { OpenAIRequestSettings chatRequestSettings = OpenAIRequestSettings.FromRequestSettings(requestSettings); - var chat = this.PrepareChatHistory(input, chatRequestSettings); + var chat = this.PrepareChatHistory(prompt, chatRequestSettings); using var request = this.GetRequest(chat, chatRequestSettings, isStreamEnabled: true); using var response = await this.SendRequestAsync(request, cancellationToken).ConfigureAwait(false); diff --git a/dotnet/src/Connectors/Connectors.AI.OpenAI/TextCompletion/AzureTextCompletion.cs b/dotnet/src/Connectors/Connectors.AI.OpenAI/TextCompletion/AzureTextCompletion.cs index e09ba66bc411..ad67d76f31fe 100644 --- a/dotnet/src/Connectors/Connectors.AI.OpenAI/TextCompletion/AzureTextCompletion.cs +++ b/dotnet/src/Connectors/Connectors.AI.OpenAI/TextCompletion/AzureTextCompletion.cs @@ -100,8 +100,8 @@ public Task> GetCompletionsAsync( } /// - public IAsyncEnumerable GetStreamingContentAsync(string input, AIRequestSettings? requestSettings = null, CancellationToken cancellationToken = default) + public IAsyncEnumerable GetStreamingContentAsync(string prompt, AIRequestSettings? requestSettings = null, CancellationToken cancellationToken = default) { - return this.InternalGetTextStreamingUpdatesAsync(input, requestSettings, cancellationToken); + return this.InternalGetTextStreamingUpdatesAsync(prompt, requestSettings, cancellationToken); } } diff --git a/dotnet/src/Connectors/Connectors.AI.OpenAI/TextCompletion/OpenAITextCompletion.cs b/dotnet/src/Connectors/Connectors.AI.OpenAI/TextCompletion/OpenAITextCompletion.cs index d565457eb126..0b44cb52d708 100644 --- a/dotnet/src/Connectors/Connectors.AI.OpenAI/TextCompletion/OpenAITextCompletion.cs +++ b/dotnet/src/Connectors/Connectors.AI.OpenAI/TextCompletion/OpenAITextCompletion.cs @@ -62,8 +62,8 @@ public Task> GetCompletionsAsync( } /// - public IAsyncEnumerable GetStreamingContentAsync(string input, AIRequestSettings? requestSettings = null, CancellationToken cancellationToken = default) + public IAsyncEnumerable GetStreamingContentAsync(string prompt, AIRequestSettings? requestSettings = null, CancellationToken cancellationToken = default) { - return this.InternalGetTextStreamingUpdatesAsync(input, requestSettings, cancellationToken); + return this.InternalGetTextStreamingUpdatesAsync(prompt, requestSettings, cancellationToken); } } diff --git a/dotnet/src/SemanticKernel.Abstractions/AI/ChatCompletion/IChatCompletion.cs b/dotnet/src/SemanticKernel.Abstractions/AI/ChatCompletion/IChatCompletion.cs index 8fde955fd376..817d04dd8a25 100644 --- a/dotnet/src/SemanticKernel.Abstractions/AI/ChatCompletion/IChatCompletion.cs +++ b/dotnet/src/SemanticKernel.Abstractions/AI/ChatCompletion/IChatCompletion.cs @@ -45,19 +45,19 @@ IAsyncEnumerable GetStreamingChatCompletionsAsync( CancellationToken cancellationToken = default); /// - /// Get streaming results for the prompt and settings of a specific type. + /// Get streaming results for the prompt using the specified request settings. /// Each modality may support for different types of streaming result. /// /// /// Usage of this method may be more efficient if the connector has a dedicated API to return this result without extra allocations for StreamingResultChunk abstraction. /// /// Throws if the specified type is not the same or fail to cast - /// The prompt to complete. + /// The prompt to complete. /// Request settings for the completion API /// The to monitor for cancellation requests. The default is . /// Streaming list of different completion streaming string updates generated by the remote model IAsyncEnumerable GetStreamingContentAsync( - string input, + string prompt, AIRequestSettings? requestSettings = null, CancellationToken cancellationToken = default); } diff --git a/dotnet/src/SemanticKernel.Abstractions/AI/TextCompletion/ITextCompletion.cs b/dotnet/src/SemanticKernel.Abstractions/AI/TextCompletion/ITextCompletion.cs index 49ef12704c99..40ee7bdd519d 100644 --- a/dotnet/src/SemanticKernel.Abstractions/AI/TextCompletion/ITextCompletion.cs +++ b/dotnet/src/SemanticKernel.Abstractions/AI/TextCompletion/ITextCompletion.cs @@ -38,19 +38,19 @@ IAsyncEnumerable GetStreamingCompletionsAsync( CancellationToken cancellationToken = default); /// - /// Get asynchronous stream of the provided generic type. + /// Get streaming results for the prompt using the specified request settings. /// Each modality may support for different types of streaming contents. /// /// /// Usage of this method with value types may be more efficient if the connector supports it. /// /// Throws if the specified type is not the same or fail to cast - /// The prompt to complete. + /// The prompt to complete. /// Request settings for the completion API /// The to monitor for cancellation requests. The default is . /// Streaming list of different completion streaming string updates generated by the remote model IAsyncEnumerable GetStreamingContentAsync( - string input, + string prompt, AIRequestSettings? requestSettings = null, CancellationToken cancellationToken = default); } diff --git a/dotnet/src/SemanticKernel.Abstractions/Functions/KernelFunction.cs b/dotnet/src/SemanticKernel.Abstractions/Functions/KernelFunction.cs index 221ea5257f57..f8d76cecf1ff 100644 --- a/dotnet/src/SemanticKernel.Abstractions/Functions/KernelFunction.cs +++ b/dotnet/src/SemanticKernel.Abstractions/Functions/KernelFunction.cs @@ -6,7 +6,6 @@ using System.Diagnostics.Metrics; using System.Linq; using System.Runtime.CompilerServices; -using System.Text; using System.Threading; using System.Threading.Tasks; using Microsoft.Extensions.Logging; @@ -99,9 +98,9 @@ public async Task InvokeAsync( { var result = await this.InvokeCoreAsync(kernel, context, requestSettings, cancellationToken).ConfigureAwait(false); - if (logger.IsEnabled(LogLevel.Trace)) + if (logger.IsEnabled(LogLevel.Information)) { - logger.LogTrace("Function succeeded. Result: {Result}", result.GetValue()); // Sensitive data, logging as trace, disabled by default + logger.LogTrace("Function succeeded."); } return result; @@ -145,28 +144,13 @@ public async IAsyncEnumerable InvokeStreamingAsync( logger.LogInformation("Function invoking streaming."); - TagList tags = new() { { "sk.function.name", this.Name } }; - long startingTimestamp = Stopwatch.GetTimestamp(); - - StringBuilder fullCompletion = new(); - await foreach (var genericChunk in this.InvokeCoreStreamingAsync(kernel, context, requestSettings, cancellationToken)) { - fullCompletion.Append(genericChunk); yield return genericChunk; } - if (logger.IsEnabled(LogLevel.Trace)) - { - logger.LogTrace("Function streaming succeeded. Result: {Result}", fullCompletion); // Sensitive data, logging as trace, disabled by default - } - - TimeSpan duration = new((long)((Stopwatch.GetTimestamp() - startingTimestamp) * (10_000_000.0 / Stopwatch.Frequency))); - s_invocationDuration.Record(duration.TotalSeconds, in tags); - if (logger.IsEnabled(LogLevel.Information)) - { - logger.LogInformation("Function streaming completed. Duration: {Duration}ms", duration.TotalMilliseconds); - } + // No logging is done here since there is no guarantee that this line will be hit + // (Only if the caller consumes the whole enumeration) } /// diff --git a/dotnet/src/SemanticKernel.Core/Functions/SKFunctionFromMethod.cs b/dotnet/src/SemanticKernel.Core/Functions/SKFunctionFromMethod.cs index 87cd8d3f605c..96fa4731d87b 100644 --- a/dotnet/src/SemanticKernel.Core/Functions/SKFunctionFromMethod.cs +++ b/dotnet/src/SemanticKernel.Core/Functions/SKFunctionFromMethod.cs @@ -159,11 +159,19 @@ protected override async IAsyncEnumerable InvokeCoreStreamingAsync( if (functionResult.Value is T) { yield return (T)functionResult.Value; + yield break; } - else if (functionResult.Value is not null) + + if (typeof(T).IsSubclassOf(typeof(StreamingContent)) || typeof(T) == typeof(StreamingContent)) { - yield return (T)(object)new StreamingMethodContent(functionResult.Value); + if (functionResult.Value is not null) + { + yield return (T)(object)new StreamingMethodContent(functionResult.Value); + } + yield break; } + + throw new NotSupportedException($"Streaming function {this.Name} does not support type {typeof(T)}"); } private FunctionInvokingEventArgs CallFunctionInvoking(Kernel kernel, SKContext context) diff --git a/dotnet/src/SemanticKernel.Core/Functions/StreamingMethodContent.cs b/dotnet/src/SemanticKernel.Core/Functions/StreamingMethodContent.cs index 335a380f5961..e9425fe3fb7a 100644 --- a/dotnet/src/SemanticKernel.Core/Functions/StreamingMethodContent.cs +++ b/dotnet/src/SemanticKernel.Core/Functions/StreamingMethodContent.cs @@ -9,7 +9,7 @@ namespace Microsoft.SemanticKernel; #pragma warning restore IDE0130 /// -/// Method function streaming result chunk. +/// This class represents the content of a streaming chunk generated by KernelFunctionsFromMethod. /// public sealed class StreamingMethodContent : StreamingContent { @@ -17,34 +17,34 @@ public sealed class StreamingMethodContent : StreamingContent public override int ChoiceIndex => 0; /// - /// Method object value that represents the chunk + /// Content of a streaming chunk result of a KernelFunctionFromMethod. /// - public object Value { get; } + public object Content { get; } /// public override byte[] ToByteArray() { - if (this.Value is byte[]) + if (this.Content is byte[]) { - return (byte[])this.Value; + return (byte[])this.Content; } // By default if a native value is not Byte[] we output the UTF8 string representation of the value - return Encoding.UTF8.GetBytes(this.Value?.ToString()); + return Encoding.UTF8.GetBytes(this.Content?.ToString()); } /// public override string ToString() { - return this.Value.ToString(); + return this.Content.ToString(); } /// /// Initializes a new instance of the class. /// - /// Underlying object that represents the chunk + /// Underlying object that represents the chunk content public StreamingMethodContent(object innerContent) : base(innerContent) { - this.Value = innerContent; + this.Content = innerContent; } } From 2b23582b8c3ec3f8f4702feef3f75942effde8d7 Mon Sep 17 00:00:00 2001 From: Roger Barreto <19890735+RogerBarreto@users.noreply.github.com> Date: Thu, 23 Nov 2023 23:32:47 +0000 Subject: [PATCH 40/46] Addressed PR comments --- .../AI/ChatCompletion/ChatCompletionExtensions.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dotnet/src/SemanticKernel.Abstractions/AI/ChatCompletion/ChatCompletionExtensions.cs b/dotnet/src/SemanticKernel.Abstractions/AI/ChatCompletion/ChatCompletionExtensions.cs index a60e255ef845..1b7d3b6cda02 100644 --- a/dotnet/src/SemanticKernel.Abstractions/AI/ChatCompletion/ChatCompletionExtensions.cs +++ b/dotnet/src/SemanticKernel.Abstractions/AI/ChatCompletion/ChatCompletionExtensions.cs @@ -61,7 +61,7 @@ public static async Task GenerateMessageAsync( } /// - /// Get asynchronous stream.of . + /// Get asynchronous stream of . /// /// Chat completion target /// The input string. (May be a JSON for complex objects, Byte64 for binary, will depend on the connector spec). From 591851207bdc9edb1810c499cfa5c738fd958c94 Mon Sep 17 00:00:00 2001 From: Roger Barreto <19890735+RogerBarreto@users.noreply.github.com> Date: Fri, 24 Nov 2023 00:41:50 +0000 Subject: [PATCH 41/46] Warning fix --- dotnet/samples/KernelSyntaxExamples/Example16_CustomLLM.cs | 2 +- .../Extensions/KernelSemanticFunctionExtensionsTests.cs | 2 +- .../Functions/OrderedIAIServiceConfigurationProviderTests.cs | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/dotnet/samples/KernelSyntaxExamples/Example16_CustomLLM.cs b/dotnet/samples/KernelSyntaxExamples/Example16_CustomLLM.cs index b369628eb4e5..6166eed5cd94 100644 --- a/dotnet/samples/KernelSyntaxExamples/Example16_CustomLLM.cs +++ b/dotnet/samples/KernelSyntaxExamples/Example16_CustomLLM.cs @@ -49,7 +49,7 @@ public async IAsyncEnumerable GetStreamingCompletionsAsync yield return new MyTextCompletionStreamingResult(); } - public async IAsyncEnumerable GetStreamingContentAsync(string input, AIRequestSettings? requestSettings = null, [EnumeratorCancellation] CancellationToken cancellationToken = default) + public async IAsyncEnumerable GetStreamingContentAsync(string prompt, AIRequestSettings? requestSettings = null, [EnumeratorCancellation] CancellationToken cancellationToken = default) { if (typeof(T) == typeof(MyStreamingContent)) { diff --git a/dotnet/src/IntegrationTests/Extensions/KernelSemanticFunctionExtensionsTests.cs b/dotnet/src/IntegrationTests/Extensions/KernelSemanticFunctionExtensionsTests.cs index 0d2aba0c73b1..6ebccaf76888 100644 --- a/dotnet/src/IntegrationTests/Extensions/KernelSemanticFunctionExtensionsTests.cs +++ b/dotnet/src/IntegrationTests/Extensions/KernelSemanticFunctionExtensionsTests.cs @@ -83,7 +83,7 @@ IAsyncEnumerable ITextCompletion.GetStreamingCompletionsAs throw new NotImplementedException(); // TODO } - public IAsyncEnumerable GetStreamingContentAsync(string input, AIRequestSettings? requestSettings = null, CancellationToken cancellationToken = default) + public IAsyncEnumerable GetStreamingContentAsync(string prompt, AIRequestSettings? requestSettings = null, CancellationToken cancellationToken = default) { throw new NotImplementedException(); } diff --git a/dotnet/src/SemanticKernel.UnitTests/Functions/OrderedIAIServiceConfigurationProviderTests.cs b/dotnet/src/SemanticKernel.UnitTests/Functions/OrderedIAIServiceConfigurationProviderTests.cs index 1691674aa526..cb59b0827eff 100644 --- a/dotnet/src/SemanticKernel.UnitTests/Functions/OrderedIAIServiceConfigurationProviderTests.cs +++ b/dotnet/src/SemanticKernel.UnitTests/Functions/OrderedIAIServiceConfigurationProviderTests.cs @@ -212,7 +212,7 @@ public Task> GetCompletionsAsync(string text, AIReque throw new NotImplementedException(); } - public IAsyncEnumerable GetStreamingContentAsync(string input, AIRequestSettings? requestSettings = null, CancellationToken cancellationToken = default) + public IAsyncEnumerable GetStreamingContentAsync(string prompt, AIRequestSettings? requestSettings = null, CancellationToken cancellationToken = default) { throw new NotImplementedException(); } From 39fe228dd07fa2a0b36b8be8bacc5f398e00cefb Mon Sep 17 00:00:00 2001 From: Roger Barreto <19890735+RogerBarreto@users.noreply.github.com> Date: Fri, 24 Nov 2023 09:03:51 +0000 Subject: [PATCH 42/46] Kernel Streaming UTs --- .../SemanticKernel.Core/KernelExtensions.cs | 22 +++ .../SemanticKernel.UnitTests/KernelTests.cs | 153 +++++++++++++++++- 2 files changed, 169 insertions(+), 6 deletions(-) diff --git a/dotnet/src/SemanticKernel.Core/KernelExtensions.cs b/dotnet/src/SemanticKernel.Core/KernelExtensions.cs index eeb1b1b512d5..0612a2d1daaa 100644 --- a/dotnet/src/SemanticKernel.Core/KernelExtensions.cs +++ b/dotnet/src/SemanticKernel.Core/KernelExtensions.cs @@ -532,6 +532,28 @@ public static IAsyncEnumerable RunStreamingAsync(this Kernel kernel, Kerne /// Streaming result of the function public static IAsyncEnumerable RunStreamingAsync(this Kernel kernel, KernelFunction function, ContextVariables? variables = null, CancellationToken cancellationToken = default) => kernel.RunStreamingAsync(function, variables ?? new ContextVariables(), CancellationToken.None); + + /// + /// Run a function in streaming mode. + /// + /// The target kernel + /// Target function to run + /// Input to process + /// The to monitor for cancellation requests. + /// Streaming result of the function + public static IAsyncEnumerable RunStreamingAsync(this Kernel kernel, KernelFunction function, string input, CancellationToken cancellationToken = default) + => function.InvokeStreamingAsync(kernel, kernel.CreateNewContext(new ContextVariables(input)), null, cancellationToken); + + /// + /// Run a function in streaming mode. + /// + /// Target kernel + /// Target function to run + /// Input to process + /// Cancellation token + /// Streaming result of the function + public static IAsyncEnumerable RunStreamingAsync(this Kernel kernel, KernelFunction function, string input, CancellationToken cancellationToken = default) + => kernel.RunStreamingAsync(function, input, CancellationToken.None); #endregion /// diff --git a/dotnet/src/SemanticKernel.UnitTests/KernelTests.cs b/dotnet/src/SemanticKernel.UnitTests/KernelTests.cs index 887da4fbfff3..427e078870c2 100644 --- a/dotnet/src/SemanticKernel.UnitTests/KernelTests.cs +++ b/dotnet/src/SemanticKernel.UnitTests/KernelTests.cs @@ -6,6 +6,7 @@ using System.Globalization; using System.Linq; using System.Reflection; +using System.Text; using System.Threading; using System.Threading.Tasks; using Microsoft.Extensions.Logging; @@ -121,13 +122,152 @@ public async Task RunAsyncHandlesPreInvocationAsync() Assert.Equal(1, handlerInvocations); } + [Fact] + public async Task RunStreamingAsyncHandlesPreInvocationAsync() + { + // Arrange + var sut = new KernelBuilder().Build(); + int functionInvocations = 0; + var function = SKFunctionFactory.CreateFromMethod(() => functionInvocations++); + + var handlerInvocations = 0; + sut.FunctionInvoking += (object? sender, FunctionInvokingEventArgs e) => + { + handlerInvocations++; + }; + + // Act + await foreach (var chunk in sut.RunStreamingAsync(function)) { } + + // Assert + Assert.Equal(1, functionInvocations); + Assert.Equal(1, handlerInvocations); + } + + [Fact] + public async Task RunStreamingAsyncHandlesPreInvocationWasCancelledAsync() + { + // Arrange + var sut = new KernelBuilder().Build(); + int functionInvocations = 0; + var function = SKFunctionFactory.CreateFromMethod(() => functionInvocations++); + + var handlerInvocations = 0; + sut.FunctionInvoking += (object? sender, FunctionInvokingEventArgs e) => + { + handlerInvocations++; + e.Cancel(); + }; + + // Act + int chunksCount = 0; + await foreach (var chunk in sut.RunStreamingAsync(function)) + { + chunksCount++; + } + + // Assert + Assert.Equal(1, handlerInvocations); + Assert.Equal(0, functionInvocations); + Assert.Equal(0, chunksCount); + } + + [Fact] + public async Task RunStreamingAsyncPreInvocationCancelationDontTriggerInvokedHandlerAsync() + { + // Arrange + var sut = new KernelBuilder().Build(); + var functions = sut.ImportPluginFromObject(); + + var invoked = 0; + sut.FunctionInvoking += (object? sender, FunctionInvokingEventArgs e) => + { + e.Cancel(); + }; + + sut.FunctionInvoked += (object? sender, FunctionInvokedEventArgs e) => + { + invoked++; + }; + + // Act + await foreach (var chunk in sut.RunStreamingAsync(functions["GetAnyValue"])) + { + } + + // Assert + Assert.Equal(0, invoked); + } + + [Fact] + public async Task RunStreamingAsyncPreInvocationSkipDontTriggerInvokedHandlerAsync() + { + // Arrange + var sut = new KernelBuilder().Build(); + int funcInvocations = 0; + var function = SKFunctionFactory.CreateFromMethod(() => funcInvocations++, functionName: "func1"); + + var invoked = 0; + var invoking = 0; + string invokedFunction = string.Empty; + + sut.FunctionInvoking += (object? sender, FunctionInvokingEventArgs e) => + { + invoking++; + if (e.FunctionMetadata.Name == "func1") + { + e.Skip(); + } + }; + + sut.FunctionInvoked += (object? sender, FunctionInvokedEventArgs e) => + { + invokedFunction = e.FunctionMetadata.Name; + invoked++; + }; + + // Act + await foreach (var chunk in sut.RunStreamingAsync(function)) + { + } + + // Assert + Assert.Equal(1, invoking); + Assert.Equal(0, invoked); + Assert.Equal(0, funcInvocations); + } + + [Fact] + public async Task RunStreamingAsyncHandlesPostInvocationAsync() + { + // Arrange + var sut = new KernelBuilder().Build(); + int functionInvocations = 0; + var function = SKFunctionFactory.CreateFromMethod(() => functionInvocations++); + + int handlerInvocations = 0; + sut.FunctionInvoked += (object? sender, FunctionInvokedEventArgs e) => + { + handlerInvocations++; + }; + + // Act + await foreach (var chunk in sut.RunStreamingAsync(function)) + { + } + + // Assert + Assert.Equal(1, functionInvocations); + Assert.Equal(1, handlerInvocations); + } + [Fact] public async Task RunAsyncHandlesPreInvocationWasCancelledAsync() { // Arrange var sut = new KernelBuilder().Build(); int functionInvocations = 0; - KernelFunction function = SKFunctionFactory.CreateFromMethod(() => functionInvocations++); + var function = SKFunctionFactory.CreateFromMethod(() => functionInvocations++); var handlerInvocations = 0; sut.FunctionInvoking += (object? sender, FunctionInvokingEventArgs e) => @@ -152,6 +292,7 @@ public async Task RunAsyncHandlesPreInvocationCancelationDontRunSubsequentFuncti var sut = new KernelBuilder().Build(); int functionInvocations = 0; var function = SKFunctionFactory.CreateFromMethod(() => functionInvocations++); + var function2 = SKFunctionFactory.CreateFromMethod(() => functionInvocations++); int handlerInvocations = 0; sut.FunctionInvoking += (object? sender, FunctionInvokingEventArgs e) => @@ -161,7 +302,7 @@ public async Task RunAsyncHandlesPreInvocationCancelationDontRunSubsequentFuncti }; // Act - var result = await sut.RunAsync(function); + var result = await sut.RunAsync(new ContextVariables(), CancellationToken.None, new[] { function, function2 }); // Assert Assert.Equal(1, handlerInvocations); @@ -199,7 +340,7 @@ public async Task RunAsyncPreInvocationSkipDontTriggerInvokedHandlerAsync() // Arrange var sut = new KernelBuilder().Build(); int funcInvocations = 0; - KernelFunction function = SKFunctionFactory.CreateFromMethod(() => funcInvocations++, functionName: "func1"); + var function = SKFunctionFactory.CreateFromMethod(() => funcInvocations++, functionName: "func1"); var invoked = 0; var invoking = 0; @@ -235,7 +376,7 @@ public async Task RunAsyncHandlesPostInvocationAsync() // Arrange var sut = new KernelBuilder().Build(); int functionInvocations = 0; - KernelFunction function = SKFunctionFactory.CreateFromMethod(() => functionInvocations++); + var function = SKFunctionFactory.CreateFromMethod(() => functionInvocations++); int handlerInvocations = 0; sut.FunctionInvoked += (object? sender, FunctionInvokedEventArgs e) => @@ -255,7 +396,7 @@ public async Task RunAsyncHandlesPostInvocationAsync() public async Task RunAsyncChangeVariableInvokingHandlerAsync() { var sut = new KernelBuilder().Build(); - KernelFunction function = SKFunctionFactory.CreateFromMethod(() => { }); + var function = SKFunctionFactory.CreateFromMethod(() => { }); var originalInput = "Importance"; var newInput = "Problems"; @@ -276,7 +417,7 @@ public async Task RunAsyncChangeVariableInvokingHandlerAsync() public async Task RunAsyncChangeVariableInvokedHandlerAsync() { var sut = new KernelBuilder().Build(); - KernelFunction function = SKFunctionFactory.CreateFromMethod(() => { }); + var function = SKFunctionFactory.CreateFromMethod(() => { }); var originalInput = "Importance"; var newInput = "Problems"; From b0757adbd2f14ef2694092dde5455d7d2ae2dcce Mon Sep 17 00:00:00 2001 From: Roger Barreto <19890735+RogerBarreto@users.noreply.github.com> Date: Fri, 24 Nov 2023 09:13:14 +0000 Subject: [PATCH 43/46] Removed Pipeline and related UT --- .../SemanticKernel.Core/KernelExtensions.cs | 61 ------------------- .../SemanticKernel.UnitTests/KernelTests.cs | 24 -------- 2 files changed, 85 deletions(-) diff --git a/dotnet/src/SemanticKernel.Core/KernelExtensions.cs b/dotnet/src/SemanticKernel.Core/KernelExtensions.cs index 0612a2d1daaa..e176105e2790 100644 --- a/dotnet/src/SemanticKernel.Core/KernelExtensions.cs +++ b/dotnet/src/SemanticKernel.Core/KernelExtensions.cs @@ -447,67 +447,6 @@ public static Task RunAsync( return kernel.RunAsync(function, variables ?? new(), cancellationToken); } - - /// - /// Run a pipeline composed of synchronous and asynchronous functions. - /// - /// The kernel. - /// Input to process - /// The to monitor for cancellation requests. The default is . - /// List of functions - /// Result of the function composition - /// - public static async Task RunAsync(this Kernel kernel, ContextVariables variables, CancellationToken cancellationToken, params KernelFunction[] pipeline) - { - var context = kernel.CreateNewContext(variables); - - FunctionResult? functionResult = null; - - int pipelineStepCount = 0; - var allFunctionResults = new List(); - - var logger = kernel.LoggerFactory.CreateLogger(typeof(Kernel)); - - foreach (KernelFunction function in pipeline) - { -repeat: - cancellationToken.ThrowIfCancellationRequested(); - - try - { - var functionDetails = function.GetMetadata(); - - functionResult = await function.InvokeAsync(kernel, context, null, cancellationToken: cancellationToken).ConfigureAwait(false); - - if (IsCancelRequested(functionResult, function, pipelineStepCount, logger)) - { - break; - } - - if (IsSkipRequested(functionResult, function, pipelineStepCount, logger)) - { - continue; - } - - // Only non-stop results are considered as Kernel results - allFunctionResults.Add(functionResult!); - - if (IsRepeatRequested(functionResult, function, pipelineStepCount, logger)) - { - goto repeat; - } - } - catch (Exception ex) - { - logger.LogError("Function {Function} call fail during pipeline step {Step} with error {Error}:", function.Name, pipelineStepCount, ex.Message); - throw; - } - - pipelineStepCount++; - } - - return allFunctionResults.LastOrDefault(); - } #endregion #region RunStreamingAsync diff --git a/dotnet/src/SemanticKernel.UnitTests/KernelTests.cs b/dotnet/src/SemanticKernel.UnitTests/KernelTests.cs index 427e078870c2..1afd6af320bf 100644 --- a/dotnet/src/SemanticKernel.UnitTests/KernelTests.cs +++ b/dotnet/src/SemanticKernel.UnitTests/KernelTests.cs @@ -285,30 +285,6 @@ public async Task RunAsyncHandlesPreInvocationWasCancelledAsync() Assert.NotNull(result); } - [Fact] - public async Task RunAsyncHandlesPreInvocationCancelationDontRunSubsequentFunctionsInThePipelineAsync() - { - // Arrange - var sut = new KernelBuilder().Build(); - int functionInvocations = 0; - var function = SKFunctionFactory.CreateFromMethod(() => functionInvocations++); - var function2 = SKFunctionFactory.CreateFromMethod(() => functionInvocations++); - - int handlerInvocations = 0; - sut.FunctionInvoking += (object? sender, FunctionInvokingEventArgs e) => - { - handlerInvocations++; - e.Cancel(); - }; - - // Act - var result = await sut.RunAsync(new ContextVariables(), CancellationToken.None, new[] { function, function2 }); - - // Assert - Assert.Equal(1, handlerInvocations); - Assert.Equal(0, functionInvocations); - } - [Fact] public async Task RunAsyncPreInvocationCancelationDontTriggerInvokedHandlerAsync() { From 76d54dc666bf421c0c08a7adf40c9ac191e3d6a6 Mon Sep 17 00:00:00 2001 From: Roger Barreto <19890735+RogerBarreto@users.noreply.github.com> Date: Fri, 24 Nov 2023 11:58:38 +0000 Subject: [PATCH 44/46] Removing try finally + Invoked support for streaming --- .../AzureOpenAIChatCompletion.cs | 4 +- .../ChatCompletion/OpenAIChatCompletion.cs | 4 +- .../Functions/KernelFunction.cs | 7 ++- .../Functions/KernelFunctionExtensions.cs | 4 +- .../Functions/SKFunctionFromMethod.cs | 14 ++++- .../Functions/SKFunctionFromPrompt.cs | 8 +-- .../SemanticKernel.Core/KernelExtensions.cs | 57 ----------------- .../Functions/SemanticFunctionTests.cs | 61 ++++++++++++++++++- .../SemanticKernel.UnitTests/KernelTests.cs | 19 ------ 9 files changed, 83 insertions(+), 95 deletions(-) diff --git a/dotnet/src/Connectors/Connectors.AI.OpenAI/ChatCompletion/AzureOpenAIChatCompletion.cs b/dotnet/src/Connectors/Connectors.AI.OpenAI/ChatCompletion/AzureOpenAIChatCompletion.cs index 98dd3ede77a2..81d81af4d577 100644 --- a/dotnet/src/Connectors/Connectors.AI.OpenAI/ChatCompletion/AzureOpenAIChatCompletion.cs +++ b/dotnet/src/Connectors/Connectors.AI.OpenAI/ChatCompletion/AzureOpenAIChatCompletion.cs @@ -127,9 +127,9 @@ public Task> GetCompletionsAsync( } /// - public IAsyncEnumerable GetStreamingContentAsync(string input, AIRequestSettings? requestSettings = null, CancellationToken cancellationToken = default) + public IAsyncEnumerable GetStreamingContentAsync(string prompt, AIRequestSettings? requestSettings = null, CancellationToken cancellationToken = default) { - var chatHistory = this.CreateNewChat(input); + var chatHistory = this.CreateNewChat(prompt); return this.InternalGetChatStreamingUpdatesAsync(chatHistory, requestSettings, cancellationToken); } } diff --git a/dotnet/src/Connectors/Connectors.AI.OpenAI/ChatCompletion/OpenAIChatCompletion.cs b/dotnet/src/Connectors/Connectors.AI.OpenAI/ChatCompletion/OpenAIChatCompletion.cs index b7e5c629813e..4bc81e8dfe9e 100644 --- a/dotnet/src/Connectors/Connectors.AI.OpenAI/ChatCompletion/OpenAIChatCompletion.cs +++ b/dotnet/src/Connectors/Connectors.AI.OpenAI/ChatCompletion/OpenAIChatCompletion.cs @@ -103,9 +103,9 @@ public Task> GetCompletionsAsync( } /// - public IAsyncEnumerable GetStreamingContentAsync(string input, AIRequestSettings? requestSettings = null, CancellationToken cancellationToken = default) + public IAsyncEnumerable GetStreamingContentAsync(string prompt, AIRequestSettings? requestSettings = null, CancellationToken cancellationToken = default) { - var chatHistory = this.CreateNewChat(input); + var chatHistory = this.CreateNewChat(prompt); return this.InternalGetChatStreamingUpdatesAsync(chatHistory, requestSettings, cancellationToken); } } diff --git a/dotnet/src/SemanticKernel.Abstractions/Functions/KernelFunction.cs b/dotnet/src/SemanticKernel.Abstractions/Functions/KernelFunction.cs index f8d76cecf1ff..b172370844a1 100644 --- a/dotnet/src/SemanticKernel.Abstractions/Functions/KernelFunction.cs +++ b/dotnet/src/SemanticKernel.Abstractions/Functions/KernelFunction.cs @@ -142,15 +142,16 @@ public async IAsyncEnumerable InvokeStreamingAsync( using var activity = s_activitySource.StartActivity(this.Name); ILogger logger = kernel.LoggerFactory.CreateLogger(this.Name); - logger.LogInformation("Function invoking streaming."); + logger.LogInformation("Function streaming invoking."); + + cancellationToken.ThrowIfCancellationRequested(); await foreach (var genericChunk in this.InvokeCoreStreamingAsync(kernel, context, requestSettings, cancellationToken)) { yield return genericChunk; } - // No logging is done here since there is no guarantee that this line will be hit - // (Only if the caller consumes the whole enumeration) + // Completion logging is not supported for streaming functions } /// diff --git a/dotnet/src/SemanticKernel.Abstractions/Functions/KernelFunctionExtensions.cs b/dotnet/src/SemanticKernel.Abstractions/Functions/KernelFunctionExtensions.cs index 010a58d8d9f5..3a6512d85d89 100644 --- a/dotnet/src/SemanticKernel.Abstractions/Functions/KernelFunctionExtensions.cs +++ b/dotnet/src/SemanticKernel.Abstractions/Functions/KernelFunctionExtensions.cs @@ -24,10 +24,10 @@ public static class KernelFunctionExtensions public static IAsyncEnumerable InvokeStreamingAsync( this KernelFunction function, Kernel kernel, - SKContext context, + SKContext? context = null, AIRequestSettings? requestSettings = null, CancellationToken cancellationToken = default) { - return function.InvokeStreamingAsync(kernel, context, requestSettings, cancellationToken); + return function.InvokeStreamingAsync(kernel, context ?? kernel.CreateNewContext(), requestSettings, cancellationToken); } } diff --git a/dotnet/src/SemanticKernel.Core/Functions/SKFunctionFromMethod.cs b/dotnet/src/SemanticKernel.Core/Functions/SKFunctionFromMethod.cs index 96fa4731d87b..9edec9c56559 100644 --- a/dotnet/src/SemanticKernel.Core/Functions/SKFunctionFromMethod.cs +++ b/dotnet/src/SemanticKernel.Core/Functions/SKFunctionFromMethod.cs @@ -155,6 +155,13 @@ protected override async IAsyncEnumerable InvokeCoreStreamingAsync( AIRequestSettings? requestSettings = null, [EnumeratorCancellation] CancellationToken cancellationToken = default) { + // Invoke pre hook, and stop if skipping requested. + var invokingEventArgs = this.CallFunctionInvoking(kernel, context); + if (invokingEventArgs.IsSkipRequested || invokingEventArgs.CancelToken.IsCancellationRequested) + { + yield break; + } + var functionResult = await this.InvokeCoreAsync(kernel, context, requestSettings, cancellationToken).ConfigureAwait(false); if (functionResult.Value is T) { @@ -162,7 +169,9 @@ protected override async IAsyncEnumerable InvokeCoreStreamingAsync( yield break; } - if (typeof(T).IsSubclassOf(typeof(StreamingContent)) || typeof(T) == typeof(StreamingContent)) + // Supports the following provided T types for Method streaming + if (typeof(T) == typeof(StreamingContent) || + typeof(T) == typeof(StreamingMethodContent)) { if (functionResult.Value is not null) { @@ -172,6 +181,9 @@ protected override async IAsyncEnumerable InvokeCoreStreamingAsync( } throw new NotSupportedException($"Streaming function {this.Name} does not support type {typeof(T)}"); + + // Invoked is not supported for streaming yet + // There is no post cancellation check to override the result as the stream data was already sent. } private FunctionInvokingEventArgs CallFunctionInvoking(Kernel kernel, SKContext context) diff --git a/dotnet/src/SemanticKernel.Core/Functions/SKFunctionFromPrompt.cs b/dotnet/src/SemanticKernel.Core/Functions/SKFunctionFromPrompt.cs index ddc5f657e704..bdd36a250d4f 100644 --- a/dotnet/src/SemanticKernel.Core/Functions/SKFunctionFromPrompt.cs +++ b/dotnet/src/SemanticKernel.Core/Functions/SKFunctionFromPrompt.cs @@ -194,17 +194,13 @@ protected override async IAsyncEnumerable InvokeCoreStreamingAsync( yield break; } - StringBuilder fullCompletion = new(); await foreach (T genericChunk in textCompletion.GetStreamingContentAsync(renderedPrompt, requestSettings ?? defaultRequestSettings, cancellationToken)) { cancellationToken.ThrowIfCancellationRequested(); - fullCompletion.Append(genericChunk); + yield return genericChunk; } - // Update the result with the completion - context.Variables.Update(fullCompletion.ToString()); - - this.CallFunctionInvoked(kernel, context, null, renderedPrompt); + // Invoked is not supported for streaming // There is no post cancellation check to override the result as the stream data was already sent. } diff --git a/dotnet/src/SemanticKernel.Core/KernelExtensions.cs b/dotnet/src/SemanticKernel.Core/KernelExtensions.cs index e176105e2790..af6c9ff87cd2 100644 --- a/dotnet/src/SemanticKernel.Core/KernelExtensions.cs +++ b/dotnet/src/SemanticKernel.Core/KernelExtensions.cs @@ -4,7 +4,6 @@ using System.Collections.Generic; using System.ComponentModel; using System.IO; -using System.Linq; using System.Reflection; using System.Text.Json; using System.Threading; @@ -494,60 +493,4 @@ public static IAsyncEnumerable RunStreamingAsync(this Kernel kernel, Kerne public static IAsyncEnumerable RunStreamingAsync(this Kernel kernel, KernelFunction function, string input, CancellationToken cancellationToken = default) => kernel.RunStreamingAsync(function, input, CancellationToken.None); #endregion - - /// - /// Checks if the handler requested to cancel the function execution. - /// - /// Function result - /// Target function - /// Current pipeline step - /// The logger. - /// Streaming result of the function - private static bool IsCancelRequested(FunctionResult result, KernelFunction function, int pipelineStepCount, ILogger logger) - { - if (result.IsCancellationRequested) - { - logger.LogInformation("Execution was cancelled for pipeline step {StepCount}: {FunctionName}.", pipelineStepCount, function.Name); - return true; - } - - return false; - } - - /// - /// Checks if the handler requested to skip the function execution. - /// - /// Function result - /// Target function - /// Current pipeline step - /// The logger. - /// Streaming result of the function - private static bool IsSkipRequested(FunctionResult result, KernelFunction function, int pipelineStepCount, ILogger logger) - { - if (result.IsSkipRequested) - { - logger.LogInformation("Execution was skipped on function invoking event of pipeline step {StepCount}: {FunctionName}.", pipelineStepCount, function.Name); - return true; - } - - return false; - } - - /// - /// Checks if the handler requested to repeat the function execution. - /// - /// Function result - /// Target function - /// Current pipeline step - /// The logger. - /// Streaming result of the function - private static bool IsRepeatRequested(FunctionResult result, KernelFunction function, int pipelineStepCount, ILogger logger) - { - if (result.IsRepeatRequested) - { - logger.LogInformation("Execution repeat request on function invoked event of pipeline step {StepCount}: {FunctionName}.", pipelineStepCount, function.Name); - return true; - } - return false; - } } diff --git a/dotnet/src/SemanticKernel.UnitTests/Functions/SemanticFunctionTests.cs b/dotnet/src/SemanticKernel.UnitTests/Functions/SemanticFunctionTests.cs index 06d03db00f2e..7b68debcb5c5 100644 --- a/dotnet/src/SemanticKernel.UnitTests/Functions/SemanticFunctionTests.cs +++ b/dotnet/src/SemanticKernel.UnitTests/Functions/SemanticFunctionTests.cs @@ -2,6 +2,7 @@ using System; using System.Collections.Generic; +using System.Linq; using System.Reflection; using System.Threading; using System.Threading.Tasks; @@ -10,6 +11,7 @@ using Microsoft.SemanticKernel.AI; using Microsoft.SemanticKernel.AI.TextCompletion; using Microsoft.SemanticKernel.Connectors.AI.OpenAI; +using Microsoft.SemanticKernel.Orchestration; using Microsoft.SemanticKernel.TemplateEngine; using Moq; using Xunit; @@ -346,6 +348,26 @@ public async Task RunAsyncChangeVariableInvokedHandlerAsync() Assert.Equal(newInput, originalInput); } + [Fact] + public async Task InvokeStreamingAsyncCallsConnectorStreamingApiAsync() + { + // Arrange + var mockTextCompletion = this.SetupStreamingMocks(new TestStreamingContent()); + var kernel = new KernelBuilder().WithAIService(null, mockTextCompletion.Object).Build(); + var prompt = "Write a simple phrase about UnitTests {{$input}}"; + var sut = SKFunctionFactory.CreateFromPrompt(prompt); + var variables = new ContextVariables("importance"); + var context = kernel.CreateNewContext(variables); + + // Act + await foreach (var chunk in sut.InvokeStreamingAsync(kernel, context)) + { + } + + // Assert + mockTextCompletion.Verify(m => m.GetStreamingContentAsync(It.IsIn("Write a simple phrase about UnitTests importance"), It.IsAny(), It.IsAny()), Times.Exactly(1)); + } + private (Mock textResultMock, Mock textCompletionMock) SetupMocks(string? completionResult = null) { var mockTextResult = new Mock(); @@ -353,12 +375,45 @@ public async Task RunAsyncChangeVariableInvokedHandlerAsync() var mockTextCompletion = new Mock(); mockTextCompletion.Setup(m => m.GetCompletionsAsync(It.IsAny(), It.IsAny(), It.IsAny())).ReturnsAsync(new List { mockTextResult.Object }); - return (mockTextResult, mockTextCompletion); } - private static MethodInfo Method(Delegate method) + private Mock SetupStreamingMocks(T completionResult) { - return method.Method; + var mockTextCompletion = new Mock(); + mockTextCompletion.Setup(m => m.GetStreamingContentAsync(It.IsAny(), It.IsAny(), It.IsAny())).Returns(this.ToAsyncEnumerable(new List { completionResult })); + + return mockTextCompletion; + } + + private class TestStreamingContent : StreamingContent + { + public TestStreamingContent() : base(null) + { + } + + public override int ChoiceIndex => 0; + + public override byte[] ToByteArray() + { + return Array.Empty(); + } + + public override string ToString() + { + return string.Empty; + } + } + +#pragma warning disable CS1998 // Async method lacks 'await' operators and will run synchronously +#pragma warning disable IDE1006 // Naming Styles + private async IAsyncEnumerable ToAsyncEnumerable(IEnumerable enumeration) +#pragma warning restore IDE1006 // Naming Styles +#pragma warning restore CS1998 // Async method lacks 'await' operators and will run synchronously + { + foreach (var enumerationItem in enumeration) + { + yield return enumerationItem; + } } } diff --git a/dotnet/src/SemanticKernel.UnitTests/KernelTests.cs b/dotnet/src/SemanticKernel.UnitTests/KernelTests.cs index 1afd6af320bf..3000e4d5f43a 100644 --- a/dotnet/src/SemanticKernel.UnitTests/KernelTests.cs +++ b/dotnet/src/SemanticKernel.UnitTests/KernelTests.cs @@ -1,17 +1,14 @@ // Copyright (c) Microsoft. All rights reserved. using System; -using System.Collections.Generic; using System.ComponentModel; using System.Globalization; using System.Linq; using System.Reflection; -using System.Text; using System.Threading; using System.Threading.Tasks; using Microsoft.Extensions.Logging; using Microsoft.SemanticKernel; -using Microsoft.SemanticKernel.AI; using Microsoft.SemanticKernel.AI.TextCompletion; using Microsoft.SemanticKernel.Events; using Microsoft.SemanticKernel.Http; @@ -594,20 +591,4 @@ public async Task ReadFunctionCollectionAsync(SKContext context, Kern return context; } } - - private (Mock textResultMock, Mock textCompletionMock) SetupMocks(string? completionResult = null) - { - var mockTextResult = new Mock(); - mockTextResult.Setup(m => m.GetCompletionAsync(It.IsAny())).ReturnsAsync(completionResult ?? "LLM Result about UnitTests"); - - var mockTextCompletion = new Mock(); - mockTextCompletion.Setup(m => m.GetCompletionsAsync(It.IsAny(), It.IsAny(), It.IsAny())).ReturnsAsync(new List { mockTextResult.Object }); - - return (mockTextResult, mockTextCompletion); - } - - private static MethodInfo Method(Delegate method) - { - return method.Method; - } } From ba33198bf4620808ac551bbcbb5c98e94336ea48 Mon Sep 17 00:00:00 2001 From: Roger Barreto <19890735+RogerBarreto@users.noreply.github.com> Date: Fri, 24 Nov 2023 12:07:48 +0000 Subject: [PATCH 45/46] Removed duplicated Invoke, UT passing --- .../Functions/SKFunctionFromMethod.cs | 11 ++--------- .../Functions/SKFunctionFromPrompt.cs | 1 - .../Functions/SemanticFunctionTests.cs | 2 -- dotnet/src/SemanticKernel.UnitTests/KernelTests.cs | 1 - 4 files changed, 2 insertions(+), 13 deletions(-) diff --git a/dotnet/src/SemanticKernel.Core/Functions/SKFunctionFromMethod.cs b/dotnet/src/SemanticKernel.Core/Functions/SKFunctionFromMethod.cs index 9edec9c56559..44278ec07d2a 100644 --- a/dotnet/src/SemanticKernel.Core/Functions/SKFunctionFromMethod.cs +++ b/dotnet/src/SemanticKernel.Core/Functions/SKFunctionFromMethod.cs @@ -155,13 +155,7 @@ protected override async IAsyncEnumerable InvokeCoreStreamingAsync( AIRequestSettings? requestSettings = null, [EnumeratorCancellation] CancellationToken cancellationToken = default) { - // Invoke pre hook, and stop if skipping requested. - var invokingEventArgs = this.CallFunctionInvoking(kernel, context); - if (invokingEventArgs.IsSkipRequested || invokingEventArgs.CancelToken.IsCancellationRequested) - { - yield break; - } - + // We don't invoke the hook here as the InvokeCoreAsync will do that for us var functionResult = await this.InvokeCoreAsync(kernel, context, requestSettings, cancellationToken).ConfigureAwait(false); if (functionResult.Value is T) { @@ -182,8 +176,7 @@ protected override async IAsyncEnumerable InvokeCoreStreamingAsync( throw new NotSupportedException($"Streaming function {this.Name} does not support type {typeof(T)}"); - // Invoked is not supported for streaming yet - // There is no post cancellation check to override the result as the stream data was already sent. + // We don't invoke the hook here as the InvokeCoreAsync will do that for us } private FunctionInvokingEventArgs CallFunctionInvoking(Kernel kernel, SKContext context) diff --git a/dotnet/src/SemanticKernel.Core/Functions/SKFunctionFromPrompt.cs b/dotnet/src/SemanticKernel.Core/Functions/SKFunctionFromPrompt.cs index bdd36a250d4f..405477937282 100644 --- a/dotnet/src/SemanticKernel.Core/Functions/SKFunctionFromPrompt.cs +++ b/dotnet/src/SemanticKernel.Core/Functions/SKFunctionFromPrompt.cs @@ -5,7 +5,6 @@ using System.Diagnostics; using System.Linq; using System.Runtime.CompilerServices; -using System.Text; using System.Text.Json; using System.Threading; using System.Threading.Tasks; diff --git a/dotnet/src/SemanticKernel.UnitTests/Functions/SemanticFunctionTests.cs b/dotnet/src/SemanticKernel.UnitTests/Functions/SemanticFunctionTests.cs index 7b68debcb5c5..f6b0a7f4f22b 100644 --- a/dotnet/src/SemanticKernel.UnitTests/Functions/SemanticFunctionTests.cs +++ b/dotnet/src/SemanticKernel.UnitTests/Functions/SemanticFunctionTests.cs @@ -2,8 +2,6 @@ using System; using System.Collections.Generic; -using System.Linq; -using System.Reflection; using System.Threading; using System.Threading.Tasks; using Microsoft.Extensions.Logging; diff --git a/dotnet/src/SemanticKernel.UnitTests/KernelTests.cs b/dotnet/src/SemanticKernel.UnitTests/KernelTests.cs index 3000e4d5f43a..415ab3016117 100644 --- a/dotnet/src/SemanticKernel.UnitTests/KernelTests.cs +++ b/dotnet/src/SemanticKernel.UnitTests/KernelTests.cs @@ -4,7 +4,6 @@ using System.ComponentModel; using System.Globalization; using System.Linq; -using System.Reflection; using System.Threading; using System.Threading.Tasks; using Microsoft.Extensions.Logging; From 7104c20a3d81ee5abad50369447acdaf48b1e526 Mon Sep 17 00:00:00 2001 From: Roger Barreto <19890735+RogerBarreto@users.noreply.github.com> Date: Fri, 24 Nov 2023 12:10:37 +0000 Subject: [PATCH 46/46] Warning fix --- .../SemanticKernel.UnitTests/Functions/SemanticFunctionTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dotnet/src/SemanticKernel.UnitTests/Functions/SemanticFunctionTests.cs b/dotnet/src/SemanticKernel.UnitTests/Functions/SemanticFunctionTests.cs index f6b0a7f4f22b..d40426f21ed5 100644 --- a/dotnet/src/SemanticKernel.UnitTests/Functions/SemanticFunctionTests.cs +++ b/dotnet/src/SemanticKernel.UnitTests/Functions/SemanticFunctionTests.cs @@ -384,7 +384,7 @@ private Mock SetupStreamingMocks(T completionResult) return mockTextCompletion; } - private class TestStreamingContent : StreamingContent + private sealed class TestStreamingContent : StreamingContent { public TestStreamingContent() : base(null) {