Merge pull request #537 from iceljc/features/refine-image-reading

Oceania2018 · web-flow · commit b4f87fdd06b5 · 2024-07-11T11:42:05.000-05:00
refine chat image reading
diff --git a/src/Infrastructure/BotSharp.Core/Using.cs b/src/Infrastructure/BotSharp.Core/Using.cs
@@ -28,7 +28,6 @@
 global using BotSharp.Abstraction.Files.Enums;
 global using BotSharp.Abstraction.Translation.Attributes;
 global using BotSharp.Abstraction.Messaging.Enums;
-global using BotSharp.Abstraction.Http.Settings;
 global using BotSharp.Core.Repository;
 global using BotSharp.Core.Routing;
 global using BotSharp.Core.Agents.Services;
diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs
@@ -85,7 +85,7 @@ public async Task<string> MultiModalCompletion([FromBody] IncomingMessageModel i
         try
         {
             var completion = CompletionProvider.GetChatCompletion(_services, provider: input.Provider ?? "openai",
-                modelId: input.ModelId ?? "gpt-4", multiModal: true);
+                model: input.Model ?? "gpt-4o", multiModal: true);
             var message = await completion.GetChatCompletions(new Agent()
             {
                 Id = Guid.Empty.ToString(),
diff --git a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Chat/ChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Chat/ChatCompletionProvider.cs
@@ -193,7 +193,6 @@ public async Task<bool> GetChatCompletionsStreamingAsync(Agent agent, List<RoleD
         return true;
     }
 
-
     protected (string, IEnumerable<ChatMessage>, ChatCompletionOptions) PrepareOptions(Agent agent, List<RoleDialogModel> conversations)
     {
         var agentService = _services.GetRequiredService<IAgentService>();
@@ -257,40 +256,34 @@ public async Task<bool> GetChatCompletionsStreamingAsync(Agent agent, List<RoleD
             {
                 var text = !string.IsNullOrWhiteSpace(message.Payload) ? message.Payload : message.Content;
                 var textPart = ChatMessageContentPart.CreateTextMessageContentPart(text);
-                var chat = new UserChatMessage(textPart)
-                {
-                    ParticipantName = message.FunctionName
-                };
+                var contentParts = new List<ChatMessageContentPart> { textPart };
 
-                if (allowMultiModal)
+                if (allowMultiModal && !message.Files.IsNullOrEmpty())
                 {
-                    if (!message.Files.IsNullOrEmpty())
+                    foreach (var file in message.Files)
                     {
-                        foreach (var file in message.Files)
+                        if (!string.IsNullOrEmpty(file.FileUrl))
+                        {
+                            var uri = new Uri(file.FileUrl);
+                            var contentPart = ChatMessageContentPart.CreateImageMessageContentPart(uri, ImageChatMessageContentPartDetail.Low);
+                            contentParts.Add(contentPart);
+                        }
+                        else if (!string.IsNullOrEmpty(file.FileData))
                         {
-                            if (!string.IsNullOrEmpty(file.FileUrl))
-                            {
-                                var uri = new Uri(file.FileUrl);
-                                var contentPart = ChatMessageContentPart.CreateImageMessageContentPart(uri, ImageChatMessageContentPartDetail.Low);
-                                chat = new UserChatMessage(textPart, contentPart) { ParticipantName = message.FunctionName };
-                            }
-                            else if (!string.IsNullOrEmpty(file.FileData))
-                            {
-                                var (contentType, bytes) = fileService.GetFileInfoFromData(file.FileData);
-                                var contentPart = ChatMessageContentPart.CreateImageMessageContentPart(BinaryData.FromBytes(bytes), contentType, ImageChatMessageContentPartDetail.Low);
-                                chat = new UserChatMessage(textPart, contentPart) { ParticipantName = message.FunctionName };
-                            }
-                            else if (!string.IsNullOrEmpty(file.FileStorageUrl))
-                            {
-                                var contentType = fileService.GetFileContentType(file.FileStorageUrl);
-                                using var stream = File.OpenRead(file.FileStorageUrl);
-                                var contentPart = ChatMessageContentPart.CreateImageMessageContentPart(BinaryData.FromStream(stream), contentType, ImageChatMessageContentPartDetail.Low);
-                                chat = new UserChatMessage(textPart, contentPart) { ParticipantName = message.FunctionName };
-                            }
+                            var (contentType, bytes) = fileService.GetFileInfoFromData(file.FileData);
+                            var contentPart = ChatMessageContentPart.CreateImageMessageContentPart(BinaryData.FromBytes(bytes), contentType, ImageChatMessageContentPartDetail.Low);
+                            contentParts.Add(contentPart);
+                        }
+                        else if (!string.IsNullOrEmpty(file.FileStorageUrl))
+                        {
+                            var contentType = fileService.GetFileContentType(file.FileStorageUrl);
+                            using var stream = File.OpenRead(file.FileStorageUrl);
+                            var contentPart = ChatMessageContentPart.CreateImageMessageContentPart(BinaryData.FromStream(stream), contentType, ImageChatMessageContentPartDetail.Low);
+                            contentParts.Add(contentPart);
                         }
                     }
                 }
-                messages.Add(chat);
+                messages.Add(new UserChatMessage(contentParts) { ParticipantName = message.FunctionName });
             }
             else if (message.Role == AgentRole.Assistant)
             {
@@ -302,7 +295,6 @@ public async Task<bool> GetChatCompletionsStreamingAsync(Agent agent, List<RoleD
         return (prompt, messages, options);
     }
 
-
     private string GetPrompt(IEnumerable<ChatMessage> messages, ChatCompletionOptions options)
     {
         var prompt = string.Empty;
diff --git a/src/Plugins/BotSharp.Plugin.HttpHandler/Functions/HandleHttpRequestFn.cs b/src/Plugins/BotSharp.Plugin.HttpHandler/Functions/HandleHttpRequestFn.cs
@@ -1,5 +1,4 @@
 using System.Net.Http;
-using BotSharp.Plugin.HttpHandler.LlmContexts;
 using Microsoft.AspNetCore.Http;
 using Microsoft.Extensions.Logging;
 
@@ -58,7 +57,7 @@ public async Task<bool> Execute(RoleDialogModel message)
     {
         if (string.IsNullOrEmpty(url)) return null;
 
-        var settings = _services.GetRequiredService<HttpSettings>();
+        var settings = _services.GetRequiredService<HttpHandlerSettings>();
         using var client = _httpClientFactory.CreateClient();
         AddRequestHeaders(client);
 
@@ -82,7 +81,7 @@ private void AddRequestHeaders(HttpClient client)
     {
         client.DefaultRequestHeaders.Add("Authorization", $"{_context.HttpContext.Request.Headers["Authorization"]}");
 
-        var settings = _services.GetRequiredService<HttpSettings>();
+        var settings = _services.GetRequiredService<HttpHandlerSettings>();
         var origin = !string.IsNullOrEmpty(settings.Origin) ? settings.Origin : $"{_context.HttpContext.Request.Headers["Origin"]}";
         if (!string.IsNullOrEmpty(origin))
         {
diff --git a/src/Plugins/BotSharp.Plugin.HttpHandler/Hooks/HttpHandlerHook.cs b/src/Plugins/BotSharp.Plugin.HttpHandler/Hooks/HttpHandlerHook.cs
@@ -3,7 +3,6 @@
 using BotSharp.Abstraction.Agents.Settings;
 using BotSharp.Abstraction.Functions.Models;
 using BotSharp.Abstraction.Repositories;
-using BotSharp.Plugin.HttpHandler.Enums;
 
 namespace BotSharp.Plugin.HttpHandler.Hooks;
 
diff --git a/src/Plugins/BotSharp.Plugin.HttpHandler/Hooks/HttpHandlerUtilityHook.cs b/src/Plugins/BotSharp.Plugin.HttpHandler/Hooks/HttpHandlerUtilityHook.cs
@@ -1,5 +1,4 @@
 using BotSharp.Abstraction.Agents;
-using BotSharp.Plugin.HttpHandler.Enums;
 
 namespace BotSharp.Plugin.HttpHandler.Hooks;
 
diff --git a/src/Plugins/BotSharp.Plugin.HttpHandler/HttpHandlerPlugin.cs b/src/Plugins/BotSharp.Plugin.HttpHandler/HttpHandlerPlugin.cs
@@ -1,6 +1,5 @@
 using BotSharp.Abstraction.Agents;
 using BotSharp.Abstraction.Settings;
-using BotSharp.Plugin.HttpHandler.Hooks;
 using Microsoft.Extensions.Configuration;
 
 namespace BotSharp.Plugin.HttpHandler;
@@ -18,7 +17,7 @@ public void RegisterDI(IServiceCollection services, IConfiguration config)
         services.AddScoped(provider =>
         {
             var settingService = provider.GetRequiredService<ISettingService>();
-            return settingService.Bind<HttpSettings>("Http");
+            return settingService.Bind<HttpHandlerSettings>("HttpHandler");
         });
 
         services.AddScoped<IAgentHook, HttpHandlerHook>();
diff --git a/src/Plugins/BotSharp.Plugin.HttpHandler/Settings/HttpHandlerSettings.cs b/src/Plugins/BotSharp.Plugin.HttpHandler/Settings/HttpHandlerSettings.cs
@@ -1,6 +1,6 @@
-namespace BotSharp.Abstraction.Http.Settings;
+namespace BotSharp.Plugin.HttpHandler.Settings;
 
-public class HttpSettings
+public class HttpHandlerSettings
 {
     public string BaseAddress { get; set; } = string.Empty;
     public string Origin { get; set; } = string.Empty;
diff --git a/src/Plugins/BotSharp.Plugin.HttpHandler/Using.cs b/src/Plugins/BotSharp.Plugin.HttpHandler/Using.cs
@@ -15,5 +15,8 @@
 global using BotSharp.Abstraction.Messaging;
 global using BotSharp.Abstraction.Messaging.Models.RichContent;
 global using BotSharp.Abstraction.Options;
-global using BotSharp.Abstraction.Http.Settings;
-global using BotSharp.Abstraction.Messaging.Enums;
+global using BotSharp.Abstraction.Messaging.Enums;
+global using BotSharp.Plugin.HttpHandler.Hooks;
+global using BotSharp.Plugin.HttpHandler.Settings;
+global using BotSharp.Plugin.HttpHandler.LlmContexts;
+global using BotSharp.Plugin.HttpHandler.Enums;
diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Chat/ChatCompletionProvider.cs
@@ -257,40 +257,34 @@ public async Task<bool> GetChatCompletionsStreamingAsync(Agent agent, List<RoleD
             {
                 var text = !string.IsNullOrWhiteSpace(message.Payload) ? message.Payload : message.Content;
                 var textPart = ChatMessageContentPart.CreateTextMessageContentPart(text);
-                var chat = new UserChatMessage(textPart)
-                {
-                    ParticipantName = message.FunctionName
-                };
+                var contentParts = new List<ChatMessageContentPart> { textPart };
 
-                if (allowMultiModal)
+                if (allowMultiModal && !message.Files.IsNullOrEmpty())
                 {
-                    if (!message.Files.IsNullOrEmpty())
+                    foreach (var file in message.Files)
                     {
-                        foreach (var file in message.Files)
+                        if (!string.IsNullOrEmpty(file.FileUrl))
+                        {
+                            var uri = new Uri(file.FileUrl);
+                            var contentPart = ChatMessageContentPart.CreateImageMessageContentPart(uri, ImageChatMessageContentPartDetail.Low);
+                            contentParts.Add(contentPart);
+                        }
+                        else if (!string.IsNullOrEmpty(file.FileData))
+                        {
+                            var (contentType, bytes) = fileService.GetFileInfoFromData(file.FileData);
+                            var contentPart = ChatMessageContentPart.CreateImageMessageContentPart(BinaryData.FromBytes(bytes), contentType, ImageChatMessageContentPartDetail.Low);
+                            contentParts.Add(contentPart);
+                        }
+                        else if (!string.IsNullOrEmpty(file.FileStorageUrl))
                         {
-                            if (!string.IsNullOrEmpty(file.FileUrl))
-                            {
-                                var uri = new Uri(file.FileUrl);
-                                var contentPart = ChatMessageContentPart.CreateImageMessageContentPart(uri, ImageChatMessageContentPartDetail.Low);
-                                chat = new UserChatMessage(textPart, contentPart) { ParticipantName = message.FunctionName };
-                            }
-                            else if (!string.IsNullOrEmpty(file.FileData))
-                            {
-                                var (contentType, bytes) = fileService.GetFileInfoFromData(file.FileData);
-                                var contentPart = ChatMessageContentPart.CreateImageMessageContentPart(BinaryData.FromBytes(bytes), contentType, ImageChatMessageContentPartDetail.Low);
-                                chat = new UserChatMessage(textPart, contentPart) { ParticipantName = message.FunctionName };
-                            }
-                            else if (!string.IsNullOrEmpty(file.FileStorageUrl))
-                            {
-                                var contentType = fileService.GetFileContentType(file.FileStorageUrl);
-                                using var stream = File.OpenRead(file.FileStorageUrl);
-                                var contentPart = ChatMessageContentPart.CreateImageMessageContentPart(BinaryData.FromStream(stream), contentType, ImageChatMessageContentPartDetail.Low);
-                                chat = new UserChatMessage(textPart, contentPart) { ParticipantName = message.FunctionName };
-                            }
+                            var contentType = fileService.GetFileContentType(file.FileStorageUrl);
+                            using var stream = File.OpenRead(file.FileStorageUrl);
+                            var contentPart = ChatMessageContentPart.CreateImageMessageContentPart(BinaryData.FromStream(stream), contentType, ImageChatMessageContentPartDetail.Low);
+                            contentParts.Add(contentPart);
                         }
                     }
                 }
-                messages.Add(chat);
+                messages.Add(new UserChatMessage(contentParts) { ParticipantName = message.FunctionName });
             }
             else if (message.Role == AgentRole.Assistant)
             {
diff --git a/src/WebStarter/appsettings.json b/src/WebStarter/appsettings.json
@@ -154,7 +154,7 @@
     "Driver": "Playwright"
   },
 
-  "Http": {
+  "HttpHandler": {
     "BaseAddress": "",
     "Origin": ""
   },

Original file line number	Diff line number	Diff line change
`@@ -85,7 +85,7 @@ public async Task<string> MultiModalCompletion([FromBody] IncomingMessageModel i`
`85`	`85`	`try`
`86`	`86`	`{`
`87`	`87`	`var completion = CompletionProvider.GetChatCompletion(_services, provider: input.Provider ?? "openai",`
`88`		`- modelId: input.ModelId ?? "gpt-4", multiModal: true);`
	`88`	`+ model: input.Model ?? "gpt-4o", multiModal: true);`
`89`	`89`	`var message = await completion.GetChatCompletions(new Agent()`
`90`	`90`	`{`
`91`	`91`	`Id = Guid.Empty.ToString(),`
Original file line number	Diff line number	Diff line change
`@@ -1,5 +1,4 @@`
`1`	`1`	`using System.Net.Http;`
`2`		`-using BotSharp.Plugin.HttpHandler.LlmContexts;`
`3`	`2`	`using Microsoft.AspNetCore.Http;`
`4`	`3`	`using Microsoft.Extensions.Logging;`
`5`	`4`
`@@ -58,7 +57,7 @@ public async Task<bool> Execute(RoleDialogModel message)`
`58`	`57`	`{`
`59`	`58`	`if (string.IsNullOrEmpty(url)) return null;`
`60`	`59`
`61`		`- var settings = _services.GetRequiredService<HttpSettings>();`
	`60`	`+ var settings = _services.GetRequiredService<HttpHandlerSettings>();`
`62`	`61`	`using var client = _httpClientFactory.CreateClient();`
`63`	`62`	`AddRequestHeaders(client);`
`64`	`63`
`@@ -82,7 +81,7 @@ private void AddRequestHeaders(HttpClient client)`
`82`	`81`	`{`
`83`	`82`	`client.DefaultRequestHeaders.Add("Authorization", $"{_context.HttpContext.Request.Headers["Authorization"]}");`
`84`	`83`
`85`		`- var settings = _services.GetRequiredService<HttpSettings>();`
	`84`	`+ var settings = _services.GetRequiredService<HttpHandlerSettings>();`
`86`	`85`	`var origin = !string.IsNullOrEmpty(settings.Origin) ? settings.Origin : $"{_context.HttpContext.Request.Headers["Origin"]}";`
`87`	`86`	`if (!string.IsNullOrEmpty(origin))`
`88`	`87`	`{`
Original file line number	Diff line number	Diff line change
`@@ -1,5 +1,4 @@`
`1`	`1`	`using BotSharp.Abstraction.Agents;`
`2`		`-using BotSharp.Plugin.HttpHandler.Enums;`
`3`	`2`
`4`	`3`	`namespace BotSharp.Plugin.HttpHandler.Hooks;`
`5`	`4`
Original file line number	Diff line number	Diff line change
`@@ -1,6 +1,6 @@`
`1`		`-namespace BotSharp.Abstraction.Http.Settings;`
	`1`	`+namespace BotSharp.Plugin.HttpHandler.Settings;`
`2`	`2`
`3`		`-public class HttpSettings`
	`3`	`+public class HttpHandlerSettings`
`4`	`4`	`{`
`5`	`5`	`public string BaseAddress { get; set; } = string.Empty;`
`6`	`6`	`public string Origin { get; set; } = string.Empty;`