From 6cc9f14b8c2fd0d98f11f1ab67780874b7d0f317 Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Mon, 24 Jun 2024 13:03:05 -0500 Subject: [PATCH 1/9] add image generation --- .../MLTasks/IChatCompletion.cs | 3 + .../MLTasks/Settings/LlmModelSetting.cs | 3 +- .../Controllers/InstructModeController.cs | 32 ++++++++++ .../Instructs/ImageGenerationViewModel.cs | 18 ++++++ .../Providers/ChatCompletionProvider.cs | 5 ++ .../Providers/ChatCompletionProvider.cs | 58 +++++++++++++++++++ .../Providers/ChatCompletionProvider.cs | 5 ++ .../Providers/ChatCompletionProvider.cs | 5 ++ .../Providers/ChatCompletionProvider.cs | 5 ++ .../Providers/ChatCompletionProvider.cs | 5 ++ .../SemanticKernelChatCompletionProvider.cs | 5 ++ .../Providers/ChatCompletionProvider.cs | 7 ++- 12 files changed, 148 insertions(+), 3 deletions(-) create mode 100644 src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/ImageGenerationViewModel.cs diff --git a/src/Infrastructure/BotSharp.Abstraction/MLTasks/IChatCompletion.cs b/src/Infrastructure/BotSharp.Abstraction/MLTasks/IChatCompletion.cs index 306e72e36..c5332b406 100644 --- a/src/Infrastructure/BotSharp.Abstraction/MLTasks/IChatCompletion.cs +++ b/src/Infrastructure/BotSharp.Abstraction/MLTasks/IChatCompletion.cs @@ -24,4 +24,7 @@ Task GetChatCompletionsAsync(Agent agent, Task GetChatCompletionsStreamingAsync(Agent agent, List conversations, Func onMessageReceived); + + Task GetImageGeneration(Agent agent, + List conversations); } diff --git a/src/Infrastructure/BotSharp.Abstraction/MLTasks/Settings/LlmModelSetting.cs b/src/Infrastructure/BotSharp.Abstraction/MLTasks/Settings/LlmModelSetting.cs index b86578fe5..0a0bd42a3 100644 --- a/src/Infrastructure/BotSharp.Abstraction/MLTasks/Settings/LlmModelSetting.cs +++ b/src/Infrastructure/BotSharp.Abstraction/MLTasks/Settings/LlmModelSetting.cs @@ -51,5 +51,6 @@ public override string ToString() public enum LlmModelType { Text = 1, - Chat = 2 + Chat = 2, + Image = 3 } diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs index 45120a263..0b013456e 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs @@ -103,4 +103,36 @@ public async Task MultiModalCompletion([FromBody] IncomingMessageModel i return $"Error in analyzing files."; } } + + [HttpPost("/instruct/image-generation")] + public async Task ImageGeneration([FromBody] IncomingMessageModel input) + { + var state = _services.GetRequiredService(); + input.States.ForEach(x => state.SetState(x.Key, x.Value, activeRounds: x.ActiveRounds, source: StateSource.External)); + var imageViewModel = new ImageGenerationViewModel(); + + try + { + var completion = CompletionProvider.GetChatCompletion(_services, provider: input.Provider ?? "openai", + modelId: input.ModelId ?? "dall-e"); + var message = await completion.GetImageGeneration(new Agent() + { + Id = Guid.Empty.ToString(), + }, new List + { + new RoleDialogModel(AgentRole.User, input.Text) + }); + + imageViewModel.Content = message.Content; + imageViewModel.Data = message.Data; + return imageViewModel; + } + catch (Exception ex) + { + var error = "Error in image generation."; + _logger.LogError($"{error} {ex.Message}"); + imageViewModel.Message = error; + return imageViewModel; + } + } } diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/ImageGenerationViewModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/ImageGenerationViewModel.cs new file mode 100644 index 000000000..179c05f40 --- /dev/null +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/ImageGenerationViewModel.cs @@ -0,0 +1,18 @@ +using System.Text.Json.Serialization; + +namespace BotSharp.OpenAPI.ViewModels.Instructs; + +public class ImageGenerationViewModel +{ + [JsonPropertyName("content")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? Content { get; set; } + + [JsonPropertyName("data")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public object? Data { get; set; } + + [JsonPropertyName("message")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? Message { get; set; } +} diff --git a/src/Plugins/BotSharp.Plugin.AnthropicAI/Providers/ChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.AnthropicAI/Providers/ChatCompletionProvider.cs index 872021098..8977e04db 100644 --- a/src/Plugins/BotSharp.Plugin.AnthropicAI/Providers/ChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.AnthropicAI/Providers/ChatCompletionProvider.cs @@ -264,4 +264,9 @@ public void SetModelName(string model) { _model = model; } + + public Task GetImageGeneration(Agent agent, List conversations) + { + throw new NotImplementedException(); + } } diff --git a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/ChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/ChatCompletionProvider.cs index 50fd33ce5..d85b4d658 100644 --- a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/ChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/ChatCompletionProvider.cs @@ -446,4 +446,62 @@ ChatRequestToolMessage GetToolCallResponseMessage(ChatCompletionsToolCall toolCa functionResultData = "31 celsius"; return new ChatRequestToolMessage(functionResultData.ToString(), toolCall.Id); } + + public async Task GetImageGeneration(Agent agent, List conversations) + { + var contentHooks = _services.GetServices().ToList(); + foreach (var hook in contentHooks) + { + await hook.BeforeGenerating(agent, conversations); + } + + var client = ProviderHelper.GetClient(Provider, _model, _services); + var options = BuildImageGenerationOptions(conversations); + var response = await client.GetImageGenerationsAsync(options); + var image = response.Value.Data.First(); + + var content = string.Empty; + if (!string.IsNullOrEmpty(image.RevisedPrompt)) + { + content = image.RevisedPrompt; + } + + var responseMessage = new RoleDialogModel(AgentRole.Assistant, content) + { + CurrentAgentId = agent.Id, + MessageId = conversations.LastOrDefault()?.MessageId ?? string.Empty, + Data = image.Url.AbsoluteUri ?? image.Base64Data + }; + + foreach (var hook in contentHooks) + { + await hook.AfterGenerated(responseMessage, new TokenStatsModel + { + Prompt = options.Prompt, + Provider = Provider, + Model = _model, + PromptCount = options.Prompt.Split(' ', StringSplitOptions.RemoveEmptyEntries).Count(), + CompletionCount = content.Split(' ', StringSplitOptions.RemoveEmptyEntries).Count() + }); + } + + return responseMessage; + } + + private ImageGenerationOptions BuildImageGenerationOptions(List conversations) + { + var state = _services.GetRequiredService(); + + var sizeValue = !string.IsNullOrEmpty(state.GetState("image_size")) ? state.GetState("image_size") : "1024x1024"; + var qualityValue = !string.IsNullOrEmpty(state.GetState("image_quality")) ? state.GetState("image_quality") : "standard"; + + var options = new ImageGenerationOptions + { + DeploymentName = _model, + Prompt = conversations.LastOrDefault()?.Payload ?? conversations.LastOrDefault()?.Content ?? string.Empty, + Size = new ImageSize(sizeValue), + Quality = new ImageGenerationQuality(qualityValue) + }; + return options; + } } diff --git a/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/ChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/ChatCompletionProvider.cs index d278b110b..c4628c6d0 100644 --- a/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/ChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/ChatCompletionProvider.cs @@ -149,4 +149,9 @@ public void SetModelName(string model) { _model = model; } + + public async Task GetImageGeneration(Agent agent, List conversations) + { + throw new NotImplementedException(); + } } diff --git a/src/Plugins/BotSharp.Plugin.HuggingFace/Providers/ChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.HuggingFace/Providers/ChatCompletionProvider.cs index 88b37dbb7..80c950c8d 100644 --- a/src/Plugins/BotSharp.Plugin.HuggingFace/Providers/ChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.HuggingFace/Providers/ChatCompletionProvider.cs @@ -139,4 +139,9 @@ public async Task GetChatCompletions(Agent agent, List GetImageGeneration(Agent agent, List conversations) + { + throw new NotImplementedException(); + } } diff --git a/src/Plugins/BotSharp.Plugin.LLamaSharp/Providers/ChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.LLamaSharp/Providers/ChatCompletionProvider.cs index 0db444ce1..cdd96a0b5 100644 --- a/src/Plugins/BotSharp.Plugin.LLamaSharp/Providers/ChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.LLamaSharp/Providers/ChatCompletionProvider.cs @@ -191,4 +191,9 @@ public void SetModelName(string model) { _model = model; } + + public async Task GetImageGeneration(Agent agent, List conversations) + { + throw new NotImplementedException(); + } } diff --git a/src/Plugins/BotSharp.Plugin.MetaGLM/Providers/ChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.MetaGLM/Providers/ChatCompletionProvider.cs index c702a7ecd..7010842db 100644 --- a/src/Plugins/BotSharp.Plugin.MetaGLM/Providers/ChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.MetaGLM/Providers/ChatCompletionProvider.cs @@ -231,6 +231,11 @@ public Task GetChatCompletionsStreamingAsync(Agent agent, List GetImageGeneration(Agent agent, List conversations) + { + throw new NotImplementedException(); + } + public void SetModelName(string model) { _model = model; diff --git a/src/Plugins/BotSharp.Plugin.SemanticKernel/SemanticKernelChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.SemanticKernel/SemanticKernelChatCompletionProvider.cs index 156f238c5..cb70b1896 100644 --- a/src/Plugins/BotSharp.Plugin.SemanticKernel/SemanticKernelChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.SemanticKernel/SemanticKernelChatCompletionProvider.cs @@ -102,5 +102,10 @@ public void SetModelName(string model) { _model = model; } + + public async Task GetImageGeneration(Agent agent, List conversations) + { + throw new NotImplementedException(); + } } } \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.SparkDesk/Providers/ChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.SparkDesk/Providers/ChatCompletionProvider.cs index 7d529df14..25e588c3b 100644 --- a/src/Plugins/BotSharp.Plugin.SparkDesk/Providers/ChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.SparkDesk/Providers/ChatCompletionProvider.cs @@ -1,8 +1,6 @@ using BotSharp.Abstraction.Agents; using BotSharp.Abstraction.Agents.Enums; using BotSharp.Abstraction.Loggers; -using BotSharp.Abstraction.Routing; -using Sdcb.SparkDesk.ResponseInternals; namespace BotSharp.Plugin.SparkDesk.Providers; @@ -270,4 +268,9 @@ private FunctionDef ConvertToFunctionDef(BotSharp.Abstraction.Functions.Models.F FunctionDef functionDef = new FunctionDef(def.Name, def.Description, fundef.ToArray()); return functionDef; } + + public async Task GetImageGeneration(Agent agent, List conversations) + { + throw new NotImplementedException(); + } } From 9d3244e075324d9cd02fd6ae3e41e41a44bdcbd8 Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Mon, 24 Jun 2024 14:32:52 -0500 Subject: [PATCH 2/9] add image generation provider --- .../MLTasks/IChatCompletion.cs | 3 - .../MLTasks/IImageGeneration.cs | 17 +++ .../MLTasks/ILlmProviderService.cs | 2 +- .../MLTasks/Settings/LlmModelSetting.cs | 5 + .../Infrastructures/CompletionProvider.cs | 72 ++++++++---- .../Infrastructures/LlmProviderService.cs | 4 +- .../Controllers/InstructModeController.cs | 13 ++- .../Controllers/LlmProviderController.cs | 3 +- .../Providers/ChatCompletionProvider.cs | 5 - .../AzureOpenAiPlugin.cs | 2 + .../Providers/ChatCompletionProvider.cs | 58 ---------- .../Providers/ImageGenerationProvider.cs | 104 ++++++++++++++++++ .../OpenAiImageGenerationProvider.cs | 16 +++ .../Providers/ChatCompletionProvider.cs | 5 - .../Providers/ChatCompletionProvider.cs | 5 - .../Providers/ChatCompletionProvider.cs | 5 - .../Providers/ChatCompletionProvider.cs | 5 - .../SemanticKernelChatCompletionProvider.cs | 5 - .../Providers/ChatCompletionProvider.cs | 5 - 19 files changed, 206 insertions(+), 128 deletions(-) create mode 100644 src/Infrastructure/BotSharp.Abstraction/MLTasks/IImageGeneration.cs create mode 100644 src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/ImageGenerationProvider.cs create mode 100644 src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/OpenAiImageGenerationProvider.cs diff --git a/src/Infrastructure/BotSharp.Abstraction/MLTasks/IChatCompletion.cs b/src/Infrastructure/BotSharp.Abstraction/MLTasks/IChatCompletion.cs index c5332b406..306e72e36 100644 --- a/src/Infrastructure/BotSharp.Abstraction/MLTasks/IChatCompletion.cs +++ b/src/Infrastructure/BotSharp.Abstraction/MLTasks/IChatCompletion.cs @@ -24,7 +24,4 @@ Task GetChatCompletionsAsync(Agent agent, Task GetChatCompletionsStreamingAsync(Agent agent, List conversations, Func onMessageReceived); - - Task GetImageGeneration(Agent agent, - List conversations); } diff --git a/src/Infrastructure/BotSharp.Abstraction/MLTasks/IImageGeneration.cs b/src/Infrastructure/BotSharp.Abstraction/MLTasks/IImageGeneration.cs new file mode 100644 index 000000000..b257e114b --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/MLTasks/IImageGeneration.cs @@ -0,0 +1,17 @@ +namespace BotSharp.Abstraction.MLTasks; + +public interface IImageGeneration +{ + /// + /// The LLM provider like Microsoft Azure, OpenAI, ClaudAI + /// + string Provider { get; } + + /// + /// Set model name, one provider can consume different model or version(s) + /// + /// deployment name + void SetModelName(string model); + + Task GetImageGeneration(Agent agent, List conversations); +} diff --git a/src/Infrastructure/BotSharp.Abstraction/MLTasks/ILlmProviderService.cs b/src/Infrastructure/BotSharp.Abstraction/MLTasks/ILlmProviderService.cs index 20762fe0f..1e203333e 100644 --- a/src/Infrastructure/BotSharp.Abstraction/MLTasks/ILlmProviderService.cs +++ b/src/Infrastructure/BotSharp.Abstraction/MLTasks/ILlmProviderService.cs @@ -6,6 +6,6 @@ public interface ILlmProviderService { LlmModelSetting GetSetting(string provider, string model); List GetProviders(); - LlmModelSetting GetProviderModel(string provider, string id, bool? multiModal = null); + LlmModelSetting GetProviderModel(string provider, string id, bool? multiModal = null, bool imageGenerate = false); List GetProviderModels(string provider); } diff --git a/src/Infrastructure/BotSharp.Abstraction/MLTasks/Settings/LlmModelSetting.cs b/src/Infrastructure/BotSharp.Abstraction/MLTasks/Settings/LlmModelSetting.cs index 0a0bd42a3..e82564170 100644 --- a/src/Infrastructure/BotSharp.Abstraction/MLTasks/Settings/LlmModelSetting.cs +++ b/src/Infrastructure/BotSharp.Abstraction/MLTasks/Settings/LlmModelSetting.cs @@ -32,6 +32,11 @@ public class LlmModelSetting /// public bool MultiModal { get; set; } + /// + /// If true, allow generating images + /// + public bool ImageGeneration { get; set; } + /// /// Prompt cost per 1K token /// diff --git a/src/Infrastructure/BotSharp.Core/Infrastructures/CompletionProvider.cs b/src/Infrastructure/BotSharp.Core/Infrastructures/CompletionProvider.cs index 4655b1de7..3c116c086 100644 --- a/src/Infrastructure/BotSharp.Core/Infrastructures/CompletionProvider.cs +++ b/src/Infrastructure/BotSharp.Core/Infrastructures/CompletionProvider.cs @@ -55,11 +55,56 @@ public static IChatCompletion GetChatCompletion(IServiceProvider services, return completer; } - private static (string, string) GetProviderAndModel(IServiceProvider services, + public static ITextCompletion GetTextCompletion(IServiceProvider services, + string? provider = null, + string? model = null, + AgentLlmConfig? agentConfig = null) + { + var completions = services.GetServices(); + + (provider, model) = GetProviderAndModel(services, provider: provider, model: model, agentConfig: agentConfig); + + var completer = completions.FirstOrDefault(x => x.Provider == provider); + if (completer == null) + { + var logger = services.GetRequiredService>(); + logger.LogError($"Can't resolve completion provider by {provider}"); + } + + completer.SetModelName(model); + + return completer; + } + + public static IImageGeneration GetImageGeneration(IServiceProvider services, + string? provider = null, + string? model = null, + string? modelId = null, + bool imageGenerate = false, + AgentLlmConfig? agentConfig = null) + { + var completions = services.GetServices(); + (provider, model) = GetProviderAndModel(services, provider: provider, model: model, modelId: modelId, + imageGenerate: imageGenerate, agentConfig: agentConfig); + + var completer = completions.FirstOrDefault(x => x.Provider == provider); + if (completer == null) + { + var logger = services.GetRequiredService>(); + logger.LogError($"Can't resolve completion provider by {provider}"); + } + + completer?.SetModelName(model); + + return completer; + } + + private static (string, string) GetProviderAndModel(IServiceProvider services, string? provider = null, string? model = null, string? modelId = null, bool? multiModal = null, + bool imageGenerate = false, AgentLlmConfig? agentConfig = null) { var agentSetting = services.GetRequiredService(); @@ -76,13 +121,14 @@ private static (string, string) GetProviderAndModel(IServiceProvider services, model = agentConfig?.Model ?? agentSetting.LlmConfig?.Model; if (state.ContainsState("model")) { - model = state.GetState("model", model ?? "gpt-35-turbo-4k"); + model = state.GetState("model", model ?? "dall-e-3"); } else if (state.ContainsState("model_id") || !string.IsNullOrEmpty(modelId)) { var modelIdentity = state.ContainsState("model_id") ? state.GetState("model_id") : modelId; var llmProviderService = services.GetRequiredService(); - model = llmProviderService.GetProviderModel(provider, modelIdentity, multiModal: multiModal)?.Name; + model = llmProviderService.GetProviderModel(provider, modelIdentity, + multiModal: multiModal, imageGenerate: imageGenerate)?.Name; } } @@ -91,24 +137,4 @@ private static (string, string) GetProviderAndModel(IServiceProvider services, return (provider, model); } - public static ITextCompletion GetTextCompletion(IServiceProvider services, - string? provider = null, - string? model = null, - AgentLlmConfig? agentConfig = null) - { - var completions = services.GetServices(); - - (provider, model) = GetProviderAndModel(services, provider: provider, model: model, agentConfig: agentConfig); - - var completer = completions.FirstOrDefault(x => x.Provider == provider); - if (completer == null) - { - var logger = services.GetRequiredService>(); - logger.LogError($"Can't resolve completion provider by {provider}"); - } - - completer.SetModelName(model); - - return completer; - } } diff --git a/src/Infrastructure/BotSharp.Core/Infrastructures/LlmProviderService.cs b/src/Infrastructure/BotSharp.Core/Infrastructures/LlmProviderService.cs index 8320bdb73..b3e86e585 100644 --- a/src/Infrastructure/BotSharp.Core/Infrastructures/LlmProviderService.cs +++ b/src/Infrastructure/BotSharp.Core/Infrastructures/LlmProviderService.cs @@ -44,7 +44,7 @@ public List GetProviderModels(string provider) ?.Models ?? new List(); } - public LlmModelSetting GetProviderModel(string provider, string id, bool? multiModal = null) + public LlmModelSetting GetProviderModel(string provider, string id, bool? multiModal = null, bool imageGenerate = false) { var models = GetProviderModels(provider) .Where(x => x.Id == id); @@ -54,6 +54,8 @@ public LlmModelSetting GetProviderModel(string provider, string id, bool? multiM models = models.Where(x => x.MultiModal == multiModal); } + models = models.Where(x => x.ImageGeneration == imageGenerate); + var random = new Random(); var index = random.Next(0, models.Count()); var modelSetting = models.ElementAt(index); diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs index 0b013456e..bcbd1e8ab 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs @@ -99,8 +99,9 @@ public async Task MultiModalCompletion([FromBody] IncomingMessageModel i } catch (Exception ex) { - _logger.LogError($"Error in analyzing files. {ex.Message}"); - return $"Error in analyzing files."; + var error = $"Error in analyzing files. {ex.Message}"; + _logger.LogError(error); + return error; } } @@ -113,8 +114,8 @@ public async Task ImageGeneration([FromBody] IncomingM try { - var completion = CompletionProvider.GetChatCompletion(_services, provider: input.Provider ?? "openai", - modelId: input.ModelId ?? "dall-e"); + var completion = CompletionProvider.GetImageGeneration(_services, provider: input.Provider ?? "openai", + modelId: input.ModelId ?? "dall-e", imageGenerate: true); var message = await completion.GetImageGeneration(new Agent() { Id = Guid.Empty.ToString(), @@ -129,8 +130,8 @@ public async Task ImageGeneration([FromBody] IncomingM } catch (Exception ex) { - var error = "Error in image generation."; - _logger.LogError($"{error} {ex.Message}"); + var error = $"Error in image generation. {ex.Message}"; + _logger.LogError(error); imageViewModel.Message = error; return imageViewModel; } diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/LlmProviderController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/LlmProviderController.cs index 94de179ec..5571d9fa9 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/LlmProviderController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/LlmProviderController.cs @@ -24,6 +24,7 @@ public IEnumerable GetLlmProviders() [HttpGet("/llm-provider/{provider}/models")] public IEnumerable GetLlmProviderModels([FromRoute] string provider) { - return _llmProvider.GetProviderModels(provider); + var list = _llmProvider.GetProviderModels(provider); + return list.Where(x => !x.ImageGeneration); } } diff --git a/src/Plugins/BotSharp.Plugin.AnthropicAI/Providers/ChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.AnthropicAI/Providers/ChatCompletionProvider.cs index 8977e04db..872021098 100644 --- a/src/Plugins/BotSharp.Plugin.AnthropicAI/Providers/ChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.AnthropicAI/Providers/ChatCompletionProvider.cs @@ -264,9 +264,4 @@ public void SetModelName(string model) { _model = model; } - - public Task GetImageGeneration(Agent agent, List conversations) - { - throw new NotImplementedException(); - } } diff --git a/src/Plugins/BotSharp.Plugin.AzureOpenAI/AzureOpenAiPlugin.cs b/src/Plugins/BotSharp.Plugin.AzureOpenAI/AzureOpenAiPlugin.cs index b3decd7b8..fb04f9610 100644 --- a/src/Plugins/BotSharp.Plugin.AzureOpenAI/AzureOpenAiPlugin.cs +++ b/src/Plugins/BotSharp.Plugin.AzureOpenAI/AzureOpenAiPlugin.cs @@ -29,5 +29,7 @@ public void RegisterDI(IServiceCollection services, IConfiguration config) services.AddScoped(); services.AddScoped(); services.AddScoped(); + services.AddScoped(); + services.AddScoped(); } } \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/ChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/ChatCompletionProvider.cs index d85b4d658..50fd33ce5 100644 --- a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/ChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/ChatCompletionProvider.cs @@ -446,62 +446,4 @@ ChatRequestToolMessage GetToolCallResponseMessage(ChatCompletionsToolCall toolCa functionResultData = "31 celsius"; return new ChatRequestToolMessage(functionResultData.ToString(), toolCall.Id); } - - public async Task GetImageGeneration(Agent agent, List conversations) - { - var contentHooks = _services.GetServices().ToList(); - foreach (var hook in contentHooks) - { - await hook.BeforeGenerating(agent, conversations); - } - - var client = ProviderHelper.GetClient(Provider, _model, _services); - var options = BuildImageGenerationOptions(conversations); - var response = await client.GetImageGenerationsAsync(options); - var image = response.Value.Data.First(); - - var content = string.Empty; - if (!string.IsNullOrEmpty(image.RevisedPrompt)) - { - content = image.RevisedPrompt; - } - - var responseMessage = new RoleDialogModel(AgentRole.Assistant, content) - { - CurrentAgentId = agent.Id, - MessageId = conversations.LastOrDefault()?.MessageId ?? string.Empty, - Data = image.Url.AbsoluteUri ?? image.Base64Data - }; - - foreach (var hook in contentHooks) - { - await hook.AfterGenerated(responseMessage, new TokenStatsModel - { - Prompt = options.Prompt, - Provider = Provider, - Model = _model, - PromptCount = options.Prompt.Split(' ', StringSplitOptions.RemoveEmptyEntries).Count(), - CompletionCount = content.Split(' ', StringSplitOptions.RemoveEmptyEntries).Count() - }); - } - - return responseMessage; - } - - private ImageGenerationOptions BuildImageGenerationOptions(List conversations) - { - var state = _services.GetRequiredService(); - - var sizeValue = !string.IsNullOrEmpty(state.GetState("image_size")) ? state.GetState("image_size") : "1024x1024"; - var qualityValue = !string.IsNullOrEmpty(state.GetState("image_quality")) ? state.GetState("image_quality") : "standard"; - - var options = new ImageGenerationOptions - { - DeploymentName = _model, - Prompt = conversations.LastOrDefault()?.Payload ?? conversations.LastOrDefault()?.Content ?? string.Empty, - Size = new ImageSize(sizeValue), - Quality = new ImageGenerationQuality(qualityValue) - }; - return options; - } } diff --git a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/ImageGenerationProvider.cs b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/ImageGenerationProvider.cs new file mode 100644 index 000000000..c0211f116 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/ImageGenerationProvider.cs @@ -0,0 +1,104 @@ +using Azure.AI.OpenAI; +using BotSharp.Abstraction.Agents.Enums; +using BotSharp.Abstraction.Agents.Models; +using BotSharp.Abstraction.Conversations; +using BotSharp.Abstraction.Conversations.Models; +using BotSharp.Abstraction.Loggers; +using BotSharp.Abstraction.MLTasks; +using BotSharp.Plugin.AzureOpenAI.Settings; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; + +namespace BotSharp.Plugin.AzureOpenAI.Providers; + +public class ImageGenerationProvider : IImageGeneration +{ + protected readonly AzureOpenAiSettings _settings; + protected readonly IServiceProvider _services; + protected readonly ILogger _logger; + + protected string _model; + + public virtual string Provider => "azure-openai"; + + public ImageGenerationProvider( + AzureOpenAiSettings settings, + ILogger logger, + IServiceProvider services) + { + _settings = settings; + _services = services; + _logger = logger; + } + + + public async Task GetImageGeneration(Agent agent, List conversations) + { + var contentHooks = _services.GetServices().ToList(); + + // Before + foreach (var hook in contentHooks) + { + await hook.BeforeGenerating(agent, conversations); + } + + var client = ProviderHelper.GetClient(Provider, _model, _services); + var options = PrepareOptions(conversations); + var response = await client.GetImageGenerationsAsync(options); + var image = response.Value.Data.First(); + + var content = string.Empty; + if (!string.IsNullOrEmpty(image.RevisedPrompt)) + { + content = image.RevisedPrompt; + } + + var responseMessage = new RoleDialogModel(AgentRole.Assistant, content) + { + CurrentAgentId = agent.Id, + MessageId = conversations.LastOrDefault()?.MessageId ?? string.Empty, + Data = image.Url.AbsoluteUri ?? image.Base64Data + }; + + // After + foreach (var hook in contentHooks) + { + await hook.AfterGenerated(responseMessage, new TokenStatsModel + { + Prompt = options.Prompt, + Provider = Provider, + Model = _model, + PromptCount = options.Prompt.Split(' ', StringSplitOptions.RemoveEmptyEntries).Count(), + CompletionCount = content.Split(' ', StringSplitOptions.RemoveEmptyEntries).Count() + }); + } + + return responseMessage; + } + + private ImageGenerationOptions PrepareOptions(List conversations) + { + var state = _services.GetRequiredService(); + + var sizeValue = !string.IsNullOrEmpty(state.GetState("image_size")) ? state.GetState("image_size") : "1024x1024"; + var qualityValue = !string.IsNullOrEmpty(state.GetState("image_quality")) ? state.GetState("image_quality") : "standard"; + + var options = new ImageGenerationOptions + { + DeploymentName = _model, + Prompt = conversations.LastOrDefault()?.Payload ?? conversations.LastOrDefault()?.Content ?? string.Empty, + Size = new ImageSize(sizeValue), + Quality = new ImageGenerationQuality(qualityValue) + }; + return options; + } + + public void SetModelName(string model) + { + _model = model; + } +} diff --git a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/OpenAiImageGenerationProvider.cs b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/OpenAiImageGenerationProvider.cs new file mode 100644 index 000000000..5a02b6ae5 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/OpenAiImageGenerationProvider.cs @@ -0,0 +1,16 @@ +using BotSharp.Plugin.AzureOpenAI.Settings; +using Microsoft.Extensions.Logging; +using System; + +namespace BotSharp.Plugin.AzureOpenAI.Providers; + +public class OpenAiImageGenerationProvider : ImageGenerationProvider +{ + public override string Provider => "openai"; + + public OpenAiImageGenerationProvider(AzureOpenAiSettings settings, + ILogger logger, + IServiceProvider services) : base(settings, logger, services) + { + } +} diff --git a/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/ChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/ChatCompletionProvider.cs index c4628c6d0..d278b110b 100644 --- a/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/ChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.GoogleAI/Providers/ChatCompletionProvider.cs @@ -149,9 +149,4 @@ public void SetModelName(string model) { _model = model; } - - public async Task GetImageGeneration(Agent agent, List conversations) - { - throw new NotImplementedException(); - } } diff --git a/src/Plugins/BotSharp.Plugin.HuggingFace/Providers/ChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.HuggingFace/Providers/ChatCompletionProvider.cs index 80c950c8d..88b37dbb7 100644 --- a/src/Plugins/BotSharp.Plugin.HuggingFace/Providers/ChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.HuggingFace/Providers/ChatCompletionProvider.cs @@ -139,9 +139,4 @@ public async Task GetChatCompletions(Agent agent, List GetImageGeneration(Agent agent, List conversations) - { - throw new NotImplementedException(); - } } diff --git a/src/Plugins/BotSharp.Plugin.LLamaSharp/Providers/ChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.LLamaSharp/Providers/ChatCompletionProvider.cs index cdd96a0b5..0db444ce1 100644 --- a/src/Plugins/BotSharp.Plugin.LLamaSharp/Providers/ChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.LLamaSharp/Providers/ChatCompletionProvider.cs @@ -191,9 +191,4 @@ public void SetModelName(string model) { _model = model; } - - public async Task GetImageGeneration(Agent agent, List conversations) - { - throw new NotImplementedException(); - } } diff --git a/src/Plugins/BotSharp.Plugin.MetaGLM/Providers/ChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.MetaGLM/Providers/ChatCompletionProvider.cs index 7010842db..c702a7ecd 100644 --- a/src/Plugins/BotSharp.Plugin.MetaGLM/Providers/ChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.MetaGLM/Providers/ChatCompletionProvider.cs @@ -231,11 +231,6 @@ public Task GetChatCompletionsStreamingAsync(Agent agent, List GetImageGeneration(Agent agent, List conversations) - { - throw new NotImplementedException(); - } - public void SetModelName(string model) { _model = model; diff --git a/src/Plugins/BotSharp.Plugin.SemanticKernel/SemanticKernelChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.SemanticKernel/SemanticKernelChatCompletionProvider.cs index cb70b1896..156f238c5 100644 --- a/src/Plugins/BotSharp.Plugin.SemanticKernel/SemanticKernelChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.SemanticKernel/SemanticKernelChatCompletionProvider.cs @@ -102,10 +102,5 @@ public void SetModelName(string model) { _model = model; } - - public async Task GetImageGeneration(Agent agent, List conversations) - { - throw new NotImplementedException(); - } } } \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.SparkDesk/Providers/ChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.SparkDesk/Providers/ChatCompletionProvider.cs index 25e588c3b..c4556c36c 100644 --- a/src/Plugins/BotSharp.Plugin.SparkDesk/Providers/ChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.SparkDesk/Providers/ChatCompletionProvider.cs @@ -268,9 +268,4 @@ private FunctionDef ConvertToFunctionDef(BotSharp.Abstraction.Functions.Models.F FunctionDef functionDef = new FunctionDef(def.Name, def.Description, fundef.ToArray()); return functionDef; } - - public async Task GetImageGeneration(Agent agent, List conversations) - { - throw new NotImplementedException(); - } } From 734be5d607ee304a7de8f6c688c9d4c9af9bbab1 Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Mon, 24 Jun 2024 16:39:08 -0500 Subject: [PATCH 3/9] refactor load attachment tool --- .../Agents/Enums/AgentType.cs | 2 + .../Agents/Services/AgentService.LoadAgent.cs | 4 +- .../BotSharp.Core/BotSharp.Core.csproj | 16 ++++++ .../Files/Functions/LoadAttachmentFn.cs | 10 ++-- .../Files/Hooks/AttachmentProcessingHook.cs | 56 +++++++++---------- .../agent.json | 13 +++++ .../functions.json | 20 +++++++ .../instruction.liquid | 0 .../templates/load_attachment_prompt.liquid | 1 + 9 files changed, 84 insertions(+), 38 deletions(-) create mode 100644 src/Infrastructure/BotSharp.Core/data/agents/00000000-0000-0000-0000-000000000000/agent.json create mode 100644 src/Infrastructure/BotSharp.Core/data/agents/00000000-0000-0000-0000-000000000000/functions.json create mode 100644 src/Infrastructure/BotSharp.Core/data/agents/00000000-0000-0000-0000-000000000000/instruction.liquid create mode 100644 src/Infrastructure/BotSharp.Core/data/agents/00000000-0000-0000-0000-000000000000/templates/load_attachment_prompt.liquid diff --git a/src/Infrastructure/BotSharp.Abstraction/Agents/Enums/AgentType.cs b/src/Infrastructure/BotSharp.Abstraction/Agents/Enums/AgentType.cs index 176894070..5bddb791d 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Agents/Enums/AgentType.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Agents/Enums/AgentType.cs @@ -18,5 +18,7 @@ public class AgentType /// Agent that cannot use external tools /// public const string Static = "static"; + + public const string Tool = "tool"; } diff --git a/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.LoadAgent.cs b/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.LoadAgent.cs index 6c2d7ea22..e3f1774ef 100644 --- a/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.LoadAgent.cs +++ b/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.LoadAgent.cs @@ -7,7 +7,7 @@ public partial class AgentService [MemoryCache(10 * 60, perInstanceCache: true)] public async Task LoadAgent(string id) { - if (string.IsNullOrEmpty(id) || id == Guid.Empty.ToString()) + if (string.IsNullOrEmpty(id)) { return null; } @@ -28,7 +28,7 @@ public async Task LoadAgent(string id) var agent = await GetAgent(id); if (agent == null) { - throw new Exception($"Can't load agent by id: {id}"); + return null; } if (agent.InheritAgentId != null) diff --git a/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj b/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj index 7bdd84645..e502e7f31 100644 --- a/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj +++ b/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj @@ -46,6 +46,10 @@ + + + + @@ -146,6 +150,18 @@ PreserveNewest + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + + + PreserveNewest + PreserveNewest diff --git a/src/Infrastructure/BotSharp.Core/Files/Functions/LoadAttachmentFn.cs b/src/Infrastructure/BotSharp.Core/Files/Functions/LoadAttachmentFn.cs index a4f041493..ca0e24bfa 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Functions/LoadAttachmentFn.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Functions/LoadAttachmentFn.cs @@ -10,9 +10,9 @@ public class LoadAttachmentFn : IFunctionCallback private readonly IServiceProvider _services; private readonly ILogger _logger; - private const string AIAssistant = "01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a"; private readonly IEnumerable _imageTypes = new List { "image", "images", "png", "jpg", "jpeg" }; private readonly IEnumerable _pdfTypes = new List { "pdf" }; + private static string TOOL_ASSISTANT = Guid.Empty.ToString(); public LoadAttachmentFn( IServiceProvider services, @@ -29,13 +29,13 @@ public async Task Execute(RoleDialogModel message) var agentService = _services.GetRequiredService(); var wholeDialogs = conv.GetDialogHistory(); - var fileTypes = args?.FileTypes?.Split(",")?.ToList() ?? new List(); + var fileTypes = args?.FileTypes?.Split(",", StringSplitOptions.RemoveEmptyEntries)?.ToList() ?? new List(); var dialogs = await AssembleFiles(conv.ConversationId, wholeDialogs, fileTypes); - var agent = await agentService.LoadAgent(!string.IsNullOrEmpty(message.CurrentAgentId) ? message.CurrentAgentId : AIAssistant); + var agent = await agentService.LoadAgent(TOOL_ASSISTANT); var fileAgent = new Agent { - Id = agent.Id, - Name = agent.Name, + Id = agent?.Id ?? Guid.Empty.ToString(), + Name = agent?.Name ?? "Unkown", Instruction = !string.IsNullOrWhiteSpace(args?.UserRequest) ? args.UserRequest : "Please describe the files.", TemplateDict = new Dictionary() }; diff --git a/src/Infrastructure/BotSharp.Core/Files/Hooks/AttachmentProcessingHook.cs b/src/Infrastructure/BotSharp.Core/Files/Hooks/AttachmentProcessingHook.cs index 558657b83..50088679a 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Hooks/AttachmentProcessingHook.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Hooks/AttachmentProcessingHook.cs @@ -1,11 +1,11 @@ - -using Microsoft.EntityFrameworkCore; +using System.Text.RegularExpressions; namespace BotSharp.Core.Files.Hooks; public class AttachmentProcessingHook : AgentHookBase { private readonly IServiceProvider _services; + private static string TOOL_ASSISTANT = Guid.Empty.ToString(); public override string SelfId => string.Empty; @@ -23,41 +23,35 @@ public override void OnAgentLoaded(Agent agent) if (hasConvFiles) { - agent.Instruction += "\r\n\r\nPlease call load_attachment if user wants to describe files, such as images, pdf.\r\n\r\n"; - - if (agent.Functions != null) + var (prompt, loadAttachmentFn) = GetLoadAttachmentFn(); + if (loadAttachmentFn != null) { - var json = JsonSerializer.Serialize(new + if (!string.IsNullOrWhiteSpace(prompt)) + { + agent.Instruction += $"\r\n\r\n{prompt}\r\n\r\n"; + } + + if (agent.Functions == null) { - user_request = new - { - type = "string", - description = "The request posted by user, which is related to analyzing requested files. User can request for multiple files to process at one time." - }, - file_types = new - { - type = "string", - description = "The file types requested by user to analyze, such as image, png, jpeg, and pdf. There can be multiple file types in a single request. An example output is, 'image,pdf'" - } - }); - - agent.Functions.Add(new FunctionDef + agent.Functions = new List { loadAttachmentFn }; + } + else { - Name = "load_attachment", - Description = "If the user's request is related to analyzing files and/or images, you can call this function to analyze files and images.", - Parameters = - { - Properties = JsonSerializer.Deserialize(json), - Required = new List - { - "user_request", - "file_types" - } - } - }); + agent.Functions.Add(loadAttachmentFn); + } } } base.OnAgentLoaded(agent); } + + private (string, FunctionDef?) GetLoadAttachmentFn() + { + var fnName = "load_attachment"; + var db = _services.GetRequiredService(); + var agent = db.GetAgent(TOOL_ASSISTANT); + var prompt = agent?.Templates?.FirstOrDefault(x => x.Name.IsEqualTo($"{fnName}_prompt"))?.Content ?? string.Empty; + var loadAttachmentFn = agent?.Functions?.FirstOrDefault(x => x.Name.IsEqualTo(fnName)); + return (prompt, loadAttachmentFn); + } } diff --git a/src/Infrastructure/BotSharp.Core/data/agents/00000000-0000-0000-0000-000000000000/agent.json b/src/Infrastructure/BotSharp.Core/data/agents/00000000-0000-0000-0000-000000000000/agent.json new file mode 100644 index 000000000..31a9590ad --- /dev/null +++ b/src/Infrastructure/BotSharp.Core/data/agents/00000000-0000-0000-0000-000000000000/agent.json @@ -0,0 +1,13 @@ +{ + "id": "00000000-0000-0000-0000-000000000000", + "name": "Tool Assistant", + "description": "Tool assistant that can be used to complete many different tasks", + "type": "tool", + "createdDateTime": "2023-06-24T10:39:32.2349685Z", + "updatedDateTime": "2023-06-24T14:39:32.2349686Z", + "iconUrl": "https://cdn.iconscout.com/icon/premium/png-256-thumb/route-1613278-1368497.png", + "disabled": false, + "isPublic": false, + "profiles": [ "tool" ], + "routingRules": [] +} \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.Core/data/agents/00000000-0000-0000-0000-000000000000/functions.json b/src/Infrastructure/BotSharp.Core/data/agents/00000000-0000-0000-0000-000000000000/functions.json new file mode 100644 index 000000000..75b0b53e6 --- /dev/null +++ b/src/Infrastructure/BotSharp.Core/data/agents/00000000-0000-0000-0000-000000000000/functions.json @@ -0,0 +1,20 @@ +[ + { + "name": "load_attachment", + "description": "If the user's request is related to analyzing files and/or images, you can call this function to analyze files and images.", + "parameters": { + "type": "object", + "properties": { + "user_request": { + "type": "string", + "description": "The request posted by user, which is related to analyzing requested files. User can request for multiple files to process at one time." + }, + "file_types": { + "type": "string", + "description": "The file types requested by user to analyze, such as image, png, jpeg, and pdf. There can be multiple file types in a single request. An example output is, 'image,pdf'." + } + }, + "required": [ "user_request", "file_types" ] + } + } +] \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.Core/data/agents/00000000-0000-0000-0000-000000000000/instruction.liquid b/src/Infrastructure/BotSharp.Core/data/agents/00000000-0000-0000-0000-000000000000/instruction.liquid new file mode 100644 index 000000000..e69de29bb diff --git a/src/Infrastructure/BotSharp.Core/data/agents/00000000-0000-0000-0000-000000000000/templates/load_attachment_prompt.liquid b/src/Infrastructure/BotSharp.Core/data/agents/00000000-0000-0000-0000-000000000000/templates/load_attachment_prompt.liquid new file mode 100644 index 000000000..1e1f0ab88 --- /dev/null +++ b/src/Infrastructure/BotSharp.Core/data/agents/00000000-0000-0000-0000-000000000000/templates/load_attachment_prompt.liquid @@ -0,0 +1 @@ +Please call load_attachment if user wants to describe files, such as images, pdf. \ No newline at end of file From dc6888c0012f4a91370b25c033b839680d48b511 Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Mon, 24 Jun 2024 17:21:41 -0500 Subject: [PATCH 4/9] minor change --- .../BotSharp.Core/Files/Hooks/AttachmentProcessingHook.cs | 2 -- src/Plugins/BotSharp.Plugin.ChatHub/WebSocketsMiddleware.cs | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/src/Infrastructure/BotSharp.Core/Files/Hooks/AttachmentProcessingHook.cs b/src/Infrastructure/BotSharp.Core/Files/Hooks/AttachmentProcessingHook.cs index 50088679a..4b70fd733 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Hooks/AttachmentProcessingHook.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Hooks/AttachmentProcessingHook.cs @@ -4,7 +4,6 @@ namespace BotSharp.Core.Files.Hooks; public class AttachmentProcessingHook : AgentHookBase { - private readonly IServiceProvider _services; private static string TOOL_ASSISTANT = Guid.Empty.ToString(); public override string SelfId => string.Empty; @@ -12,7 +11,6 @@ public class AttachmentProcessingHook : AgentHookBase public AttachmentProcessingHook(IServiceProvider services, AgentSettings settings) : base(services, settings) { - _services = services; } public override void OnAgentLoaded(Agent agent) diff --git a/src/Plugins/BotSharp.Plugin.ChatHub/WebSocketsMiddleware.cs b/src/Plugins/BotSharp.Plugin.ChatHub/WebSocketsMiddleware.cs index 773708930..fb65f150a 100644 --- a/src/Plugins/BotSharp.Plugin.ChatHub/WebSocketsMiddleware.cs +++ b/src/Plugins/BotSharp.Plugin.ChatHub/WebSocketsMiddleware.cs @@ -36,7 +36,7 @@ private bool VerifyGetRequest(HttpRequest request) { var regexes = new List { - new Regex(@"/conversation/[a-z0-9-]+/message/[a-z0-9-]+/[a-z]+/file/[a-z0-9-]+/[a-z0-9-]+", RegexOptions.IgnoreCase), + new Regex(@"/conversation/(.*?)/message/(.*?)/(.*?)/file/(.*?)/(.*?)", RegexOptions.IgnoreCase), new Regex(@"/user/avatar", RegexOptions.IgnoreCase) }; From dbf103fd47b5d81e0b80ceb08392d30bf1ee2e54 Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Mon, 24 Jun 2024 17:34:05 -0500 Subject: [PATCH 5/9] change to static --- .../BotSharp.Abstraction/Agents/Enums/AgentType.cs | 2 -- .../data/agents/00000000-0000-0000-0000-000000000000/agent.json | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/src/Infrastructure/BotSharp.Abstraction/Agents/Enums/AgentType.cs b/src/Infrastructure/BotSharp.Abstraction/Agents/Enums/AgentType.cs index 5bddb791d..176894070 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Agents/Enums/AgentType.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Agents/Enums/AgentType.cs @@ -18,7 +18,5 @@ public class AgentType /// Agent that cannot use external tools /// public const string Static = "static"; - - public const string Tool = "tool"; } diff --git a/src/Infrastructure/BotSharp.Core/data/agents/00000000-0000-0000-0000-000000000000/agent.json b/src/Infrastructure/BotSharp.Core/data/agents/00000000-0000-0000-0000-000000000000/agent.json index 31a9590ad..ef0ce2228 100644 --- a/src/Infrastructure/BotSharp.Core/data/agents/00000000-0000-0000-0000-000000000000/agent.json +++ b/src/Infrastructure/BotSharp.Core/data/agents/00000000-0000-0000-0000-000000000000/agent.json @@ -2,7 +2,7 @@ "id": "00000000-0000-0000-0000-000000000000", "name": "Tool Assistant", "description": "Tool assistant that can be used to complete many different tasks", - "type": "tool", + "type": "static", "createdDateTime": "2023-06-24T10:39:32.2349685Z", "updatedDateTime": "2023-06-24T14:39:32.2349686Z", "iconUrl": "https://cdn.iconscout.com/icon/premium/png-256-thumb/route-1613278-1368497.png", From 02b20a07323c5e0528145f2b4678ea75c8bcecfa Mon Sep 17 00:00:00 2001 From: Jicheng Lu Date: Mon, 24 Jun 2024 18:41:23 -0500 Subject: [PATCH 6/9] change name --- .../BotSharp.OpenAPI/Controllers/InstructModeController.cs | 2 +- .../ViewModels/Instructs/ImageGenerationViewModel.cs | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs index bcbd1e8ab..700d40153 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs @@ -124,7 +124,7 @@ public async Task ImageGeneration([FromBody] IncomingM new RoleDialogModel(AgentRole.User, input.Text) }); - imageViewModel.Content = message.Content; + imageViewModel.RevisedPrompt = message.Content; imageViewModel.Data = message.Data; return imageViewModel; } diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/ImageGenerationViewModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/ImageGenerationViewModel.cs index 179c05f40..1dc3872d3 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/ImageGenerationViewModel.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/ImageGenerationViewModel.cs @@ -4,9 +4,9 @@ namespace BotSharp.OpenAPI.ViewModels.Instructs; public class ImageGenerationViewModel { - [JsonPropertyName("content")] + [JsonPropertyName("revised_prompt")] [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] - public string? Content { get; set; } + public string? RevisedPrompt { get; set; } [JsonPropertyName("data")] [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] From f65eeddb8e31967849523d185ace19b8138f2ba1 Mon Sep 17 00:00:00 2001 From: Jicheng Lu Date: Mon, 24 Jun 2024 22:46:13 -0500 Subject: [PATCH 7/9] add agent tools --- .../Agents/Enums/AgentField.cs | 3 ++- .../Agents/Enums/AgentTool.cs | 8 ++++++++ .../Agents/Models/Agent.cs | 13 +++++++++++++ .../Conversations/IConversationService.cs | 2 ++ .../Services/AgentService.UpdateAgent.cs | 4 ++-- .../Services/ConversationService.cs | 5 +++++ .../Files/Hooks/AttachmentProcessingHook.cs | 8 +++----- .../FileRepository/FileRepository.Agent.cs | 17 +++++++++++++++++ .../ViewModels/Agents/AgentCreationModel.cs | 2 ++ .../ViewModels/Agents/AgentUpdateModel.cs | 6 ++++++ .../ViewModels/Agents/AgentViewModel.cs | 2 ++ .../Collections/AgentDocument.cs | 1 + .../Repository/MongoRepository.Agent.cs | 18 ++++++++++++++++++ .../Repository/MongoRepository.Transaction.cs | 2 ++ 14 files changed, 83 insertions(+), 8 deletions(-) create mode 100644 src/Infrastructure/BotSharp.Abstraction/Agents/Enums/AgentTool.cs diff --git a/src/Infrastructure/BotSharp.Abstraction/Agents/Enums/AgentField.cs b/src/Infrastructure/BotSharp.Abstraction/Agents/Enums/AgentField.cs index 61cb0f4dc..bbcc51ab5 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Agents/Enums/AgentField.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Agents/Enums/AgentField.cs @@ -16,7 +16,8 @@ public enum AgentField Template, Response, Sample, - LlmConfig + LlmConfig, + Tool } public enum AgentTaskField diff --git a/src/Infrastructure/BotSharp.Abstraction/Agents/Enums/AgentTool.cs b/src/Infrastructure/BotSharp.Abstraction/Agents/Enums/AgentTool.cs new file mode 100644 index 000000000..875062f85 --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Agents/Enums/AgentTool.cs @@ -0,0 +1,8 @@ +namespace BotSharp.Abstraction.Agents.Enums; + +public class AgentTool +{ + public const string FileAnalyzer = "file-analyzer"; + public const string ImageGenerator = "image-generator"; + public const string HttpHandler = "http-handler"; +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Agents/Models/Agent.cs b/src/Infrastructure/BotSharp.Abstraction/Agents/Models/Agent.cs index 7b48f452f..c1f82e7bd 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Agents/Models/Agent.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Agents/Models/Agent.cs @@ -90,6 +90,12 @@ public class Agent public List Profiles { get; set; } = new List(); + /// + /// Useful tools + /// + public List Tools { get; set; } + = new List(); + /// /// Inherit from agent /// @@ -121,6 +127,7 @@ public static Agent Clone(Agent agent) Functions = agent.Functions, Responses = agent.Responses, Samples = agent.Samples, + Tools = agent.Tools, Knowledges = agent.Knowledges, IsPublic = agent.IsPublic, Disabled = agent.Disabled, @@ -162,6 +169,12 @@ public Agent SetSamples(List samples) return this; } + public Agent SetTools(List tools) + { + Tools = tools ?? new List(); + return this; + } + public Agent SetResponses(List responses) { Responses = responses ?? new List(); ; diff --git a/src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationService.cs b/src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationService.cs index 8434359ed..9fd40c099 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationService.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Conversations/IConversationService.cs @@ -58,4 +58,6 @@ Task SendMessage(string agentId, Task GetConversationSummary(IEnumerable conversationId); Task GetConversationRecordOrCreateNew(string agentId); + + bool IsConversationMode(); } diff --git a/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.UpdateAgent.cs b/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.UpdateAgent.cs index daab89165..ecf2ecfae 100644 --- a/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.UpdateAgent.cs +++ b/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.UpdateAgent.cs @@ -1,8 +1,6 @@ -using BotSharp.Abstraction.Agents; using BotSharp.Abstraction.Repositories.Enums; using BotSharp.Abstraction.Routing.Models; using BotSharp.Abstraction.Users.Enums; -using Microsoft.EntityFrameworkCore.Metadata; using System.IO; namespace BotSharp.Core.Agents.Services; @@ -34,6 +32,7 @@ public async Task UpdateAgent(Agent agent, AgentField updateField) record.Templates = agent.Templates ?? new List(); record.Responses = agent.Responses ?? new List(); record.Samples = agent.Samples ?? new List(); + record.Tools = agent.Tools ?? new List(); if (agent.LlmConfig != null && !agent.LlmConfig.IsInherit) { record.LlmConfig = agent.LlmConfig; @@ -95,6 +94,7 @@ public async Task UpdateAgentFromFile(string id) .SetFunctions(foundAgent.Functions) .SetResponses(foundAgent.Responses) .SetSamples(foundAgent.Samples) + .SetTools(foundAgent.Tools) .SetLlmConfig(foundAgent.LlmConfig); _db.UpdateAgent(clonedAgent, AgentField.All); diff --git a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.cs b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.cs index 5142133aa..c810790e8 100644 --- a/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.cs +++ b/src/Infrastructure/BotSharp.Core/Conversations/Services/ConversationService.cs @@ -161,4 +161,9 @@ public async Task GetConversationRecordOrCreateNew(string agentId) return converation; } + + public bool IsConversationMode() + { + return !string.IsNullOrWhiteSpace(_conversationId); + } } diff --git a/src/Infrastructure/BotSharp.Core/Files/Hooks/AttachmentProcessingHook.cs b/src/Infrastructure/BotSharp.Core/Files/Hooks/AttachmentProcessingHook.cs index 4b70fd733..e114ceb6d 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Hooks/AttachmentProcessingHook.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Hooks/AttachmentProcessingHook.cs @@ -1,5 +1,3 @@ -using System.Text.RegularExpressions; - namespace BotSharp.Core.Files.Hooks; public class AttachmentProcessingHook : AgentHookBase @@ -15,11 +13,11 @@ public AttachmentProcessingHook(IServiceProvider services, AgentSettings setting public override void OnAgentLoaded(Agent agent) { - var fileService = _services.GetRequiredService(); var conv = _services.GetRequiredService(); - var hasConvFiles = fileService.HasConversationUserFiles(conv.ConversationId); + var isConvMode = conv.IsConversationMode(); + var isEnabled = !agent.Tools.IsNullOrEmpty() && agent.Tools.Contains(AgentTool.FileAnalyzer); - if (hasConvFiles) + if (isConvMode && isEnabled) { var (prompt, loadAttachmentFn) = GetLoadAttachmentFn(); if (loadAttachmentFn != null) diff --git a/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Agent.cs b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Agent.cs index 3488e0e75..89d5081d5 100644 --- a/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Agent.cs +++ b/src/Infrastructure/BotSharp.Core/Repository/FileRepository/FileRepository.Agent.cs @@ -53,6 +53,9 @@ public void UpdateAgent(Agent agent, AgentField field) case AgentField.LlmConfig: UpdateAgentLlmConfig(agent.Id, agent.LlmConfig); break; + case AgentField.Tool: + UpdateAgentTools(agent.Id, agent.Tools); + break; case AgentField.All: UpdateAgentAllFields(agent); break; @@ -145,6 +148,19 @@ private void UpdateAgentProfiles(string agentId, List profiles) File.WriteAllText(agentFile, json); } + private void UpdateAgentTools(string agentId, List tools) + { + if (tools == null) return; + + var (agent, agentFile) = GetAgentFromFile(agentId); + if (agent == null) return; + + agent.Tools = tools; + agent.UpdatedDateTime = DateTime.UtcNow; + var json = JsonSerializer.Serialize(agent, _options); + File.WriteAllText(agentFile, json); + } + private void UpdateAgentRoutingRules(string agentId, List rules) { if (rules == null) return; @@ -271,6 +287,7 @@ private void UpdateAgentAllFields(Agent inputAgent) agent.Disabled = inputAgent.Disabled; agent.Type = inputAgent.Type; agent.Profiles = inputAgent.Profiles; + agent.Tools = inputAgent.Tools; agent.RoutingRules = inputAgent.RoutingRules; agent.LlmConfig = inputAgent.LlmConfig; agent.UpdatedDateTime = DateTime.UtcNow; diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Agents/AgentCreationModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Agents/AgentCreationModel.cs index b5f8ea826..4078fc034 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Agents/AgentCreationModel.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Agents/AgentCreationModel.cs @@ -43,6 +43,7 @@ public class AgentCreationModel /// Combine different Agents together to form a Profile. /// public List Profiles { get; set; } = new List(); + public List Tools { get; set; } = new List(); public List RoutingRules { get; set; } = new List(); public AgentLlmConfig? LlmConfig { get; set; } @@ -57,6 +58,7 @@ public Agent ToAgent() Functions = Functions, Responses = Responses, Samples = Samples, + Tools = Tools, IsPublic = IsPublic, Type = Type, Disabled = Disabled, diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Agents/AgentUpdateModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Agents/AgentUpdateModel.cs index 0a461ea60..ba331f85d 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Agents/AgentUpdateModel.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Agents/AgentUpdateModel.cs @@ -25,6 +25,11 @@ public class AgentUpdateModel /// public List? Samples { get; set; } + /// + /// Tools + /// + public List? Tools { get; set; } + /// /// Functions /// @@ -71,6 +76,7 @@ public Agent ToAgent() Templates = Templates ?? new List(), Functions = Functions ?? new List(), Responses = Responses ?? new List(), + Tools = Tools ?? new List(), LlmConfig = LlmConfig }; diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Agents/AgentViewModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Agents/AgentViewModel.cs index 9d87a3e2a..5faa1b98c 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Agents/AgentViewModel.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Agents/AgentViewModel.cs @@ -17,6 +17,7 @@ public class AgentViewModel public List Functions { get; set; } public List Responses { get; set; } public List Samples { get; set; } + public List Tools { get; set; } [JsonPropertyName("is_public")] public bool IsPublic { get; set; } @@ -63,6 +64,7 @@ public static AgentViewModel FromAgent(Agent agent) Functions = agent.Functions, Responses = agent.Responses, Samples = agent.Samples, + Tools = agent.Tools, IsPublic= agent.IsPublic, Disabled = agent.Disabled, IconUrl = agent.IconUrl, diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/AgentDocument.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/AgentDocument.cs index 8e2f4dd44..8e4395a19 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/AgentDocument.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Collections/AgentDocument.cs @@ -14,6 +14,7 @@ public class AgentDocument : MongoBase public List Functions { get; set; } public List Responses { get; set; } public List Samples { get; set; } + public List Tools { get; set; } public bool IsPublic { get; set; } public bool Disabled { get; set; } public List Profiles { get; set; } diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Agent.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Agent.cs index 97ceee9f5..abdfb21b4 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Agent.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Agent.cs @@ -57,6 +57,9 @@ public void UpdateAgent(Agent agent, AgentField field) case AgentField.LlmConfig: UpdateAgentLlmConfig(agent.Id, agent.LlmConfig); break; + case AgentField.Tool: + UpdateAgentTools(agent.Id, agent.Tools); + break; case AgentField.All: UpdateAgentAllFields(agent); break; @@ -218,6 +221,18 @@ private void UpdateAgentSamples(string agentId, List samples) _dc.Agents.UpdateOne(filter, update); } + private void UpdateAgentTools(string agentId, List tools) + { + if (tools == null) return; + + var filter = Builders.Filter.Eq(x => x.Id, agentId); + var update = Builders.Update + .Set(x => x.Tools, tools) + .Set(x => x.UpdatedTime, DateTime.UtcNow); + + _dc.Agents.UpdateOne(filter, update); + } + private void UpdateAgentLlmConfig(string agentId, AgentLlmConfig? config) { var llmConfig = AgentLlmConfigMongoElement.ToMongoElement(config); @@ -244,6 +259,7 @@ private void UpdateAgentAllFields(Agent agent) .Set(x => x.Functions, agent.Functions.Select(f => FunctionDefMongoElement.ToMongoElement(f)).ToList()) .Set(x => x.Responses, agent.Responses.Select(r => AgentResponseMongoElement.ToMongoElement(r)).ToList()) .Set(x => x.Samples, agent.Samples) + .Set(x => x.Tools, agent.Tools) .Set(x => x.LlmConfig, AgentLlmConfigMongoElement.ToMongoElement(agent.LlmConfig)) .Set(x => x.IsPublic, agent.IsPublic) .Set(x => x.UpdatedTime, DateTime.UtcNow); @@ -369,6 +385,7 @@ public void BulkInsertAgents(List agents) .Select(r => AgentResponseMongoElement.ToMongoElement(r))? .ToList() ?? new List(), Samples = x.Samples ?? new List(), + Tools = x.Tools ?? new List(), IsPublic = x.IsPublic, Type = x.Type, InheritAgentId = x.InheritAgentId, @@ -458,6 +475,7 @@ private Agent TransformAgentDocument(AgentDocument? agentDoc) .Select(r => AgentResponseMongoElement.ToDomainElement(r)) .ToList() : new List(), Samples = agentDoc.Samples ?? new List(), + Tools = agentDoc.Tools ?? new List(), IsPublic = agentDoc.IsPublic, Disabled = agentDoc.Disabled, Type = agentDoc.Type, diff --git a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Transaction.cs b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Transaction.cs index 470b1fb19..923cf5fd6 100644 --- a/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Transaction.cs +++ b/src/Plugins/BotSharp.Plugin.MongoStorage/Repository/MongoRepository.Transaction.cs @@ -53,6 +53,7 @@ public int Transaction(Action action) .Select(r => AgentResponseMongoElement.ToMongoElement(r))? .ToList() ?? new List(), Samples = x.Samples ?? new List(), + Tools = x.Tools ?? new List(), IsPublic = x.IsPublic, Type = x.Type, InheritAgentId = x.InheritAgentId, @@ -77,6 +78,7 @@ public int Transaction(Action action) .Set(x => x.Functions, agent.Functions) .Set(x => x.Responses, agent.Responses) .Set(x => x.Samples, agent.Samples) + .Set(x => x.Tools, agent.Tools) .Set(x => x.IsPublic, agent.IsPublic) .Set(x => x.Type, agent.Type) .Set(x => x.InheritAgentId, agent.InheritAgentId) From 8d798a43993e065c66e2f1f5131fd09ce23f2e8a Mon Sep 17 00:00:00 2001 From: Jicheng Lu Date: Mon, 24 Jun 2024 23:34:30 -0500 Subject: [PATCH 8/9] add agent tools api --- .../BotSharp.Abstraction/Agents/IAgentService.cs | 2 ++ .../BotSharp.Core/Agents/Services/AgentService.cs | 11 +++++++++++ .../BotSharp.OpenAPI/Controllers/AgentController.cs | 6 ++++++ 3 files changed, 19 insertions(+) diff --git a/src/Infrastructure/BotSharp.Abstraction/Agents/IAgentService.cs b/src/Infrastructure/BotSharp.Abstraction/Agents/IAgentService.cs index b435a4d18..c2368cd26 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Agents/IAgentService.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Agents/IAgentService.cs @@ -51,4 +51,6 @@ public interface IAgentService List GetAgentsByUser(string userId); PluginDef GetPlugin(string agentId); + + IEnumerable GetAgentTools(); } diff --git a/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.cs b/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.cs index da1b8cf1f..69e67c300 100644 --- a/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.cs +++ b/src/Infrastructure/BotSharp.Core/Agents/Services/AgentService.cs @@ -1,4 +1,5 @@ using System.IO; +using System.Reflection; namespace BotSharp.Core.Agents.Services; @@ -53,4 +54,14 @@ public List GetAgentsByUser(string userId) var agents = _db.GetAgentsByUser(userId); return agents; } + + public IEnumerable GetAgentTools() + { + var tools = typeof(AgentTool).GetFields(BindingFlags.Public | BindingFlags.Static) + .Where(f => f.IsLiteral && f.FieldType == typeof(string)) + .Select(x => x.GetRawConstantValue()?.ToString()) + .ToList(); + + return tools; + } } diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/AgentController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/AgentController.cs index a7b5652df..5d34ec77f 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/AgentController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/AgentController.cs @@ -140,4 +140,10 @@ public async Task DeleteAgent([FromRoute] string agentId) { return await _agentService.DeleteAgent(agentId); } + + [HttpGet("/agent/tools")] + public IEnumerable GetAgentTools() + { + return _agentService.GetAgentTools(); + } } \ No newline at end of file From fa9e0bffe8696f27df20d1a08645c302326ade42 Mon Sep 17 00:00:00 2001 From: Jicheng Lu Date: Mon, 24 Jun 2024 23:38:24 -0500 Subject: [PATCH 9/9] minor change --- .../00000000-0000-0000-0000-000000000000/instruction.liquid | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Infrastructure/BotSharp.Core/data/agents/00000000-0000-0000-0000-000000000000/instruction.liquid b/src/Infrastructure/BotSharp.Core/data/agents/00000000-0000-0000-0000-000000000000/instruction.liquid index e69de29bb..941957e34 100644 --- a/src/Infrastructure/BotSharp.Core/data/agents/00000000-0000-0000-0000-000000000000/instruction.liquid +++ b/src/Infrastructure/BotSharp.Core/data/agents/00000000-0000-0000-0000-000000000000/instruction.liquid @@ -0,0 +1 @@ +You are a tool agent. \ No newline at end of file