From 96b891897ca28ad4c5eeb4ae6783b7a63cf75211 Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Mon, 26 Aug 2024 17:24:07 -0500 Subject: [PATCH 01/27] add instruct api and clean code --- .../Files/IFileInstructService.cs | 6 +- .../MLTasks/ISpeechToText.cs | 6 +- .../MLTasks/Settings/LlmModelSetting.cs | 3 +- .../Instruct/FileInstructService.Audio.cs | 19 ++++ .../Instruct/FileInstructService.Image.cs | 4 +- .../Instruct/FileInstructService.Pdf.cs | 6 +- .../Services/Instruct/FileInstructService.cs | 1 + .../Controllers/InstructModeController.cs | 39 +++++++- .../Instructs/AudioCompletionViewModel.cs | 5 + .../Instructs/ImageGenerationViewModel.cs | 9 +- .../Instructs/InstructBaseViewModel.cs | 13 +++ .../Instructs/InstructMessageModel.cs | 2 - .../Instructs/PdfCompletionViewModel.cs | 10 +- .../AudioHandlerPlugin.cs | 36 +++---- .../Controllers/AudioController.cs | 15 +-- .../Enums/AudioType.cs | 39 +++----- .../Enums/UtilityName.cs | 13 +-- .../Functions/HandleAudioRequestFn.cs | 59 ++++++------ .../Functions/IAudioProcessUtilities.cs | 10 -- .../AudioHelper.cs} | 76 ++++++++------- .../Helpers/IAudioHelper.cs | 6 ++ .../Hooks/AudioHandlerHook.cs | 96 +++++++++---------- .../Hooks/AudioHandlerUtilityHook.cs | 7 -- .../LlmContexts/LlmContextIn.cs | 5 - .../LlmContexts/LlmContextOut.cs | 5 - .../Models/AudioOutput.cs | 20 ++-- .../Provider/NativeWhisperProvider.cs | 72 +++++++------- .../Settings/AudioHandlerSettings.cs | 7 +- .../BotSharp.Plugin.AudioHandler/Using.cs | 2 +- src/Plugins/BotSharp.Plugin.Graph/GraphDb.cs | 2 +- .../Providers/Audio/SpeechToTextProvider.cs | 48 +++++----- .../Providers/Audio/TextToSpeechProvider.cs | 17 ++-- 32 files changed, 332 insertions(+), 326 deletions(-) create mode 100644 src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Audio.cs create mode 100644 src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/AudioCompletionViewModel.cs create mode 100644 src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/InstructBaseViewModel.cs delete mode 100644 src/Plugins/BotSharp.Plugin.AudioHandler/Functions/IAudioProcessUtilities.cs rename src/Plugins/BotSharp.Plugin.AudioHandler/{Functions/AudioProcessUtilities.cs => Helpers/AudioHelper.cs} (59%) create mode 100644 src/Plugins/BotSharp.Plugin.AudioHandler/Helpers/IAudioHelper.cs diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/IFileInstructService.cs b/src/Infrastructure/BotSharp.Abstraction/Files/IFileInstructService.cs index 78a400e1d..e9ccb4ba2 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Files/IFileInstructService.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Files/IFileInstructService.cs @@ -3,7 +3,7 @@ namespace BotSharp.Abstraction.Files; public interface IFileInstructService { #region Image - Task ReadImages(string? provider, string? model, string text, IEnumerable images); + Task ReadImages(string? provider, string? model, string text, IEnumerable images); Task GenerateImage(string? provider, string? model, string text); Task VaryImage(string? provider, string? model, BotSharpFile image); Task EditImage(string? provider, string? model, string text, BotSharpFile image); @@ -20,6 +20,10 @@ public interface IFileInstructService Task ReadPdf(string? provider, string? model, string? modelId, string prompt, List files); #endregion + #region Audio + Task ReadAudio(string? provider, string? model, BotSharpFile audio); + #endregion + #region Select file Task> SelectMessageFiles(string conversationId, SelectFileOptions options); #endregion diff --git a/src/Infrastructure/BotSharp.Abstraction/MLTasks/ISpeechToText.cs b/src/Infrastructure/BotSharp.Abstraction/MLTasks/ISpeechToText.cs index 9e0dd574e..a1af443e0 100644 --- a/src/Infrastructure/BotSharp.Abstraction/MLTasks/ISpeechToText.cs +++ b/src/Infrastructure/BotSharp.Abstraction/MLTasks/ISpeechToText.cs @@ -1,3 +1,5 @@ +using System.IO; + namespace BotSharp.Abstraction.MLTasks; public interface ISpeechToText @@ -5,6 +7,6 @@ public interface ISpeechToText string Provider { get; } Task GenerateTextFromAudioAsync(string filePath); - // Task AudioToTextTranscript(Stream stream); - Task SetModelName(string modelType); + Task GenerateTextFromAudioAsync(Stream audio, string audioFileName); + Task SetModelName(string model); } diff --git a/src/Infrastructure/BotSharp.Abstraction/MLTasks/Settings/LlmModelSetting.cs b/src/Infrastructure/BotSharp.Abstraction/MLTasks/Settings/LlmModelSetting.cs index cc7cfba07..008f528bc 100644 --- a/src/Infrastructure/BotSharp.Abstraction/MLTasks/Settings/LlmModelSetting.cs +++ b/src/Infrastructure/BotSharp.Abstraction/MLTasks/Settings/LlmModelSetting.cs @@ -68,5 +68,6 @@ public enum LlmModelType Text = 1, Chat = 2, Image = 3, - Embedding = 4 + Embedding = 4, + Audio = 5 } diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Audio.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Audio.cs new file mode 100644 index 000000000..f3f40f2c9 --- /dev/null +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Audio.cs @@ -0,0 +1,19 @@ +using System.IO; + +namespace BotSharp.Core.Files.Services; + +public partial class FileInstructService +{ + public async Task ReadAudio(string? provider, string? model, BotSharpFile audio) + { + var completion = CompletionProvider.GetSpeechToText(_services, provider: provider ?? "openai", model: model ?? "whisper-1"); + var audioBytes = await DownloadFile(audio); + using var stream = new MemoryStream(); + stream.Write(audioBytes, 0, audioBytes.Length); + stream.Position = 0; + + var content = await completion.GenerateTextFromAudioAsync(stream, audio.FileName ?? string.Empty); + stream.Close(); + return content; + } +} diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Image.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Image.cs index c9d35cb7e..244b5ac4e 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Image.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Image.cs @@ -4,7 +4,7 @@ namespace BotSharp.Core.Files.Services; public partial class FileInstructService { - public async Task ReadImages(string? provider, string? model, string text, IEnumerable images) + public async Task ReadImages(string? provider, string? model, string text, IEnumerable images) { var completion = CompletionProvider.GetChatCompletion(_services, provider: provider ?? "openai", model: model ?? "gpt-4o", multiModal: true); var message = await completion.GetChatCompletions(new Agent() @@ -17,7 +17,7 @@ public async Task ReadImages(string? provider, string? model, s Files = images?.ToList() ?? new List() } }); - return message; + return message.Content; } public async Task GenerateImage(string? provider, string? model, string text) diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Pdf.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Pdf.cs index 2aec257b4..dd504fbf6 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Pdf.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Pdf.cs @@ -80,9 +80,9 @@ private async Task> DownloadFiles(string dir, List MultiModalCompletion([FromBody] IncomingMessageModel i try { var fileInstruct = _services.GetRequiredService(); - var message = await fileInstruct.ReadImages(input.Provider, input.Model, input.Text, input.Files); - return message.Content; + var content = await fileInstruct.ReadImages(input.Provider, input.Model, input.Text, input.Files); + return content; } catch (Exception ex) { @@ -169,7 +170,7 @@ public async Task ImageEdit([FromBody] IncomingMessage var image = input.Files.FirstOrDefault(x => !string.IsNullOrWhiteSpace(x.FileUrl) || !string.IsNullOrWhiteSpace(x.FileData)); if (image == null) { - return new ImageGenerationViewModel { Message = "Error! Cannot find an image!" }; + return new ImageGenerationViewModel { Message = "Error! Cannot find a valid image file!" }; } var message = await fileInstruct.EditImage(input.Provider, input.Model, input.Text, image); imageViewModel.Content = message.Content; @@ -199,7 +200,7 @@ public async Task ImageMaskEdit([FromBody] IncomingMes var mask = input.Mask; if (image == null || mask == null) { - return new ImageGenerationViewModel { Message = "Error! Cannot find an image or mask!" }; + return new ImageGenerationViewModel { Message = "Error! Cannot find a valid image or mask!" }; } var message = await fileInstruct.EditImage(input.Provider, input.Model, input.Text, image, mask); imageViewModel.Content = message.Content; @@ -240,4 +241,34 @@ public async Task PdfCompletion([FromBody] IncomingMessa } } #endregion + + #region Audio + [HttpPost("/instruct/audio-completion")] + public async Task AudioCompletion([FromBody] IncomingMessageModel input) + { + var fileInstruct = _services.GetRequiredService(); + var state = _services.GetRequiredService(); + input.States.ForEach(x => state.SetState(x.Key, x.Value, activeRounds: x.ActiveRounds, source: StateSource.External)); + var viewModel = new AudioCompletionViewModel(); + + try + { + var audio = input.Files.FirstOrDefault(x => !string.IsNullOrWhiteSpace(x.FileUrl) || !string.IsNullOrWhiteSpace(x.FileData)); + if (audio == null) + { + return new AudioCompletionViewModel { Message = "Error! Cannot find a valid audio file!" }; + } + var content = await fileInstruct.ReadAudio(input.Provider, input.Model, audio); + viewModel.Content = content; + return viewModel; + } + catch (Exception ex) + { + var error = $"Error in audio completion. {ex.Message}"; + _logger.LogError(error); + viewModel.Message = error; + return viewModel; + } + } + #endregion } diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/AudioCompletionViewModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/AudioCompletionViewModel.cs new file mode 100644 index 000000000..3c3501aef --- /dev/null +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/AudioCompletionViewModel.cs @@ -0,0 +1,5 @@ +namespace BotSharp.OpenAPI.ViewModels.Instructs; + +public class AudioCompletionViewModel : InstructBaseViewModel +{ +} diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/ImageGenerationViewModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/ImageGenerationViewModel.cs index 0050de015..ea8dc0763 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/ImageGenerationViewModel.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/ImageGenerationViewModel.cs @@ -2,18 +2,11 @@ namespace BotSharp.OpenAPI.ViewModels.Instructs; -public class ImageGenerationViewModel +public class ImageGenerationViewModel : InstructBaseViewModel { - [JsonPropertyName("content")] - public string Content { get; set; } = string.Empty; - [JsonPropertyName("images")] [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public IEnumerable Images { get; set; } = new List(); - - [JsonPropertyName("message")] - [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] - public string? Message { get; set; } } public class ImageViewModel diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/InstructBaseViewModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/InstructBaseViewModel.cs new file mode 100644 index 000000000..0b20fca36 --- /dev/null +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/InstructBaseViewModel.cs @@ -0,0 +1,13 @@ +using System.Text.Json.Serialization; + +namespace BotSharp.OpenAPI.ViewModels.Instructs; + +public class InstructBaseViewModel +{ + [JsonPropertyName("content")] + public string Content { get; set; } = string.Empty; + + [JsonPropertyName("message")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? Message { get; set; } +} diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/InstructMessageModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/InstructMessageModel.cs index 3b265e044..b736330fc 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/InstructMessageModel.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/InstructMessageModel.cs @@ -1,5 +1,3 @@ -using BotSharp.Abstraction.Conversations.Enums; -using BotSharp.Abstraction.Conversations.Models; namespace BotSharp.OpenAPI.ViewModels.Instructs; public class InstructMessageModel : IncomingMessageModel diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/PdfCompletionViewModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/PdfCompletionViewModel.cs index 13ed3eb90..7ac594e61 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/PdfCompletionViewModel.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/PdfCompletionViewModel.cs @@ -1,13 +1,5 @@ -using System.Text.Json.Serialization; - namespace BotSharp.OpenAPI.ViewModels.Instructs; -public class PdfCompletionViewModel +public class PdfCompletionViewModel : InstructBaseViewModel { - [JsonPropertyName("content")] - public string Content { get; set; } = string.Empty; - - [JsonPropertyName("message")] - [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] - public string? Message { get; set; } } diff --git a/src/Plugins/BotSharp.Plugin.AudioHandler/AudioHandlerPlugin.cs b/src/Plugins/BotSharp.Plugin.AudioHandler/AudioHandlerPlugin.cs index 304aa97ba..855c00810 100644 --- a/src/Plugins/BotSharp.Plugin.AudioHandler/AudioHandlerPlugin.cs +++ b/src/Plugins/BotSharp.Plugin.AudioHandler/AudioHandlerPlugin.cs @@ -1,31 +1,25 @@ using BotSharp.Plugin.AudioHandler.Settings; -using BotSharp.Plugin.AudioHandler.Provider; using BotSharp.Abstraction.Settings; -namespace BotSharp.Plugin.AudioHandler +namespace BotSharp.Plugin.AudioHandler; + +public class AudioHandlerPlugin : IBotSharpPlugin { - public class AudioHandlerPlugin : IBotSharpPlugin + public string Id => "9d22014c-4f45-466a-9e82-a74e67983df8"; + public string Name => "Audio Handler"; + public string Description => "Process audio input and transform it into text output."; + public void RegisterDI(IServiceCollection services, IConfiguration config) { - public string Id => "9d22014c-4f45-466a-9e82-a74e67983df8"; - public string Name => "Audio Handler"; - public string Description => "Process audio input and transform it into text output."; - public void RegisterDI(IServiceCollection services, IConfiguration config) + services.AddScoped(provider => { - //var settings = new AudioHandlerSettings(); - //config.Bind("AudioHandler", settings); - //services.AddSingleton(x => settings); - - services.AddScoped(provider => - { - var settingService = provider.GetRequiredService(); - return settingService.Bind("AudioHandler"); - }); + var settingService = provider.GetRequiredService(); + return settingService.Bind("AudioHandler"); + }); - services.AddScoped(); - services.AddScoped(); - services.AddScoped(); - services.AddScoped(); - } + services.AddScoped(); + services.AddScoped(); + services.AddScoped(); + services.AddScoped(); } } diff --git a/src/Plugins/BotSharp.Plugin.AudioHandler/Controllers/AudioController.cs b/src/Plugins/BotSharp.Plugin.AudioHandler/Controllers/AudioController.cs index 45f4c8a49..60b2dc15c 100644 --- a/src/Plugins/BotSharp.Plugin.AudioHandler/Controllers/AudioController.cs +++ b/src/Plugins/BotSharp.Plugin.AudioHandler/Controllers/AudioController.cs @@ -1,11 +1,4 @@ -using System; -using System.Collections.Generic; using System.Diagnostics; -using System.Linq; -using System.Text; -using System.Threading.Tasks; -using BotSharp.Plugin.AudioHandler.Models; -using BotSharp.Plugin.AudioHandler.Provider; using BotSharp.Core.Infrastructures; namespace BotSharp.Plugin.AudioHandler.Controllers @@ -38,9 +31,7 @@ public async Task GetTextFromAudioController(string audioInputStr #if DEBUG stopWatch.Stop(); TimeSpan ts = stopWatch.Elapsed; - string elapsedTime = String.Format("{0:00}:{1:00}:{2:00}.{3:00}", - ts.Hours, ts.Minutes, ts.Seconds, - ts.Milliseconds / 10); + string elapsedTime = String.Format("{0:00}:{1:00}:{2:00}.{3:00}", ts.Hours, ts.Minutes, ts.Seconds, ts.Milliseconds / 10); Console.WriteLine("RunTime " + elapsedTime); #endif return Ok(result); @@ -58,9 +49,7 @@ public async Task GetTextFromAudioOpenAiController(string filePat #if DEBUG stopWatch.Stop(); TimeSpan ts = stopWatch.Elapsed; - string elapsedTime = String.Format("{0:00}:{1:00}:{2:00}.{3:00}", - ts.Hours, ts.Minutes, ts.Seconds, - ts.Milliseconds / 10); + string elapsedTime = String.Format("{0:00}:{1:00}:{2:00}.{3:00}", ts.Hours, ts.Minutes, ts.Seconds, ts.Milliseconds / 10); Console.WriteLine("RunTime " + elapsedTime); #endif return Ok(result); diff --git a/src/Plugins/BotSharp.Plugin.AudioHandler/Enums/AudioType.cs b/src/Plugins/BotSharp.Plugin.AudioHandler/Enums/AudioType.cs index 436b29227..7b299a76e 100644 --- a/src/Plugins/BotSharp.Plugin.AudioHandler/Enums/AudioType.cs +++ b/src/Plugins/BotSharp.Plugin.AudioHandler/Enums/AudioType.cs @@ -1,32 +1,23 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Runtime.CompilerServices; -using System.Text; -using System.Threading.Tasks; -using Whisper.net.Wave; +namespace BotSharp.Plugin.AudioHandler.Enums; -namespace BotSharp.Plugin.AudioHandler.Enums +public enum AudioType { - public enum AudioType - { - wav, - mp3, - } + wav, + mp3, +} - public static class AudioTypeExtensions +public static class AudioTypeExtensions +{ + public static string ToFileExtension(this AudioType audioType) => $".{audioType}"; + public static string ToFileType(this AudioType audioType) { - public static string ToFileExtension(this AudioType audioType) => $".{audioType}"; - public static string ToFileType(this AudioType audioType) + string type = audioType switch { - string type = audioType switch - { - AudioType.mp3 => "audio/mpeg", - AudioType.wav => "audio/wav", - _ => throw new NotImplementedException($"No support found for {audioType}") - }; - return type; - } + AudioType.mp3 => "audio/mpeg", + AudioType.wav => "audio/wav", + _ => throw new NotImplementedException($"No support found for {audioType}") + }; + return type; } } diff --git a/src/Plugins/BotSharp.Plugin.AudioHandler/Enums/UtilityName.cs b/src/Plugins/BotSharp.Plugin.AudioHandler/Enums/UtilityName.cs index 0deab6e9f..d11bd65ac 100644 --- a/src/Plugins/BotSharp.Plugin.AudioHandler/Enums/UtilityName.cs +++ b/src/Plugins/BotSharp.Plugin.AudioHandler/Enums/UtilityName.cs @@ -1,13 +1,6 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; +namespace BotSharp.Plugin.AudioHandler.Enums; -namespace BotSharp.Plugin.AudioHandler.Enums +public class UtilityName { - public class UtilityName - { - public const string AudioHandler = "audio-handler"; - } + public const string AudioHandler = "audio-handler"; } diff --git a/src/Plugins/BotSharp.Plugin.AudioHandler/Functions/HandleAudioRequestFn.cs b/src/Plugins/BotSharp.Plugin.AudioHandler/Functions/HandleAudioRequestFn.cs index 0c2298c60..37016220c 100644 --- a/src/Plugins/BotSharp.Plugin.AudioHandler/Functions/HandleAudioRequestFn.cs +++ b/src/Plugins/BotSharp.Plugin.AudioHandler/Functions/HandleAudioRequestFn.cs @@ -1,4 +1,3 @@ -using BotSharp.Abstraction.Agents.Models; using BotSharp.Core.Infrastructures; using Microsoft.AspNetCore.StaticFiles; @@ -12,7 +11,6 @@ public class HandleAudioRequestFn : IFunctionCallback private readonly IServiceProvider _serviceProvider; private readonly ILogger _logger; private readonly BotSharpOptions _options; - private Agent? _agent; private readonly IEnumerable _audioContentType = new List { @@ -20,12 +18,10 @@ public class HandleAudioRequestFn : IFunctionCallback AudioType.wav.ToFileType(), }; - public HandleAudioRequestFn( IServiceProvider serviceProvider, ILogger logger, - BotSharpOptions options - ) + BotSharpOptions options) { _serviceProvider = serviceProvider; _logger = logger; @@ -36,20 +32,18 @@ public async Task Execute(RoleDialogModel message) { var args = JsonSerializer.Deserialize(message.FunctionArgs, _options.JsonSerializerOptions); var conv = _serviceProvider.GetRequiredService(); - var isNeedSummary = args?.IsNeedSummary ?? false; var wholeDialogs = conv.GetDialogHistory(); - var dialogs = await AssembleFiles(conv.ConversationId, wholeDialogs); + var dialogs = AssembleFiles(conv.ConversationId, wholeDialogs); - var response = await GetResponeFromDialogs(dialogs); // isNeedSummary ? await SummarizeAudioText : TranscribeAudioToText; + var response = await GetResponeFromDialogs(dialogs); message.Content = response; return true; } - private async Task> AssembleFiles(string convId, List dialogs) + private List AssembleFiles(string convId, List dialogs) { - if (dialogs.IsNullOrEmpty()) - return new List(); + if (dialogs.IsNullOrEmpty()) return new List(); var fileService = _serviceProvider.GetRequiredService(); var messageId = dialogs.Select(x => x.MessageId).Distinct().ToList(); @@ -60,8 +54,7 @@ private async Task> AssembleFiles(string convId, List x.MessageId == dialog.MessageId).ToList(); - if (found.IsNullOrEmpty()) - continue; + if (found.IsNullOrEmpty()) continue; dialog.Files = found.Select(x => new BotSharpFile { @@ -74,28 +67,20 @@ private async Task> AssembleFiles(string convId, List(fileType, out var fileEnumType) || provider.TryGetContentType(fileType, out string contentType); - return canParse; - } - private async Task GetResponeFromDialogs(List dialogs) { - var whisperService = await PrepareModel("native"); // openai, native + var speech2Text = await PrepareModel("native"); var dialog = dialogs.Where(x => !x.Files.IsNullOrEmpty()).Last(); int transcribedCount = 0; + foreach (var file in dialog.Files) { - if (file == null) - continue; + if (file == null) continue; string extension = Path.GetExtension(file?.FileStorageUrl); if (ParseAudioFileType(extension) && File.Exists(file.FileStorageUrl)) { - file.FileData = await whisperService.GenerateTextFromAudioAsync(file.FileStorageUrl); + file.FileData = await speech2Text.GenerateTextFromAudioAsync(file.FileStorageUrl); transcribedCount++; } } @@ -104,23 +89,33 @@ private async Task GetResponeFromDialogs(List dialogs) { throw new FileNotFoundException($"No audio files found in the dialog. MessageId: {dialog.MessageId}"); } + var resList = dialog.Files.Select(x => $"{x.FileName} \r\n {x.FileData}").ToList(); return string.Join("\n\r", resList); } - private async Task PrepareModel(string modelName = "native") + private async Task PrepareModel(string provider = "native") { - var whisperService = _serviceProvider.GetServices().FirstOrDefault(x => x.Provider == modelName.ToLower()); - if (whisperService == null) + var speech2Text = _serviceProvider.GetServices().FirstOrDefault(x => x.Provider == provider.ToLower()); + if (speech2Text == null) { - throw new Exception($"Can't resolve speech2text provider by {modelName}"); + throw new Exception($"Can't resolve speech2text provider by {provider}"); } - if (modelName.Equals("openai", StringComparison.OrdinalIgnoreCase)) + if (provider.IsEqualTo("openai")) { return CompletionProvider.GetSpeechToText(_serviceProvider, provider: "openai", model: "whisper-1"); } - await whisperService.SetModelName("Tiny"); - return whisperService; + + await speech2Text.SetModelName("Tiny"); + return speech2Text; + } + + private bool ParseAudioFileType(string fileType) + { + fileType = fileType.ToLower(); + var provider = new FileExtensionContentTypeProvider(); + bool canParse = Enum.TryParse(fileType, out _) || provider.TryGetContentType(fileType, out _); + return canParse; } } diff --git a/src/Plugins/BotSharp.Plugin.AudioHandler/Functions/IAudioProcessUtilities.cs b/src/Plugins/BotSharp.Plugin.AudioHandler/Functions/IAudioProcessUtilities.cs deleted file mode 100644 index a3c8243b3..000000000 --- a/src/Plugins/BotSharp.Plugin.AudioHandler/Functions/IAudioProcessUtilities.cs +++ /dev/null @@ -1,10 +0,0 @@ - -namespace BotSharp.Plugin.AudioHandler.Functions -{ - public interface IAudioProcessUtilities - { - Stream ConvertMp3ToStream(string mp3FileName); - Stream ConvertWavToStream(string wavFileName); - Stream ConvertToStream(string fileName); - } -} \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.AudioHandler/Functions/AudioProcessUtilities.cs b/src/Plugins/BotSharp.Plugin.AudioHandler/Helpers/AudioHelper.cs similarity index 59% rename from src/Plugins/BotSharp.Plugin.AudioHandler/Functions/AudioProcessUtilities.cs rename to src/Plugins/BotSharp.Plugin.AudioHandler/Helpers/AudioHelper.cs index a65443596..122273c84 100644 --- a/src/Plugins/BotSharp.Plugin.AudioHandler/Functions/AudioProcessUtilities.cs +++ b/src/Plugins/BotSharp.Plugin.AudioHandler/Helpers/AudioHelper.cs @@ -1,19 +1,52 @@ -using BotSharp.Plugin.AudioHandler.Enums; -using NAudio; using NAudio.Wave; using NAudio.Wave.SampleProviders; -namespace BotSharp.Plugin.AudioHandler.Functions; +namespace BotSharp.Plugin.AudioHandler.Helpers; -public class AudioProcessUtilities : IAudioProcessUtilities +public class AudioHelper : IAudioHelper { - public AudioProcessUtilities() + private readonly IServiceProvider _services; + private readonly ILogger _logger; + + public AudioHelper( + IServiceProvider services, + ILogger logger) { + _services = services; + _logger = logger; } - public Stream ConvertMp3ToStream(string mp3FileName) + public Stream ConvertToStream(string fileName) + { + if (string.IsNullOrEmpty(fileName)) + { + var error = "fileName is Null when converting to stream in audio processor"; + _logger.LogWarning(error); + throw new ArgumentNullException(error); + } + + var fileExtension = Path.GetExtension(fileName).ToLower().TrimStart('.'); + if (!Enum.TryParse(fileExtension, out AudioType fileType)) + { + var error = $"File extension: '{fileExtension}' is not supported!"; + _logger.LogWarning(error); + throw new NotSupportedException(error); + } + + var stream = fileType switch + { + AudioType.mp3 => ConvertMp3ToStream(fileName), + AudioType.wav => ConvertWavToStream(fileName), + _ => throw new NotSupportedException("File extension not supported"), + }; + + return stream; + } + + + private Stream ConvertMp3ToStream(string fileName) { - var fileStream = File.OpenRead(mp3FileName); + var fileStream = File.OpenRead(fileName); using var reader = new Mp3FileReader(fileStream); if (reader.WaveFormat.SampleRate != 16000) { @@ -23,14 +56,14 @@ public Stream ConvertMp3ToStream(string mp3FileName) wavStream.Seek(0, SeekOrigin.Begin); return wavStream; } + fileStream.Seek(0, SeekOrigin.Begin); return fileStream; - } - public Stream ConvertWavToStream(string wavFileName) + private Stream ConvertWavToStream(string fileName) { - var fileStream = File.OpenRead(wavFileName); + var fileStream = File.OpenRead(fileName); using var reader = new WaveFileReader(fileStream); if (reader.WaveFormat.SampleRate != 16000) { @@ -40,29 +73,8 @@ public Stream ConvertWavToStream(string wavFileName) wavStream.Seek(0, SeekOrigin.Begin); return wavStream; } + fileStream.Seek(0, SeekOrigin.Begin); return fileStream; } - - public Stream ConvertToStream(string fileName) - { - if (string.IsNullOrEmpty(fileName)) - { - throw new ArgumentNullException("fileName is Null"); - } - string fileExtension = Path.GetExtension(fileName).ToLower().TrimStart('.'); - if (!Enum.TryParse(fileExtension, out AudioType fileType)) - { - throw new NotSupportedException($"File extension: '{fileExtension}' not supported"); - } - - var stream = fileType switch - { - AudioType.mp3 => ConvertMp3ToStream(fileName), - AudioType.wav => ConvertWavToStream(fileName), - _ => throw new NotSupportedException("File extension not supported"), - }; - - return stream; - } } diff --git a/src/Plugins/BotSharp.Plugin.AudioHandler/Helpers/IAudioHelper.cs b/src/Plugins/BotSharp.Plugin.AudioHandler/Helpers/IAudioHelper.cs new file mode 100644 index 000000000..d096a526e --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.AudioHandler/Helpers/IAudioHelper.cs @@ -0,0 +1,6 @@ +namespace BotSharp.Plugin.AudioHandler.Helpers; + +public interface IAudioHelper +{ + Stream ConvertToStream(string fileName); +} \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.AudioHandler/Hooks/AudioHandlerHook.cs b/src/Plugins/BotSharp.Plugin.AudioHandler/Hooks/AudioHandlerHook.cs index c3ff10d0b..51edcefa0 100644 --- a/src/Plugins/BotSharp.Plugin.AudioHandler/Hooks/AudioHandlerHook.cs +++ b/src/Plugins/BotSharp.Plugin.AudioHandler/Hooks/AudioHandlerHook.cs @@ -1,73 +1,67 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; using BotSharp.Abstraction.Agents.Settings; using BotSharp.Abstraction.Functions.Models; using BotSharp.Abstraction.Repositories; -namespace BotSharp.Plugin.AudioHandler.Hooks +namespace BotSharp.Plugin.AudioHandler.Hooks; + +public class AudioHandlerHook : AgentHookBase, IAgentHook { - public class AudioHandlerHook : AgentHookBase, IAgentHook + private const string HANDLER_AUDIO = "handle_audio_request"; + + public override string SelfId => string.Empty; + + public AudioHandlerHook(IServiceProvider services, AgentSettings settings) : base(services, settings) { - private const string HANDLER_AUDIO = "handle_audio_request"; + } - public override string SelfId => string.Empty; + public override void OnAgentLoaded(Agent agent) + { + var conv = _services.GetRequiredService(); + var isConvMode = conv.IsConversationMode(); + var isEnabled = !agent.Utilities.IsNullOrEmpty() && agent.Utilities.Contains(UtilityName.AudioHandler); - public AudioHandlerHook(IServiceProvider services, AgentSettings settings) : base(services, settings) + if (isEnabled && isConvMode) { + AddUtility(agent, UtilityName.AudioHandler, HANDLER_AUDIO); } - public override void OnAgentLoaded(Agent agent) - { - var conv = _services.GetRequiredService(); - var isConvMode = conv.IsConversationMode(); - var isEnabled = !agent.Utilities.IsNullOrEmpty() && agent.Utilities.Contains(UtilityName.AudioHandler); - - if (isEnabled && isConvMode) - { - AddUtility(agent, UtilityName.AudioHandler, HANDLER_AUDIO); - } + base.OnAgentLoaded(agent); + } - base.OnAgentLoaded(agent); - } + private void AddUtility(Agent agent, string utility, string functionName) + { + if (!IsEnableUtility(agent, utility)) return; - private void AddUtility(Agent agent, string utility, string functionName) + var (prompt, fn) = GetPromptAndFunction(functionName); + if (fn != null) { - if (!IsEnableUtility(agent, utility)) return; - - var (prompt, fn) = GetPromptAndFunction(functionName); - if (fn != null) + if (!string.IsNullOrWhiteSpace(prompt)) { - if (!string.IsNullOrWhiteSpace(prompt)) - { - agent.Instruction += $"\r\n\r\n{prompt}\r\n\r\n"; - } + agent.Instruction += $"\r\n\r\n{prompt}\r\n\r\n"; + } - if (agent.Functions == null) - { - agent.Functions = new List { fn }; - } - else - { - agent.Functions.Add(fn); - } + if (agent.Functions == null) + { + agent.Functions = new List { fn }; + } + else + { + agent.Functions.Add(fn); } } + } - private bool IsEnableUtility(Agent agent, string utility) - { - return !agent.Utilities.IsNullOrEmpty() && agent.Utilities.Contains(utility); - } + private bool IsEnableUtility(Agent agent, string utility) + { + return !agent.Utilities.IsNullOrEmpty() && agent.Utilities.Contains(utility); + } - private (string, FunctionDef?) GetPromptAndFunction(string functionName) - { - var db = _services.GetRequiredService(); - var agent = db.GetAgent(BuiltInAgentId.UtilityAssistant); - var prompt = agent?.Templates?.FirstOrDefault(x => x.Name.IsEqualTo($"{functionName}.fn"))?.Content ?? string.Empty; - var loadAttachmentFn = agent?.Functions?.FirstOrDefault(x => x.Name.IsEqualTo(functionName)); - return (prompt, loadAttachmentFn); - } + private (string, FunctionDef?) GetPromptAndFunction(string functionName) + { + var db = _services.GetRequiredService(); + var agent = db.GetAgent(BuiltInAgentId.UtilityAssistant); + var prompt = agent?.Templates?.FirstOrDefault(x => x.Name.IsEqualTo($"{functionName}.fn"))?.Content ?? string.Empty; + var loadAttachmentFn = agent?.Functions?.FirstOrDefault(x => x.Name.IsEqualTo(functionName)); + return (prompt, loadAttachmentFn); } } diff --git a/src/Plugins/BotSharp.Plugin.AudioHandler/Hooks/AudioHandlerUtilityHook.cs b/src/Plugins/BotSharp.Plugin.AudioHandler/Hooks/AudioHandlerUtilityHook.cs index f220acdfa..ac3f0ed73 100644 --- a/src/Plugins/BotSharp.Plugin.AudioHandler/Hooks/AudioHandlerUtilityHook.cs +++ b/src/Plugins/BotSharp.Plugin.AudioHandler/Hooks/AudioHandlerUtilityHook.cs @@ -1,10 +1,3 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; -using BotSharp.Abstraction.Agents; - namespace BotSharp.Plugin.AudioHandler.Hooks; public class AudioHandlerUtilityHook : IAgentUtilityHook diff --git a/src/Plugins/BotSharp.Plugin.AudioHandler/LlmContexts/LlmContextIn.cs b/src/Plugins/BotSharp.Plugin.AudioHandler/LlmContexts/LlmContextIn.cs index 5132ef3a9..281305e8f 100644 --- a/src/Plugins/BotSharp.Plugin.AudioHandler/LlmContexts/LlmContextIn.cs +++ b/src/Plugins/BotSharp.Plugin.AudioHandler/LlmContexts/LlmContextIn.cs @@ -1,9 +1,4 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; using System.Text.Json.Serialization; -using System.Threading.Tasks; namespace BotSharp.Plugin.AudioHandler.LlmContexts; diff --git a/src/Plugins/BotSharp.Plugin.AudioHandler/LlmContexts/LlmContextOut.cs b/src/Plugins/BotSharp.Plugin.AudioHandler/LlmContexts/LlmContextOut.cs index 3db02f717..ba75464d8 100644 --- a/src/Plugins/BotSharp.Plugin.AudioHandler/LlmContexts/LlmContextOut.cs +++ b/src/Plugins/BotSharp.Plugin.AudioHandler/LlmContexts/LlmContextOut.cs @@ -1,9 +1,4 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; using System.Text.Json.Serialization; -using System.Threading.Tasks; namespace BotSharp.Plugin.AudioHandler.LlmContexts; diff --git a/src/Plugins/BotSharp.Plugin.AudioHandler/Models/AudioOutput.cs b/src/Plugins/BotSharp.Plugin.AudioHandler/Models/AudioOutput.cs index 1b58f455d..f8f0bf51d 100644 --- a/src/Plugins/BotSharp.Plugin.AudioHandler/Models/AudioOutput.cs +++ b/src/Plugins/BotSharp.Plugin.AudioHandler/Models/AudioOutput.cs @@ -1,19 +1,13 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; using Whisper.net; -namespace BotSharp.Plugin.AudioHandler.Models +namespace BotSharp.Plugin.AudioHandler.Models; + +public class AudioOutput { - public class AudioOutput - { - public List Segments { get; set; } + public List Segments { get; set; } = new(); - public override string ToString() - { - return this.Segments.Count > 0 ? string.Join(" ", this.Segments.Select(x => x.Text)) : string.Empty; - } + public override string ToString() + { + return this.Segments.Count > 0 ? string.Join(" ", this.Segments.Select(x => x.Text)) : string.Empty; } } diff --git a/src/Plugins/BotSharp.Plugin.AudioHandler/Provider/NativeWhisperProvider.cs b/src/Plugins/BotSharp.Plugin.AudioHandler/Provider/NativeWhisperProvider.cs index 7afae1fcb..d95f02588 100644 --- a/src/Plugins/BotSharp.Plugin.AudioHandler/Provider/NativeWhisperProvider.cs +++ b/src/Plugins/BotSharp.Plugin.AudioHandler/Provider/NativeWhisperProvider.cs @@ -1,4 +1,3 @@ -using BotSharp.Core.Agents.Services; using Whisper.net; using Whisper.net.Ggml; @@ -9,47 +8,47 @@ namespace BotSharp.Plugin.AudioHandler.Provider; /// public class NativeWhisperProvider : ISpeechToText { + private readonly IAudioHelper _audioProcessor; + private static WhisperProcessor _whisperProcessor; + private readonly ILogger _logger; + public string Provider => "native"; - private readonly IAudioProcessUtilities _audioProcessUtilities; - private static WhisperProcessor _processor; - private readonly ILogger _logger; private string MODEL_DIR = "model"; private string? _currentModelPath; + private Dictionary _modelPathDict = new Dictionary(); private GgmlType? _modelType; public NativeWhisperProvider( - IAudioProcessUtilities audioProcessUtilities, + IAudioHelper audioProcessor, ILogger logger) { - _audioProcessUtilities = audioProcessUtilities; + _audioProcessor = audioProcessor; _logger = logger; } public async Task GenerateTextFromAudioAsync(string filePath) { string fileExtension = Path.GetExtension(filePath); - if (!Enum.TryParse(fileExtension.TrimStart('.').ToLower(), out AudioType audioType)) + if (!Enum.TryParse(fileExtension.TrimStart('.').ToLower(), out AudioType audioType)) { throw new Exception($"Unsupported audio type: {fileExtension}"); } - using var stream = _audioProcessUtilities.ConvertToStream(filePath); - + using var stream = _audioProcessor.ConvertToStream(filePath); if (stream == null) { throw new Exception($"Failed to convert {fileExtension} to stream"); } var textResult = new List(); - - await foreach (var result in _processor.ProcessAsync((Stream)stream).ConfigureAwait(false)) + await foreach (var result in _whisperProcessor.ProcessAsync(stream).ConfigureAwait(false)) { textResult.Add(result); } - _processor.Dispose(); + _whisperProcessor.Dispose(); var audioOutput = new AudioOutput { @@ -57,17 +56,35 @@ public async Task GenerateTextFromAudioAsync(string filePath) }; return audioOutput.ToString(); } + + public Task GenerateTextFromAudioAsync(Stream audio, string audioFileName) + { + throw new NotImplementedException(); + } + + public async Task SetModelName(string model) + { + if (Enum.TryParse(model, true, out GgmlType ggmlType)) + { + await LoadWhisperModel(ggmlType); + return; + } + + _logger.LogWarning($"Unsupported model type: {model}. Use Tiny model instead!"); + await LoadWhisperModel(GgmlType.Tiny); + } + private async Task LoadWhisperModel(GgmlType modelType) { try { if (!Directory.Exists(MODEL_DIR)) + { Directory.CreateDirectory(MODEL_DIR); + } - var availableModelPaths = Directory.GetFiles(MODEL_DIR, "*.bin") - .ToArray(); - - if (!availableModelPaths.Any()) + var availableModelPaths = Directory.GetFiles(MODEL_DIR, "*.bin").ToArray(); + if (availableModelPaths.IsNullOrEmpty()) { _currentModelPath = SetModelPath(MODEL_DIR, modelType); await DownloadModel(modelType, _currentModelPath); @@ -86,17 +103,14 @@ private async Task LoadWhisperModel(GgmlType modelType) } } - _processor = WhisperFactory - .FromPath(path: _currentModelPath) - .CreateBuilder() - .WithLanguage("auto") - .Build(); - + _whisperProcessor = WhisperFactory.FromPath(path: _currentModelPath).CreateBuilder().WithLanguage("auto").Build(); _modelType = modelType; } catch (Exception ex) { - throw new Exception($"Failed to load whisper model: {ex.Message}"); + var error = "Failed to load whisper model"; + _logger.LogWarning($"${error}: {ex.Message}\r\n{ex.InnerException}"); + throw new Exception($"{error}: {ex.Message}"); } } @@ -112,16 +126,4 @@ private string SetModelPath(string rootPath, GgmlType modelType) string currentModelPath = Path.Combine(rootPath, $"ggml-{modelType}.bin"); return currentModelPath; } - - public async Task SetModelName(string modelType) - { - if (Enum.TryParse(modelType, true, out GgmlType ggmlType)) - { - await LoadWhisperModel(ggmlType); - return; - } - - _logger.LogWarning($"Unsupported model type: {modelType}. Use Tiny model instead!"); - await LoadWhisperModel(GgmlType.Tiny); - } } diff --git a/src/Plugins/BotSharp.Plugin.AudioHandler/Settings/AudioHandlerSettings.cs b/src/Plugins/BotSharp.Plugin.AudioHandler/Settings/AudioHandlerSettings.cs index 4ace63db4..fedf123ca 100644 --- a/src/Plugins/BotSharp.Plugin.AudioHandler/Settings/AudioHandlerSettings.cs +++ b/src/Plugins/BotSharp.Plugin.AudioHandler/Settings/AudioHandlerSettings.cs @@ -1,6 +1,5 @@ -namespace BotSharp.Plugin.AudioHandler.Settings +namespace BotSharp.Plugin.AudioHandler.Settings; + +public class AudioHandlerSettings { - public class AudioHandlerSettings - { - } } diff --git a/src/Plugins/BotSharp.Plugin.AudioHandler/Using.cs b/src/Plugins/BotSharp.Plugin.AudioHandler/Using.cs index c15e6cc4a..b1ba52497 100644 --- a/src/Plugins/BotSharp.Plugin.AudioHandler/Using.cs +++ b/src/Plugins/BotSharp.Plugin.AudioHandler/Using.cs @@ -20,7 +20,7 @@ global using BotSharp.Abstraction.Utilities; global using BotSharp.Plugin.AudioHandler.Enums; -global using BotSharp.Plugin.AudioHandler.Functions; +global using BotSharp.Plugin.AudioHandler.Helpers; global using BotSharp.Plugin.AudioHandler.Hooks; global using BotSharp.Plugin.AudioHandler.Models; global using BotSharp.Plugin.AudioHandler.LlmContexts; diff --git a/src/Plugins/BotSharp.Plugin.Graph/GraphDb.cs b/src/Plugins/BotSharp.Plugin.Graph/GraphDb.cs index 9e0ceaee0..74552d174 100644 --- a/src/Plugins/BotSharp.Plugin.Graph/GraphDb.cs +++ b/src/Plugins/BotSharp.Plugin.Graph/GraphDb.cs @@ -35,7 +35,7 @@ public GraphDb( _settings = settings; } - public string Name => "Default"; + public string Name => "Neo4j"; public async Task Search(string query, GraphSearchOptions options) { diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/SpeechToTextProvider.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/SpeechToTextProvider.cs index 2314b431f..e9b54af88 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/SpeechToTextProvider.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/SpeechToTextProvider.cs @@ -1,14 +1,13 @@ -using System.Text; using OpenAI.Audio; namespace BotSharp.Plugin.OpenAI.Providers.Audio; public class SpeechToTextProvider : ISpeechToText { - public string Provider => "openai"; private readonly IServiceProvider _services; - private string? _modelName; - private AudioTranscriptionOptions? _options; + + public string Provider => "openai"; + private string? _model; public SpeechToTextProvider(IServiceProvider service) { @@ -17,30 +16,35 @@ public SpeechToTextProvider(IServiceProvider service) public async Task GenerateTextFromAudioAsync(string filePath) { - var client = ProviderHelper - .GetClient(Provider, _modelName, _services) - .GetAudioClient(_modelName); - SetOptions(); - - var transcription = await client.TranscribeAudioAsync(filePath); - - return transcription.Value.Text; + var client = ProviderHelper.GetClient(Provider, _model, _services) + .GetAudioClient(_model); + + var options = PrepareOptions(); + var result = await client.TranscribeAudioAsync(filePath, options); + return result.Value.Text; + } + + public async Task GenerateTextFromAudioAsync(Stream audio, string audioFileName) + { + var audioClient = ProviderHelper.GetClient(Provider, _model, _services) + .GetAudioClient(_model); + + var options = PrepareOptions(); + var result = await audioClient.TranscribeAudioAsync(audio, audioFileName, options); + return result.Value.Text; } - public async Task SetModelName(string modelName) + public async Task SetModelName(string model) { - _modelName = modelName; + _model = model; } - public void SetOptions(AudioTranscriptionOptions? options = null) + private AudioTranscriptionOptions PrepareOptions() { - if (_options == null) + return new AudioTranscriptionOptions { - _options = options ?? new AudioTranscriptionOptions - { - ResponseFormat = AudioTranscriptionFormat.Verbose, - Granularities = AudioTimestampGranularities.Word | AudioTimestampGranularities.Segment, - }; - } + ResponseFormat = AudioTranscriptionFormat.Verbose, + Granularities = AudioTimestampGranularities.Word | AudioTimestampGranularities.Segment, + }; } } diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/TextToSpeechProvider.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/TextToSpeechProvider.cs index e559e1096..e109dfcd8 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/TextToSpeechProvider.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/TextToSpeechProvider.cs @@ -4,8 +4,9 @@ namespace BotSharp.Plugin.OpenAI.Providers.Audio { public partial class TextToSpeechProvider : ITextToSpeech { - public string Provider => "openai"; private readonly IServiceProvider _services; + + public string Provider => "openai"; private string? _model; public TextToSpeechProvider( @@ -14,17 +15,17 @@ public TextToSpeechProvider( _services = services; } - public void SetModelName(string model) + public async Task GenerateSpeechFromTextAsync(string text, ITextToSpeechOptions? options = null) { - _model = model; + var client = ProviderHelper.GetClient(Provider, _model, _services) + .GetAudioClient(_model); + + return await client.GenerateSpeechFromTextAsync(text, GeneratedSpeechVoice.Alloy); } - public async Task GenerateSpeechFromTextAsync(string text, ITextToSpeechOptions? options = null) + public void SetModelName(string model) { - var client = ProviderHelper - .GetClient(Provider, _model, _services) - .GetAudioClient(_model); - return await client.GenerateSpeechFromTextAsync(text, GeneratedSpeechVoice.Alloy); + _model = model; } } } From 512d459c6b29c4d4028834b5f5839a9f1c246fdd Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Tue, 27 Aug 2024 10:42:48 -0500 Subject: [PATCH 02/27] refine instruct request model --- .../Files/Models/InputMessageFiles.cs | 7 -- .../BotSharp.Abstraction/Models/AiModel.cs | 10 --- .../Models/MessageConfig.cs | 6 +- .../Controllers/ConversationController.cs | 6 +- .../Controllers/InstructModeController.cs | 32 ++++----- .../Conversations/InputMessageFiles.cs | 7 ++ .../Instructs/InstructBaseRequest.cs | 66 +++++++++++++++++++ 7 files changed, 92 insertions(+), 42 deletions(-) delete mode 100644 src/Infrastructure/BotSharp.Abstraction/Files/Models/InputMessageFiles.cs delete mode 100644 src/Infrastructure/BotSharp.Abstraction/Models/AiModel.cs create mode 100644 src/Infrastructure/BotSharp.OpenAPI/ViewModels/Conversations/InputMessageFiles.cs create mode 100644 src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/InstructBaseRequest.cs diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/Models/InputMessageFiles.cs b/src/Infrastructure/BotSharp.Abstraction/Files/Models/InputMessageFiles.cs deleted file mode 100644 index 29749bce3..000000000 --- a/src/Infrastructure/BotSharp.Abstraction/Files/Models/InputMessageFiles.cs +++ /dev/null @@ -1,7 +0,0 @@ -namespace BotSharp.Abstraction.Files.Models; - -public class InputMessageFiles -{ - public List Files { get; set; } = new List(); - public BotSharpFile? Mask { get; set; } -} diff --git a/src/Infrastructure/BotSharp.Abstraction/Models/AiModel.cs b/src/Infrastructure/BotSharp.Abstraction/Models/AiModel.cs deleted file mode 100644 index 348eb4574..000000000 --- a/src/Infrastructure/BotSharp.Abstraction/Models/AiModel.cs +++ /dev/null @@ -1,10 +0,0 @@ -namespace BotSharp.Abstraction.Models; - -public class AiModel -{ - public string Id { get; set; } = string.Empty; - public string Model { get; set; } = string.Empty; - public string Name { get; set; } = string.Empty; - public int MaxLength { get; set; } - public int TokenLimit { get; set; } -} diff --git a/src/Infrastructure/BotSharp.Abstraction/Models/MessageConfig.cs b/src/Infrastructure/BotSharp.Abstraction/Models/MessageConfig.cs index 24e4152c4..9ea5917cc 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Models/MessageConfig.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Models/MessageConfig.cs @@ -1,6 +1,6 @@ namespace BotSharp.Abstraction.Models; -public class MessageConfig : InputMessageFiles +public class MessageConfig { /// /// Completion Provider @@ -15,7 +15,7 @@ public class MessageConfig : InputMessageFiles public virtual string? Model { get; set; } = null; /// - /// Model name + /// Model id /// [JsonPropertyName("model_id")] public virtual string? ModelId { get; set; } = null; @@ -34,7 +34,7 @@ public class MessageConfig : InputMessageFiles /// /// Conversation states from input /// - public List States { get; set; } = new List(); + public List States { get; set; } = new(); /// /// Agent task id diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs index 9b964dffd..565677f7c 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/ConversationController.cs @@ -294,9 +294,7 @@ await conv.SendMessage(agentId, inputMsg, } [HttpPost("/conversation/{agentId}/{conversationId}/sse")] - public async Task SendMessageSse([FromRoute] string agentId, - [FromRoute] string conversationId, - [FromBody] NewMessageModel input) + public async Task SendMessageSse([FromRoute] string agentId, [FromRoute] string conversationId, [FromBody] NewMessageModel input) { var conv = _services.GetRequiredService(); var inputMsg = new RoleDialogModel(AgentRole.User, input.Text) @@ -391,7 +389,7 @@ public IActionResult UploadAttachments([FromRoute] string conversationId, } [HttpPost("/agent/{agentId}/conversation/{conversationId}/upload")] - public async Task UploadConversationMessageFiles([FromRoute] string agentId, [FromRoute] string conversationId, [FromBody] NewMessageModel input) + public async Task UploadConversationMessageFiles([FromRoute] string agentId, [FromRoute] string conversationId, [FromBody] InputMessageFiles input) { var convService = _services.GetRequiredService(); convService.SetConversationId(conversationId, input.States); diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs index 0e60f331a..b0ecd9056 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs @@ -3,7 +3,6 @@ using BotSharp.Abstraction.Instructs.Models; using BotSharp.Core.Infrastructures; using BotSharp.OpenAPI.ViewModels.Instructs; -using static System.Net.Mime.MediaTypeNames; namespace BotSharp.OpenAPI.Controllers; @@ -21,8 +20,7 @@ public InstructModeController(IServiceProvider services, ILogger InstructCompletion([FromRoute] string agentId, - [FromBody] InstructMessageModel input) + public async Task InstructCompletion([FromRoute] string agentId, [FromBody] InstructMessageModel input) { var state = _services.GetRequiredService(); input.States.ForEach(x => state.SetState(x.Key, x.Value, activeRounds: x.ActiveRounds, source: StateSource.External)); @@ -80,7 +78,7 @@ public async Task ChatCompletion([FromBody] IncomingMessageModel input) #region Read image [HttpPost("/instruct/multi-modal")] - public async Task MultiModalCompletion([FromBody] IncomingMessageModel input) + public async Task MultiModalCompletion([FromBody] MultiModalRequest input) { var state = _services.GetRequiredService(); input.States.ForEach(x => state.SetState(x.Key, x.Value, activeRounds: x.ActiveRounds, source: StateSource.External)); @@ -102,7 +100,7 @@ public async Task MultiModalCompletion([FromBody] IncomingMessageModel i #region Generate image [HttpPost("/instruct/image-generation")] - public async Task ImageGeneration([FromBody] IncomingMessageModel input) + public async Task ImageGeneration([FromBody] ImageGenerationRequest input) { var state = _services.GetRequiredService(); input.States.ForEach(x => state.SetState(x.Key, x.Value, activeRounds: x.ActiveRounds, source: StateSource.External)); @@ -128,7 +126,7 @@ public async Task ImageGeneration([FromBody] IncomingM #region Edit image [HttpPost("/instruct/image-variation")] - public async Task ImageVariation([FromBody] IncomingMessageModel input) + public async Task ImageVariation([FromBody] ImageVariationRequest input) { var state = _services.GetRequiredService(); input.States.ForEach(x => state.SetState(x.Key, x.Value, activeRounds: x.ActiveRounds, source: StateSource.External)); @@ -136,14 +134,13 @@ public async Task ImageVariation([FromBody] IncomingMe try { - var image = input.Files.FirstOrDefault(x => !string.IsNullOrWhiteSpace(x.FileUrl) || !string.IsNullOrWhiteSpace(x.FileData)); - if (image == null) + if (input.File == null) { return new ImageGenerationViewModel { Message = "Error! Cannot find an image!" }; } var fileInstruct = _services.GetRequiredService(); - var message = await fileInstruct.VaryImage(input.Provider, input.Model, image); + var message = await fileInstruct.VaryImage(input.Provider, input.Model, input.File); imageViewModel.Content = message.Content; imageViewModel.Images = message.GeneratedImages.Select(x => ImageViewModel.ToViewModel(x)).ToList(); return imageViewModel; @@ -158,7 +155,7 @@ public async Task ImageVariation([FromBody] IncomingMe } [HttpPost("/instruct/image-edit")] - public async Task ImageEdit([FromBody] IncomingMessageModel input) + public async Task ImageEdit([FromBody] ImageEditRequest input) { var fileInstruct = _services.GetRequiredService(); var state = _services.GetRequiredService(); @@ -167,12 +164,11 @@ public async Task ImageEdit([FromBody] IncomingMessage try { - var image = input.Files.FirstOrDefault(x => !string.IsNullOrWhiteSpace(x.FileUrl) || !string.IsNullOrWhiteSpace(x.FileData)); - if (image == null) + if (input.File == null) { return new ImageGenerationViewModel { Message = "Error! Cannot find a valid image file!" }; } - var message = await fileInstruct.EditImage(input.Provider, input.Model, input.Text, image); + var message = await fileInstruct.EditImage(input.Provider, input.Model, input.Text, input.File); imageViewModel.Content = message.Content; imageViewModel.Images = message.GeneratedImages.Select(x => ImageViewModel.ToViewModel(x)).ToList(); return imageViewModel; @@ -187,7 +183,7 @@ public async Task ImageEdit([FromBody] IncomingMessage } [HttpPost("/instruct/image-mask-edit")] - public async Task ImageMaskEdit([FromBody] IncomingMessageModel input) + public async Task ImageMaskEdit([FromBody] ImageMaskEditRequest input) { var fileInstruct = _services.GetRequiredService(); var state = _services.GetRequiredService(); @@ -196,7 +192,7 @@ public async Task ImageMaskEdit([FromBody] IncomingMes try { - var image = input.Files.FirstOrDefault(x => !string.IsNullOrWhiteSpace(x.FileUrl) || !string.IsNullOrWhiteSpace(x.FileData)); + var image = input.File; var mask = input.Mask; if (image == null || mask == null) { @@ -219,7 +215,7 @@ public async Task ImageMaskEdit([FromBody] IncomingMes #region Pdf [HttpPost("/instruct/pdf-completion")] - public async Task PdfCompletion([FromBody] IncomingMessageModel input) + public async Task PdfCompletion([FromBody] MultiModalRequest input) { var state = _services.GetRequiredService(); input.States.ForEach(x => state.SetState(x.Key, x.Value, activeRounds: x.ActiveRounds, source: StateSource.External)); @@ -244,7 +240,7 @@ public async Task PdfCompletion([FromBody] IncomingMessa #region Audio [HttpPost("/instruct/audio-completion")] - public async Task AudioCompletion([FromBody] IncomingMessageModel input) + public async Task AudioCompletion([FromBody] AudioCompletionRequest input) { var fileInstruct = _services.GetRequiredService(); var state = _services.GetRequiredService(); @@ -253,7 +249,7 @@ public async Task AudioCompletion([FromBody] IncomingM try { - var audio = input.Files.FirstOrDefault(x => !string.IsNullOrWhiteSpace(x.FileUrl) || !string.IsNullOrWhiteSpace(x.FileData)); + var audio = input.File; if (audio == null) { return new AudioCompletionViewModel { Message = "Error! Cannot find a valid audio file!" }; diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Conversations/InputMessageFiles.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Conversations/InputMessageFiles.cs new file mode 100644 index 000000000..5656f7ae0 --- /dev/null +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Conversations/InputMessageFiles.cs @@ -0,0 +1,7 @@ +namespace BotSharp.OpenAPI.ViewModels.Conversations; + +public class InputMessageFiles +{ + public List States { get; set; } = new(); + public List Files { get; set; } = new(); +} diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/InstructBaseRequest.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/InstructBaseRequest.cs new file mode 100644 index 000000000..1b5d551db --- /dev/null +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/InstructBaseRequest.cs @@ -0,0 +1,66 @@ +using System.Text.Json.Serialization; + +namespace BotSharp.Abstraction.Instructs.Models; + +public class InstructBaseRequest +{ + [JsonPropertyName("provider")] + public virtual string? Provider { get; set; } = null; + + [JsonPropertyName("model")] + public virtual string? Model { get; set; } = null; + + [JsonPropertyName("model_id")] + public virtual string? ModelId { get; set; } = null; + + [JsonPropertyName("states")] + public List States { get; set; } = new(); +} + +public class MultiModalRequest : InstructBaseRequest +{ + [JsonPropertyName("text")] + public string Text { get; set; } = string.Empty; + + [JsonPropertyName("files")] + public List Files { get; set; } = new(); +} + +public class ImageGenerationRequest : InstructBaseRequest +{ + [JsonPropertyName("text")] + public string Text { get; set; } = string.Empty; +} + +public class ImageVariationRequest : InstructBaseRequest +{ + [JsonPropertyName("file")] + public BotSharpFile File { get; set; } +} + +public class ImageEditRequest : InstructBaseRequest +{ + [JsonPropertyName("text")] + public string Text { get; set; } = string.Empty; + + [JsonPropertyName("file")] + public BotSharpFile File { get; set; } +} + +public class ImageMaskEditRequest : InstructBaseRequest +{ + [JsonPropertyName("text")] + public string Text { get; set; } = string.Empty; + + [JsonPropertyName("file")] + public BotSharpFile File { get; set; } + + [JsonPropertyName("mask")] + public BotSharpFile Mask { get; set; } +} + +public class AudioCompletionRequest : InstructBaseRequest +{ + [JsonPropertyName("file")] + public BotSharpFile File { get; set; } +} \ No newline at end of file From 1ace4e44aba53d698d78f221a7ea75656426eea9 Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Tue, 27 Aug 2024 12:12:17 -0500 Subject: [PATCH 03/27] refine file models --- .../Files/IFileInstructService.cs | 12 +++--- .../Files/IFileStorageService.cs | 6 +-- .../Files/Models/BotSharpFile.cs | 9 ++++- .../Files/Models/FileBase.cs | 28 -------------- .../Files/Models/FileInfo.cs | 38 +++++++++++++++++++ .../Files/Models/InputFileModel.cs | 16 ++++++++ .../Files/Models/InstructFileModel.cs | 18 +++++++++ .../Files/Models/MessageFileModel.cs | 2 +- .../BotSharp.Abstraction/Models/AiModel.cs | 10 +++++ .../Instruct/FileInstructService.Audio.cs | 5 ++- .../Instruct/FileInstructService.Image.cs | 22 ++++++----- .../Instruct/FileInstructService.Pdf.cs | 4 +- .../LocalFileStorageService.Conversation.cs | 2 +- .../Storage/LocalFileStorageService.User.cs | 2 +- .../Controllers/UserController.cs | 7 +++- .../Conversations/InputMessageFiles.cs | 2 +- .../Instructs/InstructBaseRequest.cs | 12 +++--- .../ViewModels/Users/UserAvatarModel.cs | 18 +++++++++ .../Functions/EditImageFn.cs | 4 +- .../Functions/GenerateImageFn.cs | 2 +- .../TencentCosService.Conversation.cs | 2 +- .../Services/TencentCosService.User.cs | 2 +- 22 files changed, 155 insertions(+), 68 deletions(-) create mode 100644 src/Infrastructure/BotSharp.Abstraction/Files/Models/FileInfo.cs create mode 100644 src/Infrastructure/BotSharp.Abstraction/Files/Models/InputFileModel.cs create mode 100644 src/Infrastructure/BotSharp.Abstraction/Files/Models/InstructFileModel.cs create mode 100644 src/Infrastructure/BotSharp.Abstraction/Models/AiModel.cs create mode 100644 src/Infrastructure/BotSharp.OpenAPI/ViewModels/Users/UserAvatarModel.cs diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/IFileInstructService.cs b/src/Infrastructure/BotSharp.Abstraction/Files/IFileInstructService.cs index e9ccb4ba2..f35cdad5f 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Files/IFileInstructService.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Files/IFileInstructService.cs @@ -3,11 +3,11 @@ namespace BotSharp.Abstraction.Files; public interface IFileInstructService { #region Image - Task ReadImages(string? provider, string? model, string text, IEnumerable images); + Task ReadImages(string? provider, string? model, string text, IEnumerable images); Task GenerateImage(string? provider, string? model, string text); - Task VaryImage(string? provider, string? model, BotSharpFile image); - Task EditImage(string? provider, string? model, string text, BotSharpFile image); - Task EditImage(string? provider, string? model, string text, BotSharpFile image, BotSharpFile mask); + Task VaryImage(string? provider, string? model, InstructFileModel image); + Task EditImage(string? provider, string? model, string text, InstructFileModel image); + Task EditImage(string? provider, string? model, string text, InstructFileModel image, InstructFileModel mask); #endregion #region Pdf @@ -17,11 +17,11 @@ public interface IFileInstructService /// /// Pdf files /// - Task ReadPdf(string? provider, string? model, string? modelId, string prompt, List files); + Task ReadPdf(string? provider, string? model, string? modelId, string prompt, List files); #endregion #region Audio - Task ReadAudio(string? provider, string? model, BotSharpFile audio); + Task ReadAudio(string? provider, string? model, InstructFileModel audio); #endregion #region Select file diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/IFileStorageService.cs b/src/Infrastructure/BotSharp.Abstraction/Files/IFileStorageService.cs index a3e5a32dc..6c98e21a8 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Files/IFileStorageService.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Files/IFileStorageService.cs @@ -16,7 +16,6 @@ public interface IFileStorageService string BuildDirectory(params string[] segments); #endregion - #region Conversation /// /// Get the message file screenshots for specific content types, e.g., pdf @@ -37,7 +36,7 @@ public interface IFileStorageService IEnumerable GetMessageFiles(string conversationId, IEnumerable messageIds, string source, IEnumerable? contentTypes = null); string GetMessageFile(string conversationId, string messageId, string source, string index, string fileName); IEnumerable GetMessagesWithFile(string conversationId, IEnumerable messageIds); - bool SaveMessageFiles(string conversationId, string messageId, string source, List files); + bool SaveMessageFiles(string conversationId, string messageId, string source, List files); /// /// Delete files under messages @@ -54,8 +53,9 @@ public interface IFileStorageService #region User string GetUserAvatar(); - bool SaveUserAvatar(BotSharpFile file); + bool SaveUserAvatar(InputFileModel file); #endregion + #region Speech Task SaveSpeechFileAsync(string conversationId, string fileName, BinaryData data); Task RetrieveSpeechFileAsync(string conversationId, string fileName); diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/Models/BotSharpFile.cs b/src/Infrastructure/BotSharp.Abstraction/Files/Models/BotSharpFile.cs index 11a11e469..46cba5959 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Files/Models/BotSharpFile.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Files/Models/BotSharpFile.cs @@ -1,7 +1,12 @@ namespace BotSharp.Abstraction.Files.Models; -public class BotSharpFile : FileBase +public class BotSharpFile : FileInfo { - + /// + /// File data, e.g., "data:image/png;base64,aaaaaaaa" + /// + [JsonPropertyName("file_data")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? FileData { get; set; } = string.Empty; } diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/Models/FileBase.cs b/src/Infrastructure/BotSharp.Abstraction/Files/Models/FileBase.cs index 3483921af..f952e7f90 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Files/Models/FileBase.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Files/Models/FileBase.cs @@ -2,20 +2,6 @@ namespace BotSharp.Abstraction.Files.Models; public class FileBase { - /// - /// External file url - /// - [JsonPropertyName("file_url")] - [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] - public string? FileUrl { get; set; } = string.Empty; - - /// - /// Internal file storage url - /// - [JsonPropertyName("file_storage_url")] - [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] - public string? FileStorageUrl { get; set; } = string.Empty; - /// /// File name without extension /// @@ -29,18 +15,4 @@ public class FileBase [JsonPropertyName("file_data")] [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public string? FileData { get; set; } = string.Empty; - - /// - /// File content type - /// - [JsonPropertyName("content_type")] - [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] - public string? ContentType { get; set; } = string.Empty; - - /// - /// File extension without dot - /// - [JsonPropertyName("file_extension")] - [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] - public string? FileExtension { get; set; } = string.Empty; } diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/Models/FileInfo.cs b/src/Infrastructure/BotSharp.Abstraction/Files/Models/FileInfo.cs new file mode 100644 index 000000000..14c02f746 --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Files/Models/FileInfo.cs @@ -0,0 +1,38 @@ +namespace BotSharp.Abstraction.Files.Models; + +public class FileInfo +{ + /// + /// External file url + /// + [JsonPropertyName("file_url")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? FileUrl { get; set; } = string.Empty; + + /// + /// Internal file storage url + /// + [JsonPropertyName("file_storage_url")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? FileStorageUrl { get; set; } = string.Empty; + + /// + /// File content type + /// + [JsonPropertyName("content_type")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? ContentType { get; set; } = string.Empty; + + /// + /// File name without extension + /// + [JsonPropertyName("file_name")] + public string FileName { get; set; } = string.Empty; + + /// + /// File extension without dot + /// + [JsonPropertyName("file_extension")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? FileExtension { get; set; } = string.Empty; +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/Models/InputFileModel.cs b/src/Infrastructure/BotSharp.Abstraction/Files/Models/InputFileModel.cs new file mode 100644 index 000000000..7eb350e3d --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Files/Models/InputFileModel.cs @@ -0,0 +1,16 @@ +namespace BotSharp.Abstraction.Files.Models; + +public class InputFileModel : FileBase +{ + /// + /// File name with extension + /// + [JsonPropertyName("file_name")] + public new string FileName { get; set; } = string.Empty; + + /// + /// File data => format: "data:image/png;base64,aaaaaaaa" + /// + [JsonPropertyName("file_data")] + public new string FileData { get; set; } = string.Empty; +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/Models/InstructFileModel.cs b/src/Infrastructure/BotSharp.Abstraction/Files/Models/InstructFileModel.cs new file mode 100644 index 000000000..7eaddd5af --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Files/Models/InstructFileModel.cs @@ -0,0 +1,18 @@ +namespace BotSharp.Abstraction.Files.Models; + +public class InstructFileModel : FileBase +{ + /// + /// File extension without dot + /// + [JsonPropertyName("file_extension")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? FileExtension { get; set; } = string.Empty; + + /// + /// External file url + /// + [JsonPropertyName("file_url")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? FileUrl { get; set; } = string.Empty; +} diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/Models/MessageFileModel.cs b/src/Infrastructure/BotSharp.Abstraction/Files/Models/MessageFileModel.cs index 2a0128b66..59ada4f2f 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Files/Models/MessageFileModel.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Files/Models/MessageFileModel.cs @@ -1,6 +1,6 @@ namespace BotSharp.Abstraction.Files.Models; -public class MessageFileModel : FileBase +public class MessageFileModel : FileInfo { [JsonPropertyName("message_id")] public string MessageId { get; set; } diff --git a/src/Infrastructure/BotSharp.Abstraction/Models/AiModel.cs b/src/Infrastructure/BotSharp.Abstraction/Models/AiModel.cs new file mode 100644 index 000000000..348eb4574 --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Models/AiModel.cs @@ -0,0 +1,10 @@ +namespace BotSharp.Abstraction.Models; + +public class AiModel +{ + public string Id { get; set; } = string.Empty; + public string Model { get; set; } = string.Empty; + public string Name { get; set; } = string.Empty; + public int MaxLength { get; set; } + public int TokenLimit { get; set; } +} diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Audio.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Audio.cs index f3f40f2c9..30b6dbe30 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Audio.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Audio.cs @@ -4,7 +4,7 @@ namespace BotSharp.Core.Files.Services; public partial class FileInstructService { - public async Task ReadAudio(string? provider, string? model, BotSharpFile audio) + public async Task ReadAudio(string? provider, string? model, InstructFileModel audio) { var completion = CompletionProvider.GetSpeechToText(_services, provider: provider ?? "openai", model: model ?? "whisper-1"); var audioBytes = await DownloadFile(audio); @@ -12,7 +12,8 @@ public async Task ReadAudio(string? provider, string? model, BotSharpFil stream.Write(audioBytes, 0, audioBytes.Length); stream.Position = 0; - var content = await completion.GenerateTextFromAudioAsync(stream, audio.FileName ?? string.Empty); + var fileName = $"{audio.FileName ?? "audio"}.{audio.FileExtension ?? "wav"}"; + var content = await completion.GenerateTextFromAudioAsync(stream, fileName); stream.Close(); return content; } diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Image.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Image.cs index 244b5ac4e..cd65c16a9 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Image.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Image.cs @@ -4,7 +4,7 @@ namespace BotSharp.Core.Files.Services; public partial class FileInstructService { - public async Task ReadImages(string? provider, string? model, string text, IEnumerable images) + public async Task ReadImages(string? provider, string? model, string text, IEnumerable images) { var completion = CompletionProvider.GetChatCompletion(_services, provider: provider ?? "openai", model: model ?? "gpt-4o", multiModal: true); var message = await completion.GetChatCompletions(new Agent() @@ -14,7 +14,7 @@ public async Task ReadImages(string? provider, string? model, string tex { new RoleDialogModel(AgentRole.User, text) { - Files = images?.ToList() ?? new List() + Files = images?.Select(x => new BotSharpFile { FileUrl = x.FileUrl, FileData = x.FileData }).ToList() ?? new List() } }); return message.Content; @@ -30,7 +30,7 @@ public async Task GenerateImage(string? provider, string? model return message; } - public async Task VaryImage(string? provider, string? model, BotSharpFile image) + public async Task VaryImage(string? provider, string? model, InstructFileModel image) { if (string.IsNullOrWhiteSpace(image?.FileUrl) && string.IsNullOrWhiteSpace(image?.FileData)) { @@ -43,16 +43,17 @@ public async Task VaryImage(string? provider, string? model, Bo stream.Write(bytes, 0, bytes.Length); stream.Position = 0; + var fileName = $"{image.FileName ?? "image"}.{image.FileExtension ?? "png"}"; var message = await completion.GetImageVariation(new Agent() { Id = Guid.Empty.ToString() - }, new RoleDialogModel(AgentRole.User, string.Empty), stream, image.FileName ?? string.Empty); + }, new RoleDialogModel(AgentRole.User, string.Empty), stream, fileName); stream.Close(); return message; } - public async Task EditImage(string? provider, string? model, string text, BotSharpFile image) + public async Task EditImage(string? provider, string? model, string text, InstructFileModel image) { if (string.IsNullOrWhiteSpace(image?.FileUrl) && string.IsNullOrWhiteSpace(image?.FileData)) { @@ -65,16 +66,17 @@ public async Task EditImage(string? provider, string? model, st stream.Write(bytes, 0, bytes.Length); stream.Position = 0; + var fileName = $"{image.FileName ?? "image"}.{image.FileExtension ?? "png"}"; var message = await completion.GetImageEdits(new Agent() { Id = Guid.Empty.ToString() - }, new RoleDialogModel(AgentRole.User, text), stream, image.FileName ?? string.Empty); + }, new RoleDialogModel(AgentRole.User, text), stream, fileName); stream.Close(); return message; } - public async Task EditImage(string? provider, string? model, string text, BotSharpFile image, BotSharpFile mask) + public async Task EditImage(string? provider, string? model, string text, InstructFileModel image, InstructFileModel mask) { if ((string.IsNullOrWhiteSpace(image?.FileUrl) && string.IsNullOrWhiteSpace(image?.FileData)) || (string.IsNullOrWhiteSpace(mask?.FileUrl) && string.IsNullOrWhiteSpace(mask?.FileData))) @@ -94,10 +96,12 @@ public async Task EditImage(string? provider, string? model, st maskStream.Write(maskBytes, 0, maskBytes.Length); maskStream.Position = 0; + var imageName = $"{image.FileName ?? "image"}.{image.FileExtension ?? "png"}"; + var maskName = $"{mask.FileName ?? "mask"}.{mask.FileExtension ?? "png"}"; var message = await completion.GetImageEdits(new Agent() { Id = Guid.Empty.ToString() - }, new RoleDialogModel(AgentRole.User, text), imageStream, image.FileName ?? string.Empty, maskStream, mask.FileName ?? string.Empty); + }, new RoleDialogModel(AgentRole.User, text), imageStream, imageName, maskStream, maskName); imageStream.Close(); maskStream.Close(); @@ -105,7 +109,7 @@ public async Task EditImage(string? provider, string? model, st } #region Private methods - private async Task DownloadFile(BotSharpFile file) + private async Task DownloadFile(InstructFileModel file) { var bytes = new byte[0]; if (!string.IsNullOrEmpty(file.FileUrl)) diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Pdf.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Pdf.cs index dd504fbf6..e7228b138 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Pdf.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Pdf.cs @@ -4,7 +4,7 @@ namespace BotSharp.Core.Files.Services; public partial class FileInstructService { - public async Task ReadPdf(string? provider, string? model, string? modelId, string prompt, List files) + public async Task ReadPdf(string? provider, string? model, string? modelId, string prompt, List files) { var content = string.Empty; @@ -50,7 +50,7 @@ public async Task ReadPdf(string? provider, string? model, string? model } #region Private methods - private async Task> DownloadFiles(string dir, List files, string extension = "pdf") + private async Task> DownloadFiles(string dir, List files, string extension = "pdf") { if (string.IsNullOrWhiteSpace(dir) || files.IsNullOrEmpty()) { diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Conversation.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Conversation.cs index 8c2ed8310..248806eca 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Conversation.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Conversation.cs @@ -118,7 +118,7 @@ public IEnumerable GetMessagesWithFile(string conversationId, return foundMsgs; } - public bool SaveMessageFiles(string conversationId, string messageId, string source, List files) + public bool SaveMessageFiles(string conversationId, string messageId, string source, List files) { if (files.IsNullOrEmpty()) return false; diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.User.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.User.cs index 43ff9eed3..d1e962cd4 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.User.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.User.cs @@ -16,7 +16,7 @@ public string GetUserAvatar() return found; } - public bool SaveUserAvatar(BotSharpFile file) + public bool SaveUserAvatar(InputFileModel file) { if (file == null || string.IsNullOrEmpty(file.FileData)) return false; diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/UserController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/UserController.cs index a7d4c449f..163993349 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/UserController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/UserController.cs @@ -135,9 +135,14 @@ public async Task ModifyUserPhone([FromQuery] string phone) #region Avatar [HttpPost("/user/avatar")] - public bool UploadUserAvatar([FromBody] BotSharpFile file) + public bool UploadUserAvatar([FromBody] UserAvatarModel input) { var fileStorage = _services.GetRequiredService(); + var file = new InputFileModel + { + FileName = input.FileName, + FileData = input.FileData, + }; return fileStorage.SaveUserAvatar(file); } diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Conversations/InputMessageFiles.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Conversations/InputMessageFiles.cs index 5656f7ae0..fdf088bbc 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Conversations/InputMessageFiles.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Conversations/InputMessageFiles.cs @@ -3,5 +3,5 @@ namespace BotSharp.OpenAPI.ViewModels.Conversations; public class InputMessageFiles { public List States { get; set; } = new(); - public List Files { get; set; } = new(); + public List Files { get; set; } = new(); } diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/InstructBaseRequest.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/InstructBaseRequest.cs index 1b5d551db..45a1c8ab6 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/InstructBaseRequest.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/InstructBaseRequest.cs @@ -23,7 +23,7 @@ public class MultiModalRequest : InstructBaseRequest public string Text { get; set; } = string.Empty; [JsonPropertyName("files")] - public List Files { get; set; } = new(); + public List Files { get; set; } = new(); } public class ImageGenerationRequest : InstructBaseRequest @@ -35,7 +35,7 @@ public class ImageGenerationRequest : InstructBaseRequest public class ImageVariationRequest : InstructBaseRequest { [JsonPropertyName("file")] - public BotSharpFile File { get; set; } + public InstructFileModel File { get; set; } } public class ImageEditRequest : InstructBaseRequest @@ -44,7 +44,7 @@ public class ImageEditRequest : InstructBaseRequest public string Text { get; set; } = string.Empty; [JsonPropertyName("file")] - public BotSharpFile File { get; set; } + public InstructFileModel File { get; set; } } public class ImageMaskEditRequest : InstructBaseRequest @@ -53,14 +53,14 @@ public class ImageMaskEditRequest : InstructBaseRequest public string Text { get; set; } = string.Empty; [JsonPropertyName("file")] - public BotSharpFile File { get; set; } + public InstructFileModel File { get; set; } [JsonPropertyName("mask")] - public BotSharpFile Mask { get; set; } + public InstructFileModel Mask { get; set; } } public class AudioCompletionRequest : InstructBaseRequest { [JsonPropertyName("file")] - public BotSharpFile File { get; set; } + public InstructFileModel File { get; set; } } \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Users/UserAvatarModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Users/UserAvatarModel.cs new file mode 100644 index 000000000..0eab88e50 --- /dev/null +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Users/UserAvatarModel.cs @@ -0,0 +1,18 @@ +using System.Text.Json.Serialization; + +namespace BotSharp.OpenAPI.ViewModels.Users; + +public class UserAvatarModel +{ + /// + /// File name with extension + /// + [JsonPropertyName("file_name")] + public string FileName { get; set; } = string.Empty; + + /// + /// File data, e.g., "data:image/png;base64,aaaaaaaa" + /// + [JsonPropertyName("file_data")] + public string FileData { get; set; } = string.Empty; +} diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs index 759f94b60..7a5d8e725 100644 --- a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs +++ b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs @@ -99,9 +99,9 @@ private void SaveGeneratedImage(ImageGeneration? image) { if (image == null) return; - var files = new List() + var files = new List() { - new BotSharpFile + new InputFileModel { FileName = $"{Guid.NewGuid()}.png", FileData = $"data:{MediaTypeNames.Image.Png};base64,{image.ImageData}" diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/GenerateImageFn.cs b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/GenerateImageFn.cs index 4102ff7b2..4c8fdf9ef 100644 --- a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/GenerateImageFn.cs +++ b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/GenerateImageFn.cs @@ -77,7 +77,7 @@ private void SaveGeneratedImages(List? images) { if (images.IsNullOrEmpty()) return; - var files = images.Where(x => !string.IsNullOrEmpty(x?.ImageData)).Select(x => new BotSharpFile + var files = images.Where(x => !string.IsNullOrEmpty(x?.ImageData)).Select(x => new InputFileModel { FileName = $"{Guid.NewGuid()}.png", FileData = $"data:{MediaTypeNames.Image.Png};base64,{x.ImageData}" diff --git a/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Conversation.cs b/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Conversation.cs index 7e2462de4..4c339f127 100644 --- a/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Conversation.cs +++ b/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Conversation.cs @@ -115,7 +115,7 @@ public IEnumerable GetMessagesWithFile(string conversationId, return foundMsgs; } - public bool SaveMessageFiles(string conversationId, string messageId, string source, List files) + public bool SaveMessageFiles(string conversationId, string messageId, string source, List files) { if (files.IsNullOrEmpty()) return false; diff --git a/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.User.cs b/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.User.cs index de222c439..a23085396 100644 --- a/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.User.cs +++ b/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.User.cs @@ -16,7 +16,7 @@ public string GetUserAvatar() return found; } - public bool SaveUserAvatar(BotSharpFile file) + public bool SaveUserAvatar(InputFileModel file) { if (file == null || string.IsNullOrEmpty(file.FileData)) return false; From 35eb7dabd6e6d5ae7b570dfce2d071bae30c7364 Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Wed, 28 Aug 2024 11:08:12 -0500 Subject: [PATCH 04/27] refine api --- BotSharp.sln | 13 +- .../Files/IFileInstructService.cs | 2 +- .../Files/IFileStorageService.cs | 1 + .../Files/Models/BotSharpFile.cs | 2 +- .../Files/Utilities/FileUtility.cs | 21 ++ .../MLTasks/IAudioCompletion.cs | 13 ++ .../MLTasks/ISpeechToText.cs | 12 - .../MLTasks/ITextToSpeech.cs | 23 -- .../Instruct/FileInstructService.Audio.cs | 6 +- .../Storage/LocalFileStorageService.Common.cs | 16 ++ .../Infrastructures/CompletionProvider.cs | 28 +-- .../Controllers/InstructModeController.cs | 209 +++++++++++++++++- .../Instructs/AudioCompletionViewModel.cs | 5 - .../Instructs/InstructBaseRequest.cs | 5 +- .../Instructs/MultiModalViewModel.cs | 5 + .../Instructs/SpeechToTextViewModel.cs | 5 + .../AudioHandlerPlugin.cs | 3 +- .../BotSharp.Plugin.AudioHandler.csproj | 14 ++ .../Controllers/AudioController.cs | 58 ----- .../Functions/HandleAudioRequestFn.cs | 62 +++--- .../Helpers/AudioHelper.cs | 89 ++++---- .../Helpers/IAudioHelper.cs | 6 - .../Hooks/AudioHandlerHook.cs | 13 +- .../Provider/NativeWhisperProvider.cs | 103 ++++----- .../BotSharp.Plugin.AudioHandler/Using.cs | 1 + .../functions/handle_audio_request.json | 32 +-- .../Functions/HandleEmailSenderFn.cs | 3 +- .../Hooks/EmailReaderHook.cs | 7 - .../Hooks/EmailSenderHook.cs | 7 - src/Plugins/BotSharp.Plugin.Graph/GraphDb.cs | 2 +- .../BotSharp.Plugin.KnowledgeBase.csproj | 2 +- .../Functions/KnowledgeRetrievalFn.cs | 2 +- .../Functions/MemorizeKnowledgeFn.cs | 2 +- .../KnowledgeSettingHelper.cs} | 4 +- .../VectorHelper.cs} | 4 +- .../KnowledgeBasePlugin.cs | 2 +- .../MemVecDb/MemoryVectorDb.cs | 2 +- .../Services/KnowledgeService.cs | 2 +- .../BotSharp.Plugin.KnowledgeBase/Using.cs | 2 +- .../BotSharp.Plugin.OpenAI/OpenAiPlugin.cs | 5 +- .../AudioCompletionProvider.SpeechToText.cs | 53 +++++ .../AudioCompletionProvider.TextToSpeech.cs | 23 ++ .../Audio/AudioCompletionProvider.cs | 21 ++ .../Providers/Audio/SpeechToTextProvider.cs | 50 ----- .../Providers/Audio/TextToSpeechProvider.cs | 31 --- .../Services/TencentCosService.Common.cs | 22 +- .../Controllers/TwilioVoiceController.cs | 8 +- .../Services/TwilioMessageQueueService.cs | 9 +- src/WebStarter/appsettings.json | 1 + 49 files changed, 590 insertions(+), 421 deletions(-) create mode 100644 src/Infrastructure/BotSharp.Abstraction/MLTasks/IAudioCompletion.cs delete mode 100644 src/Infrastructure/BotSharp.Abstraction/MLTasks/ISpeechToText.cs delete mode 100644 src/Infrastructure/BotSharp.Abstraction/MLTasks/ITextToSpeech.cs delete mode 100644 src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/AudioCompletionViewModel.cs create mode 100644 src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/MultiModalViewModel.cs create mode 100644 src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/SpeechToTextViewModel.cs delete mode 100644 src/Plugins/BotSharp.Plugin.AudioHandler/Controllers/AudioController.cs delete mode 100644 src/Plugins/BotSharp.Plugin.AudioHandler/Helpers/IAudioHelper.cs rename src/Plugins/BotSharp.Plugin.KnowledgeBase/{Utilities/KnowledgeSettingUtility.cs => Helpers/KnowledgeSettingHelper.cs} (87%) rename src/Plugins/BotSharp.Plugin.KnowledgeBase/{Utilities/VectorUtility.cs => Helpers/VectorHelper.cs} (96%) create mode 100644 src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/AudioCompletionProvider.SpeechToText.cs create mode 100644 src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/AudioCompletionProvider.TextToSpeech.cs create mode 100644 src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/AudioCompletionProvider.cs delete mode 100644 src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/SpeechToTextProvider.cs delete mode 100644 src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/TextToSpeechProvider.cs diff --git a/BotSharp.sln b/BotSharp.sln index 8dda48e83..d281450e8 100644 --- a/BotSharp.sln +++ b/BotSharp.sln @@ -111,7 +111,9 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "BotSharp.Plugin.PythonInter EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Graph", "Graph", "{97A0B191-64D7-4F8A-BFE8-1BFCC5E247E1}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BotSharp.Plugin.Graph", "src\Plugins\BotSharp.Plugin.Graph\BotSharp.Plugin.Graph.csproj", "{EBFE97DA-D0BA-48BA-8B5D-083B60348D1D}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "BotSharp.Plugin.Graph", "src\Plugins\BotSharp.Plugin.Graph\BotSharp.Plugin.Graph.csproj", "{EBFE97DA-D0BA-48BA-8B5D-083B60348D1D}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "BotSharp.Plugin.AudioHandler", "src\Plugins\BotSharp.Plugin.AudioHandler\BotSharp.Plugin.AudioHandler.csproj", "{F57F4862-F8D4-44A1-AC12-5C131B5C9785}" EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution @@ -449,6 +451,14 @@ Global {EBFE97DA-D0BA-48BA-8B5D-083B60348D1D}.Release|Any CPU.Build.0 = Release|Any CPU {EBFE97DA-D0BA-48BA-8B5D-083B60348D1D}.Release|x64.ActiveCfg = Release|Any CPU {EBFE97DA-D0BA-48BA-8B5D-083B60348D1D}.Release|x64.Build.0 = Release|Any CPU + {F57F4862-F8D4-44A1-AC12-5C131B5C9785}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {F57F4862-F8D4-44A1-AC12-5C131B5C9785}.Debug|Any CPU.Build.0 = Debug|Any CPU + {F57F4862-F8D4-44A1-AC12-5C131B5C9785}.Debug|x64.ActiveCfg = Debug|Any CPU + {F57F4862-F8D4-44A1-AC12-5C131B5C9785}.Debug|x64.Build.0 = Debug|Any CPU + {F57F4862-F8D4-44A1-AC12-5C131B5C9785}.Release|Any CPU.ActiveCfg = Release|Any CPU + {F57F4862-F8D4-44A1-AC12-5C131B5C9785}.Release|Any CPU.Build.0 = Release|Any CPU + {F57F4862-F8D4-44A1-AC12-5C131B5C9785}.Release|x64.ActiveCfg = Release|Any CPU + {F57F4862-F8D4-44A1-AC12-5C131B5C9785}.Release|x64.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -503,6 +513,7 @@ Global {05E6E405-5021-406E-8A5E-0A7CEC881F6D} = {C4C59872-3C8A-450D-83D5-2BE402D610D5} {97A0B191-64D7-4F8A-BFE8-1BFCC5E247E1} = {2635EC9B-2E5F-4313-AC21-0B847F31F36C} {EBFE97DA-D0BA-48BA-8B5D-083B60348D1D} = {97A0B191-64D7-4F8A-BFE8-1BFCC5E247E1} + {F57F4862-F8D4-44A1-AC12-5C131B5C9785} = {51AFE054-AE99-497D-A593-69BAEFB5106F} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {A9969D89-C98B-40A5-A12B-FC87E55B3A19} diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/IFileInstructService.cs b/src/Infrastructure/BotSharp.Abstraction/Files/IFileInstructService.cs index f35cdad5f..f8f4c605a 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Files/IFileInstructService.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Files/IFileInstructService.cs @@ -21,7 +21,7 @@ public interface IFileInstructService #endregion #region Audio - Task ReadAudio(string? provider, string? model, InstructFileModel audio); + Task SpeechToText(string? provider, string? model, InstructFileModel audio, string? text = null); #endregion #region Select file diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/IFileStorageService.cs b/src/Infrastructure/BotSharp.Abstraction/Files/IFileStorageService.cs index 6c98e21a8..ebd7d61f6 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Files/IFileStorageService.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Files/IFileStorageService.cs @@ -6,6 +6,7 @@ public interface IFileStorageService { #region Common string GetDirectory(string conversationId); + IEnumerable GetFiles(string relativePath, string? searchQuery = null); byte[] GetFileBytes(string fileStorageUrl); bool SaveFileStreamToPath(string filePath, Stream stream); bool SaveFileBytesToPath(string filePath, byte[] bytes); diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/Models/BotSharpFile.cs b/src/Infrastructure/BotSharp.Abstraction/Files/Models/BotSharpFile.cs index 46cba5959..7a606b3d3 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Files/Models/BotSharpFile.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Files/Models/BotSharpFile.cs @@ -4,7 +4,7 @@ namespace BotSharp.Abstraction.Files.Models; public class BotSharpFile : FileInfo { /// - /// File data, e.g., "data:image/png;base64,aaaaaaaa" + /// File data => format: "data:image/png;base64,aaaaaaaa" /// [JsonPropertyName("file_data")] [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/Utilities/FileUtility.cs b/src/Infrastructure/BotSharp.Abstraction/Files/Utilities/FileUtility.cs index df33906df..5c021159a 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Files/Utilities/FileUtility.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Files/Utilities/FileUtility.cs @@ -1,4 +1,6 @@ +using Microsoft.AspNetCore.Http; using Microsoft.AspNetCore.StaticFiles; +using System.IO; namespace BotSharp.Abstraction.Files.Utilities; @@ -26,6 +28,25 @@ public static (string, byte[]) GetFileInfoFromData(string data) return (contentType, Convert.FromBase64String(base64Str)); } + public static string BuildFileDataFromFile(string fileName, byte[] bytes) + { + var contentType = GetFileContentType(fileName); + var base64 = Convert.ToBase64String(bytes); + return $"data:{contentType};base64,{base64}"; + } + + public static string BuildFileDataFromFile(IFormFile file) + { + using var stream = new MemoryStream(); + file.CopyTo(stream); + stream.Position = 0; + var contentType = GetFileContentType(file.FileName); + var base64 = Convert.ToBase64String(stream.ToArray()); + stream.Close(); + + return $"data:{contentType};base64,{base64}"; + } + public static string GetFileContentType(string filePath) { string contentType; diff --git a/src/Infrastructure/BotSharp.Abstraction/MLTasks/IAudioCompletion.cs b/src/Infrastructure/BotSharp.Abstraction/MLTasks/IAudioCompletion.cs new file mode 100644 index 000000000..d32624c0e --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/MLTasks/IAudioCompletion.cs @@ -0,0 +1,13 @@ +using System.IO; + +namespace BotSharp.Abstraction.MLTasks; + +public interface IAudioCompletion +{ + string Provider { get; } + + Task GenerateTextFromAudioAsync(Stream audio, string audioFileName, string? text = null); + Task GenerateSpeechFromTextAsync(string text); + + void SetModelName(string model); +} diff --git a/src/Infrastructure/BotSharp.Abstraction/MLTasks/ISpeechToText.cs b/src/Infrastructure/BotSharp.Abstraction/MLTasks/ISpeechToText.cs deleted file mode 100644 index a1af443e0..000000000 --- a/src/Infrastructure/BotSharp.Abstraction/MLTasks/ISpeechToText.cs +++ /dev/null @@ -1,12 +0,0 @@ -using System.IO; - -namespace BotSharp.Abstraction.MLTasks; - -public interface ISpeechToText -{ - string Provider { get; } - - Task GenerateTextFromAudioAsync(string filePath); - Task GenerateTextFromAudioAsync(Stream audio, string audioFileName); - Task SetModelName(string model); -} diff --git a/src/Infrastructure/BotSharp.Abstraction/MLTasks/ITextToSpeech.cs b/src/Infrastructure/BotSharp.Abstraction/MLTasks/ITextToSpeech.cs deleted file mode 100644 index 344fad0ea..000000000 --- a/src/Infrastructure/BotSharp.Abstraction/MLTasks/ITextToSpeech.cs +++ /dev/null @@ -1,23 +0,0 @@ -namespace BotSharp.Abstraction.MLTasks -{ - public interface ITextToSpeech - { - /// - /// The LLM provider like Microsoft Azure, OpenAI, ClaudAI - /// - string Provider { get; } - - /// - /// Set model name, one provider can consume different model or version(s) - /// - /// deployment name - void SetModelName(string model); - - Task GenerateSpeechFromTextAsync(string text, ITextToSpeechOptions? options = null); - } - - public interface ITextToSpeechOptions - { - - } -} diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Audio.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Audio.cs index 30b6dbe30..e2aa844cd 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Audio.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Audio.cs @@ -4,16 +4,16 @@ namespace BotSharp.Core.Files.Services; public partial class FileInstructService { - public async Task ReadAudio(string? provider, string? model, InstructFileModel audio) + public async Task SpeechToText(string? provider, string? model, InstructFileModel audio, string? text = null) { - var completion = CompletionProvider.GetSpeechToText(_services, provider: provider ?? "openai", model: model ?? "whisper-1"); + var completion = CompletionProvider.GetAudioCompletion(_services, provider: provider ?? "openai", model: model ?? "whisper-1"); var audioBytes = await DownloadFile(audio); using var stream = new MemoryStream(); stream.Write(audioBytes, 0, audioBytes.Length); stream.Position = 0; var fileName = $"{audio.FileName ?? "audio"}.{audio.FileExtension ?? "wav"}"; - var content = await completion.GenerateTextFromAudioAsync(stream, fileName); + var content = await completion.GenerateTextFromAudioAsync(stream, fileName, text); stream.Close(); return content; } diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Common.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Common.cs index c49edbce6..b16c2c2cf 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Common.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Common.cs @@ -14,6 +14,22 @@ public string GetDirectory(string conversationId) return dir; } + public IEnumerable GetFiles(string relativePath, string? searchPattern = null) + { + if (string.IsNullOrWhiteSpace(relativePath)) + { + return Enumerable.Empty(); + } + + var path = Path.Combine(_baseDir, relativePath); + + if (!string.IsNullOrWhiteSpace(searchPattern)) + { + return Directory.GetFiles(path, searchPattern); + } + return Directory.GetFiles(path); + } + public byte[] GetFileBytes(string fileStorageUrl) { using var stream = File.OpenRead(fileStorageUrl); diff --git a/src/Infrastructure/BotSharp.Core/Infrastructures/CompletionProvider.cs b/src/Infrastructure/BotSharp.Core/Infrastructures/CompletionProvider.cs index 874a063e1..d6746e09a 100644 --- a/src/Infrastructure/BotSharp.Core/Infrastructures/CompletionProvider.cs +++ b/src/Infrastructure/BotSharp.Core/Infrastructures/CompletionProvider.cs @@ -28,6 +28,10 @@ public static object GetCompletion(IServiceProvider services, { return GetImageCompletion(services, provider: provider, model: model); } + else if (settings.Type == LlmModelType.Audio) + { + return GetAudioCompletion(services, provider: provider, model: model); + } else { return GetChatCompletion(services, provider: provider, model: model, agentConfig: agentConfig); @@ -108,7 +112,7 @@ public static ITextEmbedding GetTextEmbedding(IServiceProvider services, if (completer == null) { var logger = services.GetRequiredService>(); - logger.LogError($"Can't resolve completion provider by {provider}"); + logger.LogError($"Can't resolve text-embedding provider by {provider}"); } @@ -120,35 +124,19 @@ public static ITextEmbedding GetTextEmbedding(IServiceProvider services, return completer; } - public static ITextToSpeech GetTextToSpeech( + public static IAudioCompletion GetAudioCompletion( IServiceProvider services, string provider, string model) { - var completions = services.GetServices(); + var completions = services.GetServices(); var completer = completions.FirstOrDefault(x => x.Provider == provider); if (completer == null) { var logger = services.GetRequiredService>(); - logger.LogError($"Can't resolve text2speech provider by {provider}"); + logger.LogError($"Can't resolve audio-completion provider by {provider}"); } - completer.SetModelName(model); - return completer; - } - public static ISpeechToText GetSpeechToText( - IServiceProvider services, - string provider, - string model - ) - { - var completions = services.GetServices(); - var completer = completions.FirstOrDefault(x => x.Provider == provider); - if (completer == null) - { - var logger = services.GetRequiredService>(); - logger.LogError($"Can't resolve speech2text provider by {provider}"); - } completer.SetModelName(model); return completer; } diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs index b0ecd9056..170cf2d20 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs @@ -1,4 +1,5 @@ using BotSharp.Abstraction.Agents.Models; +using BotSharp.Abstraction.Files.Utilities; using BotSharp.Abstraction.Instructs; using BotSharp.Abstraction.Instructs.Models; using BotSharp.Core.Infrastructures; @@ -91,11 +92,40 @@ public async Task MultiModalCompletion([FromBody] MultiModalRequest inpu } catch (Exception ex) { - var error = $"Error in analyzing files. {ex.Message}"; + var error = $"Error in reading images. {ex.Message}"; _logger.LogError(error); return error; } } + + [HttpPost("/instruct/multi-modal/upload")] + public async Task MultiModalCompletion(IFormFile file, [FromForm] string text, [FromForm] string? provider = null, + [FromForm] string? model = null, [FromForm] List? states = null) + { + var state = _services.GetRequiredService(); + states?.ForEach(x => state.SetState(x.Key, x.Value, activeRounds: x.ActiveRounds, source: StateSource.External)); + var viewModel = new MultiModalViewModel(); + + try + { + var data = FileUtility.BuildFileDataFromFile(file); + var files = new List + { + new InstructFileModel { FileData = data } + }; + var fileInstruct = _services.GetRequiredService(); + var content = await fileInstruct.ReadImages(provider, model, text, files); + viewModel.Content = content; + return viewModel; + } + catch (Exception ex) + { + var error = $"Error in reading image upload. {ex.Message}"; + _logger.LogError(error); + viewModel.Message = error; + return viewModel; + } + } #endregion #region Generate image @@ -154,6 +184,38 @@ public async Task ImageVariation([FromBody] ImageVaria } } + [HttpPost("/instruct/image-variation/upload")] + public async Task ImageVariation(IFormFile file, [FromForm] string? provider = null, + [FromForm] string? model = null, [FromForm] List? states = null) + { + var state = _services.GetRequiredService(); + states?.ForEach(x => state.SetState(x.Key, x.Value, activeRounds: x.ActiveRounds, source: StateSource.External)); + var imageViewModel = new ImageGenerationViewModel(); + + try + { + var data = FileUtility.BuildFileDataFromFile(file); + var image = new InstructFileModel + { + FileName = Path.GetFileNameWithoutExtension(file.FileName), + FileExtension = Path.GetExtension(file.FileName).Substring(1), + FileData = data + }; + var fileInstruct = _services.GetRequiredService(); + var message = await fileInstruct.VaryImage(provider, model, image); + imageViewModel.Content = message.Content; + imageViewModel.Images = message.GeneratedImages.Select(x => ImageViewModel.ToViewModel(x)).ToList(); + return imageViewModel; + } + catch (Exception ex) + { + var error = $"Error in image variation upload. {ex.Message}"; + _logger.LogError(error); + imageViewModel.Message = error; + return imageViewModel; + } + } + [HttpPost("/instruct/image-edit")] public async Task ImageEdit([FromBody] ImageEditRequest input) { @@ -182,6 +244,38 @@ public async Task ImageEdit([FromBody] ImageEditReques } } + [HttpPost("/instruct/image-edit/upload")] + public async Task ImageEdit(IFormFile file, [FromForm] string text, [FromForm] string? provider = null, + [FromForm] string? model = null, [FromForm] List? states = null) + { + var fileInstruct = _services.GetRequiredService(); + var state = _services.GetRequiredService(); + states?.ForEach(x => state.SetState(x.Key, x.Value, activeRounds: x.ActiveRounds, source: StateSource.External)); + var imageViewModel = new ImageGenerationViewModel(); + + try + { + var data = FileUtility.BuildFileDataFromFile(file); + var image = new InstructFileModel + { + FileName = Path.GetFileNameWithoutExtension(file.FileName), + FileExtension = Path.GetExtension(file.FileName).Substring(1), + FileData = data + }; + var message = await fileInstruct.EditImage(provider, model, text, image); + imageViewModel.Content = message.Content; + imageViewModel.Images = message.GeneratedImages.Select(x => ImageViewModel.ToViewModel(x)).ToList(); + return imageViewModel; + } + catch (Exception ex) + { + var error = $"Error in image edit upload. {ex.Message}"; + _logger.LogError(error); + imageViewModel.Message = error; + return imageViewModel; + } + } + [HttpPost("/instruct/image-mask-edit")] public async Task ImageMaskEdit([FromBody] ImageMaskEditRequest input) { @@ -211,6 +305,47 @@ public async Task ImageMaskEdit([FromBody] ImageMaskEd return imageViewModel; } } + + [HttpPost("/instruct/image-mask-edit/upload")] + public async Task ImageMaskEdit(IFormFile image, IFormFile mask, [FromForm] string text, [FromForm] string? provider = null, + [FromForm] string? model = null, [FromForm] List? states = null) + { + var fileInstruct = _services.GetRequiredService(); + var state = _services.GetRequiredService(); + states?.ForEach(x => state.SetState(x.Key, x.Value, activeRounds: x.ActiveRounds, source: StateSource.External)); + var imageViewModel = new ImageGenerationViewModel(); + + try + { + var imageData = FileUtility.BuildFileDataFromFile(image); + var imageFile = new InstructFileModel + { + FileName = Path.GetFileNameWithoutExtension(image.FileName), + FileExtension = Path.GetExtension(image.FileName).Substring(1), + FileData = imageData + }; + + var maskData = FileUtility.BuildFileDataFromFile(mask); + var maskFile = new InstructFileModel + { + FileName = Path.GetFileNameWithoutExtension(mask.FileName), + FileExtension = Path.GetExtension(mask.FileName).Substring(1), + FileData = maskData + }; + + var message = await fileInstruct.EditImage(provider, model, text, imageFile, maskFile); + imageViewModel.Content = message.Content; + imageViewModel.Images = message.GeneratedImages.Select(x => ImageViewModel.ToViewModel(x)).ToList(); + return imageViewModel; + } + catch (Exception ex) + { + var error = $"Error in image mask edit upload. {ex.Message}"; + _logger.LogError(error); + imageViewModel.Message = error; + return imageViewModel; + } + } #endregion #region Pdf @@ -236,31 +371,91 @@ public async Task PdfCompletion([FromBody] MultiModalReq return viewModel; } } + + [HttpPost("/instruct/pdf-completion/upload")] + public async Task PdfCompletion(IFormFile file, [FromForm] string text, [FromForm] string? provider = null, + [FromForm] string? model = null, [FromForm] string? modelId = null, [FromForm] List? states = null) + { + var state = _services.GetRequiredService(); + states?.ForEach(x => state.SetState(x.Key, x.Value, activeRounds: x.ActiveRounds, source: StateSource.External)); + var viewModel = new PdfCompletionViewModel(); + + try + { + var data = FileUtility.BuildFileDataFromFile(file); + var files = new List + { + new InstructFileModel { FileData = data } + }; + + var fileInstruct = _services.GetRequiredService(); + var content = await fileInstruct.ReadPdf(provider, model, modelId, text, files); + viewModel.Content = content; + return viewModel; + } + catch (Exception ex) + { + var error = $"Error in pdf completion upload. {ex.Message}"; + _logger.LogError(error); + viewModel.Message = error; + return viewModel; + } + } #endregion #region Audio - [HttpPost("/instruct/audio-completion")] - public async Task AudioCompletion([FromBody] AudioCompletionRequest input) + [HttpPost("/instruct/speech-to-text")] + public async Task SpeechToText([FromBody] SpeechToTextRequest input) { var fileInstruct = _services.GetRequiredService(); var state = _services.GetRequiredService(); input.States.ForEach(x => state.SetState(x.Key, x.Value, activeRounds: x.ActiveRounds, source: StateSource.External)); - var viewModel = new AudioCompletionViewModel(); + var viewModel = new SpeechToTextViewModel(); try { var audio = input.File; if (audio == null) { - return new AudioCompletionViewModel { Message = "Error! Cannot find a valid audio file!" }; + return new SpeechToTextViewModel { Message = "Error! Cannot find a valid audio file!" }; } - var content = await fileInstruct.ReadAudio(input.Provider, input.Model, audio); + var content = await fileInstruct.SpeechToText(input.Provider, input.Model, audio); + viewModel.Content = content; + return viewModel; + } + catch (Exception ex) + { + var error = $"Error in speech to text. {ex.Message}"; + _logger.LogError(error); + viewModel.Message = error; + return viewModel; + } + } + + [HttpPost("/instruct/speech-to-text/upload")] + public async Task SpeechToText(IFormFile file, [FromForm] string? provider = null, [FromForm] string? model = null, + [FromForm] string? text = null, [FromForm] List? states = null) + { + var fileInstruct = _services.GetRequiredService(); + var state = _services.GetRequiredService(); + states?.ForEach(x => state.SetState(x.Key, x.Value, activeRounds: x.ActiveRounds, source: StateSource.External)); + var viewModel = new SpeechToTextViewModel(); + + try + { + using var stream = new MemoryStream(); + file.CopyTo(stream); + stream.Position = 0; + + var completion = CompletionProvider.GetAudioCompletion(_services, provider: provider ?? "openai", model: model ?? "whisper-1"); + var content = await completion.GenerateTextFromAudioAsync(stream, file.FileName, text); viewModel.Content = content; + stream.Close(); return viewModel; } catch (Exception ex) { - var error = $"Error in audio completion. {ex.Message}"; + var error = $"Error in speech-to-text upload. {ex.Message}"; _logger.LogError(error); viewModel.Message = error; return viewModel; diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/AudioCompletionViewModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/AudioCompletionViewModel.cs deleted file mode 100644 index 3c3501aef..000000000 --- a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/AudioCompletionViewModel.cs +++ /dev/null @@ -1,5 +0,0 @@ -namespace BotSharp.OpenAPI.ViewModels.Instructs; - -public class AudioCompletionViewModel : InstructBaseViewModel -{ -} diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/InstructBaseRequest.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/InstructBaseRequest.cs index 45a1c8ab6..396fff2f5 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/InstructBaseRequest.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/InstructBaseRequest.cs @@ -59,8 +59,11 @@ public class ImageMaskEditRequest : InstructBaseRequest public InstructFileModel Mask { get; set; } } -public class AudioCompletionRequest : InstructBaseRequest +public class SpeechToTextRequest : InstructBaseRequest { + [JsonPropertyName("text")] + public string? Text { get; set; } + [JsonPropertyName("file")] public InstructFileModel File { get; set; } } \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/MultiModalViewModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/MultiModalViewModel.cs new file mode 100644 index 000000000..a7b15262d --- /dev/null +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/MultiModalViewModel.cs @@ -0,0 +1,5 @@ +namespace BotSharp.OpenAPI.ViewModels.Instructs; + +public class MultiModalViewModel : InstructBaseViewModel +{ +} diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/SpeechToTextViewModel.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/SpeechToTextViewModel.cs new file mode 100644 index 000000000..d681986d3 --- /dev/null +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/SpeechToTextViewModel.cs @@ -0,0 +1,5 @@ +namespace BotSharp.OpenAPI.ViewModels.Instructs; + +public class SpeechToTextViewModel : InstructBaseViewModel +{ +} diff --git a/src/Plugins/BotSharp.Plugin.AudioHandler/AudioHandlerPlugin.cs b/src/Plugins/BotSharp.Plugin.AudioHandler/AudioHandlerPlugin.cs index 855c00810..2c289907e 100644 --- a/src/Plugins/BotSharp.Plugin.AudioHandler/AudioHandlerPlugin.cs +++ b/src/Plugins/BotSharp.Plugin.AudioHandler/AudioHandlerPlugin.cs @@ -16,8 +16,7 @@ public void RegisterDI(IServiceCollection services, IConfiguration config) return settingService.Bind("AudioHandler"); }); - services.AddScoped(); - services.AddScoped(); + services.AddScoped(); services.AddScoped(); services.AddScoped(); } diff --git a/src/Plugins/BotSharp.Plugin.AudioHandler/BotSharp.Plugin.AudioHandler.csproj b/src/Plugins/BotSharp.Plugin.AudioHandler/BotSharp.Plugin.AudioHandler.csproj index 7218d40c1..4ae8c7d27 100644 --- a/src/Plugins/BotSharp.Plugin.AudioHandler/BotSharp.Plugin.AudioHandler.csproj +++ b/src/Plugins/BotSharp.Plugin.AudioHandler/BotSharp.Plugin.AudioHandler.csproj @@ -22,5 +22,19 @@ + + + + + + + + + PreserveNewest + + + PreserveNewest + + diff --git a/src/Plugins/BotSharp.Plugin.AudioHandler/Controllers/AudioController.cs b/src/Plugins/BotSharp.Plugin.AudioHandler/Controllers/AudioController.cs deleted file mode 100644 index 60b2dc15c..000000000 --- a/src/Plugins/BotSharp.Plugin.AudioHandler/Controllers/AudioController.cs +++ /dev/null @@ -1,58 +0,0 @@ -using System.Diagnostics; -using BotSharp.Core.Infrastructures; - -namespace BotSharp.Plugin.AudioHandler.Controllers -{ -#if DEBUG - [AllowAnonymous] -#endif - [ApiController] - public class AudioController : ControllerBase - { - private readonly ISpeechToText _nativeWhisperProvider; - private readonly IServiceProvider _services; - - public AudioController(ISpeechToText nativeWhisperProvider, IServiceProvider service) - { - _nativeWhisperProvider = nativeWhisperProvider; - _services = service; - } - - [HttpGet("audio/transcript")] - public async Task GetTextFromAudioController(string audioInputString, string modelType = "") - { -#if DEBUG - Stopwatch stopWatch = new Stopwatch(); - stopWatch.Start(); -#endif - await _nativeWhisperProvider.SetModelName(modelType); - - var result = await _nativeWhisperProvider.GenerateTextFromAudioAsync(audioInputString); -#if DEBUG - stopWatch.Stop(); - TimeSpan ts = stopWatch.Elapsed; - string elapsedTime = String.Format("{0:00}:{1:00}:{2:00}.{3:00}", ts.Hours, ts.Minutes, ts.Seconds, ts.Milliseconds / 10); - Console.WriteLine("RunTime " + elapsedTime); -#endif - return Ok(result); - } - - [HttpPost("openai/audio/transcript")] - public async Task GetTextFromAudioOpenAiController(string filePath) - { -#if DEBUG - Stopwatch stopWatch = new Stopwatch(); - stopWatch.Start(); -#endif - var client = CompletionProvider.GetSpeechToText(_services, "openai", "whisper-1"); - var result = await client.GenerateTextFromAudioAsync(filePath); -#if DEBUG - stopWatch.Stop(); - TimeSpan ts = stopWatch.Elapsed; - string elapsedTime = String.Format("{0:00}:{1:00}:{2:00}.{3:00}", ts.Hours, ts.Minutes, ts.Seconds, ts.Milliseconds / 10); - Console.WriteLine("RunTime " + elapsedTime); -#endif - return Ok(result); - } - } -} diff --git a/src/Plugins/BotSharp.Plugin.AudioHandler/Functions/HandleAudioRequestFn.cs b/src/Plugins/BotSharp.Plugin.AudioHandler/Functions/HandleAudioRequestFn.cs index 37016220c..34ec530c6 100644 --- a/src/Plugins/BotSharp.Plugin.AudioHandler/Functions/HandleAudioRequestFn.cs +++ b/src/Plugins/BotSharp.Plugin.AudioHandler/Functions/HandleAudioRequestFn.cs @@ -9,6 +9,7 @@ public class HandleAudioRequestFn : IFunctionCallback public string Indication => "Handling audio request"; private readonly IServiceProvider _serviceProvider; + private readonly IFileStorageService _fileStorage; private readonly ILogger _logger; private readonly BotSharpOptions _options; @@ -19,10 +20,12 @@ public class HandleAudioRequestFn : IFunctionCallback }; public HandleAudioRequestFn( + IFileStorageService fileStorage, IServiceProvider serviceProvider, ILogger logger, BotSharpOptions options) { + _fileStorage = fileStorage; _serviceProvider = serviceProvider; _logger = logger; _options = options; @@ -43,11 +46,13 @@ public async Task Execute(RoleDialogModel message) private List AssembleFiles(string convId, List dialogs) { - if (dialogs.IsNullOrEmpty()) return new List(); + if (dialogs.IsNullOrEmpty()) + { + return new List(); + } - var fileService = _serviceProvider.GetRequiredService(); var messageId = dialogs.Select(x => x.MessageId).Distinct().ToList(); - var audioMessageFiles = fileService.GetMessageFiles(convId, messageId, FileSourceType.User, _audioContentType); + var audioMessageFiles = _fileStorage.GetMessageFiles(convId, messageId, FileSourceType.User, _audioContentType); audioMessageFiles = audioMessageFiles.Where(x => x.ContentType.Contains("audio")).ToList(); @@ -69,53 +74,46 @@ private List AssembleFiles(string convId, List private async Task GetResponeFromDialogs(List dialogs) { - var speech2Text = await PrepareModel("native"); + var audioCompletion = PrepareModel(); var dialog = dialogs.Where(x => !x.Files.IsNullOrEmpty()).Last(); - int transcribedCount = 0; + var transcripts = new List(); foreach (var file in dialog.Files) { - if (file == null) continue; + if (file == null || string.IsNullOrWhiteSpace(file.FileStorageUrl)) continue; - string extension = Path.GetExtension(file?.FileStorageUrl); - if (ParseAudioFileType(extension) && File.Exists(file.FileStorageUrl)) - { - file.FileData = await speech2Text.GenerateTextFromAudioAsync(file.FileStorageUrl); - transcribedCount++; - } + var extension = Path.GetExtension(file.FileStorageUrl); + + var fileName = Path.GetFileName(file.FileStorageUrl); + if (!ParseAudioFileType(fileName)) continue; + + var bytes = _fileStorage.GetFileBytes(file.FileStorageUrl); + using var stream = new MemoryStream(bytes); + stream.Position = 0; + + var result = await audioCompletion.GenerateTextFromAudioAsync(stream, fileName); + transcripts.Add(result); + stream.Close(); } - if (transcribedCount == 0) + if (transcripts.IsNullOrEmpty()) { throw new FileNotFoundException($"No audio files found in the dialog. MessageId: {dialog.MessageId}"); } - var resList = dialog.Files.Select(x => $"{x.FileName} \r\n {x.FileData}").ToList(); - return string.Join("\n\r", resList); + return string.Join("\r\n\r\n", transcripts); } - private async Task PrepareModel(string provider = "native") + private IAudioCompletion PrepareModel() { - var speech2Text = _serviceProvider.GetServices().FirstOrDefault(x => x.Provider == provider.ToLower()); - if (speech2Text == null) - { - throw new Exception($"Can't resolve speech2text provider by {provider}"); - } - - if (provider.IsEqualTo("openai")) - { - return CompletionProvider.GetSpeechToText(_serviceProvider, provider: "openai", model: "whisper-1"); - } - - await speech2Text.SetModelName("Tiny"); - return speech2Text; + return CompletionProvider.GetAudioCompletion(_serviceProvider, provider: "openai", model: "whisper-1"); } - private bool ParseAudioFileType(string fileType) + private bool ParseAudioFileType(string fileName) { - fileType = fileType.ToLower(); + var extension = Path.GetExtension(fileName).TrimStart('.').ToLower(); var provider = new FileExtensionContentTypeProvider(); - bool canParse = Enum.TryParse(fileType, out _) || provider.TryGetContentType(fileType, out _); + bool canParse = Enum.TryParse(extension, out _) || provider.TryGetContentType(fileName, out _); return canParse; } } diff --git a/src/Plugins/BotSharp.Plugin.AudioHandler/Helpers/AudioHelper.cs b/src/Plugins/BotSharp.Plugin.AudioHandler/Helpers/AudioHelper.cs index 122273c84..4737a5e9d 100644 --- a/src/Plugins/BotSharp.Plugin.AudioHandler/Helpers/AudioHelper.cs +++ b/src/Plugins/BotSharp.Plugin.AudioHandler/Helpers/AudioHelper.cs @@ -3,78 +3,83 @@ namespace BotSharp.Plugin.AudioHandler.Helpers; -public class AudioHelper : IAudioHelper +public static class AudioHelper { - private readonly IServiceProvider _services; - private readonly ILogger _logger; + private const int DEFAULT_SAMPLE_RATE = 16000; - public AudioHelper( - IServiceProvider services, - ILogger logger) - { - _services = services; - _logger = logger; - } - - public Stream ConvertToStream(string fileName) + public static Stream ConvertToStream(string fileName) { if (string.IsNullOrEmpty(fileName)) { - var error = "fileName is Null when converting to stream in audio processor"; - _logger.LogWarning(error); - throw new ArgumentNullException(error); + throw new ArgumentNullException("fileName is Null when converting to stream in audio processor"); } var fileExtension = Path.GetExtension(fileName).ToLower().TrimStart('.'); if (!Enum.TryParse(fileExtension, out AudioType fileType)) { - var error = $"File extension: '{fileExtension}' is not supported!"; - _logger.LogWarning(error); - throw new NotSupportedException(error); + throw new NotSupportedException($"File extension: '{fileExtension}' is not supported!"); } var stream = fileType switch { AudioType.mp3 => ConvertMp3ToStream(fileName), - AudioType.wav => ConvertWavToStream(fileName), - _ => throw new NotSupportedException("File extension not supported"), + _ => ConvertWavToStream(fileName) }; return stream; } - - private Stream ConvertMp3ToStream(string fileName) + public static Stream Transform(Stream stream, string fileName) { - var fileStream = File.OpenRead(fileName); - using var reader = new Mp3FileReader(fileStream); - if (reader.WaveFormat.SampleRate != 16000) + var fileExtension = Path.GetExtension(fileName).ToLower().TrimStart('.'); + if (!Enum.TryParse(fileExtension, out AudioType fileType)) { - var wavStream = new MemoryStream(); - var resampler = new WdlResamplingSampleProvider(reader.ToSampleProvider(), 16000); - WaveFileWriter.WriteWavFileToStream(wavStream, resampler.ToWaveProvider16()); - wavStream.Seek(0, SeekOrigin.Begin); - return wavStream; + throw new NotSupportedException($"File extension: '{fileExtension}' is not supported!"); } - fileStream.Seek(0, SeekOrigin.Begin); - return fileStream; + Stream resultStream = new MemoryStream(); + stream.CopyTo(resultStream); + resultStream.Seek(0, SeekOrigin.Begin); + + WaveStream reader = fileType switch + { + AudioType.mp3 => new Mp3FileReader(resultStream), + _ => new WaveFileReader(resultStream) + }; + + resultStream = ChangeSampleRate(reader); + reader.Close(); + return resultStream; + } + + private static Stream ConvertMp3ToStream(string fileName) + { + using var fileStream = File.OpenRead(fileName); + using var reader = new Mp3FileReader(fileStream); + return ChangeSampleRate(reader); } - private Stream ConvertWavToStream(string fileName) + private static Stream ConvertWavToStream(string fileName) { - var fileStream = File.OpenRead(fileName); + using var fileStream = File.OpenRead(fileName); using var reader = new WaveFileReader(fileStream); - if (reader.WaveFormat.SampleRate != 16000) + return ChangeSampleRate(reader); + } + + private static Stream ChangeSampleRate(WaveStream ws) + { + var ms = new MemoryStream(); + if (ws.WaveFormat.SampleRate != DEFAULT_SAMPLE_RATE) + { + var resampler = new WdlResamplingSampleProvider(ws.ToSampleProvider(), DEFAULT_SAMPLE_RATE); + WaveFileWriter.WriteWavFileToStream(ms, resampler.ToWaveProvider16()); + } + else { - var wavStream = new MemoryStream(); - var resampler = new WdlResamplingSampleProvider(reader.ToSampleProvider(), 16000); - WaveFileWriter.WriteWavFileToStream(wavStream, resampler.ToWaveProvider16()); - wavStream.Seek(0, SeekOrigin.Begin); - return wavStream; + ws.CopyTo(ms); } - fileStream.Seek(0, SeekOrigin.Begin); - return fileStream; + ms.Seek(0, SeekOrigin.Begin); + return ms; } } diff --git a/src/Plugins/BotSharp.Plugin.AudioHandler/Helpers/IAudioHelper.cs b/src/Plugins/BotSharp.Plugin.AudioHandler/Helpers/IAudioHelper.cs deleted file mode 100644 index d096a526e..000000000 --- a/src/Plugins/BotSharp.Plugin.AudioHandler/Helpers/IAudioHelper.cs +++ /dev/null @@ -1,6 +0,0 @@ -namespace BotSharp.Plugin.AudioHandler.Helpers; - -public interface IAudioHelper -{ - Stream ConvertToStream(string fileName); -} \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.AudioHandler/Hooks/AudioHandlerHook.cs b/src/Plugins/BotSharp.Plugin.AudioHandler/Hooks/AudioHandlerHook.cs index 51edcefa0..80acb149a 100644 --- a/src/Plugins/BotSharp.Plugin.AudioHandler/Hooks/AudioHandlerHook.cs +++ b/src/Plugins/BotSharp.Plugin.AudioHandler/Hooks/AudioHandlerHook.cs @@ -1,6 +1,5 @@ using BotSharp.Abstraction.Agents.Settings; using BotSharp.Abstraction.Functions.Models; -using BotSharp.Abstraction.Repositories; namespace BotSharp.Plugin.AudioHandler.Hooks; @@ -22,17 +21,16 @@ public override void OnAgentLoaded(Agent agent) if (isEnabled && isConvMode) { - AddUtility(agent, UtilityName.AudioHandler, HANDLER_AUDIO); + AddUtility(agent, HANDLER_AUDIO); } base.OnAgentLoaded(agent); } - private void AddUtility(Agent agent, string utility, string functionName) + private void AddUtility(Agent agent, string functionName) { - if (!IsEnableUtility(agent, utility)) return; - var (prompt, fn) = GetPromptAndFunction(functionName); + if (fn != null) { if (!string.IsNullOrWhiteSpace(prompt)) @@ -51,11 +49,6 @@ private void AddUtility(Agent agent, string utility, string functionName) } } - private bool IsEnableUtility(Agent agent, string utility) - { - return !agent.Utilities.IsNullOrEmpty() && agent.Utilities.Contains(utility); - } - private (string, FunctionDef?) GetPromptAndFunction(string functionName) { var db = _services.GetRequiredService(); diff --git a/src/Plugins/BotSharp.Plugin.AudioHandler/Provider/NativeWhisperProvider.cs b/src/Plugins/BotSharp.Plugin.AudioHandler/Provider/NativeWhisperProvider.cs index d95f02588..9bf049ac6 100644 --- a/src/Plugins/BotSharp.Plugin.AudioHandler/Provider/NativeWhisperProvider.cs +++ b/src/Plugins/BotSharp.Plugin.AudioHandler/Provider/NativeWhisperProvider.cs @@ -6,49 +6,39 @@ namespace BotSharp.Plugin.AudioHandler.Provider; /// /// Native Whisper provider for speech to text conversion /// -public class NativeWhisperProvider : ISpeechToText +public class NativeWhisperProvider : IAudioCompletion { - private readonly IAudioHelper _audioProcessor; private static WhisperProcessor _whisperProcessor; + + private readonly IServiceProvider _services; + private readonly IFileStorageService _fileStorage; private readonly ILogger _logger; public string Provider => "native"; - private string MODEL_DIR = "model"; - private string? _currentModelPath; - - private Dictionary _modelPathDict = new Dictionary(); - private GgmlType? _modelType; - public NativeWhisperProvider( - IAudioHelper audioProcessor, + BotSharpDatabaseSettings dbSettings, + IFileStorageService fileStorage, + IServiceProvider services, ILogger logger) { - _audioProcessor = audioProcessor; + _fileStorage = fileStorage; + _services = services; _logger = logger; } - public async Task GenerateTextFromAudioAsync(string filePath) + public async Task GenerateTextFromAudioAsync(Stream audio, string audioFileName, string? text = null) { - string fileExtension = Path.GetExtension(filePath); - if (!Enum.TryParse(fileExtension.TrimStart('.').ToLower(), out AudioType audioType)) - { - throw new Exception($"Unsupported audio type: {fileExtension}"); - } - - using var stream = _audioProcessor.ConvertToStream(filePath); - if (stream == null) - { - throw new Exception($"Failed to convert {fileExtension} to stream"); - } - var textResult = new List(); + + using var stream = AudioHelper.Transform(audio, audioFileName); await foreach (var result in _whisperProcessor.ProcessAsync(stream).ConfigureAwait(false)) { textResult.Add(result); } _whisperProcessor.Dispose(); + stream.Close(); var audioOutput = new AudioOutput { @@ -57,54 +47,45 @@ public async Task GenerateTextFromAudioAsync(string filePath) return audioOutput.ToString(); } - public Task GenerateTextFromAudioAsync(Stream audio, string audioFileName) + public async Task GenerateSpeechFromTextAsync(string text) { throw new NotImplementedException(); } - public async Task SetModelName(string model) + public void SetModelName(string model) { if (Enum.TryParse(model, true, out GgmlType ggmlType)) { - await LoadWhisperModel(ggmlType); - return; + LoadWhisperModel(ggmlType); + } + else + { + _logger.LogWarning($"Unsupported model type: {model}. Use Tiny model instead!"); + LoadWhisperModel(GgmlType.Tiny); } - - _logger.LogWarning($"Unsupported model type: {model}. Use Tiny model instead!"); - await LoadWhisperModel(GgmlType.Tiny); } - private async Task LoadWhisperModel(GgmlType modelType) + private void LoadWhisperModel(GgmlType modelType) { try { - if (!Directory.Exists(MODEL_DIR)) + var modelDir = _fileStorage.BuildDirectory("models", "whisper"); + var exist = _fileStorage.ExistDirectory(modelDir); + if (!exist) { - Directory.CreateDirectory(MODEL_DIR); + _fileStorage.CreateDirectory(modelDir); } - var availableModelPaths = Directory.GetFiles(MODEL_DIR, "*.bin").ToArray(); - if (availableModelPaths.IsNullOrEmpty()) + var files = _fileStorage.GetFiles("models/whisper", "*.bin"); + var modelLoc = files.FirstOrDefault(x => Path.GetFileName(x) == BuildModelFile(modelType)); + if (string.IsNullOrEmpty(modelLoc)) { - _currentModelPath = SetModelPath(MODEL_DIR, modelType); - await DownloadModel(modelType, _currentModelPath); - } - else - { - var modelFilePath = availableModelPaths.FirstOrDefault(x => Path.GetFileName(x) == $"ggml-{modelType}.bin"); - if (modelFilePath == null) - { - _currentModelPath = SetModelPath(MODEL_DIR, modelType); - await DownloadModel(modelType, _currentModelPath); - } - else - { - _currentModelPath = modelFilePath; - } + modelLoc = BuildModelPath(modelType); + DownloadModel(modelType, modelLoc); } - _whisperProcessor = WhisperFactory.FromPath(path: _currentModelPath).CreateBuilder().WithLanguage("auto").Build(); - _modelType = modelType; + var bytes = _fileStorage.GetFileBytes(modelLoc); + _whisperProcessor = WhisperFactory.FromBuffer(buffer: bytes).CreateBuilder().WithLanguage("auto").Build(); } catch (Exception ex) { @@ -114,16 +95,20 @@ private async Task LoadWhisperModel(GgmlType modelType) } } - private async Task DownloadModel(GgmlType modelType, string modelDir) + private void DownloadModel(GgmlType modelType, string modelDir) + { + using var modelStream = WhisperGgmlDownloader.GetGgmlModelAsync(modelType).ConfigureAwait(false).GetAwaiter().GetResult(); + _fileStorage.SaveFileStreamToPath(modelDir, modelStream); + modelStream.Close(); + } + + private string BuildModelPath(GgmlType modelType) { - using var modelStream = await WhisperGgmlDownloader.GetGgmlModelAsync(modelType); - using var fileWriter = File.OpenWrite(modelDir); - await modelStream.CopyToAsync(fileWriter); + return _fileStorage.BuildDirectory("models", "whisper", BuildModelFile(modelType)); } - private string SetModelPath(string rootPath, GgmlType modelType) + private string BuildModelFile(GgmlType modelType) { - string currentModelPath = Path.Combine(rootPath, $"ggml-{modelType}.bin"); - return currentModelPath; + return $"ggml-{modelType}.bin"; } } diff --git a/src/Plugins/BotSharp.Plugin.AudioHandler/Using.cs b/src/Plugins/BotSharp.Plugin.AudioHandler/Using.cs index b1ba52497..1a1188fd8 100644 --- a/src/Plugins/BotSharp.Plugin.AudioHandler/Using.cs +++ b/src/Plugins/BotSharp.Plugin.AudioHandler/Using.cs @@ -5,6 +5,7 @@ global using System.Text.Json; global using System.Threading.Tasks; +global using BotSharp.Abstraction.Repositories; global using BotSharp.Abstraction.Agents; global using BotSharp.Abstraction.Agents.Enums; global using BotSharp.Abstraction.Agents.Models; diff --git a/src/Plugins/BotSharp.Plugin.AudioHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/handle_audio_request.json b/src/Plugins/BotSharp.Plugin.AudioHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/handle_audio_request.json index b223c7ae9..c0603b1af 100644 --- a/src/Plugins/BotSharp.Plugin.AudioHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/handle_audio_request.json +++ b/src/Plugins/BotSharp.Plugin.AudioHandler/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/handle_audio_request.json @@ -1,18 +1,18 @@ { - "name": "handle_audio_request", - "description": "If the user requests to transcribe or summarize audio content, you need to call this function to transcribe the audio content to raw texts or provide sunmmary based on raw texts transcribed from audio", - "parameters": { - "type": "object", - "properties": { - "user_request": { - "type": "string", - "description": "The request posted by user, which is related to trascribe a aduio based on the inputted audio file" - }, - "is_need_summary": { - "type": "boolean", - "description": "If the user request is to summarize the audio content, set this value to true, otherwise, set it to false" - } - }, - "required": [ "user_request" ] - } + "name": "handle_audio_request", + "description": "If the user requests to transcribe or summarize audio content, you need to call this function to transcribe the audio content to raw texts or provide sunmmary based on raw texts transcribed from audio", + "parameters": { + "type": "object", + "properties": { + "user_request": { + "type": "string", + "description": "The request posted by user, which is related to trascribe a aduio based on the inputted audio file" + }, + "is_need_summary": { + "type": "boolean", + "description": "If the user request is to summarize the audio content, set this value to true, otherwise, set it to false" + } + }, + "required": [ "user_request" ] + } } \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.EmailHandler/Functions/HandleEmailSenderFn.cs b/src/Plugins/BotSharp.Plugin.EmailHandler/Functions/HandleEmailSenderFn.cs index b4f9a6e4e..fa038811d 100644 --- a/src/Plugins/BotSharp.Plugin.EmailHandler/Functions/HandleEmailSenderFn.cs +++ b/src/Plugins/BotSharp.Plugin.EmailHandler/Functions/HandleEmailSenderFn.cs @@ -84,11 +84,12 @@ private void BuildEmailAttachments(BodyBuilder builder, IEnumerable(); + foreach (var file in files) { if (string.IsNullOrEmpty(file.FileStorageUrl)) continue; - var fileStorage = _services.GetRequiredService(); var fileBytes = fileStorage.GetFileBytes(file.FileStorageUrl); builder.Attachments.Add($"{file.FileName}.{file.FileExtension}", fileBytes, ContentType.Parse(file.ContentType)); Thread.Sleep(100); diff --git a/src/Plugins/BotSharp.Plugin.EmailHandler/Hooks/EmailReaderHook.cs b/src/Plugins/BotSharp.Plugin.EmailHandler/Hooks/EmailReaderHook.cs index 222c27b91..beece9e28 100644 --- a/src/Plugins/BotSharp.Plugin.EmailHandler/Hooks/EmailReaderHook.cs +++ b/src/Plugins/BotSharp.Plugin.EmailHandler/Hooks/EmailReaderHook.cs @@ -1,14 +1,7 @@ using BotSharp.Abstraction.Agents; -using BotSharp.Abstraction.Agents.Enums; using BotSharp.Abstraction.Agents.Settings; using BotSharp.Abstraction.Functions.Models; -using BotSharp.Abstraction.Repositories; using BotSharp.Plugin.EmailHandler.Enums; -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; namespace BotSharp.Plugin.EmailHandler.Hooks; diff --git a/src/Plugins/BotSharp.Plugin.EmailHandler/Hooks/EmailSenderHook.cs b/src/Plugins/BotSharp.Plugin.EmailHandler/Hooks/EmailSenderHook.cs index d4ae8c16a..274910157 100644 --- a/src/Plugins/BotSharp.Plugin.EmailHandler/Hooks/EmailSenderHook.cs +++ b/src/Plugins/BotSharp.Plugin.EmailHandler/Hooks/EmailSenderHook.cs @@ -1,14 +1,7 @@ using BotSharp.Abstraction.Agents; -using BotSharp.Abstraction.Agents.Enums; using BotSharp.Abstraction.Agents.Settings; using BotSharp.Abstraction.Functions.Models; -using BotSharp.Abstraction.Repositories; using BotSharp.Plugin.EmailHandler.Enums; -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; namespace BotSharp.Plugin.EmailHandler.Hooks; diff --git a/src/Plugins/BotSharp.Plugin.Graph/GraphDb.cs b/src/Plugins/BotSharp.Plugin.Graph/GraphDb.cs index 74552d174..55a11ec9e 100644 --- a/src/Plugins/BotSharp.Plugin.Graph/GraphDb.cs +++ b/src/Plugins/BotSharp.Plugin.Graph/GraphDb.cs @@ -35,7 +35,7 @@ public GraphDb( _settings = settings; } - public string Name => "Neo4j"; + public string Name => "Remote"; public async Task Search(string query, GraphSearchOptions options) { diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/BotSharp.Plugin.KnowledgeBase.csproj b/src/Plugins/BotSharp.Plugin.KnowledgeBase/BotSharp.Plugin.KnowledgeBase.csproj index 42663445e..29c81ec73 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/BotSharp.Plugin.KnowledgeBase.csproj +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/BotSharp.Plugin.KnowledgeBase.csproj @@ -1,4 +1,4 @@ - + $(TargetFramework) diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/KnowledgeRetrievalFn.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/KnowledgeRetrievalFn.cs index 060e21500..7a7db8aec 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/KnowledgeRetrievalFn.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/KnowledgeRetrievalFn.cs @@ -18,7 +18,7 @@ public async Task Execute(RoleDialogModel message) var args = JsonSerializer.Deserialize(message.FunctionArgs ?? "{}"); var collectionName = _settings.Default.CollectionName ?? KnowledgeCollectionName.BotSharp; - var embedding = KnowledgeSettingUtility.GetTextEmbeddingSetting(_services, collectionName); + var embedding = KnowledgeSettingHelper.GetTextEmbeddingSetting(_services, collectionName); var vector = await embedding.GetVectorAsync(args.Question); var vectorDb = _services.GetServices().FirstOrDefault(x => x.Name == _settings.VectorDb); diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/MemorizeKnowledgeFn.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/MemorizeKnowledgeFn.cs index f709e3b89..f77954d8f 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/MemorizeKnowledgeFn.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/MemorizeKnowledgeFn.cs @@ -18,7 +18,7 @@ public async Task Execute(RoleDialogModel message) var args = JsonSerializer.Deserialize(message.FunctionArgs ?? "{}"); var collectionName = _settings.Default.CollectionName ?? KnowledgeCollectionName.BotSharp; - var embedding = KnowledgeSettingUtility.GetTextEmbeddingSetting(_services, collectionName); + var embedding = KnowledgeSettingHelper.GetTextEmbeddingSetting(_services, collectionName); var vector = await embedding.GetVectorsAsync(new List { diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Utilities/KnowledgeSettingUtility.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Helpers/KnowledgeSettingHelper.cs similarity index 87% rename from src/Plugins/BotSharp.Plugin.KnowledgeBase/Utilities/KnowledgeSettingUtility.cs rename to src/Plugins/BotSharp.Plugin.KnowledgeBase/Helpers/KnowledgeSettingHelper.cs index 770e1efaa..b1b280eba 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Utilities/KnowledgeSettingUtility.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Helpers/KnowledgeSettingHelper.cs @@ -1,6 +1,6 @@ -namespace BotSharp.Plugin.KnowledgeBase.Utilities; +namespace BotSharp.Plugin.KnowledgeBase.Helpers; -public static class KnowledgeSettingUtility +public static class KnowledgeSettingHelper { public static ITextEmbedding GetTextEmbeddingSetting(IServiceProvider services, string collectionName) { diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Utilities/VectorUtility.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Helpers/VectorHelper.cs similarity index 96% rename from src/Plugins/BotSharp.Plugin.KnowledgeBase/Utilities/VectorUtility.cs rename to src/Plugins/BotSharp.Plugin.KnowledgeBase/Helpers/VectorHelper.cs index f0538af0e..88a6ebb5a 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Utilities/VectorUtility.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Helpers/VectorHelper.cs @@ -2,9 +2,9 @@ using Tensorflow.NumPy; using static Tensorflow.Binding; -namespace BotSharp.Plugin.KnowledgeBase.Utilities; +namespace BotSharp.Plugin.KnowledgeBase.Helpers; -public static class VectorUtility +public static class VectorHelper { public static float[] CalEuclideanDistance(float[] vec, List records) { diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/KnowledgeBasePlugin.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/KnowledgeBasePlugin.cs index 213825b5a..4350e33c1 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/KnowledgeBasePlugin.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/KnowledgeBasePlugin.cs @@ -35,7 +35,7 @@ public bool AttachMenu(List menu) SubMenu = new List { new PluginMenuDef("Q & A", link: "page/knowledge-base/question-answer"), - new PluginMenuDef("Relations", link: "page/knowledge-base/relations") + new PluginMenuDef("Relationships", link: "page/knowledge-base/relationships") } }); return true; diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemoryVectorDb.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemoryVectorDb.cs index 0d4be8198..aa5c0f4e8 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemoryVectorDb.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemoryVectorDb.cs @@ -40,7 +40,7 @@ public async Task> Search(string collectionNam return new List(); } - var similarities = VectorUtility.CalCosineSimilarity(vector, _vectors[collectionName]); + var similarities = VectorHelper.CalCosineSimilarity(vector, _vectors[collectionName]); // var similarities = VectorUtility.CalEuclideanDistance(vector, _vectors[collectionName]); var results = np.argsort(similarities).ToArray() diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.cs index eec498910..4e40fea4b 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.cs @@ -33,6 +33,6 @@ private IGraphDb GetGraphDb() private ITextEmbedding GetTextEmbedding(string collection) { - return KnowledgeSettingUtility.GetTextEmbeddingSetting(_services, collection); + return KnowledgeSettingHelper.GetTextEmbeddingSetting(_services, collection); } } diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Using.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Using.cs index 7eeea4f08..a78afefd6 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Using.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Using.cs @@ -33,4 +33,4 @@ global using BotSharp.Abstraction.Repositories; global using BotSharp.Plugin.KnowledgeBase.Services; global using BotSharp.Plugin.KnowledgeBase.Enum; -global using BotSharp.Plugin.KnowledgeBase.Utilities; \ No newline at end of file +global using BotSharp.Plugin.KnowledgeBase.Helpers; \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/OpenAiPlugin.cs b/src/Plugins/BotSharp.Plugin.OpenAI/OpenAiPlugin.cs index 4b41ec466..e6d0637c5 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/OpenAiPlugin.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/OpenAiPlugin.cs @@ -4,8 +4,8 @@ using BotSharp.Plugin.OpenAI.Providers.Image; using BotSharp.Plugin.OpenAI.Providers.Text; using BotSharp.Plugin.OpenAI.Providers.Chat; -using Microsoft.Extensions.Configuration; using BotSharp.Plugin.OpenAI.Providers.Audio; +using Microsoft.Extensions.Configuration; namespace BotSharp.Plugin.OpenAI; @@ -31,7 +31,6 @@ public void RegisterDI(IServiceCollection services, IConfiguration config) services.AddScoped(); services.AddScoped(); services.AddScoped(); - services.AddScoped(); - services.AddScoped(); + services.AddScoped(); } } \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/AudioCompletionProvider.SpeechToText.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/AudioCompletionProvider.SpeechToText.cs new file mode 100644 index 000000000..cf4837a11 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/AudioCompletionProvider.SpeechToText.cs @@ -0,0 +1,53 @@ +using OpenAI.Audio; + +namespace BotSharp.Plugin.OpenAI.Providers.Audio; + +public partial class AudioCompletionProvider +{ + public async Task GenerateTextFromAudioAsync(Stream audio, string audioFileName, string? text = null) + { + var audioClient = ProviderHelper.GetClient(Provider, _model, _services) + .GetAudioClient(_model); + + var options = PrepareTranscriptionOptions(text); + var result = await audioClient.TranscribeAudioAsync(audio, audioFileName, options); + return result.Value.Text; + } + + private AudioTranscriptionOptions PrepareTranscriptionOptions(string? text) + { + var state = _services.GetRequiredService(); + var options = new AudioTranscriptionOptions + { + ResponseFormat = AudioTranscriptionFormat.Verbose, + Granularities = AudioTimestampGranularities.Word | AudioTimestampGranularities.Segment, + Prompt = text + }; + + return options; + } + + private AudioTranscriptionFormat GetTranscriptionResponseFormat(string format) + { + var value = !string.IsNullOrEmpty(format) ? format : "verbose"; + + AudioTranscriptionFormat retFormat; + switch (value) + { + case "json": + retFormat = AudioTranscriptionFormat.Simple; + break; + case "srt": + retFormat = AudioTranscriptionFormat.Srt; + break; + case "vtt": + retFormat = AudioTranscriptionFormat.Vtt; + break; + default: + retFormat = AudioTranscriptionFormat.Verbose; + break; + } + + return retFormat; + } +} diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/AudioCompletionProvider.TextToSpeech.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/AudioCompletionProvider.TextToSpeech.cs new file mode 100644 index 000000000..a1ad2a1f1 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/AudioCompletionProvider.TextToSpeech.cs @@ -0,0 +1,23 @@ +using OpenAI.Audio; + +namespace BotSharp.Plugin.OpenAI.Providers.Audio; + +public partial class AudioCompletionProvider +{ + public async Task GenerateSpeechFromTextAsync(string text) + { + var audioClient = ProviderHelper.GetClient(Provider, _model, _services) + .GetAudioClient(_model); + + var result = await audioClient.GenerateSpeechFromTextAsync(text, GeneratedSpeechVoice.Alloy); + return result.Value; + } + + private SpeechGenerationOptions PrepareGenerationOptions() + { + return new SpeechGenerationOptions + { + + }; + } +} diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/AudioCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/AudioCompletionProvider.cs new file mode 100644 index 000000000..3311fd43f --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/AudioCompletionProvider.cs @@ -0,0 +1,21 @@ +using OpenAI.Audio; + +namespace BotSharp.Plugin.OpenAI.Providers.Audio; + +public partial class AudioCompletionProvider : IAudioCompletion +{ + private readonly IServiceProvider _services; + + public string Provider => "openai"; + private string _model; + + public AudioCompletionProvider(IServiceProvider service) + { + _services = service; + } + + public void SetModelName(string model) + { + _model = model; + } +} diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/SpeechToTextProvider.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/SpeechToTextProvider.cs deleted file mode 100644 index e9b54af88..000000000 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/SpeechToTextProvider.cs +++ /dev/null @@ -1,50 +0,0 @@ -using OpenAI.Audio; - -namespace BotSharp.Plugin.OpenAI.Providers.Audio; - -public class SpeechToTextProvider : ISpeechToText -{ - private readonly IServiceProvider _services; - - public string Provider => "openai"; - private string? _model; - - public SpeechToTextProvider(IServiceProvider service) - { - _services = service; - } - - public async Task GenerateTextFromAudioAsync(string filePath) - { - var client = ProviderHelper.GetClient(Provider, _model, _services) - .GetAudioClient(_model); - - var options = PrepareOptions(); - var result = await client.TranscribeAudioAsync(filePath, options); - return result.Value.Text; - } - - public async Task GenerateTextFromAudioAsync(Stream audio, string audioFileName) - { - var audioClient = ProviderHelper.GetClient(Provider, _model, _services) - .GetAudioClient(_model); - - var options = PrepareOptions(); - var result = await audioClient.TranscribeAudioAsync(audio, audioFileName, options); - return result.Value.Text; - } - - public async Task SetModelName(string model) - { - _model = model; - } - - private AudioTranscriptionOptions PrepareOptions() - { - return new AudioTranscriptionOptions - { - ResponseFormat = AudioTranscriptionFormat.Verbose, - Granularities = AudioTimestampGranularities.Word | AudioTimestampGranularities.Segment, - }; - } -} diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/TextToSpeechProvider.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/TextToSpeechProvider.cs deleted file mode 100644 index e109dfcd8..000000000 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/TextToSpeechProvider.cs +++ /dev/null @@ -1,31 +0,0 @@ -using OpenAI.Audio; - -namespace BotSharp.Plugin.OpenAI.Providers.Audio -{ - public partial class TextToSpeechProvider : ITextToSpeech - { - private readonly IServiceProvider _services; - - public string Provider => "openai"; - private string? _model; - - public TextToSpeechProvider( - IServiceProvider services) - { - _services = services; - } - - public async Task GenerateSpeechFromTextAsync(string text, ITextToSpeechOptions? options = null) - { - var client = ProviderHelper.GetClient(Provider, _model, _services) - .GetAudioClient(_model); - - return await client.GenerateSpeechFromTextAsync(text, GeneratedSpeechVoice.Alloy); - } - - public void SetModelName(string model) - { - _model = model; - } - } -} diff --git a/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Common.cs b/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Common.cs index e94ab6ce3..d21a4c1af 100644 --- a/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Common.cs +++ b/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Common.cs @@ -7,6 +7,24 @@ public string GetDirectory(string conversationId) return $"{CONVERSATION_FOLDER}/{conversationId}/attachments/"; } + public IEnumerable GetFiles(string relativePath, string? searchPattern = null) + { + if (string.IsNullOrEmpty(relativePath)) + { + return Enumerable.Empty(); + } + + try + { + return _cosClient.BucketClient.GetDirFiles(relativePath); + } + catch (Exception ex) + { + _logger.LogWarning($"Error when getting files: {ex.Message}\r\n{ex.InnerException}"); + return Enumerable.Empty(); + } + } + public byte[] GetFileBytes(string fileStorageUrl) { try @@ -15,9 +33,9 @@ public byte[] GetFileBytes(string fileStorageUrl) } catch (Exception ex) { - _logger.LogWarning($"Error when get file bytes: {ex.Message}\r\n{ex.InnerException}"); + _logger.LogWarning($"Error when getting file bytes: {ex.Message}\r\n{ex.InnerException}"); + return Array.Empty(); } - return Array.Empty(); } public bool SaveFileStreamToPath(string filePath, Stream stream) diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs index 9521e8ff0..eb7d775ca 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs @@ -112,11 +112,11 @@ public async Task ReplyCallerMessage([FromRoute] string conversatio } else { - var textToSpeechService = CompletionProvider.GetTextToSpeech(_services, "openai", "tts-1"); - var fileService = _services.GetRequiredService(); - var data = await textToSpeechService.GenerateSpeechFromTextAsync(indication); + var completion = CompletionProvider.GetAudioCompletion(_services, "openai", "tts-1"); + var fileStorage = _services.GetRequiredService(); + var data = await completion.GenerateSpeechFromTextAsync(indication); var fileName = $"indication_{seqNum}.mp3"; - await fileService.SaveSpeechFileAsync(conversationId, fileName, data); + await fileStorage.SaveSpeechFileAsync(conversationId, fileName, data); speechPath = $"twilio/voice/speeches/{conversationId}/{fileName}"; } response = twilio.ReturnInstructions(speechPath, $"twilio/voice/{conversationId}/reply/{seqNum}?states={states}", true, 2); diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioMessageQueueService.cs b/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioMessageQueueService.cs index 19e0ce681..e11284bde 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioMessageQueueService.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioMessageQueueService.cs @@ -3,7 +3,6 @@ using BotSharp.Core.Infrastructures; using BotSharp.Plugin.Twilio.Models; using Microsoft.Extensions.Hosting; -using System; using System.Threading; using Task = System.Threading.Tasks.Task; @@ -97,11 +96,11 @@ private async Task ProcessUserMessageAsync(CallerMessage message) async functionExecuted => { } ); - var textToSpeechService = CompletionProvider.GetTextToSpeech(sp, "openai", "tts-1"); - var fileService = sp.GetRequiredService(); - var data = await textToSpeechService.GenerateSpeechFromTextAsync(reply.Content); + var completion = CompletionProvider.GetAudioCompletion(sp, "openai", "tts-1"); + var fileStorage = sp.GetRequiredService(); + var data = await completion.GenerateSpeechFromTextAsync(reply.Content); var fileName = $"reply_{reply.MessageId}.mp3"; - await fileService.SaveSpeechFileAsync(message.ConversationId, fileName, data); + await fileStorage.SaveSpeechFileAsync(message.ConversationId, fileName, data); reply.SpeechFileName = fileName; reply.Content = null; await sessionManager.SetAssistantReplyAsync(message.ConversationId, message.SeqNumber, reply); diff --git a/src/WebStarter/appsettings.json b/src/WebStarter/appsettings.json index c34fc7911..5b2203fbb 100644 --- a/src/WebStarter/appsettings.json +++ b/src/WebStarter/appsettings.json @@ -345,6 +345,7 @@ "BotSharp.Plugin.HttpHandler", "BotSharp.Plugin.FileHandler", "BotSharp.Plugin.EmailHandler", + "BotSharp.Plugin.AudioHandler", "BotSharp.Plugin.TencentCos", "BotSharp.Plugin.PythonInterpreter" ] From 4b739a04d967b63ce02b2fd19f605b8899901b85 Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Wed, 28 Aug 2024 11:41:42 -0500 Subject: [PATCH 05/27] refine api --- .../Files/Utilities/FileUtility.cs | 4 +- .../Controllers/InstructModeController.cs | 66 +++++++++++-------- 2 files changed, 39 insertions(+), 31 deletions(-) diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/Utilities/FileUtility.cs b/src/Infrastructure/BotSharp.Abstraction/Files/Utilities/FileUtility.cs index 5c021159a..bba60b101 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Files/Utilities/FileUtility.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Files/Utilities/FileUtility.cs @@ -47,11 +47,11 @@ public static string BuildFileDataFromFile(IFormFile file) return $"data:{contentType};base64,{base64}"; } - public static string GetFileContentType(string filePath) + public static string GetFileContentType(string fileName) { string contentType; var provider = new FileExtensionContentTypeProvider(); - if (!provider.TryGetContentType(filePath, out contentType)) + if (!provider.TryGetContentType(fileName, out contentType)) { contentType = string.Empty; } diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs index 170cf2d20..b8aedadc2 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs @@ -173,6 +173,7 @@ public async Task ImageVariation([FromBody] ImageVaria var message = await fileInstruct.VaryImage(input.Provider, input.Model, input.File); imageViewModel.Content = message.Content; imageViewModel.Images = message.GeneratedImages.Select(x => ImageViewModel.ToViewModel(x)).ToList(); + return imageViewModel; } catch (Exception ex) @@ -194,17 +195,20 @@ public async Task ImageVariation(IFormFile file, [From try { - var data = FileUtility.BuildFileDataFromFile(file); - var image = new InstructFileModel + using var stream = new MemoryStream(); + file.CopyTo(stream); + stream.Position = 0; + + var completion = CompletionProvider.GetImageCompletion(_services, provider: provider ?? "openai", model: model ?? "dall-e-2"); + var message = await completion.GetImageVariation(new Agent() { - FileName = Path.GetFileNameWithoutExtension(file.FileName), - FileExtension = Path.GetExtension(file.FileName).Substring(1), - FileData = data - }; - var fileInstruct = _services.GetRequiredService(); - var message = await fileInstruct.VaryImage(provider, model, image); + Id = Guid.Empty.ToString() + }, new RoleDialogModel(AgentRole.User, string.Empty), stream, file.FileName); + imageViewModel.Content = message.Content; imageViewModel.Images = message.GeneratedImages.Select(x => ImageViewModel.ToViewModel(x)).ToList(); + stream.Close(); + return imageViewModel; } catch (Exception ex) @@ -255,16 +259,20 @@ public async Task ImageEdit(IFormFile file, [FromForm] try { - var data = FileUtility.BuildFileDataFromFile(file); - var image = new InstructFileModel + using var stream = new MemoryStream(); + file.CopyTo(stream); + stream.Position = 0; + + var completion = CompletionProvider.GetImageCompletion(_services, provider: provider ?? "openai", model: model ?? "dall-e-2"); + var message = await completion.GetImageEdits(new Agent() { - FileName = Path.GetFileNameWithoutExtension(file.FileName), - FileExtension = Path.GetExtension(file.FileName).Substring(1), - FileData = data - }; - var message = await fileInstruct.EditImage(provider, model, text, image); + Id = Guid.Empty.ToString() + }, new RoleDialogModel(AgentRole.User, text), stream, file.FileName); + imageViewModel.Content = message.Content; imageViewModel.Images = message.GeneratedImages.Select(x => ImageViewModel.ToViewModel(x)).ToList(); + stream.Close(); + return imageViewModel; } catch (Exception ex) @@ -317,25 +325,25 @@ public async Task ImageMaskEdit(IFormFile image, IForm try { - var imageData = FileUtility.BuildFileDataFromFile(image); - var imageFile = new InstructFileModel - { - FileName = Path.GetFileNameWithoutExtension(image.FileName), - FileExtension = Path.GetExtension(image.FileName).Substring(1), - FileData = imageData - }; + using var imageStream = new MemoryStream(); + image.CopyTo(imageStream); + imageStream.Position = 0; - var maskData = FileUtility.BuildFileDataFromFile(mask); - var maskFile = new InstructFileModel + using var maskStream = new MemoryStream(); + mask.CopyTo(maskStream); + maskStream.Position = 0; + + var completion = CompletionProvider.GetImageCompletion(_services, provider: provider ?? "openai", model: model ?? "dall-e-2"); + var message = await completion.GetImageEdits(new Agent() { - FileName = Path.GetFileNameWithoutExtension(mask.FileName), - FileExtension = Path.GetExtension(mask.FileName).Substring(1), - FileData = maskData - }; + Id = Guid.Empty.ToString() + }, new RoleDialogModel(AgentRole.User, text), imageStream, image.FileName, maskStream, mask.FileName); - var message = await fileInstruct.EditImage(provider, model, text, imageFile, maskFile); imageViewModel.Content = message.Content; imageViewModel.Images = message.GeneratedImages.Select(x => ImageViewModel.ToViewModel(x)).ToList(); + imageStream.Close(); + maskStream.Close(); + return imageViewModel; } catch (Exception ex) From a4be2e867184cfb23160515ab19517353a409060 Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Wed, 28 Aug 2024 12:06:23 -0500 Subject: [PATCH 06/27] add speech2text options --- .../Image/ImageCompletionProvider.Edit.cs | 2 +- .../ImageCompletionProvider.Generation.cs | 2 +- .../ImageCompletionProvider.Variation.cs | 2 +- .../Functions/EditImageFn.cs | 2 +- .../Functions/GenerateImageFn.cs | 2 +- .../AudioCompletionProvider.SpeechToText.cs | 56 +++++++++++++++---- .../Image/ImageCompletionProvider.Edit.cs | 2 +- .../ImageCompletionProvider.Generation.cs | 2 +- .../ImageCompletionProvider.Variation.cs | 2 +- 9 files changed, 54 insertions(+), 18 deletions(-) diff --git a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Image/ImageCompletionProvider.Edit.cs b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Image/ImageCompletionProvider.Edit.cs index 43272f8eb..52e1a845a 100644 --- a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Image/ImageCompletionProvider.Edit.cs +++ b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Image/ImageCompletionProvider.Edit.cs @@ -53,7 +53,7 @@ public async Task GetImageEdits(Agent agent, RoleDialogModel me var state = _services.GetRequiredService(); var size = GetImageSize(state.GetState("image_size")); - var format = GetImageFormat(state.GetState("image_format")); + var format = GetImageFormat(state.GetState("image_response_format")); var count = GetImageCount(state.GetState("image_count", "1")); var options = new ImageEditOptions diff --git a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Image/ImageCompletionProvider.Generation.cs b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Image/ImageCompletionProvider.Generation.cs index 7d608fd14..f5d8c1f92 100644 --- a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Image/ImageCompletionProvider.Generation.cs +++ b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Image/ImageCompletionProvider.Generation.cs @@ -33,7 +33,7 @@ public async Task GetImageGeneration(Agent agent, RoleDialogMod var size = GetImageSize(state.GetState("image_size")); var quality = GetImageQuality(state.GetState("image_quality")); var style = GetImageStyle(state.GetState("image_style")); - var format = GetImageFormat(state.GetState("image_format")); + var format = GetImageFormat(state.GetState("image_response_format")); var count = GetImageCount(state.GetState("image_count", "1")); var options = new ImageGenerationOptions diff --git a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Image/ImageCompletionProvider.Variation.cs b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Image/ImageCompletionProvider.Variation.cs index 88f393e06..b2e4c5fe1 100644 --- a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Image/ImageCompletionProvider.Variation.cs +++ b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Image/ImageCompletionProvider.Variation.cs @@ -29,7 +29,7 @@ public async Task GetImageVariation(Agent agent, RoleDialogMode { var state = _services.GetRequiredService(); var size = GetImageSize(state.GetState("image_size")); - var format = GetImageFormat(state.GetState("image_format")); + var format = GetImageFormat(state.GetState("image_response_format")); var count = GetImageCount(state.GetState("image_count", "1")); var options = new ImageVariationOptions diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs index 7a5d8e725..dc4ddef03 100644 --- a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs +++ b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/EditImageFn.cs @@ -43,7 +43,7 @@ private void Init(RoleDialogModel message) private void SetImageOptions() { var state = _services.GetRequiredService(); - state.SetState("image_format", "bytes"); + state.SetState("image_response_format", "bytes"); state.SetState("image_count", "1"); } diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/GenerateImageFn.cs b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/GenerateImageFn.cs index 4c8fdf9ef..9592ef80c 100644 --- a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/GenerateImageFn.cs +++ b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/GenerateImageFn.cs @@ -50,7 +50,7 @@ private void Init(RoleDialogModel message) private void SetImageOptions() { var state = _services.GetRequiredService(); - state.SetState("image_format", "bytes"); + state.SetState("image_response_format", "bytes"); state.SetState("image_count", "1"); } diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/AudioCompletionProvider.SpeechToText.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/AudioCompletionProvider.SpeechToText.cs index cf4837a11..b31c30357 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/AudioCompletionProvider.SpeechToText.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/AudioCompletionProvider.SpeechToText.cs @@ -17,37 +17,73 @@ public async Task GenerateTextFromAudioAsync(Stream audio, string audioF private AudioTranscriptionOptions PrepareTranscriptionOptions(string? text) { var state = _services.GetRequiredService(); + var format = GetTranscriptionResponseFormat(state.GetState("audio_response_format")); + var granularity = GetGranularity(state.GetState("audio_granularity")); + var temperature = GetTemperature(state.GetState("audio_temperature")); + var options = new AudioTranscriptionOptions { - ResponseFormat = AudioTranscriptionFormat.Verbose, - Granularities = AudioTimestampGranularities.Word | AudioTimestampGranularities.Segment, + ResponseFormat = format, + Granularities = granularity, + Temperature = temperature, Prompt = text }; return options; } - private AudioTranscriptionFormat GetTranscriptionResponseFormat(string format) + private AudioTranscriptionFormat GetTranscriptionResponseFormat(string input) { - var value = !string.IsNullOrEmpty(format) ? format : "verbose"; + var value = !string.IsNullOrEmpty(input) ? input : "verbose"; - AudioTranscriptionFormat retFormat; + AudioTranscriptionFormat format; switch (value) { case "json": - retFormat = AudioTranscriptionFormat.Simple; + format = AudioTranscriptionFormat.Simple; break; case "srt": - retFormat = AudioTranscriptionFormat.Srt; + format = AudioTranscriptionFormat.Srt; break; case "vtt": - retFormat = AudioTranscriptionFormat.Vtt; + format = AudioTranscriptionFormat.Vtt; break; default: - retFormat = AudioTranscriptionFormat.Verbose; + format = AudioTranscriptionFormat.Verbose; + break; + } + + return format; + } + + private AudioTimestampGranularities GetGranularity(string input) + { + var value = !string.IsNullOrEmpty(input) ? input : "default"; + + AudioTimestampGranularities granularity; + switch (value) + { + case "word": + granularity = AudioTimestampGranularities.Word; break; + case "segment": + granularity = AudioTimestampGranularities.Segment; + break; + default: + granularity = AudioTimestampGranularities.Default; + break; + } + + return granularity; + } + + private float GetTemperature(string input) + { + if (!float.TryParse(input, out var temperature)) + { + return 0.0f; } - return retFormat; + return temperature; } } diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Image/ImageCompletionProvider.Edit.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Image/ImageCompletionProvider.Edit.cs index 717002fba..47df077ef 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Image/ImageCompletionProvider.Edit.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Image/ImageCompletionProvider.Edit.cs @@ -54,7 +54,7 @@ public async Task GetImageEdits(Agent agent, RoleDialogModel me var state = _services.GetRequiredService(); var size = GetImageSize(state.GetState("image_size")); - var format = GetImageFormat(state.GetState("image_format")); + var format = GetImageFormat(state.GetState("image_response_format")); var count = GetImageCount(state.GetState("image_count", "1")); var options = new ImageEditOptions diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Image/ImageCompletionProvider.Generation.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Image/ImageCompletionProvider.Generation.cs index 85c156864..2f55d9172 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Image/ImageCompletionProvider.Generation.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Image/ImageCompletionProvider.Generation.cs @@ -33,7 +33,7 @@ public async Task GetImageGeneration(Agent agent, RoleDialogMod var size = GetImageSize(state.GetState("image_size")); var quality = GetImageQuality(state.GetState("image_quality")); var style = GetImageStyle(state.GetState("image_style")); - var format = GetImageFormat(state.GetState("image_format")); + var format = GetImageFormat(state.GetState("image_response_format")); var count = GetImageCount(state.GetState("image_count", "1")); var options = new ImageGenerationOptions diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Image/ImageCompletionProvider.Variation.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Image/ImageCompletionProvider.Variation.cs index 8a93df83a..30c8d68fb 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Image/ImageCompletionProvider.Variation.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Image/ImageCompletionProvider.Variation.cs @@ -29,7 +29,7 @@ public async Task GetImageVariation(Agent agent, RoleDialogMode { var state = _services.GetRequiredService(); var size = GetImageSize(state.GetState("image_size")); - var format = GetImageFormat(state.GetState("image_format")); + var format = GetImageFormat(state.GetState("image_response_format")); var count = GetImageCount(state.GetState("image_count", "1")); var options = new ImageVariationOptions From 5cb87e8c8fa297bf43b772d0418ca67b4d8555d0 Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Wed, 28 Aug 2024 13:43:37 -0500 Subject: [PATCH 07/27] add text to speech api --- .../Files/Models/BotSharpFile.cs | 2 +- .../{FileInfo.cs => FileInformation.cs} | 2 +- .../Files/Models/MessageFileModel.cs | 2 +- .../MLTasks/IAudioCompletion.cs | 2 +- .../BotSharp.OpenAPI/BotSharp.OpenAPI.csproj | 3 +- .../Controllers/InstructModeController.cs | 14 +++ .../Instructs/InstructBaseRequest.cs | 6 ++ .../Provider/NativeWhisperProvider.cs | 2 +- .../AudioCompletionProvider.SpeechToText.cs | 4 +- .../AudioCompletionProvider.TextToSpeech.cs | 89 +++++++++++++++++-- .../Controllers/TwilioVoiceController.cs | 2 +- .../Services/TwilioMessageQueueService.cs | 2 +- 12 files changed, 115 insertions(+), 15 deletions(-) rename src/Infrastructure/BotSharp.Abstraction/Files/Models/{FileInfo.cs => FileInformation.cs} (97%) diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/Models/BotSharpFile.cs b/src/Infrastructure/BotSharp.Abstraction/Files/Models/BotSharpFile.cs index 7a606b3d3..bc6706269 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Files/Models/BotSharpFile.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Files/Models/BotSharpFile.cs @@ -1,7 +1,7 @@ namespace BotSharp.Abstraction.Files.Models; -public class BotSharpFile : FileInfo +public class BotSharpFile : FileInformation { /// /// File data => format: "data:image/png;base64,aaaaaaaa" diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/Models/FileInfo.cs b/src/Infrastructure/BotSharp.Abstraction/Files/Models/FileInformation.cs similarity index 97% rename from src/Infrastructure/BotSharp.Abstraction/Files/Models/FileInfo.cs rename to src/Infrastructure/BotSharp.Abstraction/Files/Models/FileInformation.cs index 14c02f746..fe767e163 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Files/Models/FileInfo.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Files/Models/FileInformation.cs @@ -1,6 +1,6 @@ namespace BotSharp.Abstraction.Files.Models; -public class FileInfo +public class FileInformation { /// /// External file url diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/Models/MessageFileModel.cs b/src/Infrastructure/BotSharp.Abstraction/Files/Models/MessageFileModel.cs index 59ada4f2f..da2ddec66 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Files/Models/MessageFileModel.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Files/Models/MessageFileModel.cs @@ -1,6 +1,6 @@ namespace BotSharp.Abstraction.Files.Models; -public class MessageFileModel : FileInfo +public class MessageFileModel : FileInformation { [JsonPropertyName("message_id")] public string MessageId { get; set; } diff --git a/src/Infrastructure/BotSharp.Abstraction/MLTasks/IAudioCompletion.cs b/src/Infrastructure/BotSharp.Abstraction/MLTasks/IAudioCompletion.cs index d32624c0e..54ed2d7d3 100644 --- a/src/Infrastructure/BotSharp.Abstraction/MLTasks/IAudioCompletion.cs +++ b/src/Infrastructure/BotSharp.Abstraction/MLTasks/IAudioCompletion.cs @@ -7,7 +7,7 @@ public interface IAudioCompletion string Provider { get; } Task GenerateTextFromAudioAsync(Stream audio, string audioFileName, string? text = null); - Task GenerateSpeechFromTextAsync(string text); + Task GenerateAudioFromTextAsync(string text); void SetModelName(string model); } diff --git a/src/Infrastructure/BotSharp.OpenAPI/BotSharp.OpenAPI.csproj b/src/Infrastructure/BotSharp.OpenAPI/BotSharp.OpenAPI.csproj index 240022258..9094107da 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/BotSharp.OpenAPI.csproj +++ b/src/Infrastructure/BotSharp.OpenAPI/BotSharp.OpenAPI.csproj @@ -1,4 +1,4 @@ - + $(TargetFramework) @@ -19,6 +19,7 @@ + diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs index b8aedadc2..bc02e8464 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/InstructModeController.cs @@ -469,5 +469,19 @@ public async Task SpeechToText(IFormFile file, [FromForm] return viewModel; } } + + [HttpPost("/instruct/text-to-speech")] + public async Task TextToSpeech([FromBody] TextToSpeechRequest input) + { + var state = _services.GetRequiredService(); + input.States.ForEach(x => state.SetState(x.Key, x.Value, activeRounds: x.ActiveRounds, source: StateSource.External)); + + var completion = CompletionProvider.GetAudioCompletion(_services, provider: input.Provider ?? "openai", model: input.Model ?? "tts-1"); + var binaryData = await completion.GenerateAudioFromTextAsync(input.Text); + var stream = binaryData.ToStream(); + stream.Position = 0; + + return new FileStreamResult(stream, "audio/mpeg") { FileDownloadName = "output.mp3" }; + } #endregion } diff --git a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/InstructBaseRequest.cs b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/InstructBaseRequest.cs index 396fff2f5..39a6439d9 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/InstructBaseRequest.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/ViewModels/Instructs/InstructBaseRequest.cs @@ -66,4 +66,10 @@ public class SpeechToTextRequest : InstructBaseRequest [JsonPropertyName("file")] public InstructFileModel File { get; set; } +} + +public class TextToSpeechRequest : InstructBaseRequest +{ + [JsonPropertyName("text")] + public string Text { get; set; } } \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.AudioHandler/Provider/NativeWhisperProvider.cs b/src/Plugins/BotSharp.Plugin.AudioHandler/Provider/NativeWhisperProvider.cs index 9bf049ac6..e6ac230e9 100644 --- a/src/Plugins/BotSharp.Plugin.AudioHandler/Provider/NativeWhisperProvider.cs +++ b/src/Plugins/BotSharp.Plugin.AudioHandler/Provider/NativeWhisperProvider.cs @@ -47,7 +47,7 @@ public async Task GenerateTextFromAudioAsync(Stream audio, string audioF return audioOutput.ToString(); } - public async Task GenerateSpeechFromTextAsync(string text) + public async Task GenerateAudioFromTextAsync(string text) { throw new NotImplementedException(); } diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/AudioCompletionProvider.SpeechToText.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/AudioCompletionProvider.SpeechToText.cs index b31c30357..9f2480865 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/AudioCompletionProvider.SpeechToText.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/AudioCompletionProvider.SpeechToText.cs @@ -77,11 +77,11 @@ private AudioTimestampGranularities GetGranularity(string input) return granularity; } - private float GetTemperature(string input) + private float? GetTemperature(string input) { if (!float.TryParse(input, out var temperature)) { - return 0.0f; + return null; } return temperature; diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/AudioCompletionProvider.TextToSpeech.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/AudioCompletionProvider.TextToSpeech.cs index a1ad2a1f1..09acf389f 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/AudioCompletionProvider.TextToSpeech.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Audio/AudioCompletionProvider.TextToSpeech.cs @@ -4,20 +4,99 @@ namespace BotSharp.Plugin.OpenAI.Providers.Audio; public partial class AudioCompletionProvider { - public async Task GenerateSpeechFromTextAsync(string text) + public async Task GenerateAudioFromTextAsync(string text) { var audioClient = ProviderHelper.GetClient(Provider, _model, _services) .GetAudioClient(_model); - var result = await audioClient.GenerateSpeechFromTextAsync(text, GeneratedSpeechVoice.Alloy); + var (voice, options) = PrepareGenerationOptions(); + var result = await audioClient.GenerateSpeechFromTextAsync(text, voice, options); return result.Value; } - private SpeechGenerationOptions PrepareGenerationOptions() + private (GeneratedSpeechVoice, SpeechGenerationOptions) PrepareGenerationOptions() { - return new SpeechGenerationOptions - { + var state = _services.GetRequiredService(); + var voice = GetVoice(state.GetState("speech_generate_voice")); + var format = GetSpeechFormat(state.GetState("speech_generate_format")); + var speed = GetSpeed(state.GetState("speech_generate_speed")); + var options = new SpeechGenerationOptions + { + ResponseFormat = format, + Speed = speed }; + + return (voice, options); + } + + private GeneratedSpeechVoice GetVoice(string input) + { + var value = !string.IsNullOrEmpty(input) ? input : "alloy"; + + GeneratedSpeechVoice voice; + switch (value) + { + case "echo": + voice = GeneratedSpeechVoice.Echo; + break; + case "fable": + voice = GeneratedSpeechVoice.Fable; + break; + case "onyx": + voice = GeneratedSpeechVoice.Onyx; + break; + case "nova": + voice = GeneratedSpeechVoice.Nova; + break; + case "shimmer": + voice = GeneratedSpeechVoice.Shimmer; + break; + default: + voice = GeneratedSpeechVoice.Alloy; + break; + } + + return voice; + } + + private GeneratedSpeechFormat GetSpeechFormat(string input) + { + var value = !string.IsNullOrEmpty(input) ? input : "mp3"; + + GeneratedSpeechFormat format; + switch (value) + { + case "wav": + format = GeneratedSpeechFormat.Wav; + break; + case "opus": + format = GeneratedSpeechFormat.Opus; + break; + case "aac": + format = GeneratedSpeechFormat.Aac; + break; + case "flac": + format = GeneratedSpeechFormat.Flac; + break; + case "pcm": + format = GeneratedSpeechFormat.Pcm; + break; + default: + format = GeneratedSpeechFormat.Mp3; + break; + } + + return format; + } + + private float? GetSpeed(string input) + { + if (!float.TryParse(input, out var speed)) + { + return null; + } + + return speed; } } diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs index e4bb9e448..571a51013 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs @@ -115,7 +115,7 @@ public async Task ReplyCallerMessage([FromRoute] string conversatio { var completion = CompletionProvider.GetAudioCompletion(_services, "openai", "tts-1"); var fileStorage = _services.GetRequiredService(); - var data = await completion.GenerateSpeechFromTextAsync(indication); + var data = await completion.GenerateAudioFromTextAsync(indication); var fileName = $"indication_{seqNum}.mp3"; await fileStorage.SaveSpeechFileAsync(conversationId, fileName, data); speechPath = $"twilio/voice/speeches/{conversationId}/{fileName}"; diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioMessageQueueService.cs b/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioMessageQueueService.cs index e11284bde..6b558aad2 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioMessageQueueService.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioMessageQueueService.cs @@ -98,7 +98,7 @@ private async Task ProcessUserMessageAsync(CallerMessage message) ); var completion = CompletionProvider.GetAudioCompletion(sp, "openai", "tts-1"); var fileStorage = sp.GetRequiredService(); - var data = await completion.GenerateSpeechFromTextAsync(reply.Content); + var data = await completion.GenerateAudioFromTextAsync(reply.Content); var fileName = $"reply_{reply.MessageId}.mp3"; await fileStorage.SaveSpeechFileAsync(message.ConversationId, fileName, data); reply.SpeechFileName = fileName; From 4a870a397229a47a1f16ff04c766b63e003bf70f Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Wed, 28 Aug 2024 13:47:09 -0500 Subject: [PATCH 08/27] add azure audio completion --- .../AzureOpenAiPlugin.cs | 2 + .../AudioCompletionProvider.SpeechToText.cs | 90 ++++++++++++++++ .../AudioCompletionProvider.TextToSpeech.cs | 102 ++++++++++++++++++ .../Audio/AudioCompletionProvider.cs | 19 ++++ 4 files changed, 213 insertions(+) create mode 100644 src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Audio/AudioCompletionProvider.SpeechToText.cs create mode 100644 src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Audio/AudioCompletionProvider.TextToSpeech.cs create mode 100644 src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Audio/AudioCompletionProvider.cs diff --git a/src/Plugins/BotSharp.Plugin.AzureOpenAI/AzureOpenAiPlugin.cs b/src/Plugins/BotSharp.Plugin.AzureOpenAI/AzureOpenAiPlugin.cs index dba74d274..eba22bfa4 100644 --- a/src/Plugins/BotSharp.Plugin.AzureOpenAI/AzureOpenAiPlugin.cs +++ b/src/Plugins/BotSharp.Plugin.AzureOpenAI/AzureOpenAiPlugin.cs @@ -1,5 +1,6 @@ using BotSharp.Abstraction.Plugins; using BotSharp.Abstraction.Settings; +using BotSharp.Plugin.AzureOpenAI.Providers.Audio; using BotSharp.Plugin.AzureOpenAI.Providers.Chat; using BotSharp.Plugin.AzureOpenAI.Providers.Embedding; using BotSharp.Plugin.AzureOpenAI.Providers.Image; @@ -30,5 +31,6 @@ public void RegisterDI(IServiceCollection services, IConfiguration config) services.AddScoped(); services.AddScoped(); services.AddScoped(); + services.AddScoped(); } } \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Audio/AudioCompletionProvider.SpeechToText.cs b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Audio/AudioCompletionProvider.SpeechToText.cs new file mode 100644 index 000000000..5a436feff --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Audio/AudioCompletionProvider.SpeechToText.cs @@ -0,0 +1,90 @@ +using OpenAI.Audio; + +namespace BotSharp.Plugin.AzureOpenAI.Providers.Audio; + +public partial class AudioCompletionProvider +{ + public async Task GenerateTextFromAudioAsync(Stream audio, string audioFileName, string? text = null) + { + var audioClient = ProviderHelper.GetClient(Provider, _model, _services) + .GetAudioClient(_model); + + var options = PrepareTranscriptionOptions(text); + var result = await audioClient.TranscribeAudioAsync(audio, audioFileName, options); + return result.Value.Text; + } + + private AudioTranscriptionOptions PrepareTranscriptionOptions(string? text) + { + var state = _services.GetRequiredService(); + var format = GetTranscriptionResponseFormat(state.GetState("audio_response_format")); + var granularity = GetGranularity(state.GetState("audio_granularity")); + var temperature = GetTemperature(state.GetState("audio_temperature")); + + var options = new AudioTranscriptionOptions + { + ResponseFormat = format, + Granularities = granularity, + Temperature = temperature, + Prompt = text + }; + + return options; + } + + private AudioTranscriptionFormat GetTranscriptionResponseFormat(string input) + { + var value = !string.IsNullOrEmpty(input) ? input : "verbose"; + + AudioTranscriptionFormat format; + switch (value) + { + case "json": + format = AudioTranscriptionFormat.Simple; + break; + case "srt": + format = AudioTranscriptionFormat.Srt; + break; + case "vtt": + format = AudioTranscriptionFormat.Vtt; + break; + default: + format = AudioTranscriptionFormat.Verbose; + break; + } + + return format; + } + + private AudioTimestampGranularities GetGranularity(string input) + { + var value = !string.IsNullOrEmpty(input) ? input : "default"; + + AudioTimestampGranularities granularity; + switch (value) + { + case "word": + granularity = AudioTimestampGranularities.Word; + break; + case "segment": + granularity = AudioTimestampGranularities.Segment; + break; + default: + granularity = AudioTimestampGranularities.Default; + break; + } + + return granularity; + } + + private float? GetTemperature(string input) + { + if (!float.TryParse(input, out var temperature)) + { + return null; + } + + return temperature; + } +} + diff --git a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Audio/AudioCompletionProvider.TextToSpeech.cs b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Audio/AudioCompletionProvider.TextToSpeech.cs new file mode 100644 index 000000000..4e5fc3fa9 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Audio/AudioCompletionProvider.TextToSpeech.cs @@ -0,0 +1,102 @@ +using OpenAI.Audio; + +namespace BotSharp.Plugin.AzureOpenAI.Providers.Audio; + +public partial class AudioCompletionProvider +{ + public async Task GenerateAudioFromTextAsync(string text) + { + var audioClient = ProviderHelper.GetClient(Provider, _model, _services) + .GetAudioClient(_model); + + var (voice, options) = PrepareGenerationOptions(); + var result = await audioClient.GenerateSpeechFromTextAsync(text, voice, options); + return result.Value; + } + + private (GeneratedSpeechVoice, SpeechGenerationOptions) PrepareGenerationOptions() + { + var state = _services.GetRequiredService(); + var voice = GetVoice(state.GetState("speech_generate_voice")); + var format = GetSpeechFormat(state.GetState("speech_generate_format")); + var speed = GetSpeed(state.GetState("speech_generate_speed")); + + var options = new SpeechGenerationOptions + { + ResponseFormat = format, + Speed = speed + }; + + return (voice, options); + } + + private GeneratedSpeechVoice GetVoice(string input) + { + var value = !string.IsNullOrEmpty(input) ? input : "alloy"; + + GeneratedSpeechVoice voice; + switch (value) + { + case "echo": + voice = GeneratedSpeechVoice.Echo; + break; + case "fable": + voice = GeneratedSpeechVoice.Fable; + break; + case "onyx": + voice = GeneratedSpeechVoice.Onyx; + break; + case "nova": + voice = GeneratedSpeechVoice.Nova; + break; + case "shimmer": + voice = GeneratedSpeechVoice.Shimmer; + break; + default: + voice = GeneratedSpeechVoice.Alloy; + break; + } + + return voice; + } + + private GeneratedSpeechFormat GetSpeechFormat(string input) + { + var value = !string.IsNullOrEmpty(input) ? input : "mp3"; + + GeneratedSpeechFormat format; + switch (value) + { + case "wav": + format = GeneratedSpeechFormat.Wav; + break; + case "opus": + format = GeneratedSpeechFormat.Opus; + break; + case "aac": + format = GeneratedSpeechFormat.Aac; + break; + case "flac": + format = GeneratedSpeechFormat.Flac; + break; + case "pcm": + format = GeneratedSpeechFormat.Pcm; + break; + default: + format = GeneratedSpeechFormat.Mp3; + break; + } + + return format; + } + + private float? GetSpeed(string input) + { + if (!float.TryParse(input, out var speed)) + { + return null; + } + + return speed; + } +} diff --git a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Audio/AudioCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Audio/AudioCompletionProvider.cs new file mode 100644 index 000000000..8cc75ad8e --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Audio/AudioCompletionProvider.cs @@ -0,0 +1,19 @@ +namespace BotSharp.Plugin.AzureOpenAI.Providers.Audio; + +public partial class AudioCompletionProvider : IAudioCompletion +{ + private readonly IServiceProvider _services; + + public string Provider => "openai"; + private string _model; + + public AudioCompletionProvider(IServiceProvider service) + { + _services = service; + } + + public void SetModelName(string model) + { + _model = model; + } +} From 8674ac9830f7af8540be94d505d3fb8f038f3601 Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Wed, 28 Aug 2024 15:34:41 -0500 Subject: [PATCH 09/27] refine speech storage --- .../Files/IFileStorageService.cs | 6 ++-- .../Storage/LocalFileStorageService.Audio.cs | 34 ++++++++++++------- .../LocalFileStorageService.Conversation.cs | 2 +- .../Storage/LocalFileStorageService.cs | 6 ---- .../Functions/ReadPdfFn.cs | 2 +- .../Modules/BucketClient.cs | 26 ++++++++++++++ .../Services/TencentCosService.Audio.cs | 33 +++++++++++++++--- .../TencentCosService.Conversation.cs | 2 +- .../Services/TencentCosService.cs | 2 +- .../Controllers/TwilioVoiceController.cs | 26 ++++++++++---- .../Services/TwilioMessageQueue.cs | 7 +++- .../Services/TwilioMessageQueueService.cs | 11 ++++-- .../Services/TwilioService.cs | 3 ++ .../Services/TwilioSessionManager.cs | 4 +-- .../BotSharp.Plugin.Twilio/TwilioPlugin.cs | 3 ++ 15 files changed, 124 insertions(+), 43 deletions(-) diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/IFileStorageService.cs b/src/Infrastructure/BotSharp.Abstraction/Files/IFileStorageService.cs index ebd7d61f6..373466c41 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Files/IFileStorageService.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Files/IFileStorageService.cs @@ -24,7 +24,7 @@ public interface IFileStorageService /// /// /// - Task> GetMessageFileScreenshots(string conversationId, IEnumerable messageIds); + Task> GetMessageFileScreenshotsAsync(string conversationId, IEnumerable messageIds); /// /// Get the files that have been uploaded in the chat. No screenshot images are included. @@ -58,7 +58,7 @@ public interface IFileStorageService #endregion #region Speech - Task SaveSpeechFileAsync(string conversationId, string fileName, BinaryData data); - Task RetrieveSpeechFileAsync(string conversationId, string fileName); + bool SaveSpeechFile(string conversationId, string fileName, BinaryData data); + BinaryData GetSpeechFile(string conversationId, string fileName); #endregion } diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Audio.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Audio.cs index e90c7812d..1aaba5755 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Audio.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Audio.cs @@ -1,28 +1,38 @@ using System.IO; -namespace BotSharp.Core.Files.Services +namespace BotSharp.Core.Files.Services; + +public partial class LocalFileStorageService { - public partial class LocalFileStorageService + public bool SaveSpeechFile(string conversationId, string fileName, BinaryData data) { - public async Task SaveSpeechFileAsync(string conversationId, string fileName, BinaryData data) + try { var dir = Path.Combine(_baseDir, CONVERSATION_FOLDER, conversationId, TEXT_TO_SPEECH_FOLDER); if (!Directory.Exists(dir)) { Directory.CreateDirectory(dir); } + var filePath = Path.Combine(dir, fileName); - if (File.Exists(filePath)) return; - using var file = File.Create(filePath); - using var input = data.ToStream(); - await input.CopyToAsync(file); - } + if (File.Exists(filePath)) return false; - public async Task RetrieveSpeechFileAsync(string conversationId, string fileName) + using var fs = File.Create(filePath); + using var ds = data.ToStream(); + ds.CopyTo(fs); + return true; + } + catch (Exception ex) { - var path = Path.Combine(_baseDir, CONVERSATION_FOLDER, conversationId, TEXT_TO_SPEECH_FOLDER, fileName); - using var file = new FileStream(path, FileMode.Open, FileAccess.Read); - return await BinaryData.FromStreamAsync(file); + _logger.LogWarning($"Error when saving speech file. {fileName} ({conversationId})\r\n{ex.Message}\r\n{ex.InnerException}"); + return false; } } + + public BinaryData GetSpeechFile(string conversationId, string fileName) + { + var path = Path.Combine(_baseDir, CONVERSATION_FOLDER, conversationId, TEXT_TO_SPEECH_FOLDER, fileName); + using var file = new FileStream(path, FileMode.Open, FileAccess.Read); + return BinaryData.FromStream(file); + } } diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Conversation.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Conversation.cs index 248806eca..c40e2d593 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Conversation.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Conversation.cs @@ -6,7 +6,7 @@ namespace BotSharp.Core.Files.Services; public partial class LocalFileStorageService { - public async Task> GetMessageFileScreenshots(string conversationId, IEnumerable messageIds) + public async Task> GetMessageFileScreenshotsAsync(string conversationId, IEnumerable messageIds) { var files = new List(); if (string.IsNullOrEmpty(conversationId) || messageIds.IsNullOrEmpty()) diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.cs index 94ee2beb9..750803c42 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.cs @@ -10,12 +10,6 @@ public partial class LocalFileStorageService : IFileStorageService private readonly ILogger _logger; private readonly string _baseDir; - private readonly IEnumerable _audioTypes = new List - { - "mp3", - "wav" - }; - private const string CONVERSATION_FOLDER = "conversations"; private const string FILE_FOLDER = "files"; private const string USER_FILE_FOLDER = "user"; diff --git a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadPdfFn.cs b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadPdfFn.cs index e2b465c8e..814f943e3 100644 --- a/src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadPdfFn.cs +++ b/src/Plugins/BotSharp.Plugin.FileHandler/Functions/ReadPdfFn.cs @@ -52,7 +52,7 @@ private async Task> AssembleFiles(string conversationId, L var fileStorage = _services.GetRequiredService(); var messageIds = dialogs.Select(x => x.MessageId).Distinct().ToList(); - var screenshots = await fileStorage.GetMessageFileScreenshots(conversationId, messageIds); + var screenshots = await fileStorage.GetMessageFileScreenshotsAsync(conversationId, messageIds); if (screenshots.IsNullOrEmpty()) return dialogs; diff --git a/src/Plugins/BotSharp.Plugin.TencentCos/Modules/BucketClient.cs b/src/Plugins/BotSharp.Plugin.TencentCos/Modules/BucketClient.cs index 1886cfa07..fe6f0b7b8 100644 --- a/src/Plugins/BotSharp.Plugin.TencentCos/Modules/BucketClient.cs +++ b/src/Plugins/BotSharp.Plugin.TencentCos/Modules/BucketClient.cs @@ -12,6 +12,7 @@ public class BucketClient private readonly string _fullBucketName; private readonly string _appId; private readonly string _region; + public BucketClient(CosXmlServer cosXml, string fullBucketName, string appId, string region) { _cosXml = cosXml; @@ -141,7 +142,32 @@ public List GetDirFiles(string dir) var objects = info.contentsList; return objects.Where(o => o.size > 0).Select(o => o.key).ToList(); + } + catch (CosClientException clientEx) + { + throw new Exception(clientEx.Message); + } + catch (CosServerException serverEx) + { + throw new Exception(serverEx.Message); + } + } + + public string? GetDirFile(string dir, string key) + { + try + { + var request = new GetBucketRequest(_fullBucketName); + request.SetPrefix($"{dir.TrimEnd('/')}/"); + request.SetDelimiter("/"); + + var result = _cosXml.GetBucket(request); + + var info = result.listBucket; + + var objects = info.contentsList; + return objects.Where(o => o.size > 0).FirstOrDefault(o => o.key == key)?.key; } catch (CosClientException clientEx) { diff --git a/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Audio.cs b/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Audio.cs index 4d803628d..ec1a41617 100644 --- a/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Audio.cs +++ b/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Audio.cs @@ -2,13 +2,38 @@ namespace BotSharp.Plugin.TencentCos.Services; public partial class TencentCosService { - public Task SaveSpeechFileAsync(string conversationId, string fileName, BinaryData data) + public bool SaveSpeechFile(string conversationId, string fileName, BinaryData data) { - throw new NotImplementedException(); + try + { + var file = $"{CONVERSATION_FOLDER}/{conversationId}/{TEXT_TO_SPEECH_FOLDER}/{fileName}"; + var exist = _cosClient.BucketClient.DoesObjectExist(file); + if (exist) + { + return false; + } + + return _cosClient.BucketClient.UploadBytes(file, data.ToArray()); + } + catch (Exception ex) + { + _logger.LogWarning($"Error when saving speech file. {fileName} ({conversationId})\r\n{ex.Message}\r\n{ex.InnerException}"); + return false; + } } - public Task RetrieveSpeechFileAsync(string conversationId, string fileName) + public BinaryData GetSpeechFile(string conversationId, string fileName) { - throw new NotImplementedException(); + var dir = $"{CONVERSATION_FOLDER}/{conversationId}/{TEXT_TO_SPEECH_FOLDER}"; + var key = $"{dir}/{fileName}"; + var file = _cosClient.BucketClient.GetDirFile(dir, key); + + if (string.IsNullOrWhiteSpace(file)) + { + return BinaryData.Empty; + } + + var bytes = _cosClient.BucketClient.DownloadFileBytes(file); + return BinaryData.FromBytes(bytes); } } diff --git a/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Conversation.cs b/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Conversation.cs index 4c339f127..9badec03a 100644 --- a/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Conversation.cs +++ b/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Conversation.cs @@ -8,7 +8,7 @@ namespace BotSharp.Plugin.TencentCos.Services; public partial class TencentCosService { - public async Task> GetMessageFileScreenshots(string conversationId, IEnumerable messageIds) + public async Task> GetMessageFileScreenshotsAsync(string conversationId, IEnumerable messageIds) { var files = new List(); if (string.IsNullOrEmpty(conversationId) || messageIds.IsNullOrEmpty()) diff --git a/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.cs b/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.cs index ddb5e031c..c48812f1f 100644 --- a/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.cs +++ b/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.cs @@ -27,7 +27,7 @@ public partial class TencentCosService : IFileStorageService private const string USERS_FOLDER = "users"; private const string USER_AVATAR_FOLDER = "avatar"; private const string SESSION_FOLDER = "sessions"; - + private const string TEXT_TO_SPEECH_FOLDER = "speeches"; public TencentCosService( TencentCosSettings settings, diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs index 571a51013..cf49141b7 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs @@ -31,7 +31,11 @@ public TwilioVoiceController(TwilioSetting settings, IServiceProvider services, [HttpPost("twilio/voice/welcome")] public TwiMLResult InitiateConversation(VoiceRequest request, [FromQuery] string states) { - if (request?.CallSid == null) throw new ArgumentNullException(nameof(VoiceRequest.CallSid)); + if (request?.CallSid == null) + { + throw new ArgumentNullException(nameof(VoiceRequest.CallSid)); + } + string conversationId = $"TwilioVoice_{request.CallSid}"; var twilio = _services.GetRequiredService(); var url = $"twilio/voice/{conversationId}/receive/0?states={states}"; @@ -46,13 +50,16 @@ public async Task ReceiveCallerMessage([FromRoute] string conversat var twilio = _services.GetRequiredService(); var messageQueue = _services.GetRequiredService(); var sessionManager = _services.GetRequiredService(); + var messages = await sessionManager.RetrieveStagedCallerMessagesAsync(conversationId, seqNum); string text = (request.SpeechResult + "\r\n" + request.Digits).Trim(); + if (!string.IsNullOrWhiteSpace(text)) { messages.Add(text); await sessionManager.StageCallerMessageAsync(conversationId, seqNum, text); } + VoiceResponse response; if (messages.Count == 0 && seqNum == 0) { @@ -64,6 +71,7 @@ public async Task ReceiveCallerMessage([FromRoute] string conversat { messages = await sessionManager.RetrieveStagedCallerMessagesAsync(conversationId, seqNum - 1); } + var messageContent = string.Join("\r\n", messages); var callerMessage = new CallerMessage() { @@ -72,6 +80,7 @@ public async Task ReceiveCallerMessage([FromRoute] string conversat Content = messageContent, From = request.From }; + if (!string.IsNullOrEmpty(states)) { var kvp = states.Split(':'); @@ -80,11 +89,12 @@ public async Task ReceiveCallerMessage([FromRoute] string conversat callerMessage.States.Add(kvp[0], kvp[1]); } } - await messageQueue.EnqueueAsync(callerMessage); + await messageQueue.EnqueueAsync(callerMessage); int audioIndex = Random.Shared.Next(1, 5); response = twilio.ReturnInstructions($"twilio/hold-on-{audioIndex}.mp3", $"twilio/voice/{conversationId}/reply/{seqNum}?states={states}", true, 1); } + return TwiML(response); } @@ -95,10 +105,12 @@ public async Task ReplyCallerMessage([FromRoute] string conversatio var nextSeqNum = seqNum + 1; var sessionManager = _services.GetRequiredService(); var twilio = _services.GetRequiredService(); + if (request.SpeechResult != null) { await sessionManager.StageCallerMessageAsync(conversationId, nextSeqNum, request.SpeechResult); } + var reply = await sessionManager.GetAssistantReplyAsync(conversationId, seqNum); VoiceResponse response; if (reply == null) @@ -117,7 +129,7 @@ public async Task ReplyCallerMessage([FromRoute] string conversatio var fileStorage = _services.GetRequiredService(); var data = await completion.GenerateAudioFromTextAsync(indication); var fileName = $"indication_{seqNum}.mp3"; - await fileStorage.SaveSpeechFileAsync(conversationId, fileName, data); + fileStorage.SaveSpeechFile(conversationId, fileName, data); speechPath = $"twilio/voice/speeches/{conversationId}/{fileName}"; } response = twilio.ReturnInstructions(speechPath, $"twilio/voice/{conversationId}/reply/{seqNum}?states={states}", true, 2); @@ -137,17 +149,17 @@ public async Task ReplyCallerMessage([FromRoute] string conversatio { response = twilio.ReturnInstructions($"twilio/voice/speeches/{conversationId}/{reply.SpeechFileName}", $"twilio/voice/{conversationId}/receive/{nextSeqNum}?states={states}", true); } - } + return TwiML(response); } [ValidateRequest] [HttpGet("twilio/voice/speeches/{conversationId}/{fileName}")] - public async Task RetrieveSpeechFile([FromRoute] string conversationId, [FromRoute] string fileName) + public async Task GetSpeechFile([FromRoute] string conversationId, [FromRoute] string fileName) { - var fileService = _services.GetRequiredService(); - var data = await fileService.RetrieveSpeechFileAsync(conversationId, fileName); + var fileStorage = _services.GetRequiredService(); + var data = fileStorage.GetSpeechFile(conversationId, fileName); var result = new FileContentResult(data.ToArray(), "audio/mpeg"); result.FileDownloadName = fileName; return result; diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioMessageQueue.cs b/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioMessageQueue.cs index 455e314ce..dfdc66c33 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioMessageQueue.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioMessageQueue.cs @@ -7,6 +7,7 @@ public class TwilioMessageQueue { private readonly Channel _queue; internal ChannelReader Reader => _queue.Reader; + public TwilioMessageQueue() { BoundedChannelOptions options = new(100) @@ -18,7 +19,11 @@ public TwilioMessageQueue() public async ValueTask EnqueueAsync(CallerMessage request) { - if (request == null) throw new ArgumentNullException(nameof(request)); + if (request == null) + { + throw new ArgumentNullException(nameof(request)); + } + Console.WriteLine($"[{DateTime.UtcNow}] Enqueue {request}"); await _queue.Writer.WriteAsync(request); } diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioMessageQueueService.cs b/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioMessageQueueService.cs index 6b558aad2..825bdb3b8 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioMessageQueueService.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioMessageQueueService.cs @@ -57,22 +57,26 @@ private async Task ProcessUserMessageAsync(CallerMessage message) { using var scope = _serviceProvider.CreateScope(); var sp = scope.ServiceProvider; + AssistantMessage reply = null; var inputMsg = new RoleDialogModel(AgentRole.User, message.Content); var conv = sp.GetRequiredService(); var routing = sp.GetRequiredService(); var config = sp.GetRequiredService(); + routing.Context.SetMessageId(message.ConversationId, inputMsg.MessageId); var states = new List { new MessageState("channel", ConversationChannel.Phone), new MessageState("calling_phone", message.From) }; + foreach (var kvp in message.States) { states.Add(new MessageState(kvp.Key, kvp.Value)); } conv.SetConversationId(message.ConversationId, states); + var sessionManager = sp.GetRequiredService(); var result = await conv.SendMessage(config.AgentId, inputMsg, @@ -93,14 +97,15 @@ private async Task ProcessUserMessageAsync(CallerMessage message) await sessionManager.SetReplyIndicationAsync(message.ConversationId, message.SeqNumber, msg.Indication); } }, - async functionExecuted => - { } + async functionExecuted => { } ); + var completion = CompletionProvider.GetAudioCompletion(sp, "openai", "tts-1"); var fileStorage = sp.GetRequiredService(); var data = await completion.GenerateAudioFromTextAsync(reply.Content); var fileName = $"reply_{reply.MessageId}.mp3"; - await fileStorage.SaveSpeechFileAsync(message.ConversationId, fileName, data); + fileStorage.SaveSpeechFile(message.ConversationId, fileName, data); + reply.SpeechFileName = fileName; reply.Content = null; await sessionManager.SetAssistantReplyAsync(message.ConversationId, message.SeqNumber, reply); diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs b/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs index 309d627c8..e6f930bb6 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs @@ -59,6 +59,7 @@ public VoiceResponse ReturnInstructions(string message) }, Action = new Uri($"{_settings.CallbackHost}/twilio/voice/{twilioSetting.AgentId}") }; + gather.Say(message); response.Append(gather); return response; @@ -80,6 +81,7 @@ public VoiceResponse ReturnInstructions(string speechPath, string callbackPath, Timeout = timeout > 0 ? timeout : 3, ActionOnEmptyResult = actionOnEmptyResult }; + if (!string.IsNullOrEmpty(speechPath)) { gather.Play(new Uri($"{_settings.CallbackHost}/{speechPath}")); @@ -119,6 +121,7 @@ public VoiceResponse HoldOn(int interval, string message = null) Action = new Uri($"{_settings.CallbackHost}/twilio/voice/{twilioSetting.AgentId}"), ActionOnEmptyResult = true }; + if (!string.IsNullOrEmpty(message)) { gather.Say(message); diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioSessionManager.cs b/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioSessionManager.cs index eae0b238e..117fc3bbd 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioSessionManager.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioSessionManager.cs @@ -25,9 +25,7 @@ public async Task> RetrieveStagedCallerMessagesAsync(string convers { var db = _redis.GetDatabase(); var key = $"{conversationId}:Caller:{seqNum}"; - return (await db.ListRangeAsync(key)) - .Select(x => (string)x) - .ToList(); + return (await db.ListRangeAsync(key)).Select(x => (string)x).ToList(); } public async Task SetAssistantReplyAsync(string conversationId, int seqNum, AssistantMessage message) diff --git a/src/Plugins/BotSharp.Plugin.Twilio/TwilioPlugin.cs b/src/Plugins/BotSharp.Plugin.Twilio/TwilioPlugin.cs index fa971a7c5..2a6d7c221 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/TwilioPlugin.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/TwilioPlugin.cs @@ -17,9 +17,12 @@ public void RegisterDI(IServiceCollection services, IConfiguration config) var settingService = provider.GetRequiredService(); return settingService.Bind("Twilio"); }); + services.AddScoped(); + var conn = ConnectionMultiplexer.Connect(config["Twilio:RedisConnectionString"]); var sessionManager = new TwilioSessionManager(conn); + services.AddSingleton(sessionManager); services.AddSingleton(); services.AddHostedService(); From f4e20ac64c6ba7628c718f8e657667d9f339be23 Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Wed, 28 Aug 2024 17:45:33 -0500 Subject: [PATCH 10/27] add search filter --- .../BotSharp.Abstraction/Models/KeyValue.cs | 10 +++ .../VectorStorage/Models/VectorFilter.cs | 5 +- .../BotSharp.Plugin.Qdrant/QdrantDb.cs | 79 ++++++++++--------- .../SemanticKernelMemoryStoreProvider.cs | 2 +- 4 files changed, 58 insertions(+), 38 deletions(-) create mode 100644 src/Infrastructure/BotSharp.Abstraction/Models/KeyValue.cs diff --git a/src/Infrastructure/BotSharp.Abstraction/Models/KeyValue.cs b/src/Infrastructure/BotSharp.Abstraction/Models/KeyValue.cs new file mode 100644 index 000000000..f981ba6e9 --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/Models/KeyValue.cs @@ -0,0 +1,10 @@ +namespace BotSharp.Abstraction.Models; + +public class KeyValue +{ + [JsonPropertyName("key")] + public string Key { get; set; } + + [JsonPropertyName("value")] + public string Value { get; set; } +} diff --git a/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/VectorFilter.cs b/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/VectorFilter.cs index 91abcc25f..85b9dec29 100644 --- a/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/VectorFilter.cs +++ b/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/VectorFilter.cs @@ -4,4 +4,7 @@ public class VectorFilter : StringIdPagination { [JsonPropertyName("with_vector")] public bool WithVector { get; set; } -} + + [JsonPropertyName("search_pairs")] + public IEnumerable? SearchPairs { get; set; } +} \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs b/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs index 99d917d44..496a8fc21 100644 --- a/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs +++ b/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs @@ -51,10 +51,34 @@ public async Task> GetPagedCollectionDa return new StringIdPagedItems(); } - var totalPointCount = await client.CountAsync(collectionName); + // Build query filter + Filter? queryFilter = null; + if (!filter.SearchPairs.IsNullOrEmpty()) + { + var conditions = filter.SearchPairs.Select(x => new Condition + { + Field = new FieldCondition + { + Key = x.Key, + Match = new Match { Text = x.Value }, + } + }); + + queryFilter = new Filter + { + Should = + { + conditions + } + }; + } + + var totalPointCount = await client.CountAsync(collectionName, filter: queryFilter); var response = await client.ScrollAsync(collectionName, limit: (uint)filter.Size, - offset: !string.IsNullOrWhiteSpace(filter.StartId) ? new PointId { Uuid = filter.StartId } : 0, + offset: !string.IsNullOrWhiteSpace(filter.StartId) ? new PointId { Uuid = filter.StartId } : null, + filter: queryFilter, vectorsSelector: filter.WithVector); + var points = response?.Result?.Select(x => new VectorCollectionData { Id = x.Id?.Uuid ?? string.Empty, @@ -160,43 +184,26 @@ public async Task> Search(string collectionNam return results; } + var payloadSelector = new WithPayloadSelector { Enable = true }; + if (fields != null) + { + payloadSelector.Include = new PayloadIncludeSelector { Fields = { fields.ToArray() } }; + } + var points = await client.SearchAsync(collectionName, - vector, - limit: (ulong)limit, - scoreThreshold: confidence, - vectorsSelector: new WithVectorsSelector { Enable = withVector }); + vector, + limit: (ulong)limit, + scoreThreshold: confidence, + payloadSelector: payloadSelector, + vectorsSelector: withVector); - var pickFields = fields != null; - foreach (var point in points) + results = points.Select(x => new VectorCollectionData { - var data = new Dictionary(); - if (pickFields) - { - foreach (var field in fields) - { - if (point.Payload.ContainsKey(field)) - { - data[field] = point.Payload[field].StringValue; - } - else - { - data[field] = ""; - } - } - } - else - { - data = point.Payload.ToDictionary(k => k.Key, v => v.Value.StringValue); - } - - results.Add(new VectorCollectionData - { - Id = point.Id.Uuid, - Data = data, - Score = point.Score, - Vector = withVector ? point.Vectors?.Vector?.Data?.ToArray() : null - }); - } + Id = x.Id.Uuid, + Data = x.Payload.ToDictionary(x => x.Key, x => x.Value.StringValue), + Score = x.Score, + Vector = x.Vectors?.Vector?.Data?.ToArray() + }).ToList(); return results; } diff --git a/src/Plugins/BotSharp.Plugin.SemanticKernel/SemanticKernelMemoryStoreProvider.cs b/src/Plugins/BotSharp.Plugin.SemanticKernel/SemanticKernelMemoryStoreProvider.cs index a797b47fe..89510f4d0 100644 --- a/src/Plugins/BotSharp.Plugin.SemanticKernel/SemanticKernelMemoryStoreProvider.cs +++ b/src/Plugins/BotSharp.Plugin.SemanticKernel/SemanticKernelMemoryStoreProvider.cs @@ -31,7 +31,7 @@ public async Task CreateCollection(string collectionName, int dim) public Task> GetPagedCollectionData(string collectionName, VectorFilter filter) { - throw new System.NotImplementedException(); + throw new NotImplementedException(); } public Task> GetCollectionData(string collectionName, IEnumerable ids, From 3fd7f749cac45ca8517e33bd245896dc803b988b Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Thu, 29 Aug 2024 14:30:47 -0500 Subject: [PATCH 11/27] add create / delete vector collection --- .../Knowledges/IKnowledgeService.cs | 7 +++ .../VectorStorage/IVectorDb.cs | 3 +- .../Controllers/KnowledgeBaseController.cs | 19 +++++++ .../MemVecDb/MemoryVectorDb.cs | 10 +++- .../Services/KnowledgeService.Create.cs | 19 +++++++ .../Services/KnowledgeService.Delete.cs | 19 +++++++ .../Services/KnowledgeService.cs | 1 + .../Providers/FaissDb.cs | 7 ++- .../BotSharp.Plugin.Qdrant/QdrantDb.cs | 50 +++++++++++-------- .../SemanticKernelMemoryStoreProvider.cs | 9 +++- .../Modules/BucketClient.cs | 23 +-------- 11 files changed, 119 insertions(+), 48 deletions(-) diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs index 20825791f..1021ea1c0 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/IKnowledgeService.cs @@ -5,6 +5,9 @@ namespace BotSharp.Abstraction.Knowledges; public interface IKnowledgeService { + #region Vector + Task CreateVectorCollection(string collectionName, int dimension); + Task DeleteVectorCollection(string collectionName); Task> GetVectorCollections(); Task> SearchVectorKnowledge(string query, string collectionName, VectorSearchOptions options); Task FeedVectorKnowledge(string collectionName, KnowledgeCreationModel model); @@ -12,6 +15,10 @@ public interface IKnowledgeService Task DeleteVectorCollectionData(string collectionName, string id); Task CreateVectorCollectionData(string collectionName, VectorCreateModel create); Task UpdateVectorCollectionData(string collectionName, VectorUpdateModel update); + #endregion + + #region Graph Task SearchGraphKnowledge(string query, GraphSearchOptions options); Task SearchKnowledge(string query, string collectionName, VectorSearchOptions vectorOptions, GraphSearchOptions graphOptions); + #endregion } diff --git a/src/Infrastructure/BotSharp.Abstraction/VectorStorage/IVectorDb.cs b/src/Infrastructure/BotSharp.Abstraction/VectorStorage/IVectorDb.cs index 53b921a1a..c169988b1 100644 --- a/src/Infrastructure/BotSharp.Abstraction/VectorStorage/IVectorDb.cs +++ b/src/Infrastructure/BotSharp.Abstraction/VectorStorage/IVectorDb.cs @@ -9,7 +9,8 @@ public interface IVectorDb Task> GetCollections(); Task> GetPagedCollectionData(string collectionName, VectorFilter filter); Task> GetCollectionData(string collectionName, IEnumerable ids, bool withPayload = false, bool withVector = false); - Task CreateCollection(string collectionName, int dim); + Task CreateCollection(string collectionName, int dimension); + Task DeleteCollection(string collectionName); Task Upsert(string collectionName, Guid id, float[] vector, string text, Dictionary? payload = null); Task> Search(string collectionName, float[] vector, IEnumerable? fields, int limit = 5, float confidence = 0.5f, bool withVector = false); Task DeleteCollectionData(string collectionName, Guid id); diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs index deb21378a..4a045e4b0 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs @@ -18,12 +18,25 @@ public KnowledgeBaseController(IKnowledgeService knowledgeService, IServiceProvi _services = services; } + #region Vector [HttpGet("knowledge/vector/collections")] public async Task> GetVectorCollections() { return await _knowledgeService.GetVectorCollections(); } + [HttpPost("knowledge/vector/{collection}/create-collection/{dimension}")] + public async Task CreateVectorCollection([FromRoute] string collection, [FromRoute] int dimension) + { + return await _knowledgeService.CreateVectorCollection(collection, dimension); + } + + [HttpDelete("knowledge/vector/{collection}/delete-collection")] + public async Task GetVectorCollections([FromRoute] string collection) + { + return await _knowledgeService.DeleteVectorCollection(collection); + } + [HttpPost("/knowledge/vector/{collection}/search")] public async Task> SearchVectorKnowledge([FromRoute] string collection, [FromBody] SearchVectorKnowledgeRequest request) { @@ -109,7 +122,10 @@ public async Task UploadVectorKnowledge([FromRoute] string collec System.IO.File.Delete(filePath); return Ok(new { count = 1, file.Length }); } + #endregion + + #region Graph [HttpPost("/knowledge/graph/search")] public async Task SearchGraphKnowledge([FromBody] SearchGraphKnowledgeRequest request) { @@ -124,7 +140,9 @@ public async Task SearchGraphKnowledge([FromBody] Searc Result = result.Result }; } + #endregion + #region Knowledge [HttpPost("/knowledge/search")] public async Task SearchKnowledge([FromBody] SearchKnowledgeRequest request) { @@ -148,4 +166,5 @@ public async Task SearchKnowledge([FromBody] SearchKno GraphResult = result?.GraphResult != null ? new GraphKnowledgeViewModel { Result = result.GraphResult.Result } : null }; } + #endregion } diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemoryVectorDb.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemoryVectorDb.cs index aa5c0f4e8..2f4e2db6a 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemoryVectorDb.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemoryVectorDb.cs @@ -10,10 +10,16 @@ public class MemoryVectorDb : IVectorDb public string Name => "MemoryVector"; - public async Task CreateCollection(string collectionName, int dim) + public async Task CreateCollection(string collectionName, int dimension) { - _collections[collectionName] = dim; + _collections[collectionName] = dimension; _vectors[collectionName] = new List(); + return true; + } + + public async Task DeleteCollection(string collectionName) + { + return false; } public async Task> GetCollections() diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Create.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Create.cs index 9b185867e..1df460a0c 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Create.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Create.cs @@ -25,6 +25,25 @@ public async Task FeedVectorKnowledge(string collectionName, KnowledgeCreationMo } } + public async Task CreateVectorCollection(string collectionName, int dimension) + { + try + { + if (string.IsNullOrWhiteSpace(collectionName)) + { + return false; + } + + var db = GetVectorDb(); + return await db.CreateCollection(collectionName, dimension); + } + catch (Exception ex) + { + _logger.LogWarning($"Error when creating a vector collection ({collectionName}). {ex.Message}\r\n{ex.InnerException}"); + return false; + } + } + public async Task CreateVectorCollectionData(string collectionName, VectorCreateModel create) { try diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Delete.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Delete.cs index e78990a79..b0973a996 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Delete.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Delete.cs @@ -2,6 +2,25 @@ namespace BotSharp.Plugin.KnowledgeBase.Services; public partial class KnowledgeService { + public async Task DeleteVectorCollection(string collectionName) + { + try + { + if (string.IsNullOrWhiteSpace(collectionName)) + { + return false; + } + + var db = GetVectorDb(); + return await db.DeleteCollection(collectionName); + } + catch (Exception ex) + { + _logger.LogWarning($"Error when deleting collection ({collectionName}). {ex.Message}\r\n{ex.InnerException}"); + return false; + } + } + public async Task DeleteVectorCollectionData(string collectionName, string id) { try diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.cs index 4e40fea4b..aec32dc5b 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.cs @@ -1,3 +1,4 @@ + namespace BotSharp.Plugin.KnowledgeBase.Services; public partial class KnowledgeService : IKnowledgeService diff --git a/src/Plugins/BotSharp.Plugin.MetaAI/Providers/FaissDb.cs b/src/Plugins/BotSharp.Plugin.MetaAI/Providers/FaissDb.cs index 902087d78..30150d4c1 100644 --- a/src/Plugins/BotSharp.Plugin.MetaAI/Providers/FaissDb.cs +++ b/src/Plugins/BotSharp.Plugin.MetaAI/Providers/FaissDb.cs @@ -11,7 +11,12 @@ public class FaissDb : IVectorDb { public string Name => "Faiss"; - public Task CreateCollection(string collectionName, int dim) + public Task CreateCollection(string collectionName, int dimension) + { + throw new NotImplementedException(); + } + + public Task DeleteCollection(string collectionName) { throw new NotImplementedException(); } diff --git a/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs b/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs index 496a8fc21..927f36164 100644 --- a/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs +++ b/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs @@ -35,6 +35,34 @@ private QdrantClient GetClient() return _client; } + public async Task CreateCollection(string collectionName, int dim) + { + var client = GetClient(); + var exist = await DoesCollectionExist(client, collectionName); + + if (exist) return false; + + // Create a new collection + await client.CreateCollectionAsync(collectionName, new VectorParams() + { + Size = (ulong)dim, + Distance = Distance.Cosine + }); + + return true; + } + + public async Task DeleteCollection(string collectionName) + { + var client = GetClient(); + var exist = await DoesCollectionExist(client, collectionName); + + if (!exist) return false; + + await client.DeleteCollectionAsync(collectionName); + return true; + } + public async Task> GetCollections() { // List all the collections @@ -117,27 +145,7 @@ public async Task> GetCollectionData(string co }); } - public async Task CreateCollection(string collectionName, int dim) - { - var client = GetClient(); - var exist = await DoesCollectionExist(client, collectionName); - if (!exist) - { - // Create a new collection - await client.CreateCollectionAsync(collectionName, new VectorParams() - { - Size = (ulong)dim, - Distance = Distance.Cosine - }); - } - - // Get collection info - var collectionInfo = await client.GetCollectionInfoAsync(collectionName); - if (collectionInfo == null) - { - throw new Exception($"Create {collectionName} failed."); - } - } + public async Task Upsert(string collectionName, Guid id, float[] vector, string text, Dictionary? payload = null) { diff --git a/src/Plugins/BotSharp.Plugin.SemanticKernel/SemanticKernelMemoryStoreProvider.cs b/src/Plugins/BotSharp.Plugin.SemanticKernel/SemanticKernelMemoryStoreProvider.cs index 89510f4d0..11df098c3 100644 --- a/src/Plugins/BotSharp.Plugin.SemanticKernel/SemanticKernelMemoryStoreProvider.cs +++ b/src/Plugins/BotSharp.Plugin.SemanticKernel/SemanticKernelMemoryStoreProvider.cs @@ -24,9 +24,16 @@ public SemanticKernelMemoryStoreProvider(IMemoryStore memoryStore) public string Name => "SemanticKernel"; - public async Task CreateCollection(string collectionName, int dim) + public async Task CreateCollection(string collectionName, int dimension) { await _memoryStore.CreateCollectionAsync(collectionName); + return true; + } + + public async Task DeleteCollection(string collectionName) + { + await _memoryStore.DeleteCollectionAsync(collectionName); + return false; } public Task> GetPagedCollectionData(string collectionName, VectorFilter filter) diff --git a/src/Plugins/BotSharp.Plugin.TencentCos/Modules/BucketClient.cs b/src/Plugins/BotSharp.Plugin.TencentCos/Modules/BucketClient.cs index fe6f0b7b8..c0c2d0766 100644 --- a/src/Plugins/BotSharp.Plugin.TencentCos/Modules/BucketClient.cs +++ b/src/Plugins/BotSharp.Plugin.TencentCos/Modules/BucketClient.cs @@ -155,28 +155,7 @@ public List GetDirFiles(string dir) public string? GetDirFile(string dir, string key) { - try - { - var request = new GetBucketRequest(_fullBucketName); - request.SetPrefix($"{dir.TrimEnd('/')}/"); - request.SetDelimiter("/"); - - var result = _cosXml.GetBucket(request); - - var info = result.listBucket; - - var objects = info.contentsList; - - return objects.Where(o => o.size > 0).FirstOrDefault(o => o.key == key)?.key; - } - catch (CosClientException clientEx) - { - throw new Exception(clientEx.Message); - } - catch (CosServerException serverEx) - { - throw new Exception(serverEx.Message); - } + return GetDirFiles(dir).FirstOrDefault(x => x == key); } public List GetDirectories(string dir) From 09137ff8e7b30e9e5ec60961c35c217ad7df2be5 Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Thu, 29 Aug 2024 14:57:58 -0500 Subject: [PATCH 12/27] get knowledge base text embedding dimension from llm --- .../Embedding/TextEmbeddingProvider.cs | 2 +- .../Helpers/KnowledgeSettingHelper.cs | 18 +++++++++++++++++- .../Embedding/TextEmbeddingProvider.cs | 2 +- 3 files changed, 19 insertions(+), 3 deletions(-) diff --git a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Embedding/TextEmbeddingProvider.cs b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Embedding/TextEmbeddingProvider.cs index 39119050d..762e870f8 100644 --- a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Embedding/TextEmbeddingProvider.cs +++ b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Embedding/TextEmbeddingProvider.cs @@ -8,7 +8,7 @@ public class TextEmbeddingProvider : ITextEmbedding protected readonly IServiceProvider _services; protected readonly ILogger _logger; - private const int DEFAULT_DIMENSION = 3072; + private const int DEFAULT_DIMENSION = 1536; protected string _model; protected int _dimension; diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Helpers/KnowledgeSettingHelper.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Helpers/KnowledgeSettingHelper.cs index b1b280eba..3b05bdba0 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Helpers/KnowledgeSettingHelper.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Helpers/KnowledgeSettingHelper.cs @@ -1,3 +1,5 @@ +using BotSharp.Abstraction.Settings; + namespace BotSharp.Plugin.KnowledgeBase.Helpers; public static class KnowledgeSettingHelper @@ -12,8 +14,22 @@ public static ITextEmbedding GetTextEmbeddingSetting(IServiceProvider services, } var embedding = services.GetServices().FirstOrDefault(x => x.Provider == found.Provider); + var dimension = found.Dimension; + + if (found.Dimension <= 0) + { + dimension = GetLlmTextEmbeddingDimension(services, found.Provider, found.Model); + } + embedding.SetModelName(found.Model); - embedding.SetDimension(found.Dimension); + embedding.SetDimension(dimension); return embedding; } + + private static int GetLlmTextEmbeddingDimension(IServiceProvider services, string provider, string model) + { + var settings = services.GetRequiredService(); + var found = settings.GetSetting(provider, model); + return found?.Dimension ?? 0; + } } diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Embedding/TextEmbeddingProvider.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Embedding/TextEmbeddingProvider.cs index bebf6853a..e27d4fcb1 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Embedding/TextEmbeddingProvider.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Embedding/TextEmbeddingProvider.cs @@ -8,7 +8,7 @@ public class TextEmbeddingProvider : ITextEmbedding protected readonly IServiceProvider _services; protected readonly ILogger _logger; - private const int DEFAULT_DIMENSION = 3072; + private const int DEFAULT_DIMENSION = 1536; protected string _model = "text-embedding-3-large"; protected int _dimension = DEFAULT_DIMENSION; From c3f0bdea14fae5690bba06c6bf5fcfcbbf742bb6 Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Thu, 29 Aug 2024 15:00:28 -0500 Subject: [PATCH 13/27] add comments --- .../Conversations/Models/RoleDialogModel.cs | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/RoleDialogModel.cs b/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/RoleDialogModel.cs index 3d8a25e9c..f93ef96f1 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/RoleDialogModel.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/RoleDialogModel.cs @@ -75,6 +75,9 @@ public class RoleDialogModel : ITrackableMessage [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public RichContent? RichContent { get; set; } + /// + /// Rich content for secondary language + /// [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public RichContent? SecondaryRichContent { get; set; } @@ -86,6 +89,9 @@ public class RoleDialogModel : ITrackableMessage public FunctionCallFromLlm Instruction { get; set; } + /// + /// Files to be used in conversation + /// public List Files { get; set; } = new List(); /// @@ -94,7 +100,14 @@ public class RoleDialogModel : ITrackableMessage [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] [JsonPropertyName("generated_images")] public List GeneratedImages { get; set; } = new List(); + + /// + /// Knowledge confidence + /// + [JsonPropertyName("knowledge_confidence")] public float KnowledgeConfidence { get; set; } = 0.5f; + + private RoleDialogModel() { } From f74237e8bb5ee73363ccc5b9b71b343decd07516 Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Fri, 30 Aug 2024 11:29:21 -0500 Subject: [PATCH 14/27] minor change --- .../Controllers/TwilioVoiceController.cs | 21 ++++++++++++------- .../Services/TwilioService.cs | 4 +++- 2 files changed, 17 insertions(+), 8 deletions(-) diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs index 9d0368f6f..912dd122b 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs @@ -115,6 +115,7 @@ public async Task ReplyCallerMessage([FromRoute] string conversatio var nextSeqNum = seqNum + 1; var sessionManager = _services.GetRequiredService(); var twilio = _services.GetRequiredService(); + var fileStorage = _services.GetRequiredService(); if (request.SpeechResult != null) { @@ -123,6 +124,7 @@ public async Task ReplyCallerMessage([FromRoute] string conversatio var reply = await sessionManager.GetAssistantReplyAsync(conversationId, seqNum); VoiceResponse response; + if (reply == null) { var indication = await sessionManager.GetReplyIndicationAsync(conversationId, seqNum); @@ -139,11 +141,11 @@ public async Task ReplyCallerMessage([FromRoute] string conversatio } else { - var textToSpeechService = CompletionProvider.GetTextToSpeech(_services, "openai", "tts-1"); - var fileService = _services.GetRequiredService(); - var data = await textToSpeechService.GenerateSpeechFromTextAsync(seg); + var completion = CompletionProvider.GetAudioCompletion(_services, "openai", "tts-1"); + var data = await completion.GenerateAudioFromTextAsync(seg); + var fileName = $"indication_{seqNum}_{segIndex}.mp3"; - await fileService.SaveSpeechFileAsync(conversationId, fileName, data); + fileStorage.SaveSpeechFile(conversationId, fileName, data); speechPaths.Add($"twilio/voice/speeches/{conversationId}/{fileName}"); segIndex++; } @@ -168,7 +170,10 @@ public async Task ReplyCallerMessage([FromRoute] string conversatio } else { - response = twilio.ReturnInstructions(new List { $"twilio/voice/speeches/{conversationId}/{reply.SpeechFileName}" }, $"twilio/voice/{conversationId}/receive/{nextSeqNum}?states={states}", true); + response = twilio.ReturnInstructions(new List + { + $"twilio/voice/speeches/{conversationId}/{reply.SpeechFileName}" + }, $"twilio/voice/{conversationId}/receive/{nextSeqNum}?states={states}", true); } } @@ -181,8 +186,10 @@ public async Task GetSpeechFile([FromRoute] string conversati { var fileStorage = _services.GetRequiredService(); var data = fileStorage.GetSpeechFile(conversationId, fileName); - var result = new FileContentResult(data.ToArray(), "audio/mpeg"); - result.FileDownloadName = fileName; + var result = new FileContentResult(data.ToArray(), "audio/mpeg") + { + FileDownloadName = fileName + }; return result; } } diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs b/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs index 611007faa..c43782ab7 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioService.cs @@ -1,3 +1,4 @@ +using BotSharp.Abstraction.Utilities; using Twilio.Jwt.AccessToken; using Token = Twilio.Jwt.AccessToken.Token; @@ -81,7 +82,8 @@ public VoiceResponse ReturnInstructions(List speechPaths, string callbac Timeout = timeout > 0 ? timeout : 2, ActionOnEmptyResult = actionOnEmptyResult }; - if (speechPaths != null && speechPaths.Any()) + + if (!speechPaths.IsNullOrEmpty()) { foreach (var speechPath in speechPaths) { From 6aece1d73c7f781a4e1b69a29cd5989b93dcde04 Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Fri, 30 Aug 2024 13:09:28 -0500 Subject: [PATCH 15/27] clean comments --- .../BotSharp.Plugin.KnowledgeBase/MemVecDb/MemoryVectorDb.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemoryVectorDb.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemoryVectorDb.cs index 2f4e2db6a..3a695e935 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemoryVectorDb.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemoryVectorDb.cs @@ -47,7 +47,6 @@ public async Task> Search(string collectionNam } var similarities = VectorHelper.CalCosineSimilarity(vector, _vectors[collectionName]); - // var similarities = VectorUtility.CalEuclideanDistance(vector, _vectors[collectionName]); var results = np.argsort(similarities).ToArray() .Reverse() From 4102f6fa1f83d7c9dc40a6885f8ea2f5fb983eb6 Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Fri, 30 Aug 2024 16:12:20 -0500 Subject: [PATCH 16/27] refine planner --- .../Routing/RoutingService.GetPlanner.cs | 1 - .../templates/database_knowledge.liquid | 8 +- .../Controllers/LlmProviderController.cs | 2 +- .../Embedding/TextEmbeddingProvider.cs | 2 +- .../Functions/PrimaryStagePlanFn.cs | 33 +++++--- .../Functions/SecondaryStagePlanFn.cs | 43 ++++++----- .../Functions/SummaryPlanFn.cs | 14 ++-- .../templates/two_stage.1st.plan.liquid | 6 +- .../templates/two_stage.2nd.plan.liquid | 2 +- .../Functions/AddDatabaseKnowledge.cs | 75 +++++++++++-------- .../Functions/GetTableDefinitionFn.cs | 30 +++----- 11 files changed, 118 insertions(+), 98 deletions(-) diff --git a/src/Infrastructure/BotSharp.Core/Routing/RoutingService.GetPlanner.cs b/src/Infrastructure/BotSharp.Core/Routing/RoutingService.GetPlanner.cs index 72549e517..0fcaa5a16 100644 --- a/src/Infrastructure/BotSharp.Core/Routing/RoutingService.GetPlanner.cs +++ b/src/Infrastructure/BotSharp.Core/Routing/RoutingService.GetPlanner.cs @@ -1,4 +1,3 @@ -using BotSharp.Abstraction.Agents.Models; using BotSharp.Abstraction.Routing.Enums; using BotSharp.Abstraction.Routing.Planning; using BotSharp.Core.Routing.Planning; diff --git a/src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/templates/database_knowledge.liquid b/src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/templates/database_knowledge.liquid index 0332af76c..4c9f51c37 100644 --- a/src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/templates/database_knowledge.liquid +++ b/src/Infrastructure/BotSharp.Core/data/agents/01fcc3e5-9af7-49e6-ad7a-a760bd12dc4a/templates/database_knowledge.liquid @@ -1,8 +1,10 @@ -You are a knowledge generator assistant. Based on the provided mysql table structure, including tablename, fieldname,data type and comments, generate the related knowledge for DBA and BA. When user ask the question, they don't know the table name. -the summarized question/answer should: +You are a knowledge generator assistant. Based on the provided mysql table structure, including tablename, fieldname, data type and comments, generate the related knowledge for DBA and BA. When users ask the question, they don't know the table name. + +The summarized question/answer should: 1. help user to identify the location of tables to find further information 2. identify the table structure and data relationship based on the task description -3. summarize all the table to table relationship information based on the FOREIGN KEY, and include both table in the answer +3. summarize all the tables to table relationship information based on the FOREIGN KEY, and include both tables in the answer + Go through all the columns and generate multiple question & answer pairs. The output should be question/answer pair list in JSON: [{"question":"","answer":""}]. And the new line should be replaced with \r\n. diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/LlmProviderController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/LlmProviderController.cs index 5571d9fa9..02282b858 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/LlmProviderController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/LlmProviderController.cs @@ -25,6 +25,6 @@ public IEnumerable GetLlmProviders() public IEnumerable GetLlmProviderModels([FromRoute] string provider) { var list = _llmProvider.GetProviderModels(provider); - return list.Where(x => !x.ImageGeneration); + return list.Where(x => x.Type == LlmModelType.Chat); } } diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Embedding/TextEmbeddingProvider.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Embedding/TextEmbeddingProvider.cs index e27d4fcb1..cb9e7d15e 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Embedding/TextEmbeddingProvider.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Embedding/TextEmbeddingProvider.cs @@ -9,7 +9,7 @@ public class TextEmbeddingProvider : ITextEmbedding protected readonly ILogger _logger; private const int DEFAULT_DIMENSION = 1536; - protected string _model = "text-embedding-3-large"; + protected string _model = "text-embedding-3-small"; protected int _dimension = DEFAULT_DIMENSION; public virtual string Provider => "openai"; diff --git a/src/Plugins/BotSharp.Plugin.Planner/Functions/PrimaryStagePlanFn.cs b/src/Plugins/BotSharp.Plugin.Planner/Functions/PrimaryStagePlanFn.cs index a12cafed3..2842397bf 100644 --- a/src/Plugins/BotSharp.Plugin.Planner/Functions/PrimaryStagePlanFn.cs +++ b/src/Plugins/BotSharp.Plugin.Planner/Functions/PrimaryStagePlanFn.cs @@ -13,9 +13,9 @@ namespace BotSharp.Plugin.Planner.Functions; public class PrimaryStagePlanFn : IFunctionCallback { public string Name => "plan_primary_stage"; + private readonly IServiceProvider _services; - private readonly ILogger _logger; - private object aiAssistant; + private readonly ILogger _logger; public PrimaryStagePlanFn(IServiceProvider services, ILogger logger) { @@ -25,31 +25,33 @@ public PrimaryStagePlanFn(IServiceProvider services, ILogger public async Task Execute(RoleDialogModel message) { - //debug + // Debug var state = _services.GetRequiredService(); state.SetState("max_tokens", "4096"); var task = JsonSerializer.Deserialize(message.FunctionArgs); - //get knowledge from vectordb + // Get knowledge from vectordb var fn = _services.GetRequiredService(); var msg = new ExtractedKnowledge { Question = task.Question, }; + var retrievalMessage = new RoleDialogModel(AgentRole.User, task.Requirements) { FunctionArgs = JsonSerializer.Serialize(msg), KnowledgeConfidence = 0.1f, - Content = "" + Content = string.Empty }; + await fn.InvokeFunction("knowledge_retrieval", retrievalMessage); message.Content = retrievalMessage.Content; var agentService = _services.GetRequiredService(); var currentAgent = await agentService.LoadAgent(message.CurrentAgentId); - //send knowledge to AI to refine and summarize the primary planning + // Send knowledge to AI to refine and summarize the primary planning var firstPlanningPrompt = await GetFirstStagePlanPrompt(task, message); var plannerAgent = new Agent { @@ -59,7 +61,7 @@ public async Task Execute(RoleDialogModel message) TemplateDict = new Dictionary(), LlmConfig = currentAgent.LlmConfig }; - var response = await GetAIResponse(plannerAgent); + var response = await GetAiResponse(plannerAgent); message.Content = response.Content; /*await fn.InvokeFunction("plan_secondary_stage", message); @@ -96,12 +98,14 @@ public async Task Execute(RoleDialogModel message) message.StopCompletion = true;*/ return true; } + private async Task GetFirstStagePlanPrompt(PrimaryRequirementRequest task, RoleDialogModel message) { var agentService = _services.GetRequiredService(); - var aiAssistant = await agentService.GetAgent(BuiltInAgentId.Planner); var render = _services.GetRequiredService(); - var template = aiAssistant.Templates.First(x => x.Name == "two_stage.1st.plan").Content; + + var aiAssistant = await agentService.GetAgent(BuiltInAgentId.Planner); + var template = aiAssistant.Templates.FirstOrDefault(x => x.Name == "two_stage.1st.plan")?.Content ?? string.Empty; var responseFormat = JsonSerializer.Serialize(new FirstStagePlan { Parameters = [JsonDocument.Parse("{}")], @@ -115,13 +119,15 @@ private async Task GetFirstStagePlanPrompt(PrimaryRequirementRequest tas { "response_format", responseFormat } }); } + private async Task GetPlanSummaryPrompt(PrimaryRequirementRequest task, RoleDialogModel message) { // save to knowledge base var agentService = _services.GetRequiredService(); - var aiAssistant = await agentService.GetAgent(BuiltInAgentId.AIAssistant); var render = _services.GetRequiredService(); - var template = aiAssistant.Templates.First(x => x.Name == "planner_prompt.two_stage.summarize").Content; + + var aiAssistant = await agentService.GetAgent(BuiltInAgentId.AIAssistant); + var template = aiAssistant.Templates.FirstOrDefault(x => x.Name == "planner_prompt.two_stage.summarize")?.Content ?? string.Empty; var responseFormat = JsonSerializer.Serialize(new FirstStagePlan { Parameters = [JsonDocument.Parse("{}")], @@ -136,10 +142,12 @@ private async Task GetPlanSummaryPrompt(PrimaryRequirementRequest task, { "response_format", responseFormat } }); } - private async Task GetAIResponse(Agent plannerAgent) + + private async Task GetAiResponse(Agent plannerAgent) { var conv = _services.GetRequiredService(); var wholeDialogs = conv.GetDialogHistory(); + //add "test" to wholeDialogs' last element if(plannerAgent.Name == "planner_summary") { @@ -147,6 +155,7 @@ private async Task GetAIResponse(Agent plannerAgent) wholeDialogs.Last().Content += "\n\nIf the table structure didn't mention auto incremental, the data field id needs to insert id manually and you need to use max(id) instead of LAST_INSERT_ID function.\nFor example, you should use SET @id = select max(id) from table;"; wholeDialogs.Last().Content += "\n\nTry if you can generate a single query to fulfill the needs"; } + if (plannerAgent.Name == "planning_1st") { //add "test" to wholeDialogs' last element in a new paragraph diff --git a/src/Plugins/BotSharp.Plugin.Planner/Functions/SecondaryStagePlanFn.cs b/src/Plugins/BotSharp.Plugin.Planner/Functions/SecondaryStagePlanFn.cs index d6e022996..2723d0708 100644 --- a/src/Plugins/BotSharp.Plugin.Planner/Functions/SecondaryStagePlanFn.cs +++ b/src/Plugins/BotSharp.Plugin.Planner/Functions/SecondaryStagePlanFn.cs @@ -1,13 +1,10 @@ -using Azure; using BotSharp.Abstraction.Conversations.Models; using BotSharp.Abstraction.Functions; using BotSharp.Abstraction.Knowledges.Models; -using BotSharp.Abstraction.MLTasks; using BotSharp.Abstraction.Routing; using BotSharp.Abstraction.Templating; using BotSharp.Core.Infrastructures; using BotSharp.Plugin.Planner.TwoStaging.Models; -using NetTopologySuite.Index.HPRtree; using System.Threading.Tasks; using Microsoft.Extensions.Logging; @@ -16,8 +13,9 @@ namespace BotSharp.Plugin.Planner.Functions; public class SecondaryStagePlanFn : IFunctionCallback { public string Name => "plan_secondary_stage"; + private readonly IServiceProvider _services; - private readonly ILogger _logger; + private readonly ILogger _logger; public SecondaryStagePlanFn(IServiceProvider services, ILogger logger) { @@ -28,45 +26,49 @@ public SecondaryStagePlanFn(IServiceProvider services, ILogger Execute(RoleDialogModel message) { var fn = _services.GetRequiredService(); - var msg_secondary = RoleDialogModel.From(message); - var task_primary = JsonSerializer.Deserialize(message.FunctionArgs); - msg_secondary.FunctionArgs = JsonSerializer.Serialize(new SecondaryBreakdownTask + + var msgSecondary = RoleDialogModel.From(message); + var taskPrimary = JsonSerializer.Deserialize(message.FunctionArgs); + + msgSecondary.FunctionArgs = JsonSerializer.Serialize(new SecondaryBreakdownTask { - TaskDescription = task_primary.Requirements + TaskDescription = taskPrimary.Requirements }); - var task_secondary = JsonSerializer.Deserialize(msg_secondary.FunctionArgs); - var items = msg_secondary.Content.JsonArrayContent(); - msg_secondary.KnowledgeConfidence = 0.5f; + var taskSecondary = JsonSerializer.Deserialize(msgSecondary.FunctionArgs); + var items = msgSecondary.Content.JsonArrayContent(); + + msgSecondary.KnowledgeConfidence = 0.5f; foreach (var item in items) { if (item.NeedAdditionalInformation) { - msg_secondary.FunctionArgs = JsonSerializer.Serialize(new ExtractedKnowledge + msgSecondary.FunctionArgs = JsonSerializer.Serialize(new ExtractedKnowledge { Question = item.Task }); - await fn.InvokeFunction("knowledge_retrieval", msg_secondary); - message.Content += msg_secondary.Content; + await fn.InvokeFunction("knowledge_retrieval", msgSecondary); + message.Content += msgSecondary.Content; } } + // load agent var agentService = _services.GetRequiredService(); var currentAgent = await agentService.LoadAgent(message.CurrentAgentId); - var secondPlanningPrompt = await GetSecondStagePlanPrompt(task_secondary, message); + var secondPlanningPrompt = await GetSecondStagePlanPrompt(taskSecondary, message); _logger.LogInformation(secondPlanningPrompt); var plannerAgent = new Agent { - Id = "", + Id = string.Empty, Name = "test", Instruction = secondPlanningPrompt, TemplateDict = new Dictionary(), LlmConfig = currentAgent.LlmConfig }; - var response = await GetAIResponse(plannerAgent); + var response = await GetAiResponse(plannerAgent); message.Content = response.Content; _logger.LogInformation(response.Content); return true; @@ -74,9 +76,10 @@ public async Task Execute(RoleDialogModel message) private async Task GetSecondStagePlanPrompt(SecondaryBreakdownTask task, RoleDialogModel message) { var agentService = _services.GetRequiredService(); - var planner = await agentService.GetAgent(message.CurrentAgentId); var render = _services.GetRequiredService(); - var template = planner.Templates.First(x => x.Name == "two_stage.2nd.plan").Content; + + var planner = await agentService.GetAgent(message.CurrentAgentId); + var template = planner.Templates.FirstOrDefault(x => x.Name == "two_stage.2nd.plan")?.Content ?? string.Empty; var responseFormat = JsonSerializer.Serialize(new SecondStagePlan { Tool = "tool name if task solution provided", @@ -91,7 +94,7 @@ private async Task GetSecondStagePlanPrompt(SecondaryBreakdownTask task, { "response_format", responseFormat } }); } - private async Task GetAIResponse(Agent plannerAgent) + private async Task GetAiResponse(Agent plannerAgent) { var conv = _services.GetRequiredService(); var wholeDialogs = conv.GetDialogHistory(); diff --git a/src/Plugins/BotSharp.Plugin.Planner/Functions/SummaryPlanFn.cs b/src/Plugins/BotSharp.Plugin.Planner/Functions/SummaryPlanFn.cs index 098edd533..5fd9fe943 100644 --- a/src/Plugins/BotSharp.Plugin.Planner/Functions/SummaryPlanFn.cs +++ b/src/Plugins/BotSharp.Plugin.Planner/Functions/SummaryPlanFn.cs @@ -32,6 +32,7 @@ public async Task Execute(RoleDialogModel message) // summarize and generate query var summaryPlanningPrompt = await GetPlanSummaryPrompt(task, message); _logger.LogInformation(summaryPlanningPrompt); + var plannerAgent = new Agent { Id = BuiltInAgentId.Planner, @@ -39,7 +40,7 @@ public async Task Execute(RoleDialogModel message) Instruction = summaryPlanningPrompt, TemplateDict = new Dictionary() }; - var response_summary = await GetAIResponse(plannerAgent); + var response_summary = await GetAiResponse(plannerAgent); message.Content = response_summary.Content; message.StopCompletion = true; @@ -51,9 +52,10 @@ private async Task GetPlanSummaryPrompt(string task, RoleDialogModel mes { // save to knowledge base var agentService = _services.GetRequiredService(); - var aiAssistant = await agentService.GetAgent(message.CurrentAgentId); var render = _services.GetRequiredService(); - var template = aiAssistant.Templates.First(x => x.Name == "two_stage.summarize").Content; + + var aiAssistant = await agentService.GetAgent(BuiltInAgentId.Planner); + var template = aiAssistant.Templates.FirstOrDefault(x => x.Name == "two_stage.summarize")?.Content ?? string.Empty; var responseFormat = JsonSerializer.Serialize(new FirstStagePlan { Parameters = [JsonDocument.Parse("{}")], @@ -68,17 +70,19 @@ private async Task GetPlanSummaryPrompt(string task, RoleDialogModel mes { "response_format", responseFormat } }); } - private async Task GetAIResponse(Agent plannerAgent) + private async Task GetAiResponse(Agent plannerAgent) { var conv = _services.GetRequiredService(); var wholeDialogs = conv.GetDialogHistory(); + //add "test" to wholeDialogs' last element - if(plannerAgent.Name == "planner_summary") + if (plannerAgent.Name == "planner_summary") { //add "test" to wholeDialogs' last element in a new paragraph wholeDialogs.Last().Content += "\n\nIf the table structure didn't mention auto incremental, the data field id needs to insert id manually and you need to use max(id) instead of LAST_INSERT_ID function.\nFor example, you should use SET @id = select max(id) from table;"; wholeDialogs.Last().Content += "\n\nTry if you can generate a single query to fulfill the needs"; } + if (plannerAgent.Name == "planning_1st") { //add "test" to wholeDialogs' last element in a new paragraph diff --git a/src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/templates/two_stage.1st.plan.liquid b/src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/templates/two_stage.1st.plan.liquid index 0a1f62928..fb13b0c14 100644 --- a/src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/templates/two_stage.1st.plan.liquid +++ b/src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/templates/two_stage.1st.plan.liquid @@ -3,7 +3,7 @@ You are a Task Planner. you will breakdown user business requirements into excut Thinking process: 1. Reference to "Task Knowledge" if there is relevant knowledge; 2. Breakdown task into subtasks. - - The subtask should contains all needed parameters for subsequent steps. + - The subtask should contain all needed parameters for subsequent steps. - If limited information provided and there are furture information needed, or miss relationship between steps, set the need_additional_information to true. - If there is extra knowledge or relationship needed between steps, set the need_additional_information to true for both steps. - If the solution mentioned "related solutions" is needed, set the need_additional_information to true. @@ -22,9 +22,7 @@ Task Knowledge: {{ k }} {% endfor %} {%- endif %} - ===== Task description: -{{ task_description }} - +{{ task_description }} \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/templates/two_stage.2nd.plan.liquid b/src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/templates/two_stage.2nd.plan.liquid index ccec8033b..d5340e2e4 100644 --- a/src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/templates/two_stage.2nd.plan.liquid +++ b/src/Plugins/BotSharp.Plugin.Planner/data/agents/282a7128-69a1-44b0-878c-a9159b88f3b9/templates/two_stage.2nd.plan.liquid @@ -1,5 +1,5 @@ Reference to "Primary Planning" and the additional knowledge included. Breakdown task into multiple steps. -* the step should contains all needed parameters. +* The step should contains all needed parameters. * The parameters can be extracted from the original task. * You need to list all the steps in detail. Finding relationships should also be a step. * When generate the steps, you should find the relationships between data structure based on the provided knowledge strictly. diff --git a/src/Plugins/BotSharp.Plugin.SqlDriver/Functions/AddDatabaseKnowledge.cs b/src/Plugins/BotSharp.Plugin.SqlDriver/Functions/AddDatabaseKnowledge.cs index 5c2bfa456..5a50e355b 100644 --- a/src/Plugins/BotSharp.Plugin.SqlDriver/Functions/AddDatabaseKnowledge.cs +++ b/src/Plugins/BotSharp.Plugin.SqlDriver/Functions/AddDatabaseKnowledge.cs @@ -3,58 +3,69 @@ using MySqlConnector; using static Dapper.SqlMapper; using BotSharp.Abstraction.Agents.Enums; - +using Microsoft.Extensions.Logging; namespace BotSharp.Plugin.Planner.Functions; + public class AddDatabaseKnowledgeFn : IFunctionCallback { public string Name => "add_database_knowledge"; + private readonly IServiceProvider _services; - private object aiAssistant; + private readonly ILogger _logger; - public AddDatabaseKnowledgeFn(IServiceProvider services) + public AddDatabaseKnowledgeFn( + IServiceProvider services, + ILogger logger) { _services = services; + _logger = logger; } + public async Task Execute(RoleDialogModel message) { var agentService = _services.GetRequiredService(); var sqlDriver = _services.GetRequiredService(); var fn = _services.GetRequiredService(); var settings = _services.GetRequiredService(); + + + var allTables = new HashSet(); using var connection = new MySqlConnection(settings.MySqlConnectionString); - var dictionary = new Dictionary(); - List allTables = new List(); var sql = $"select table_name from information_schema.tables;"; - var result = connection.Query(sql: sql,dictionary); - foreach (var item in result) + var results = connection.Query(sql, new Dictionary()); + + foreach (var item in results) { + if (item == null) continue; + allTables.Add(item.TABLE_NAME); } - message.Data = allTables.Distinct().ToList(); + message.Data = allTables.ToList(); var currentAgent = await agentService.LoadAgent(message.CurrentAgentId); - var note = ""; - foreach (var item in allTables) + var errorNote = string.Empty; + + foreach (var table in allTables) { - message.Data = new List { item }; + message.Data = new List { table }; await fn.InvokeFunction("get_table_definition", message); - var PlanningPrompt = await GetPrompt(message); + var planningPrompt = await GetPrompt(message); var plannerAgent = new Agent { - Id = "", - Name = "database_knowledge", - Instruction = PlanningPrompt, - TemplateDict = new Dictionary(), + Id = string.Empty, + Name = "Database Knowledge", + Instruction = planningPrompt, LlmConfig = currentAgent.LlmConfig }; - var response = await GetAIResponse(plannerAgent); + try { - var knowledge = response.Content.JsonArrayContent(); - foreach (var k in knowledge) + var response = await GetAiResponse(plannerAgent); + var knowledges = response.Content.JsonArrayContent(); + foreach (var k in knowledges) { try { @@ -64,42 +75,44 @@ public async Task Execute(RoleDialogModel message) Answer = k.Answer }); await fn.InvokeFunction("memorize_knowledge", message); - message.SecondaryContent += $"Table: {item}, Question:{k.Question}, {message.Content} \r\n"; + message.SecondaryContent += $"Table: {table}, Question: {k.Question}, {message.Content}\r\n"; } catch (Exception e) { - note += $"Error processing table {item}: {e.Message}\r\n{e.InnerException}"; + var note = $"Error processing table {table}: {e.Message}\r\n{e.InnerException}"; + errorNote += note; + _logger.LogWarning(note); } } } catch (Exception e) { - note += $"Error processing table {item}: {e.Message}\r\n{e.InnerException}"; + errorNote += $"Error processing table {table}: {e.Message}\r\n{e.InnerException}\r\n"; + _logger.LogWarning(errorNote); } } return true; } - private async Task GetAIResponse(Agent plannerAgent) + + private async Task GetAiResponse(Agent plannerAgent) { var conv = _services.GetRequiredService(); var wholeDialogs = conv.GetDialogHistory(); - var completion = CompletionProvider.GetChatCompletion(_services, + + var completion = CompletionProvider.GetChatCompletion(_services, provider: plannerAgent.LlmConfig.Provider, model: plannerAgent.LlmConfig.Model); return await completion.GetChatCompletions(plannerAgent, wholeDialogs); } + private async Task GetPrompt(RoleDialogModel message) { var agentService = _services.GetRequiredService(); - var aiAssistant = await agentService.GetAgent(BuiltInAgentId.AIAssistant); var render = _services.GetRequiredService(); - var template = aiAssistant.Templates.First(x => x.Name == "database_knowledge").Content; - var responseFormat = JsonSerializer.Serialize(new ExtractedKnowledge - { - Question = "question", - Answer = "answer" - }); + + var aiAssistant = await agentService.GetAgent(BuiltInAgentId.AIAssistant); + var template = aiAssistant.Templates.FirstOrDefault(x => x.Name == "database_knowledge")?.Content ?? string.Empty; return render.Render(template, new Dictionary { diff --git a/src/Plugins/BotSharp.Plugin.SqlDriver/Functions/GetTableDefinitionFn.cs b/src/Plugins/BotSharp.Plugin.SqlDriver/Functions/GetTableDefinitionFn.cs index 74ae8f729..47f64a1f3 100644 --- a/src/Plugins/BotSharp.Plugin.SqlDriver/Functions/GetTableDefinitionFn.cs +++ b/src/Plugins/BotSharp.Plugin.SqlDriver/Functions/GetTableDefinitionFn.cs @@ -1,5 +1,3 @@ -using BotSharp.Plugin.SqlDriver.Models; -using Microsoft.EntityFrameworkCore.Metadata.Internal; using MySqlConnector; using static Dapper.SqlMapper; @@ -17,37 +15,31 @@ public GetTableDefinitionFn(IServiceProvider services) public async Task Execute(RoleDialogModel message) { - // get agent service var agentService = _services.GetRequiredService(); - - // var args = JsonSerializer.Deserialize(message.FunctionArgs); var sqlDriver = _services.GetRequiredService(); - - //get table DDL from database var settings = _services.GetRequiredService(); + + // Get table DDL from database using var connection = new MySqlConnection(settings.MySqlConnectionString); var dictionary = new Dictionary(); + var tableDdls = new List(); - var table_ddl = ""; foreach (var p in (List)message.Data) { - dictionary["@" + "table_name"] = p; var escapedTableName = MySqlHelper.EscapeString(p); + dictionary["@" + "table_name"] = p; dictionary["table_name"] = escapedTableName; - // can you replace this with a parameterized query? - var sql = $"select * from information_schema.tables where table_name ='{dictionary["table_name"]}'"; + var sql = $"select * from information_schema.tables where table_name ='{escapedTableName}'"; var result = connection.QueryFirstOrDefault(sql: sql, dictionary); - if (result != null) - { - sql = $"SHOW CREATE TABLE `{dictionary["table_name"]}`"; - result = connection.QueryFirstOrDefault(sql: sql, dictionary); - table_ddl += "\r\n" + result; - } - + if (result == null) continue; + + sql = $"SHOW CREATE TABLE `{escapedTableName}`"; + result = connection.QueryFirstOrDefault(sql: sql, dictionary); + tableDdls.Add(result); } - message.Content = table_ddl; + message.Content = string.Join("\r\n", tableDdls); return true; } } From a528a3f6b9242c4b7223d42844a32e857721e749 Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Fri, 30 Aug 2024 16:59:37 -0500 Subject: [PATCH 17/27] upgrade apphost --- src/BotSharp.AppHost/BotSharp.AppHost.csproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/BotSharp.AppHost/BotSharp.AppHost.csproj b/src/BotSharp.AppHost/BotSharp.AppHost.csproj index 6e8a2ae32..2a2705db0 100644 --- a/src/BotSharp.AppHost/BotSharp.AppHost.csproj +++ b/src/BotSharp.AppHost/BotSharp.AppHost.csproj @@ -10,7 +10,7 @@ - + From f38fdf7497e441034c72006f51f68feced7f239b Mon Sep 17 00:00:00 2001 From: Jicheng Lu Date: Fri, 30 Aug 2024 19:31:18 -0500 Subject: [PATCH 18/27] update settings --- .../Files/Converters/IPdf2ImageConverter.cs | 2 +- .../Files/FileCoreSettings.cs | 9 +++++++-- .../BotSharp.Abstraction/Graph/IGraphDb.cs | 2 +- .../Knowledges/IPdf2TextConverter.cs | 2 +- .../Knowledges/Settings/KnowledgeBaseSettings.cs | 12 ++++++++---- .../VectorStorage/IVectorDb.cs | 2 +- .../Services/Instruct/FileInstructService.Pdf.cs | 2 +- .../LocalFileStorageService.Conversation.cs | 2 +- .../Controllers/KnowledgeBaseController.cs | 2 +- src/Plugins/BotSharp.Plugin.Graph/GraphDb.cs | 2 +- .../Functions/KnowledgeRetrievalFn.cs | 2 +- .../Functions/MemorizeKnowledgeFn.cs | 2 +- .../Helpers/KnowledgeSettingHelper.cs | 2 -- .../MemVecDb/MemoryVectorDb.cs | 2 +- .../Services/KnowledgeService.cs | 4 ++-- .../Services/PigPdf2TextConverter.cs | 2 +- .../BotSharp.Plugin.MetaAI/Providers/FaissDb.cs | 2 +- .../Providers/Pdf2TextConverter.cs | 2 +- src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs | 2 +- .../SemanticKernelMemoryStoreProvider.cs | 2 +- .../Services/TencentCosService.Conversation.cs | 2 +- src/WebStarter/appsettings.json | 16 ++++++++++++---- 22 files changed, 46 insertions(+), 31 deletions(-) diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/Converters/IPdf2ImageConverter.cs b/src/Infrastructure/BotSharp.Abstraction/Files/Converters/IPdf2ImageConverter.cs index 87df61374..24b284029 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Files/Converters/IPdf2ImageConverter.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Files/Converters/IPdf2ImageConverter.cs @@ -2,7 +2,7 @@ namespace BotSharp.Abstraction.Files.Converters; public interface IPdf2ImageConverter { - public string Name { get; } + public string Provider { get; } /// /// Convert pdf pages to images, and return a list of image file paths diff --git a/src/Infrastructure/BotSharp.Abstraction/Files/FileCoreSettings.cs b/src/Infrastructure/BotSharp.Abstraction/Files/FileCoreSettings.cs index 10ccd1a09..dc4ac40bb 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Files/FileCoreSettings.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Files/FileCoreSettings.cs @@ -5,6 +5,11 @@ namespace BotSharp.Abstraction.Files; public class FileCoreSettings { public string Storage { get; set; } = FileStorageEnum.LocalFileStorage; - public string Pdf2TextConverter { get; set; } - public string Pdf2ImageConverter { get; set; } + public SettingBase Pdf2TextConverter { get; set; } + public SettingBase Pdf2ImageConverter { get; set; } } + +public class SettingBase +{ + public string Provider { get; set; } +} \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.Abstraction/Graph/IGraphDb.cs b/src/Infrastructure/BotSharp.Abstraction/Graph/IGraphDb.cs index 64c5388b9..b15503e22 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Graph/IGraphDb.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Graph/IGraphDb.cs @@ -4,7 +4,7 @@ namespace BotSharp.Abstraction.Graph; public interface IGraphDb { - public string Name { get; } + public string Provider { get; } Task Search(string query, GraphSearchOptions options); } diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/IPdf2TextConverter.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/IPdf2TextConverter.cs index b8f2d47b1..d6edad54c 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Knowledges/IPdf2TextConverter.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/IPdf2TextConverter.cs @@ -2,7 +2,7 @@ namespace BotSharp.Abstraction.Knowledges { public interface IPdf2TextConverter { - public string Name { get; } + public string Provider { get; } Task ConvertPdfToText(string filePath, int? startPageNum, int? endPageNum); } } \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Settings/KnowledgeBaseSettings.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Settings/KnowledgeBaseSettings.cs index 963a2b3b3..217dafb9e 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Knowledges/Settings/KnowledgeBaseSettings.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Knowledges/Settings/KnowledgeBaseSettings.cs @@ -4,8 +4,8 @@ namespace BotSharp.Abstraction.Knowledges.Settings; public class KnowledgeBaseSettings { - public string VectorDb { get; set; } - public string GraphDb { get; set; } + public SettingBase VectorDb { get; set; } + public SettingBase GraphDb { get; set; } public DefaultKnowledgeBaseSetting Default { get; set; } public List Collections { get; set; } = new(); @@ -23,9 +23,13 @@ public class VectorCollectionSetting public KnowledgeTextEmbeddingSetting TextEmbedding { get; set; } } -public class KnowledgeTextEmbeddingSetting +public class KnowledgeTextEmbeddingSetting : SettingBase { - public string Provider { get; set; } public string Model { get; set; } public int Dimension { get; set; } +} + +public class SettingBase +{ + public string Provider { get; set; } } \ No newline at end of file diff --git a/src/Infrastructure/BotSharp.Abstraction/VectorStorage/IVectorDb.cs b/src/Infrastructure/BotSharp.Abstraction/VectorStorage/IVectorDb.cs index c169988b1..cedacb46c 100644 --- a/src/Infrastructure/BotSharp.Abstraction/VectorStorage/IVectorDb.cs +++ b/src/Infrastructure/BotSharp.Abstraction/VectorStorage/IVectorDb.cs @@ -4,7 +4,7 @@ namespace BotSharp.Abstraction.VectorStorage; public interface IVectorDb { - string Name { get; } + string Provider { get; } Task> GetCollections(); Task> GetPagedCollectionData(string collectionName, VectorFilter filter); diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Pdf.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Pdf.cs index e7228b138..29dfdaf9a 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Pdf.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Pdf.cs @@ -98,7 +98,7 @@ private async Task> ConvertPdfToImages(IEnumerable f { var images = new List(); var settings = _services.GetRequiredService(); - var converter = _services.GetServices().FirstOrDefault(x => x.Name == settings.Pdf2ImageConverter); + var converter = _services.GetServices().FirstOrDefault(x => x.Provider == settings.Pdf2ImageConverter.Provider); if (converter == null || files.IsNullOrEmpty()) { return images; diff --git a/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Conversation.cs b/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Conversation.cs index c40e2d593..5638179cf 100644 --- a/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Conversation.cs +++ b/src/Infrastructure/BotSharp.Core/Files/Services/Storage/LocalFileStorageService.Conversation.cs @@ -276,7 +276,7 @@ private async Task> ConvertPdfToImages(string pdfLoc, string private IPdf2ImageConverter? GetPdf2ImageConverter() { var settings = _services.GetRequiredService(); - var converter = _services.GetServices().FirstOrDefault(x => x.Name == settings.Pdf2ImageConverter); + var converter = _services.GetServices().FirstOrDefault(x => x.Provider == settings.Pdf2ImageConverter.Provider); return converter; } diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs index 4a045e4b0..260dc166c 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs @@ -104,7 +104,7 @@ public async Task DeleteVectorCollectionData([FromRoute] string collection public async Task UploadVectorKnowledge([FromRoute] string collection, IFormFile file, [FromForm] int? startPageNum, [FromForm] int? endPageNum) { var setttings = _services.GetRequiredService(); - var textConverter = _services.GetServices().FirstOrDefault(x => x.Name == setttings.Pdf2TextConverter); + var textConverter = _services.GetServices().FirstOrDefault(x => x.Provider == setttings.Pdf2TextConverter.Provider); var filePath = Path.GetTempFileName(); using (var stream = new FileStream(filePath, FileMode.Create, FileAccess.Write, FileShare.None)) diff --git a/src/Plugins/BotSharp.Plugin.Graph/GraphDb.cs b/src/Plugins/BotSharp.Plugin.Graph/GraphDb.cs index 55a11ec9e..9aaf68ebc 100644 --- a/src/Plugins/BotSharp.Plugin.Graph/GraphDb.cs +++ b/src/Plugins/BotSharp.Plugin.Graph/GraphDb.cs @@ -35,7 +35,7 @@ public GraphDb( _settings = settings; } - public string Name => "Remote"; + public string Provider => "Remote"; public async Task Search(string query, GraphSearchOptions options) { diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/KnowledgeRetrievalFn.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/KnowledgeRetrievalFn.cs index 7a7db8aec..d6d719d75 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/KnowledgeRetrievalFn.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/KnowledgeRetrievalFn.cs @@ -21,7 +21,7 @@ public async Task Execute(RoleDialogModel message) var embedding = KnowledgeSettingHelper.GetTextEmbeddingSetting(_services, collectionName); var vector = await embedding.GetVectorAsync(args.Question); - var vectorDb = _services.GetServices().FirstOrDefault(x => x.Name == _settings.VectorDb); + var vectorDb = _services.GetServices().FirstOrDefault(x => x.Provider == _settings.VectorDb.Provider); var knowledges = await vectorDb.Search(collectionName, vector, new List { KnowledgePayloadName.Text, KnowledgePayloadName.Answer }); if (!knowledges.IsNullOrEmpty()) diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/MemorizeKnowledgeFn.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/MemorizeKnowledgeFn.cs index f77954d8f..ca545c658 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/MemorizeKnowledgeFn.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/MemorizeKnowledgeFn.cs @@ -25,7 +25,7 @@ public async Task Execute(RoleDialogModel message) args.Question }); - var vectorDb = _services.GetServices().FirstOrDefault(x => x.Name == _settings.VectorDb); + var vectorDb = _services.GetServices().FirstOrDefault(x => x.Provider == _settings.VectorDb.Provider); await vectorDb.CreateCollection(collectionName, vector[0].Length); var result = await vectorDb.Upsert(collectionName, Guid.NewGuid(), vector[0], diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Helpers/KnowledgeSettingHelper.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Helpers/KnowledgeSettingHelper.cs index 3b05bdba0..48033c912 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Helpers/KnowledgeSettingHelper.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Helpers/KnowledgeSettingHelper.cs @@ -1,5 +1,3 @@ -using BotSharp.Abstraction.Settings; - namespace BotSharp.Plugin.KnowledgeBase.Helpers; public static class KnowledgeSettingHelper diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemoryVectorDb.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemoryVectorDb.cs index 3a695e935..17a2c7572 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemoryVectorDb.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/MemVecDb/MemoryVectorDb.cs @@ -8,7 +8,7 @@ public class MemoryVectorDb : IVectorDb private readonly Dictionary> _vectors = new Dictionary>(); - public string Name => "MemoryVector"; + public string Provider => "MemoryVector"; public async Task CreateCollection(string collectionName, int dimension) { diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.cs index aec32dc5b..bf5973953 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.cs @@ -22,13 +22,13 @@ public KnowledgeService( private IVectorDb GetVectorDb() { - var db = _services.GetServices().FirstOrDefault(x => x.Name == _settings.VectorDb); + var db = _services.GetServices().FirstOrDefault(x => x.Provider == _settings.VectorDb.Provider); return db; } private IGraphDb GetGraphDb() { - var db = _services.GetServices().FirstOrDefault(x => x.Name == _settings.GraphDb); + var db = _services.GetServices().FirstOrDefault(x => x.Provider == _settings.GraphDb.Provider); return db; } diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/PigPdf2TextConverter.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/PigPdf2TextConverter.cs index 09401f207..c334505ed 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/PigPdf2TextConverter.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/PigPdf2TextConverter.cs @@ -5,7 +5,7 @@ namespace BotSharp.Plugin.KnowledgeBase.Services; public class PigPdf2TextConverter : IPdf2TextConverter { - public string Name => "Pig"; + public string Provider => "Pig"; public Task ConvertPdfToText(string filePath, int? startPageNum, int? endPageNum) { diff --git a/src/Plugins/BotSharp.Plugin.MetaAI/Providers/FaissDb.cs b/src/Plugins/BotSharp.Plugin.MetaAI/Providers/FaissDb.cs index 30150d4c1..58fb18512 100644 --- a/src/Plugins/BotSharp.Plugin.MetaAI/Providers/FaissDb.cs +++ b/src/Plugins/BotSharp.Plugin.MetaAI/Providers/FaissDb.cs @@ -9,7 +9,7 @@ namespace BotSharp.Plugin.MetaAI.Providers; public class FaissDb : IVectorDb { - public string Name => "Faiss"; + public string Provider => "Faiss"; public Task CreateCollection(string collectionName, int dimension) { diff --git a/src/Plugins/BotSharp.Plugin.PaddleSharp/Providers/Pdf2TextConverter.cs b/src/Plugins/BotSharp.Plugin.PaddleSharp/Providers/Pdf2TextConverter.cs index 81c53f835..37ea35c43 100644 --- a/src/Plugins/BotSharp.Plugin.PaddleSharp/Providers/Pdf2TextConverter.cs +++ b/src/Plugins/BotSharp.Plugin.PaddleSharp/Providers/Pdf2TextConverter.cs @@ -29,7 +29,7 @@ public Pdf2TextConverter(PaddleSharpSettings paddleSharpSettings) _paddleSharpSettings = paddleSharpSettings; } - public string Name => "Paddle"; + public string Provider => "Paddle"; public async Task ConvertPdfToText(string filePath, int? startPageNum, int? endPageNum) { diff --git a/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs b/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs index 927f36164..be89fec05 100644 --- a/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs +++ b/src/Plugins/BotSharp.Plugin.Qdrant/QdrantDb.cs @@ -19,7 +19,7 @@ public QdrantDb( _services = services; } - public string Name => "Qdrant"; + public string Provider => "Qdrant"; private QdrantClient GetClient() { diff --git a/src/Plugins/BotSharp.Plugin.SemanticKernel/SemanticKernelMemoryStoreProvider.cs b/src/Plugins/BotSharp.Plugin.SemanticKernel/SemanticKernelMemoryStoreProvider.cs index 11df098c3..4912e1276 100644 --- a/src/Plugins/BotSharp.Plugin.SemanticKernel/SemanticKernelMemoryStoreProvider.cs +++ b/src/Plugins/BotSharp.Plugin.SemanticKernel/SemanticKernelMemoryStoreProvider.cs @@ -22,7 +22,7 @@ public SemanticKernelMemoryStoreProvider(IMemoryStore memoryStore) } - public string Name => "SemanticKernel"; + public string Provider => "SemanticKernel"; public async Task CreateCollection(string collectionName, int dimension) { diff --git a/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Conversation.cs b/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Conversation.cs index 9badec03a..ff8dfa17c 100644 --- a/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Conversation.cs +++ b/src/Plugins/BotSharp.Plugin.TencentCos/Services/TencentCosService.Conversation.cs @@ -254,7 +254,7 @@ private async Task> ConvertPdfToImages(string pdfLoc, string private IPdf2ImageConverter? GetPdf2ImageConverter() { var settings = _services.GetRequiredService(); - var converter = _services.GetServices().FirstOrDefault(x => x.Name == settings.Pdf2ImageConverter); + var converter = _services.GetServices().FirstOrDefault(x => x.Provider == settings.Pdf2ImageConverter.Provider); return converter; } diff --git a/src/WebStarter/appsettings.json b/src/WebStarter/appsettings.json index 5b2203fbb..f470fe25f 100644 --- a/src/WebStarter/appsettings.json +++ b/src/WebStarter/appsettings.json @@ -233,8 +233,12 @@ "FileCore": { "Storage": "LocalFileStorage", - "Pdf2TextConverter": "", - "Pdf2ImageConverter": "" + "Pdf2TextConverter": { + "Provider": "" + }, + "Pdf2ImageConverter": { + "Provider": "" + } }, "TencentCos": { @@ -262,8 +266,12 @@ }, "KnowledgeBase": { - "VectorDb": "Qdrant", - "GraphDb": "Default", + "VectorDb": { + "Provider": "Qdrant" + }, + "GraphDb": { + "Provider": "Remote" + }, "Default": { "CollectionName": "BotSharp", "TextEmbedding": { From e660957c469fdd6c8af2cfcc2c861f7867a6a1a1 Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Fri, 30 Aug 2024 20:10:00 -0500 Subject: [PATCH 19/27] update settings --- src/Plugins/BotSharp.Plugin.Graph/GraphDb.cs | 2 +- src/Plugins/BotSharp.Plugin.Graph/GraphDbSettings.cs | 1 + src/WebStarter/appsettings.json | 3 ++- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/Plugins/BotSharp.Plugin.Graph/GraphDb.cs b/src/Plugins/BotSharp.Plugin.Graph/GraphDb.cs index 9aaf68ebc..0465dc4e0 100644 --- a/src/Plugins/BotSharp.Plugin.Graph/GraphDb.cs +++ b/src/Plugins/BotSharp.Plugin.Graph/GraphDb.cs @@ -44,7 +44,7 @@ public async Task Search(string query, GraphSearchOptions optio return new GraphSearchData(); } - var url = $"{_settings.BaseUrl}/query"; + var url = $"{_settings.BaseUrl}{_settings.SearchPath}"; var request = new GraphQueryRequest { Query = query, diff --git a/src/Plugins/BotSharp.Plugin.Graph/GraphDbSettings.cs b/src/Plugins/BotSharp.Plugin.Graph/GraphDbSettings.cs index 893513989..34e0b5fd8 100644 --- a/src/Plugins/BotSharp.Plugin.Graph/GraphDbSettings.cs +++ b/src/Plugins/BotSharp.Plugin.Graph/GraphDbSettings.cs @@ -3,4 +3,5 @@ namespace BotSharp.Plugin.Graph; public class GraphDbSettings { public string BaseUrl { get; set; } + public string SearchPath { get; set; } } diff --git a/src/WebStarter/appsettings.json b/src/WebStarter/appsettings.json index f470fe25f..4fcc745ca 100644 --- a/src/WebStarter/appsettings.json +++ b/src/WebStarter/appsettings.json @@ -254,7 +254,8 @@ }, "Graph": { - "BaseUrl": "" + "BaseUrl": "", + "SearchPath": "" }, "WeChat": { From 495ac9949c0494b45f7342adfb795f11cc1c2938 Mon Sep 17 00:00:00 2001 From: Jicheng Lu Date: Sat, 31 Aug 2024 00:57:49 -0500 Subject: [PATCH 20/27] relocate knowledge service --- .../BotSharp.Core/BotSharp.Core.csproj | 2 +- .../Helpers/KnowledgeSettingHelper.cs | 5 ++++- .../Knowledges/KnowledgeCorePlugin.cs | 19 +++++++++++++++++++ .../Services/KnowledgeService.Create.cs | 9 +++++++-- .../Services/KnowledgeService.Delete.cs | 2 +- .../Services/KnowledgeService.Get.cs | 4 +++- .../Services/KnowledgeService.Update.cs | 4 +++- .../Knowledges}/Services/KnowledgeService.cs | 10 ++++++---- .../Functions/KnowledgeRetrievalFn.cs | 2 ++ .../Functions/MemorizeKnowledgeFn.cs | 2 ++ .../KnowledgeBasePlugin.cs | 1 - 11 files changed, 48 insertions(+), 12 deletions(-) rename src/{Plugins/BotSharp.Plugin.KnowledgeBase => Infrastructure/BotSharp.Core/Knowledges}/Helpers/KnowledgeSettingHelper.cs (89%) create mode 100644 src/Infrastructure/BotSharp.Core/Knowledges/KnowledgeCorePlugin.cs rename src/{Plugins/BotSharp.Plugin.KnowledgeBase => Infrastructure/BotSharp.Core/Knowledges}/Services/KnowledgeService.Create.cs (88%) rename src/{Plugins/BotSharp.Plugin.KnowledgeBase => Infrastructure/BotSharp.Core/Knowledges}/Services/KnowledgeService.Delete.cs (95%) rename src/{Plugins/BotSharp.Plugin.KnowledgeBase => Infrastructure/BotSharp.Core/Knowledges}/Services/KnowledgeService.Get.cs (96%) rename src/{Plugins/BotSharp.Plugin.KnowledgeBase => Infrastructure/BotSharp.Core/Knowledges}/Services/KnowledgeService.Update.cs (91%) rename src/{Plugins/BotSharp.Plugin.KnowledgeBase => Infrastructure/BotSharp.Core/Knowledges}/Services/KnowledgeService.cs (80%) diff --git a/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj b/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj index ce4c82d20..ddd39d0cf 100644 --- a/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj +++ b/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj @@ -1,4 +1,4 @@ - + $(TargetFramework) diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Helpers/KnowledgeSettingHelper.cs b/src/Infrastructure/BotSharp.Core/Knowledges/Helpers/KnowledgeSettingHelper.cs similarity index 89% rename from src/Plugins/BotSharp.Plugin.KnowledgeBase/Helpers/KnowledgeSettingHelper.cs rename to src/Infrastructure/BotSharp.Core/Knowledges/Helpers/KnowledgeSettingHelper.cs index 48033c912..a278d7bff 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Helpers/KnowledgeSettingHelper.cs +++ b/src/Infrastructure/BotSharp.Core/Knowledges/Helpers/KnowledgeSettingHelper.cs @@ -1,4 +1,7 @@ -namespace BotSharp.Plugin.KnowledgeBase.Helpers; +using BotSharp.Abstraction.Knowledges.Settings; +using BotSharp.Abstraction.MLTasks; + +namespace BotSharp.Core.Knowledges.Helpers; public static class KnowledgeSettingHelper { diff --git a/src/Infrastructure/BotSharp.Core/Knowledges/KnowledgeCorePlugin.cs b/src/Infrastructure/BotSharp.Core/Knowledges/KnowledgeCorePlugin.cs new file mode 100644 index 000000000..6118647b1 --- /dev/null +++ b/src/Infrastructure/BotSharp.Core/Knowledges/KnowledgeCorePlugin.cs @@ -0,0 +1,19 @@ +using BotSharp.Core.Knowledges.Services; +using Microsoft.Extensions.Configuration; + +namespace BotSharp.Core.Knowledges; + +public class KnowledgeCorePlugin : IBotSharpPlugin +{ + public string Id => "a5ebc8f9-d089-44d0-bf00-4eac97a050bc"; + + public string Name => "Knowledge Core"; + + public string Description => "Provides core knowledge services."; + + + public void RegisterDI(IServiceCollection services, IConfiguration config) + { + services.AddScoped(); + } +} \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Create.cs b/src/Infrastructure/BotSharp.Core/Knowledges/Services/KnowledgeService.Create.cs similarity index 88% rename from src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Create.cs rename to src/Infrastructure/BotSharp.Core/Knowledges/Services/KnowledgeService.Create.cs index 1df460a0c..2927d9529 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Create.cs +++ b/src/Infrastructure/BotSharp.Core/Knowledges/Services/KnowledgeService.Create.cs @@ -1,11 +1,16 @@ -namespace BotSharp.Plugin.KnowledgeBase.Services; +using BotSharp.Abstraction.Knowledges.Models; +using BotSharp.Abstraction.VectorStorage.Models; + +namespace BotSharp.Core.Knowledges.Services; public partial class KnowledgeService { public async Task FeedVectorKnowledge(string collectionName, KnowledgeCreationModel knowledge) { var index = 0; - var lines = _textChopper.Chop(knowledge.Content, new ChunkOption + + var textChopper = _services.GetRequiredService(); + var lines = textChopper.Chop(knowledge.Content, new ChunkOption { Size = 1024, Conjunction = 32, diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Delete.cs b/src/Infrastructure/BotSharp.Core/Knowledges/Services/KnowledgeService.Delete.cs similarity index 95% rename from src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Delete.cs rename to src/Infrastructure/BotSharp.Core/Knowledges/Services/KnowledgeService.Delete.cs index b0973a996..404b4e2ca 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Delete.cs +++ b/src/Infrastructure/BotSharp.Core/Knowledges/Services/KnowledgeService.Delete.cs @@ -1,4 +1,4 @@ -namespace BotSharp.Plugin.KnowledgeBase.Services; +namespace BotSharp.Core.Knowledges.Services; public partial class KnowledgeService { diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Get.cs b/src/Infrastructure/BotSharp.Core/Knowledges/Services/KnowledgeService.Get.cs similarity index 96% rename from src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Get.cs rename to src/Infrastructure/BotSharp.Core/Knowledges/Services/KnowledgeService.Get.cs index ee36780ec..69093659a 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Get.cs +++ b/src/Infrastructure/BotSharp.Core/Knowledges/Services/KnowledgeService.Get.cs @@ -1,6 +1,8 @@ using BotSharp.Abstraction.Graph.Models; +using BotSharp.Abstraction.Knowledges.Models; +using BotSharp.Abstraction.VectorStorage.Models; -namespace BotSharp.Plugin.KnowledgeBase.Services; +namespace BotSharp.Core.Knowledges.Services; public partial class KnowledgeService { diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Update.cs b/src/Infrastructure/BotSharp.Core/Knowledges/Services/KnowledgeService.Update.cs similarity index 91% rename from src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Update.cs rename to src/Infrastructure/BotSharp.Core/Knowledges/Services/KnowledgeService.Update.cs index 0fa1f5cd8..b9652f617 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Update.cs +++ b/src/Infrastructure/BotSharp.Core/Knowledges/Services/KnowledgeService.Update.cs @@ -1,4 +1,6 @@ -namespace BotSharp.Plugin.KnowledgeBase.Services; +using BotSharp.Abstraction.VectorStorage.Models; + +namespace BotSharp.Core.Knowledges.Services; public partial class KnowledgeService { diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.cs b/src/Infrastructure/BotSharp.Core/Knowledges/Services/KnowledgeService.cs similarity index 80% rename from src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.cs rename to src/Infrastructure/BotSharp.Core/Knowledges/Services/KnowledgeService.cs index bf5973953..2c0853d0a 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.cs +++ b/src/Infrastructure/BotSharp.Core/Knowledges/Services/KnowledgeService.cs @@ -1,22 +1,24 @@ +using BotSharp.Abstraction.Graph; +using BotSharp.Abstraction.Knowledges.Settings; +using BotSharp.Abstraction.MLTasks; +using BotSharp.Abstraction.VectorStorage; +using BotSharp.Core.Knowledges.Helpers; -namespace BotSharp.Plugin.KnowledgeBase.Services; +namespace BotSharp.Core.Knowledges.Services; public partial class KnowledgeService : IKnowledgeService { private readonly IServiceProvider _services; private readonly KnowledgeBaseSettings _settings; - private readonly ITextChopper _textChopper; private readonly ILogger _logger; public KnowledgeService( IServiceProvider services, KnowledgeBaseSettings settings, - ITextChopper textChopper, ILogger logger) { _services = services; _settings = settings; - _textChopper = textChopper; _logger = logger; } diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/KnowledgeRetrievalFn.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/KnowledgeRetrievalFn.cs index d6d719d75..400b17927 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/KnowledgeRetrievalFn.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/KnowledgeRetrievalFn.cs @@ -1,3 +1,5 @@ +using BotSharp.Core.Knowledges.Helpers; + namespace BotSharp.Plugin.KnowledgeBase.Functions; public class KnowledgeRetrievalFn : IFunctionCallback diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/MemorizeKnowledgeFn.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/MemorizeKnowledgeFn.cs index ca545c658..08b55d1c5 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/MemorizeKnowledgeFn.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/MemorizeKnowledgeFn.cs @@ -1,3 +1,5 @@ +using BotSharp.Core.Knowledges.Helpers; + namespace BotSharp.Plugin.KnowledgeBase.Functions; public class MemorizeKnowledgeFn : IFunctionCallback diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/KnowledgeBasePlugin.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/KnowledgeBasePlugin.cs index 4350e33c1..dd864a732 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/KnowledgeBasePlugin.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/KnowledgeBasePlugin.cs @@ -21,7 +21,6 @@ public void RegisterDI(IServiceCollection services, IConfiguration config) }); services.AddScoped(); - services.AddScoped(); services.AddSingleton(); services.AddScoped(); services.AddScoped(); From 48be12a92085b40c4b5253c99ccab05a6db98301 Mon Sep 17 00:00:00 2001 From: Jicheng Lu Date: Sat, 31 Aug 2024 01:05:53 -0500 Subject: [PATCH 21/27] relocate text chopper --- .../BotSharp.Abstraction/Knowledges/ITextChopper.cs | 11 ----------- .../Knowledges/Helpers}/TextChopperService.cs | 11 ++++++----- .../Knowledges/Services/KnowledgeService.Create.cs | 5 ++--- .../KnowledgeBasePlugin.cs | 1 - 4 files changed, 8 insertions(+), 20 deletions(-) delete mode 100644 src/Infrastructure/BotSharp.Abstraction/Knowledges/ITextChopper.cs rename src/{Plugins/BotSharp.Plugin.KnowledgeBase/Services => Infrastructure/BotSharp.Core/Knowledges/Helpers}/TextChopperService.cs (78%) diff --git a/src/Infrastructure/BotSharp.Abstraction/Knowledges/ITextChopper.cs b/src/Infrastructure/BotSharp.Abstraction/Knowledges/ITextChopper.cs deleted file mode 100644 index 7d06a8a41..000000000 --- a/src/Infrastructure/BotSharp.Abstraction/Knowledges/ITextChopper.cs +++ /dev/null @@ -1,11 +0,0 @@ -using BotSharp.Abstraction.Knowledges.Models; - -namespace BotSharp.Abstraction.Knowledges; - -/// -/// Chop large content into chunks -/// -public interface ITextChopper -{ - List Chop(string content, ChunkOption option); -} diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/TextChopperService.cs b/src/Infrastructure/BotSharp.Core/Knowledges/Helpers/TextChopperService.cs similarity index 78% rename from src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/TextChopperService.cs rename to src/Infrastructure/BotSharp.Core/Knowledges/Helpers/TextChopperService.cs index 88c77641a..68b585393 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/TextChopperService.cs +++ b/src/Infrastructure/BotSharp.Core/Knowledges/Helpers/TextChopperService.cs @@ -1,17 +1,18 @@ +using BotSharp.Abstraction.Knowledges.Models; using System.Text.RegularExpressions; -namespace BotSharp.Plugin.KnowledgeBase.Services; +namespace BotSharp.Core.Knowledges.Helpers; -public class TextChopperService : ITextChopper +public static class TextChopper { - public List Chop(string content, ChunkOption option) + public static List Chop(string content, ChunkOption option) { content = Regex.Replace(content, @"\.{2,}", " "); content = Regex.Replace(content, @"_{2,}", " "); return option.SplitByWord ? ChopByWord(content, option) : ChopByChar(content, option); } - private List ChopByWord(string content, ChunkOption option) + private static List ChopByWord(string content, ChunkOption option) { var chunks = new List(); @@ -34,7 +35,7 @@ private List ChopByWord(string content, ChunkOption option) return chunks; } - private List ChopByChar(string content, ChunkOption option) + private static List ChopByChar(string content, ChunkOption option) { var chunks = new List(); var currentPos = 0; diff --git a/src/Infrastructure/BotSharp.Core/Knowledges/Services/KnowledgeService.Create.cs b/src/Infrastructure/BotSharp.Core/Knowledges/Services/KnowledgeService.Create.cs index 2927d9529..d8b82fb51 100644 --- a/src/Infrastructure/BotSharp.Core/Knowledges/Services/KnowledgeService.Create.cs +++ b/src/Infrastructure/BotSharp.Core/Knowledges/Services/KnowledgeService.Create.cs @@ -1,5 +1,6 @@ using BotSharp.Abstraction.Knowledges.Models; using BotSharp.Abstraction.VectorStorage.Models; +using BotSharp.Core.Knowledges.Helpers; namespace BotSharp.Core.Knowledges.Services; @@ -8,9 +9,7 @@ public partial class KnowledgeService public async Task FeedVectorKnowledge(string collectionName, KnowledgeCreationModel knowledge) { var index = 0; - - var textChopper = _services.GetRequiredService(); - var lines = textChopper.Chop(knowledge.Content, new ChunkOption + var lines = TextChopper.Chop(knowledge.Content, new ChunkOption { Size = 1024, Conjunction = 32, diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/KnowledgeBasePlugin.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/KnowledgeBasePlugin.cs index dd864a732..9a955a5b8 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/KnowledgeBasePlugin.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/KnowledgeBasePlugin.cs @@ -20,7 +20,6 @@ public void RegisterDI(IServiceCollection services, IConfiguration config) return settingService.Bind("KnowledgeBase"); }); - services.AddScoped(); services.AddSingleton(); services.AddScoped(); services.AddScoped(); From 2ea8ddfe194cf77fb27b0a5bbedfbfb4a334524a Mon Sep 17 00:00:00 2001 From: Jicheng Lu Date: Sun, 1 Sep 2024 00:15:13 -0500 Subject: [PATCH 22/27] relocate --- .../BotSharp.Core/BotSharp.Core.csproj | 2 +- .../Knowledges/KnowledgeCorePlugin.cs | 19 ------------------- .../PigPdf2TextConverter.cs | 2 +- .../Functions/KnowledgeRetrievalFn.cs | 2 -- .../Functions/MemorizeKnowledgeFn.cs | 2 -- .../Helpers/KnowledgeSettingHelper.cs | 5 +---- .../Helpers/TextChopper.cs} | 3 +-- .../KnowledgeBasePlugin.cs | 2 ++ .../Services/KnowledgeService.Create.cs | 6 +----- .../Services/KnowledgeService.Delete.cs | 2 +- .../Services/KnowledgeService.Get.cs | 4 +--- .../Services/KnowledgeService.Update.cs | 4 +--- .../Services/KnowledgeService.cs | 8 +------- 13 files changed, 11 insertions(+), 50 deletions(-) delete mode 100644 src/Infrastructure/BotSharp.Core/Knowledges/KnowledgeCorePlugin.cs rename src/Plugins/BotSharp.Plugin.KnowledgeBase/{Services => Converters}/PigPdf2TextConverter.cs (95%) rename src/{Infrastructure/BotSharp.Core/Knowledges => Plugins/BotSharp.Plugin.KnowledgeBase}/Helpers/KnowledgeSettingHelper.cs (89%) rename src/{Infrastructure/BotSharp.Core/Knowledges/Helpers/TextChopperService.cs => Plugins/BotSharp.Plugin.KnowledgeBase/Helpers/TextChopper.cs} (94%) rename src/{Infrastructure/BotSharp.Core/Knowledges => Plugins/BotSharp.Plugin.KnowledgeBase}/Services/KnowledgeService.Create.cs (92%) rename src/{Infrastructure/BotSharp.Core/Knowledges => Plugins/BotSharp.Plugin.KnowledgeBase}/Services/KnowledgeService.Delete.cs (95%) rename src/{Infrastructure/BotSharp.Core/Knowledges => Plugins/BotSharp.Plugin.KnowledgeBase}/Services/KnowledgeService.Get.cs (96%) rename src/{Infrastructure/BotSharp.Core/Knowledges => Plugins/BotSharp.Plugin.KnowledgeBase}/Services/KnowledgeService.Update.cs (91%) rename src/{Infrastructure/BotSharp.Core/Knowledges => Plugins/BotSharp.Plugin.KnowledgeBase}/Services/KnowledgeService.cs (80%) diff --git a/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj b/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj index ddd39d0cf..ce4c82d20 100644 --- a/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj +++ b/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj @@ -1,4 +1,4 @@ - + $(TargetFramework) diff --git a/src/Infrastructure/BotSharp.Core/Knowledges/KnowledgeCorePlugin.cs b/src/Infrastructure/BotSharp.Core/Knowledges/KnowledgeCorePlugin.cs deleted file mode 100644 index 6118647b1..000000000 --- a/src/Infrastructure/BotSharp.Core/Knowledges/KnowledgeCorePlugin.cs +++ /dev/null @@ -1,19 +0,0 @@ -using BotSharp.Core.Knowledges.Services; -using Microsoft.Extensions.Configuration; - -namespace BotSharp.Core.Knowledges; - -public class KnowledgeCorePlugin : IBotSharpPlugin -{ - public string Id => "a5ebc8f9-d089-44d0-bf00-4eac97a050bc"; - - public string Name => "Knowledge Core"; - - public string Description => "Provides core knowledge services."; - - - public void RegisterDI(IServiceCollection services, IConfiguration config) - { - services.AddScoped(); - } -} \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/PigPdf2TextConverter.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Converters/PigPdf2TextConverter.cs similarity index 95% rename from src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/PigPdf2TextConverter.cs rename to src/Plugins/BotSharp.Plugin.KnowledgeBase/Converters/PigPdf2TextConverter.cs index c334505ed..0e39bef0d 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/PigPdf2TextConverter.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Converters/PigPdf2TextConverter.cs @@ -1,7 +1,7 @@ using UglyToad.PdfPig; using UglyToad.PdfPig.Content; -namespace BotSharp.Plugin.KnowledgeBase.Services; +namespace BotSharp.Plugin.KnowledgeBase.Converters; public class PigPdf2TextConverter : IPdf2TextConverter { diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/KnowledgeRetrievalFn.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/KnowledgeRetrievalFn.cs index 400b17927..d6d719d75 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/KnowledgeRetrievalFn.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/KnowledgeRetrievalFn.cs @@ -1,5 +1,3 @@ -using BotSharp.Core.Knowledges.Helpers; - namespace BotSharp.Plugin.KnowledgeBase.Functions; public class KnowledgeRetrievalFn : IFunctionCallback diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/MemorizeKnowledgeFn.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/MemorizeKnowledgeFn.cs index 08b55d1c5..ca545c658 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/MemorizeKnowledgeFn.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/MemorizeKnowledgeFn.cs @@ -1,5 +1,3 @@ -using BotSharp.Core.Knowledges.Helpers; - namespace BotSharp.Plugin.KnowledgeBase.Functions; public class MemorizeKnowledgeFn : IFunctionCallback diff --git a/src/Infrastructure/BotSharp.Core/Knowledges/Helpers/KnowledgeSettingHelper.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Helpers/KnowledgeSettingHelper.cs similarity index 89% rename from src/Infrastructure/BotSharp.Core/Knowledges/Helpers/KnowledgeSettingHelper.cs rename to src/Plugins/BotSharp.Plugin.KnowledgeBase/Helpers/KnowledgeSettingHelper.cs index a278d7bff..48033c912 100644 --- a/src/Infrastructure/BotSharp.Core/Knowledges/Helpers/KnowledgeSettingHelper.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Helpers/KnowledgeSettingHelper.cs @@ -1,7 +1,4 @@ -using BotSharp.Abstraction.Knowledges.Settings; -using BotSharp.Abstraction.MLTasks; - -namespace BotSharp.Core.Knowledges.Helpers; +namespace BotSharp.Plugin.KnowledgeBase.Helpers; public static class KnowledgeSettingHelper { diff --git a/src/Infrastructure/BotSharp.Core/Knowledges/Helpers/TextChopperService.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Helpers/TextChopper.cs similarity index 94% rename from src/Infrastructure/BotSharp.Core/Knowledges/Helpers/TextChopperService.cs rename to src/Plugins/BotSharp.Plugin.KnowledgeBase/Helpers/TextChopper.cs index 68b585393..9e9b5f9dc 100644 --- a/src/Infrastructure/BotSharp.Core/Knowledges/Helpers/TextChopperService.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Helpers/TextChopper.cs @@ -1,7 +1,6 @@ -using BotSharp.Abstraction.Knowledges.Models; using System.Text.RegularExpressions; -namespace BotSharp.Core.Knowledges.Helpers; +namespace BotSharp.Plugin.KnowledgeBase.Helpers; public static class TextChopper { diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/KnowledgeBasePlugin.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/KnowledgeBasePlugin.cs index 9a955a5b8..75d3284bf 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/KnowledgeBasePlugin.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/KnowledgeBasePlugin.cs @@ -1,5 +1,6 @@ using BotSharp.Abstraction.Plugins.Models; using BotSharp.Abstraction.Settings; +using BotSharp.Plugin.KnowledgeBase.Converters; using BotSharp.Plugin.KnowledgeBase.Hooks; using Microsoft.Extensions.Configuration; @@ -20,6 +21,7 @@ public void RegisterDI(IServiceCollection services, IConfiguration config) return settingService.Bind("KnowledgeBase"); }); + services.AddScoped(); services.AddSingleton(); services.AddScoped(); services.AddScoped(); diff --git a/src/Infrastructure/BotSharp.Core/Knowledges/Services/KnowledgeService.Create.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Create.cs similarity index 92% rename from src/Infrastructure/BotSharp.Core/Knowledges/Services/KnowledgeService.Create.cs rename to src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Create.cs index d8b82fb51..2c4004fc8 100644 --- a/src/Infrastructure/BotSharp.Core/Knowledges/Services/KnowledgeService.Create.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Create.cs @@ -1,8 +1,4 @@ -using BotSharp.Abstraction.Knowledges.Models; -using BotSharp.Abstraction.VectorStorage.Models; -using BotSharp.Core.Knowledges.Helpers; - -namespace BotSharp.Core.Knowledges.Services; +namespace BotSharp.Plugin.KnowledgeBase.Services; public partial class KnowledgeService { diff --git a/src/Infrastructure/BotSharp.Core/Knowledges/Services/KnowledgeService.Delete.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Delete.cs similarity index 95% rename from src/Infrastructure/BotSharp.Core/Knowledges/Services/KnowledgeService.Delete.cs rename to src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Delete.cs index 404b4e2ca..b0973a996 100644 --- a/src/Infrastructure/BotSharp.Core/Knowledges/Services/KnowledgeService.Delete.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Delete.cs @@ -1,4 +1,4 @@ -namespace BotSharp.Core.Knowledges.Services; +namespace BotSharp.Plugin.KnowledgeBase.Services; public partial class KnowledgeService { diff --git a/src/Infrastructure/BotSharp.Core/Knowledges/Services/KnowledgeService.Get.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Get.cs similarity index 96% rename from src/Infrastructure/BotSharp.Core/Knowledges/Services/KnowledgeService.Get.cs rename to src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Get.cs index 69093659a..ee36780ec 100644 --- a/src/Infrastructure/BotSharp.Core/Knowledges/Services/KnowledgeService.Get.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Get.cs @@ -1,8 +1,6 @@ using BotSharp.Abstraction.Graph.Models; -using BotSharp.Abstraction.Knowledges.Models; -using BotSharp.Abstraction.VectorStorage.Models; -namespace BotSharp.Core.Knowledges.Services; +namespace BotSharp.Plugin.KnowledgeBase.Services; public partial class KnowledgeService { diff --git a/src/Infrastructure/BotSharp.Core/Knowledges/Services/KnowledgeService.Update.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Update.cs similarity index 91% rename from src/Infrastructure/BotSharp.Core/Knowledges/Services/KnowledgeService.Update.cs rename to src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Update.cs index b9652f617..0fa1f5cd8 100644 --- a/src/Infrastructure/BotSharp.Core/Knowledges/Services/KnowledgeService.Update.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Update.cs @@ -1,6 +1,4 @@ -using BotSharp.Abstraction.VectorStorage.Models; - -namespace BotSharp.Core.Knowledges.Services; +namespace BotSharp.Plugin.KnowledgeBase.Services; public partial class KnowledgeService { diff --git a/src/Infrastructure/BotSharp.Core/Knowledges/Services/KnowledgeService.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.cs similarity index 80% rename from src/Infrastructure/BotSharp.Core/Knowledges/Services/KnowledgeService.cs rename to src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.cs index 2c0853d0a..c668e07eb 100644 --- a/src/Infrastructure/BotSharp.Core/Knowledges/Services/KnowledgeService.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.cs @@ -1,10 +1,4 @@ -using BotSharp.Abstraction.Graph; -using BotSharp.Abstraction.Knowledges.Settings; -using BotSharp.Abstraction.MLTasks; -using BotSharp.Abstraction.VectorStorage; -using BotSharp.Core.Knowledges.Helpers; - -namespace BotSharp.Core.Knowledges.Services; +namespace BotSharp.Plugin.KnowledgeBase.Services; public partial class KnowledgeService : IKnowledgeService { From 638a9d6a387b260bc9218304d16f652a088c0eec Mon Sep 17 00:00:00 2001 From: Jicheng Lu Date: Mon, 2 Sep 2024 18:54:40 -0500 Subject: [PATCH 23/27] refine image options --- .../Providers/Image/ImageCompletionProvider.cs | 12 ------------ .../Providers/Image/ImageCompletionProvider.cs | 12 ------------ 2 files changed, 24 deletions(-) diff --git a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Image/ImageCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Image/ImageCompletionProvider.cs index 7d0641547..967f3e082 100644 --- a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Image/ImageCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Image/ImageCompletionProvider.cs @@ -72,9 +72,6 @@ private GeneratedImageSize GetImageSize(string size) case "512x512": retSize = GeneratedImageSize.W512xH512; break; - case "1024x1024": - retSize = GeneratedImageSize.W1024xH1024; - break; case "1024x1792": retSize = GeneratedImageSize.W1024xH1792; break; @@ -96,9 +93,6 @@ private GeneratedImageQuality GetImageQuality(string quality) GeneratedImageQuality retQuality; switch (value) { - case "standard": - retQuality = GeneratedImageQuality.Standard; - break; case "hd": retQuality = GeneratedImageQuality.High; break; @@ -117,9 +111,6 @@ private GeneratedImageStyle GetImageStyle(string style) GeneratedImageStyle retStyle; switch (value) { - case "natural": - retStyle = GeneratedImageStyle.Natural; - break; case "vivid": retStyle = GeneratedImageStyle.Vivid; break; @@ -138,9 +129,6 @@ private GeneratedImageFormat GetImageFormat(string format) GeneratedImageFormat retFormat; switch (value) { - case "uri": - retFormat = GeneratedImageFormat.Uri; - break; case "bytes": retFormat = GeneratedImageFormat.Bytes; break; diff --git a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Image/ImageCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Image/ImageCompletionProvider.cs index 185cf17c9..0e45e7892 100644 --- a/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Image/ImageCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.OpenAI/Providers/Image/ImageCompletionProvider.cs @@ -72,9 +72,6 @@ private GeneratedImageSize GetImageSize(string size) case "512x512": retSize = GeneratedImageSize.W512xH512; break; - case "1024x1024": - retSize = GeneratedImageSize.W1024xH1024; - break; case "1024x1792": retSize = GeneratedImageSize.W1024xH1792; break; @@ -96,9 +93,6 @@ private GeneratedImageQuality GetImageQuality(string quality) GeneratedImageQuality retQuality; switch (value) { - case "standard": - retQuality = GeneratedImageQuality.Standard; - break; case "hd": retQuality = GeneratedImageQuality.High; break; @@ -117,9 +111,6 @@ private GeneratedImageStyle GetImageStyle(string style) GeneratedImageStyle retStyle; switch (value) { - case "natural": - retStyle = GeneratedImageStyle.Natural; - break; case "vivid": retStyle = GeneratedImageStyle.Vivid; break; @@ -138,9 +129,6 @@ private GeneratedImageFormat GetImageFormat(string format) GeneratedImageFormat retFormat; switch (value) { - case "uri": - retFormat = GeneratedImageFormat.Uri; - break; case "bytes": retFormat = GeneratedImageFormat.Bytes; break; From 949a2cce9f0cc7c4243ce0a7a20bd9ba6bd360f3 Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Tue, 3 Sep 2024 12:52:33 -0500 Subject: [PATCH 24/27] refine knowledge searching --- .../Conversations/Models/RoleDialogModel.cs | 1 - .../Helpers/VectorStorageHelper.cs | 29 ++++++++++++++++ .../Models/VectorSearchResult.cs | 2 ++ .../Functions/KnowledgeRetrievalFn.cs | 17 ++++++---- .../Functions/MemorizeKnowledgeFn.cs | 23 +++++-------- .../Functions/PrimaryStagePlanFn.cs | 34 ++++++------------- .../Functions/SecondaryStagePlanFn.cs | 26 +++++--------- .../Functions/SummaryPlanFn.cs | 6 ---- .../TwoStaging/TwoStageTaskPlanner.cs | 7 ---- src/Plugins/BotSharp.Plugin.Planner/Using.cs | 18 ++++++++-- 10 files changed, 86 insertions(+), 77 deletions(-) create mode 100644 src/Infrastructure/BotSharp.Abstraction/VectorStorage/Helpers/VectorStorageHelper.cs diff --git a/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/RoleDialogModel.cs b/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/RoleDialogModel.cs index f93ef96f1..84f194c40 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/RoleDialogModel.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/RoleDialogModel.cs @@ -1,7 +1,6 @@ using BotSharp.Abstraction.Functions.Models; using BotSharp.Abstraction.Messaging; using BotSharp.Abstraction.Messaging.Models.RichContent; -using BotSharp.Abstraction.MLTasks; namespace BotSharp.Abstraction.Conversations.Models; diff --git a/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Helpers/VectorStorageHelper.cs b/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Helpers/VectorStorageHelper.cs new file mode 100644 index 000000000..f1f0ebde0 --- /dev/null +++ b/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Helpers/VectorStorageHelper.cs @@ -0,0 +1,29 @@ +using BotSharp.Abstraction.Knowledges.Enums; +using BotSharp.Abstraction.VectorStorage.Models; + +namespace BotSharp.Abstraction.VectorStorage.Helpers; + +public static class VectorStorageHelper +{ + public static string ToQuestionAnswer(this VectorSearchResult data) + { + if (data?.Data == null) return string.Empty; + + return $"Question: {data.Data[KnowledgePayloadName.Text]}\r\nAnswer: {data.Data[KnowledgePayloadName.Answer]}"; + } + + public static string ToPayloadPair(this VectorSearchResult data, IList payloads) + { + if (data?.Data == null || payloads.IsNullOrEmpty()) return string.Empty; + + var results = data.Data.Where(x => payloads.Contains(x.Key)) + .OrderBy(x => payloads.IndexOf(x.Key)) + .Select(x => + { + return $"{x.Key}: {x.Value}"; + }) + .ToList(); + + return string.Join("\r\n", results.Where(x => !string.IsNullOrWhiteSpace(x))); + } +} diff --git a/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/VectorSearchResult.cs b/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/VectorSearchResult.cs index ce39edbf9..f2c582ce1 100644 --- a/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/VectorSearchResult.cs +++ b/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Models/VectorSearchResult.cs @@ -1,3 +1,5 @@ +using BotSharp.Abstraction.Knowledges.Enums; + namespace BotSharp.Abstraction.VectorStorage.Models; public class VectorSearchResult : VectorCollectionData diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/KnowledgeRetrievalFn.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/KnowledgeRetrievalFn.cs index d6d719d75..a93374195 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/KnowledgeRetrievalFn.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/KnowledgeRetrievalFn.cs @@ -1,9 +1,13 @@ +using BotSharp.Abstraction.VectorStorage.Helpers; + namespace BotSharp.Plugin.KnowledgeBase.Functions; public class KnowledgeRetrievalFn : IFunctionCallback { public string Name => "knowledge_retrieval"; + public string Indication => "searching my brain"; + private readonly IServiceProvider _services; private readonly KnowledgeBaseSettings _settings; @@ -18,15 +22,16 @@ public async Task Execute(RoleDialogModel message) var args = JsonSerializer.Deserialize(message.FunctionArgs ?? "{}"); var collectionName = _settings.Default.CollectionName ?? KnowledgeCollectionName.BotSharp; - var embedding = KnowledgeSettingHelper.GetTextEmbeddingSetting(_services, collectionName); - - var vector = await embedding.GetVectorAsync(args.Question); - var vectorDb = _services.GetServices().FirstOrDefault(x => x.Provider == _settings.VectorDb.Provider); - var knowledges = await vectorDb.Search(collectionName, vector, new List { KnowledgePayloadName.Text, KnowledgePayloadName.Answer }); + var knowledgeService = _services.GetRequiredService(); + var knowledges = await knowledgeService.SearchVectorKnowledge(args.Question, collectionName, new VectorSearchOptions + { + Fields = new List { KnowledgePayloadName.Text, KnowledgePayloadName.Answer }, + Confidence = 0.2f + }); if (!knowledges.IsNullOrEmpty()) { - var answers = knowledges.Select(x => $"Question: {x.Data[KnowledgePayloadName.Text]}\r\nAnswer: {x.Data[KnowledgePayloadName.Answer]}").ToList(); + var answers = knowledges.Select(x => x.ToQuestionAnswer()).ToList(); message.Content = string.Join("\r\n\r\n=====\r\n", answers); } else diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/MemorizeKnowledgeFn.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/MemorizeKnowledgeFn.cs index ca545c658..62388f197 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/MemorizeKnowledgeFn.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/MemorizeKnowledgeFn.cs @@ -4,6 +4,8 @@ public class MemorizeKnowledgeFn : IFunctionCallback { public string Name => "memorize_knowledge"; + public string Indication => "remembering knowledge"; + private readonly IServiceProvider _services; private readonly KnowledgeBaseSettings _settings; @@ -18,23 +20,16 @@ public async Task Execute(RoleDialogModel message) var args = JsonSerializer.Deserialize(message.FunctionArgs ?? "{}"); var collectionName = _settings.Default.CollectionName ?? KnowledgeCollectionName.BotSharp; - var embedding = KnowledgeSettingHelper.GetTextEmbeddingSetting(_services, collectionName); - - var vector = await embedding.GetVectorsAsync(new List + var knowledgeService = _services.GetRequiredService(); + var result = await knowledgeService.CreateVectorCollectionData(collectionName, new VectorCreateModel { - args.Question + Text = args.Question, + Payload = new Dictionary + { + { KnowledgePayloadName.Answer, args.Answer } + } }); - var vectorDb = _services.GetServices().FirstOrDefault(x => x.Provider == _settings.VectorDb.Provider); - await vectorDb.CreateCollection(collectionName, vector[0].Length); - - var result = await vectorDb.Upsert(collectionName, Guid.NewGuid(), vector[0], - args.Question, - new Dictionary - { - { KnowledgePayloadName.Answer, args.Answer } - }); - message.Content = result ? "Saved to my brain" : "I forgot it"; return true; } diff --git a/src/Plugins/BotSharp.Plugin.Planner/Functions/PrimaryStagePlanFn.cs b/src/Plugins/BotSharp.Plugin.Planner/Functions/PrimaryStagePlanFn.cs index 2842397bf..01d2bfdf2 100644 --- a/src/Plugins/BotSharp.Plugin.Planner/Functions/PrimaryStagePlanFn.cs +++ b/src/Plugins/BotSharp.Plugin.Planner/Functions/PrimaryStagePlanFn.cs @@ -1,12 +1,4 @@ -using BotSharp.Abstraction.Conversations.Models; -using BotSharp.Abstraction.Functions; -using BotSharp.Abstraction.Templating; -using System.Threading.Tasks; -using BotSharp.Abstraction.Routing; -using BotSharp.Core.Infrastructures; using BotSharp.Plugin.Planner.TwoStaging.Models; -using Microsoft.Extensions.Logging; -using BotSharp.Abstraction.Knowledges.Models; namespace BotSharp.Plugin.Planner.Functions; @@ -27,26 +19,20 @@ public async Task Execute(RoleDialogModel message) { // Debug var state = _services.GetRequiredService(); + var knowledgeService = _services.GetRequiredService(); + var knowledgeSettings = _services.GetRequiredService(); + var fn = _services.GetRequiredService(); + state.SetState("max_tokens", "4096"); var task = JsonSerializer.Deserialize(message.FunctionArgs); - // Get knowledge from vectordb - var fn = _services.GetRequiredService(); - - var msg = new ExtractedKnowledge - { - Question = task.Question, - }; - - var retrievalMessage = new RoleDialogModel(AgentRole.User, task.Requirements) + // Get knowledge from vectordb + var collectionName = knowledgeSettings.Default.CollectionName ?? KnowledgeCollectionName.BotSharp; ; + var knowledges = await knowledgeService.SearchVectorKnowledge(task.Question, collectionName, new VectorSearchOptions { - FunctionArgs = JsonSerializer.Serialize(msg), - KnowledgeConfidence = 0.1f, - Content = string.Empty - }; - - await fn.InvokeFunction("knowledge_retrieval", retrievalMessage); - message.Content = retrievalMessage.Content; + Confidence = 0.1f + }); + message.Content = string.Join("\r\n\r\n=====\r\n", knowledges.Select(x => x.ToQuestionAnswer())); var agentService = _services.GetRequiredService(); var currentAgent = await agentService.LoadAgent(message.CurrentAgentId); diff --git a/src/Plugins/BotSharp.Plugin.Planner/Functions/SecondaryStagePlanFn.cs b/src/Plugins/BotSharp.Plugin.Planner/Functions/SecondaryStagePlanFn.cs index 2723d0708..211eea53a 100644 --- a/src/Plugins/BotSharp.Plugin.Planner/Functions/SecondaryStagePlanFn.cs +++ b/src/Plugins/BotSharp.Plugin.Planner/Functions/SecondaryStagePlanFn.cs @@ -1,12 +1,4 @@ -using BotSharp.Abstraction.Conversations.Models; -using BotSharp.Abstraction.Functions; -using BotSharp.Abstraction.Knowledges.Models; -using BotSharp.Abstraction.Routing; -using BotSharp.Abstraction.Templating; -using BotSharp.Core.Infrastructures; using BotSharp.Plugin.Planner.TwoStaging.Models; -using System.Threading.Tasks; -using Microsoft.Extensions.Logging; namespace BotSharp.Plugin.Planner.Functions; @@ -26,6 +18,9 @@ public SecondaryStagePlanFn(IServiceProvider services, ILogger Execute(RoleDialogModel message) { var fn = _services.GetRequiredService(); + var knowledgeService = _services.GetRequiredService(); + var knowledgeSettings = _services.GetRequiredService(); + var collectionName = knowledgeSettings.Default.CollectionName ?? KnowledgeCollectionName.BotSharp; var msgSecondary = RoleDialogModel.From(message); var taskPrimary = JsonSerializer.Deserialize(message.FunctionArgs); @@ -38,18 +33,15 @@ public async Task Execute(RoleDialogModel message) var taskSecondary = JsonSerializer.Deserialize(msgSecondary.FunctionArgs); var items = msgSecondary.Content.JsonArrayContent(); - msgSecondary.KnowledgeConfidence = 0.5f; foreach (var item in items) { - if (item.NeedAdditionalInformation) + if (!item.NeedAdditionalInformation) continue; + + var knowledges = await knowledgeService.SearchVectorKnowledge(item.Task, collectionName, new VectorSearchOptions { - msgSecondary.FunctionArgs = JsonSerializer.Serialize(new ExtractedKnowledge - { - Question = item.Task - }); - await fn.InvokeFunction("knowledge_retrieval", msgSecondary); - message.Content += msgSecondary.Content; - } + Confidence = 0.5f + }); + message.Content += string.Join("\r\n\r\n=====\r\n", knowledges.Select(x => x.ToQuestionAnswer())); } // load agent diff --git a/src/Plugins/BotSharp.Plugin.Planner/Functions/SummaryPlanFn.cs b/src/Plugins/BotSharp.Plugin.Planner/Functions/SummaryPlanFn.cs index 5fd9fe943..0f9da4aab 100644 --- a/src/Plugins/BotSharp.Plugin.Planner/Functions/SummaryPlanFn.cs +++ b/src/Plugins/BotSharp.Plugin.Planner/Functions/SummaryPlanFn.cs @@ -1,10 +1,4 @@ -using BotSharp.Abstraction.Conversations.Models; -using BotSharp.Abstraction.Functions; -using BotSharp.Abstraction.Templating; -using System.Threading.Tasks; -using BotSharp.Core.Infrastructures; using BotSharp.Plugin.Planner.TwoStaging.Models; -using Microsoft.Extensions.Logging; namespace BotSharp.Plugin.Planner.Functions; diff --git a/src/Plugins/BotSharp.Plugin.Planner/TwoStaging/TwoStageTaskPlanner.cs b/src/Plugins/BotSharp.Plugin.Planner/TwoStaging/TwoStageTaskPlanner.cs index be627c752..c6f819e5e 100644 --- a/src/Plugins/BotSharp.Plugin.Planner/TwoStaging/TwoStageTaskPlanner.cs +++ b/src/Plugins/BotSharp.Plugin.Planner/TwoStaging/TwoStageTaskPlanner.cs @@ -1,13 +1,6 @@ -using BotSharp.Abstraction.Conversations.Models; -using BotSharp.Abstraction.Knowledges; using BotSharp.Abstraction.MLTasks; -using BotSharp.Abstraction.Routing; using BotSharp.Abstraction.Routing.Planning; -using BotSharp.Abstraction.Templating; -using BotSharp.Core.Infrastructures; using BotSharp.Core.Routing.Planning; -using Microsoft.Extensions.Logging; -using System.Threading.Tasks; namespace BotSharp.Plugin.Planner.TwoStaging; diff --git a/src/Plugins/BotSharp.Plugin.Planner/Using.cs b/src/Plugins/BotSharp.Plugin.Planner/Using.cs index 20880e816..fa59ee23b 100644 --- a/src/Plugins/BotSharp.Plugin.Planner/Using.cs +++ b/src/Plugins/BotSharp.Plugin.Planner/Using.cs @@ -3,12 +3,13 @@ global using System.Text.Json.Serialization; global using System.Collections.Generic; global using System.Linq; +global using System.Threading.Tasks; global using Microsoft.Extensions.Configuration; global using Microsoft.Extensions.DependencyInjection; +global using Microsoft.Extensions.Logging; global using BotSharp.Abstraction.Plugins; -global using BotSharp.Abstraction.Planning; global using BotSharp.Abstraction.Agents; global using BotSharp.Abstraction.Agents.Enums; global using BotSharp.Abstraction.Agents.Models; @@ -18,5 +19,18 @@ global using BotSharp.Abstraction.Repositories; global using BotSharp.Abstraction.Utilities; +global using BotSharp.Abstraction.Conversations.Models; +global using BotSharp.Abstraction.Functions; +global using BotSharp.Abstraction.Routing; +global using BotSharp.Abstraction.Templating; + +global using BotSharp.Abstraction.Knowledges; +global using BotSharp.Abstraction.Knowledges.Settings; +global using BotSharp.Abstraction.Knowledges.Enums; +global using BotSharp.Abstraction.VectorStorage.Models; +global using BotSharp.Abstraction.VectorStorage.Helpers; + global using BotSharp.Plugin.Planner.Hooks; -global using BotSharp.Plugin.Planner.Enums; \ No newline at end of file +global using BotSharp.Plugin.Planner.Enums; + +global using BotSharp.Core.Infrastructures; \ No newline at end of file From e21a44ec3d9b6fcd5105634bc07e982dfd8a9e72 Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Tue, 3 Sep 2024 15:41:12 -0500 Subject: [PATCH 25/27] refine db knowledge --- .../BotSharp.Logger/Hooks/VerboseLogHook.cs | 11 +- .../Controllers/KnowledgeBaseController.cs | 1 + .../Providers/Chat/ChatCompletionProvider.cs | 6 +- .../Providers/Chat/ChatCompletionProvider.cs | 6 +- .../BotSharp.Plugin.SqlDriver.csproj | 3 +- .../Controllers/SqlDriverController.cs | 24 +++ .../Functions/GetTableDefinitionFn.cs | 31 ++-- .../Models/RequestBase.cs | 19 +++ .../Services/DbKnowledgeService.cs | 140 ++++++++++++++++++ .../SqlDriverPlugin.cs | 1 + 10 files changed, 221 insertions(+), 21 deletions(-) create mode 100644 src/Plugins/BotSharp.Plugin.SqlDriver/Controllers/SqlDriverController.cs create mode 100644 src/Plugins/BotSharp.Plugin.SqlDriver/Models/RequestBase.cs create mode 100644 src/Plugins/BotSharp.Plugin.SqlDriver/Services/DbKnowledgeService.cs diff --git a/src/Infrastructure/BotSharp.Logger/Hooks/VerboseLogHook.cs b/src/Infrastructure/BotSharp.Logger/Hooks/VerboseLogHook.cs index cf6985e9e..afb8dd019 100644 --- a/src/Infrastructure/BotSharp.Logger/Hooks/VerboseLogHook.cs +++ b/src/Infrastructure/BotSharp.Logger/Hooks/VerboseLogHook.cs @@ -23,10 +23,13 @@ public async Task BeforeGenerating(Agent agent, List conversati { if (!_convSettings.ShowVerboseLog) return; - var dialog = conversations.Last(); - var log = $"{dialog.Role}: {dialog.Content} [msg_id: {dialog.MessageId}] ==>"; - _logger.LogInformation(log); - + var dialog = conversations.LastOrDefault(); + if (dialog != null) + { + var log = $"{dialog.Role}: {dialog.Content} [msg_id: {dialog.MessageId}] ==>"; + _logger.LogInformation(log); + } + await Task.CompletedTask; } diff --git a/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs b/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs index 260dc166c..827c2d7ce 100644 --- a/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs +++ b/src/Infrastructure/BotSharp.OpenAPI/Controllers/KnowledgeBaseController.cs @@ -142,6 +142,7 @@ public async Task SearchGraphKnowledge([FromBody] Searc } #endregion + #region Knowledge [HttpPost("/knowledge/search")] public async Task SearchKnowledge([FromBody] SearchKnowledgeRequest request) diff --git a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Chat/ChatCompletionProvider.cs b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Chat/ChatCompletionProvider.cs index af82b0374..678ee82be 100644 --- a/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Chat/ChatCompletionProvider.cs +++ b/src/Plugins/BotSharp.Plugin.AzureOpenAI/Providers/Chat/ChatCompletionProvider.cs @@ -49,7 +49,7 @@ public async Task GetChatCompletions(Agent agent, List GetChatCompletions(Agent agent, List GetChatCompletions(Agent agent, List GetChatCompletions(Agent agent, List GetChatCompletions(Agent agent, List GetChatCompletions(Agent agent, List + $(TargetFramework) @@ -53,6 +53,7 @@ + diff --git a/src/Plugins/BotSharp.Plugin.SqlDriver/Controllers/SqlDriverController.cs b/src/Plugins/BotSharp.Plugin.SqlDriver/Controllers/SqlDriverController.cs new file mode 100644 index 000000000..f863adb79 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.SqlDriver/Controllers/SqlDriverController.cs @@ -0,0 +1,24 @@ +using BotSharp.Plugin.SqlDriver.Models; +using Microsoft.AspNetCore.Authorization; +using Microsoft.AspNetCore.Mvc; + +namespace BotSharp.Plugin.SqlDriver.Controllers; + +[Authorize] +[ApiController] +public class SqlDriverController : ControllerBase +{ + private readonly IServiceProvider _services; + + public SqlDriverController(IServiceProvider services) + { + _services = services; + } + + [HttpPost("/knowledge/database/import")] + public async Task ImportDbKnowledge(ImportDbKnowledgeRequest request) + { + var dbKnowledge = _services.GetRequiredService(); + return await dbKnowledge.Import(request.Provider ?? "openai", request.Model ?? "gpt-4o", request.Schema); + } +} diff --git a/src/Plugins/BotSharp.Plugin.SqlDriver/Functions/GetTableDefinitionFn.cs b/src/Plugins/BotSharp.Plugin.SqlDriver/Functions/GetTableDefinitionFn.cs index 47f64a1f3..6fcee0bd3 100644 --- a/src/Plugins/BotSharp.Plugin.SqlDriver/Functions/GetTableDefinitionFn.cs +++ b/src/Plugins/BotSharp.Plugin.SqlDriver/Functions/GetTableDefinitionFn.cs @@ -20,25 +20,36 @@ public async Task Execute(RoleDialogModel message) var settings = _services.GetRequiredService(); // Get table DDL from database - using var connection = new MySqlConnection(settings.MySqlConnectionString); - var dictionary = new Dictionary(); var tableDdls = new List(); + using var connection = new MySqlConnection(settings.MySqlConnectionString); + connection.Open(); - foreach (var p in (List)message.Data) + foreach (var table in (List)message.Data) { - var escapedTableName = MySqlHelper.EscapeString(p); - dictionary["@" + "table_name"] = p; - dictionary["table_name"] = escapedTableName; + var escapedTableName = MySqlHelper.EscapeString(table); + + var sql = $"select * from information_schema.tables where table_name = @tableName"; + var result = connection.QueryFirstOrDefault(sql, new + { + tableName = escapedTableName + }); - var sql = $"select * from information_schema.tables where table_name ='{escapedTableName}'"; - var result = connection.QueryFirstOrDefault(sql: sql, dictionary); if (result == null) continue; sql = $"SHOW CREATE TABLE `{escapedTableName}`"; - result = connection.QueryFirstOrDefault(sql: sql, dictionary); - tableDdls.Add(result); + using var command = new MySqlCommand(sql, connection); + using var reader = command.ExecuteReader(); + if (reader.Read()) + { + result = reader.GetString(1); + tableDdls.Add(result); + } + + reader.Close(); + command.Dispose(); } + connection.Close(); message.Content = string.Join("\r\n", tableDdls); return true; } diff --git a/src/Plugins/BotSharp.Plugin.SqlDriver/Models/RequestBase.cs b/src/Plugins/BotSharp.Plugin.SqlDriver/Models/RequestBase.cs new file mode 100644 index 000000000..48ba820cf --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.SqlDriver/Models/RequestBase.cs @@ -0,0 +1,19 @@ +using System.Text.Json.Serialization; + +namespace BotSharp.Plugin.SqlDriver.Models; + +public class RequestBase +{ + [JsonPropertyName("provider")] + public string? Provider { get; set; } + + [JsonPropertyName("model")] + public string? Model { get; set; } +} + + +public class ImportDbKnowledgeRequest : RequestBase +{ + [JsonPropertyName("schema")] + public string Schema { get; set; } +} \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.SqlDriver/Services/DbKnowledgeService.cs b/src/Plugins/BotSharp.Plugin.SqlDriver/Services/DbKnowledgeService.cs new file mode 100644 index 000000000..7807f87f9 --- /dev/null +++ b/src/Plugins/BotSharp.Plugin.SqlDriver/Services/DbKnowledgeService.cs @@ -0,0 +1,140 @@ +using static Dapper.SqlMapper; +using Microsoft.Extensions.Logging; +using BotSharp.Core.Infrastructures; +using MySqlConnector; +using BotSharp.Abstraction.Agents.Enums; +using BotSharp.Abstraction.Knowledges.Settings; +using BotSharp.Abstraction.Knowledges.Enums; +using BotSharp.Abstraction.VectorStorage.Models; + +namespace BotSharp.Plugin.SqlDriver.Services; + +public class DbKnowledgeService +{ + private readonly IServiceProvider _services; + private readonly ILogger _logger; + + public DbKnowledgeService( + IServiceProvider services, + ILogger logger) + { + _services = services; + _logger = logger; + } + + public async Task Import(string provider, string model, string schema) + { + var sqlDriverSettings = _services.GetRequiredService(); + var knowledgeSettings = _services.GetRequiredService(); + var knowledgeService = _services.GetRequiredService(); + var collectionName = knowledgeSettings.Default.CollectionName ?? KnowledgeCollectionName.BotSharp; + + var tables = new HashSet(); + using var connection = new MySqlConnection(sqlDriverSettings.MySqlConnectionString); + + var sql = $"select table_name from information_schema.tables where table_schema = @tableSchema"; + var results = connection.Query(sql, new + { + tableSchema = schema + }); + + foreach (var item in results) + { + if (item == null) continue; + + tables.Add(item.TABLE_NAME); + } + + foreach (var table in tables) + { + try + { + _logger.LogInformation($"Start processing table {table}\r\n"); + + var ddl = GetTableStructure(table); + if (string.IsNullOrEmpty(ddl)) continue; + + var prompt = await GetPrompt(ddl); + var response = await GetAiResponse(prompt, provider, model); + var knowledges = response.Content.JsonArrayContent(); + + if (knowledges.IsNullOrEmpty()) + { + _logger.LogInformation($"No knowledge for table {table}"); + continue; + } + + foreach (var item in knowledges) + { + await knowledgeService.CreateVectorCollectionData(collectionName, new VectorCreateModel + { + Text = item.Question, + Payload = new Dictionary + { + { KnowledgePayloadName.Answer, item.Answer } + } + }); + + _logger.LogInformation($"Knowledge {table} is saved =>\r\nQuestion: {item.Question}\r\nAnswer: {item.Answer}\r\n"); + } + } + catch (Exception ex) + { + var note = $"Error processing table {table}: {ex.Message}\r\n{ex.InnerException}"; + _logger.LogWarning(note); + } + } + + return true; + } + + private string GetTableStructure(string table) + { + var settings = _services.GetRequiredService(); + using var connection = new MySqlConnection(settings.MySqlConnectionString); + connection.Open(); + + var result = string.Empty; + var escapedTableName = MySqlHelper.EscapeString(table); + var sql = $"SHOW CREATE TABLE `{escapedTableName}`"; + + using var command = new MySqlCommand(sql, connection); + using var reader = command.ExecuteReader(); + if (reader.Read()) + { + result = reader.GetString(1); + } + + reader.Close(); + command.Dispose(); + connection.Close(); + return result; + } + + private async Task GetPrompt(string content) + { + var agentService = _services.GetRequiredService(); + var render = _services.GetRequiredService(); + + var aiAssistant = await agentService.GetAgent(BuiltInAgentId.AIAssistant); + var template = aiAssistant.Templates.FirstOrDefault(x => x.Name == "database_knowledge")?.Content ?? string.Empty; + + return render.Render(template, new Dictionary + { + { "table_structure", content } + }); + } + + private async Task GetAiResponse(string prompt, string provider, string model) + { + var agent = new Agent + { + Id = string.Empty, + Name = "Db knowledge", + Instruction = prompt, + }; + + var completion = CompletionProvider.GetChatCompletion(_services, provider, model); + return await completion.GetChatCompletions(agent, new List()); + } +} diff --git a/src/Plugins/BotSharp.Plugin.SqlDriver/SqlDriverPlugin.cs b/src/Plugins/BotSharp.Plugin.SqlDriver/SqlDriverPlugin.cs index aaad1a0fa..c059edfb0 100644 --- a/src/Plugins/BotSharp.Plugin.SqlDriver/SqlDriverPlugin.cs +++ b/src/Plugins/BotSharp.Plugin.SqlDriver/SqlDriverPlugin.cs @@ -16,6 +16,7 @@ public void RegisterDI(IServiceCollection services, IConfiguration config) }); services.AddScoped(); + services.AddScoped(); services.AddScoped(); services.AddScoped(); services.AddScoped(); From 32c99dee5f6218bb05064a942464c12fae9de451 Mon Sep 17 00:00:00 2001 From: Jicheng Lu Date: Tue, 3 Sep 2024 22:13:25 -0500 Subject: [PATCH 26/27] relocate --- .../BotSharp.Abstraction/BotSharp.Abstraction.csproj | 2 +- .../VectorStorageExtension.cs} | 4 ++-- src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj | 2 +- .../BotSharp.Core/Conversations/ConversationPlugin.cs | 3 +++ .../Knowledges}/Helpers/KnowledgeSettingHelper.cs | 5 ++++- .../BotSharp.Core/Knowledges}/Helpers/TextChopper.cs | 3 ++- .../Knowledges}/Services/KnowledgeService.Create.cs | 6 +++++- .../Knowledges}/Services/KnowledgeService.Delete.cs | 2 +- .../Knowledges}/Services/KnowledgeService.Get.cs | 4 +++- .../Knowledges}/Services/KnowledgeService.Update.cs | 4 +++- .../Knowledges}/Services/KnowledgeService.cs | 8 +++++++- .../Functions/KnowledgeRetrievalFn.cs | 2 +- .../BotSharp.Plugin.KnowledgeBase/KnowledgeBasePlugin.cs | 1 - src/Plugins/BotSharp.Plugin.KnowledgeBase/Using.cs | 1 - src/Plugins/BotSharp.Plugin.Planner/Using.cs | 2 +- .../Controllers/TwilioVoiceController.cs | 3 +-- 16 files changed, 35 insertions(+), 17 deletions(-) rename src/Infrastructure/BotSharp.Abstraction/VectorStorage/{Helpers/VectorStorageHelper.cs => Extensions/VectorStorageExtension.cs} (90%) rename src/{Plugins/BotSharp.Plugin.KnowledgeBase => Infrastructure/BotSharp.Core/Knowledges}/Helpers/KnowledgeSettingHelper.cs (89%) rename src/{Plugins/BotSharp.Plugin.KnowledgeBase => Infrastructure/BotSharp.Core/Knowledges}/Helpers/TextChopper.cs (94%) rename src/{Plugins/BotSharp.Plugin.KnowledgeBase => Infrastructure/BotSharp.Core/Knowledges}/Services/KnowledgeService.Create.cs (92%) rename src/{Plugins/BotSharp.Plugin.KnowledgeBase => Infrastructure/BotSharp.Core/Knowledges}/Services/KnowledgeService.Delete.cs (95%) rename src/{Plugins/BotSharp.Plugin.KnowledgeBase => Infrastructure/BotSharp.Core/Knowledges}/Services/KnowledgeService.Get.cs (96%) rename src/{Plugins/BotSharp.Plugin.KnowledgeBase => Infrastructure/BotSharp.Core/Knowledges}/Services/KnowledgeService.Update.cs (91%) rename src/{Plugins/BotSharp.Plugin.KnowledgeBase => Infrastructure/BotSharp.Core/Knowledges}/Services/KnowledgeService.cs (80%) diff --git a/src/Infrastructure/BotSharp.Abstraction/BotSharp.Abstraction.csproj b/src/Infrastructure/BotSharp.Abstraction/BotSharp.Abstraction.csproj index 94c9e27c5..697f94a56 100644 --- a/src/Infrastructure/BotSharp.Abstraction/BotSharp.Abstraction.csproj +++ b/src/Infrastructure/BotSharp.Abstraction/BotSharp.Abstraction.csproj @@ -1,4 +1,4 @@ - + $(TargetFramework) diff --git a/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Helpers/VectorStorageHelper.cs b/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Extensions/VectorStorageExtension.cs similarity index 90% rename from src/Infrastructure/BotSharp.Abstraction/VectorStorage/Helpers/VectorStorageHelper.cs rename to src/Infrastructure/BotSharp.Abstraction/VectorStorage/Extensions/VectorStorageExtension.cs index f1f0ebde0..ec519f046 100644 --- a/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Helpers/VectorStorageHelper.cs +++ b/src/Infrastructure/BotSharp.Abstraction/VectorStorage/Extensions/VectorStorageExtension.cs @@ -1,9 +1,9 @@ using BotSharp.Abstraction.Knowledges.Enums; using BotSharp.Abstraction.VectorStorage.Models; -namespace BotSharp.Abstraction.VectorStorage.Helpers; +namespace BotSharp.Abstraction.VectorStorage.Extensions; -public static class VectorStorageHelper +public static class VectorStorageExtension { public static string ToQuestionAnswer(this VectorSearchResult data) { diff --git a/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj b/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj index 05d485b74..ff10a5176 100644 --- a/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj +++ b/src/Infrastructure/BotSharp.Core/BotSharp.Core.csproj @@ -1,4 +1,4 @@ - + $(TargetFramework) diff --git a/src/Infrastructure/BotSharp.Core/Conversations/ConversationPlugin.cs b/src/Infrastructure/BotSharp.Core/Conversations/ConversationPlugin.cs index f06e2ba2b..05653d9c6 100644 --- a/src/Infrastructure/BotSharp.Core/Conversations/ConversationPlugin.cs +++ b/src/Infrastructure/BotSharp.Core/Conversations/ConversationPlugin.cs @@ -6,6 +6,7 @@ using BotSharp.Abstraction.Settings; using BotSharp.Abstraction.Templating; using BotSharp.Core.Instructs; +using BotSharp.Core.Knowledges.Services; using BotSharp.Core.Messaging; using BotSharp.Core.Routing.Planning; using BotSharp.Core.Templating; @@ -54,6 +55,8 @@ public void RegisterDI(IServiceCollection services, IConfiguration config) services.AddScoped(); services.AddScoped(); services.AddScoped(); + + services.AddScoped(); } public bool AttachMenu(List menu) diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Helpers/KnowledgeSettingHelper.cs b/src/Infrastructure/BotSharp.Core/Knowledges/Helpers/KnowledgeSettingHelper.cs similarity index 89% rename from src/Plugins/BotSharp.Plugin.KnowledgeBase/Helpers/KnowledgeSettingHelper.cs rename to src/Infrastructure/BotSharp.Core/Knowledges/Helpers/KnowledgeSettingHelper.cs index 48033c912..a278d7bff 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Helpers/KnowledgeSettingHelper.cs +++ b/src/Infrastructure/BotSharp.Core/Knowledges/Helpers/KnowledgeSettingHelper.cs @@ -1,4 +1,7 @@ -namespace BotSharp.Plugin.KnowledgeBase.Helpers; +using BotSharp.Abstraction.Knowledges.Settings; +using BotSharp.Abstraction.MLTasks; + +namespace BotSharp.Core.Knowledges.Helpers; public static class KnowledgeSettingHelper { diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Helpers/TextChopper.cs b/src/Infrastructure/BotSharp.Core/Knowledges/Helpers/TextChopper.cs similarity index 94% rename from src/Plugins/BotSharp.Plugin.KnowledgeBase/Helpers/TextChopper.cs rename to src/Infrastructure/BotSharp.Core/Knowledges/Helpers/TextChopper.cs index 9e9b5f9dc..68b585393 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Helpers/TextChopper.cs +++ b/src/Infrastructure/BotSharp.Core/Knowledges/Helpers/TextChopper.cs @@ -1,6 +1,7 @@ +using BotSharp.Abstraction.Knowledges.Models; using System.Text.RegularExpressions; -namespace BotSharp.Plugin.KnowledgeBase.Helpers; +namespace BotSharp.Core.Knowledges.Helpers; public static class TextChopper { diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Create.cs b/src/Infrastructure/BotSharp.Core/Knowledges/Services/KnowledgeService.Create.cs similarity index 92% rename from src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Create.cs rename to src/Infrastructure/BotSharp.Core/Knowledges/Services/KnowledgeService.Create.cs index 2c4004fc8..d8b82fb51 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Create.cs +++ b/src/Infrastructure/BotSharp.Core/Knowledges/Services/KnowledgeService.Create.cs @@ -1,4 +1,8 @@ -namespace BotSharp.Plugin.KnowledgeBase.Services; +using BotSharp.Abstraction.Knowledges.Models; +using BotSharp.Abstraction.VectorStorage.Models; +using BotSharp.Core.Knowledges.Helpers; + +namespace BotSharp.Core.Knowledges.Services; public partial class KnowledgeService { diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Delete.cs b/src/Infrastructure/BotSharp.Core/Knowledges/Services/KnowledgeService.Delete.cs similarity index 95% rename from src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Delete.cs rename to src/Infrastructure/BotSharp.Core/Knowledges/Services/KnowledgeService.Delete.cs index b0973a996..404b4e2ca 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Delete.cs +++ b/src/Infrastructure/BotSharp.Core/Knowledges/Services/KnowledgeService.Delete.cs @@ -1,4 +1,4 @@ -namespace BotSharp.Plugin.KnowledgeBase.Services; +namespace BotSharp.Core.Knowledges.Services; public partial class KnowledgeService { diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Get.cs b/src/Infrastructure/BotSharp.Core/Knowledges/Services/KnowledgeService.Get.cs similarity index 96% rename from src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Get.cs rename to src/Infrastructure/BotSharp.Core/Knowledges/Services/KnowledgeService.Get.cs index ee36780ec..69093659a 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Get.cs +++ b/src/Infrastructure/BotSharp.Core/Knowledges/Services/KnowledgeService.Get.cs @@ -1,6 +1,8 @@ using BotSharp.Abstraction.Graph.Models; +using BotSharp.Abstraction.Knowledges.Models; +using BotSharp.Abstraction.VectorStorage.Models; -namespace BotSharp.Plugin.KnowledgeBase.Services; +namespace BotSharp.Core.Knowledges.Services; public partial class KnowledgeService { diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Update.cs b/src/Infrastructure/BotSharp.Core/Knowledges/Services/KnowledgeService.Update.cs similarity index 91% rename from src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Update.cs rename to src/Infrastructure/BotSharp.Core/Knowledges/Services/KnowledgeService.Update.cs index 0fa1f5cd8..b9652f617 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.Update.cs +++ b/src/Infrastructure/BotSharp.Core/Knowledges/Services/KnowledgeService.Update.cs @@ -1,4 +1,6 @@ -namespace BotSharp.Plugin.KnowledgeBase.Services; +using BotSharp.Abstraction.VectorStorage.Models; + +namespace BotSharp.Core.Knowledges.Services; public partial class KnowledgeService { diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.cs b/src/Infrastructure/BotSharp.Core/Knowledges/Services/KnowledgeService.cs similarity index 80% rename from src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.cs rename to src/Infrastructure/BotSharp.Core/Knowledges/Services/KnowledgeService.cs index c668e07eb..2c0853d0a 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Services/KnowledgeService.cs +++ b/src/Infrastructure/BotSharp.Core/Knowledges/Services/KnowledgeService.cs @@ -1,4 +1,10 @@ -namespace BotSharp.Plugin.KnowledgeBase.Services; +using BotSharp.Abstraction.Graph; +using BotSharp.Abstraction.Knowledges.Settings; +using BotSharp.Abstraction.MLTasks; +using BotSharp.Abstraction.VectorStorage; +using BotSharp.Core.Knowledges.Helpers; + +namespace BotSharp.Core.Knowledges.Services; public partial class KnowledgeService : IKnowledgeService { diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/KnowledgeRetrievalFn.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/KnowledgeRetrievalFn.cs index a93374195..ffbef7708 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/KnowledgeRetrievalFn.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Functions/KnowledgeRetrievalFn.cs @@ -1,4 +1,4 @@ -using BotSharp.Abstraction.VectorStorage.Helpers; +using BotSharp.Abstraction.VectorStorage.Extensions; namespace BotSharp.Plugin.KnowledgeBase.Functions; diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/KnowledgeBasePlugin.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/KnowledgeBasePlugin.cs index 75d3284bf..ffe4d8f14 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/KnowledgeBasePlugin.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/KnowledgeBasePlugin.cs @@ -21,7 +21,6 @@ public void RegisterDI(IServiceCollection services, IConfiguration config) return settingService.Bind("KnowledgeBase"); }); - services.AddScoped(); services.AddSingleton(); services.AddScoped(); services.AddScoped(); diff --git a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Using.cs b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Using.cs index a78afefd6..1437f41e1 100644 --- a/src/Plugins/BotSharp.Plugin.KnowledgeBase/Using.cs +++ b/src/Plugins/BotSharp.Plugin.KnowledgeBase/Using.cs @@ -31,6 +31,5 @@ global using BotSharp.Abstraction.Agents.Models; global using BotSharp.Abstraction.Functions.Models; global using BotSharp.Abstraction.Repositories; -global using BotSharp.Plugin.KnowledgeBase.Services; global using BotSharp.Plugin.KnowledgeBase.Enum; global using BotSharp.Plugin.KnowledgeBase.Helpers; \ No newline at end of file diff --git a/src/Plugins/BotSharp.Plugin.Planner/Using.cs b/src/Plugins/BotSharp.Plugin.Planner/Using.cs index fa59ee23b..1e6737dfa 100644 --- a/src/Plugins/BotSharp.Plugin.Planner/Using.cs +++ b/src/Plugins/BotSharp.Plugin.Planner/Using.cs @@ -28,7 +28,7 @@ global using BotSharp.Abstraction.Knowledges.Settings; global using BotSharp.Abstraction.Knowledges.Enums; global using BotSharp.Abstraction.VectorStorage.Models; -global using BotSharp.Abstraction.VectorStorage.Helpers; +global using BotSharp.Abstraction.VectorStorage.Extensions; global using BotSharp.Plugin.Planner.Hooks; global using BotSharp.Plugin.Planner.Enums; diff --git a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs index 23a7b469f..8304e3b27 100644 --- a/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs +++ b/src/Plugins/BotSharp.Plugin.Twilio/Controllers/TwilioVoiceController.cs @@ -82,8 +82,7 @@ public async Task ReceiveCallerMessage([FromRoute] string conversat } } - response = new VoiceResponse() - .Redirect(new Uri($"{_settings.CallbackHost}/twilio/voice/{conversationId}/reply/{seqNum}?states={states}"), HttpMethod.Post); + response = new VoiceResponse().Redirect(new Uri($"{_settings.CallbackHost}/twilio/voice/{conversationId}/reply/{seqNum}?states={states}"), HttpMethod.Post); } else { From b244e07e1d0055a35a3da82628ffca9f687866c3 Mon Sep 17 00:00:00 2001 From: Jicheng Lu <103353@smsassist.com> Date: Wed, 4 Sep 2024 10:16:12 -0500 Subject: [PATCH 27/27] remove knowledge confidence --- .../Conversations/Models/RoleDialogModel.cs | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/RoleDialogModel.cs b/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/RoleDialogModel.cs index 2c43df1c0..8484c5fb0 100644 --- a/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/RoleDialogModel.cs +++ b/src/Infrastructure/BotSharp.Abstraction/Conversations/Models/RoleDialogModel.cs @@ -100,12 +100,6 @@ public class RoleDialogModel : ITrackableMessage [JsonPropertyName("generated_images")] public List GeneratedImages { get; set; } = new List(); - /// - /// Knowledge confidence - /// - [JsonPropertyName("knowledge_confidence")] - public float KnowledgeConfidence { get; set; } = 0.5f; - private RoleDialogModel() { @@ -146,8 +140,7 @@ public static RoleDialogModel From(RoleDialogModel source, Payload = source.Payload, StopCompletion = source.StopCompletion, Instruction = source.Instruction, - Data = source.Data, - KnowledgeConfidence = source.KnowledgeConfidence + Data = source.Data }; } }