Merge pull request #964 from hchen2020/master

IAudioSynthesis

Author: Oceania2018

src/Infrastructure/BotSharp.Abstraction/MLTasks/IAudioCompletion.cs

@@ -0,0 +1,15 @@
+namespace BotSharp.Abstraction.MLTasks;
+
+/// <summary>
+/// Text to speech synthesis
+/// </summary>
+public interface IAudioSynthesis
+{
+    string Provider { get; }
+
+    string Model { get; }
+
+    void SetModelName(string model);
+
+    Task<BinaryData> GenerateAudioAsync(string text, string? voice = "alloy", string? format = "mp3", string? instructions = null);
+}

src/Infrastructure/BotSharp.Abstraction/MLTasks/IAudioSynthesis.cs

@@ -0,0 +1,17 @@
+using System.IO;
+
+namespace BotSharp.Abstraction.MLTasks;
+
+/// <summary>
+/// Audio transcription service
+/// </summary>
+public interface IAudioTranscription
+{
+    string Provider { get; }
+
+    string Model { get; }
+
+    Task<string> TranscriptTextAsync(Stream audio, string audioFileName, string? text = null);
+
+    void SetModelName(string model);
+}

src/Infrastructure/BotSharp.Abstraction/MLTasks/IAudioTranscription.cs

@@ -3,14 +3,14 @@ namespace BotSharp.Abstraction.MLTasks.Settings;
 public class LlmModelSetting
 {
     /// <summary>
-    /// Model Id, like "gpt-3.5" and "gpt-4".
+    /// Model Id, like "gpt-4", "gpt-4o", "o1".
     /// </summary>
-    public string? Id { get; set; }
+    public string Id { get; set; } = null!;
 
     /// <summary>
     /// Deployment model name
     /// </summary>
-    public string Name { get; set; }
+    public string Name { get; set; } = null!;
 
     /// <summary>
     /// Model version
@@ -28,8 +28,8 @@ public class LlmModelSetting
     /// </summary>
     public string? Group { get; set; }
 
-    public string ApiKey { get; set; }
-    public string Endpoint { get; set; }
+    public string ApiKey { get; set; } = null!;
+    public string? Endpoint { get; set; }
     public LlmModelType Type { get; set; } = LlmModelType.Chat;
 
     /// <summary>

src/Infrastructure/BotSharp.Abstraction/MLTasks/Settings/LlmModelSetting.cs

@@ -11,6 +11,6 @@ public class ModelTurnDetection
 
 public class AudioTranscription
 {
-    public string Model { get; set; } = "whisper-1";
-    public string Language { get; set; } = "en";
+    public string Model { get; set; } = "gpt-4o-mini-transcribe";
+    public string? Language { get; set; }
 }

src/Infrastructure/BotSharp.Abstraction/Realtime/Models/ModelTurnDetection.cs

@@ -74,7 +74,7 @@ private async Task ConnectToModel(WebSocket userWebSocket)
         if (!model.Contains("-realtime-"))
         {
             var llmProviderService = _services.GetRequiredService<ILlmProviderService>();
-            model = llmProviderService.GetProviderModel("openai", "gpt-4", realTime: true).Name;
+            model = llmProviderService.GetProviderModel("openai", "gpt-4o", realTime: true).Name;
         }
 
         _completer.SetModelName(model);

src/Infrastructure/BotSharp.Core.Realtime/Services/RealtimeHub.cs

@@ -6,14 +6,14 @@ public partial class FileInstructService
 {
     public async Task<string> SpeechToText(string? provider, string? model, InstructFileModel audio, string? text = null)
     {
-        var completion = CompletionProvider.GetAudioCompletion(_services, provider: provider ?? "openai", model: model ?? "whisper-1");
+        var completion = CompletionProvider.GetAudioTranscriber(_services, provider: provider, model: model);
         var audioBytes = await DownloadFile(audio);
         using var stream = new MemoryStream();
         stream.Write(audioBytes, 0, audioBytes.Length);
         stream.Position = 0;
 
         var fileName = $"{audio.FileName ?? "audio"}.{audio.FileExtension ?? "wav"}";
-        var content = await completion.GenerateTextFromAudioAsync(stream, fileName, text);
+        var content = await completion.TranscriptTextAsync(stream, fileName, text);
         stream.Close();
         return content;
     }

src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Audio.cs

@@ -27,7 +27,7 @@ public async Task<string> ReadPdf(string? provider, string? model, string? model
 
             var innerAgentId = agentId ?? Guid.Empty.ToString();
             var completion = CompletionProvider.GetChatCompletion(_services, provider: provider ?? "openai",
-                model: model, modelId: modelId ?? "gpt-4", multiModal: true);
+                model: model, modelId: modelId ?? "gpt-4o", multiModal: true);
             var message = await completion.GetChatCompletions(new Agent()
             {
                 Id = innerAgentId,

src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.Pdf.cs

@@ -93,7 +93,7 @@ private async Task<IEnumerable<MessageFileModel>> SelectFiles(IEnumerable<Messag
             }
 
             var providerName = options.Provider ?? "openai";
-            var modelId = options?.ModelId ?? "gpt-4";
+            var modelId = options?.ModelId ?? "gpt-4o";
             var provider = llmProviderService.GetProviders().FirstOrDefault(x => x == providerName);
             var model = llmProviderService.GetProviderModel(provider: provider, id: modelId);
             var completion = CompletionProvider.GetChatCompletion(_services, provider: provider, model: model.Name);

src/Infrastructure/BotSharp.Core/Files/Services/Instruct/FileInstructService.SelectFile.cs

@@ -30,7 +30,7 @@ public static object GetCompletion(IServiceProvider services,
         }
         else if (settings.Type == LlmModelType.Audio)
         {
-            return GetAudioCompletion(services, provider: provider, model: model);
+            return GetAudioTranscriber(services, provider: provider, model: model);
         }
         else
         {
@@ -126,20 +126,39 @@ public static ITextEmbedding GetTextEmbedding(IServiceProvider services,
         return completer;
     }
 
-    public static IAudioCompletion GetAudioCompletion(
+    public static IAudioTranscription GetAudioTranscriber(
         IServiceProvider services,
-        string provider,
-        string model)
+        string? provider = null,
+        string? model = null)
     {
-        var completions = services.GetServices<IAudioCompletion>();
-        var completer = completions.FirstOrDefault(x => x.Provider == provider);
+        var completions = services.GetServices<IAudioTranscription>();
+        var completer = completions.FirstOrDefault(x => x.Provider == (provider ?? "openai"));
         if (completer == null)
         {
             var logger = services.GetRequiredService<ILogger<CompletionProvider>>();
-            logger.LogError($"Can't resolve audio-completion provider by {provider}");
+            logger.LogError($"Can't resolve audio-transcriber provider by {provider}");
+            return default!;
         }
 
-        completer.SetModelName(model);
+        completer.SetModelName(model ?? "gpt-4o-mini-transcribe");
+        return completer;
+    }
+
+    public static IAudioSynthesis GetAudioSynthesizer(
+        IServiceProvider services,
+        string? provider = null,
+        string? model = null)
+    {
+        var completions = services.GetServices<IAudioSynthesis>();
+        var completer = completions.FirstOrDefault(x => x.Provider == (provider ?? "openai"));
+        if (completer == null)
+        {
+            var logger = services.GetRequiredService<ILogger<CompletionProvider>>();
+            logger.LogError($"Can't resolve audio-synthesizer provider by {provider}");
+            return default!;
+        }
+
+        completer.SetModelName(model ?? "gpt-4o-mini-tts");
         return completer;
     }