Merge pull request #999 from SciSharp/realtimhub-refactor

realtime-hub-refactor

Author: Oceania2018

BotSharp.sln

@@ -143,6 +143,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BotSharp.Test.BrowserUse",
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BotSharp.LLM.Tests", "tests\BotSharp.LLM.Tests\BotSharp.LLM.Tests.csproj", "{7C0C7D13-D161-4AB0-9C29-83A0F1FF990E}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BotSharp.Test.RealtimeVoice", "tests\BotSharp.Test.RealtimeVoice\BotSharp.Test.RealtimeVoice.csproj", "{B067B126-88CD-4282-BEEF-7369B64423EF}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -599,6 +601,14 @@ Global
 		{7C0C7D13-D161-4AB0-9C29-83A0F1FF990E}.Release|Any CPU.Build.0 = Release|Any CPU
 		{7C0C7D13-D161-4AB0-9C29-83A0F1FF990E}.Release|x64.ActiveCfg = Release|Any CPU
 		{7C0C7D13-D161-4AB0-9C29-83A0F1FF990E}.Release|x64.Build.0 = Release|Any CPU
+		{B067B126-88CD-4282-BEEF-7369B64423EF}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{B067B126-88CD-4282-BEEF-7369B64423EF}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{B067B126-88CD-4282-BEEF-7369B64423EF}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{B067B126-88CD-4282-BEEF-7369B64423EF}.Debug|x64.Build.0 = Debug|Any CPU
+		{B067B126-88CD-4282-BEEF-7369B64423EF}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{B067B126-88CD-4282-BEEF-7369B64423EF}.Release|Any CPU.Build.0 = Release|Any CPU
+		{B067B126-88CD-4282-BEEF-7369B64423EF}.Release|x64.ActiveCfg = Release|Any CPU
+		{B067B126-88CD-4282-BEEF-7369B64423EF}.Release|x64.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -668,6 +678,7 @@ Global
 		{970BE341-9AC8-99A5-6572-E703C1E02FCB} = {E29DC6C4-5E57-48C5-BCB0-6B8F84782749}
 		{7D0DB012-9798-4BB9-B15B-A5B0B7B3B094} = {32FAFFFE-A4CB-4FEE-BF7C-84518BBC6DCC}
 		{7C0C7D13-D161-4AB0-9C29-83A0F1FF990E} = {32FAFFFE-A4CB-4FEE-BF7C-84518BBC6DCC}
+		{B067B126-88CD-4282-BEEF-7369B64423EF} = {32FAFFFE-A4CB-4FEE-BF7C-84518BBC6DCC}
 	EndGlobalSection
 	GlobalSection(ExtensibilityGlobals) = postSolution
 		SolutionGuid = {A9969D89-C98B-40A5-A12B-FC87E55B3A19}

src/Infrastructure/BotSharp.Abstraction/Agents/Settings/AgentSettings.cs

@@ -2,7 +2,7 @@ namespace BotSharp.Abstraction.Agents.Settings;
 
 public class AgentSettings
 {
-    public string DataDir { get; set; } = string.Empty;
+    public string DataDir { get; set; } = "agents";
     public string TemplateFormat { get; set; } = "liquid";
     public string HostAgentId { get; set; } = string.Empty;
     public bool EnableTranslator { get; set; } = false;

src/Infrastructure/BotSharp.Abstraction/Conversations/Settings/ConversationSetting.cs

@@ -2,7 +2,7 @@ namespace BotSharp.Abstraction.Conversations.Settings;
 
 public class ConversationSetting
 {
-    public string DataDir { get; set; }
+    public string DataDir { get; set; } = "conversations";
     public string ChatCompletion { get; set; }
     public bool EnableKnowledgeBase { get; set; }
     public bool ShowVerboseLog { get; set; }

src/Infrastructure/BotSharp.Abstraction/MLTasks/IRealTimeCompletion.cs

@@ -18,6 +18,7 @@ Task Connect(RealtimeHubConnection conn,
         Action<RoleDialogModel> onInputAudioTranscriptionCompleted,
         Action onUserInterrupted);
     Task AppenAudioBuffer(string message);
+    Task AppenAudioBuffer(ArraySegment<byte> data, int length);
 
     Task SendEventToModel(object message);
     Task Disconnect();

src/Infrastructure/BotSharp.Abstraction/Realtime/Enums/StreamChannelStatus.cs

@@ -0,0 +1,7 @@
+namespace BotSharp.Abstraction.Realtime.Enums;
+
+public enum StreamChannelStatus
+{
+    Open = 1,
+    Closed = 2
+}

src/Infrastructure/BotSharp.Abstraction/Realtime/IRealtimeHub.cs

@@ -15,5 +15,5 @@ public interface IRealtimeHub
     IRealTimeCompletion Completer { get; }
     IRealTimeCompletion SetCompleter(string provider);
 
-    Task Listen(WebSocket userWebSocket, Action<string> onUserMessageReceived);
+    Task ConnectToModel(Func<string, Task> responseToUser);
 }

src/Infrastructure/BotSharp.Abstraction/Realtime/IStreamChannel.cs

@@ -0,0 +1,13 @@
+using BotSharp.Abstraction.Realtime.Enums;
+using BotSharp.Abstraction.Realtime.Models;
+using System.Threading;
+
+namespace BotSharp.Abstraction.Realtime;
+
+public interface IStreamChannel
+{
+    Task ConnectAsync(string conversationId);
+    Task<StreamReceiveResult> ReceiveAsync(ArraySegment<byte> buffer, CancellationToken cancellation);
+    Task SendAsync(byte[] data, CancellationToken cancellation);
+    Task CloseAsync(StreamChannelStatus status, string description, CancellationToken cancellation);
+}

src/Infrastructure/BotSharp.Abstraction/Realtime/Models/ModelResponseEvent.cs

@@ -0,0 +1,7 @@
+namespace BotSharp.Abstraction.Realtime.Models;
+
+public class ModelResponseEvent
+{
+    [JsonPropertyName("event")]
+    public string Event { get; set; } = string.Empty;
+}

src/Infrastructure/BotSharp.Abstraction/Realtime/Models/ModelResponseMediaEvent.cs

@@ -0,0 +1,7 @@
+namespace BotSharp.Abstraction.Realtime.Models;
+
+public class ModelResponseMediaEvent : ModelResponseEvent
+{
+    [JsonPropertyName("media")]
+    public string Media { get; set; } = null!;
+}

src/Infrastructure/BotSharp.Abstraction/Realtime/Models/RealtimeHubConnection.cs

@@ -4,7 +4,6 @@ namespace BotSharp.Abstraction.Realtime.Models;
 
 public class RealtimeHubConnection
 {
-    public string Event { get; set; } = null!;
     public string StreamId { get; set; } = null!;
     public string? LastAssistantItemId { get; set; } = null!;
     public long LatestMediaTimestamp { get; set; }
@@ -13,10 +12,9 @@ public class RealtimeHubConnection
     public ConcurrentQueue<string> MarkQueue { get; set; } = new();
     public string CurrentAgentId { get; set; } = null!;
     public string ConversationId { get; set; } = null!;
-    public string Data { get; set; } = string.Empty;
-    public Func<string, object> OnModelMessageReceived { get; set; } = null!;
-    public Func<object> OnModelAudioResponseDone { get; set; } = null!;
-    public Func<object> OnModelUserInterrupted { get; set; } = null!;
+    public Func<string, string> OnModelMessageReceived { get; set; } = null!;
+    public Func<string> OnModelAudioResponseDone { get; set; } = null!;
+    public Func<string> OnModelUserInterrupted { get; set; } = null!;
 
     public void ResetResponseState()
     {