Merge pull request #1047 from hchen2020/master

Twilio.GetHints

Author: Oceania2018

src/Plugins/BotSharp.Plugin.OpenAI/Providers/Realtime/RealTimeCompletionProvider.cs

@@ -231,7 +231,6 @@ private async Task ReceiveMessage(
                 _logger.LogInformation($"{response.Type}: {receivedText}");
 
                 var data = JsonSerializer.Deserialize<ConversationItemCreated>(receivedText);
-                await Task.Delay(500);
                 onConversationItemCreated(receivedText);
             }
             else if (response.Type == "conversation.item.input_audio_transcription.completed")
@@ -253,7 +252,6 @@ private async Task ReceiveMessage(
             else if (response.Type == "input_audio_buffer.speech_stopped")
             {
                 _logger.LogInformation($"{response.Type}: {receivedText}");
-                await Task.Delay(500);
             }
             else if (response.Type == "input_audio_buffer.committed")
             {

src/Plugins/BotSharp.Plugin.Twilio/Services/TwilioMessageQueueService.cs

@@ -1,4 +1,5 @@
 using BotSharp.Abstraction.Files;
+using BotSharp.Abstraction.Realtime;
 using BotSharp.Abstraction.Routing;
 using BotSharp.Core.Infrastructures;
 using BotSharp.Plugin.Twilio.Interfaces;
@@ -110,7 +111,7 @@ private async Task ProcessUserMessageAsync(CallerMessage message)
             reply.SpeechFileName = await GetReplySpeechFileName(message.ConversationId, reply, sp);
         }
 
-        reply.Hints = GetHints(reply);
+        reply.Hints = GetHints(agentId, reply, sp);
         await sessionManager.SetAssistantReplyAsync(message.ConversationId, message.SeqNumber, reply);
     }
 
@@ -151,8 +152,13 @@ private static async Task<string> GetReplySpeechFileName(string conversationId,
         return fileName;
     }
 
-    private static string GetHints(AssistantMessage reply)
+    private static string GetHints(string agentId, AssistantMessage reply, IServiceProvider sp)
     {
+        var agentService = sp.GetRequiredService<IAgentService>();
+        var agent = agentService.GetAgent(agentId).Result;
+        var extraWords = new List<string>();
+        HookEmitter.Emit<IRealtimeHook>(sp, hook => extraWords.AddRange(hook.OnModelTranscriptPrompt(agent)));
+
         var phrases = reply.Content.Split(',', StringSplitOptions.RemoveEmptyEntries);
         int capcity = 100;
         var hints = new List<string>(capcity);
@@ -174,6 +180,7 @@ private static string GetHints(AssistantMessage reply)
         }
         // add frequency short words
         hints.AddRange(["yes", "no", "correct", "right"]);
+        hints.AddRange(extraWords);
         return string.Join(", ", hints.Select(x => x.ToLower()).Distinct().Reverse());
     }