Merge pull request #12 from rakeshv247/debug-mode

Introducting LLM debug mode

CHANGELOG.md

@@ -4,6 +4,14 @@ All notable changes to this project will be documented in this file.
 
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 
+## [0.1.0] - 2025-02-13
+
+### Added
+
+- Added Debug mode
+  - Introduced a new debug field in `StartConversation`.
+  - When LLM debug mode is enabled, the client will receive debug messages with LLM request and response content
+
 ## [0.0.9] - 2025-01-24
 
 ### Fixed

VERSION

@@ -1 +1 @@
-0.0.9
+0.1.0

speechmatics_flow/cli.py

@@ -24,6 +24,7 @@
     Interaction,
     ConnectionSettings,
     PlaybackSettings,
+    DebugMode,
 )
 from speechmatics_flow.templates import TemplateOptions
 
@@ -159,6 +160,10 @@ def get_playback_settings(args):
     )
 
 
+def get_debug_mode_settings(args):
+    return DebugMode(llm=args.get("llm_debug_enabled"))
+
+
 # pylint: disable=too-many-arguments,too-many-statements
 def add_printing_handlers(
     api,
@@ -284,6 +289,7 @@ def run(stream):
                 audio_settings=get_audio_settings(args),
                 conversation_config=get_conversation_config(args),
                 playback_settings=get_playback_settings(args),
+                debug_mode=get_debug_mode_settings(args),
                 from_cli=True,
             )
         except KeyboardInterrupt:

speechmatics_flow/cli_parser.py

@@ -153,6 +153,12 @@ def get_arg_parser():
         choices=[k for k in TemplateOptions.keys()],
         help="Choose your assistant.",
     )
+    parser.add_argument(
+        "--llm-debug-enabled",
+        default=False,
+        action="store_true",
+        help="Flag indicating whether to receive conversations between the LLM and the Flow backend in debug messages.",
+    )
 
     return parser
 

speechmatics_flow/client.py

@@ -30,6 +30,7 @@
     Interaction,
     PlaybackSettings,
     ServerMessageType,
+    DebugMode,
 )
 from speechmatics_flow.tool_function_param import ToolFunctionParam
 from speechmatics_flow.utils import read_in_chunks, json_utf8
@@ -67,6 +68,7 @@ def __init__(
         self.audio_settings = None
         self.playback_settings = None
         self.tools = None
+        self.debug_mode = None
 
         self.event_handlers = {x: [] for x in ServerMessageType}
         self.middlewares = {x: [] for x in ClientMessageType}
@@ -148,6 +150,10 @@ def _start_conversation(self):
         }
         if self.tools is not None:
             msg["tools"] = self.tools
+
+        if self.debug_mode:
+            msg["debug"] = self.debug_mode.asdict()
+
         self.session_running = True
         self._call_middleware(ClientMessageType.StartConversation, msg, False)
         LOGGER.debug(msg)
@@ -564,6 +570,7 @@ async def run(
         from_cli: bool = False,
         tools: Optional[List[ToolFunctionParam]] = None,
         playback_settings: PlaybackSettings = PlaybackSettings(),
+        debug_mode: DebugMode = None,
     ):
         """
         Begin a new recognition session.
@@ -586,6 +593,9 @@ async def run(
         :param playback_settings: Configuration for the playback stream.
         :type playback_settings: models.PlaybackSettings
 
+        :param debug_mode: Configuration to receive debug messages from Flow
+        :type debug_mode: models.DebugMode
+
         :raises Exception: Can raise any exception returned by the
             consumer/producer tasks.
 
@@ -596,6 +606,7 @@ async def run(
         self.audio_settings = audio_settings
         self.playback_settings = playback_settings
         self.tools = tools
+        self.debug_mode = debug_mode
 
         await self._init_synchronization_primitives()
 

speechmatics_flow/models.py

@@ -91,6 +91,18 @@ def asdict(self):
         )
 
 
+@dataclass
+class DebugMode:
+    """Defines debug flags for monitoring and troubleshooting Flow"""
+
+    llm: bool = False
+    """Optional Flag indicating whether to receive conversations between the LLM and the Flow backend as
+    debug messages."""
+
+    def asdict(self):
+        return asdict(self)
+
+
 class ClientMessageType(str, Enum):
     # pylint: disable=invalid-name
     """Defines various messages sent from client to server."""
@@ -179,7 +191,10 @@ class ServerMessageType(str, Enum):
     """Indicates a generic warning message."""
 
     Error = "Error"
-    """Indicates n generic error message."""
+    """Indicates a generic error message."""
+
+    Debug = "Debug"
+    """Indicates a debug message"""
 
 
 @dataclass

tests/test_cli.py

@@ -52,3 +52,18 @@ def test_get_conversation_config(args, exp_values):
     assert config.asdict() == exp_values, "Expecting {} but got {}".format(
         exp_values, config.asdict()
     )
+
+
+@mark.parametrize(
+    "args, exp_values",
+    [
+        param([], {"llm": False}),
+        param(["--llm-debug-enabled"], {"llm": True}),
+    ],
+)
+def test_get_debug_mode_settings(args, exp_values):
+    test_args = vars(cli.parse_args(args=args))
+    config = cli.get_debug_mode_settings(test_args)
+    assert (
+        config.asdict() == exp_values
+    ), f"Expecting {exp_values}, got {config.asdict()}"

tests/test_client.py

@@ -4,6 +4,7 @@
 import pytest
 from pytest import param
 
+from speechmatics_flow import DebugMode
 from speechmatics_flow.client import WebsocketClient
 from speechmatics_flow.models import (
     ServerMessageType,
@@ -33,23 +34,26 @@ def ws_client():
 
 
 @pytest.mark.parametrize(
-    "audio_format, conversation_config, tools, expected_start_message",
+    "audio_format, conversation_config, tools, debug_mode, expected_start_message",
     [
         param(
             AudioSettings(),
             ConversationConfig(),
             None,
+            DebugMode(),
             {
                 "message": ClientMessageType.StartConversation.value,
                 "audio_format": AudioSettings().asdict(),
                 "conversation_config": ConversationConfig().asdict(),
+                "debug": DebugMode().asdict(),
             },
             id="with default values",
         ),
         param(
             AudioSettings(),
             ConversationConfig(),
             [TOOL_FUNCTION],
+            None,
             {
                 "message": ClientMessageType.StartConversation.value,
                 "audio_format": AudioSettings().asdict(),
@@ -58,13 +62,27 @@ def ws_client():
             },
             id="with default values and tools",
         ),
+        param(
+            AudioSettings(),
+            ConversationConfig(),
+            None,
+            DebugMode(llm=True),
+            {
+                "message": ClientMessageType.StartConversation.value,
+                "audio_format": AudioSettings().asdict(),
+                "conversation_config": ConversationConfig().asdict(),
+                "debug": DebugMode(llm=True).asdict(),
+            },
+            id="with default values and llm debug mode enabled",
+        ),
     ],
 )
 def test_start_conversation(
     ws_client: WebsocketClient,
     audio_format: AudioSettings,
     conversation_config: ConversationConfig,
     tools: Optional[List[ToolFunctionParam]],
+    debug_mode: Optional[DebugMode],
     expected_start_message: Dict,
 ):
     handler_called = False
@@ -77,6 +95,7 @@ def handler(*_):
     ws_client.audio_settings = audio_format
     ws_client.conversation_config = conversation_config
     ws_client.tools = tools
+    ws_client.debug_mode = debug_mode
     start_conversation_msg = ws_client._start_conversation()
     assert start_conversation_msg == json.dumps(
         expected_start_message

tests/test_models.py

@@ -57,3 +57,17 @@ def test_playback_settings(config, want):
     audio_settings = models.PlaybackSettings(**config)
     got = asdict(audio_settings)
     assert got == want
+
+
+@pytest.mark.parametrize(
+    "config, want",
+    [
+        ({}, {"llm": False}),
+        ({"llm": False}, {"llm": False}),
+        ({"llm": True}, {"llm": True}),
+    ],
+)
+def test_debug_mode(config, want):
+    debug_mode_settings = models.DebugMode(**config)
+    got = debug_mode_settings.asdict()
+    assert got == want