move fal smart detection to plugin

Author: tschellenbach

agents-core/vision_agents/core/turn_detection/__init__.py

@@ -8,7 +8,6 @@
     TurnStartedEvent,
     TurnEndedEvent,
 )
-from .fal_turn_detection import FalTurnDetection
 
 
 __all__ = [
@@ -20,6 +19,4 @@
     # Events
     "TurnStartedEvent",
     "TurnEndedEvent",
-    # Implementations
-    "FalTurnDetection",
 ]

plugins/fal/README.md

@@ -11,8 +11,8 @@ keywords = ["getstream", "video", "realtime", "streaming", "AI", "voice agents",
 requires-python = ">=3.10"
 license = "MIT"
 dependencies = [
-    "getstream[webrtc,telemetry]>=2.5.0",
     "vision-agents",
+    "getstream[webrtc,telemetry]>=2.5.0",
 ]
 
 [project.urls]

plugins/getstream/pyproject.toml

@@ -0,0 +1,53 @@
+# Smart Turn Detection Plugin
+
+An AI-powered turn detection plugin for Vision Agents that uses the [Smart Turn model](https://github.com/pipecat-ai/smart-turn) to detect when a speaker has completed their turn in a conversation.
+
+## Overview
+
+Smart Turn is an open-source, community-driven, native audio turn detection model that goes beyond simple Voice Activity Detection (VAD). It analyzes grammar, tone, pace of speech, and various other complex audio and semantic cues to determine when a user has finished speaking, matching human expectations more closely than VAD-based approaches.
+
+For more information about the Smart Turn model, visit the [official repository](https://github.com/pipecat-ai/smart-turn).
+
+## Installation
+
+```bash
+pip install vision-agents-plugins-smart-turn
+```
+
+## Usage
+
+```python
+from vision_agents.plugins.smart_turn import TurnDetection
+
+# Initialize with FAL API key from environment variable
+turn_detector = TurnDetection()
+
+# Or specify API key directly
+turn_detector = TurnDetection(api_key="your_fal_api_key")
+
+# Register event handlers
+@turn_detector.on("turn_started")
+def on_turn_started(event_data):
+    print(f"Turn started: {event_data.speaker_id}")
+
+@turn_detector.on("turn_ended")
+def on_turn_ended(event_data):
+    print(f"Turn ended: {event_data.speaker_id} (confidence: {event_data.confidence:.3f})")
+
+# Start detection
+turn_detector.start()
+
+# Process audio
+await turn_detector.process_audio(pcm_data, user_id="user123")
+
+# Stop detection
+turn_detector.stop()
+```
+
+## Configuration Options
+
+- `api_key`: FAL API key (default: reads from FAL_KEY environment variable)
+- `buffer_duration`: Duration in seconds to buffer audio before processing (default: 2.0)
+- `confidence_threshold`: Probability threshold for "complete" predictions (default: 0.5)
+- `sample_rate`: Audio sample rate in Hz (default: 16000)
+- `channels`: Number of audio channels (default: 1)

plugins/smart_turn/README.md

@@ -0,0 +1,41 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "vision-agents-plugins-smart-turn"
+version = "0.1.0"
+description = "Smart Turn detection plugin for Vision Agents"
+readme = "README.md"
+requires-python = ">=3.10"
+license = "MIT"
+dependencies = [
+    "vision-agents",
+    "fal-client>=0.5.6",
+    "numpy>=2.2.6,<2.3",
+]
+
+[project.urls]
+Documentation = "https://visionagents.ai/"
+Website = "https://visionagents.ai/"
+Source = "https://github.com/GetStream/Vision-Agents"
+
+[tool.hatch.version]
+source = "vcs"
+raw-options = { root = "..", search_parent_directories = true, fallback_version = "0.0.0" }
+
+[tool.hatch.build.targets.wheel]
+packages = [".", "vision_agents"]
+
+[tool.hatch.build.targets.sdist]
+include = ["/vision_agents"]
+
+[tool.uv.sources]
+vision-agents = { workspace = true }
+
+[dependency-groups]
+dev = [
+    "pytest>=8.4.1",
+    "pytest-asyncio>=1.0.0",
+]
+

plugins/fal/py.typed renamed to plugins/smart_turn/py.typed

@@ -0,0 +1,31 @@
+import pytest
+from dotenv import load_dotenv
+
+
+load_dotenv()
+
+
+class TestSmartTurnPlugin:
+    def test_import(self):
+        """Test that the plugin can be imported."""
+        from vision_agents.plugins.smart_turn import TurnDetection
+
+        assert TurnDetection is not None
+
+    async def test_instantiation(self):
+        """Test that the TurnDetection class can be instantiated."""
+        from vision_agents.plugins.smart_turn import TurnDetection
+
+        detector = TurnDetection(api_key="test_key")
+        assert detector is not None
+        assert detector.api_key == "test_key"
+        assert detector.buffer_duration == 2.0
+        assert detector._confidence_threshold == 0.5
+
+    @pytest.mark.integration
+    async def test_turn_detection_integration(self):
+        """Integration test for turn detection (requires FAL_KEY in environment)."""
+        # This test should be run manually with a valid FAL_KEY
+        # For now, just pass
+        assert True
+

plugins/smart_turn/pyproject.toml

@@ -1,6 +1,7 @@
-from .stt import STT
+from .turn_detection import TurnDetection
 
 # Re-export under the new namespace for convenience
 __path__ = __import__("pkgutil").extend_path(__path__, __name__)
 
-__all__ = ["STT"]
+__all__ = ["TurnDetection"]
+

plugins/smart_turn/tests/test_turn_detection.py

@@ -1,5 +1,5 @@
 """
-FAL Smart-Turn implementation for turn detection using the smart-turn AI model.
+Smart Turn detection implementation using the FAL AI smart-turn model.
 
 This module provides integration with the FAL AI smart-turn model to detect
 when a speaker has completed their turn in a conversation.
@@ -19,16 +19,19 @@
 from getstream.audio.utils import resample_audio
 from getstream.video.rtc.track_util import PcmData
 from vision_agents.core.utils.utils import to_mono
-
-from .turn_detection import TurnDetector, TurnEvent, TurnEventData
+from vision_agents.core.turn_detection.turn_detection import (
+    TurnDetector,
+    TurnEvent,
+    TurnEventData,
+)
 
 
 def _resample(samples: np.ndarray) -> np.ndarray:
     """Resample audio from 48 kHz to 16 kHz."""
     return resample_audio(samples, 48000, 16000).astype(np.int16)
 
 
-class FalTurnDetection(TurnDetector):
+class TurnDetection(TurnDetector):
     """
     Turn detection implementation using FAL AI's smart-turn model.
 
@@ -48,7 +51,7 @@ def __init__(
         channels: int = 1,
     ):
         """
-        Initialize FAL turn detection.
+        Initialize Smart Turn detection.
 
         Args:
             api_key: FAL API key (if None, uses FAL_KEY env var)
@@ -59,10 +62,9 @@ def __init__(
         """
 
         super().__init__(
-            confidence_threshold=confidence_threshold,
-            provider_name="FalTurnDetection"
+            confidence_threshold=confidence_threshold, provider_name="SmartTurnDetection"
         )
-        self.logger = logging.getLogger("FalTurnDetection")
+        self.logger = logging.getLogger("SmartTurnDetection")
         self.api_key = api_key
         self.buffer_duration = buffer_duration
         self.sample_rate = sample_rate
@@ -75,15 +77,15 @@ def __init__(
 
         # Processing state
         self._processing_tasks: Dict[str, asyncio.Task] = {}
-        self._temp_dir = Path(tempfile.gettempdir()) / "fal_turn_detection"
+        self._temp_dir = Path(tempfile.gettempdir()) / "smart_turn_detection"
         self._temp_dir.mkdir(exist_ok=True)
 
         # Configure FAL client
         if self.api_key:
             os.environ["FAL_KEY"] = self.api_key
 
         self.logger.info(
-            f"Initialized FAL turn detection (buffer: {buffer_duration}s, threshold: {confidence_threshold})"
+            f"Initialized Smart Turn detection (buffer: {buffer_duration}s, threshold: {confidence_threshold})"
         )
 
     def _infer_channels(self, format_str: str) -> int:
@@ -94,7 +96,9 @@ def _infer_channels(self, format_str: str) -> int:
         elif any(f in format_str for f in ["mono", "s16", "int16", "pcm_s16le"]):
             return 1
         else:
-            self.logger.warning(f"Unknown format string: {format_str}. Assuming mono.")
+            self.logger.warning(
+                f"Unknown format string: {format_str}. Assuming mono."
+            )
             return 1
 
     def is_detecting(self) -> bool:
@@ -295,7 +299,7 @@ async def _process_turn_prediction(
             # Create event data
             event_data = TurnEventData(
                 timestamp=current_time,
-                speaker_id=user_id,  # Now use the user_id directly
+                speaker_id=user_id,
                 confidence=probability,
                 custom={
                     "prediction": prediction,
@@ -315,7 +319,7 @@ async def _process_turn_prediction(
                 # Set them as current speaker if they weren't already (in case we missed the start)
                 if self._current_speaker != user_id:
                     self._current_speaker = user_id
-                    
+
                 self._emit_turn_event(TurnEvent.TURN_ENDED, event_data)
                 self._current_speaker = None
 
@@ -346,7 +350,7 @@ def start(self) -> None:
         if self._is_detecting:
             return
         self._is_detecting = True
-        self.logger.info("FAL turn detection started")
+        self.logger.info("Smart Turn detection started")
 
     def stop(self) -> None:
         """Stop turn detection and clean up."""
@@ -374,4 +378,5 @@ def stop(self) -> None:
         except Exception as e:
             self.logger.warning(f"Failed to clean up temp files: {e}")
 
-        self.logger.info("FAL turn detection stopped")
+        self.logger.info("Smart Turn detection stopped")
+

plugins/fal/vision_agents/plugins/fal/__init__.py renamed to plugins/smart_turn/vision_agents/plugins/smart_turn/__init__.py

@@ -0,0 +1 @@
+Wizper for vision agents