diff --git a/screenpipe-app-tauri/app/timeline/page.tsx b/screenpipe-app-tauri/app/timeline/page.tsx index e89b51081..052b22afa 100644 --- a/screenpipe-app-tauri/app/timeline/page.tsx +++ b/screenpipe-app-tauri/app/timeline/page.tsx @@ -25,6 +25,7 @@ import { TimelineDockIcon, TimelineIconsSection, } from "@/components/timeline/timeline-dock"; +import { AudioTranscript } from "@/components/timeline/audio-transcript"; export interface StreamTimeSeriesResponse { timestamp: string; @@ -46,7 +47,7 @@ interface DeviceMetadata { timestamp: string; } -interface AudioData { +export interface AudioData { device_name: string; is_input: boolean; transcription: string; @@ -302,7 +303,9 @@ export default function Timeline() { const handleScroll = (e: React.WheelEvent) => { const isWithinAiPanel = aiPanelRef.current?.contains(e.target as Node); - if (isWithinAiPanel) { + const isWithinAudioPanel = document.querySelector('.audio-transcript-panel')?.contains(e.target as Node); + + if (isWithinAiPanel || isWithinAudioPanel) { return; } @@ -754,6 +757,13 @@ export default function Timeline() { alt="Current frame" /> )} + {currentFrame && ( + + )}
diff --git a/screenpipe-app-tauri/components/timeline/audio-transcript.tsx b/screenpipe-app-tauri/components/timeline/audio-transcript.tsx new file mode 100644 index 000000000..63152335f --- /dev/null +++ b/screenpipe-app-tauri/components/timeline/audio-transcript.tsx @@ -0,0 +1,253 @@ +import { useState, useEffect, useRef } from "react"; +import { AudioData, StreamTimeSeriesResponse } from "@/app/timeline/page"; +import { Button } from "@/components/ui/button"; +import { Card } from "@/components/ui/card"; +import { Play, Pause, Volume2, GripHorizontal, X } from "lucide-react"; +import { VideoComponent } from "@/components/video"; + +interface AudioGroup { + deviceName: string; + isInput: boolean; + audioItems: AudioData[]; + startTime: Date; + endTime: Date; +} + +interface AudioTranscriptProps { + frames: StreamTimeSeriesResponse[]; + currentIndex: number; + groupingWindowMs?: number; // how many ms to group audio files together + onClose?: () => void; +} + +export function AudioTranscript({ + frames, + currentIndex, + groupingWindowMs = 30000, + onClose, +}: AudioTranscriptProps) { + const [audioGroups, setAudioGroups] = useState([]); + const [playing, setPlaying] = useState(null); + const [position, setPosition] = useState({ + x: window.innerWidth - 320, + y: 100, + }); + const [isDragging, setIsDragging] = useState(false); + const [dragOffset, setDragOffset] = useState({ x: 0, y: 0 }); + const [windowSize, setWindowSize] = useState({ width: 300, height: 500 }); + const resizerRef = useRef(null); + const panelRef = useRef(null); + + // Group audio files from nearby frames + useEffect(() => { + if (!frames.length) return; + + const currentFrame = frames[currentIndex]; + const currentTime = new Date(currentFrame.timestamp); + const windowStart = new Date(currentTime.getTime() - groupingWindowMs); + const windowEnd = new Date(currentTime.getTime() + groupingWindowMs); + + // Get frames within our time window + const nearbyFrames = frames.filter((frame) => { + const frameTime = new Date(frame.timestamp); + return frameTime >= windowStart && frameTime <= windowEnd; + }); + + // Group audio by device + const groups = new Map(); + + nearbyFrames.forEach((frame) => { + frame.devices.forEach((device) => { + device.audio.forEach((audio) => { + const key = `${audio.device_name}-${audio.is_input}`; + + if (!groups.has(key)) { + groups.set(key, { + deviceName: audio.device_name, + isInput: audio.is_input, + audioItems: [], + startTime: new Date(frame.timestamp), + endTime: new Date(frame.timestamp), + }); + } + + const group = groups.get(key)!; + group.audioItems.push(audio); + + // Update time range + const frameTime = new Date(frame.timestamp); + if (frameTime < group.startTime) group.startTime = frameTime; + if (frameTime > group.endTime) group.endTime = frameTime; + }); + }); + }); + + setAudioGroups(Array.from(groups.values())); + }, [frames, currentIndex, groupingWindowMs]); + + const handlePlay = (audioPath: string) => { + if (playing === audioPath) { + setPlaying(null); + } else { + setPlaying(audioPath); + } + }; + + const handlePanelMouseDown = (e: React.MouseEvent) => { + setIsDragging(true); + setDragOffset({ + x: e.clientX - position.x, + y: e.clientY - position.y, + }); + }; + + const handlePanelMouseMove = (e: React.MouseEvent) => { + if (isDragging) { + setPosition({ + x: e.clientX - dragOffset.x, + y: e.clientY - dragOffset.y, + }); + } + }; + + const handlePanelMouseUp = () => { + if (isDragging) { + setIsDragging(false); + } + }; + + const handleResizeMouseDown = (e: React.MouseEvent) => { + e.preventDefault(); + const startX = e.clientX; + const startY = e.clientY; + const startWidth = windowSize.width; + const startHeight = windowSize.height; + + const handleMouseMove = (moveEvent: MouseEvent) => { + const newWidth = Math.max(200, startWidth + moveEvent.clientX - startX); + const newHeight = Math.max(200, startHeight + moveEvent.clientY - startY); + setWindowSize({ width: newWidth, height: newHeight }); + }; + + const handleMouseUp = () => { + document.removeEventListener("mousemove", handleMouseMove); + document.removeEventListener("mouseup", handleMouseUp); + }; + + document.addEventListener("mousemove", handleMouseMove); + document.addEventListener("mouseup", handleMouseUp); + }; + + // Modify the content scroll handler + const handleContentScroll = (e: React.WheelEvent) => { + e.stopPropagation(); // Stop the event from reaching the timeline + }; + + return ( +
+
+
+
+ + audio transcripts +
+ +
+
+ +
+ {audioGroups.map((group, groupIndex) => ( + +
+ {group.deviceName} ({group.isInput ? "input" : "output"}) +
+ {group.startTime.toLocaleTimeString()} -{" "} + {group.endTime.toLocaleTimeString()} +
+
+ + {group.audioItems.map((audio, index) => ( +
+
+ +
+ + {Math.round(audio.duration_secs)}s +
+
+ + {audio.transcription && ( +
+ {audio.transcription} +
+ )} + + {playing === audio.audio_file_path && ( +
+ +
+ )} +
+ ))} +
+ ))} +
+ +
+
+ ); +} diff --git a/screenpipe-app-tauri/components/timeline/timeline-block.tsx b/screenpipe-app-tauri/components/timeline/timeline-block.tsx index 5110b6220..0a5e2111d 100644 --- a/screenpipe-app-tauri/components/timeline/timeline-block.tsx +++ b/screenpipe-app-tauri/components/timeline/timeline-block.tsx @@ -93,9 +93,6 @@ export function TimelineBlocks({ frames, timeRange }: TimelineBlocksProps) { return blocks; }, [frames, timeRange]); - // Debug output - console.log("timeline blocks:", blocks); - return (
{blocks.map((block, index) => { diff --git a/screenpipe-app-tauri/components/timeline/timeline-dock.tsx b/screenpipe-app-tauri/components/timeline/timeline-dock.tsx index 8e598a0c8..0b92c4eb1 100644 --- a/screenpipe-app-tauri/components/timeline/timeline-dock.tsx +++ b/screenpipe-app-tauri/components/timeline/timeline-dock.tsx @@ -273,6 +273,7 @@ export function TimelineIconsSection({ return (
+ {processedBlocks.map((block, i) => (
{isAudio ? (
-

Audio Device Recording