diff --git a/.changeset/real-crabs-beam.md b/.changeset/real-crabs-beam.md new file mode 100644 index 000000000..15bf6c0f3 --- /dev/null +++ b/.changeset/real-crabs-beam.md @@ -0,0 +1,6 @@ +--- +"@livekit/components-core": patch +"@livekit/components-react": patch +--- + +Add activeSegements to useTrackTranscription diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 0cf789972..0fb9dda5d 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -13,7 +13,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - uses: pnpm/action-setup@v2 + - uses: pnpm/action-setup@v4 - name: Use Node.js 20 uses: actions/setup-node@v4 with: diff --git a/.github/workflows/size-limit.yaml b/.github/workflows/size-limit.yaml index 50cad3bbc..79ad03364 100644 --- a/.github/workflows/size-limit.yaml +++ b/.github/workflows/size-limit.yaml @@ -11,7 +11,7 @@ jobs: CI_JOB_NUMBER: 1 steps: - uses: actions/checkout@v4 - - uses: pnpm/action-setup@v2 + - uses: pnpm/action-setup@v4 - name: Use Node.js 20 uses: actions/setup-node@v4 with: diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 1d675f442..dbe6daab1 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -16,7 +16,7 @@ jobs: with: fetch-depth: 2 - - uses: pnpm/action-setup@v2 + - uses: pnpm/action-setup@v4 - name: Use Node.js 20 uses: actions/setup-node@v4 diff --git a/examples/nextjs/pages/transcriptions.tsx b/examples/nextjs/pages/transcriptions.tsx new file mode 100644 index 000000000..d7be2700e --- /dev/null +++ b/examples/nextjs/pages/transcriptions.tsx @@ -0,0 +1,77 @@ +import { + LiveKitRoom, + useToken, + setLogLevel, + useParticipantTracks, + useTrackTranscription, +} from '@livekit/components-react'; +import { Track } from 'livekit-client'; +import type { NextPage } from 'next'; +import { generateRandomUserId } from '../lib/helper'; +import { useMemo } from 'react'; + +const TranscriptionExample: NextPage = () => { + const params = typeof window !== 'undefined' ? new URLSearchParams(location.search) : null; + const roomName = useMemo(() => params?.get('room') ?? generateRandomUserId(), []); + setLogLevel('info', { liveKitClientLogLevel: 'debug' }); + const userId = useMemo(() => params?.get('user') ?? 'test-user', []); + + const tokenOptions = useMemo(() => { + return { + userInfo: { + identity: userId, + name: userId, + }, + }; + }, [userId]); + + const token = useToken(process.env.NEXT_PUBLIC_LK_TOKEN_ENDPOINT, roomName, tokenOptions); + + return ( +
+ { + console.error(e); + alert( + 'Error acquiring camera or microphone permissions. Please make sure you grant the necessary permissions in your browser and reload the tab', + ); + }} + > +
+
+

All

+ +
+
+

Active only

+ +
+
+ +
+ ); +}; + +function ParticipantTranscriptions({ + identity, + activeOnly = true, + ...props +}: React.HtmlHTMLAttributes & { identity: string; activeOnly?: boolean }) { + const audioTracks = useParticipantTracks([Track.Source.Microphone], identity); + const transcriptions = useTrackTranscription(audioTracks[0]); + const segments = activeOnly ? transcriptions.activeSegments : transcriptions.segments; + + return ( +
+ {segments.map((segment) => ( +

{segment.text}

+ ))} +
+ ); +} + +export default TranscriptionExample; diff --git a/packages/core/src/helper/transcriptions.ts b/packages/core/src/helper/transcriptions.ts index c60f16784..035229a3b 100644 --- a/packages/core/src/helper/transcriptions.ts +++ b/packages/core/src/helper/transcriptions.ts @@ -8,11 +8,11 @@ export type ReceivedTranscriptionSegment = TranscriptionSegment & { export function getActiveTranscriptionSegments( segments: ReceivedTranscriptionSegment[], syncTimes: { timestamp: number; rtpTimestamp?: number }, - maxAge = 0, + maxAge = 5_000, ) { return segments.filter((segment) => { - const hasTrackSync = !!syncTimes.rtpTimestamp; - const currentTrackTime = syncTimes.rtpTimestamp ?? performance.timeOrigin + performance.now(); + const hasTrackSync = !!syncTimes.rtpTimestamp && segment.startTime !== 0; + const currentTrackTime = syncTimes.rtpTimestamp || performance.timeOrigin + performance.now(); // if a segment arrives late, consider startTime to be the media timestamp from when the segment was received client side const displayStartTime = hasTrackSync ? Math.max(segment.receivedAtMediaTimestamp, segment.startTime) @@ -29,6 +29,7 @@ export function addMediaTimestampToTranscription( segment: TranscriptionSegment, timestamps: { timestamp: number; rtpTimestamp?: number }, ): ReceivedTranscriptionSegment { + console.log('new media timestamp', { segment, receivedAt: timestamps.timestamp }); return { ...segment, receivedAtMediaTimestamp: timestamps.rtpTimestamp ?? 0, diff --git a/packages/react/etc/components-react.api.md b/packages/react/etc/components-react.api.md index eb7c573dd..b6846bb0c 100644 --- a/packages/react/etc/components-react.api.md +++ b/packages/react/etc/components-react.api.md @@ -667,6 +667,7 @@ export interface TrackToggleProps extends Omit extends Omit + participantIdentity + ? participantIdentity === room.localParticipant.identity + ? room.localParticipant + : room.getParticipantByIdentity(participantIdentity) + : participantContext, + [connectionState, participantIdentity, room, participantContext], + ); const observable = React.useMemo( () => (p ? participantTracksObservable(p, { sources }) : undefined), [p?.sid, p?.identity, JSON.stringify(sources)], diff --git a/packages/react/src/hooks/useTrackSyncTime.ts b/packages/react/src/hooks/useTrackSyncTime.ts index a1f2ef737..ff9d72871 100644 --- a/packages/react/src/hooks/useTrackSyncTime.ts +++ b/packages/react/src/hooks/useTrackSyncTime.ts @@ -5,13 +5,14 @@ import { useObservableState } from './internal'; /** * @internal */ -export function useTrackSyncTime({ publication }: TrackReferenceOrPlaceholder) { +export function useTrackSyncTime(trackRef?: TrackReferenceOrPlaceholder) { const observable = React.useMemo( - () => (publication?.track ? trackSyncTimeObserver(publication.track) : undefined), - [publication?.track], + () => + trackRef?.publication?.track ? trackSyncTimeObserver(trackRef?.publication.track) : undefined, + [trackRef?.publication?.track], ); return useObservableState(observable, { - timestamp: Date.now(), - rtpTimestamp: publication?.track?.rtpTimestamp, + timestamp: performance.timeOrigin + performance.now(), + rtpTimestamp: trackRef?.publication?.track?.rtpTimestamp, }); } diff --git a/packages/react/src/hooks/useTrackTranscription.ts b/packages/react/src/hooks/useTrackTranscription.ts index d99589252..4b84157e3 100644 --- a/packages/react/src/hooks/useTrackTranscription.ts +++ b/packages/react/src/hooks/useTrackTranscription.ts @@ -1,12 +1,12 @@ import { type ReceivedTranscriptionSegment, - addMediaTimestampToTranscription as addTimestampsToTranscription, + addMediaTimestampToTranscription, dedupeSegments, - // getActiveTranscriptionSegments, + getActiveTranscriptionSegments, getTrackReferenceId, trackTranscriptionObserver, type TrackReferenceOrPlaceholder, - // didActiveSegmentsChange, + didActiveSegmentsChange, } from '@livekit/components-core'; import type { TranscriptionSegment } from 'livekit-client'; import * as React from 'react'; @@ -22,12 +22,12 @@ export interface TrackTranscriptionOptions { */ bufferSize?: number; /** amount of time (in ms) that the segment is considered `active` past its original segment duration, defaults to 2_000 */ - // maxAge?: number; + maxAge?: number; } const TRACK_TRANSCRIPTION_DEFAULTS = { bufferSize: 100, - // maxAge: 2_000, + maxAge: 5_000, } as const satisfies TrackTranscriptionOptions; /** @@ -35,28 +35,31 @@ const TRACK_TRANSCRIPTION_DEFAULTS = { * @alpha */ export function useTrackTranscription( - trackRef: TrackReferenceOrPlaceholder, + trackRef?: TrackReferenceOrPlaceholder, options?: TrackTranscriptionOptions, ) { const opts = { ...TRACK_TRANSCRIPTION_DEFAULTS, ...options }; const [segments, setSegments] = React.useState>([]); - // const [activeSegments, setActiveSegments] = React.useState>( - // [], - // ); - // const prevActiveSegments = React.useRef([]); + const [activeSegments, setActiveSegments] = React.useState>( + [], + ); + const prevActiveSegments = React.useRef([]); const syncTimestamps = useTrackSyncTime(trackRef); - const handleSegmentMessage = (newSegments: TranscriptionSegment[]) => { - setSegments((prevSegments) => - dedupeSegments( - prevSegments, - // when first receiving a segment, add the current media timestamp to it - newSegments.map((s) => addTimestampsToTranscription(s, syncTimestamps)), - opts.bufferSize, - ), - ); - }; + const handleSegmentMessage = React.useCallback( + (newSegments: TranscriptionSegment[]) => { + setSegments((prevSegments) => + dedupeSegments( + prevSegments, + // when first receiving a segment, add the current media timestamp to it + newSegments.map((s) => addMediaTimestampToTranscription(s, syncTimestamps)), + opts.bufferSize, + ), + ); + }, + [syncTimestamps, opts.bufferSize], + ); React.useEffect(() => { - if (!trackRef.publication) { + if (!trackRef?.publication) { return; } const subscription = trackTranscriptionObserver(trackRef.publication).subscribe((evt) => { @@ -65,22 +68,21 @@ export function useTrackTranscription( return () => { subscription.unsubscribe(); }; - }, [getTrackReferenceId(trackRef), handleSegmentMessage]); + }, [trackRef && getTrackReferenceId(trackRef), handleSegmentMessage]); - // React.useEffect(() => { - // if (syncTimestamps) { - // const newActiveSegments = getActiveTranscriptionSegments( - // segments, - // syncTimestamps, - // opts.maxAge, - // ); - // // only update active segment array if content actually changed - // if (didActiveSegmentsChange(prevActiveSegments.current, newActiveSegments)) { - // setActiveSegments(newActiveSegments); - // prevActiveSegments.current = newActiveSegments; - // } - // } - // }, [syncTimestamps, segments, opts.maxAge]); - - return { segments }; + React.useEffect(() => { + if (syncTimestamps) { + const newActiveSegments = getActiveTranscriptionSegments( + segments, + syncTimestamps, + opts.maxAge, + ); + // only update active segment array if content actually changed + if (didActiveSegmentsChange(prevActiveSegments.current, newActiveSegments)) { + setActiveSegments(newActiveSegments); + prevActiveSegments.current = newActiveSegments; + } + } + }, [syncTimestamps, segments, opts.maxAge]); + return { segments, activeSegments }; }