Skip to content

Commit b00181b

Browse files
thomasyuill-livekitthomasyuill-livekit
committed
AudioGridVisualizer
1 parent 7800c99 commit b00181b

File tree

4 files changed

+590
-4
lines changed

4 files changed

+590
-4
lines changed

‎app/ui/_components.tsx‎

Lines changed: 142 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -13,15 +13,20 @@ import { MicrophoneIcon } from '@phosphor-icons/react/dist/ssr';
1313
import { useSession } from '@/components/app/session-provider';
1414
import { AgentControlBar } from '@/components/livekit/agent-control-bar/agent-control-bar';
1515
import { TrackControl } from '@/components/livekit/agent-control-bar/track-control';
16-
import { TrackDeviceSelect } from '@/components/livekit/agent-control-bar/track-device-select';
17-
import { TrackToggle } from '@/components/livekit/agent-control-bar/track-toggle';
16+
// import { TrackDeviceSelect } from '@/components/livekit/agent-control-bar/track-device-select';
17+
// import { TrackToggle } from '@/components/livekit/agent-control-bar/track-toggle';
1818
import { Alert, AlertDescription, AlertTitle, alertVariants } from '@/components/livekit/alert';
1919
import { AlertToast } from '@/components/livekit/alert-toast';
2020
import { BarVisualizer } from '@/components/livekit/audio-visualizer/audio-bar-visualizer/_bar-visualizer';
2121
import {
2222
AudioBarVisualizer,
2323
audioBarVisualizerVariants,
2424
} from '@/components/livekit/audio-visualizer/audio-bar-visualizer/audio-bar-visualizer';
25+
import {
26+
AudioGridVisualizer,
27+
type GridOptions,
28+
} from '@/components/livekit/audio-visualizer/audio-grid-visualizer/audio-grid-visualizer';
29+
import { gridVariants } from '@/components/livekit/audio-visualizer/audio-grid-visualizer/demos';
2530
import { Button, buttonVariants } from '@/components/livekit/button';
2631
import { ChatEntry } from '@/components/livekit/chat-entry';
2732
import {
@@ -191,8 +196,8 @@ export const COMPONENTS = {
191196
</Container>
192197
),
193198

194-
// Audio visualizer
195-
AudioVisualizer: () => {
199+
// Audio bar visualizer
200+
AudioBarVisualizer: () => {
196201
const barCounts = ['0', '3', '5', '7', '9'];
197202
const sizes = ['icon', 'xs', 'sm', 'md', 'lg', 'xl'];
198203
const states = [
@@ -309,6 +314,139 @@ export const COMPONENTS = {
309314
);
310315
},
311316

317+
// Audio grid visualizer
318+
AudioGridVisualizer: () => {
319+
const rowCounts = ['3', '5', '7', '9', '11', '13', '15'];
320+
const columnCounts = ['3', '5', '7', '9', '11', '13', '15'];
321+
const states = [
322+
'disconnected',
323+
'connecting',
324+
'initializing',
325+
'listening',
326+
'thinking',
327+
'speaking',
328+
] as AgentState[];
329+
330+
const { microphoneTrack, localParticipant } = useLocalParticipant();
331+
const [rowCount, setRowCount] = useState(rowCounts[0]);
332+
const [columnCount, setColumnCount] = useState(columnCounts[0]);
333+
const [state, setState] = useState<AgentState>(states[0]);
334+
const [demoIndex, setDemoIndex] = useState(0);
335+
336+
const micTrackRef = useMemo<TrackReferenceOrPlaceholder | undefined>(() => {
337+
return state === 'speaking'
338+
? ({
339+
participant: localParticipant,
340+
source: Track.Source.Microphone,
341+
publication: microphoneTrack,
342+
} as TrackReference)
343+
: undefined;
344+
}, [state, localParticipant, microphoneTrack]);
345+
346+
useMicrophone();
347+
348+
const demoOptions = {
349+
rowCount: parseInt(rowCount),
350+
columnCount: parseInt(columnCount),
351+
...gridVariants[demoIndex],
352+
};
353+
354+
return (
355+
<Container componentName="AudioVisualizer">
356+
<div className="flex items-center gap-2">
357+
<div className="flex-1">
358+
<label className="font-mono text-xs uppercase" htmlFor="state">
359+
State
360+
</label>
361+
<Select value={state} onValueChange={(value) => setState(value as AgentState)}>
362+
<SelectTrigger id="state" className="w-full">
363+
<SelectValue placeholder="Select a state" />
364+
</SelectTrigger>
365+
<SelectContent>
366+
{states.map((state) => (
367+
<SelectItem key={state} value={state}>
368+
{state}
369+
</SelectItem>
370+
))}
371+
</SelectContent>
372+
</Select>
373+
</div>
374+
375+
<div className="flex-1">
376+
<label className="font-mono text-xs uppercase" htmlFor="rowCount">
377+
Row count
378+
</label>
379+
<Select value={rowCount.toString()} onValueChange={(value) => setRowCount(value)}>
380+
<SelectTrigger id="rowCount" className="w-full">
381+
<SelectValue placeholder="Select a bar count" />
382+
</SelectTrigger>
383+
<SelectContent>
384+
{rowCounts.map((rowCount) => (
385+
<SelectItem key={rowCount} value={rowCount.toString()}>
386+
{parseInt(rowCount) || 'Default'}
387+
</SelectItem>
388+
))}
389+
</SelectContent>
390+
</Select>
391+
</div>
392+
393+
<div className="flex-1">
394+
<label className="font-mono text-xs uppercase" htmlFor="columnCount">
395+
Column count
396+
</label>
397+
<Select value={columnCount.toString()} onValueChange={(value) => setColumnCount(value)}>
398+
<SelectTrigger id="columnCount" className="w-full">
399+
<SelectValue placeholder="Select a column count" />
400+
</SelectTrigger>
401+
<SelectContent>
402+
{columnCounts.map((columnCount) => (
403+
<SelectItem key={columnCount} value={columnCount.toString()}>
404+
{parseInt(columnCount) || 'Default'}
405+
</SelectItem>
406+
))}
407+
</SelectContent>
408+
</Select>
409+
</div>
410+
411+
<div className="flex-1">
412+
<label className="font-mono text-xs uppercase" htmlFor="demoIndex">
413+
Demo
414+
</label>
415+
<Select
416+
value={demoIndex.toString()}
417+
onValueChange={(value) => setDemoIndex(parseInt(value))}
418+
>
419+
<SelectTrigger id="demoIndex" className="w-full">
420+
<SelectValue placeholder="Select a demo" />
421+
</SelectTrigger>
422+
<SelectContent>
423+
{gridVariants.map((_, idx) => (
424+
<SelectItem key={idx} value={idx.toString()}>
425+
Demo {String(idx + 1)}
426+
</SelectItem>
427+
))}
428+
</SelectContent>
429+
</Select>
430+
</div>
431+
</div>
432+
433+
<div className="grid place-items-center py-12">
434+
<AudioGridVisualizer
435+
key={`${demoIndex}-${rowCount}-${columnCount}`}
436+
state={state}
437+
audioTrack={micTrackRef!}
438+
options={demoOptions}
439+
/>
440+
</div>
441+
<div className="border-border bg-muted overflow-x-auto rounded-xl border p-8">
442+
<pre className="text-muted-foreground text-sm">
443+
<code>{JSON.stringify(demoOptions, null, 2)}</code>
444+
</pre>
445+
</div>
446+
</Container>
447+
);
448+
},
449+
312450
// Agent control bar
313451
AgentControlBar: () => {
314452
useMicrophone();

‎components/livekit/audio-visualizer/audio-grid-visualizer/audio-grid-visualizer.tsx‎

Lines changed: 136 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,136 @@
1+
import { CSSProperties, ComponentType, JSX, useMemo } from 'react';
2+
import { LocalAudioTrack, RemoteAudioTrack } from 'livekit-client';
3+
import {
4+
type AgentState,
5+
type TrackReferenceOrPlaceholder,
6+
useMultibandTrackVolume,
7+
} from '@livekit/components-react';
8+
import { cn } from '@/lib/utils';
9+
import { type Coordinate, useGridAnimator } from './hooks/useGridAnimator';
10+
11+
type GridComponentType =
12+
| ComponentType<{ style?: CSSProperties; className?: string }>
13+
| keyof JSX.IntrinsicElements;
14+
15+
export interface GridOptions {
16+
radius?: number;
17+
interval?: number;
18+
rowCount?: number;
19+
columnCount?: number;
20+
className?: string;
21+
baseClassName?: string;
22+
offClassName?: string;
23+
onClassName?: string;
24+
gridComponent?: GridComponentType;
25+
transformer?: (index: number, rowCount: number, columnCount: number) => CSSProperties;
26+
}
27+
28+
function useGrid(options: GridOptions) {
29+
return useMemo(() => {
30+
const { columnCount = 5, rowCount } = options;
31+
32+
const _columnCount = columnCount;
33+
const _rowCount = rowCount ?? columnCount;
34+
const items = new Array(_columnCount * _rowCount).fill(0).map((_, idx) => idx);
35+
36+
return { columnCount: _columnCount, rowCount: _rowCount, items };
37+
}, [options]);
38+
}
39+
40+
interface GridCellProps {
41+
index: number;
42+
state: AgentState;
43+
options: GridOptions;
44+
rowCount: number;
45+
volumeBands: number[];
46+
columnCount: number;
47+
highlightedCoordinate: Coordinate;
48+
Component: GridComponentType;
49+
}
50+
51+
function GridCell({
52+
index,
53+
state,
54+
options,
55+
rowCount,
56+
volumeBands,
57+
columnCount,
58+
highlightedCoordinate,
59+
Component,
60+
}: GridCellProps) {
61+
const { interval = 100, baseClassName, onClassName, offClassName, transformer } = options;
62+
const rowMidPoint = Math.floor(rowCount / 2);
63+
64+
if (state === 'speaking') {
65+
const y = Math.floor(index / columnCount);
66+
const volumeChunks = 1 / (rowMidPoint + 1);
67+
const distanceToMid = Math.abs(rowMidPoint - y);
68+
const threshold = distanceToMid * volumeChunks;
69+
const isOn = volumeBands[index % columnCount] >= threshold;
70+
71+
return <Component className={cn(baseClassName, isOn ? onClassName : offClassName)} />;
72+
}
73+
74+
let transformerStyle: CSSProperties | undefined;
75+
if (transformer) {
76+
transformerStyle = transformer(index, rowCount, columnCount);
77+
}
78+
79+
const isOn =
80+
highlightedCoordinate.x === index % columnCount &&
81+
highlightedCoordinate.y === Math.floor(index / columnCount);
82+
83+
const transitionDurationInSeconds = interval / (isOn ? 1000 : 100);
84+
85+
return (
86+
<Component
87+
style={{
88+
transitionProperty: 'all',
89+
transitionDuration: `${transitionDurationInSeconds}s`,
90+
transitionTimingFunction: 'ease-out',
91+
...transformerStyle,
92+
}}
93+
className={cn(baseClassName, isOn ? onClassName : offClassName)}
94+
/>
95+
);
96+
}
97+
98+
export interface AudioGridVisualizerProps {
99+
state: AgentState;
100+
options: GridOptions;
101+
audioTrack?: LocalAudioTrack | RemoteAudioTrack | TrackReferenceOrPlaceholder;
102+
}
103+
104+
export function AudioGridVisualizer({ state, options, audioTrack }: AudioGridVisualizerProps) {
105+
const { radius, interval = 100, className, gridComponent } = options;
106+
const { columnCount, rowCount, items } = useGrid(options);
107+
const highlightedCoordinate = useGridAnimator(state, rowCount, columnCount, interval, radius);
108+
const volumeBands = useMultibandTrackVolume(audioTrack, {
109+
bands: columnCount,
110+
loPass: 100,
111+
hiPass: 200,
112+
});
113+
114+
const GridComponent = gridComponent || 'div';
115+
116+
return (
117+
<div
118+
className={cn('grid gap-1', className)}
119+
style={{ gridTemplateColumns: `repeat(${columnCount}, 1fr)` }}
120+
>
121+
{items.map((idx) => (
122+
<GridCell
123+
key={idx}
124+
index={idx}
125+
state={state}
126+
options={options}
127+
rowCount={rowCount}
128+
columnCount={columnCount}
129+
volumeBands={volumeBands}
130+
highlightedCoordinate={highlightedCoordinate}
131+
Component={GridComponent}
132+
/>
133+
))}
134+
</div>
135+
);
136+
}

0 commit comments

Comments
 (0)