Skip to content

Commit cb48bcd

Browse files
thomasyuill-livekitthomasyuill-livekit
committed
AudioShaderVisualizer
1 parent 082d433 commit cb48bcd

File tree

16 files changed

+2094
-637
lines changed

16 files changed

+2094
-637
lines changed

‎app/ui/_components.tsx‎

Lines changed: 124 additions & 69 deletions
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,10 @@ import {
3030
AudioRadialVisualizer,
3131
audioRadialVisualizerVariants,
3232
} from '@/components/livekit/audio-visualizer/audio-radial-visualizer/audio-radial-visualizer';
33-
import { AudioShaderVisualizer } from '@/components/livekit/audio-visualizer/audio-shader-visualizer/audio-shader-visualizer';
33+
import {
34+
AudioShaderVisualizer,
35+
audioShaderVisualizerVariants,
36+
} from '@/components/livekit/audio-visualizer/audio-shader-visualizer/audio-shader-visualizer';
3437
import { Button, buttonVariants } from '@/components/livekit/button';
3538
import { ChatEntry } from '@/components/livekit/chat-entry';
3639
import {
@@ -52,6 +55,9 @@ type audioBarVisualizerVariantsSizeType = VariantProps<typeof audioBarVisualizer
5255
type audioRadialVisualizerVariantsSizeType = VariantProps<
5356
typeof audioRadialVisualizerVariants
5457
>['size'];
58+
type audioShaderVisualizerVariantsSizeType = VariantProps<
59+
typeof audioShaderVisualizerVariants
60+
>['size'];
5561

5662
export function useMicrophone() {
5763
const { startSession } = useSession();
@@ -563,93 +569,142 @@ export const COMPONENTS = {
563569
},
564570

565571
AudioShaderVisualizer: () => {
566-
const [presetIndex, setPresetIndex] = useState(3);
567-
568-
// speed
569-
const [a, setA] = useState(50);
570-
// // color scale
571-
const [h, setH] = useState(0.1);
572-
// // color position
573-
const [i, setI] = useState(0.15);
574-
// blur
575-
const [f, setF] = useState(0.1);
572+
const { startSession, endSession } = useSession();
573+
const { localParticipant } = useLocalParticipant();
574+
576575
// shape
577-
const [g, setG] = useState(1.0);
576+
const [shape, setShape] = useState(1.0);
577+
// color scale
578+
const [colorScale, setColorScale] = useState(0.1);
579+
// color position
580+
const [colorPosition, setColorPosition] = useState(0.15);
581+
582+
const sizes = ['icon', 'sm', 'md', 'lg', 'xl'];
583+
const states = [
584+
'disconnected',
585+
'connecting',
586+
'initializing',
587+
'listening',
588+
'thinking',
589+
'speaking',
590+
] as AgentState[];
591+
592+
const [size, setSize] = useState<audioShaderVisualizerVariantsSizeType>('lg');
593+
const [state, setState] = useState<AgentState>(states[0]);
578594

579595
const {
580596
// state,
581597
audioTrack,
582598
} = useVoiceAssistant();
583599

584-
useMicrophone();
600+
useEffect(() => {
601+
if (state === 'speaking') {
602+
startSession();
603+
localParticipant.setMicrophoneEnabled(true, undefined);
604+
} else {
605+
endSession();
606+
localParticipant.setMicrophoneEnabled(false, undefined);
607+
}
608+
}, [startSession, endSession, state, localParticipant]);
585609

586610
const fields = [
587-
['speed', a, setA, 0, 250, 10],
588-
['color position', i, setI, 0, 1, 0.01],
589-
['color scale', h, setH, 0, 1, 0.01],
590-
['blur', f, setF, 0, 2, 0.01],
591-
['shape', g, setG, 1, 5, 1],
611+
['color position', colorPosition, setColorPosition, 0, 1, 0.01],
612+
['color scale', colorScale, setColorScale, 0, 1, 0.01],
592613
] as const;
593614

594615
return (
595616
<Container componentName="AudioShaderVisualizer">
596617
<StartAudio label="Start Audio" />
597618
<RoomAudioRenderer />
598-
<div className="grid grid-cols-2 gap-4">
619+
620+
<div className="flex gap-4">
621+
<div className="flex-1">
622+
<label className="font-mono text-xs uppercase" htmlFor="state">
623+
State
624+
</label>
625+
<Select value={state} onValueChange={(value) => setState(value as AgentState)}>
626+
<SelectTrigger id="state" className="w-full">
627+
<SelectValue placeholder="Select a state" />
628+
</SelectTrigger>
629+
<SelectContent>
630+
{states.map((state) => (
631+
<SelectItem key={state} value={state}>
632+
{state}
633+
</SelectItem>
634+
))}
635+
</SelectContent>
636+
</Select>
637+
</div>
638+
639+
<div className="flex-1">
640+
<label className="font-mono text-xs uppercase" htmlFor="size">
641+
Size
642+
</label>
643+
<Select
644+
value={size as string}
645+
onValueChange={(value) => setSize(value as audioShaderVisualizerVariantsSizeType)}
646+
>
647+
<SelectTrigger id="size" className="w-full">
648+
<SelectValue placeholder="Select a size" />
649+
</SelectTrigger>
650+
<SelectContent>
651+
{sizes.map((size) => (
652+
<SelectItem key={size} value={size as string}>
653+
{size.toUpperCase()}
654+
</SelectItem>
655+
))}
656+
</SelectContent>
657+
</Select>
658+
</div>
659+
660+
<div className="flex-1">
661+
<label className="font-mono text-xs uppercase" htmlFor="shape">
662+
Shape
663+
</label>
664+
<Select value={shape.toString()} onValueChange={(value) => setShape(parseInt(value))}>
665+
<SelectTrigger id="shape" className="w-full">
666+
<SelectValue placeholder="Select a shape" />
667+
</SelectTrigger>
668+
<SelectContent>
669+
<SelectItem value="1">Circle</SelectItem>
670+
<SelectItem value="2">Line</SelectItem>
671+
</SelectContent>
672+
</Select>
673+
</div>
674+
</div>
675+
676+
<div className="py-12">
599677
<AudioShaderVisualizer
600-
speed={a}
601-
blur={f}
602-
shape={g}
603-
colorScale={h}
604-
colorPosition={i}
605-
audioTrack={audioTrack}
606-
presetIndex={presetIndex}
607-
// className="bg-amber-100"
678+
size={size}
679+
state={state}
680+
shape={shape}
681+
colorScale={colorScale}
682+
colorPosition={colorPosition}
683+
audioTrack={audioTrack as TrackReferenceOrPlaceholder}
684+
className="mx-auto bg-black"
608685
/>
609-
<div>
610-
<div className="mb-4">
611-
<StoryTitle>Preset</StoryTitle>
612-
<Select
613-
value={String(presetIndex)}
614-
onValueChange={(value) => setPresetIndex(parseInt(value))}
615-
>
616-
<SelectTrigger id="presetIndex" className="w-full">
617-
<SelectValue placeholder="Select a preset" />
618-
</SelectTrigger>
619-
<SelectContent>
620-
<SelectItem value="0">Preset 1</SelectItem>
621-
<SelectItem value="1">Preset 2</SelectItem>
622-
<SelectItem value="2">Preset 3</SelectItem>
623-
<SelectItem value="3">Preset 4</SelectItem>
624-
</SelectContent>
625-
</Select>
626-
</div>
686+
</div>
627687

628-
{fields.map(([name, value, setValue, min = 0.1, max = 10, step = 0.1]) => {
629-
// Use 0-1 range for color phase channels
630-
const isColorPhase = name.toString().startsWith('colorPhase');
631-
632-
return (
633-
<div key={name}>
634-
<div className="flex items-center justify-between">
635-
<StoryTitle>{name}</StoryTitle>
636-
<div className="text-muted-foreground mb-2 text-xs">
637-
{isColorPhase ? Number(value).toFixed(2) : String(value)}
638-
</div>
639-
</div>
640-
<input
641-
type="range"
642-
value={String(value)}
643-
min={min}
644-
max={max}
645-
step={step}
646-
onChange={(e) => setValue(parseFloat(e.target.value))}
647-
className="w-full"
648-
/>
688+
<div className="grid grid-cols-2 gap-4">
689+
{fields.map(([name, value, setValue, min = 0.1, max = 10, step = 0.1]) => {
690+
return (
691+
<div key={name}>
692+
<div className="flex items-center justify-between">
693+
<StoryTitle>{name}</StoryTitle>
694+
<div className="text-muted-foreground mb-2 text-xs">{String(value)}</div>
649695
</div>
650-
);
651-
})}
652-
</div>
696+
<input
697+
type="range"
698+
value={String(value)}
699+
min={min}
700+
max={max}
701+
step={step}
702+
onChange={(e) => setValue(parseFloat(e.target.value))}
703+
className="w-full"
704+
/>
705+
</div>
706+
);
707+
})}
653708
</div>
654709
</Container>
655710
);

‎components/app/tile-layout.tsx‎

Lines changed: 11 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,8 @@ import {
88
useTracks,
99
useVoiceAssistant,
1010
} from '@livekit/components-react';
11-
import { AudioBarVisualizer } from '@/components/livekit/audio-visualizer/audio-bar-visualizer/audio-bar-visualizer';
11+
// import { AudioBarVisualizer } from '@/components/livekit/audio-visualizer/audio-bar-visualizer/audio-bar-visualizer';
12+
import { AudioShaderVisualizer } from '@/components/livekit/audio-visualizer/audio-shader-visualizer/audio-shader-visualizer';
1213
import { cn } from '@/lib/utils';
1314

1415
const MotionContainer = motion.create('div');
@@ -92,7 +93,7 @@ export function TileLayout({ chatOpen }: TileLayoutProps) {
9293
const videoHeight = agentVideoTrack?.publication.dimensions?.height ?? 0;
9394

9495
return (
95-
<div className="pointer-events-none fixed inset-x-0 top-8 bottom-32 z-50 md:top-12 md:bottom-40">
96+
<div className="fixed inset-x-0 top-8 bottom-32 z-50 md:top-12 md:bottom-40">
9697
<div className="relative mx-auto h-full max-w-2xl px-4 md:px-0">
9798
<div className={cn(classNames.grid)}>
9899
{/* Agent */}
@@ -112,7 +113,7 @@ export function TileLayout({ chatOpen }: TileLayoutProps) {
112113
layoutId="agent"
113114
initial={{
114115
opacity: 0,
115-
scale: 0,
116+
scale: chatOpen ? 1 : 6,
116117
}}
117118
animate={{
118119
opacity: 1,
@@ -127,11 +128,17 @@ export function TileLayout({ chatOpen }: TileLayoutProps) {
127128
chatOpen && 'border-input/50 drop-shadow-lg/10 delay-200'
128129
)}
129130
>
130-
<AudioBarVisualizer
131+
{/* <AudioBarVisualizer
131132
size="sm"
133+
barCount={5}
132134
state={agentState}
133135
audioTrack={agentAudioTrack!}
134136
className="mx-auto"
137+
/> */}
138+
<AudioShaderVisualizer
139+
size="sm"
140+
state={agentState}
141+
audioTrack={agentAudioTrack!}
135142
/>
136143
</MotionContainer>
137144
)}

‎components/livekit/audio-visualizer/audio-bar-visualizer/audio-bar-visualizer.tsx‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -84,7 +84,7 @@ export function AudioBarVisualizer({
8484
case 'connecting':
8585
return 2000 / _barCount;
8686
case 'initializing':
87-
return 500;
87+
return 2000;
8888
case 'listening':
8989
return 500;
9090
case 'thinking':

0 commit comments

Comments
 (0)