diff --git a/app/ui/(landing-page)/page.tsx b/app/ui/(landing-page)/page.tsx new file mode 100644 index 00000000..384bf396 --- /dev/null +++ b/app/ui/(landing-page)/page.tsx @@ -0,0 +1,102 @@ +'use client'; + +import Link from 'next/link'; +import { useVoiceAssistant } from '@livekit/components-react'; +import { AgentControlBar } from '@/components/livekit/agent-control-bar/agent-control-bar'; +import { AudioBarVisualizer } from '@/components/livekit/audio-visualizer/audio-bar-visualizer/audio-bar-visualizer'; +import { Button } from '@/components/livekit/button'; +import { ChatEntry } from '@/components/livekit/chat-entry'; +import { useMicrophone } from '../_components'; + +export default function Page() { + const { state, audioTrack } = useVoiceAssistant(); + + useMicrophone(); + + return ( + <> +
+

+ + + + UI +

+

+ A set of Open Source UI components for +
+ building beautiful voice experiences. +

+
+ + +
+
+ +
+
+
+
+
+ +
+ +
+
+
+
+
+ + +
+ +
+
+
+
+ + ); +} diff --git a/app/ui/README.md b/app/ui/README.md new file mode 100644 index 00000000..472efb95 --- /dev/null +++ b/app/ui/README.md @@ -0,0 +1,13 @@ +THIS IS NOT PART OF THE MAIN APPLICATION CODE. + +This folder contains code for testing and previewing LiveKit's UI component library in isolation. + +## Getting started + +To run the development server, run the following command: + +```bash +npm run dev +``` + +Then, navigate to `http://localhost:3000/ui` to see the components. diff --git a/app/ui/_components.tsx b/app/ui/_components.tsx new file mode 100644 index 00000000..6f7f03c3 --- /dev/null +++ b/app/ui/_components.tsx @@ -0,0 +1,932 @@ +'use client'; + +import { useEffect, useMemo, useState } from 'react'; +import { type VariantProps } from 'class-variance-authority'; +import { Track } from 'livekit-client'; +// import { RoomAudioRenderer, StartAudio } from '@livekit/components-react'; +import { + type AgentState, + type TrackReference, + type TrackReferenceOrPlaceholder, + useLocalParticipant, + // useVoiceAssistant, +} from '@livekit/components-react'; +import { MicrophoneIcon } from '@phosphor-icons/react/dist/ssr'; +import { useSession } from '@/components/app/session-provider'; +import { AgentControlBar } from '@/components/livekit/agent-control-bar/agent-control-bar'; +import { TrackControl } from '@/components/livekit/agent-control-bar/track-control'; +// import { TrackDeviceSelect } from '@/components/livekit/agent-control-bar/track-device-select'; +// import { TrackToggle } from '@/components/livekit/agent-control-bar/track-toggle'; +import { Alert, AlertDescription, AlertTitle, alertVariants } from '@/components/livekit/alert'; +import { AlertToast } from '@/components/livekit/alert-toast'; +import { BarVisualizer } from '@/components/livekit/audio-visualizer/audio-bar-visualizer/_bar-visualizer'; +import { + AudioBarVisualizer, + audioBarVisualizerVariants, +} from '@/components/livekit/audio-visualizer/audio-bar-visualizer/audio-bar-visualizer'; +import { AudioGridVisualizer } from '@/components/livekit/audio-visualizer/audio-grid-visualizer/audio-grid-visualizer'; +import { gridVariants } from '@/components/livekit/audio-visualizer/audio-grid-visualizer/demos'; +import { AudioOscilloscopeVisualizer } from '@/components/livekit/audio-visualizer/audio-oscilloscope-visualizer/audio-oscilloscope-visualizer'; +import { + AudioRadialVisualizer, + audioRadialVisualizerVariants, +} from '@/components/livekit/audio-visualizer/audio-radial-visualizer/audio-radial-visualizer'; +import { + AudioShaderVisualizer, + audioShaderVisualizerVariants, +} from '@/components/livekit/audio-visualizer/audio-shader-visualizer/audio-shader-visualizer'; +import { Button, buttonVariants } from '@/components/livekit/button'; +import { ChatEntry } from '@/components/livekit/chat-entry'; +import { + Select, + SelectContent, + SelectItem, + SelectTrigger, + SelectValue, +} from '@/components/livekit/select'; +import { ShimmerText } from '@/components/livekit/shimmer-text'; +import { Toggle, toggleVariants } from '@/components/livekit/toggle'; + +type toggleVariantsType = VariantProps['variant']; +type toggleVariantsSizeType = VariantProps['size']; +type buttonVariantsType = VariantProps['variant']; +type buttonVariantsSizeType = VariantProps['size']; +type alertVariantsType = VariantProps['variant']; +type audioBarVisualizerVariantsSizeType = VariantProps['size']; +type audioRadialVisualizerVariantsSizeType = VariantProps< + typeof audioRadialVisualizerVariants +>['size']; +type audioShaderVisualizerVariantsSizeType = VariantProps< + typeof audioShaderVisualizerVariants +>['size']; + +export function useMicrophone() { + const { startSession } = useSession(); + const { localParticipant } = useLocalParticipant(); + + useEffect(() => { + startSession(); + localParticipant.setMicrophoneEnabled(true, undefined); + }, [startSession, localParticipant]); +} + +interface ContainerProps { + componentName: string; + children: React.ReactNode; + className?: string; +} + +function Container({ children, className }: ContainerProps) { + return ( +
+
+ {children} +
+
+ ); +} + +function StoryTitle({ children }: { children: React.ReactNode }) { + return

{children}

; +} + +export const COMPONENTS = { + // Button + Button: () => ( + + + + + + + + + + + + + {['default', 'primary', 'secondary', 'outline', 'ghost', 'link', 'destructive'].map( + (variant) => ( + + + {['sm', 'default', 'lg', 'icon'].map((size) => ( + + ))} + + ) + )} + +
SmallDefaultLargeIcon
{variant} + +
+ + ), + + // Toggle + Toggle: () => ( + + + + + + + + + + + + + {['default', 'primary', 'secondary', 'outline'].map((variant) => ( + + + {['sm', 'default', 'lg', 'icon'].map((size) => ( + + ))} + + ))} + +
SmallDefaultLargeIcon
{variant} + + {size === 'icon' ? : 'Toggle'} + +
+ + ), + + // Alert + Alert: () => ( + + {['default', 'destructive'].map((variant) => ( +
+ {variant} + + Alert {variant} title + This is a {variant} alert description. + +
+ ))} + + ), + + // Select + Select: () => ( + +
+
+ Size default + +
+
+ Size sm + +
+
+ + ), + + // Audio bar visualizer + AudioBarVisualizer: () => { + const barCounts = ['0', '3', '5', '7', '9']; + const sizes = ['icon', 'sm', 'md', 'lg', 'xl']; + const states = [ + 'disconnected', + 'connecting', + 'initializing', + 'listening', + 'thinking', + 'speaking', + ] as AgentState[]; + + const { microphoneTrack, localParticipant } = useLocalParticipant(); + const [barCount, setBarCount] = useState(barCounts[0]); + const [size, setSize] = useState( + 'md' as audioBarVisualizerVariantsSizeType + ); + const [state, setState] = useState(states[0]); + + const micTrackRef = useMemo(() => { + return state === 'speaking' + ? ({ + participant: localParticipant, + source: Track.Source.Microphone, + publication: microphoneTrack, + } as TrackReference) + : undefined; + }, [state, localParticipant, microphoneTrack]); + + useMicrophone(); + + return ( + +
+
+ + +
+ +
+ + +
+
+ +
+
+ +
+
+ + Original BarVisualizer + +
+ +
+
+
+ +
+ {states.map((stateType) => ( + + ))} +
+ + ); + }, + + // Audio bar visualizer + AudioRadialVisualizer: () => { + const barCounts = ['0', '4', '8', '12', '16', '24']; + const sizes = ['icon', 'sm', 'md', 'lg', 'xl']; + const states = [ + 'disconnected', + 'connecting', + 'initializing', + 'listening', + 'thinking', + 'speaking', + ] as AgentState[]; + + const { microphoneTrack, localParticipant } = useLocalParticipant(); + const [barCount, setBarCount] = useState(barCounts[0]); + const [size, setSize] = useState( + 'md' as audioRadialVisualizerVariantsSizeType + ); + const [state, setState] = useState(states[0]); + + const micTrackRef = useMemo(() => { + return state === 'speaking' + ? ({ + participant: localParticipant, + source: Track.Source.Microphone, + publication: microphoneTrack, + } as TrackReference) + : undefined; + }, [state, localParticipant, microphoneTrack]); + + useMicrophone(); + + return ( + +
+
+ + +
+ +
+ + +
+
+ +
+
+ +
+
+ +
+ {states.map((stateType) => ( + + ))} +
+ + ); + }, + + // Audio bar visualizer + AudioGridVisualizer: () => { + const rowCounts = ['3', '5', '7', '9', '11', '13', '15']; + const columnCounts = ['3', '5', '7', '9', '11', '13', '15']; + const states = [ + 'disconnected', + 'connecting', + 'initializing', + 'listening', + 'thinking', + 'speaking', + ] as AgentState[]; + + const [rowCount, setRowCount] = useState(rowCounts[0]); + const [columnCount, setColumnCount] = useState(columnCounts[0]); + const [state, setState] = useState(states[0]); + const [demoIndex, setDemoIndex] = useState(0); + const { microphoneTrack, localParticipant } = useLocalParticipant(); + + const micTrackRef = useMemo(() => { + return state === 'speaking' + ? ({ + participant: localParticipant, + source: Track.Source.Microphone, + publication: microphoneTrack, + } as TrackReference) + : undefined; + }, [state, localParticipant, microphoneTrack]); + + useMicrophone(); + + const demoOptions = { + rowCount: parseInt(rowCount), + columnCount: parseInt(columnCount), + ...gridVariants[demoIndex], + }; + + return ( + +
+
+ + +
+ +
+ + +
+ +
+ + +
+
+ +
+ +
+ +
+ {states.map((stateType) => ( + + ))} +
+ +
+ Demo options +
+ 
+              {JSON.stringify(demoOptions, null, 2)}
+
+
+
+ + ); + }, + + AudioShaderVisualizer: () => { + // shape + const [shape, setShape] = useState(1.0); + // color scale + const [colorScale, setColorScale] = useState(0.1); + // color position + const [colorPosition, setColorPosition] = useState(0.15); + + const sizes = ['icon', 'sm', 'md', 'lg', 'xl']; + const states = [ + 'disconnected', + 'connecting', + 'initializing', + 'listening', + 'thinking', + 'speaking', + ] as AgentState[]; + + const [size, setSize] = useState('lg'); + const [state, setState] = useState(states[0]); + + const { microphoneTrack, localParticipant } = useLocalParticipant(); + const micTrackRef = useMemo(() => { + return state === 'speaking' + ? ({ + participant: localParticipant, + source: Track.Source.Microphone, + publication: microphoneTrack, + } as TrackReference) + : undefined; + }, [state, localParticipant, microphoneTrack]); + + useMicrophone(); + + const fields = [ + ['color position', colorPosition, setColorPosition, 0, 1, 0.01], + ['color scale', colorScale, setColorScale, 0, 1, 0.01], + ] as const; + + return ( + +
+
+ + +
+ +
+ + +
+
+ +
+ +
+ +
+ {states.map((stateType) => ( + + ))} +
+ +
+ {fields.map(([name, value, setValue, min = 0.1, max = 10, step = 0.1]) => { + return ( +
+
+ {name} +
{String(value)}
+
+ setValue(parseFloat(e.target.value))} + className="w-full" + /> +
+ ); + })} +
+ + ); + }, + + AudioOscilloscopeVisualizer: () => { + // shape + const [shape, setShape] = useState(1.0); + + const sizes = ['icon', 'sm', 'md', 'lg', 'xl']; + const states = [ + 'disconnected', + 'connecting', + 'initializing', + 'listening', + 'thinking', + 'speaking', + ] as AgentState[]; + + const [size, setSize] = useState('lg'); + const [state, setState] = useState(states[0]); + + const { microphoneTrack, localParticipant } = useLocalParticipant(); + const micTrackRef = useMemo(() => { + return state === 'speaking' + ? ({ + participant: localParticipant, + source: Track.Source.Microphone, + publication: microphoneTrack, + } as TrackReference) + : undefined; + }, [state, localParticipant, microphoneTrack]); + + useMicrophone(); + + return ( + +
+
+ + +
+ +
+ + +
+
+ +
+ +
+ +
+ {states.map((stateType) => ( + + ))} +
+ + ); + }, + + // Agent control bar + AgentControlBar: () => { + useMicrophone(); + + return ( + +
+ +
+ + ); + }, + + // Track device select + // TrackDeviceSelect: () => ( + // + //
+ //
+ // Size default + // + //
+ //
+ // Size sm + // + //
+ //
+ // + // ), + + // Track toggle + // TrackToggle: () => ( + // + //
+ //
+ // Track.Source.Microphone + // + //
+ //
+ // Track.Source.Camera + // + //
+ //
+ // + // ), + + // Track control + TrackControl: () => { + const { microphoneTrack, localParticipant } = useLocalParticipant(); + const micTrackRef = useMemo(() => { + return { + participant: localParticipant, + source: Track.Source.Microphone, + publication: microphoneTrack, + } as TrackReference; + }, [localParticipant, microphoneTrack]); + + useMicrophone(); + + return ( + +
+
+
+ Track.Source.Microphone + +
+
+ Track.Source.Microphone + +
+
+ +
+ Track.Source.Camera + +
+
+ + ); + }, + + // Chat entry + ChatEntry: () => ( + +
+ + +
+ + ), + + // Shimmer text + ShimmerText: () => ( + +
+ This is shimmer text +
+ + ), + + // Alert toast + AlertToast: () => ( + + Alert toast +
+ +
+ + ), +}; diff --git a/app/ui/components/[...slug]/page.tsx b/app/ui/components/[...slug]/page.tsx new file mode 100644 index 00000000..6300a5e8 --- /dev/null +++ b/app/ui/components/[...slug]/page.tsx @@ -0,0 +1,21 @@ +'use client'; + +import { redirect, useParams } from 'next/navigation'; +import { COMPONENTS } from '../../_components'; + +export default function Page() { + const { slug = [] } = useParams(); + const [componentName] = slug; + const component = COMPONENTS[componentName as keyof typeof COMPONENTS]; + + if (!component) { + return redirect('/ui'); + } + + return ( + <> +

{componentName}

+ {component()} + + ); +} diff --git a/app/ui/components/layout.tsx b/app/ui/components/layout.tsx new file mode 100644 index 00000000..56e79f5d --- /dev/null +++ b/app/ui/components/layout.tsx @@ -0,0 +1,51 @@ +'use client'; + +import Link from 'next/link'; +import { usePathname } from 'next/navigation'; +import { cn } from '@/lib/utils'; +import { COMPONENTS } from '../_components'; + +export default function Layout({ children }: { children: React.ReactNode }) { + const pathname = usePathname(); + const isActive = (path: string) => pathname === path; + + return ( +
+ + +
+
{children}
+
+ + +
+ ); +} diff --git a/app/ui/components/page.tsx b/app/ui/components/page.tsx new file mode 100644 index 00000000..b0cce846 --- /dev/null +++ b/app/ui/components/page.tsx @@ -0,0 +1,40 @@ +'use client'; + +import Link from 'next/link'; +import { COMPONENTS } from '../_components'; + +export default function Page() { + return ( + <> +

+ Components +

+

+ Build beautiful voice experiences with our components. +

+ +
+ {Object.entries(COMPONENTS) + .sort(([a], [b]) => a.localeCompare(b)) + .map(([componentName]) => ( + + {componentName} + + ))} +
+ +
+ {Object.entries(COMPONENTS).map(([componentName, component]) => ( +
+

{componentName}

+ {component()} +
+ ))} +
+ + ); +} diff --git a/app/ui/layout.tsx b/app/ui/layout.tsx index c7202785..4205c28b 100644 --- a/app/ui/layout.tsx +++ b/app/ui/layout.tsx @@ -1,41 +1,69 @@ -import * as React from 'react'; import { headers } from 'next/headers'; +import Link from 'next/link'; import { SessionProvider } from '@/components/app/session-provider'; import { getAppConfig } from '@/lib/utils'; -export default async function ComponentsLayout({ children }: { children: React.ReactNode }) { +export default async function Layout({ children }: { children: React.ReactNode }) { const hdrs = await headers(); const appConfig = await getAppConfig(hdrs); return ( -
-
-
-

LiveKit UI

-

- A set of UI components for building LiveKit-powered voice experiences. -

-

- Built with{' '} - - Shadcn - - ,{' '} - - Motion - - , and{' '} - - LiveKit - - . -

-

Open Source.

+
+
+
+ + + + + UI + + + Docs + + + Components +
-
{children}
+ {children}
+ +
); diff --git a/app/ui/page.tsx b/app/ui/page.tsx deleted file mode 100644 index 83e1a7ba..00000000 --- a/app/ui/page.tsx +++ /dev/null @@ -1,268 +0,0 @@ -import { type VariantProps } from 'class-variance-authority'; -import { Track } from 'livekit-client'; -import { MicrophoneIcon } from '@phosphor-icons/react/dist/ssr'; -import { AgentControlBar } from '@/components/livekit/agent-control-bar/agent-control-bar'; -import { TrackDeviceSelect } from '@/components/livekit/agent-control-bar/track-device-select'; -import { TrackSelector } from '@/components/livekit/agent-control-bar/track-selector'; -import { TrackToggle } from '@/components/livekit/agent-control-bar/track-toggle'; -import { Alert, AlertDescription, AlertTitle, alertVariants } from '@/components/livekit/alert'; -import { AlertToast } from '@/components/livekit/alert-toast'; -import { Button, buttonVariants } from '@/components/livekit/button'; -import { ChatEntry } from '@/components/livekit/chat-entry'; -import { - Select, - SelectContent, - SelectItem, - SelectTrigger, - SelectValue, -} from '@/components/livekit/select'; -import { ShimmerText } from '@/components/livekit/shimmer-text'; -import { Toggle, toggleVariants } from '@/components/livekit/toggle'; -import { cn } from '@/lib/utils'; - -type toggleVariantsType = VariantProps['variant']; -type toggleVariantsSizeType = VariantProps['size']; -type buttonVariantsType = VariantProps['variant']; -type buttonVariantsSizeType = VariantProps['size']; -type alertVariantsType = VariantProps['variant']; - -interface ContainerProps { - componentName?: string; - children: React.ReactNode; - className?: string; -} - -function Container({ componentName, children, className }: ContainerProps) { - return ( -
-

- {componentName} -

-
- {children} -
-
- ); -} - -function StoryTitle({ children }: { children: React.ReactNode }) { - return

{children}

; -} - -export default function Base() { - return ( - <> -

Primitives

- - {/* Button */} - - - - - - - - - - - - - {['default', 'primary', 'secondary', 'outline', 'ghost', 'link', 'destructive'].map( - (variant) => ( - - - {['sm', 'default', 'lg', 'icon'].map((size) => ( - - ))} - - ) - )} - -
SmallDefaultLargeIcon
{variant} - -
- - - {/* Toggle */} - - - - - - - - - - - - - {['default', 'primary', 'secondary', 'outline'].map((variant) => ( - - - {['sm', 'default', 'lg', 'icon'].map((size) => ( - - ))} - - ))} - -
SmallDefaultLargeIcon
{variant} - - {size === 'icon' ? : 'Toggle'} - -
- - - {/* Alert */} - - {['default', 'destructive'].map((variant) => ( -
- {variant} - - Alert {variant} title - This is a {variant} alert description. - -
- ))} - - - {/* Select */} - -
-
- Size default - -
-
- Size sm - -
-
- - -

Components

- - {/* Agent control bar */} - -
- -
- - - {/* Track device select */} - -
-
- Size default - -
-
- Size sm - -
-
- - - {/* Track toggle */} - -
-
- Track.Source.Microphone - -
-
- Track.Source.Camera - -
-
- - - {/* Track selector */} - -
-
- Track.Source.Camera - -
-
- Track.Source.Microphone - -
-
- - - {/* Chat entry */} - -
- - -
- - - {/* Shimmer text */} - -
- This is shimmer text -
- - - {/* Alert toast */} - - Alert toast -
- -
- - - ); -} diff --git a/components/app/tile-layout.tsx b/components/app/tile-layout.tsx index 33372276..38a7e481 100644 --- a/components/app/tile-layout.tsx +++ b/components/app/tile-layout.tsx @@ -2,13 +2,14 @@ import React, { useMemo } from 'react'; import { Track } from 'livekit-client'; import { AnimatePresence, motion } from 'motion/react'; import { - BarVisualizer, type TrackReference, VideoTrack, useLocalParticipant, useTracks, useVoiceAssistant, } from '@livekit/components-react'; +// import { AudioBarVisualizer } from '@/components/livekit/audio-visualizer/audio-bar-visualizer/audio-bar-visualizer'; +import { AudioShaderVisualizer } from '@/components/livekit/audio-visualizer/audio-shader-visualizer/audio-shader-visualizer'; import { cn } from '@/lib/utils'; const MotionContainer = motion.create('div'); @@ -92,7 +93,7 @@ export function TileLayout({ chatOpen }: TileLayoutProps) { const videoHeight = agentVideoTrack?.publication.dimensions?.height ?? 0; return ( -
+
{/* Agent */} @@ -112,36 +113,33 @@ export function TileLayout({ chatOpen }: TileLayoutProps) { layoutId="agent" initial={{ opacity: 0, - scale: 0, + scale: chatOpen ? 1 : 6, }} animate={{ opacity: 1, - scale: chatOpen ? 1 : 5, + scale: chatOpen ? 1 : 6, }} transition={{ ...ANIMATION_TRANSITION, delay: animationDelay, }} className={cn( - 'bg-background aspect-square h-[90px] rounded-md border border-transparent transition-[border,drop-shadow]', + 'bg-background flex aspect-square h-[90px] items-center justify-center rounded-md border border-transparent transition-[border,drop-shadow]', chatOpen && 'border-input/50 drop-shadow-lg/10 delay-200' )} > - - - + audioTrack={agentAudioTrack!} + className="mx-auto" + /> */} + )} diff --git a/components/livekit/agent-control-bar/agent-control-bar.tsx b/components/livekit/agent-control-bar/agent-control-bar.tsx index 1b53b1c4..f7b2787d 100644 --- a/components/livekit/agent-control-bar/agent-control-bar.tsx +++ b/components/livekit/agent-control-bar/agent-control-bar.tsx @@ -12,7 +12,7 @@ import { cn } from '@/lib/utils'; import { ChatInput } from './chat-input'; import { UseInputControlsProps, useInputControls } from './hooks/use-input-controls'; import { usePublishPermissions } from './hooks/use-publish-permissions'; -import { TrackSelector } from './track-selector'; +import { TrackControl } from './track-control'; export interface ControlBarControls { leave?: boolean; @@ -107,7 +107,7 @@ export function AgentControlBar({
{/* Toggle Microphone */} {visibleControls.microphone && ( - [0]['source']; pressed?: boolean; @@ -22,7 +19,7 @@ interface TrackSelectorProps { onActiveDeviceChange?: (deviceId: string) => void; } -export function TrackSelector({ +export function TrackControl({ kind, source, pressed, @@ -33,7 +30,7 @@ export function TrackSelector({ onPressedChange, onMediaDeviceError, onActiveDeviceChange, -}: TrackSelectorProps) { +}: TrackControlProps) { return (
{audioTrackRef && ( - - - + )}
diff --git a/components/livekit/audio-visualizer/audio-bar-visualizer/_bar-visualizer.tsx b/components/livekit/audio-visualizer/audio-bar-visualizer/_bar-visualizer.tsx new file mode 100644 index 00000000..aa3ab23b --- /dev/null +++ b/components/livekit/audio-visualizer/audio-bar-visualizer/_bar-visualizer.tsx @@ -0,0 +1,99 @@ +import { useEffect, useMemo, useRef } from 'react'; +import { type VariantProps, cva } from 'class-variance-authority'; +import { + type AgentState, + BarVisualizer as LiveKitBarVisualizer, + type TrackReferenceOrPlaceholder, +} from '@livekit/components-react'; +import { cn } from '@/lib/utils'; + +const MIN_HEIGHT = 15; // 15% + +export const barVisualizerVariants = cva( + ['relative flex aspect-square h-36 items-center justify-center'], + { + variants: { + size: { + default: 'h-32', + icon: 'h-6', + xs: 'h-8', + sm: 'h-16', + md: 'h-32', + lg: 'h-64', + xl: 'h-96', + '2xl': 'h-128', + }, + }, + defaultVariants: { + size: 'default', + }, + } +); + +interface BarVisualizerProps { + state?: AgentState; + barCount?: number; + audioTrack?: TrackReferenceOrPlaceholder; + className?: string; +} + +export function BarVisualizer({ + size, + state, + barCount, + audioTrack, + className, +}: BarVisualizerProps & VariantProps) { + const ref = useRef(null); + const _barCount = useMemo(() => { + if (barCount) { + return barCount; + } + switch (size) { + case 'icon': + case 'xs': + return 3; + default: + return 5; + } + }, [barCount, size]); + + const x = (1 / (_barCount + (_barCount + 1) / 2)) * 100; + + // reset bars height when audio track is disconnected + useEffect(() => { + if (ref.current && !audioTrack) { + const bars = [...(ref.current.querySelectorAll('& > span') ?? [])] as HTMLElement[]; + + bars.forEach((bar) => { + bar.style.height = `${MIN_HEIGHT}%`; + }); + } + }, [audioTrack]); + + return ( + + + + ); +} diff --git a/components/livekit/audio-visualizer/audio-bar-visualizer/audio-bar-visualizer.tsx b/components/livekit/audio-visualizer/audio-bar-visualizer/audio-bar-visualizer.tsx new file mode 100644 index 00000000..b7657097 --- /dev/null +++ b/components/livekit/audio-visualizer/audio-bar-visualizer/audio-bar-visualizer.tsx @@ -0,0 +1,113 @@ +import { useMemo } from 'react'; +import { type VariantProps, cva } from 'class-variance-authority'; +import { type LocalAudioTrack, type RemoteAudioTrack } from 'livekit-client'; +import { + type AgentState, + type TrackReferenceOrPlaceholder, + useMultibandTrackVolume, +} from '@livekit/components-react'; +import { cn } from '@/lib/utils'; +import { useBarAnimator } from './hooks/useBarAnimator'; + +export const audioBarVisualizerVariants = cva(['relative flex items-center justify-center'], { + variants: { + size: { + icon: 'h-[24px] gap-[2px]', + sm: 'h-[56px] gap-[4px]', + md: 'h-[112px] gap-[8px]', + lg: 'h-[224px] gap-[16px]', + xl: 'h-[448px] gap-[32px]', + }, + }, + defaultVariants: { + size: 'md', + }, +}); + +export const audioBarVisualizerBarVariants = cva( + [ + 'rounded-full transition-colors duration-250 ease-linear bg-(--audio-visualizer-idle) data-[lk-highlighted=true]:bg-(--audio-visualizer-active)', + ], + { + variants: { + size: { + icon: 'w-[4px] min-h-[4px]', + sm: 'w-[8px] min-h-[8px]', + md: 'w-[16px] min-h-[16px]', + lg: 'w-[32px] min-h-[32px]', + xl: 'w-[64px] min-h-[64px]', + }, + }, + defaultVariants: { + size: 'md', + }, + } +); + +interface AudioBarVisualizerProps { + state?: AgentState; + barCount?: number; + audioTrack?: LocalAudioTrack | RemoteAudioTrack | TrackReferenceOrPlaceholder; + className?: string; + barClassName?: string; +} + +export function AudioBarVisualizer({ + size, + state, + barCount, + audioTrack, + className, + barClassName, +}: AudioBarVisualizerProps & VariantProps) { + const _barCount = useMemo(() => { + if (barCount) { + return barCount; + } + switch (size) { + case 'icon': + case 'sm': + return 3; + default: + return 5; + } + }, [barCount, size]); + + const volumeBands = useMultibandTrackVolume(audioTrack, { + bands: _barCount, + loPass: 100, + hiPass: 200, + }); + + const sequencerInterval = useMemo(() => { + switch (state) { + case 'connecting': + return 2000 / _barCount; + case 'initializing': + return 2000; + case 'listening': + return 500; + case 'thinking': + return 150; + default: + return 1000; + } + }, [state, _barCount]); + + const highlightedIndices = useBarAnimator(state, _barCount, sequencerInterval); + + const bands = audioTrack ? volumeBands : new Array(_barCount).fill(0); + return ( +
+ {bands.map((band, idx) => ( +
+ ))} +
+ ); +} diff --git a/components/livekit/audio-visualizer/audio-bar-visualizer/hooks/useBarAnimator.ts b/components/livekit/audio-visualizer/audio-bar-visualizer/hooks/useBarAnimator.ts new file mode 100644 index 00000000..070bfa0b --- /dev/null +++ b/components/livekit/audio-visualizer/audio-bar-visualizer/hooks/useBarAnimator.ts @@ -0,0 +1,84 @@ +import { useEffect, useRef, useState } from 'react'; +import { type AgentState } from '@livekit/components-react'; + +function generateConnectingSequenceBar(columns: number): number[][] { + const seq = []; + + for (let x = 0; x < columns; x++) { + seq.push([x, columns - 1 - x]); + } + + return seq; +} + +function generateListeningSequenceBar(columns: number): number[][] { + const center = Math.floor(columns / 2); + const noIndex = -1; + + return [[center], [noIndex]]; +} + +// function generateThinkingSequenceBar(columns: number): number[][] { +// const seq = []; +// for (let x = 0; x < columns; x++) { +// seq.push([x]); +// } + +// for (let x = columns - 1; x >= 0; x--) { +// seq.push([x]); +// } + +// return seq; +// } + +export const useBarAnimator = ( + state: AgentState | undefined, + columns: number, + interval: number +): number[] => { + const [index, setIndex] = useState(0); + const [sequence, setSequence] = useState([[]]); + + useEffect(() => { + if (state === 'thinking') { + setSequence(generateListeningSequenceBar(columns)); + // setSequence(generateThinkingSequenceBar(columns)); + } else if (state === 'connecting' || state === 'initializing') { + const sequence = [...generateConnectingSequenceBar(columns)]; + setSequence(sequence); + } else if (state === 'listening') { + setSequence(generateListeningSequenceBar(columns)); + } else if (state === undefined || state === 'speaking') { + setSequence([new Array(columns).fill(0).map((_, idx) => idx)]); + } else { + setSequence([[]]); + } + setIndex(0); + }, [state, columns]); + + const animationFrameId = useRef(null); + useEffect(() => { + let startTime = performance.now(); + + const animate = (time: DOMHighResTimeStamp) => { + const timeElapsed = time - startTime; + + if (timeElapsed >= interval) { + setIndex((prev) => prev + 1); + startTime = time; + } + + animationFrameId.current = requestAnimationFrame(animate); + }; + + animationFrameId.current = requestAnimationFrame(animate); + + return () => { + if (animationFrameId.current !== null) { + cancelAnimationFrame(animationFrameId.current); + } + }; + }, [interval, columns, state, sequence.length]); + + return sequence[index % sequence.length]; +}; diff --git a/components/livekit/audio-visualizer/audio-grid-visualizer/audio-grid-visualizer.tsx b/components/livekit/audio-visualizer/audio-grid-visualizer/audio-grid-visualizer.tsx new file mode 100644 index 00000000..3d412f26 --- /dev/null +++ b/components/livekit/audio-visualizer/audio-grid-visualizer/audio-grid-visualizer.tsx @@ -0,0 +1,134 @@ +import { CSSProperties, ComponentType, JSX, memo, useMemo } from 'react'; +import { LocalAudioTrack, RemoteAudioTrack } from 'livekit-client'; +import { + type AgentState, + type TrackReferenceOrPlaceholder, + useMultibandTrackVolume, +} from '@livekit/components-react'; +import { cn } from '@/lib/utils'; +import { type Coordinate, useGridAnimator } from './hooks/useGridAnimator'; + +type GridComponentType = + | ComponentType<{ style?: CSSProperties; className?: string }> + | keyof JSX.IntrinsicElements; + +export interface GridOptions { + radius?: number; + interval?: number; + rowCount?: number; + columnCount?: number; + className?: string; + baseClassName?: string; + offClassName?: string; + onClassName?: string; + GridComponent?: GridComponentType; + transformer?: (index: number, rowCount: number, columnCount: number) => CSSProperties; +} + +function useGrid(options: GridOptions) { + return useMemo(() => { + const { columnCount = 5, rowCount } = options; + + const _columnCount = columnCount; + const _rowCount = rowCount ?? columnCount; + const items = new Array(_columnCount * _rowCount).fill(0).map((_, idx) => idx); + + return { columnCount: _columnCount, rowCount: _rowCount, items }; + }, [options]); +} + +interface GridCellProps { + index: number; + state: AgentState; + options: GridOptions; + rowCount: number; + volumeBands: number[]; + columnCount: number; + highlightedCoordinate: Coordinate; + Component: GridComponentType; +} + +const GridCell = memo(function GridCell({ + index, + state, + options, + rowCount, + volumeBands, + columnCount, + highlightedCoordinate, + Component, +}: GridCellProps) { + const { interval = 100, baseClassName, onClassName, offClassName, transformer } = options; + + if (state === 'speaking') { + const y = Math.floor(index / columnCount); + const rowMidPoint = Math.floor(rowCount / 2); + const volumeChunks = 1 / (rowMidPoint + 1); + const distanceToMid = Math.abs(rowMidPoint - y); + const threshold = distanceToMid * volumeChunks; + const isOn = volumeBands[index % columnCount] >= threshold; + + return ; + } + + let transformerStyle: CSSProperties | undefined; + if (transformer) { + transformerStyle = transformer(index, rowCount, columnCount); + } + + const isOn = + highlightedCoordinate.x === index % columnCount && + highlightedCoordinate.y === Math.floor(index / columnCount); + + const transitionDurationInSeconds = interval / (isOn ? 1000 : 100); + + return ( + + ); +}); + +export interface AudioGridVisualizerProps { + state: AgentState; + options: GridOptions; + audioTrack?: LocalAudioTrack | RemoteAudioTrack | TrackReferenceOrPlaceholder; +} + +export function AudioGridVisualizer({ state, options, audioTrack }: AudioGridVisualizerProps) { + const { radius, interval = 100, className, GridComponent = 'div' } = options; + const { columnCount, rowCount, items } = useGrid(options); + const highlightedCoordinate = useGridAnimator(state, rowCount, columnCount, interval, radius); + const volumeBands = useMultibandTrackVolume(audioTrack, { + bands: columnCount, + loPass: 100, + hiPass: 200, + }); + + return ( +
+ {items.map((idx) => ( + + ))} +
+ ); +} diff --git a/components/livekit/audio-visualizer/audio-grid-visualizer/demos.tsx b/components/livekit/audio-visualizer/audio-grid-visualizer/demos.tsx new file mode 100644 index 00000000..4fb0237c --- /dev/null +++ b/components/livekit/audio-visualizer/audio-grid-visualizer/demos.tsx @@ -0,0 +1,197 @@ +import { GridOptions } from './audio-grid-visualizer'; + +type SVGIconProps = React.SVGProps; + +function SVGIcon({ className, children, ...props }: SVGIconProps) { + return ( + <> + + {children} + + + ); +} + +function EyeSVG(props: SVGIconProps) { + return ( + + + + ); +} + +function PlusSVG(props: SVGIconProps) { + return ( + + + + ); +} + +export const gridVariants: GridOptions[] = [ + // 1 + { + radius: 6, + interval: 75, + className: 'gap-4', + baseClassName: 'size-1 rounded-full', + offClassName: 'bg-foreground/10 scale-100', + onClassName: 'bg-foreground scale-125 shadow-[0px_0px_10px_2px_rgba(255,255,255,0.4)]', + }, + // 2 + { + interval: 50, + className: 'gap-2', + baseClassName: 'w-4 h-1', + offClassName: 'bg-foreground/10', + onClassName: 'bg-[#F9B11F] shadow-[0px_0px_14.8px_2px_#F9B11F]', + }, + // 3 + { + className: 'gap-2', + baseClassName: 'size-2 rounded-full', + offClassName: 'bg-foreground/10', + onClassName: 'bg-[#1FD5F9] shadow-[0px_0px_8px_3px_rgba(31,213,249,0.44)]', + }, + // 4 + { + className: 'gap-4', + baseClassName: 'size-4 rounded-full', + offClassName: 'bg-foreground/5', + onClassName: 'bg-[#FF6352] shadow-[0px_0px_32px_8px_rgba(255,99,82,0.8)]', + }, + // 5 + { + className: 'gap-4', + baseClassName: 'size-2 rounded-full', + offClassName: 'bg-foreground/10', + onClassName: 'bg-[#1F8CF9] shadow-[0px_0px_14.8px_2px_#1F8CF9]', + transformer: (index: number, rowCount: number, columnCount: number) => { + const rowMidPoint = Math.floor(rowCount / 2); + const distanceFromCenter = Math.sqrt( + Math.pow(rowMidPoint - (index % columnCount), 2) + + Math.pow(rowMidPoint - Math.floor(index / columnCount), 2) + ); + + return { + opacity: 1 - distanceFromCenter / columnCount, + transform: `scale(${1 - (distanceFromCenter / (columnCount * 2)) * 1.75})`, + }; + }, + }, + // 6 + { + radius: 4, + interval: 150, + className: 'gap-2', + baseClassName: 'size-2', + offClassName: 'bg-foreground/8', + onClassName: 'bg-[#F91F8C] shadow-[0px_0px_14.8px_4px_#F91F8C] scale-150', + transformer: (index: number, rowCount: number, columnCount: number) => { + const rowMidPoint = Math.floor(rowCount / 2); + const distanceFromCenter = Math.sqrt( + Math.pow(rowMidPoint - (index % columnCount), 2) + + Math.pow(rowMidPoint - Math.floor(index / columnCount), 2) + ); + + return { + opacity: 0.5 - distanceFromCenter / columnCount, + }; + }, + }, + // 7 + { + interval: 50, + className: 'gap-4', + baseClassName: 'size-2.5 rounded-1.5', + offClassName: 'bg-foreground/15', + onClassName: 'bg-[#FFB6C1] shadow-[0px_0px_24px_3px_rgba(255,182,193,0.8)]', + }, + // 8 + { + interval: 50, + className: 'gap-8', + baseClassName: 'size-2.5', + offClassName: 'bg-foreground/5', + onClassName: 'bg-[#FFB6C1] shadow-[0px_0px_8px_1px_rgba(255,182,193,0.8)]', + transformer: (index: number, rowCount: number, columnCount: number) => { + const rowMidPoint = Math.floor(rowCount / 2); + const distanceFromCenter = Math.sqrt( + Math.pow(rowMidPoint - (index % columnCount), 2) + + Math.pow(rowMidPoint - Math.floor(index / columnCount), 2) + ); + const maxDistance = Math.sqrt(4); // maximum distance in a normalized grid + const scaleFactor = 1 + (maxDistance - distanceFromCenter / maxDistance); + + return { + transform: `scale(${scaleFactor})`, + }; + }, + }, + // 9 + { + radius: 4, + interval: 75, + className: 'gap-2.5', + baseClassName: 'w-1.5 h-px rotate-45', + offClassName: 'bg-foreground/10 rotate-45 scale-100', + onClassName: + 'bg-foreground drop-shadow-[0px_0px_8px_2px_rgba(255,182,193,0.4)] rotate-[405deg] scale-200', + }, + // 10 + { + radius: 4, + interval: 75, + GridComponent: PlusSVG, + className: 'gap-2', + baseClassName: 'size-2 rotate-45', + offClassName: 'text-foreground/10 rotate-45', + onClassName: 'text-[#F97A1F] drop-shadow-[0px_0px_3.2px_#F97A1F] rotate-[405deg] scale-250', + }, + // 11 + { + radius: 3, + interval: 75, + GridComponent: EyeSVG, + className: 'gap-2', + baseClassName: 'size-2', + offClassName: 'text-foreground/10', + onClassName: 'text-[#B11FF9] drop-shadow-[0px_0px_16px_2px_rgba(177,31,249,0.6)] scale-150', + transformer: (index: number, rowCount: number, columnCount: number) => { + const rowMidPoint = Math.floor(rowCount / 2); + const distanceFromCenter = Math.sqrt( + Math.pow(rowMidPoint - (index % columnCount), 2) + + Math.pow(rowMidPoint - Math.floor(index / columnCount), 2) + ); + + return { + opacity: 0.5 - distanceFromCenter / columnCount, + }; + }, + }, + // 12 + { + radius: 6, + interval: 75, + className: 'gap-2.5', + baseClassName: 'w-3 h-px rotate-45', + offClassName: 'bg-foreground/10 rotate-45 scale-100', + onClassName: + 'bg-[#FFB6C1] shadow-[0px_0px_8px_2px_rgba(255,182,193,0.4)] rotate-[405deg] scale-200', + }, +]; diff --git a/components/livekit/audio-visualizer/audio-grid-visualizer/hooks/useGridAnimator.ts b/components/livekit/audio-visualizer/audio-grid-visualizer/hooks/useGridAnimator.ts new file mode 100644 index 00000000..ce9272eb --- /dev/null +++ b/components/livekit/audio-visualizer/audio-grid-visualizer/hooks/useGridAnimator.ts @@ -0,0 +1,115 @@ +import { useEffect, useState } from 'react'; +import { type AgentState } from '@livekit/components-react'; + +export interface Coordinate { + x: number; + y: number; +} + +export function generateConnectingSequence(rows: number, columns: number, radius: number) { + const seq = []; + // const centerX = Math.floor(columns / 2); + const centerY = Math.floor(rows / 2); + + // Calculate the boundaries of the ring based on the ring distance + const topLeft = { + x: Math.max(0, centerY - radius), + y: Math.max(0, centerY - radius), + }; + const bottomRight = { + x: columns - 1 - topLeft.x, + y: Math.min(rows - 1, centerY + radius), + }; + + // Top edge + for (let x = topLeft.x; x <= bottomRight.x; x++) { + seq.push({ x, y: topLeft.y }); + } + + // Right edge + for (let y = topLeft.y + 1; y <= bottomRight.y; y++) { + seq.push({ x: bottomRight.x, y }); + } + + // Bottom edge + for (let x = bottomRight.x - 1; x >= topLeft.x; x--) { + seq.push({ x, y: bottomRight.y }); + } + + // Left edge + for (let y = bottomRight.y - 1; y > topLeft.y; y--) { + seq.push({ x: topLeft.x, y }); + } + + return seq; +} + +export function generateListeningSequence(rows: number, columns: number) { + const center = { x: Math.floor(columns / 2), y: Math.floor(rows / 2) }; + const noIndex = { x: -1, y: -1 }; + + return [center, noIndex, noIndex, noIndex, noIndex, noIndex, noIndex, noIndex, noIndex]; +} + +export function generateThinkingSequence(rows: number, columns: number) { + const seq = []; + const y = Math.floor(rows / 2); + for (let x = 0; x < columns; x++) { + seq.push({ x, y }); + } + for (let x = columns - 1; x >= 0; x--) { + seq.push({ x, y }); + } + + return seq; +} + +export function useGridAnimator( + state: AgentState, + rows: number, + columns: number, + interval: number, + radius?: number +): Coordinate { + const [index, setIndex] = useState(0); + const [sequence, setSequence] = useState(() => [ + { + x: Math.floor(columns / 2), + y: Math.floor(rows / 2), + }, + ]); + + useEffect(() => { + const clampedRadius = radius + ? Math.min(radius, Math.floor(Math.max(rows, columns) / 2)) + : Math.floor(Math.max(rows, columns) / 2); + + if (state === 'thinking') { + setSequence(generateThinkingSequence(rows, columns)); + } else if (state === 'connecting' || state === 'initializing') { + const sequence = [...generateConnectingSequence(rows, columns, clampedRadius)]; + setSequence(sequence); + } else if (state === 'listening') { + setSequence(generateListeningSequence(rows, columns)); + } else { + setSequence([{ x: Math.floor(columns / 2), y: Math.floor(rows / 2) }]); + } + setIndex(0); + }, [state, rows, columns, radius]); + + useEffect(() => { + if (state === 'speaking') { + return; + } + + const indexInterval = setInterval(() => { + setIndex((prev) => { + return prev + 1; + }); + }, interval); + + return () => clearInterval(indexInterval); + }, [interval, columns, rows, state, sequence.length]); + + return sequence[index % sequence.length]; +} diff --git a/components/livekit/audio-visualizer/audio-oscilloscope-visualizer/audio-oscilloscope-visualizer.tsx b/components/livekit/audio-visualizer/audio-oscilloscope-visualizer/audio-oscilloscope-visualizer.tsx new file mode 100644 index 00000000..8d3bcbf4 --- /dev/null +++ b/components/livekit/audio-visualizer/audio-oscilloscope-visualizer/audio-oscilloscope-visualizer.tsx @@ -0,0 +1,142 @@ +'use client'; + +import { useCallback, useEffect, useMemo, useRef, useState } from 'react'; +import { type VariantProps, cva } from 'class-variance-authority'; +import { + type AnimationPlaybackControlsWithThen, + type ValueAnimationTransition, + animate, + useMotionValue, + useMotionValueEvent, +} from 'motion/react'; +import { + type AgentState, + type TrackReference, + type TrackReferenceOrPlaceholder, + // useMultibandTrackVolume, + useTrackVolume, +} from '@livekit/components-react'; +import { cn } from '@/lib/utils'; +import { OscilliscopeShaders, type OscilliscopeShadersProps } from './oscilliscope-shaders'; + +const DEFAULT_SPEED = 5; +const DEFAULT_AMPLITUDE = 0.025; +const DEFAULT_FREQUENCY = 10; +const DEFAULT_TRANSITION: ValueAnimationTransition = { duration: 0.2, ease: 'easeOut' }; + +function useAnimatedValue(initialValue: T) { + const [value, setValue] = useState(initialValue); + const motionValue = useMotionValue(initialValue); + const controlsRef = useRef(null); + useMotionValueEvent(motionValue, 'change', (value) => setValue(value as T)); + + const animateFn = useCallback( + (targetValue: T | T[], transition: ValueAnimationTransition) => { + controlsRef.current = animate(motionValue, targetValue, transition); + }, + [motionValue] + ); + + return { value, controls: controlsRef, animate: animateFn }; +} + +export const audioShaderVisualizerVariants = cva(['aspect-square'], { + variants: { + size: { + icon: 'h-[24px] gap-[2px]', + sm: 'h-[56px] gap-[4px]', + md: 'h-[112px] gap-[8px]', + lg: 'h-[224px] gap-[16px]', + xl: 'h-[448px] gap-[32px]', + }, + }, + defaultVariants: { + size: 'md', + }, +}); + +interface AudioOscilloscopeVisualizerProps { + speed?: number; + state?: AgentState; + audioTrack: TrackReferenceOrPlaceholder; + className?: string; +} + +export function AudioOscilloscopeVisualizer({ + size = 'lg', + state = 'speaking', + speed = DEFAULT_SPEED, + audioTrack, + className, +}: AudioOscilloscopeVisualizerProps & + OscilliscopeShadersProps & + VariantProps) { + const { value: amplitude, animate: animateAmplitude } = useAnimatedValue(DEFAULT_AMPLITUDE); + const { value: frequency, animate: animateFrequency } = useAnimatedValue(DEFAULT_FREQUENCY); + const { value: opacity, animate: animateOpacity } = useAnimatedValue(1.0); + + const volume = useTrackVolume(audioTrack as TrackReference, { + fftSize: 512, + smoothingTimeConstant: 0.55, + }); + + useEffect(() => { + switch (state) { + case 'disconnected': + animateAmplitude(0, DEFAULT_TRANSITION); + animateFrequency(0, DEFAULT_TRANSITION); + animateOpacity(1.0, DEFAULT_TRANSITION); + return; + case 'listening': + case 'connecting': + animateAmplitude(DEFAULT_AMPLITUDE, DEFAULT_TRANSITION); + animateFrequency(DEFAULT_FREQUENCY, DEFAULT_TRANSITION); + animateOpacity([1.0, 0.2], { + duration: 0.75, + repeat: Infinity, + repeatType: 'mirror', + }); + return; + case 'thinking': + case 'initializing': + animateAmplitude(DEFAULT_AMPLITUDE, DEFAULT_TRANSITION); + animateFrequency(DEFAULT_FREQUENCY, DEFAULT_TRANSITION); + animateOpacity([1.0, 0.2], { + duration: 0.2, + repeat: Infinity, + repeatType: 'mirror', + }); + return; + case 'speaking': + default: + animateAmplitude(DEFAULT_AMPLITUDE, DEFAULT_TRANSITION); + animateFrequency(DEFAULT_FREQUENCY, DEFAULT_TRANSITION); + animateOpacity(1.0, DEFAULT_TRANSITION); + return; + } + }, [state, animateAmplitude, animateFrequency, animateOpacity]); + + useEffect(() => { + if (state === 'speaking' && volume > 0) { + animateAmplitude(0.02 + 0.4 * volume, { duration: 0 }); + animateFrequency(20 + 60 * volume, { duration: 0 }); + } + }, [state, volume, animateAmplitude, animateFrequency]); + + return ( + + ); +} diff --git a/components/livekit/audio-visualizer/audio-oscilloscope-visualizer/oscilliscope-shaders.tsx b/components/livekit/audio-visualizer/audio-oscilloscope-visualizer/oscilliscope-shaders.tsx new file mode 100644 index 00000000..8b413716 --- /dev/null +++ b/components/livekit/audio-visualizer/audio-oscilloscope-visualizer/oscilliscope-shaders.tsx @@ -0,0 +1,168 @@ +'use client'; + +import React, { forwardRef } from 'react'; +import { Shader } from '@/components/livekit/react-shader/react-shader'; + +const oscilliscopeShaderSource = ` +const float TAU = 6.28318530718; + +// Cosine palette generator +vec3 pal(in float t, in vec3 a, in vec3 b, in vec3 c, in vec3 d) { + return a + b * cos(TAU * (c * t + d)); +} + +// Noise for dithering +vec2 randFibo(vec2 p) { + p = fract(p * vec2(443.897, 441.423)); + p += dot(p, p.yx + 19.19); + return fract((p.xx + p.yx) * p.xy); +} + +// Luma for alpha +float luma(vec3 color) { + return dot(color, vec3(0.299, 0.587, 0.114)); +} + +// Bell curve function for attenuation from center with rounded top +float bellCurve(float distanceFromCenter, float maxDistance) { + float normalizedDistance = distanceFromCenter / maxDistance; + // Use cosine with high power for smooth rounded top + return pow(cos(normalizedDistance * (3.14159265359 / 4.0)), 16.0); +} + +// Calculate the sine wave +float oscilloscopeWave(float x, float centerX, float time) { + float relativeX = x - centerX; + float maxDistance = centerX; + float distanceFromCenter = abs(relativeX); + + // Apply bell curve for amplitude attenuation + float bell = bellCurve(distanceFromCenter, maxDistance); + + // Calculate wave with uniforms and bell curve attenuation + float wave = sin(relativeX * uFrequency + time * uSpeed) * uAmplitude * bell; + + return wave; +} + +void mainImage(out vec4 fragColor, in vec2 fragCoord) { + vec2 uv = fragCoord / iResolution.xy; + vec2 pos = uv - 0.5; + + // Calculate center and positions + float centerX = 0.5; + float centerY = 0.5; + float x = uv.x; + float y = uv.y; + + // Find minimum distance to the wave by sampling nearby points + // This gives us consistent line width without high-frequency artifacts + const int NUM_SAMPLES = 50; // Must be const for GLSL loop + float minDist = 1000.0; + float sampleRange = 0.02; // Range to search for closest point + + for(int i = 0; i < NUM_SAMPLES; i++) { + float offset = (float(i) / float(NUM_SAMPLES - 1) - 0.5) * sampleRange; + float sampleX = x + offset; + float waveY = centerY + oscilloscopeWave(sampleX, centerX, iTime); + + // Calculate distance from current pixel to this point on the wave + vec2 wavePoint = vec2(sampleX, waveY); + vec2 currentPoint = vec2(x, y); + float dist = distance(currentPoint, wavePoint); + + minDist = min(minDist, dist); + } + + // Create solid line with anti-aliasing + float lineWidth = uLineWidth; + float smoothing = uSmoothing; + + // Solid line with smooth edges using minimum distance + float line = smoothstep(lineWidth + smoothing, lineWidth - smoothing, minDist); + + // Calculate color position based on x position for gradient effect + float colorPos = x; + vec3 color = pal( + colorPos * uColorScale + uColorPosition * 2.0, + vec3(0.5), + vec3(0.5), + vec3(1.0), + vec3(0.0, 0.33, 0.67) + ); + + // Apply line intensity + color *= line; + + // Add dithering for smoother gradients + // color += (randFibo(fragCoord).x - 0.5) / 255.0; + + // Calculate alpha based on line intensity + float alpha = line * uMix; + + fragColor = vec4(color * uMix, alpha); +}`; + +export interface OscilliscopeShadersProps extends React.HTMLAttributes { + className?: string; + speed?: number; + amplitude?: number; + frequency?: number; + colorScale?: number; + colorPosition?: number; + mix?: number; + lineWidth?: number; + smoothing?: number; +} + +export const OscilliscopeShaders = forwardRef( + ( + { + className, + speed = 10, + amplitude = 0.02, + frequency = 20.0, + colorScale = 0.12, + colorPosition = 0.18, + mix = 1.0, + lineWidth = 0.01, + smoothing = 0.0, + ...props + }, + ref + ) => { + const globalThis = typeof window !== 'undefined' ? window : global; + + console.log('OscilliscopeShaders rendering'); + + return ( +
+ { + console.error('Shader error:', error); + }} + onWarning={(warning) => { + console.warn('Shader warning:', warning); + }} + style={{ width: '100%', height: '100%' } as CSSStyleDeclaration} + /> +
+ ); + } +); + +OscilliscopeShaders.displayName = 'OscilliscopeShaders'; + +export default OscilliscopeShaders; diff --git a/components/livekit/audio-visualizer/audio-radial-visualizer/_bar-visualizer.tsx b/components/livekit/audio-visualizer/audio-radial-visualizer/_bar-visualizer.tsx new file mode 100644 index 00000000..aa3ab23b --- /dev/null +++ b/components/livekit/audio-visualizer/audio-radial-visualizer/_bar-visualizer.tsx @@ -0,0 +1,99 @@ +import { useEffect, useMemo, useRef } from 'react'; +import { type VariantProps, cva } from 'class-variance-authority'; +import { + type AgentState, + BarVisualizer as LiveKitBarVisualizer, + type TrackReferenceOrPlaceholder, +} from '@livekit/components-react'; +import { cn } from '@/lib/utils'; + +const MIN_HEIGHT = 15; // 15% + +export const barVisualizerVariants = cva( + ['relative flex aspect-square h-36 items-center justify-center'], + { + variants: { + size: { + default: 'h-32', + icon: 'h-6', + xs: 'h-8', + sm: 'h-16', + md: 'h-32', + lg: 'h-64', + xl: 'h-96', + '2xl': 'h-128', + }, + }, + defaultVariants: { + size: 'default', + }, + } +); + +interface BarVisualizerProps { + state?: AgentState; + barCount?: number; + audioTrack?: TrackReferenceOrPlaceholder; + className?: string; +} + +export function BarVisualizer({ + size, + state, + barCount, + audioTrack, + className, +}: BarVisualizerProps & VariantProps) { + const ref = useRef(null); + const _barCount = useMemo(() => { + if (barCount) { + return barCount; + } + switch (size) { + case 'icon': + case 'xs': + return 3; + default: + return 5; + } + }, [barCount, size]); + + const x = (1 / (_barCount + (_barCount + 1) / 2)) * 100; + + // reset bars height when audio track is disconnected + useEffect(() => { + if (ref.current && !audioTrack) { + const bars = [...(ref.current.querySelectorAll('& > span') ?? [])] as HTMLElement[]; + + bars.forEach((bar) => { + bar.style.height = `${MIN_HEIGHT}%`; + }); + } + }, [audioTrack]); + + return ( + + + + ); +} diff --git a/components/livekit/audio-visualizer/audio-radial-visualizer/audio-radial-visualizer.tsx b/components/livekit/audio-visualizer/audio-radial-visualizer/audio-radial-visualizer.tsx new file mode 100644 index 00000000..d36ac939 --- /dev/null +++ b/components/livekit/audio-visualizer/audio-radial-visualizer/audio-radial-visualizer.tsx @@ -0,0 +1,150 @@ +import { useMemo } from 'react'; +import { type VariantProps, cva } from 'class-variance-authority'; +import { type LocalAudioTrack, type RemoteAudioTrack } from 'livekit-client'; +import { + type AgentState, + type TrackReferenceOrPlaceholder, + useMultibandTrackVolume, +} from '@livekit/components-react'; +import { cn } from '@/lib/utils'; +import { useBarAnimator } from './hooks/useBarAnimator'; + +export const audioRadialVisualizerVariants = cva(['relative flex items-center justify-center'], { + variants: { + size: { + icon: 'h-[24px] gap-[2px]', + sm: 'h-[56px] gap-[4px]', + md: 'h-[112px] gap-[8px]', + lg: 'h-[224px] gap-[16px]', + xl: 'h-[448px] gap-[32px]', + }, + }, + defaultVariants: { + size: 'md', + }, +}); + +export const audioRadialVisualizerBarVariants = cva( + [ + 'rounded-full transition-colors duration-250 ease-linear bg-(--audio-visualizer-idle) data-[lk-highlighted=true]:bg-(--audio-visualizer-active)', + ], + { + variants: { + size: { + icon: 'w-[4px] min-h-[4px]', + sm: 'w-[8px] min-h-[8px]', + md: 'w-[16px] min-h-[16px]', + lg: 'w-[32px] min-h-[32px]', + xl: 'w-[64px] min-h-[64px]', + }, + }, + defaultVariants: { + size: 'md', + }, + } +); + +interface AudioRadialVisualizerProps { + state?: AgentState; + radius?: number; + barCount?: number; + audioTrack?: LocalAudioTrack | RemoteAudioTrack | TrackReferenceOrPlaceholder; + className?: string; + barClassName?: string; +} + +export function AudioRadialVisualizer({ + size, + state, + radius, + barCount, + audioTrack, + className, + barClassName, +}: AudioRadialVisualizerProps & VariantProps) { + const _barCount = useMemo(() => { + if (barCount) { + return barCount; + } + switch (size) { + case 'icon': + case 'sm': + return 9; + default: + return 12; + } + }, [barCount, size]); + + const volumeBands = useMultibandTrackVolume(audioTrack, { + bands: Math.floor(_barCount / 2), + loPass: 100, + hiPass: 200, + }); + + const sequencerInterval = useMemo(() => { + switch (state) { + case 'connecting': + return 2000 / _barCount; + case 'initializing': + return 500; + case 'listening': + return 500; + case 'thinking': + return 150; + default: + return 1000; + } + }, [state, _barCount]); + + const distanceFromCenter = useMemo(() => { + if (radius) { + return radius; + } + switch (size) { + case 'icon': + return 6; + case 'xl': + return 128; + case 'lg': + return 64; + case 'sm': + return 16; + case 'md': + default: + return 32; + } + }, [size, radius]); + + const highlightedIndices = useBarAnimator(state, _barCount, sequencerInterval); + const bands = audioTrack ? [...volumeBands, ...volumeBands] : new Array(_barCount).fill(0); + + return ( +
+ {bands.map((band, idx) => { + const angle = (idx / _barCount) * Math.PI * 2; + + return ( +
+
+
+ ); + })} +
+ ); +} diff --git a/components/livekit/audio-visualizer/audio-radial-visualizer/hooks/useBarAnimator.ts b/components/livekit/audio-visualizer/audio-radial-visualizer/hooks/useBarAnimator.ts new file mode 100644 index 00000000..e0d1c808 --- /dev/null +++ b/components/livekit/audio-visualizer/audio-radial-visualizer/hooks/useBarAnimator.ts @@ -0,0 +1,83 @@ +import { useEffect, useRef, useState } from 'react'; +import { type AgentState } from '@livekit/components-react'; + +function generateConnectingSequenceBar(columns: number): number[][] { + const seq = []; + + for (let x = 0; x < columns; x++) { + seq.push([x]); + } + + return seq; +} + +function generateListeningSequenceBar(columns: number): number[][] { + const center = Math.floor(columns / 2); + const noIndex = -1; + + return [[center], [noIndex]]; +} + +// function generateThinkingSequenceBar(columns: number): number[][] { +// const seq = []; +// for (let x = 0; x < columns; x++) { +// seq.push([x]); +// } + +// for (let x = columns - 1; x >= 0; x--) { +// seq.push([x]); +// } + +// return seq; +// } + +export const useBarAnimator = ( + state: AgentState | undefined, + columns: number, + interval: number +): number[] => { + const [index, setIndex] = useState(0); + const [sequence, setSequence] = useState([[]]); + + useEffect(() => { + if (state === 'thinking') { + setSequence(generateListeningSequenceBar(columns)); + // setSequence(generateThinkingSequenceBar(columns)); + } else if (state === 'connecting' || state === 'initializing') { + setSequence(generateConnectingSequenceBar(columns)); + } else if (state === 'listening') { + setSequence(generateListeningSequenceBar(columns)); + } else if (state === undefined || state === 'speaking') { + setSequence([new Array(columns).fill(0).map((_, idx) => idx)]); + } else { + setSequence([[]]); + } + setIndex(0); + }, [state, columns]); + + const animationFrameId = useRef(null); + useEffect(() => { + let startTime = performance.now(); + + const animate = (time: DOMHighResTimeStamp) => { + const timeElapsed = time - startTime; + + if (timeElapsed >= interval) { + setIndex((prev) => prev + 1); + startTime = time; + } + + animationFrameId.current = requestAnimationFrame(animate); + }; + + animationFrameId.current = requestAnimationFrame(animate); + + return () => { + if (animationFrameId.current !== null) { + cancelAnimationFrame(animationFrameId.current); + } + }; + }, [interval, columns, state, sequence.length]); + + return sequence[index % sequence.length]; +}; diff --git a/components/livekit/audio-visualizer/audio-shader-visualizer/audio-shader-visualizer.tsx b/components/livekit/audio-visualizer/audio-shader-visualizer/audio-shader-visualizer.tsx new file mode 100644 index 00000000..460a3341 --- /dev/null +++ b/components/livekit/audio-visualizer/audio-shader-visualizer/audio-shader-visualizer.tsx @@ -0,0 +1,183 @@ +'use client'; + +import { useCallback, useEffect, useRef, useState } from 'react'; +import { type VariantProps, cva } from 'class-variance-authority'; +import { + type AnimationPlaybackControlsWithThen, + type KeyframesTarget, + type ValueAnimationTransition, + animate, + useMotionValue, + useMotionValueEvent, +} from 'motion/react'; +import { + type AgentState, + type TrackReference, + type TrackReferenceOrPlaceholder, + // useMultibandTrackVolume, + useTrackVolume, +} from '@livekit/components-react'; +import { cn } from '@/lib/utils'; +import { AuroraShaders, type AuroraShadersProps } from './aurora-shaders'; + +const DEFAULT_SPEED = 10; +const DEFAULT_AMPLITUDE = 2; +const DEFAULT_FREQUENCY = 0.5; +const DEFAULT_SCALE = 0.2; +const DEFAULT_BRIGHTNESS = 1.5; +const DEFAULT_TRANSITION: ValueAnimationTransition = { duration: 0.5, ease: 'easeOut' }; +const DEFAULT_PULSE_TRANSITION: ValueAnimationTransition = { + duration: 0.5, + ease: 'easeOut', + repeat: Infinity, + repeatType: 'mirror', +}; + +function useAnimatedValue(initialValue: T) { + const [value, setValue] = useState(initialValue); + const motionValue = useMotionValue(initialValue); + const controlsRef = useRef(null); + useMotionValueEvent(motionValue, 'change', (value) => setValue(value as T)); + + const animateFn = useCallback( + (targetValue: T | KeyframesTarget, transition: ValueAnimationTransition) => { + controlsRef.current = animate(motionValue, targetValue, transition); + }, + [motionValue] + ); + + return { value, motionValue, controls: controlsRef, animate: animateFn }; +} + +export const audioShaderVisualizerVariants = cva(['aspect-square'], { + variants: { + size: { + icon: 'h-[24px] gap-[2px]', + sm: 'h-[56px] gap-[4px]', + md: 'h-[112px] gap-[8px]', + lg: 'h-[224px] gap-[16px]', + xl: 'h-[448px] gap-[32px]', + }, + }, + defaultVariants: { + size: 'md', + }, +}); + +interface AudioShaderVisualizerProps { + state?: AgentState; + audioTrack: TrackReferenceOrPlaceholder; +} + +export function AudioShaderVisualizer({ + size = 'md', + state = 'speaking', + shape = 1, + colorScale = 0.05, + colorPosition = 0.18, + audioTrack, + className, +}: AudioShaderVisualizerProps & + AuroraShadersProps & + VariantProps) { + const [speed, setSpeed] = useState(DEFAULT_SPEED); + const { + value: scale, + animate: animateScale, + motionValue: scaleMotionValue, + } = useAnimatedValue(DEFAULT_SCALE); + const { value: amplitude, animate: animateAmplitude } = useAnimatedValue(DEFAULT_AMPLITUDE); + const { value: frequency, animate: animateFrequency } = useAnimatedValue(DEFAULT_FREQUENCY); + const { value: brightness, animate: animateBrightness } = useAnimatedValue(DEFAULT_BRIGHTNESS); + + const volume = useTrackVolume(audioTrack as TrackReference, { + fftSize: 512, + smoothingTimeConstant: 0.55, + }); + + useEffect(() => { + switch (state) { + case 'disconnected': + setSpeed(5); + animateScale(0.2, DEFAULT_TRANSITION); + animateAmplitude(1.2, DEFAULT_TRANSITION); + animateFrequency(0.4, DEFAULT_TRANSITION); + animateBrightness(1.0, DEFAULT_TRANSITION); + return; + case 'listening': + case 'connecting': + setSpeed(20); + animateScale(0.35, DEFAULT_TRANSITION); + animateAmplitude(1, DEFAULT_TRANSITION); + animateFrequency(0.7, DEFAULT_TRANSITION); + // animateBrightness(2.0, DEFAULT_TRANSITION); + animateBrightness([1.5, 2.0], DEFAULT_PULSE_TRANSITION); + return; + case 'initializing': + setSpeed(30); + animateScale(0.3, DEFAULT_TRANSITION); + animateAmplitude(0.5, DEFAULT_TRANSITION); + animateFrequency(1, DEFAULT_TRANSITION); + animateBrightness([0.5, 2.5], DEFAULT_PULSE_TRANSITION); + return; + case 'thinking': + setSpeed(30); + animateScale(0.1, DEFAULT_TRANSITION); + animateAmplitude(1.0, DEFAULT_TRANSITION); + animateFrequency(3.0, DEFAULT_TRANSITION); + animateBrightness([1.0, 2.0], DEFAULT_PULSE_TRANSITION); + return; + case 'speaking': + setSpeed(50); + animateScale(0.3, DEFAULT_TRANSITION); + animateAmplitude(1.0, DEFAULT_TRANSITION); + animateFrequency(0.7, DEFAULT_TRANSITION); + animateBrightness(1.5, DEFAULT_TRANSITION); + return; + } + }, [ + state, + shape, + colorScale, + animateScale, + animateAmplitude, + animateFrequency, + animateBrightness, + ]); + + useEffect(() => { + if (state === 'speaking' && volume > 0 && !scaleMotionValue.isAnimating()) { + animateScale(0.3 - 0.1 * volume, { duration: 0 }); + animateAmplitude(1.0 + 0.2 * volume, { duration: 0 }); + animateFrequency(0.7 - 0.3 * volume, { duration: 0 }); + animateBrightness(1.5 + 1.0 * volume, { duration: 0 }); + } + }, [ + state, + volume, + scaleMotionValue, + animateScale, + animateAmplitude, + animateFrequency, + animateBrightness, + ]); + + return ( + + ); +} diff --git a/components/livekit/audio-visualizer/audio-shader-visualizer/aurora-shaders.tsx b/components/livekit/audio-visualizer/audio-shader-visualizer/aurora-shaders.tsx new file mode 100644 index 00000000..57bf62fb --- /dev/null +++ b/components/livekit/audio-visualizer/audio-shader-visualizer/aurora-shaders.tsx @@ -0,0 +1,250 @@ +'use client'; + +import React, { forwardRef } from 'react'; +import { Shader } from '@/components/livekit/react-shader/react-shader'; + +const auroraShaderSource = ` +const float TAU = 6.28318530718; + +// Cosine palette generator +vec3 pal(in float t, in vec3 a, in vec3 b, in vec3 c, in vec3 d) { + return a + b * cos(TAU * (c * t + d)); +} + +// Noise for dithering +vec2 randFibo(vec2 p) { + p = fract(p * vec2(443.897, 441.423)); + p += dot(p, p.yx + 19.19); + return fract((p.xx + p.yx) * p.xy); +} + +// Tonemap +vec3 Tonemap_Reinhard(vec3 x) { + x *= 4.0; + return x / (1.0 + x); +} + +// Luma for alpha +float luma(vec3 color) { + return dot(color, vec3(0.299, 0.587, 0.114)); +} + +// SDF shapes +float sdCircle(vec2 st, float r) { + return length(st) - r; +} + +float sdLine(vec2 p, float r) { + float halfLen = r * 2.0; + vec2 a = vec2(-halfLen, 0.0); + vec2 b = vec2(halfLen, 0.0); + vec2 pa = p - a; + vec2 ba = b - a; + float h = clamp(dot(pa, ba) / dot(ba, ba), 0.0, 1.0); + return length(pa - ba * h); +} + +float getSdf(vec2 st) { + if(uShape == 1.0) return sdCircle(st, uScale); + else if(uShape == 2.0) return sdLine(st, uScale); + return sdCircle(st, uScale); // Default +} + +vec2 turb(vec2 pos, float t, float it) { + // mat2 rot = mat2(0.6, -0.8, 0.8, 0.6); + mat2 rot = mat2(0.6, -0.25, 0.25, 0.9); + float freq = mix(2.0, 15.0, uFrequency); + float amp = uAmplitude; + float xp = 1.4; + float time = t * 0.1 * uSpeed; + + for(float i = 0.0; i < 4.0; i++) { + vec2 s = sin(freq * (pos * rot) + i * time + it); + pos += amp * rot[0] * s / freq; + rot *= mat2(0.6, -0.8, 0.8, 0.6); + amp *= mix(1.0, max(s.y, s.x), uVariance); + freq *= xp; + } + + return pos; +} + +const float ITERATIONS = 36.0; + +void mainImage(out vec4 fragColor, in vec2 fragCoord) { + vec2 uv = fragCoord / iResolution.xy; + + vec3 pp = vec3(0.0); + vec3 bloom = vec3(0.0); + float t = iTime * 0.5; + vec2 pos = uv - 0.5; + + vec2 prevPos = turb(pos, t, 0.0 - 1.0 / ITERATIONS); + float spacing = mix(1.0, TAU, uSpacing); + + for(float i = 1.0; i < ITERATIONS + 1.0; i++) { + float iter = i / ITERATIONS; + vec2 st = turb(pos, t, iter * spacing); + float d = abs(getSdf(st)); + float pd = distance(st, prevPos); + prevPos = st; + float dynamicBlur = exp2(pd * 2.0 * 1.4426950408889634) - 1.0; + float ds = smoothstep(0.0, uBlur * 0.05 + max(dynamicBlur * uSmoothing, 0.001), d); + + // Generate color using cosine palette + vec3 color = pal( + iter * mix(0.0, 3.0, uColorScale) + uColorPosition * 2.0, + vec3(0.5), + vec3(0.5), + vec3(1.0), + vec3(0.0, 0.33, 0.67) + ); + + float invd = 1.0 / max(d + dynamicBlur, 0.001); + pp += (ds - 1.0) * color; + bloom += clamp(invd, 0.0, 250.0) * color; + } + + pp *= 1.0 / ITERATIONS; + bloom = bloom / (bloom + 2e4); + + vec3 color = (-pp + bloom * 3.0 * uBloom); + color *= 1.2; + color += (randFibo(fragCoord).x - 0.5) / 255.0; + color = Tonemap_Reinhard(color); + + float alpha = luma(color) * uMix; + fragColor = vec4(color * uMix, alpha); +}`; + +export interface AuroraShadersProps extends React.HTMLAttributes { + /** + * Aurora wave speed + * @default 1.0 + */ + speed?: number; + + /** + * Turbulence amplitude + * @default 0.5 + */ + amplitude?: number; + + /** + * Wave frequency and complexity + * @default 0.5 + */ + frequency?: number; + + /** + * Shape scale + * @default 0.3 + */ + scale?: number; + + /** + * Shape type: 1=circle, 2=line + * @default 1 + */ + shape?: number; + + /** + * Edge blur/softness + * @default 1.0 + */ + blur?: number; + + /** + * Color palette offset - shifts colors along the gradient (0-1) + * Lower values shift toward start colors, higher toward end colors + * @default 0.5 + * @example 0.0 - cool tones dominate + * @example 0.5 - balanced (default) + * @example 1.0 - warm tones dominate + */ + colorPosition?: number; + + /** + * Color variation across layers (0-1) + * Controls how much colors change between iterations + * @default 0.5 + * @example 0.0 - minimal color variation (more uniform) + * @example 0.5 - moderate variation (default) + * @example 1.0 - maximum variation (rainbow effect) + */ + colorScale?: number; + + /** + * Brightness of the aurora (0-1) + * @default 1.0 + */ + brightness?: number; +} + +export const AuroraShaders = forwardRef( + ( + { + className, + shape = 1.0, + speed = 1.0, + amplitude = 0.5, + frequency = 0.5, + scale = 0.2, + blur = 1.0, + colorPosition = 1.0, + colorScale = 1.0, + brightness = 1.0, + ...props + }, + ref + ) => { + const globalThis = typeof window !== 'undefined' ? window : global; + return ( +
+ { + console.log('error', error); + }} + onWarning={(warning) => { + console.log('warning', warning); + }} + style={{ width: '100%', height: '100%' } as CSSStyleDeclaration} + /> +
+ ); + } +); + +AuroraShaders.displayName = 'AuroraShaders'; + +export default AuroraShaders; diff --git a/components/livekit/react-shader/logging.ts b/components/livekit/react-shader/logging.ts new file mode 100644 index 00000000..aa742d8c --- /dev/null +++ b/components/livekit/react-shader/logging.ts @@ -0,0 +1 @@ +export const log = (text: string) => `react-shaders: ${text}`; diff --git a/components/livekit/react-shader/react-shader.original.tsx b/components/livekit/react-shader/react-shader.original.tsx new file mode 100644 index 00000000..aa709302 --- /dev/null +++ b/components/livekit/react-shader/react-shader.original.tsx @@ -0,0 +1,614 @@ +import { type CSSProperties, Component } from 'react'; +import { log } from './logging'; +import { + ClampToEdgeWrapping, + LinearFilter, + LinearMipMapLinearFilter, + LinearMipMapNearestFilter, + MirroredRepeatWrapping, + NearestFilter, + NearestMipMapLinearFilter, + NearestMipMapNearestFilter, + RepeatWrapping, + Texture, +} from './texture'; +import { + type UniformType, + type Vector2, + type Vector3, + type Vector4, + isMatrixType, + isVectorListType, + processUniform, + uniformTypeToGLSLType, +} from './uniforms'; + +export { + ClampToEdgeWrapping, + LinearFilter, + LinearMipMapLinearFilter, + LinearMipMapNearestFilter, + MirroredRepeatWrapping, + NearestFilter, + NearestMipMapLinearFilter, + NearestMipMapNearestFilter, + RepeatWrapping, +}; + +export type { Vector2, Vector3, Vector4 }; + +const PRECISIONS = ['lowp', 'mediump', 'highp']; +const FS_MAIN_SHADER = `\nvoid main(void){ + vec4 color = vec4(0.0,0.0,0.0,1.0); + mainImage( color, gl_FragCoord.xy ); + gl_FragColor = color; +}`; +const BASIC_FS = `void mainImage( out vec4 fragColor, in vec2 fragCoord ) { + vec2 uv = fragCoord/iResolution.xy; + vec3 col = 0.5 + 0.5*cos(iTime+uv.xyx+vec3(0,2,4)); + fragColor = vec4(col,1.0); +}`; +const BASIC_VS = `attribute vec3 aVertexPosition; +void main(void) { + gl_Position = vec4(aVertexPosition, 1.0); +}`; +const UNIFORM_TIME = 'iTime'; +const UNIFORM_TIMEDELTA = 'iTimeDelta'; +const UNIFORM_DATE = 'iDate'; +const UNIFORM_FRAME = 'iFrame'; +const UNIFORM_MOUSE = 'iMouse'; +const UNIFORM_RESOLUTION = 'iResolution'; +const UNIFORM_CHANNEL = 'iChannel'; +const UNIFORM_CHANNELRESOLUTION = 'iChannelResolution'; +const UNIFORM_DEVICEORIENTATION = 'iDeviceOrientation'; + +const latestPointerClientCoords = (e: MouseEvent | TouchEvent) => { + // @ts-expect-error TODO: Deal with this. + return [e.clientX || e.changedTouches[0].clientX, e.clientY || e.changedTouches[0].clientY]; +}; +const lerpVal = (v0: number, v1: number, t: number) => v0 * (1 - t) + v1 * t; +const insertStringAtIndex = (currentString: string, string: string, index: number) => + index > 0 + ? currentString.substring(0, index) + + string + + currentString.substring(index, currentString.length) + : string + currentString; + +type Uniform = { type: string; value: number[] | number }; +export type Uniforms = Record; +type TextureParams = { + url: string; + wrapS?: number; + wrapT?: number; + minFilter?: number; + magFilter?: number; + flipY?: number; +}; + +type Props = { + /** Fragment shader GLSL code. */ + fs: string; + + /** Vertex shader GLSL code. */ + vs?: string; + + /** + * Textures to be passed to the shader. Textures need to be squared or will be + * automatically resized. + * + * Options default to: + * + * ```js + * { + * minFilter: LinearMipMapLinearFilter, + * magFilter: LinearFilter, + * wrapS: RepeatWrapping, + * wrapT: RepeatWrapping, + * } + * ``` + * + * See [textures in the docs](https://rysana.com/docs/react-shaders#textures) + * for details. + */ + textures?: TextureParams[]; + + /** + * Custom uniforms to be passed to the shader. + * + * See [custom uniforms in the + * docs](https://rysana.com/docs/react-shaders#custom-uniforms) for details. + */ + uniforms?: Uniforms; + + /** + * Color used when clearing the canvas. + * + * See [the WebGL + * docs](https://developer.mozilla.org/en-US/docs/Web/API/WebGLRenderingContext/clearColor) + * for details. + */ + clearColor?: Vector4; + + /** + * GLSL precision qualifier. Defaults to `'highp'`. Balance between + * performance and quality. + */ + precision?: 'highp' | 'lowp' | 'mediump'; + + /** Custom inline style for canvas. */ + style?: CSSStyleDeclaration; + + /** Customize WebGL context attributes. See [the WebGL docs](https://developer.mozilla.org/en-US/docs/Web/API/WebGLRenderingContext/getContextAttributes) for details. */ + contextAttributes?: Record; + + /** Lerp value for `iMouse` built-in uniform. Must be between 0 and 1. */ + lerp?: number; + + /** Device pixel ratio. */ + devicePixelRatio?: number; + + /** + * Callback for when the textures are done loading. Useful if you want to do + * something like e.g. hide the canvas until textures are done loading. + */ + onDoneLoadingTextures?: () => void; + + /** Custom callback to handle errors. Defaults to `console.error`. */ + onError?: (error: string) => void; + + /** Custom callback to handle warnings. Defaults to `console.warn`. */ + onWarning?: (warning: string) => void; +}; +export class Shader extends Component { + uniforms: Record< + string, + { type: string; isNeeded: boolean; value?: number[] | number; arraySize?: string } + >; + constructor(props: Props) { + super(props); + this.uniforms = { + [UNIFORM_TIME]: { type: 'float', isNeeded: false, value: 0 }, + [UNIFORM_TIMEDELTA]: { type: 'float', isNeeded: false, value: 0 }, + [UNIFORM_DATE]: { type: 'vec4', isNeeded: false, value: [0, 0, 0, 0] }, + [UNIFORM_MOUSE]: { type: 'vec4', isNeeded: false, value: [0, 0, 0, 0] }, + [UNIFORM_RESOLUTION]: { type: 'vec2', isNeeded: false, value: [0, 0] }, + [UNIFORM_FRAME]: { type: 'int', isNeeded: false, value: 0 }, + [UNIFORM_DEVICEORIENTATION]: { type: 'vec4', isNeeded: false, value: [0, 0, 0, 0] }, + }; + } + static defaultProps = { + textures: [], + contextAttributes: {}, + devicePixelRatio: 1, + vs: BASIC_VS, + precision: 'highp', + onError: console.error, + onWarn: console.warn, + }; + componentDidMount = () => { + this.initWebGL(); + const { fs, vs, clearColor = [0, 0, 0, 1] } = this.props; + const { gl } = this; + if (gl && this.canvas) { + gl.clearColor(...clearColor); + gl.clearDepth(1.0); + gl.enable(gl.DEPTH_TEST); + gl.depthFunc(gl.LEQUAL); + gl.viewport(0, 0, this.canvas.width, this.canvas.height); + this.canvas.height = this.canvas.clientHeight; + this.canvas.width = this.canvas.clientWidth; + this.processCustomUniforms(); + this.processTextures(); + this.initShaders(this.preProcessFragment(fs || BASIC_FS), vs || BASIC_VS); + this.initBuffers(); + requestAnimationFrame(this.drawScene); + this.addEventListeners(); + this.onResize(); + } + }; + shouldComponentUpdate = () => false; + componentWillUnmount() { + const { gl } = this; + if (gl) { + gl.getExtension('WEBGL_lose_context')?.loseContext(); + gl.useProgram(null); + gl.deleteProgram(this.shaderProgram ?? null); + if (this.texturesArr.length > 0) { + for (const texture of this.texturesArr as Texture[]) { + gl.deleteTexture(texture._webglTexture); + } + } + this.shaderProgram = null; + } + this.removeEventListeners(); + cancelAnimationFrame(this.animFrameId ?? 0); + } + setupChannelRes = ({ width, height }: Texture, id: number) => { + const { devicePixelRatio = 1 } = this.props; + // @ts-expect-error TODO: Deal with this. + this.uniforms.iChannelResolution.value[id * 3] = width * devicePixelRatio; + // @ts-expect-error TODO: Deal with this. + this.uniforms.iChannelResolution.value[id * 3 + 1] = height * devicePixelRatio; + // @ts-expect-error TODO: Deal with this. + this.uniforms.iChannelResolution.value[id * 3 + 2] = 0; + // console.log(this.uniforms); + }; + initWebGL = () => { + const { contextAttributes } = this.props; + if (!this.canvas) return; + this.gl = (this.canvas.getContext('webgl', contextAttributes) || + this.canvas.getContext( + 'experimental-webgl', + contextAttributes + )) as WebGLRenderingContext | null; + this.gl?.getExtension('OES_standard_derivatives'); + this.gl?.getExtension('EXT_shader_texture_lod'); + }; + initBuffers = () => { + const { gl } = this; + this.squareVerticesBuffer = gl?.createBuffer(); + gl?.bindBuffer(gl.ARRAY_BUFFER, this.squareVerticesBuffer ?? null); + const vertices = [1.0, 1.0, 0.0, -1.0, 1.0, 0.0, 1.0, -1.0, 0.0, -1.0, -1.0, 0.0]; + gl?.bufferData(gl.ARRAY_BUFFER, new Float32Array(vertices), gl.STATIC_DRAW); + }; + addEventListeners = () => { + const options = { passive: true }; + if (this.uniforms.iMouse?.isNeeded && this.canvas) { + this.canvas.addEventListener('mousemove', this.mouseMove, options); + this.canvas.addEventListener('mouseout', this.mouseUp, options); + this.canvas.addEventListener('mouseup', this.mouseUp, options); + this.canvas.addEventListener('mousedown', this.mouseDown, options); + this.canvas.addEventListener('touchmove', this.mouseMove, options); + this.canvas.addEventListener('touchend', this.mouseUp, options); + this.canvas.addEventListener('touchstart', this.mouseDown, options); + } + if (this.uniforms.iDeviceOrientation?.isNeeded) { + window.addEventListener('deviceorientation', this.onDeviceOrientationChange, options); + } + if (this.canvas) { + this.resizeObserver = new ResizeObserver(this.onResize); + this.resizeObserver.observe(this.canvas); + } + }; + removeEventListeners = () => { + const options = { passive: true } as EventListenerOptions; + if (this.uniforms.iMouse?.isNeeded && this.canvas) { + this.canvas.removeEventListener('mousemove', this.mouseMove, options); + this.canvas.removeEventListener('mouseout', this.mouseUp, options); + this.canvas.removeEventListener('mouseup', this.mouseUp, options); + this.canvas.removeEventListener('mousedown', this.mouseDown, options); + this.canvas.removeEventListener('touchmove', this.mouseMove, options); + this.canvas.removeEventListener('touchend', this.mouseUp, options); + this.canvas.removeEventListener('touchstart', this.mouseDown, options); + } + if (this.uniforms.iDeviceOrientation?.isNeeded) { + window.removeEventListener('deviceorientation', this.onDeviceOrientationChange, options); + } + if (this.resizeObserver) { + this.resizeObserver.disconnect(); + } + }; + onDeviceOrientationChange = ({ alpha, beta, gamma }: DeviceOrientationEvent) => { + this.uniforms.iDeviceOrientation.value = [ + alpha ?? 0, + beta ?? 0, + gamma ?? 0, + window.orientation || 0, + ]; + }; + mouseDown = (e: MouseEvent | TouchEvent) => { + const [clientX, clientY] = latestPointerClientCoords(e); + const mouseX = clientX - (this.canvasPosition?.left ?? 0) - window.pageXOffset; + const mouseY = + (this.canvasPosition?.height ?? 0) - + clientY - + (this.canvasPosition?.top ?? 0) - + window.pageYOffset; + this.mousedown = true; + // @ts-expect-error TODO: Deal with this. + this.uniforms.iMouse.value[2] = mouseX; + // @ts-expect-error TODO: Deal with this. + this.uniforms.iMouse.value[3] = mouseY; + this.lastMouseArr[0] = mouseX; + this.lastMouseArr[1] = mouseY; + }; + mouseMove = (e: MouseEvent | TouchEvent) => { + this.canvasPosition = this.canvas?.getBoundingClientRect(); + const { lerp = 1 } = this.props; + const [clientX, clientY] = latestPointerClientCoords(e); + const mouseX = clientX - (this.canvasPosition?.left ?? 0); + const mouseY = (this.canvasPosition?.height ?? 0) - clientY - (this.canvasPosition?.top ?? 0); + if (lerp !== 1) { + this.lastMouseArr[0] = mouseX; + this.lastMouseArr[1] = mouseY; + } else { + // @ts-expect-error TODO: Deal with this. + this.uniforms.iMouse.value[0] = mouseX; + // @ts-expect-error TODO: Deal with this. + this.uniforms.iMouse.value[1] = mouseY; + } + }; + mouseUp = () => { + // @ts-expect-error TODO: Deal with this. + this.uniforms.iMouse.value[2] = 0; + // @ts-expect-error TODO: Deal with this. + this.uniforms.iMouse.value[3] = 0; + }; + onResize = () => { + const { gl } = this; + const { devicePixelRatio = 1 } = this.props; + if (!gl) return; + this.canvasPosition = this.canvas?.getBoundingClientRect(); + // Force pixel ratio to be one to avoid expensive calculus on retina display. + const realToCSSPixels = devicePixelRatio; + const displayWidth = Math.floor((this.canvasPosition?.width ?? 1) * realToCSSPixels); + const displayHeight = Math.floor((this.canvasPosition?.height ?? 1) * realToCSSPixels); + gl.canvas.width = displayWidth; + gl.canvas.height = displayHeight; + if (this.uniforms.iResolution?.isNeeded && this.shaderProgram) { + const rUniform = gl.getUniformLocation(this.shaderProgram, UNIFORM_RESOLUTION); + gl.uniform2fv(rUniform, [gl.canvas.width, gl.canvas.height]); + } + }; + drawScene = (timestamp: number) => { + const { gl } = this; + const { lerp = 1 } = this.props; + if (!gl) return; + gl.viewport(0, 0, gl.drawingBufferWidth, gl.drawingBufferHeight); + gl.clear(gl.COLOR_BUFFER_BIT | gl.DEPTH_BUFFER_BIT); + gl.bindBuffer(gl.ARRAY_BUFFER, this.squareVerticesBuffer ?? null); + gl.vertexAttribPointer(this.vertexPositionAttribute ?? 0, 3, gl.FLOAT, false, 0, 0); + this.setUniforms(timestamp); + gl.drawArrays(gl.TRIANGLE_STRIP, 0, 4); + if (this.uniforms.iMouse?.isNeeded && lerp !== 1) { + // @ts-expect-error TODO: Deal with this. + this.uniforms.iMouse.value[0] = lerpVal( + // @ts-expect-error TODO: Deal with this. + this.uniforms.iMouse.value[0], + this.lastMouseArr[0], + lerp + ); + // @ts-expect-error TODO: Deal with this. + this.uniforms.iMouse.value[1] = lerpVal( + // @ts-expect-error TODO: Deal with this. + this.uniforms.iMouse.value[1], + this.lastMouseArr[1], + lerp + ); + } + this.animFrameId = requestAnimationFrame(this.drawScene); + }; + createShader = (type: number, shaderCodeAsText: string) => { + const { gl } = this; + if (!gl) return null; + const shader = gl.createShader(type); + if (!shader) return null; + gl.shaderSource(shader, shaderCodeAsText); + gl.compileShader(shader); + if (!gl.getShaderParameter(shader, gl.COMPILE_STATUS)) { + this.props.onWarning?.(log(`Error compiling the shader:\n${shaderCodeAsText}`)); + const compilationLog = gl.getShaderInfoLog(shader); + gl.deleteShader(shader); + this.props.onError?.(log(`Shader compiler log: ${compilationLog}`)); + } + return shader; + }; + initShaders = (fs: string, vs: string) => { + const { gl } = this; + if (!gl) return; + // console.log(fs, vs); + const fragmentShader = this.createShader(gl.FRAGMENT_SHADER, fs); + const vertexShader = this.createShader(gl.VERTEX_SHADER, vs); + this.shaderProgram = gl.createProgram(); + if (!this.shaderProgram || !vertexShader || !fragmentShader) return; + gl.attachShader(this.shaderProgram, vertexShader); + gl.attachShader(this.shaderProgram, fragmentShader); + gl.linkProgram(this.shaderProgram); + if (!gl.getProgramParameter(this.shaderProgram, gl.LINK_STATUS)) { + this.props.onError?.( + log(`Unable to initialize the shader program: ${gl.getProgramInfoLog(this.shaderProgram)}`) + ); + return; + } + gl.useProgram(this.shaderProgram); + this.vertexPositionAttribute = gl.getAttribLocation(this.shaderProgram, 'aVertexPosition'); + gl.enableVertexAttribArray(this.vertexPositionAttribute); + }; + processCustomUniforms = () => { + const { uniforms } = this.props; + if (uniforms) { + for (const name of Object.keys(uniforms)) { + const uniform = this.props.uniforms?.[name]; + if (!uniform) return; + const { value, type } = uniform; + const glslType = uniformTypeToGLSLType(type); + if (!glslType) return; + const tempObject: { arraySize?: string } = {}; + if (isMatrixType(type, value)) { + const arrayLength = type.length; + const val = Number.parseInt(type.charAt(arrayLength - 3)); + const numberOfMatrices = Math.floor(value.length / (val * val)); + if (value.length > val * val) tempObject.arraySize = `[${numberOfMatrices}]`; + } else if (isVectorListType(type, value)) { + tempObject.arraySize = `[${Math.floor(value.length / Number.parseInt(type.charAt(0)))}]`; + } + this.uniforms[name] = { type: glslType, isNeeded: false, value, ...tempObject }; + } + } + }; + processTextures = () => { + const { gl } = this; + const { textures, onDoneLoadingTextures } = this.props; + if (!gl) return; + if (textures && textures.length > 0) { + this.uniforms[`${UNIFORM_CHANNELRESOLUTION}`] = { + type: 'vec3', + isNeeded: false, + arraySize: `[${textures.length}]`, + value: [], + }; + const texturePromisesArr = textures.map((texture: TextureParams, id: number) => { + // Dynamically add textures uniforms. + this.uniforms[`${UNIFORM_CHANNEL}${id}`] = { + type: 'sampler2D', + isNeeded: false, + }; + // Initialize array with 0s: + // @ts-expect-error TODO: Deal with this. + this.setupChannelRes(texture, id); + this.texturesArr[id] = new Texture(gl); + return ( + this.texturesArr[id] + // @ts-expect-error TODO: Deal with this. + ?.load(texture, id) + .then((t: Texture) => { + this.setupChannelRes(t, id); + }) + ); + }); + Promise.all(texturePromisesArr) + .then(() => { + if (onDoneLoadingTextures) onDoneLoadingTextures(); + }) + .catch((e) => { + this.props.onError?.(e); + if (onDoneLoadingTextures) onDoneLoadingTextures(); + }); + } else if (onDoneLoadingTextures) onDoneLoadingTextures(); + }; + preProcessFragment = (fragment: string) => { + const { precision, devicePixelRatio = 1 } = this.props; + const isValidPrecision = PRECISIONS.includes(precision ?? 'highp'); + const precisionString = `precision ${isValidPrecision ? precision : PRECISIONS[1]} float;\n`; + if (!isValidPrecision) { + this.props.onWarning?.( + log( + `wrong precision type ${precision}, please make sure to pass one of a valid precision lowp, mediump, highp, by default you shader precision will be set to highp.` + ) + ); + } + let fs = precisionString + .concat(`#define DPR ${devicePixelRatio.toFixed(1)}\n`) + .concat(fragment.replace(/texture\(/g, 'texture2D(')); + for (const uniform of Object.keys(this.uniforms)) { + if (fragment.includes(uniform)) { + const u = this.uniforms[uniform]; + if (!u) continue; + fs = insertStringAtIndex( + fs, + `uniform ${u.type} ${uniform}${u.arraySize || ''}; \n`, + fs.lastIndexOf(precisionString) + precisionString.length + ); + u.isNeeded = true; + } + } + const isShadertoy = fragment.includes('mainImage'); + if (isShadertoy) fs = fs.concat(FS_MAIN_SHADER); + return fs; + }; + setUniforms = (timestamp: number) => { + const { gl } = this; + if (!gl || !this.shaderProgram) return; + const delta = this.lastTime ? (timestamp - this.lastTime) / 1000 : 0; + this.lastTime = timestamp; + if (this.props.uniforms) { + for (const name of Object.keys(this.props.uniforms)) { + const currentUniform = this.props.uniforms?.[name]; + if (!currentUniform) return; + if (this.uniforms[name]?.isNeeded) { + if (!this.shaderProgram) return; + const customUniformLocation = gl.getUniformLocation(this.shaderProgram, name); + if (!customUniformLocation) return; + processUniform( + gl, + customUniformLocation, + currentUniform.type as UniformType, + currentUniform.value + ); + } + } + } + if (this.uniforms.iMouse?.isNeeded) { + const mouseUniform = gl.getUniformLocation(this.shaderProgram, UNIFORM_MOUSE); + gl.uniform4fv(mouseUniform, this.uniforms.iMouse.value as number[]); + } + if (this.uniforms.iChannelResolution?.isNeeded) { + const channelResUniform = gl.getUniformLocation( + this.shaderProgram, + UNIFORM_CHANNELRESOLUTION + ); + gl.uniform3fv(channelResUniform, this.uniforms.iChannelResolution.value as number[]); + } + if (this.uniforms.iDeviceOrientation?.isNeeded) { + const deviceOrientationUniform = gl.getUniformLocation( + this.shaderProgram, + UNIFORM_DEVICEORIENTATION + ); + gl.uniform4fv(deviceOrientationUniform, this.uniforms.iDeviceOrientation.value as number[]); + } + if (this.uniforms.iTime?.isNeeded) { + const timeUniform = gl.getUniformLocation(this.shaderProgram, UNIFORM_TIME); + gl.uniform1f(timeUniform, (this.timer += delta)); + } + if (this.uniforms.iTimeDelta?.isNeeded) { + const timeDeltaUniform = gl.getUniformLocation(this.shaderProgram, UNIFORM_TIMEDELTA); + gl.uniform1f(timeDeltaUniform, delta); + } + if (this.uniforms.iDate?.isNeeded) { + const d = new Date(); + const month = d.getMonth() + 1; + const day = d.getDate(); + const year = d.getFullYear(); + const time = + d.getHours() * 60 * 60 + d.getMinutes() * 60 + d.getSeconds() + d.getMilliseconds() * 0.001; + const dateUniform = gl.getUniformLocation(this.shaderProgram, UNIFORM_DATE); + gl.uniform4fv(dateUniform, [year, month, day, time]); + } + if (this.uniforms.iFrame?.isNeeded) { + const timeDeltaUniform = gl.getUniformLocation(this.shaderProgram, UNIFORM_FRAME); + gl.uniform1i(timeDeltaUniform, (this.uniforms.iFrame.value as number)++); + } + if (this.texturesArr.length > 0) { + for (let index = 0; index < this.texturesArr.length; index++) { + // TODO: Don't use this casting if possible: + const texture = this.texturesArr[index] as Texture | undefined; + if (!texture) return; + const { isVideo, _webglTexture, source, flipY, isLoaded } = texture; + if (!isLoaded || !_webglTexture || !source) return; + if (this.uniforms[`iChannel${index}`]?.isNeeded) { + if (!this.shaderProgram) return; + const iChannel = gl.getUniformLocation(this.shaderProgram, `iChannel${index}`); + // @ts-expect-error TODO: Fix. Can't index WebGL context with this dynamic value. + gl.activeTexture(gl[`TEXTURE${index}`]); + gl.bindTexture(gl.TEXTURE_2D, _webglTexture); + gl.uniform1i(iChannel, index); + if (isVideo) { + texture.updateTexture(_webglTexture, source as HTMLVideoElement, flipY); + } + } + } + } + }; + registerCanvas = (r: HTMLCanvasElement) => { + this.canvas = r; + }; + gl?: WebGLRenderingContext | null; + squareVerticesBuffer?: WebGLBuffer | null; + shaderProgram?: WebGLProgram | null; + vertexPositionAttribute?: number; + animFrameId?: number; + canvas?: HTMLCanvasElement; + mousedown = false; + canvasPosition?: DOMRect; + timer = 0; + lastMouseArr: number[] = [0, 0]; + texturesArr: WebGLTexture[] = []; + lastTime = 0; + resizeObserver?: ResizeObserver; + render = () => ( + + ); +} diff --git a/components/livekit/react-shader/react-shader.tsx b/components/livekit/react-shader/react-shader.tsx new file mode 100644 index 00000000..b5ea8b76 --- /dev/null +++ b/components/livekit/react-shader/react-shader.tsx @@ -0,0 +1,649 @@ +import { type CSSProperties, useEffect, useRef } from 'react'; +import { log } from './logging'; +import { + ClampToEdgeWrapping, + LinearFilter, + LinearMipMapLinearFilter, + LinearMipMapNearestFilter, + MirroredRepeatWrapping, + NearestFilter, + NearestMipMapLinearFilter, + NearestMipMapNearestFilter, + RepeatWrapping, + Texture, +} from './texture'; +import { + type UniformType, + type Vector2, + type Vector3, + type Vector4, + isMatrixType, + isVectorListType, + processUniform, + uniformTypeToGLSLType, +} from './uniforms'; + +export { + ClampToEdgeWrapping, + LinearFilter, + LinearMipMapLinearFilter, + LinearMipMapNearestFilter, + MirroredRepeatWrapping, + NearestFilter, + NearestMipMapLinearFilter, + NearestMipMapNearestFilter, + RepeatWrapping, +}; + +export type { Vector2, Vector3, Vector4 }; + +const PRECISIONS = ['lowp', 'mediump', 'highp']; +const FS_MAIN_SHADER = `\nvoid main(void){ + vec4 color = vec4(0.0,0.0,0.0,1.0); + mainImage( color, gl_FragCoord.xy ); + gl_FragColor = color; +}`; +const BASIC_FS = `void mainImage( out vec4 fragColor, in vec2 fragCoord ) { + vec2 uv = fragCoord/iResolution.xy; + vec3 col = 0.5 + 0.5*cos(iTime+uv.xyx+vec3(0,2,4)); + fragColor = vec4(col,1.0); +}`; +const BASIC_VS = `attribute vec3 aVertexPosition; +void main(void) { + gl_Position = vec4(aVertexPosition, 1.0); +}`; +const UNIFORM_TIME = 'iTime'; +const UNIFORM_TIMEDELTA = 'iTimeDelta'; +const UNIFORM_DATE = 'iDate'; +const UNIFORM_FRAME = 'iFrame'; +const UNIFORM_MOUSE = 'iMouse'; +const UNIFORM_RESOLUTION = 'iResolution'; +const UNIFORM_CHANNEL = 'iChannel'; +const UNIFORM_CHANNELRESOLUTION = 'iChannelResolution'; +const UNIFORM_DEVICEORIENTATION = 'iDeviceOrientation'; + +const latestPointerClientCoords = (e: MouseEvent | TouchEvent) => { + // @ts-expect-error TODO: Deal with this. + return [e.clientX || e.changedTouches[0].clientX, e.clientY || e.changedTouches[0].clientY]; +}; +const lerpVal = (v0: number, v1: number, t: number) => v0 * (1 - t) + v1 * t; +const insertStringAtIndex = (currentString: string, string: string, index: number) => + index > 0 + ? currentString.substring(0, index) + + string + + currentString.substring(index, currentString.length) + : string + currentString; + +type Uniform = { type: string; value: number[] | number }; +export type Uniforms = Record; +type TextureParams = { + url: string; + wrapS?: number; + wrapT?: number; + minFilter?: number; + magFilter?: number; + flipY?: number; +}; + +type Props = { + /** Fragment shader GLSL code. */ + fs: string; + + /** Vertex shader GLSL code. */ + vs?: string; + + /** + * Textures to be passed to the shader. Textures need to be squared or will be + * automatically resized. + * + * Options default to: + * + * ```js + * { + * minFilter: LinearMipMapLinearFilter, + * magFilter: LinearFilter, + * wrapS: RepeatWrapping, + * wrapT: RepeatWrapping, + * } + * ``` + * + * See [textures in the docs](https://rysana.com/docs/react-shaders#textures) + * for details. + */ + textures?: TextureParams[]; + + /** + * Custom uniforms to be passed to the shader. + * + * See [custom uniforms in the + * docs](https://rysana.com/docs/react-shaders#custom-uniforms) for details. + */ + uniforms?: Uniforms; + + /** + * Color used when clearing the canvas. + * + * See [the WebGL + * docs](https://developer.mozilla.org/en-US/docs/Web/API/WebGLRenderingContext/clearColor) + * for details. + */ + clearColor?: Vector4; + + /** + * GLSL precision qualifier. Defaults to `'highp'`. Balance between + * performance and quality. + */ + precision?: 'highp' | 'lowp' | 'mediump'; + + /** Custom inline style for canvas. */ + style?: CSSStyleDeclaration; + + /** Customize WebGL context attributes. See [the WebGL docs](https://developer.mozilla.org/en-US/docs/Web/API/WebGLRenderingContext/getContextAttributes) for details. */ + contextAttributes?: Record; + + /** Lerp value for `iMouse` built-in uniform. Must be between 0 and 1. */ + lerp?: number; + + /** Device pixel ratio. */ + devicePixelRatio?: number; + + /** + * Callback for when the textures are done loading. Useful if you want to do + * something like e.g. hide the canvas until textures are done loading. + */ + onDoneLoadingTextures?: () => void; + + /** Custom callback to handle errors. Defaults to `console.error`. */ + onError?: (error: string) => void; + + /** Custom callback to handle warnings. Defaults to `console.warn`. */ + onWarning?: (warning: string) => void; +}; +export const Shader = ({ + fs, + vs = BASIC_VS, + textures = [], + uniforms: propUniforms, + clearColor = [0, 0, 0, 1], + precision = 'highp', + style, + contextAttributes = {}, + lerp = 1, + devicePixelRatio = 1, + onDoneLoadingTextures, + onError = console.error, + onWarning = console.warn, +}: Props) => { + // Refs for WebGL state + const canvasRef = useRef(null); + const glRef = useRef(null); + const squareVerticesBufferRef = useRef(null); + const shaderProgramRef = useRef(null); + const vertexPositionAttributeRef = useRef(undefined); + const animFrameIdRef = useRef(undefined); + const mousedownRef = useRef(false); + const canvasPositionRef = useRef(undefined); + const timerRef = useRef(0); + const lastMouseArrRef = useRef([0, 0]); + const texturesArrRef = useRef([]); + const lastTimeRef = useRef(0); + const resizeObserverRef = useRef(undefined); + const uniformsRef = useRef< + Record< + string, + { type: string; isNeeded: boolean; value?: number[] | number; arraySize?: string } + > + >({ + [UNIFORM_TIME]: { type: 'float', isNeeded: false, value: 0 }, + [UNIFORM_TIMEDELTA]: { type: 'float', isNeeded: false, value: 0 }, + [UNIFORM_DATE]: { type: 'vec4', isNeeded: false, value: [0, 0, 0, 0] }, + [UNIFORM_MOUSE]: { type: 'vec4', isNeeded: false, value: [0, 0, 0, 0] }, + [UNIFORM_RESOLUTION]: { type: 'vec2', isNeeded: false, value: [0, 0] }, + [UNIFORM_FRAME]: { type: 'int', isNeeded: false, value: 0 }, + [UNIFORM_DEVICEORIENTATION]: { type: 'vec4', isNeeded: false, value: [0, 0, 0, 0] }, + }); + const propsUniformsRef = useRef(propUniforms); + + const setupChannelRes = ({ width, height }: Texture, id: number) => { + // @ts-expect-error TODO: Deal with this. + uniformsRef.current.iChannelResolution.value[id * 3] = width * devicePixelRatio; + // @ts-expect-error TODO: Deal with this. + uniformsRef.current.iChannelResolution.value[id * 3 + 1] = height * devicePixelRatio; + // @ts-expect-error TODO: Deal with this. + uniformsRef.current.iChannelResolution.value[id * 3 + 2] = 0; + }; + + const initWebGL = () => { + if (!canvasRef.current) return; + glRef.current = (canvasRef.current.getContext('webgl', contextAttributes) || + canvasRef.current.getContext( + 'experimental-webgl', + contextAttributes + )) as WebGLRenderingContext | null; + glRef.current?.getExtension('OES_standard_derivatives'); + glRef.current?.getExtension('EXT_shader_texture_lod'); + }; + + const initBuffers = () => { + const gl = glRef.current; + squareVerticesBufferRef.current = gl?.createBuffer() ?? null; + gl?.bindBuffer(gl.ARRAY_BUFFER, squareVerticesBufferRef.current); + const vertices = [1.0, 1.0, 0.0, -1.0, 1.0, 0.0, 1.0, -1.0, 0.0, -1.0, -1.0, 0.0]; + gl?.bufferData(gl.ARRAY_BUFFER, new Float32Array(vertices), gl.STATIC_DRAW); + }; + + const onDeviceOrientationChange = ({ alpha, beta, gamma }: DeviceOrientationEvent) => { + uniformsRef.current.iDeviceOrientation.value = [ + alpha ?? 0, + beta ?? 0, + gamma ?? 0, + window.orientation || 0, + ]; + }; + + const mouseDown = (e: MouseEvent | TouchEvent) => { + const [clientX, clientY] = latestPointerClientCoords(e); + const mouseX = clientX - (canvasPositionRef.current?.left ?? 0) - window.pageXOffset; + const mouseY = + (canvasPositionRef.current?.height ?? 0) - + clientY - + (canvasPositionRef.current?.top ?? 0) - + window.pageYOffset; + mousedownRef.current = true; + // @ts-expect-error TODO: Deal with this. + uniformsRef.current.iMouse.value[2] = mouseX; + // @ts-expect-error TODO: Deal with this. + uniformsRef.current.iMouse.value[3] = mouseY; + lastMouseArrRef.current[0] = mouseX; + lastMouseArrRef.current[1] = mouseY; + }; + + const mouseMove = (e: MouseEvent | TouchEvent) => { + canvasPositionRef.current = canvasRef.current?.getBoundingClientRect(); + const [clientX, clientY] = latestPointerClientCoords(e); + const mouseX = clientX - (canvasPositionRef.current?.left ?? 0); + const mouseY = + (canvasPositionRef.current?.height ?? 0) - clientY - (canvasPositionRef.current?.top ?? 0); + if (lerp !== 1) { + lastMouseArrRef.current[0] = mouseX; + lastMouseArrRef.current[1] = mouseY; + } else { + // @ts-expect-error TODO: Deal with this. + uniformsRef.current.iMouse.value[0] = mouseX; + // @ts-expect-error TODO: Deal with this. + uniformsRef.current.iMouse.value[1] = mouseY; + } + }; + + const mouseUp = () => { + // @ts-expect-error TODO: Deal with this. + uniformsRef.current.iMouse.value[2] = 0; + // @ts-expect-error TODO: Deal with this. + uniformsRef.current.iMouse.value[3] = 0; + }; + + const onResize = () => { + const gl = glRef.current; + if (!gl) return; + canvasPositionRef.current = canvasRef.current?.getBoundingClientRect(); + // Force pixel ratio to be one to avoid expensive calculus on retina display. + const realToCSSPixels = devicePixelRatio; + const displayWidth = Math.floor((canvasPositionRef.current?.width ?? 1) * realToCSSPixels); + const displayHeight = Math.floor((canvasPositionRef.current?.height ?? 1) * realToCSSPixels); + gl.canvas.width = displayWidth; + gl.canvas.height = displayHeight; + if (uniformsRef.current.iResolution?.isNeeded && shaderProgramRef.current) { + const rUniform = gl.getUniformLocation(shaderProgramRef.current, UNIFORM_RESOLUTION); + gl.uniform2fv(rUniform, [gl.canvas.width, gl.canvas.height]); + } + }; + + const createShader = (type: number, shaderCodeAsText: string) => { + const gl = glRef.current; + if (!gl) return null; + const shader = gl.createShader(type); + if (!shader) return null; + gl.shaderSource(shader, shaderCodeAsText); + gl.compileShader(shader); + if (!gl.getShaderParameter(shader, gl.COMPILE_STATUS)) { + onWarning?.(log(`Error compiling the shader:\n${shaderCodeAsText}`)); + const compilationLog = gl.getShaderInfoLog(shader); + gl.deleteShader(shader); + onError?.(log(`Shader compiler log: ${compilationLog}`)); + } + return shader; + }; + + const initShaders = (fragmentShader: string, vertexShader: string) => { + const gl = glRef.current; + if (!gl) return; + const fragmentShaderObj = createShader(gl.FRAGMENT_SHADER, fragmentShader); + const vertexShaderObj = createShader(gl.VERTEX_SHADER, vertexShader); + shaderProgramRef.current = gl.createProgram(); + if (!shaderProgramRef.current || !vertexShaderObj || !fragmentShaderObj) return; + gl.attachShader(shaderProgramRef.current, vertexShaderObj); + gl.attachShader(shaderProgramRef.current, fragmentShaderObj); + gl.linkProgram(shaderProgramRef.current); + if (!gl.getProgramParameter(shaderProgramRef.current, gl.LINK_STATUS)) { + onError?.( + log( + `Unable to initialize the shader program: ${gl.getProgramInfoLog(shaderProgramRef.current)}` + ) + ); + return; + } + gl.useProgram(shaderProgramRef.current); + vertexPositionAttributeRef.current = gl.getAttribLocation( + shaderProgramRef.current, + 'aVertexPosition' + ); + gl.enableVertexAttribArray(vertexPositionAttributeRef.current); + }; + + const processCustomUniforms = () => { + if (propUniforms) { + for (const name of Object.keys(propUniforms)) { + const uniform = propUniforms[name]; + if (!uniform) return; + const { value, type } = uniform; + const glslType = uniformTypeToGLSLType(type); + if (!glslType) return; + const tempObject: { arraySize?: string } = {}; + if (isMatrixType(type, value)) { + const arrayLength = type.length; + const val = Number.parseInt(type.charAt(arrayLength - 3)); + const numberOfMatrices = Math.floor(value.length / (val * val)); + if (value.length > val * val) tempObject.arraySize = `[${numberOfMatrices}]`; + } else if (isVectorListType(type, value)) { + tempObject.arraySize = `[${Math.floor(value.length / Number.parseInt(type.charAt(0)))}]`; + } + uniformsRef.current[name] = { type: glslType, isNeeded: false, value, ...tempObject }; + } + } + }; + + const processTextures = () => { + const gl = glRef.current; + if (!gl) return; + if (textures && textures.length > 0) { + uniformsRef.current[`${UNIFORM_CHANNELRESOLUTION}`] = { + type: 'vec3', + isNeeded: false, + arraySize: `[${textures.length}]`, + value: [], + }; + const texturePromisesArr = textures.map((texture: TextureParams, id: number) => { + // Dynamically add textures uniforms. + uniformsRef.current[`${UNIFORM_CHANNEL}${id}`] = { + type: 'sampler2D', + isNeeded: false, + }; + // Initialize array with 0s: + // @ts-expect-error TODO: Deal with this. + setupChannelRes(texture, id); + texturesArrRef.current[id] = new Texture(gl); + return ( + texturesArrRef.current[id] + // @ts-expect-error TODO: Deal with this. + ?.load(texture, id) + .then((t: Texture) => { + setupChannelRes(t, id); + }) + ); + }); + Promise.all(texturePromisesArr) + .then(() => { + if (onDoneLoadingTextures) onDoneLoadingTextures(); + }) + .catch((e) => { + onError?.(e); + if (onDoneLoadingTextures) onDoneLoadingTextures(); + }); + } else if (onDoneLoadingTextures) onDoneLoadingTextures(); + }; + + const preProcessFragment = (fragment: string) => { + const isValidPrecision = PRECISIONS.includes(precision ?? 'highp'); + const precisionString = `precision ${isValidPrecision ? precision : PRECISIONS[1]} float;\n`; + if (!isValidPrecision) { + onWarning?.( + log( + `wrong precision type ${precision}, please make sure to pass one of a valid precision lowp, mediump, highp, by default you shader precision will be set to highp.` + ) + ); + } + let fragmentShader = precisionString + .concat(`#define DPR ${devicePixelRatio.toFixed(1)}\n`) + .concat(fragment.replace(/texture\(/g, 'texture2D(')); + for (const uniform of Object.keys(uniformsRef.current)) { + if (fragment.includes(uniform)) { + const u = uniformsRef.current[uniform]; + if (!u) continue; + fragmentShader = insertStringAtIndex( + fragmentShader, + `uniform ${u.type} ${uniform}${u.arraySize || ''}; \n`, + fragmentShader.lastIndexOf(precisionString) + precisionString.length + ); + u.isNeeded = true; + } + } + const isShadertoy = fragment.includes('mainImage'); + if (isShadertoy) fragmentShader = fragmentShader.concat(FS_MAIN_SHADER); + return fragmentShader; + }; + + const setUniforms = (timestamp: number) => { + const gl = glRef.current; + if (!gl || !shaderProgramRef.current) return; + const delta = lastTimeRef.current ? (timestamp - lastTimeRef.current) / 1000 : 0; + lastTimeRef.current = timestamp; + const propUniforms = propsUniformsRef.current; + if (propUniforms) { + for (const name of Object.keys(propUniforms)) { + const currentUniform = propUniforms[name]; + if (!currentUniform) return; + if (uniformsRef.current[name]?.isNeeded) { + if (!shaderProgramRef.current) return; + const customUniformLocation = gl.getUniformLocation(shaderProgramRef.current, name); + if (!customUniformLocation) return; + processUniform( + gl, + customUniformLocation, + currentUniform.type as UniformType, + currentUniform.value + ); + } + } + } + if (uniformsRef.current.iMouse?.isNeeded) { + const mouseUniform = gl.getUniformLocation(shaderProgramRef.current, UNIFORM_MOUSE); + gl.uniform4fv(mouseUniform, uniformsRef.current.iMouse.value as number[]); + } + if (uniformsRef.current.iChannelResolution?.isNeeded) { + const channelResUniform = gl.getUniformLocation( + shaderProgramRef.current, + UNIFORM_CHANNELRESOLUTION + ); + gl.uniform3fv(channelResUniform, uniformsRef.current.iChannelResolution.value as number[]); + } + if (uniformsRef.current.iDeviceOrientation?.isNeeded) { + const deviceOrientationUniform = gl.getUniformLocation( + shaderProgramRef.current, + UNIFORM_DEVICEORIENTATION + ); + gl.uniform4fv( + deviceOrientationUniform, + uniformsRef.current.iDeviceOrientation.value as number[] + ); + } + if (uniformsRef.current.iTime?.isNeeded) { + const timeUniform = gl.getUniformLocation(shaderProgramRef.current, UNIFORM_TIME); + gl.uniform1f(timeUniform, (timerRef.current += delta)); + } + if (uniformsRef.current.iTimeDelta?.isNeeded) { + const timeDeltaUniform = gl.getUniformLocation(shaderProgramRef.current, UNIFORM_TIMEDELTA); + gl.uniform1f(timeDeltaUniform, delta); + } + if (uniformsRef.current.iDate?.isNeeded) { + const d = new Date(); + const month = d.getMonth() + 1; + const day = d.getDate(); + const year = d.getFullYear(); + const time = + d.getHours() * 60 * 60 + d.getMinutes() * 60 + d.getSeconds() + d.getMilliseconds() * 0.001; + const dateUniform = gl.getUniformLocation(shaderProgramRef.current, UNIFORM_DATE); + gl.uniform4fv(dateUniform, [year, month, day, time]); + } + if (uniformsRef.current.iFrame?.isNeeded) { + const timeDeltaUniform = gl.getUniformLocation(shaderProgramRef.current, UNIFORM_FRAME); + gl.uniform1i(timeDeltaUniform, (uniformsRef.current.iFrame.value as number)++); + } + if (texturesArrRef.current.length > 0) { + for (let index = 0; index < texturesArrRef.current.length; index++) { + // TODO: Don't use this casting if possible: + const texture = texturesArrRef.current[index] as Texture | undefined; + if (!texture) return; + const { isVideo, _webglTexture, source, flipY, isLoaded } = texture; + if (!isLoaded || !_webglTexture || !source) return; + if (uniformsRef.current[`iChannel${index}`]?.isNeeded) { + if (!shaderProgramRef.current) return; + const iChannel = gl.getUniformLocation(shaderProgramRef.current, `iChannel${index}`); + // @ts-expect-error TODO: Fix. Can't index WebGL context with this dynamic value. + gl.activeTexture(gl[`TEXTURE${index}`]); + gl.bindTexture(gl.TEXTURE_2D, _webglTexture); + gl.uniform1i(iChannel, index); + if (isVideo) { + texture.updateTexture(_webglTexture, source as HTMLVideoElement, flipY); + } + } + } + } + }; + + const drawScene = (timestamp: number) => { + const gl = glRef.current; + if (!gl) return; + gl.viewport(0, 0, gl.drawingBufferWidth, gl.drawingBufferHeight); + gl.clear(gl.COLOR_BUFFER_BIT | gl.DEPTH_BUFFER_BIT); + gl.bindBuffer(gl.ARRAY_BUFFER, squareVerticesBufferRef.current); + gl.vertexAttribPointer(vertexPositionAttributeRef.current ?? 0, 3, gl.FLOAT, false, 0, 0); + setUniforms(timestamp); + gl.drawArrays(gl.TRIANGLE_STRIP, 0, 4); + if (uniformsRef.current.iMouse?.isNeeded && lerp !== 1) { + // @ts-expect-error TODO: Deal with this. + uniformsRef.current.iMouse.value[0] = lerpVal( + // @ts-expect-error TODO: Deal with this. + uniformsRef.current.iMouse.value[0], + lastMouseArrRef.current[0], + lerp + ); + // @ts-expect-error TODO: Deal with this. + uniformsRef.current.iMouse.value[1] = lerpVal( + // @ts-expect-error TODO: Deal with this. + uniformsRef.current.iMouse.value[1], + lastMouseArrRef.current[1], + lerp + ); + } + animFrameIdRef.current = requestAnimationFrame(drawScene); + }; + + const addEventListeners = () => { + const options = { passive: true }; + if (uniformsRef.current.iMouse?.isNeeded && canvasRef.current) { + canvasRef.current.addEventListener('mousemove', mouseMove, options); + canvasRef.current.addEventListener('mouseout', mouseUp, options); + canvasRef.current.addEventListener('mouseup', mouseUp, options); + canvasRef.current.addEventListener('mousedown', mouseDown, options); + canvasRef.current.addEventListener('touchmove', mouseMove, options); + canvasRef.current.addEventListener('touchend', mouseUp, options); + canvasRef.current.addEventListener('touchstart', mouseDown, options); + } + if (uniformsRef.current.iDeviceOrientation?.isNeeded) { + window.addEventListener('deviceorientation', onDeviceOrientationChange, options); + } + if (canvasRef.current) { + resizeObserverRef.current = new ResizeObserver(onResize); + resizeObserverRef.current.observe(canvasRef.current); + window.addEventListener('resize', onResize, options); + } + }; + + const removeEventListeners = () => { + const options = { passive: true } as EventListenerOptions; + if (uniformsRef.current.iMouse?.isNeeded && canvasRef.current) { + canvasRef.current.removeEventListener('mousemove', mouseMove, options); + canvasRef.current.removeEventListener('mouseout', mouseUp, options); + canvasRef.current.removeEventListener('mouseup', mouseUp, options); + canvasRef.current.removeEventListener('mousedown', mouseDown, options); + canvasRef.current.removeEventListener('touchmove', mouseMove, options); + canvasRef.current.removeEventListener('touchend', mouseUp, options); + canvasRef.current.removeEventListener('touchstart', mouseDown, options); + } + if (uniformsRef.current.iDeviceOrientation?.isNeeded) { + window.removeEventListener('deviceorientation', onDeviceOrientationChange, options); + } + if (resizeObserverRef.current) { + resizeObserverRef.current.disconnect(); + window.removeEventListener('resize', onResize, options); + } + }; + + useEffect(() => { + propsUniformsRef.current = propUniforms; + }, [propUniforms]); + + // Main effect for initialization and cleanup + useEffect(() => { + const textures = texturesArrRef.current; + + function init() { + initWebGL(); + const gl = glRef.current; + if (gl && canvasRef.current) { + gl.clearColor(...clearColor); + gl.clearDepth(1.0); + gl.enable(gl.DEPTH_TEST); + gl.depthFunc(gl.LEQUAL); + gl.viewport(0, 0, canvasRef.current.width, canvasRef.current.height); + console.log('canvasRef.current', canvasRef.current); + console.log('canvasRef.current.width', canvasRef.current.width); + console.log('canvasRef.current.height', canvasRef.current.height); + canvasRef.current.height = canvasRef.current.clientHeight; + canvasRef.current.width = canvasRef.current.clientWidth; + processCustomUniforms(); + processTextures(); + initShaders(preProcessFragment(fs || BASIC_FS), vs || BASIC_VS); + initBuffers(); + requestAnimationFrame(drawScene); + addEventListeners(); + onResize(); + } + } + + requestAnimationFrame(init); + + // Cleanup function + return () => { + const gl = glRef.current; + if (gl) { + gl.getExtension('WEBGL_lose_context')?.loseContext(); + gl.useProgram(null); + gl.deleteProgram(shaderProgramRef.current ?? null); + if (textures.length > 0) { + for (const texture of textures as Texture[]) { + gl.deleteTexture(texture._webglTexture); + } + } + shaderProgramRef.current = null; + } + removeEventListeners(); + cancelAnimationFrame(animFrameIdRef.current ?? 0); + }; + // eslint-disable-next-line react-hooks/exhaustive-deps + }, []); // Empty dependency array to run only once on mount + + return ( + + ); +}; diff --git a/components/livekit/react-shader/texture.ts b/components/livekit/react-shader/texture.ts new file mode 100644 index 00000000..19282c17 --- /dev/null +++ b/components/livekit/react-shader/texture.ts @@ -0,0 +1,197 @@ +import { log } from './logging'; + +export const LinearFilter = 9729; +export const NearestFilter = 9728; +export const LinearMipMapLinearFilter = 9987; +export const NearestMipMapLinearFilter = 9986; +export const LinearMipMapNearestFilter = 9985; +export const NearestMipMapNearestFilter = 9984; +export const MirroredRepeatWrapping = 33648; +export const ClampToEdgeWrapping = 33071; +export const RepeatWrapping = 10497; + +export class Texture { + gl: WebGLRenderingContext; + url?: string; + wrapS?: number; + wrapT?: number; + minFilter?: number; + magFilter?: number; + source?: HTMLImageElement | HTMLVideoElement; + pow2canvas?: HTMLCanvasElement; + isLoaded = false; + isVideo = false; + flipY = -1; + width = 0; + height = 0; + _webglTexture: WebGLTexture | null = null; + constructor(gl: WebGLRenderingContext) { + this.gl = gl; + } + updateTexture = (texture: WebGLTexture, video: HTMLVideoElement, flipY: number) => { + const { gl } = this; + const level = 0; + const internalFormat = gl.RGBA; + const srcFormat = gl.RGBA; + const srcType = gl.UNSIGNED_BYTE; + gl.bindTexture(gl.TEXTURE_2D, texture); + gl.pixelStorei(gl.UNPACK_FLIP_Y_WEBGL, flipY); + gl.texImage2D(gl.TEXTURE_2D, level, internalFormat, srcFormat, srcType, video); + }; + setupVideo = (url: string) => { + const video = document.createElement('video'); + let playing = false; + let timeupdate = false; + video.autoplay = true; + video.muted = true; + video.loop = true; + video.crossOrigin = 'anonymous'; + const checkReady = () => { + if (playing && timeupdate) { + this.isLoaded = true; + } + }; + video.addEventListener( + 'playing', + () => { + playing = true; + this.width = video.videoWidth || 0; + this.height = video.videoHeight || 0; + checkReady(); + }, + true + ); + video.addEventListener( + 'timeupdate', + () => { + timeupdate = true; + checkReady(); + }, + true + ); + video.src = url; + // video.play(); // Not sure why this is here nor commented out. From STR. + return video; + }; + makePowerOf2 = (image: T): T => { + if ( + image instanceof HTMLImageElement || + image instanceof HTMLCanvasElement || + image instanceof ImageBitmap + ) { + if (this.pow2canvas === undefined) this.pow2canvas = document.createElement('canvas'); + this.pow2canvas.width = 2 ** Math.floor(Math.log(image.width) / Math.LN2); + this.pow2canvas.height = 2 ** Math.floor(Math.log(image.height) / Math.LN2); + const context = this.pow2canvas.getContext('2d'); + context?.drawImage(image, 0, 0, this.pow2canvas.width, this.pow2canvas.height); + console.warn( + log( + `Image is not power of two ${image.width} x ${image.height}. Resized to ${this.pow2canvas.width} x ${this.pow2canvas.height};` + ) + ); + return this.pow2canvas as T; + } + return image; + }; + load = async ( + textureArgs: Texture + // channelId: number // Not sure why this is here nor commented out. From STR. + ) => { + const { gl } = this; + const { url, wrapS, wrapT, minFilter, magFilter, flipY = -1 }: Texture = textureArgs; + if (!url) { + return Promise.reject( + new Error(log('Missing url, please make sure to pass the url of your texture { url: ... }')) + ); + } + const isImage = /(\.jpg|\.jpeg|\.png|\.gif|\.bmp)$/i.exec(url); + const isVideo = /(\.mp4|\.3gp|\.webm|\.ogv)$/i.exec(url); + if (isImage === null && isVideo === null) { + return Promise.reject( + new Error(log(`Please upload a video or an image with a valid format (url: ${url})`)) + ); + } + Object.assign(this, { url, wrapS, wrapT, minFilter, magFilter, flipY }); + const level = 0; + const internalFormat = gl.RGBA; + const width = 1; + const height = 1; + const border = 0; + const srcFormat = gl.RGBA; + const srcType = gl.UNSIGNED_BYTE; + const pixel = new Uint8Array([255, 255, 255, 0]); + const texture = gl.createTexture(); + gl.bindTexture(gl.TEXTURE_2D, texture); + gl.texImage2D( + gl.TEXTURE_2D, + level, + internalFormat, + width, + height, + border, + srcFormat, + srcType, + pixel + ); + if (isVideo) { + const video = this.setupVideo(url); + gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE); + gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE); + gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.LINEAR); + this._webglTexture = texture; + this.source = video; + this.isVideo = true; + return video.play().then(() => this); + } + async function loadImage() { + return new Promise((resolve, reject) => { + const image = new Image(); + image.crossOrigin = 'anonymous'; + image.onload = () => { + resolve(image); + }; + image.onerror = () => { + reject(new Error(log(`failed loading url: ${url}`))); + }; + image.src = url ?? ''; + }); + } + let image = (await loadImage()) as HTMLImageElement; + let isPowerOf2 = + (image.width & (image.width - 1)) === 0 && (image.height & (image.height - 1)) === 0; + if ( + (textureArgs.wrapS !== ClampToEdgeWrapping || + textureArgs.wrapT !== ClampToEdgeWrapping || + (textureArgs.minFilter !== NearestFilter && textureArgs.minFilter !== LinearFilter)) && + !isPowerOf2 + ) { + image = this.makePowerOf2(image); + isPowerOf2 = true; + } + gl.bindTexture(gl.TEXTURE_2D, texture); + gl.pixelStorei(gl.UNPACK_FLIP_Y_WEBGL, flipY); + gl.texImage2D(gl.TEXTURE_2D, level, internalFormat, srcFormat, srcType, image); + if ( + isPowerOf2 && + textureArgs.minFilter !== NearestFilter && + textureArgs.minFilter !== LinearFilter + ) { + gl.generateMipmap(gl.TEXTURE_2D); + } + gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, this.wrapS || RepeatWrapping); + gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, this.wrapT || RepeatWrapping); + gl.texParameteri( + gl.TEXTURE_2D, + gl.TEXTURE_MIN_FILTER, + this.minFilter || LinearMipMapLinearFilter + ); + gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, this.magFilter || LinearFilter); + this._webglTexture = texture; + this.source = image; + this.isVideo = false; + this.isLoaded = true; + this.width = image.width || 0; + this.height = image.height || 0; + return this; + }; +} diff --git a/components/livekit/react-shader/uniforms.ts b/components/livekit/react-shader/uniforms.ts new file mode 100644 index 00000000..39405a9a --- /dev/null +++ b/components/livekit/react-shader/uniforms.ts @@ -0,0 +1,145 @@ +import { log } from './logging'; + +export type Vector2 = [T, T]; +export type Vector3 = [T, T, T]; +export type Vector4 = [T, T, T, T]; +// biome-ignore format: +export type Matrix2 = [T, T, T, T]; +// biome-ignore format: +export type Matrix3 = [T, T, T, T, T, T, T, T, T]; +// biome-ignore format: +export type Matrix4 = [T, T, T, T, T, T, T, T, T, T, T, T, T, T, T, T]; +export type Uniforms = { + '1i': number; + '2i': Vector2; + '3i': Vector3; + '4i': Vector4; + '1f': number; + '2f': Vector2; + '3f': Vector3; + '4f': Vector4; + '1iv': Float32List; + '2iv': Float32List; + '3iv': Float32List; + '4iv': Float32List; + '1fv': Float32List; + '2fv': Float32List; + '3fv': Float32List; + '4fv': Float32List; + Matrix2fv: Float32List; + Matrix3fv: Float32List; + Matrix4fv: Float32List; +}; +export type UniformType = keyof Uniforms; + +export function isMatrixType(t: string, v: number[] | number): v is number[] { + return t.includes('Matrix') && Array.isArray(v); +} +export function isVectorListType(t: string, v: number[] | number): v is number[] { + return t.includes('v') && Array.isArray(v) && v.length > Number.parseInt(t.charAt(0)); +} +function isVectorType(t: string, v: number[] | number): v is Vector4 { + return !t.includes('v') && Array.isArray(v) && v.length > Number.parseInt(t.charAt(0)); +} +export const processUniform = ( + gl: WebGLRenderingContext, + location: WebGLUniformLocation, + t: T, + value: number | number[] +) => { + if (isVectorType(t, value)) { + switch (t) { + case '2f': + return gl.uniform2f(location, value[0], value[1]); + case '3f': + return gl.uniform3f(location, value[0], value[1], value[2]); + case '4f': + return gl.uniform4f(location, value[0], value[1], value[2], value[3]); + case '2i': + return gl.uniform2i(location, value[0], value[1]); + case '3i': + return gl.uniform3i(location, value[0], value[1], value[2]); + case '4i': + return gl.uniform4i(location, value[0], value[1], value[2], value[3]); + } + } + if (typeof value === 'number') { + switch (t) { + case '1i': + return gl.uniform1i(location, value); + default: + return gl.uniform1f(location, value); + } + } + switch (t) { + case '1iv': + return gl.uniform1iv(location, value); + case '2iv': + return gl.uniform2iv(location, value); + case '3iv': + return gl.uniform3iv(location, value); + case '4iv': + return gl.uniform4iv(location, value); + case '1fv': + return gl.uniform1fv(location, value); + case '2fv': + return gl.uniform2fv(location, value); + case '3fv': + return gl.uniform3fv(location, value); + case '4fv': + return gl.uniform4fv(location, value); + case 'Matrix2fv': + return gl.uniformMatrix2fv(location, false, value); + case 'Matrix3fv': + return gl.uniformMatrix3fv(location, false, value); + case 'Matrix4fv': + return gl.uniformMatrix4fv(location, false, value); + } +}; + +export const uniformTypeToGLSLType = (t: string) => { + switch (t) { + case '1f': + return 'float'; + case '2f': + return 'vec2'; + case '3f': + return 'vec3'; + case '4f': + return 'vec4'; + case '1i': + return 'int'; + case '2i': + return 'ivec2'; + case '3i': + return 'ivec3'; + case '4i': + return 'ivec4'; + case '1iv': + return 'int'; + case '2iv': + return 'ivec2'; + case '3iv': + return 'ivec3'; + case '4iv': + return 'ivec4'; + case '1fv': + return 'float'; + case '2fv': + return 'vec2'; + case '3fv': + return 'vec3'; + case '4fv': + return 'vec4'; + case 'Matrix2fv': + return 'mat2'; + case 'Matrix3fv': + return 'mat3'; + case 'Matrix4fv': + return 'mat4'; + default: + console.error( + log(`The uniform type "${t}" is not valid, please make sure your uniform type is valid`) + ); + } +}; diff --git a/components/ui/shadcn-io/aurora-shaders/aurora-shader.glsl b/components/ui/shadcn-io/aurora-shaders/aurora-shader.glsl new file mode 100644 index 00000000..9d089c6e --- /dev/null +++ b/components/ui/shadcn-io/aurora-shaders/aurora-shader.glsl @@ -0,0 +1,111 @@ +const float TAU = 6.28318530718; + +// Cosine palette generator +vec3 pal(in float t, in vec3 a, in vec3 b, in vec3 c, in vec3 d) { + return a + b * cos(TAU * (c * t + d)); +} + +// Noise for dithering +vec2 randFibo(vec2 p) { + p = fract(p * vec2(443.897, 441.423)); + p += dot(p, p.yx + 19.19); + return fract((p.xx + p.yx) * p.xy); +} + +// Tonemap +vec3 Tonemap_Reinhard(vec3 x) { + x *= 4.0; + return x / (1.0 + x); +} + +// Luma for alpha +float luma(vec3 color) { + return dot(color, vec3(0.299, 0.587, 0.114)); +} + +// SDF shapes +float sdCircle(vec2 st, float r) { + return length(st) - r; +} + +float sdLine(vec2 p, float r) { + float halfLen = r * 2.0; + vec2 a = vec2(-halfLen, 0.0); + vec2 b = vec2(halfLen, 0.0); + vec2 pa = p - a; + vec2 ba = b - a; + float h = clamp(dot(pa, ba) / dot(ba, ba), 0.0, 1.0); + return length(pa - ba * h); +} + +float getSdf(vec2 st) { + if(uShape == 1) return sdCircle(st, uScale); + else if(uShape == 2) return sdLine(st, uScale); + return sdCircle(st, uScale); // Default +} + +vec2 turb(vec2 pos, float t, float it) { + mat2 rot = mat2(0.6, -0.8, 0.8, 0.6); + float freq = mix(2.0, 15.0, uFrequency); + float amp = uAmplitude; + float xp = 1.4; + float time = t * 0.1 * uSpeed; + + for(float i = 0.0; i < 4.0; i++) { + vec2 s = sin(freq * (pos * rot) + i * time + it); + pos += amp * rot[0] * s / freq; + rot *= mat2(0.6, -0.8, 0.8, 0.6); + amp *= mix(1.0, max(s.y, s.x), uVariance); + freq *= xp; + } + + return pos; +} + +const float ITERATIONS = 36.0; + +void mainImage(out vec4 fragColor, in vec2 fragCoord) { + vec2 uv = fragCoord / iResolution.xy; + + vec3 pp = vec3(0.0); + vec3 bloom = vec3(0.0); + float t = iTime * 0.5; + vec2 pos = uv - 0.5; + + vec2 prevPos = turb(pos, t, 0.0 - 1.0 / ITERATIONS); + float spacing = mix(1.0, TAU, uSpacing); + + for(float i = 1.0; i < ITERATIONS + 1.0; i++) { + float iter = i / ITERATIONS; + vec2 st = turb(pos, t, iter * spacing); + float d = abs(getSdf(st)); + float pd = distance(st, prevPos); + prevPos = st; + float dynamicBlur = exp2(pd * 2.0 * 1.4426950408889634) - 1.0; + float ds = smoothstep(0.0, uBlur * 0.05 + max(dynamicBlur * uSmoothing, 0.001), d); + + // Generate color using cosine palette + vec3 color = pal( + iter * mix(0.0, 3.0, uColorScale) + uColorPosition * 2.0, + vec3(0.5), + vec3(0.5), + vec3(1.0), + vec3(0.0, 0.33, 0.67) + ); + + float invd = 1.0 / max(d + dynamicBlur, 0.001); + pp += (ds - 1.0) * color; + bloom += clamp(invd, 0.0, 250.0) * color; + } + + pp *= 1.0 / ITERATIONS; + bloom = bloom / (bloom + 2e4); + + vec3 color = (-pp + bloom * 3.0 * uBloom); + color *= 1.2; + color += (randFibo(fragCoord).x - 0.5) / 255.0; + color = Tonemap_Reinhard(color); + + float alpha = luma(color) * uMix; + fragColor = vec4(color * uMix, alpha); +} \ No newline at end of file diff --git a/components/ui/shadcn-io/aurora-shaders/aurora-shader.original.glsl b/components/ui/shadcn-io/aurora-shaders/aurora-shader.original.glsl new file mode 100644 index 00000000..cdd061d4 --- /dev/null +++ b/components/ui/shadcn-io/aurora-shaders/aurora-shader.original.glsl @@ -0,0 +1,231 @@ +#version 300 es +precision mediump float; + +in vec2 vTextureCoord; +uniform sampler2D uTexture; +uniform sampler2D uCustomTexture; +uniform vec2 uPos; +uniform float uBlur; +uniform float uScale; +uniform float uFrequency; +uniform float uAngle; +uniform float uAmplitude; +uniform float uTime; +uniform float uBloom; +uniform float uMix; +uniform float uSpacing; +uniform int uBlendMode; +uniform int uShape; +uniform int uWaveType; +uniform int uColorPalette; +uniform float uColorScale; +uniform float uColorPosition; +uniform float uVariance; +uniform float uSmoothing; +uniform float uPhase; +uniform float uMouseInfluence; +uniform vec3 uColor; +${Fe} +${ys} +${Ts} + +out vec4 fragColor; + +ivec2 customTexSize; +float customTexAspect; + +const float PI = 3.14159265359; +const float TAU = 6.28318530718; + +vec3 pal( in float t, in vec3 a, in vec3 b, in vec3 c, in vec3 d ) { + return a + b*cos( TAU*(c*t+d) ); +} + +vec3 Tonemap_Reinhard(vec3 x) { + x *= 4.; + return x / (1.0 + x); +} + +float sdCircle(vec2 st, float r) { + return length(st) - r; +} + +float sdEllipse(vec2 st, float r) { + float a = length(st + vec2(0, r * 0.8)) - r; + float b = length(st + vec2(0, -r * 0.8)) - r; + return (a + b); +} + +float sdArc(vec2 st, float r) { + return length(st * vec2(0, r)) - r; +} + +float sdLine(vec2 p, float r) { + float halfLen = r * 2.; + vec2 a = vec2(-halfLen, 0.0); + vec2 b = vec2(halfLen, 0.0); + vec2 pa = p - a; + vec2 ba = b - a; + float h = clamp(dot(pa, ba) / dot(ba, ba), 0.0, 1.0); + return length(pa - ba * h); +} + +float sdBox(vec2 p, float r, float md) { + vec2 q = abs(p)-vec2(r); + return length(max(q,0.0)) + min(max(q.x,q.y),0.0) - r * mix(0., 0.3333 * md, uAmplitude); +} + +float sdEquilateralTriangle(vec2 p, float r, float md) { + const float k = sqrt(3.0); + p.x = abs(p.x) - r; + p.y = p.y + r/k; + if( p.x+k*p.y>0.0 ) p = vec2(p.x-k*p.y,-k*p.x-p.y)/2.0; + p.x -= clamp( p.x, -2.0*r, 0.0 ); + return -length(p)*sign(p.y) - r * mix(0., 0.3333 * md, uAmplitude); +} + +float median(float r, float g, float b) { + return max(min(r, g), min(max(r, g), b)); +} + +float screenPxRange() { + vec2 unitRange = 85./vec2(512); + vec2 screenTexSize = vec2(1.0)/fwidth(vTextureCoord); + return max(0.5*dot(unitRange, screenTexSize), 1.0); +} + +float sdCustom(vec2 uv) { + ivec2 customTexSize = textureSize(uCustomTexture, 0); + float customTexAspect = float(customTexSize.x) / float(customTexSize.y); + + uv.x /= customTexAspect; + uv /= (uScale * 2.5); + uv += 0.5; + + if(uv.x < 0. || uv.x > 1. || uv.y < 0. || uv.y > 1.) { + return 1.; + } + + vec4 sdColor = texture(uCustomTexture, uv); + float msdf = median(sdColor.r, sdColor.g, sdColor.b); + float sd = msdf; + float screenPxDistance = -(sd - 0.51); + return screenPxDistance * 2.; +} + +float getSdf(vec2 st, float iter, float md) { + switch(uShape) { + case 0: return sdCustom(st); break; + case 1: return sdCircle(st, uScale); break; + case 2: return sdEllipse(st, uScale); break; + case 3: return sdLine(st, uScale); break; + case 4: return sdBox(st, uScale, md); break; + case 5: return sdEquilateralTriangle(st, uScale, md); break; + default: return 0.; break; + } +} + +vec2 turb(vec2 pos, float t, float it, float md, vec2 mPos) { + mat2 rot = mat2(0.6, -0.8, 0.8, 0.6); + float freq = mix(2., 15., uFrequency); + float amp = (uAmplitude) * md; + float xp = 1.4; + float time = t * 0.1 + uPhase; + + for(float i = 0.; i < 4.; i++) { + vec2 s = sin(freq * ((pos - mPos) * rot) + i * time + it); + pos += amp * rot[0] * s / freq; + rot *= mat2(0.6, -0.8, 0.8, 0.6); + amp *= mix(1., max(s.y, s.x), uVariance); + freq *= xp; + } + + return pos; +} + + +float luma(vec3 color) { + return dot(color, vec3(0.299, 0.587, 0.114)); +} + +const float ITERATIONS = 36.; + +float expApprox(float x) { + x = clamp(x, -4.0, 4.0); + float x2 = x * x; + return 1.0 + x + 0.5 * x2 + (1.0/6.0) * x2 * x; + } + +void main() { + vec2 uv = vTextureCoord; + vec4 bg = texture(uTexture, uv); + + if(uShape == 0) { + customTexSize = textureSize(uCustomTexture, 0); + customTexAspect = float(customTexSize.x) / float(customTexSize.y); + } + + vec3 pp = vec3(0.); + vec3 bloom = vec3(0.); + float t = uTime * 0.5 + uPhase; + vec2 aspect = vec2(uResolution.x/uResolution.y, 1); + vec2 mousePos = mix(vec2(0), uMousePos - 0.5, uTrackMouse); + vec2 pos = (uv * aspect - uPos * aspect); + float mDist = length(uv * aspect - uMousePos * aspect); + float md = mix(1., smoothstep(1., 5., 1./mDist), uMouseInfluence); + float rotation = uAngle * -2.0 * 3.14159265359; + mat2 rotMatrix = mat2(cos(rotation), -sin(rotation), sin(rotation), cos(rotation)); + pos = rotMatrix * pos; + float bm = 0.05; + + // #ifelseopen + if(uShape == 0) { + bm = 0.2; + } + // #ifelseclose + + vec2 prevPos = turb(pos, t, 0. - 1./ITERATIONS, md, mousePos); + float spacing = mix(1., TAU, uSpacing); + float smoothing = uShape == 0 ? uSmoothing * 2. : uSmoothing; + + for(float i = 1.; i < ITERATIONS + 1.; i++) { + float iter = i/ITERATIONS; + vec2 st = turb(pos, t, iter * spacing, md, mousePos); + float d = abs(getSdf(st, iter, md)); + float pd = distance(st, prevPos); + prevPos = st; + float dynamicBlur = exp2(pd * 2.0 * 1.4426950408889634) - 1.0; + float ds = smoothstep(0., uBlur * bm + max(dynamicBlur * smoothing, 0.001), d); + vec3 color = pal(iter * mix(0.1, 1.9, uColorScale) + uColorPosition, vec3(0.5), vec3(0.5), vec3(1), uColor); + float invd = 1./max(d + dynamicBlur, 0.001); + pp += (ds - 1.) * color; + bloom += clamp(invd, 0., 250.) * color; + } + + pp *= 1./ITERATIONS; + bloom = bloom / (bloom + 2e4); + + // #ifelseopen + if(uShape == 0) { + pp *= 2.; + bloom *= 2.; + } + // #ifelseclose + + + vec3 color = (-pp + bloom * 3. * uBloom); + color *= 1.2; + color += (randFibo(gl_FragCoord.xy) - 0.5) / 255.0; + color = (Tonemap_Reinhard(color)); + vec4 auroraColor = vec4(color, 1.); + + // #ifelseopen + if(uBlendMode > 0) { + auroraColor.rgb = blend(uBlendMode, bg.rgb, auroraColor.rgb); + } + // #ifelseclose + + auroraColor = vec4(mix(bg.rgb, auroraColor.rgb, uMix), max(bg.a, luma(auroraColor.rgb))); + + ${ze("auroraColor")} +} diff --git a/components/ui/shadcn-io/aurora-shaders/index.tsx b/components/ui/shadcn-io/aurora-shaders/index.tsx new file mode 100644 index 00000000..922f9d59 --- /dev/null +++ b/components/ui/shadcn-io/aurora-shaders/index.tsx @@ -0,0 +1,249 @@ +'use client'; + +import React, { forwardRef } from 'react'; +import { Shader } from '../../../livekit/react-shader/react-shader'; + +const auroraShaderSource = ` +const float TAU = 6.28318530718; + +// Cosine palette generator +vec3 pal(in float t, in vec3 a, in vec3 b, in vec3 c, in vec3 d) { + return a + b * cos(TAU * (c * t + d)); +} + +// Noise for dithering +vec2 randFibo(vec2 p) { + p = fract(p * vec2(443.897, 441.423)); + p += dot(p, p.yx + 19.19); + return fract((p.xx + p.yx) * p.xy); +} + +// Tonemap +vec3 Tonemap_Reinhard(vec3 x) { + x *= 4.0; + return x / (1.0 + x); +} + +// Luma for alpha +float luma(vec3 color) { + return dot(color, vec3(0.299, 0.587, 0.114)); +} + +// SDF shapes +float sdCircle(vec2 st, float r) { + return length(st) - r; +} + +float sdLine(vec2 p, float r) { + float halfLen = r * 2.0; + vec2 a = vec2(-halfLen, 0.0); + vec2 b = vec2(halfLen, 0.0); + vec2 pa = p - a; + vec2 ba = b - a; + float h = clamp(dot(pa, ba) / dot(ba, ba), 0.0, 1.0); + return length(pa - ba * h); +} + +float getSdf(vec2 st) { + if(uShape == 1.0) return sdCircle(st, uScale); + else if(uShape == 2.0) return sdLine(st, uScale); + return sdCircle(st, uScale); // Default +} + +vec2 turb(vec2 pos, float t, float it) { + // mat2 rot = mat2(0.6, -0.8, 0.8, 0.6); + mat2 rot = mat2(0.6, -0.25, 0.25, 0.9); + float freq = mix(2.0, 15.0, uFrequency); + float amp = uAmplitude; + float xp = 1.4; + float time = t * 0.1 * uSpeed; + + for(float i = 0.0; i < 4.0; i++) { + vec2 s = sin(freq * (pos * rot) + i * time + it); + pos += amp * rot[0] * s / freq; + rot *= mat2(0.6, -0.8, 0.8, 0.6); + amp *= mix(1.0, max(s.y, s.x), uVariance); + freq *= xp; + } + + return pos; +} + +const float ITERATIONS = 36.0; + +void mainImage(out vec4 fragColor, in vec2 fragCoord) { + vec2 uv = fragCoord / iResolution.xy; + + vec3 pp = vec3(0.0); + vec3 bloom = vec3(0.0); + float t = iTime * 0.5; + vec2 pos = uv - 0.5; + + vec2 prevPos = turb(pos, t, 0.0 - 1.0 / ITERATIONS); + float spacing = mix(1.0, TAU, uSpacing); + + for(float i = 1.0; i < ITERATIONS + 1.0; i++) { + float iter = i / ITERATIONS; + vec2 st = turb(pos, t, iter * spacing); + float d = abs(getSdf(st)); + float pd = distance(st, prevPos); + prevPos = st; + float dynamicBlur = exp2(pd * 2.0 * 1.4426950408889634) - 1.0; + float ds = smoothstep(0.0, uBlur * 0.05 + max(dynamicBlur * uSmoothing, 0.001), d); + + // Generate color using cosine palette + vec3 color = pal( + iter * mix(0.0, 3.0, uColorScale) + uColorPosition * 2.0, + vec3(0.5), + vec3(0.5), + vec3(1.0), + vec3(0.0, 0.33, 0.67) + ); + + float invd = 1.0 / max(d + dynamicBlur, 0.001); + pp += (ds - 1.0) * color; + bloom += clamp(invd, 0.0, 250.0) * color; + } + + pp *= 1.0 / ITERATIONS; + bloom = bloom / (bloom + 2e4); + + vec3 color = (-pp + bloom * 3.0 * uBloom); + color *= 1.2; + color += (randFibo(fragCoord).x - 0.5) / 255.0; + color = Tonemap_Reinhard(color); + + float alpha = luma(color) * uMix; + fragColor = vec4(color * uMix, alpha); +}`; + +export interface AuroraShadersProps extends React.HTMLAttributes { + /** + * Aurora wave speed + * @default 1.0 + */ + speed?: number; + + /** + * Turbulence amplitude + * @default 0.5 + */ + amplitude?: number; + + /** + * Wave frequency and complexity + * @default 0.5 + */ + frequency?: number; + + /** + * Shape scale + * @default 0.3 + */ + scale?: number; + + /** + * Shape type: 1=circle, 2=line + * @default 1 + */ + shape?: number; + + /** + * Edge blur/softness + * @default 1.0 + */ + blur?: number; + + /** + * Color palette offset - shifts colors along the gradient (0-1) + * Lower values shift toward start colors, higher toward end colors + * @default 0.5 + * @example 0.0 - cool tones dominate + * @example 0.5 - balanced (default) + * @example 1.0 - warm tones dominate + */ + colorPosition?: number; + + /** + * Color variation across layers (0-1) + * Controls how much colors change between iterations + * @default 0.5 + * @example 0.0 - minimal color variation (more uniform) + * @example 0.5 - moderate variation (default) + * @example 1.0 - maximum variation (rainbow effect) + */ + colorScale?: number; + + /** + * Brightness of the aurora (0-1) + * @default 1.0 + */ + brightness?: number; +} + +export const AuroraShaders = forwardRef( + ( + { + className, + shape = 1.0, + speed = 1.0, + amplitude = 0.5, + frequency = 0.5, + scale = 0.2, + blur = 1.0, + colorPosition = 1.0, + colorScale = 1.0, + brightness = 1.0, + ...props + }, + ref + ) => { + return ( +
+ { + console.log('error', error); + }} + onWarning={(warning) => { + console.log('warning', warning); + }} + style={{ width: '100%', height: '100%' } as CSSStyleDeclaration} + /> +
+ ); + } +); + +AuroraShaders.displayName = 'AuroraShaders'; + +export default AuroraShaders; diff --git a/components/ui/shadcn-io/cosmic-waves-shaders/index.tsx b/components/ui/shadcn-io/cosmic-waves-shaders/index.tsx new file mode 100644 index 00000000..d9daa457 --- /dev/null +++ b/components/ui/shadcn-io/cosmic-waves-shaders/index.tsx @@ -0,0 +1,168 @@ +'use client'; + +import React, { forwardRef } from 'react'; +import { Shader } from '@/components/livekit/react-shader/react-shader'; +import { cn } from '@/lib/utils'; + +export interface CosmicWavesShadersProps extends React.HTMLAttributes { + speed?: number; + amplitude?: number; + frequency?: number; + starDensity?: number; + colorShift?: number; +} + +const cosmicWavesFragment = ` + +// Hash function for pseudo-random values +float hash(vec2 p) { + return fract(sin(dot(p, vec2(127.1, 311.7))) * 43758.5453); +} + +// Smooth noise function +float noise(vec2 p) { + vec2 i = floor(p); + vec2 f = fract(p); + f = f * f * (3.0 - 2.0 * f); + + float a = hash(i); + float b = hash(i + vec2(1.0, 0.0)); + float c = hash(i + vec2(0.0, 1.0)); + float d = hash(i + vec2(1.0, 1.0)); + + return mix(mix(a, b, f.x), mix(c, d, f.x), f.y); +} + +// Fractal noise +float fbm(vec2 p) { + float value = 0.0; + float amplitude = 0.5; + for(int i = 0; i < 4; i++) { + value += amplitude * noise(p); + p *= 2.0; + amplitude *= 0.5; + } + return value; +} + +// Star field generation +float stars(vec2 p, float density) { + vec2 grid = floor(p * density); + vec2 local = fract(p * density); + + float h = hash(grid); + if(h > 0.95) { + float d = length(local - 0.5); + float star = exp(-d * 20.0); + return star * (0.5 + 0.5 * sin(iTime * 2.0 + h * 10.0)); + } + return 0.0; +} + +void mainImage( out vec4 fragColor, in vec2 fragCoord ) { + vec2 uv = fragCoord.xy / iResolution.xy; + vec2 p = uv * 2.0 - 1.0; + p.x *= iResolution.x / iResolution.y; + + float time = iTime * u_speed; + + // Create flowing wave patterns + vec2 wavePos = p * u_frequency; + wavePos.y += time * 0.3; + + // Multiple wave layers + float wave1 = sin(wavePos.x + cos(wavePos.y + time) * 0.5) * u_amplitude; + float wave2 = sin(wavePos.x * 1.3 - wavePos.y * 0.7 + time * 1.2) * u_amplitude * 0.7; + float wave3 = sin(wavePos.x * 0.8 + wavePos.y * 1.1 - time * 0.8) * u_amplitude * 0.5; + + // Combine waves + float waves = (wave1 + wave2 + wave3) * 0.3; + + // Add fractal noise for organic texture + vec2 noisePos = p * 1.5 + vec2(time * 0.1, time * 0.05); + float noiseValue = fbm(noisePos) * 0.4; + + // Combine waves and noise + float pattern = waves + noiseValue; + + // Create flowing cosmic gradient + float gradient = length(p) * 0.8; + gradient += pattern; + + // Color cycling through cosmic spectrum + vec3 color1 = vec3(0.1, 0.2, 0.8); // Deep blue + vec3 color2 = vec3(0.6, 0.1, 0.9); // Purple + vec3 color3 = vec3(0.1, 0.8, 0.9); // Cyan + vec3 color4 = vec3(0.9, 0.3, 0.6); // Pink + + // Color interpolation based on pattern and time + float colorTime = time * u_colorShift + pattern * 2.0; + vec3 finalColor; + + float t = fract(colorTime * 0.2); + if(t < 0.25) { + finalColor = mix(color1, color2, t * 4.0); + } else if(t < 0.5) { + finalColor = mix(color2, color3, (t - 0.25) * 4.0); + } else if(t < 0.75) { + finalColor = mix(color3, color4, (t - 0.5) * 4.0); + } else { + finalColor = mix(color4, color1, (t - 0.75) * 4.0); + } + + // Apply wave intensity + finalColor *= (0.5 + pattern * 0.8); + + // Add star field + float starField = stars(p + vec2(time * 0.02, time * 0.01), u_starDensity * 15.0); + starField += stars(p * 1.5 + vec2(-time * 0.015, time * 0.008), u_starDensity * 12.0); + + finalColor += vec3(starField * 0.8); + + // Add subtle glow effect + float glow = exp(-length(p) * 0.5) * 0.3; + finalColor += glow * vec3(0.2, 0.4, 0.8); + + // Vignette effect + float vignette = 1.0 - length(uv - 0.5) * 1.2; + vignette = smoothstep(0.0, 1.0, vignette); + + finalColor *= vignette; + + fragColor = vec4(finalColor, 1.0); +} +`; + +export const CosmicWavesShaders = forwardRef( + ( + { + speed = 1.0, + amplitude = 1.0, + frequency = 1.0, + starDensity = 1.0, + colorShift = 1.0, + className, + children, + ...props + }, + ref + ) => { + return ( +
+ +
+ ); + } +); + +CosmicWavesShaders.displayName = 'CosmicWavesShaders'; diff --git a/components/ui/shadcn-io/singularity-shaders/index.tsx b/components/ui/shadcn-io/singularity-shaders/index.tsx new file mode 100644 index 00000000..93a19105 --- /dev/null +++ b/components/ui/shadcn-io/singularity-shaders/index.tsx @@ -0,0 +1,78 @@ +'use client'; + +import React, { forwardRef } from 'react'; +import { Shader } from '@/components/livekit/react-shader/react-shader'; +import { cn } from '@/lib/utils'; + +export interface SingularityShadersProps extends React.HTMLAttributes { + speed?: number; + intensity?: number; + size?: number; + waveStrength?: number; + colorShift?: number; +} + +const fragmentShader = ` +void mainImage(out vec4 O, vec2 F) +{ + float i = .2 * u_speed, a; + vec2 r = iResolution.xy, + p = ( F+F - r ) / r.y / (.7 * u_size), + d = vec2(-1,1), + b = p - i*d, + c = p * mat2(1, 1, d/(.1 + i/dot(b,b))), + v = c * mat2(cos(.5*log(a=dot(c,c)) + iTime*i*u_speed + vec4(0,33,11,0)))/i, + w = vec2(0.0); + + for(float j = 0.0; j < 9.0; j++) { + i++; + w += 1.0 + sin(v * u_waveStrength); + v += .7 * sin(v.yx * i + iTime * u_speed) / i + .5; + } + + i = length( sin(v/.3)*.4 + c*(3.+d) ); + + vec4 colorGrad = vec4(.6,-.4,-1,0) * u_colorShift; + + O = 1. - exp( -exp( c.x * colorGrad ) + / w.xyyx + / ( 2. + i*i/4. - i ) + / ( .5 + 1. / a ) + / ( .03 + abs( length(p)-.7 ) ) + * u_intensity + ); +} +`; + +export const SingularityShaders = forwardRef( + ( + { + className, + speed = 1.0, + intensity = 1.0, + size = 1.0, + waveStrength = 1.0, + colorShift = 1.0, + ...props + }, + ref + ) => { + return ( +
+ +
+ ); + } +); + +SingularityShaders.displayName = 'SingularityShaders'; diff --git a/styles/globals.css b/styles/globals.css index a1b5f7d4..ba5e9d1b 100644 --- a/styles/globals.css +++ b/styles/globals.css @@ -111,6 +111,10 @@ --color-sidebar-accent-foreground: var(--sidebar-accent-foreground); --color-sidebar-border: var(--sidebar-border); --color-sidebar-ring: var(--sidebar-ring); + + /* LiveKit UI tokens */ + --audio-visualizer-idle: var(--color-muted); + --audio-visualizer-active: var(--color-foreground); } @layer base {