From ab7e878c59e5131a4d26a7081b5011061c6b2b70 Mon Sep 17 00:00:00 2001 From: Thomas Yuill Date: Tue, 21 Oct 2025 21:36:05 -0400 Subject: [PATCH 01/10] update /ui --- app/ui/(landing-page)/page.tsx | 64 ++++++ app/ui/README.md | 13 ++ app/ui/_components.tsx | 281 +++++++++++++++++++++++++ app/ui/components/[[...slug]]/page.tsx | 23 ++ app/ui/components/layout.tsx | 33 +++ app/ui/components/page.tsx | 38 ++++ app/ui/layout.tsx | 79 ++++--- app/ui/page.tsx | 268 ----------------------- 8 files changed, 506 insertions(+), 293 deletions(-) create mode 100644 app/ui/(landing-page)/page.tsx create mode 100644 app/ui/README.md create mode 100644 app/ui/_components.tsx create mode 100644 app/ui/components/[[...slug]]/page.tsx create mode 100644 app/ui/components/layout.tsx create mode 100644 app/ui/components/page.tsx delete mode 100644 app/ui/page.tsx diff --git a/app/ui/(landing-page)/page.tsx b/app/ui/(landing-page)/page.tsx new file mode 100644 index 00000000..669dfd74 --- /dev/null +++ b/app/ui/(landing-page)/page.tsx @@ -0,0 +1,64 @@ +'use client'; + +import { AgentControlBar } from '@/components/livekit/agent-control-bar/agent-control-bar'; +import { Button } from '@/components/livekit/button'; +import { ChatEntry } from '@/components/livekit/chat-entry'; + +export default function Page() { + return ( + <> +
+

+ LiveKit{' '} + UI +

+

+ A set of Open Source UI components for +
+ building beautiful voice experiences. +

+
+ + +
+
+ +
+
+
+
+ + +
+ +
+
+
+ + ); +} diff --git a/app/ui/README.md b/app/ui/README.md new file mode 100644 index 00000000..472efb95 --- /dev/null +++ b/app/ui/README.md @@ -0,0 +1,13 @@ +THIS IS NOT PART OF THE MAIN APPLICATION CODE. + +This folder contains code for testing and previewing LiveKit's UI component library in isolation. + +## Getting started + +To run the development server, run the following command: + +```bash +npm run dev +``` + +Then, navigate to `http://localhost:3000/ui` to see the components. diff --git a/app/ui/_components.tsx b/app/ui/_components.tsx new file mode 100644 index 00000000..0e828d09 --- /dev/null +++ b/app/ui/_components.tsx @@ -0,0 +1,281 @@ +'use client'; + +import { type VariantProps } from 'class-variance-authority'; +import { Track } from 'livekit-client'; +import { MicrophoneIcon } from '@phosphor-icons/react/dist/ssr'; +import { AgentControlBar } from '@/components/livekit/agent-control-bar/agent-control-bar'; +import { TrackDeviceSelect } from '@/components/livekit/agent-control-bar/track-device-select'; +import { TrackSelector } from '@/components/livekit/agent-control-bar/track-selector'; +import { TrackToggle } from '@/components/livekit/agent-control-bar/track-toggle'; +import { Alert, AlertDescription, AlertTitle, alertVariants } from '@/components/livekit/alert'; +import { AlertToast } from '@/components/livekit/alert-toast'; +import { Button, buttonVariants } from '@/components/livekit/button'; +import { ChatEntry } from '@/components/livekit/chat-entry'; +import { + Select, + SelectContent, + SelectItem, + SelectTrigger, + SelectValue, +} from '@/components/livekit/select'; +import { ShimmerText } from '@/components/livekit/shimmer-text'; +import { Toggle, toggleVariants } from '@/components/livekit/toggle'; +import { cn } from '@/lib/utils'; + +type toggleVariantsType = VariantProps['variant']; +type toggleVariantsSizeType = VariantProps['size']; +type buttonVariantsType = VariantProps['variant']; +type buttonVariantsSizeType = VariantProps['size']; +type alertVariantsType = VariantProps['variant']; + +interface ContainerProps { + componentName: string; + children: React.ReactNode; + className?: string; +} + +function Container({ componentName, children, className }: ContainerProps) { + return ( +
+
+ {children} +
+
+ ); +} + +function StoryTitle({ children }: { children: React.ReactNode }) { + return

{children}

; +} + +export const COMPONENTS = { + // Button + Button: () => ( + + + + + + + + + + + + + {['default', 'primary', 'secondary', 'outline', 'ghost', 'link', 'destructive'].map( + (variant) => ( + + + {['sm', 'default', 'lg', 'icon'].map((size) => ( + + ))} + + ) + )} + +
SmallDefaultLargeIcon
{variant} + +
+ + ), + + // Toggle + Toggle: () => ( + + + + + + + + + + + + + {['default', 'primary', 'secondary', 'outline'].map((variant) => ( + + + {['sm', 'default', 'lg', 'icon'].map((size) => ( + + ))} + + ))} + +
SmallDefaultLargeIcon
{variant} + + {size === 'icon' ? : 'Toggle'} + +
+ + ), + + // Alert + Alert: () => ( + + {['default', 'destructive'].map((variant) => ( +
+ {variant} + + Alert {variant} title + This is a {variant} alert description. + +
+ ))} + + ), + + // Select + Select: () => ( + +
+
+ Size default + +
+
+ Size sm + +
+
+ + ), + + // Agent control bar + AgentControlBar: () => ( + +
+ +
+ + ), + + // Track device select + TrackDeviceSelect: () => ( + +
+
+ Size default + +
+
+ Size sm + +
+
+ + ), + + // Track toggle + TrackToggle: () => ( + +
+
+ Track.Source.Microphone + +
+
+ Track.Source.Camera + +
+
+ + ), + + // Track selector + TrackSelector: () => ( + +
+
+ Track.Source.Camera + +
+
+ Track.Source.Microphone + +
+
+ + ), + + // Chat entry + ChatEntry: () => ( + +
+ + +
+ + ), + + // Shimmer text + ShimmerText: () => ( + +
+ This is shimmer text +
+ + ), + + // Alert toast + AlertToast: () => ( + + Alert toast +
+ +
+ + ), +}; diff --git a/app/ui/components/[[...slug]]/page.tsx b/app/ui/components/[[...slug]]/page.tsx new file mode 100644 index 00000000..dd0c1154 --- /dev/null +++ b/app/ui/components/[[...slug]]/page.tsx @@ -0,0 +1,23 @@ +'use client'; + +import { redirect, useParams } from 'next/navigation'; +import { COMPONENTS } from '../../_components'; + +export default function Page() { + const { slug = [] } = useParams(); + const [componentName] = slug; + const component = COMPONENTS[componentName as keyof typeof COMPONENTS]; + + if (!component) { + return redirect('/ui'); + } + + return ( + <> +
+

{componentName}

+ {component()} +
+ + ); +} diff --git a/app/ui/components/layout.tsx b/app/ui/components/layout.tsx new file mode 100644 index 00000000..9ff209ca --- /dev/null +++ b/app/ui/components/layout.tsx @@ -0,0 +1,33 @@ +'use client'; + +import Link from 'next/link'; +import { COMPONENTS } from '../_components'; + +export default function Layout({ children }: { children: React.ReactNode }) { + return ( +
+ + +
+
{children}
+
+ + +
+ ); +} diff --git a/app/ui/components/page.tsx b/app/ui/components/page.tsx new file mode 100644 index 00000000..20160cc2 --- /dev/null +++ b/app/ui/components/page.tsx @@ -0,0 +1,38 @@ +'use client'; + +import Link from 'next/link'; +import { COMPONENTS } from '../_components'; + +export default function Page() { + return ( + <> +

+ Components +

+

+ Build beautiful voice experiences with our components. +

+ +
+ {Object.entries(COMPONENTS).map(([componentName]) => ( + + {componentName} + + ))} +
+ +
+ {Object.entries(COMPONENTS).map(([componentName, component]) => ( +
+

{componentName}

+ {component()} +
+ ))} +
+ + ); +} diff --git a/app/ui/layout.tsx b/app/ui/layout.tsx index c7202785..8b3dffd0 100644 --- a/app/ui/layout.tsx +++ b/app/ui/layout.tsx @@ -1,41 +1,70 @@ import * as React from 'react'; import { headers } from 'next/headers'; +import Link from 'next/link'; import { SessionProvider } from '@/components/app/session-provider'; import { getAppConfig } from '@/lib/utils'; -export default async function ComponentsLayout({ children }: { children: React.ReactNode }) { +export default async function Layout({ children }: { children: React.ReactNode }) { const hdrs = await headers(); const appConfig = await getAppConfig(hdrs); return ( -
-
-
-

LiveKit UI

-

- A set of UI components for building LiveKit-powered voice experiences. -

-

- Built with{' '} - - Shadcn - - ,{' '} - - Motion - - , and{' '} - - LiveKit - - . -

-

Open Source.

+
+
+
+ + + + + UI + + + Docs + + + Components +
-
{children}
+ {children}
+ +
); diff --git a/app/ui/page.tsx b/app/ui/page.tsx deleted file mode 100644 index 83e1a7ba..00000000 --- a/app/ui/page.tsx +++ /dev/null @@ -1,268 +0,0 @@ -import { type VariantProps } from 'class-variance-authority'; -import { Track } from 'livekit-client'; -import { MicrophoneIcon } from '@phosphor-icons/react/dist/ssr'; -import { AgentControlBar } from '@/components/livekit/agent-control-bar/agent-control-bar'; -import { TrackDeviceSelect } from '@/components/livekit/agent-control-bar/track-device-select'; -import { TrackSelector } from '@/components/livekit/agent-control-bar/track-selector'; -import { TrackToggle } from '@/components/livekit/agent-control-bar/track-toggle'; -import { Alert, AlertDescription, AlertTitle, alertVariants } from '@/components/livekit/alert'; -import { AlertToast } from '@/components/livekit/alert-toast'; -import { Button, buttonVariants } from '@/components/livekit/button'; -import { ChatEntry } from '@/components/livekit/chat-entry'; -import { - Select, - SelectContent, - SelectItem, - SelectTrigger, - SelectValue, -} from '@/components/livekit/select'; -import { ShimmerText } from '@/components/livekit/shimmer-text'; -import { Toggle, toggleVariants } from '@/components/livekit/toggle'; -import { cn } from '@/lib/utils'; - -type toggleVariantsType = VariantProps['variant']; -type toggleVariantsSizeType = VariantProps['size']; -type buttonVariantsType = VariantProps['variant']; -type buttonVariantsSizeType = VariantProps['size']; -type alertVariantsType = VariantProps['variant']; - -interface ContainerProps { - componentName?: string; - children: React.ReactNode; - className?: string; -} - -function Container({ componentName, children, className }: ContainerProps) { - return ( -
-

- {componentName} -

-
- {children} -
-
- ); -} - -function StoryTitle({ children }: { children: React.ReactNode }) { - return

{children}

; -} - -export default function Base() { - return ( - <> -

Primitives

- - {/* Button */} - - - - - - - - - - - - - {['default', 'primary', 'secondary', 'outline', 'ghost', 'link', 'destructive'].map( - (variant) => ( - - - {['sm', 'default', 'lg', 'icon'].map((size) => ( - - ))} - - ) - )} - -
SmallDefaultLargeIcon
{variant} - -
- - - {/* Toggle */} - - - - - - - - - - - - - {['default', 'primary', 'secondary', 'outline'].map((variant) => ( - - - {['sm', 'default', 'lg', 'icon'].map((size) => ( - - ))} - - ))} - -
SmallDefaultLargeIcon
{variant} - - {size === 'icon' ? : 'Toggle'} - -
- - - {/* Alert */} - - {['default', 'destructive'].map((variant) => ( -
- {variant} - - Alert {variant} title - This is a {variant} alert description. - -
- ))} - - - {/* Select */} - -
-
- Size default - -
-
- Size sm - -
-
- - -

Components

- - {/* Agent control bar */} - -
- -
- - - {/* Track device select */} - -
-
- Size default - -
-
- Size sm - -
-
- - - {/* Track toggle */} - -
-
- Track.Source.Microphone - -
-
- Track.Source.Camera - -
-
- - - {/* Track selector */} - -
-
- Track.Source.Camera - -
-
- Track.Source.Microphone - -
-
- - - {/* Chat entry */} - -
- - -
- - - {/* Shimmer text */} - -
- This is shimmer text -
- - - {/* Alert toast */} - - Alert toast -
- -
- - - ); -} From 037de5c8019190c216b12dca5de9c0c80884f89b Mon Sep 17 00:00:00 2001 From: Thomas Yuill Date: Wed, 22 Oct 2025 16:35:19 -0400 Subject: [PATCH 02/10] BarVisualizer --- app/ui/(landing-page)/page.tsx | 100 ++++-- app/ui/_components.tsx | 291 ++++++++++++++---- .../{[[...slug]] => [...slug]}/page.tsx | 6 +- app/ui/components/layout.tsx | 32 +- app/ui/components/page.tsx | 20 +- app/ui/layout.tsx | 1 - components/app/tile-layout.tsx | 25 +- .../agent-control-bar/agent-control-bar.tsx | 6 +- .../{track-selector.tsx => track-control.tsx} | 32 +- .../audio-bar-visualizer/_bar-visualizer.tsx | 99 ++++++ .../audio-bar-visualizer.tsx | 115 +++++++ .../hooks/useBarAnimator.ts | 84 +++++ styles/globals.css | 4 + 13 files changed, 660 insertions(+), 155 deletions(-) rename app/ui/components/{[[...slug]] => [...slug]}/page.tsx (71%) rename components/livekit/agent-control-bar/{track-selector.tsx => track-control.tsx} (72%) create mode 100644 components/livekit/audio-visualizer/audio-bar-visualizer/_bar-visualizer.tsx create mode 100644 components/livekit/audio-visualizer/audio-bar-visualizer/audio-bar-visualizer.tsx create mode 100644 components/livekit/audio-visualizer/audio-bar-visualizer/hooks/useBarAnimator.ts diff --git a/app/ui/(landing-page)/page.tsx b/app/ui/(landing-page)/page.tsx index 669dfd74..384bf396 100644 --- a/app/ui/(landing-page)/page.tsx +++ b/app/ui/(landing-page)/page.tsx @@ -1,16 +1,35 @@ 'use client'; +import Link from 'next/link'; +import { useVoiceAssistant } from '@livekit/components-react'; import { AgentControlBar } from '@/components/livekit/agent-control-bar/agent-control-bar'; +import { AudioBarVisualizer } from '@/components/livekit/audio-visualizer/audio-bar-visualizer/audio-bar-visualizer'; import { Button } from '@/components/livekit/button'; import { ChatEntry } from '@/components/livekit/chat-entry'; +import { useMicrophone } from '../_components'; export default function Page() { + const { state, audioTrack } = useVoiceAssistant(); + + useMicrophone(); + return ( <>
-

- LiveKit{' '} - UI +

+ + + + UI

A set of Open Source UI components for @@ -19,43 +38,62 @@ export default function Page() {

-
-
-
-
- +
+
+
+
+ +
+ - +
+
+
+
+ + +
+
-
diff --git a/app/ui/_components.tsx b/app/ui/_components.tsx index 0e828d09..ef2c02d6 100644 --- a/app/ui/_components.tsx +++ b/app/ui/_components.tsx @@ -1,14 +1,27 @@ 'use client'; +import { useEffect, useMemo, useState } from 'react'; import { type VariantProps } from 'class-variance-authority'; import { Track } from 'livekit-client'; +import { + type AgentState, + type TrackReference, + type TrackReferenceOrPlaceholder, + useLocalParticipant, +} from '@livekit/components-react'; import { MicrophoneIcon } from '@phosphor-icons/react/dist/ssr'; +import { useSession } from '@/components/app/session-provider'; import { AgentControlBar } from '@/components/livekit/agent-control-bar/agent-control-bar'; +import { TrackControl } from '@/components/livekit/agent-control-bar/track-control'; import { TrackDeviceSelect } from '@/components/livekit/agent-control-bar/track-device-select'; -import { TrackSelector } from '@/components/livekit/agent-control-bar/track-selector'; import { TrackToggle } from '@/components/livekit/agent-control-bar/track-toggle'; import { Alert, AlertDescription, AlertTitle, alertVariants } from '@/components/livekit/alert'; import { AlertToast } from '@/components/livekit/alert-toast'; +import { BarVisualizer } from '@/components/livekit/audio-visualizer/audio-bar-visualizer/_bar-visualizer'; +import { + AudioBarVisualizer, + audioBarVisualizerVariants, +} from '@/components/livekit/audio-visualizer/audio-bar-visualizer/audio-bar-visualizer'; import { Button, buttonVariants } from '@/components/livekit/button'; import { ChatEntry } from '@/components/livekit/chat-entry'; import { @@ -20,13 +33,23 @@ import { } from '@/components/livekit/select'; import { ShimmerText } from '@/components/livekit/shimmer-text'; import { Toggle, toggleVariants } from '@/components/livekit/toggle'; -import { cn } from '@/lib/utils'; type toggleVariantsType = VariantProps['variant']; type toggleVariantsSizeType = VariantProps['size']; type buttonVariantsType = VariantProps['variant']; type buttonVariantsSizeType = VariantProps['size']; type alertVariantsType = VariantProps['variant']; +type audioBarVisualizerVariantsSizeType = VariantProps['size']; + +export function useMicrophone() { + const { startSession } = useSession(); + const { localParticipant } = useLocalParticipant(); + + useEffect(() => { + startSession(); + localParticipant.setMicrophoneEnabled(true, undefined); + }, [startSession, localParticipant]); +} interface ContainerProps { componentName: string; @@ -34,7 +57,7 @@ interface ContainerProps { className?: string; } -function Container({ componentName, children, className }: ContainerProps) { +function Container({ children, className }: ContainerProps) { return (
@@ -168,71 +191,217 @@ export const COMPONENTS = { ), - // Agent control bar - AgentControlBar: () => ( - -
- -
- - ), + // Audio visualizer + AudioVisualizer: () => { + const barCounts = ['0', '3', '5', '7', '9']; + const sizes = ['icon', 'xs', 'sm', 'md', 'lg', 'xl']; + const states = [ + 'disconnected', + 'connecting', + 'initializing', + 'listening', + 'thinking', + 'speaking', + ] as AgentState[]; - // Track device select - TrackDeviceSelect: () => ( - -
-
- Size default - -
-
- Size sm - -
-
- - ), + const { microphoneTrack, localParticipant } = useLocalParticipant(); + const [barCount, setBarCount] = useState(barCounts[0]); + const [size, setSize] = useState( + sizes[3] as audioBarVisualizerVariantsSizeType + ); + const [state, setState] = useState(states[0]); - // Track toggle - TrackToggle: () => ( - -
-
- Track.Source.Microphone - + const micTrackRef = useMemo(() => { + return state === 'speaking' + ? ({ + participant: localParticipant, + source: Track.Source.Microphone, + publication: microphoneTrack, + } as TrackReference) + : undefined; + }, [state, localParticipant, microphoneTrack]); + + useMicrophone(); + + return ( + +
+
+ + +
+ +
+ + +
+ +
+ + +
-
- Track.Source.Camera - + +
+
+ +
+
Original BarVisualizer
+
+ +
-
- - ), + + ); + }, - // Track selector - TrackSelector: () => ( - -
-
- Track.Source.Camera - + // Agent control bar + AgentControlBar: () => { + useMicrophone(); + + return ( + +
+
-
- Track.Source.Microphone - + + ); + }, + + // Track device select + // TrackDeviceSelect: () => ( + // + //
+ //
+ // Size default + // + //
+ //
+ // Size sm + // + //
+ //
+ // + // ), + + // Track toggle + // TrackToggle: () => ( + // + //
+ //
+ // Track.Source.Microphone + // + //
+ //
+ // Track.Source.Camera + // + //
+ //
+ // + // ), + + // Track control + TrackControl: () => { + const { microphoneTrack, localParticipant } = useLocalParticipant(); + const micTrackRef = useMemo(() => { + return { + participant: localParticipant, + source: Track.Source.Microphone, + publication: microphoneTrack, + } as TrackReference; + }, [localParticipant, microphoneTrack]); + + useMicrophone(); + + return ( + +
+
+
+ Track.Source.Microphone + +
+
+ Track.Source.Microphone + +
+
+ +
+ Track.Source.Camera + +
-
- - ), + + ); + }, // Chat entry ChatEntry: () => ( diff --git a/app/ui/components/[[...slug]]/page.tsx b/app/ui/components/[...slug]/page.tsx similarity index 71% rename from app/ui/components/[[...slug]]/page.tsx rename to app/ui/components/[...slug]/page.tsx index dd0c1154..6300a5e8 100644 --- a/app/ui/components/[[...slug]]/page.tsx +++ b/app/ui/components/[...slug]/page.tsx @@ -14,10 +14,8 @@ export default function Page() { return ( <> -
-

{componentName}

- {component()} -
+

{componentName}

+ {component()} ); } diff --git a/app/ui/components/layout.tsx b/app/ui/components/layout.tsx index 9ff209ca..56e79f5d 100644 --- a/app/ui/components/layout.tsx +++ b/app/ui/components/layout.tsx @@ -1,25 +1,43 @@ 'use client'; import Link from 'next/link'; +import { usePathname } from 'next/navigation'; +import { cn } from '@/lib/utils'; import { COMPONENTS } from '../_components'; export default function Layout({ children }: { children: React.ReactNode }) { + const pathname = usePathname(); + const isActive = (path: string) => pathname === path; + return (
- diff --git a/app/ui/components/page.tsx b/app/ui/components/page.tsx index 20160cc2..b0cce846 100644 --- a/app/ui/components/page.tsx +++ b/app/ui/components/page.tsx @@ -14,15 +14,17 @@ export default function Page() {

- {Object.entries(COMPONENTS).map(([componentName]) => ( - - {componentName} - - ))} + {Object.entries(COMPONENTS) + .sort(([a], [b]) => a.localeCompare(b)) + .map(([componentName]) => ( + + {componentName} + + ))}
diff --git a/app/ui/layout.tsx b/app/ui/layout.tsx index 8b3dffd0..4205c28b 100644 --- a/app/ui/layout.tsx +++ b/app/ui/layout.tsx @@ -1,4 +1,3 @@ -import * as React from 'react'; import { headers } from 'next/headers'; import Link from 'next/link'; import { SessionProvider } from '@/components/app/session-provider'; diff --git a/components/app/tile-layout.tsx b/components/app/tile-layout.tsx index 33372276..9de67c97 100644 --- a/components/app/tile-layout.tsx +++ b/components/app/tile-layout.tsx @@ -2,13 +2,13 @@ import React, { useMemo } from 'react'; import { Track } from 'livekit-client'; import { AnimatePresence, motion } from 'motion/react'; import { - BarVisualizer, type TrackReference, VideoTrack, useLocalParticipant, useTracks, useVoiceAssistant, } from '@livekit/components-react'; +import { AudioBarVisualizer } from '@/components/livekit/audio-visualizer/audio-bar-visualizer/audio-bar-visualizer'; import { cn } from '@/lib/utils'; const MotionContainer = motion.create('div'); @@ -116,32 +116,23 @@ export function TileLayout({ chatOpen }: TileLayoutProps) { }} animate={{ opacity: 1, - scale: chatOpen ? 1 : 5, + scale: chatOpen ? 1 : 6, }} transition={{ ...ANIMATION_TRANSITION, delay: animationDelay, }} className={cn( - 'bg-background aspect-square h-[90px] rounded-md border border-transparent transition-[border,drop-shadow]', + 'bg-background flex aspect-square h-[90px] items-center justify-center rounded-md border border-transparent transition-[border,drop-shadow]', chatOpen && 'border-input/50 drop-shadow-lg/10 delay-200' )} > - - - + audioTrack={agentAudioTrack!} + className="mx-auto" + /> )} diff --git a/components/livekit/agent-control-bar/agent-control-bar.tsx b/components/livekit/agent-control-bar/agent-control-bar.tsx index 1b53b1c4..f7b2787d 100644 --- a/components/livekit/agent-control-bar/agent-control-bar.tsx +++ b/components/livekit/agent-control-bar/agent-control-bar.tsx @@ -12,7 +12,7 @@ import { cn } from '@/lib/utils'; import { ChatInput } from './chat-input'; import { UseInputControlsProps, useInputControls } from './hooks/use-input-controls'; import { usePublishPermissions } from './hooks/use-publish-permissions'; -import { TrackSelector } from './track-selector'; +import { TrackControl } from './track-control'; export interface ControlBarControls { leave?: boolean; @@ -107,7 +107,7 @@ export function AgentControlBar({
{/* Toggle Microphone */} {visibleControls.microphone && ( - [0]['source']; pressed?: boolean; @@ -22,7 +19,7 @@ interface TrackSelectorProps { onActiveDeviceChange?: (deviceId: string) => void; } -export function TrackSelector({ +export function TrackControl({ kind, source, pressed, @@ -33,7 +30,7 @@ export function TrackSelector({ onPressedChange, onMediaDeviceError, onActiveDeviceChange, -}: TrackSelectorProps) { +}: TrackControlProps) { return (
{audioTrackRef && ( - - - + )}
diff --git a/components/livekit/audio-visualizer/audio-bar-visualizer/_bar-visualizer.tsx b/components/livekit/audio-visualizer/audio-bar-visualizer/_bar-visualizer.tsx new file mode 100644 index 00000000..aa3ab23b --- /dev/null +++ b/components/livekit/audio-visualizer/audio-bar-visualizer/_bar-visualizer.tsx @@ -0,0 +1,99 @@ +import { useEffect, useMemo, useRef } from 'react'; +import { type VariantProps, cva } from 'class-variance-authority'; +import { + type AgentState, + BarVisualizer as LiveKitBarVisualizer, + type TrackReferenceOrPlaceholder, +} from '@livekit/components-react'; +import { cn } from '@/lib/utils'; + +const MIN_HEIGHT = 15; // 15% + +export const barVisualizerVariants = cva( + ['relative flex aspect-square h-36 items-center justify-center'], + { + variants: { + size: { + default: 'h-32', + icon: 'h-6', + xs: 'h-8', + sm: 'h-16', + md: 'h-32', + lg: 'h-64', + xl: 'h-96', + '2xl': 'h-128', + }, + }, + defaultVariants: { + size: 'default', + }, + } +); + +interface BarVisualizerProps { + state?: AgentState; + barCount?: number; + audioTrack?: TrackReferenceOrPlaceholder; + className?: string; +} + +export function BarVisualizer({ + size, + state, + barCount, + audioTrack, + className, +}: BarVisualizerProps & VariantProps) { + const ref = useRef(null); + const _barCount = useMemo(() => { + if (barCount) { + return barCount; + } + switch (size) { + case 'icon': + case 'xs': + return 3; + default: + return 5; + } + }, [barCount, size]); + + const x = (1 / (_barCount + (_barCount + 1) / 2)) * 100; + + // reset bars height when audio track is disconnected + useEffect(() => { + if (ref.current && !audioTrack) { + const bars = [...(ref.current.querySelectorAll('& > span') ?? [])] as HTMLElement[]; + + bars.forEach((bar) => { + bar.style.height = `${MIN_HEIGHT}%`; + }); + } + }, [audioTrack]); + + return ( + + + + ); +} diff --git a/components/livekit/audio-visualizer/audio-bar-visualizer/audio-bar-visualizer.tsx b/components/livekit/audio-visualizer/audio-bar-visualizer/audio-bar-visualizer.tsx new file mode 100644 index 00000000..3f236ce7 --- /dev/null +++ b/components/livekit/audio-visualizer/audio-bar-visualizer/audio-bar-visualizer.tsx @@ -0,0 +1,115 @@ +import { useMemo } from 'react'; +import { type VariantProps, cva } from 'class-variance-authority'; +import { type LocalAudioTrack, type RemoteAudioTrack } from 'livekit-client'; +import { + type AgentState, + type TrackReferenceOrPlaceholder, + useMultibandTrackVolume, +} from '@livekit/components-react'; +import { cn } from '@/lib/utils'; +import { useBarAnimator } from './hooks/useBarAnimator'; + +export const audioBarVisualizerVariants = cva(['relative flex items-center justify-center'], { + variants: { + size: { + icon: 'h-[24px] gap-[2px]', + xs: 'h-[32px] gap-[2px]', + sm: 'h-[56px] gap-[4px]', + md: 'h-[112px] gap-[8px]', + lg: 'h-[224px] gap-[16px]', + xl: 'h-[448px] gap-[32px]', + }, + }, + defaultVariants: { + size: 'md', + }, +}); + +export const audioBarVisualizerBarVariants = cva( + [ + 'rounded-full transition-colors duration-250 ease-linear bg-(--audio-visualizer-idle) data-[lk-highlighted=true]:bg-(--audio-visualizer-active)', + ], + { + variants: { + size: { + icon: 'w-[4px] min-h-[4px]', + xs: 'w-[4px] min-h-[4px]', + sm: 'w-[8px] min-h-[8px]', + md: 'w-[16px] min-h-[16px]', + lg: 'w-[32px] min-h-[32px]', + xl: 'w-[64px] min-h-[64px]', + }, + }, + defaultVariants: { + size: 'md', + }, + } +); + +interface AudioBarVisualizerProps { + state?: AgentState; + barCount?: number; + audioTrack?: LocalAudioTrack | RemoteAudioTrack | TrackReferenceOrPlaceholder; + className?: string; + barClassName?: string; +} + +export function AudioBarVisualizer({ + size, + state, + barCount, + audioTrack, + className, + barClassName, +}: AudioBarVisualizerProps & VariantProps) { + const _barCount = useMemo(() => { + if (barCount) { + return barCount; + } + switch (size) { + case 'icon': + case 'xs': + return 3; + default: + return 5; + } + }, [barCount, size]); + + const volumeBands = useMultibandTrackVolume(audioTrack, { + bands: _barCount, + loPass: 100, + hiPass: 200, + }); + + const sequencerInterval = useMemo(() => { + switch (state) { + case 'connecting': + return 2000 / _barCount; + case 'initializing': + return 500; + case 'listening': + return 500; + case 'thinking': + return 150; + default: + return 1000; + } + }, [state, _barCount]); + + const highlightedIndices = useBarAnimator(state, _barCount, sequencerInterval); + + const bands = audioTrack ? volumeBands : new Array(_barCount).fill(0); + return ( +
+ {bands.map((band, idx) => ( +
+ ))} +
+ ); +} diff --git a/components/livekit/audio-visualizer/audio-bar-visualizer/hooks/useBarAnimator.ts b/components/livekit/audio-visualizer/audio-bar-visualizer/hooks/useBarAnimator.ts new file mode 100644 index 00000000..070bfa0b --- /dev/null +++ b/components/livekit/audio-visualizer/audio-bar-visualizer/hooks/useBarAnimator.ts @@ -0,0 +1,84 @@ +import { useEffect, useRef, useState } from 'react'; +import { type AgentState } from '@livekit/components-react'; + +function generateConnectingSequenceBar(columns: number): number[][] { + const seq = []; + + for (let x = 0; x < columns; x++) { + seq.push([x, columns - 1 - x]); + } + + return seq; +} + +function generateListeningSequenceBar(columns: number): number[][] { + const center = Math.floor(columns / 2); + const noIndex = -1; + + return [[center], [noIndex]]; +} + +// function generateThinkingSequenceBar(columns: number): number[][] { +// const seq = []; +// for (let x = 0; x < columns; x++) { +// seq.push([x]); +// } + +// for (let x = columns - 1; x >= 0; x--) { +// seq.push([x]); +// } + +// return seq; +// } + +export const useBarAnimator = ( + state: AgentState | undefined, + columns: number, + interval: number +): number[] => { + const [index, setIndex] = useState(0); + const [sequence, setSequence] = useState([[]]); + + useEffect(() => { + if (state === 'thinking') { + setSequence(generateListeningSequenceBar(columns)); + // setSequence(generateThinkingSequenceBar(columns)); + } else if (state === 'connecting' || state === 'initializing') { + const sequence = [...generateConnectingSequenceBar(columns)]; + setSequence(sequence); + } else if (state === 'listening') { + setSequence(generateListeningSequenceBar(columns)); + } else if (state === undefined || state === 'speaking') { + setSequence([new Array(columns).fill(0).map((_, idx) => idx)]); + } else { + setSequence([[]]); + } + setIndex(0); + }, [state, columns]); + + const animationFrameId = useRef(null); + useEffect(() => { + let startTime = performance.now(); + + const animate = (time: DOMHighResTimeStamp) => { + const timeElapsed = time - startTime; + + if (timeElapsed >= interval) { + setIndex((prev) => prev + 1); + startTime = time; + } + + animationFrameId.current = requestAnimationFrame(animate); + }; + + animationFrameId.current = requestAnimationFrame(animate); + + return () => { + if (animationFrameId.current !== null) { + cancelAnimationFrame(animationFrameId.current); + } + }; + }, [interval, columns, state, sequence.length]); + + return sequence[index % sequence.length]; +}; diff --git a/styles/globals.css b/styles/globals.css index a1b5f7d4..ba5e9d1b 100644 --- a/styles/globals.css +++ b/styles/globals.css @@ -111,6 +111,10 @@ --color-sidebar-accent-foreground: var(--sidebar-accent-foreground); --color-sidebar-border: var(--sidebar-border); --color-sidebar-ring: var(--sidebar-ring); + + /* LiveKit UI tokens */ + --audio-visualizer-idle: var(--color-muted); + --audio-visualizer-active: var(--color-foreground); } @layer base { From b0b0bad2597ca0a96efe225bc5a6a44866763921 Mon Sep 17 00:00:00 2001 From: Thomas Yuill Date: Thu, 23 Oct 2025 10:46:38 -0400 Subject: [PATCH 03/10] AudioRadialVisualizer --- app/ui/_components.tsx | 133 ++++++++++++++-- .../audio-bar-visualizer.tsx | 4 +- .../_bar-visualizer.tsx | 99 ++++++++++++ .../audio-radial-visualizer.tsx | 150 ++++++++++++++++++ .../hooks/useBarAnimator.ts | 83 ++++++++++ 5 files changed, 457 insertions(+), 12 deletions(-) create mode 100644 components/livekit/audio-visualizer/audio-radial-visualizer/_bar-visualizer.tsx create mode 100644 components/livekit/audio-visualizer/audio-radial-visualizer/audio-radial-visualizer.tsx create mode 100644 components/livekit/audio-visualizer/audio-radial-visualizer/hooks/useBarAnimator.ts diff --git a/app/ui/_components.tsx b/app/ui/_components.tsx index ef2c02d6..d1e0c46e 100644 --- a/app/ui/_components.tsx +++ b/app/ui/_components.tsx @@ -13,8 +13,8 @@ import { MicrophoneIcon } from '@phosphor-icons/react/dist/ssr'; import { useSession } from '@/components/app/session-provider'; import { AgentControlBar } from '@/components/livekit/agent-control-bar/agent-control-bar'; import { TrackControl } from '@/components/livekit/agent-control-bar/track-control'; -import { TrackDeviceSelect } from '@/components/livekit/agent-control-bar/track-device-select'; -import { TrackToggle } from '@/components/livekit/agent-control-bar/track-toggle'; +// import { TrackDeviceSelect } from '@/components/livekit/agent-control-bar/track-device-select'; +// import { TrackToggle } from '@/components/livekit/agent-control-bar/track-toggle'; import { Alert, AlertDescription, AlertTitle, alertVariants } from '@/components/livekit/alert'; import { AlertToast } from '@/components/livekit/alert-toast'; import { BarVisualizer } from '@/components/livekit/audio-visualizer/audio-bar-visualizer/_bar-visualizer'; @@ -22,6 +22,10 @@ import { AudioBarVisualizer, audioBarVisualizerVariants, } from '@/components/livekit/audio-visualizer/audio-bar-visualizer/audio-bar-visualizer'; +import { + AudioRadialVisualizer, + audioRadialVisualizerVariants, +} from '@/components/livekit/audio-visualizer/audio-radial-visualizer/audio-radial-visualizer'; import { Button, buttonVariants } from '@/components/livekit/button'; import { ChatEntry } from '@/components/livekit/chat-entry'; import { @@ -40,6 +44,9 @@ type buttonVariantsType = VariantProps['variant']; type buttonVariantsSizeType = VariantProps['size']; type alertVariantsType = VariantProps['variant']; type audioBarVisualizerVariantsSizeType = VariantProps['size']; +type audioRadialVisualizerVariantsSizeType = VariantProps< + typeof audioRadialVisualizerVariants +>['size']; export function useMicrophone() { const { startSession } = useSession(); @@ -191,10 +198,10 @@ export const COMPONENTS = { ), - // Audio visualizer - AudioVisualizer: () => { + // Audio bar visualizer + AudioBarVisualizer: () => { const barCounts = ['0', '3', '5', '7', '9']; - const sizes = ['icon', 'xs', 'sm', 'md', 'lg', 'xl']; + const sizes = ['icon', 'sm', 'md', 'lg', 'xl']; const states = [ 'disconnected', 'connecting', @@ -207,7 +214,7 @@ export const COMPONENTS = { const { microphoneTrack, localParticipant } = useLocalParticipant(); const [barCount, setBarCount] = useState(barCounts[0]); const [size, setSize] = useState( - sizes[3] as audioBarVisualizerVariantsSizeType + 'md' as audioBarVisualizerVariantsSizeType ); const [state, setState] = useState(states[0]); @@ -258,7 +265,7 @@ export const COMPONENTS = { {sizes.map((size) => ( - {size} + {size.toUpperCase()} ))} @@ -285,7 +292,7 @@ export const COMPONENTS = {
-
+
Original BarVisualizer
-
+
{ + const barCounts = ['0', '4', '8', '12', '16', '24']; + const sizes = ['icon', 'sm', 'md', 'lg', 'xl']; + const states = [ + 'disconnected', + 'connecting', + 'initializing', + 'listening', + 'thinking', + 'speaking', + ] as AgentState[]; + + const { microphoneTrack, localParticipant } = useLocalParticipant(); + const [barCount, setBarCount] = useState(barCounts[0]); + const [size, setSize] = useState( + 'md' as audioRadialVisualizerVariantsSizeType + ); + const [state, setState] = useState(states[0]); + + const micTrackRef = useMemo(() => { + return state === 'speaking' + ? ({ + participant: localParticipant, + source: Track.Source.Microphone, + publication: microphoneTrack, + } as TrackReference) + : undefined; + }, [state, localParticipant, microphoneTrack]); + + useMicrophone(); + + return ( + +
+
+ + +
+ +
+ + +
+ +
+ + +
+
+ +
+
+ +
+
+ + ); + }, + // Agent control bar AgentControlBar: () => { useMicrophone(); diff --git a/components/livekit/audio-visualizer/audio-bar-visualizer/audio-bar-visualizer.tsx b/components/livekit/audio-visualizer/audio-bar-visualizer/audio-bar-visualizer.tsx index 3f236ce7..8701c967 100644 --- a/components/livekit/audio-visualizer/audio-bar-visualizer/audio-bar-visualizer.tsx +++ b/components/livekit/audio-visualizer/audio-bar-visualizer/audio-bar-visualizer.tsx @@ -13,7 +13,6 @@ export const audioBarVisualizerVariants = cva(['relative flex items-center justi variants: { size: { icon: 'h-[24px] gap-[2px]', - xs: 'h-[32px] gap-[2px]', sm: 'h-[56px] gap-[4px]', md: 'h-[112px] gap-[8px]', lg: 'h-[224px] gap-[16px]', @@ -33,7 +32,6 @@ export const audioBarVisualizerBarVariants = cva( variants: { size: { icon: 'w-[4px] min-h-[4px]', - xs: 'w-[4px] min-h-[4px]', sm: 'w-[8px] min-h-[8px]', md: 'w-[16px] min-h-[16px]', lg: 'w-[32px] min-h-[32px]', @@ -68,7 +66,7 @@ export function AudioBarVisualizer({ } switch (size) { case 'icon': - case 'xs': + case 'sm': return 3; default: return 5; diff --git a/components/livekit/audio-visualizer/audio-radial-visualizer/_bar-visualizer.tsx b/components/livekit/audio-visualizer/audio-radial-visualizer/_bar-visualizer.tsx new file mode 100644 index 00000000..aa3ab23b --- /dev/null +++ b/components/livekit/audio-visualizer/audio-radial-visualizer/_bar-visualizer.tsx @@ -0,0 +1,99 @@ +import { useEffect, useMemo, useRef } from 'react'; +import { type VariantProps, cva } from 'class-variance-authority'; +import { + type AgentState, + BarVisualizer as LiveKitBarVisualizer, + type TrackReferenceOrPlaceholder, +} from '@livekit/components-react'; +import { cn } from '@/lib/utils'; + +const MIN_HEIGHT = 15; // 15% + +export const barVisualizerVariants = cva( + ['relative flex aspect-square h-36 items-center justify-center'], + { + variants: { + size: { + default: 'h-32', + icon: 'h-6', + xs: 'h-8', + sm: 'h-16', + md: 'h-32', + lg: 'h-64', + xl: 'h-96', + '2xl': 'h-128', + }, + }, + defaultVariants: { + size: 'default', + }, + } +); + +interface BarVisualizerProps { + state?: AgentState; + barCount?: number; + audioTrack?: TrackReferenceOrPlaceholder; + className?: string; +} + +export function BarVisualizer({ + size, + state, + barCount, + audioTrack, + className, +}: BarVisualizerProps & VariantProps) { + const ref = useRef(null); + const _barCount = useMemo(() => { + if (barCount) { + return barCount; + } + switch (size) { + case 'icon': + case 'xs': + return 3; + default: + return 5; + } + }, [barCount, size]); + + const x = (1 / (_barCount + (_barCount + 1) / 2)) * 100; + + // reset bars height when audio track is disconnected + useEffect(() => { + if (ref.current && !audioTrack) { + const bars = [...(ref.current.querySelectorAll('& > span') ?? [])] as HTMLElement[]; + + bars.forEach((bar) => { + bar.style.height = `${MIN_HEIGHT}%`; + }); + } + }, [audioTrack]); + + return ( + + + + ); +} diff --git a/components/livekit/audio-visualizer/audio-radial-visualizer/audio-radial-visualizer.tsx b/components/livekit/audio-visualizer/audio-radial-visualizer/audio-radial-visualizer.tsx new file mode 100644 index 00000000..d36ac939 --- /dev/null +++ b/components/livekit/audio-visualizer/audio-radial-visualizer/audio-radial-visualizer.tsx @@ -0,0 +1,150 @@ +import { useMemo } from 'react'; +import { type VariantProps, cva } from 'class-variance-authority'; +import { type LocalAudioTrack, type RemoteAudioTrack } from 'livekit-client'; +import { + type AgentState, + type TrackReferenceOrPlaceholder, + useMultibandTrackVolume, +} from '@livekit/components-react'; +import { cn } from '@/lib/utils'; +import { useBarAnimator } from './hooks/useBarAnimator'; + +export const audioRadialVisualizerVariants = cva(['relative flex items-center justify-center'], { + variants: { + size: { + icon: 'h-[24px] gap-[2px]', + sm: 'h-[56px] gap-[4px]', + md: 'h-[112px] gap-[8px]', + lg: 'h-[224px] gap-[16px]', + xl: 'h-[448px] gap-[32px]', + }, + }, + defaultVariants: { + size: 'md', + }, +}); + +export const audioRadialVisualizerBarVariants = cva( + [ + 'rounded-full transition-colors duration-250 ease-linear bg-(--audio-visualizer-idle) data-[lk-highlighted=true]:bg-(--audio-visualizer-active)', + ], + { + variants: { + size: { + icon: 'w-[4px] min-h-[4px]', + sm: 'w-[8px] min-h-[8px]', + md: 'w-[16px] min-h-[16px]', + lg: 'w-[32px] min-h-[32px]', + xl: 'w-[64px] min-h-[64px]', + }, + }, + defaultVariants: { + size: 'md', + }, + } +); + +interface AudioRadialVisualizerProps { + state?: AgentState; + radius?: number; + barCount?: number; + audioTrack?: LocalAudioTrack | RemoteAudioTrack | TrackReferenceOrPlaceholder; + className?: string; + barClassName?: string; +} + +export function AudioRadialVisualizer({ + size, + state, + radius, + barCount, + audioTrack, + className, + barClassName, +}: AudioRadialVisualizerProps & VariantProps) { + const _barCount = useMemo(() => { + if (barCount) { + return barCount; + } + switch (size) { + case 'icon': + case 'sm': + return 9; + default: + return 12; + } + }, [barCount, size]); + + const volumeBands = useMultibandTrackVolume(audioTrack, { + bands: Math.floor(_barCount / 2), + loPass: 100, + hiPass: 200, + }); + + const sequencerInterval = useMemo(() => { + switch (state) { + case 'connecting': + return 2000 / _barCount; + case 'initializing': + return 500; + case 'listening': + return 500; + case 'thinking': + return 150; + default: + return 1000; + } + }, [state, _barCount]); + + const distanceFromCenter = useMemo(() => { + if (radius) { + return radius; + } + switch (size) { + case 'icon': + return 6; + case 'xl': + return 128; + case 'lg': + return 64; + case 'sm': + return 16; + case 'md': + default: + return 32; + } + }, [size, radius]); + + const highlightedIndices = useBarAnimator(state, _barCount, sequencerInterval); + const bands = audioTrack ? [...volumeBands, ...volumeBands] : new Array(_barCount).fill(0); + + return ( +
+ {bands.map((band, idx) => { + const angle = (idx / _barCount) * Math.PI * 2; + + return ( +
+
+
+ ); + })} +
+ ); +} diff --git a/components/livekit/audio-visualizer/audio-radial-visualizer/hooks/useBarAnimator.ts b/components/livekit/audio-visualizer/audio-radial-visualizer/hooks/useBarAnimator.ts new file mode 100644 index 00000000..e0d1c808 --- /dev/null +++ b/components/livekit/audio-visualizer/audio-radial-visualizer/hooks/useBarAnimator.ts @@ -0,0 +1,83 @@ +import { useEffect, useRef, useState } from 'react'; +import { type AgentState } from '@livekit/components-react'; + +function generateConnectingSequenceBar(columns: number): number[][] { + const seq = []; + + for (let x = 0; x < columns; x++) { + seq.push([x]); + } + + return seq; +} + +function generateListeningSequenceBar(columns: number): number[][] { + const center = Math.floor(columns / 2); + const noIndex = -1; + + return [[center], [noIndex]]; +} + +// function generateThinkingSequenceBar(columns: number): number[][] { +// const seq = []; +// for (let x = 0; x < columns; x++) { +// seq.push([x]); +// } + +// for (let x = columns - 1; x >= 0; x--) { +// seq.push([x]); +// } + +// return seq; +// } + +export const useBarAnimator = ( + state: AgentState | undefined, + columns: number, + interval: number +): number[] => { + const [index, setIndex] = useState(0); + const [sequence, setSequence] = useState([[]]); + + useEffect(() => { + if (state === 'thinking') { + setSequence(generateListeningSequenceBar(columns)); + // setSequence(generateThinkingSequenceBar(columns)); + } else if (state === 'connecting' || state === 'initializing') { + setSequence(generateConnectingSequenceBar(columns)); + } else if (state === 'listening') { + setSequence(generateListeningSequenceBar(columns)); + } else if (state === undefined || state === 'speaking') { + setSequence([new Array(columns).fill(0).map((_, idx) => idx)]); + } else { + setSequence([[]]); + } + setIndex(0); + }, [state, columns]); + + const animationFrameId = useRef(null); + useEffect(() => { + let startTime = performance.now(); + + const animate = (time: DOMHighResTimeStamp) => { + const timeElapsed = time - startTime; + + if (timeElapsed >= interval) { + setIndex((prev) => prev + 1); + startTime = time; + } + + animationFrameId.current = requestAnimationFrame(animate); + }; + + animationFrameId.current = requestAnimationFrame(animate); + + return () => { + if (animationFrameId.current !== null) { + cancelAnimationFrame(animationFrameId.current); + } + }; + }, [interval, columns, state, sequence.length]); + + return sequence[index % sequence.length]; +}; From a5c272ffaa040574d8abe00f556ceded37807f18 Mon Sep 17 00:00:00 2001 From: Thomas Yuill Date: Wed, 22 Oct 2025 21:26:13 -0400 Subject: [PATCH 04/10] AudioGridVisualizer --- app/ui/_components.tsx | 138 ++++++++++++ .../audio-grid-visualizer.tsx | 134 ++++++++++++ .../audio-grid-visualizer/demos.tsx | 197 ++++++++++++++++++ .../hooks/useGridAnimator.ts | 115 ++++++++++ 4 files changed, 584 insertions(+) create mode 100644 components/livekit/audio-visualizer/audio-grid-visualizer/audio-grid-visualizer.tsx create mode 100644 components/livekit/audio-visualizer/audio-grid-visualizer/demos.tsx create mode 100644 components/livekit/audio-visualizer/audio-grid-visualizer/hooks/useGridAnimator.ts diff --git a/app/ui/_components.tsx b/app/ui/_components.tsx index d1e0c46e..6dea0072 100644 --- a/app/ui/_components.tsx +++ b/app/ui/_components.tsx @@ -22,6 +22,11 @@ import { AudioBarVisualizer, audioBarVisualizerVariants, } from '@/components/livekit/audio-visualizer/audio-bar-visualizer/audio-bar-visualizer'; +import { + AudioGridVisualizer, + type GridOptions, +} from '@/components/livekit/audio-visualizer/audio-grid-visualizer/audio-grid-visualizer'; +import { gridVariants } from '@/components/livekit/audio-visualizer/audio-grid-visualizer/demos'; import { AudioRadialVisualizer, audioRadialVisualizerVariants, @@ -424,6 +429,139 @@ export const COMPONENTS = { ); }, + // Audio bar visualizer + AudioGridVisualizer: () => { + const rowCounts = ['3', '5', '7', '9', '11', '13', '15']; + const columnCounts = ['3', '5', '7', '9', '11', '13', '15']; + const states = [ + 'disconnected', + 'connecting', + 'initializing', + 'listening', + 'thinking', + 'speaking', + ] as AgentState[]; + + const { microphoneTrack, localParticipant } = useLocalParticipant(); + const [rowCount, setRowCount] = useState(rowCounts[0]); + const [columnCount, setColumnCount] = useState(columnCounts[0]); + const [state, setState] = useState(states[0]); + const [demoIndex, setDemoIndex] = useState(0); + + const micTrackRef = useMemo(() => { + return state === 'speaking' + ? ({ + participant: localParticipant, + source: Track.Source.Microphone, + publication: microphoneTrack, + } as TrackReference) + : undefined; + }, [state, localParticipant, microphoneTrack]); + + useMicrophone(); + + const demoOptions = { + rowCount: parseInt(rowCount), + columnCount: parseInt(columnCount), + ...gridVariants[demoIndex], + }; + + return ( + +
+
+ + +
+ +
+ + +
+ +
+ + +
+ +
+ + +
+
+ +
+ +
+
+ 
+            {JSON.stringify(demoOptions, null, 2)}
+
+
+ + ); + }, + // Agent control bar AgentControlBar: () => { useMicrophone(); diff --git a/components/livekit/audio-visualizer/audio-grid-visualizer/audio-grid-visualizer.tsx b/components/livekit/audio-visualizer/audio-grid-visualizer/audio-grid-visualizer.tsx new file mode 100644 index 00000000..3d412f26 --- /dev/null +++ b/components/livekit/audio-visualizer/audio-grid-visualizer/audio-grid-visualizer.tsx @@ -0,0 +1,134 @@ +import { CSSProperties, ComponentType, JSX, memo, useMemo } from 'react'; +import { LocalAudioTrack, RemoteAudioTrack } from 'livekit-client'; +import { + type AgentState, + type TrackReferenceOrPlaceholder, + useMultibandTrackVolume, +} from '@livekit/components-react'; +import { cn } from '@/lib/utils'; +import { type Coordinate, useGridAnimator } from './hooks/useGridAnimator'; + +type GridComponentType = + | ComponentType<{ style?: CSSProperties; className?: string }> + | keyof JSX.IntrinsicElements; + +export interface GridOptions { + radius?: number; + interval?: number; + rowCount?: number; + columnCount?: number; + className?: string; + baseClassName?: string; + offClassName?: string; + onClassName?: string; + GridComponent?: GridComponentType; + transformer?: (index: number, rowCount: number, columnCount: number) => CSSProperties; +} + +function useGrid(options: GridOptions) { + return useMemo(() => { + const { columnCount = 5, rowCount } = options; + + const _columnCount = columnCount; + const _rowCount = rowCount ?? columnCount; + const items = new Array(_columnCount * _rowCount).fill(0).map((_, idx) => idx); + + return { columnCount: _columnCount, rowCount: _rowCount, items }; + }, [options]); +} + +interface GridCellProps { + index: number; + state: AgentState; + options: GridOptions; + rowCount: number; + volumeBands: number[]; + columnCount: number; + highlightedCoordinate: Coordinate; + Component: GridComponentType; +} + +const GridCell = memo(function GridCell({ + index, + state, + options, + rowCount, + volumeBands, + columnCount, + highlightedCoordinate, + Component, +}: GridCellProps) { + const { interval = 100, baseClassName, onClassName, offClassName, transformer } = options; + + if (state === 'speaking') { + const y = Math.floor(index / columnCount); + const rowMidPoint = Math.floor(rowCount / 2); + const volumeChunks = 1 / (rowMidPoint + 1); + const distanceToMid = Math.abs(rowMidPoint - y); + const threshold = distanceToMid * volumeChunks; + const isOn = volumeBands[index % columnCount] >= threshold; + + return ; + } + + let transformerStyle: CSSProperties | undefined; + if (transformer) { + transformerStyle = transformer(index, rowCount, columnCount); + } + + const isOn = + highlightedCoordinate.x === index % columnCount && + highlightedCoordinate.y === Math.floor(index / columnCount); + + const transitionDurationInSeconds = interval / (isOn ? 1000 : 100); + + return ( + + ); +}); + +export interface AudioGridVisualizerProps { + state: AgentState; + options: GridOptions; + audioTrack?: LocalAudioTrack | RemoteAudioTrack | TrackReferenceOrPlaceholder; +} + +export function AudioGridVisualizer({ state, options, audioTrack }: AudioGridVisualizerProps) { + const { radius, interval = 100, className, GridComponent = 'div' } = options; + const { columnCount, rowCount, items } = useGrid(options); + const highlightedCoordinate = useGridAnimator(state, rowCount, columnCount, interval, radius); + const volumeBands = useMultibandTrackVolume(audioTrack, { + bands: columnCount, + loPass: 100, + hiPass: 200, + }); + + return ( +
+ {items.map((idx) => ( + + ))} +
+ ); +} diff --git a/components/livekit/audio-visualizer/audio-grid-visualizer/demos.tsx b/components/livekit/audio-visualizer/audio-grid-visualizer/demos.tsx new file mode 100644 index 00000000..4fb0237c --- /dev/null +++ b/components/livekit/audio-visualizer/audio-grid-visualizer/demos.tsx @@ -0,0 +1,197 @@ +import { GridOptions } from './audio-grid-visualizer'; + +type SVGIconProps = React.SVGProps; + +function SVGIcon({ className, children, ...props }: SVGIconProps) { + return ( + <> + + {children} + + + ); +} + +function EyeSVG(props: SVGIconProps) { + return ( + + + + ); +} + +function PlusSVG(props: SVGIconProps) { + return ( + + + + ); +} + +export const gridVariants: GridOptions[] = [ + // 1 + { + radius: 6, + interval: 75, + className: 'gap-4', + baseClassName: 'size-1 rounded-full', + offClassName: 'bg-foreground/10 scale-100', + onClassName: 'bg-foreground scale-125 shadow-[0px_0px_10px_2px_rgba(255,255,255,0.4)]', + }, + // 2 + { + interval: 50, + className: 'gap-2', + baseClassName: 'w-4 h-1', + offClassName: 'bg-foreground/10', + onClassName: 'bg-[#F9B11F] shadow-[0px_0px_14.8px_2px_#F9B11F]', + }, + // 3 + { + className: 'gap-2', + baseClassName: 'size-2 rounded-full', + offClassName: 'bg-foreground/10', + onClassName: 'bg-[#1FD5F9] shadow-[0px_0px_8px_3px_rgba(31,213,249,0.44)]', + }, + // 4 + { + className: 'gap-4', + baseClassName: 'size-4 rounded-full', + offClassName: 'bg-foreground/5', + onClassName: 'bg-[#FF6352] shadow-[0px_0px_32px_8px_rgba(255,99,82,0.8)]', + }, + // 5 + { + className: 'gap-4', + baseClassName: 'size-2 rounded-full', + offClassName: 'bg-foreground/10', + onClassName: 'bg-[#1F8CF9] shadow-[0px_0px_14.8px_2px_#1F8CF9]', + transformer: (index: number, rowCount: number, columnCount: number) => { + const rowMidPoint = Math.floor(rowCount / 2); + const distanceFromCenter = Math.sqrt( + Math.pow(rowMidPoint - (index % columnCount), 2) + + Math.pow(rowMidPoint - Math.floor(index / columnCount), 2) + ); + + return { + opacity: 1 - distanceFromCenter / columnCount, + transform: `scale(${1 - (distanceFromCenter / (columnCount * 2)) * 1.75})`, + }; + }, + }, + // 6 + { + radius: 4, + interval: 150, + className: 'gap-2', + baseClassName: 'size-2', + offClassName: 'bg-foreground/8', + onClassName: 'bg-[#F91F8C] shadow-[0px_0px_14.8px_4px_#F91F8C] scale-150', + transformer: (index: number, rowCount: number, columnCount: number) => { + const rowMidPoint = Math.floor(rowCount / 2); + const distanceFromCenter = Math.sqrt( + Math.pow(rowMidPoint - (index % columnCount), 2) + + Math.pow(rowMidPoint - Math.floor(index / columnCount), 2) + ); + + return { + opacity: 0.5 - distanceFromCenter / columnCount, + }; + }, + }, + // 7 + { + interval: 50, + className: 'gap-4', + baseClassName: 'size-2.5 rounded-1.5', + offClassName: 'bg-foreground/15', + onClassName: 'bg-[#FFB6C1] shadow-[0px_0px_24px_3px_rgba(255,182,193,0.8)]', + }, + // 8 + { + interval: 50, + className: 'gap-8', + baseClassName: 'size-2.5', + offClassName: 'bg-foreground/5', + onClassName: 'bg-[#FFB6C1] shadow-[0px_0px_8px_1px_rgba(255,182,193,0.8)]', + transformer: (index: number, rowCount: number, columnCount: number) => { + const rowMidPoint = Math.floor(rowCount / 2); + const distanceFromCenter = Math.sqrt( + Math.pow(rowMidPoint - (index % columnCount), 2) + + Math.pow(rowMidPoint - Math.floor(index / columnCount), 2) + ); + const maxDistance = Math.sqrt(4); // maximum distance in a normalized grid + const scaleFactor = 1 + (maxDistance - distanceFromCenter / maxDistance); + + return { + transform: `scale(${scaleFactor})`, + }; + }, + }, + // 9 + { + radius: 4, + interval: 75, + className: 'gap-2.5', + baseClassName: 'w-1.5 h-px rotate-45', + offClassName: 'bg-foreground/10 rotate-45 scale-100', + onClassName: + 'bg-foreground drop-shadow-[0px_0px_8px_2px_rgba(255,182,193,0.4)] rotate-[405deg] scale-200', + }, + // 10 + { + radius: 4, + interval: 75, + GridComponent: PlusSVG, + className: 'gap-2', + baseClassName: 'size-2 rotate-45', + offClassName: 'text-foreground/10 rotate-45', + onClassName: 'text-[#F97A1F] drop-shadow-[0px_0px_3.2px_#F97A1F] rotate-[405deg] scale-250', + }, + // 11 + { + radius: 3, + interval: 75, + GridComponent: EyeSVG, + className: 'gap-2', + baseClassName: 'size-2', + offClassName: 'text-foreground/10', + onClassName: 'text-[#B11FF9] drop-shadow-[0px_0px_16px_2px_rgba(177,31,249,0.6)] scale-150', + transformer: (index: number, rowCount: number, columnCount: number) => { + const rowMidPoint = Math.floor(rowCount / 2); + const distanceFromCenter = Math.sqrt( + Math.pow(rowMidPoint - (index % columnCount), 2) + + Math.pow(rowMidPoint - Math.floor(index / columnCount), 2) + ); + + return { + opacity: 0.5 - distanceFromCenter / columnCount, + }; + }, + }, + // 12 + { + radius: 6, + interval: 75, + className: 'gap-2.5', + baseClassName: 'w-3 h-px rotate-45', + offClassName: 'bg-foreground/10 rotate-45 scale-100', + onClassName: + 'bg-[#FFB6C1] shadow-[0px_0px_8px_2px_rgba(255,182,193,0.4)] rotate-[405deg] scale-200', + }, +]; diff --git a/components/livekit/audio-visualizer/audio-grid-visualizer/hooks/useGridAnimator.ts b/components/livekit/audio-visualizer/audio-grid-visualizer/hooks/useGridAnimator.ts new file mode 100644 index 00000000..ce9272eb --- /dev/null +++ b/components/livekit/audio-visualizer/audio-grid-visualizer/hooks/useGridAnimator.ts @@ -0,0 +1,115 @@ +import { useEffect, useState } from 'react'; +import { type AgentState } from '@livekit/components-react'; + +export interface Coordinate { + x: number; + y: number; +} + +export function generateConnectingSequence(rows: number, columns: number, radius: number) { + const seq = []; + // const centerX = Math.floor(columns / 2); + const centerY = Math.floor(rows / 2); + + // Calculate the boundaries of the ring based on the ring distance + const topLeft = { + x: Math.max(0, centerY - radius), + y: Math.max(0, centerY - radius), + }; + const bottomRight = { + x: columns - 1 - topLeft.x, + y: Math.min(rows - 1, centerY + radius), + }; + + // Top edge + for (let x = topLeft.x; x <= bottomRight.x; x++) { + seq.push({ x, y: topLeft.y }); + } + + // Right edge + for (let y = topLeft.y + 1; y <= bottomRight.y; y++) { + seq.push({ x: bottomRight.x, y }); + } + + // Bottom edge + for (let x = bottomRight.x - 1; x >= topLeft.x; x--) { + seq.push({ x, y: bottomRight.y }); + } + + // Left edge + for (let y = bottomRight.y - 1; y > topLeft.y; y--) { + seq.push({ x: topLeft.x, y }); + } + + return seq; +} + +export function generateListeningSequence(rows: number, columns: number) { + const center = { x: Math.floor(columns / 2), y: Math.floor(rows / 2) }; + const noIndex = { x: -1, y: -1 }; + + return [center, noIndex, noIndex, noIndex, noIndex, noIndex, noIndex, noIndex, noIndex]; +} + +export function generateThinkingSequence(rows: number, columns: number) { + const seq = []; + const y = Math.floor(rows / 2); + for (let x = 0; x < columns; x++) { + seq.push({ x, y }); + } + for (let x = columns - 1; x >= 0; x--) { + seq.push({ x, y }); + } + + return seq; +} + +export function useGridAnimator( + state: AgentState, + rows: number, + columns: number, + interval: number, + radius?: number +): Coordinate { + const [index, setIndex] = useState(0); + const [sequence, setSequence] = useState(() => [ + { + x: Math.floor(columns / 2), + y: Math.floor(rows / 2), + }, + ]); + + useEffect(() => { + const clampedRadius = radius + ? Math.min(radius, Math.floor(Math.max(rows, columns) / 2)) + : Math.floor(Math.max(rows, columns) / 2); + + if (state === 'thinking') { + setSequence(generateThinkingSequence(rows, columns)); + } else if (state === 'connecting' || state === 'initializing') { + const sequence = [...generateConnectingSequence(rows, columns, clampedRadius)]; + setSequence(sequence); + } else if (state === 'listening') { + setSequence(generateListeningSequence(rows, columns)); + } else { + setSequence([{ x: Math.floor(columns / 2), y: Math.floor(rows / 2) }]); + } + setIndex(0); + }, [state, rows, columns, radius]); + + useEffect(() => { + if (state === 'speaking') { + return; + } + + const indexInterval = setInterval(() => { + setIndex((prev) => { + return prev + 1; + }); + }, interval); + + return () => clearInterval(indexInterval); + }, [interval, columns, rows, state, sequence.length]); + + return sequence[index % sequence.length]; +} From 082d4332753d977e5a42fe909fee6c5ed40fc4e7 Mon Sep 17 00:00:00 2001 From: Thomas Yuill Date: Sat, 25 Oct 2025 22:19:02 -0400 Subject: [PATCH 05/10] AudioGridVisualizer --- app/ui/_components.tsx | 103 ++++++- .../audio-shader-visualizer.tsx | 129 +++++++++ .../shadcn-io/aurora-shaders/aurora-shader.gl | 231 ++++++++++++++++ .../shadcn-io/aurora-shaders/index copy 2.tsx | 213 ++++++++++++++ .../shadcn-io/aurora-shaders/index copy.tsx | 178 ++++++++++++ .../ui/shadcn-io/aurora-shaders/index.tsx | 261 ++++++++++++++++++ .../shadcn-io/cosmic-waves-shaders/index.tsx | 168 +++++++++++ .../shadcn-io/singularity-shaders/index.tsx | 78 ++++++ package.json | 1 + pnpm-lock.yaml | 14 + 10 files changed, 1371 insertions(+), 5 deletions(-) create mode 100644 components/livekit/audio-visualizer/audio-shader-visualizer/audio-shader-visualizer.tsx create mode 100644 components/ui/shadcn-io/aurora-shaders/aurora-shader.gl create mode 100644 components/ui/shadcn-io/aurora-shaders/index copy 2.tsx create mode 100644 components/ui/shadcn-io/aurora-shaders/index copy.tsx create mode 100644 components/ui/shadcn-io/aurora-shaders/index.tsx create mode 100644 components/ui/shadcn-io/cosmic-waves-shaders/index.tsx create mode 100644 components/ui/shadcn-io/singularity-shaders/index.tsx diff --git a/app/ui/_components.tsx b/app/ui/_components.tsx index 6dea0072..06b93c31 100644 --- a/app/ui/_components.tsx +++ b/app/ui/_components.tsx @@ -3,11 +3,13 @@ import { useEffect, useMemo, useState } from 'react'; import { type VariantProps } from 'class-variance-authority'; import { Track } from 'livekit-client'; +import { RoomAudioRenderer, StartAudio } from '@livekit/components-react'; import { type AgentState, type TrackReference, type TrackReferenceOrPlaceholder, useLocalParticipant, + useVoiceAssistant, } from '@livekit/components-react'; import { MicrophoneIcon } from '@phosphor-icons/react/dist/ssr'; import { useSession } from '@/components/app/session-provider'; @@ -22,15 +24,13 @@ import { AudioBarVisualizer, audioBarVisualizerVariants, } from '@/components/livekit/audio-visualizer/audio-bar-visualizer/audio-bar-visualizer'; -import { - AudioGridVisualizer, - type GridOptions, -} from '@/components/livekit/audio-visualizer/audio-grid-visualizer/audio-grid-visualizer'; +import { AudioGridVisualizer } from '@/components/livekit/audio-visualizer/audio-grid-visualizer/audio-grid-visualizer'; import { gridVariants } from '@/components/livekit/audio-visualizer/audio-grid-visualizer/demos'; import { AudioRadialVisualizer, audioRadialVisualizerVariants, } from '@/components/livekit/audio-visualizer/audio-radial-visualizer/audio-radial-visualizer'; +import { AudioShaderVisualizer } from '@/components/livekit/audio-visualizer/audio-shader-visualizer/audio-shader-visualizer'; import { Button, buttonVariants } from '@/components/livekit/button'; import { ChatEntry } from '@/components/livekit/chat-entry'; import { @@ -442,11 +442,11 @@ export const COMPONENTS = { 'speaking', ] as AgentState[]; - const { microphoneTrack, localParticipant } = useLocalParticipant(); const [rowCount, setRowCount] = useState(rowCounts[0]); const [columnCount, setColumnCount] = useState(columnCounts[0]); const [state, setState] = useState(states[0]); const [demoIndex, setDemoIndex] = useState(0); + const { microphoneTrack, localParticipant } = useLocalParticipant(); const micTrackRef = useMemo(() => { return state === 'speaking' @@ -562,6 +562,99 @@ export const COMPONENTS = { ); }, + AudioShaderVisualizer: () => { + const [presetIndex, setPresetIndex] = useState(3); + + // speed + const [a, setA] = useState(50); + // // color scale + const [h, setH] = useState(0.1); + // // color position + const [i, setI] = useState(0.15); + // blur + const [f, setF] = useState(0.1); + // shape + const [g, setG] = useState(1.0); + + const { + // state, + audioTrack, + } = useVoiceAssistant(); + + useMicrophone(); + + const fields = [ + ['speed', a, setA, 0, 250, 10], + ['color position', i, setI, 0, 1, 0.01], + ['color scale', h, setH, 0, 1, 0.01], + ['blur', f, setF, 0, 2, 0.01], + ['shape', g, setG, 1, 5, 1], + ] as const; + + return ( + + + +
+ +
+
+ Preset + +
+ + {fields.map(([name, value, setValue, min = 0.1, max = 10, step = 0.1]) => { + // Use 0-1 range for color phase channels + const isColorPhase = name.toString().startsWith('colorPhase'); + + return ( +
+
+ {name} +
+ {isColorPhase ? Number(value).toFixed(2) : String(value)} +
+
+ setValue(parseFloat(e.target.value))} + className="w-full" + /> +
+ ); + })} +
+
+ + ); + }, + // Agent control bar AgentControlBar: () => { useMicrophone(); diff --git a/components/livekit/audio-visualizer/audio-shader-visualizer/audio-shader-visualizer.tsx b/components/livekit/audio-visualizer/audio-shader-visualizer/audio-shader-visualizer.tsx new file mode 100644 index 00000000..dc716908 --- /dev/null +++ b/components/livekit/audio-visualizer/audio-shader-visualizer/audio-shader-visualizer.tsx @@ -0,0 +1,129 @@ +'use client'; + +import { + type TrackReference, + type TrackReferenceOrPlaceholder, + // useMultibandTrackVolume, + useTrackVolume, +} from '@livekit/components-react'; +import { AuroraShaders, type AuroraShadersProps } from '@/components/ui/shadcn-io/aurora-shaders'; +import { cn } from '@/lib/utils'; + +const PRESETS = [ + (volume: number) => ({ + amplitude: 0.3, + scale: 0.3 - 0.1 * (volume * 1.5), + frequency: 0.25, + brightness: 0.5 + 2.5 * volume, + }), + (volume: number) => ({ + amplitude: 0.2 + 1 * volume, + scale: 0.3 - 0.1 * (volume * 1.5), + frequency: 0.25 + 5 * volume, + brightness: 0.5 + 2.5 * volume, + }), + (volume: number) => ({ + amplitude: 0.5 + 0.1 * volume, + scale: 0.2 + 0.1 * (volume * 1.5), + frequency: 5 - 6 * volume, + brightness: 0.5 + 2.5 * volume, + }), + (volume: number) => ({ + amplitude: 0.5 + 0.1 * volume, + scale: 0.2 + 0.1 * volume * 0.5, + frequency: 1 - 1 * volume, + brightness: 0.5 + 2.5 * volume, + }), +]; + +export function AudioShaderVisualizer({ + speed = 1.0, + // intensity = 0, + blur = 0.2, + shape = 1, + colorPosition = 0.5, + colorScale = 0.1, + audioTrack, + presetIndex = 0, + className, +}: AuroraShadersProps & { presetIndex?: number; audioTrack?: TrackReferenceOrPlaceholder }) { + // const [volume] = useMultibandTrackVolume(audioTrack, { + // bands: 1, + // loPass: 100, + // hiPass: 150, + // }); + const volume = useTrackVolume(audioTrack as TrackReference, { + fftSize: 2048, + smoothingTimeConstant: 0.5, + }); + + const { amplitude, scale, frequency, brightness } = PRESETS[presetIndex](volume); + + return ( +
+ +
+ ); +} + +// import { +// CosmicWavesShaders, +// type CosmicWavesShadersProps, +// } from '@/components/ui/shadcn-io/cosmic-waves-shaders'; + +// export function AudioShaderVisualizer({ +// speed = 1.0, +// amplitude = 1.0, +// frequency = 1.0, +// starDensity = 1.0, +// colorShift = 1.0, +// }: CosmicWavesShadersProps) { +// return ( +//
+// +//
+// ); +// } + +// import { +// SingularityShaders, +// type SingularityShadersProps, +// } from '@/components/ui/shadcn-io/singularity-shaders'; + +// export function AudioShaderVisualizer({ +// speed = 1.0, +// intensity = 1.0, +// size = 1.0, +// waveStrength = 1.0, +// colorShift = 1.0, +// }: SingularityShadersProps) { +// return ( +//
+// +//
+// ); +// } diff --git a/components/ui/shadcn-io/aurora-shaders/aurora-shader.gl b/components/ui/shadcn-io/aurora-shaders/aurora-shader.gl new file mode 100644 index 00000000..cdd061d4 --- /dev/null +++ b/components/ui/shadcn-io/aurora-shaders/aurora-shader.gl @@ -0,0 +1,231 @@ +#version 300 es +precision mediump float; + +in vec2 vTextureCoord; +uniform sampler2D uTexture; +uniform sampler2D uCustomTexture; +uniform vec2 uPos; +uniform float uBlur; +uniform float uScale; +uniform float uFrequency; +uniform float uAngle; +uniform float uAmplitude; +uniform float uTime; +uniform float uBloom; +uniform float uMix; +uniform float uSpacing; +uniform int uBlendMode; +uniform int uShape; +uniform int uWaveType; +uniform int uColorPalette; +uniform float uColorScale; +uniform float uColorPosition; +uniform float uVariance; +uniform float uSmoothing; +uniform float uPhase; +uniform float uMouseInfluence; +uniform vec3 uColor; +${Fe} +${ys} +${Ts} + +out vec4 fragColor; + +ivec2 customTexSize; +float customTexAspect; + +const float PI = 3.14159265359; +const float TAU = 6.28318530718; + +vec3 pal( in float t, in vec3 a, in vec3 b, in vec3 c, in vec3 d ) { + return a + b*cos( TAU*(c*t+d) ); +} + +vec3 Tonemap_Reinhard(vec3 x) { + x *= 4.; + return x / (1.0 + x); +} + +float sdCircle(vec2 st, float r) { + return length(st) - r; +} + +float sdEllipse(vec2 st, float r) { + float a = length(st + vec2(0, r * 0.8)) - r; + float b = length(st + vec2(0, -r * 0.8)) - r; + return (a + b); +} + +float sdArc(vec2 st, float r) { + return length(st * vec2(0, r)) - r; +} + +float sdLine(vec2 p, float r) { + float halfLen = r * 2.; + vec2 a = vec2(-halfLen, 0.0); + vec2 b = vec2(halfLen, 0.0); + vec2 pa = p - a; + vec2 ba = b - a; + float h = clamp(dot(pa, ba) / dot(ba, ba), 0.0, 1.0); + return length(pa - ba * h); +} + +float sdBox(vec2 p, float r, float md) { + vec2 q = abs(p)-vec2(r); + return length(max(q,0.0)) + min(max(q.x,q.y),0.0) - r * mix(0., 0.3333 * md, uAmplitude); +} + +float sdEquilateralTriangle(vec2 p, float r, float md) { + const float k = sqrt(3.0); + p.x = abs(p.x) - r; + p.y = p.y + r/k; + if( p.x+k*p.y>0.0 ) p = vec2(p.x-k*p.y,-k*p.x-p.y)/2.0; + p.x -= clamp( p.x, -2.0*r, 0.0 ); + return -length(p)*sign(p.y) - r * mix(0., 0.3333 * md, uAmplitude); +} + +float median(float r, float g, float b) { + return max(min(r, g), min(max(r, g), b)); +} + +float screenPxRange() { + vec2 unitRange = 85./vec2(512); + vec2 screenTexSize = vec2(1.0)/fwidth(vTextureCoord); + return max(0.5*dot(unitRange, screenTexSize), 1.0); +} + +float sdCustom(vec2 uv) { + ivec2 customTexSize = textureSize(uCustomTexture, 0); + float customTexAspect = float(customTexSize.x) / float(customTexSize.y); + + uv.x /= customTexAspect; + uv /= (uScale * 2.5); + uv += 0.5; + + if(uv.x < 0. || uv.x > 1. || uv.y < 0. || uv.y > 1.) { + return 1.; + } + + vec4 sdColor = texture(uCustomTexture, uv); + float msdf = median(sdColor.r, sdColor.g, sdColor.b); + float sd = msdf; + float screenPxDistance = -(sd - 0.51); + return screenPxDistance * 2.; +} + +float getSdf(vec2 st, float iter, float md) { + switch(uShape) { + case 0: return sdCustom(st); break; + case 1: return sdCircle(st, uScale); break; + case 2: return sdEllipse(st, uScale); break; + case 3: return sdLine(st, uScale); break; + case 4: return sdBox(st, uScale, md); break; + case 5: return sdEquilateralTriangle(st, uScale, md); break; + default: return 0.; break; + } +} + +vec2 turb(vec2 pos, float t, float it, float md, vec2 mPos) { + mat2 rot = mat2(0.6, -0.8, 0.8, 0.6); + float freq = mix(2., 15., uFrequency); + float amp = (uAmplitude) * md; + float xp = 1.4; + float time = t * 0.1 + uPhase; + + for(float i = 0.; i < 4.; i++) { + vec2 s = sin(freq * ((pos - mPos) * rot) + i * time + it); + pos += amp * rot[0] * s / freq; + rot *= mat2(0.6, -0.8, 0.8, 0.6); + amp *= mix(1., max(s.y, s.x), uVariance); + freq *= xp; + } + + return pos; +} + + +float luma(vec3 color) { + return dot(color, vec3(0.299, 0.587, 0.114)); +} + +const float ITERATIONS = 36.; + +float expApprox(float x) { + x = clamp(x, -4.0, 4.0); + float x2 = x * x; + return 1.0 + x + 0.5 * x2 + (1.0/6.0) * x2 * x; + } + +void main() { + vec2 uv = vTextureCoord; + vec4 bg = texture(uTexture, uv); + + if(uShape == 0) { + customTexSize = textureSize(uCustomTexture, 0); + customTexAspect = float(customTexSize.x) / float(customTexSize.y); + } + + vec3 pp = vec3(0.); + vec3 bloom = vec3(0.); + float t = uTime * 0.5 + uPhase; + vec2 aspect = vec2(uResolution.x/uResolution.y, 1); + vec2 mousePos = mix(vec2(0), uMousePos - 0.5, uTrackMouse); + vec2 pos = (uv * aspect - uPos * aspect); + float mDist = length(uv * aspect - uMousePos * aspect); + float md = mix(1., smoothstep(1., 5., 1./mDist), uMouseInfluence); + float rotation = uAngle * -2.0 * 3.14159265359; + mat2 rotMatrix = mat2(cos(rotation), -sin(rotation), sin(rotation), cos(rotation)); + pos = rotMatrix * pos; + float bm = 0.05; + + // #ifelseopen + if(uShape == 0) { + bm = 0.2; + } + // #ifelseclose + + vec2 prevPos = turb(pos, t, 0. - 1./ITERATIONS, md, mousePos); + float spacing = mix(1., TAU, uSpacing); + float smoothing = uShape == 0 ? uSmoothing * 2. : uSmoothing; + + for(float i = 1.; i < ITERATIONS + 1.; i++) { + float iter = i/ITERATIONS; + vec2 st = turb(pos, t, iter * spacing, md, mousePos); + float d = abs(getSdf(st, iter, md)); + float pd = distance(st, prevPos); + prevPos = st; + float dynamicBlur = exp2(pd * 2.0 * 1.4426950408889634) - 1.0; + float ds = smoothstep(0., uBlur * bm + max(dynamicBlur * smoothing, 0.001), d); + vec3 color = pal(iter * mix(0.1, 1.9, uColorScale) + uColorPosition, vec3(0.5), vec3(0.5), vec3(1), uColor); + float invd = 1./max(d + dynamicBlur, 0.001); + pp += (ds - 1.) * color; + bloom += clamp(invd, 0., 250.) * color; + } + + pp *= 1./ITERATIONS; + bloom = bloom / (bloom + 2e4); + + // #ifelseopen + if(uShape == 0) { + pp *= 2.; + bloom *= 2.; + } + // #ifelseclose + + + vec3 color = (-pp + bloom * 3. * uBloom); + color *= 1.2; + color += (randFibo(gl_FragCoord.xy) - 0.5) / 255.0; + color = (Tonemap_Reinhard(color)); + vec4 auroraColor = vec4(color, 1.); + + // #ifelseopen + if(uBlendMode > 0) { + auroraColor.rgb = blend(uBlendMode, bg.rgb, auroraColor.rgb); + } + // #ifelseclose + + auroraColor = vec4(mix(bg.rgb, auroraColor.rgb, uMix), max(bg.a, luma(auroraColor.rgb))); + + ${ze("auroraColor")} +} diff --git a/components/ui/shadcn-io/aurora-shaders/index copy 2.tsx b/components/ui/shadcn-io/aurora-shaders/index copy 2.tsx new file mode 100644 index 00000000..c8f389f0 --- /dev/null +++ b/components/ui/shadcn-io/aurora-shaders/index copy 2.tsx @@ -0,0 +1,213 @@ +'use client'; + +import React, { forwardRef } from 'react'; +import { Shader } from 'react-shaders'; +import { cn } from '@/lib/utils'; + +export interface AuroraShadersProps extends React.HTMLAttributes { + /** + * Aurora wave speed + * @default 1.0 + */ + speed?: number; + + /** + * Light intensity and brightness + * @default 1.0 + */ + intensity?: number; + + /** + * Color vibrancy and saturation + * @default 1.0 + */ + vibrancy?: number; + + /** + * Wave frequency and complexity + * @default 1.0 + */ + frequency?: number; + + /** + * Vertical stretch of aurora bands (ring thickness) + * @default 1.0 + */ + stretch?: number; + + /** + * Ring radius from center + * @default 0.3 + */ + radius?: number; +} + +const auroraShader = ` +// Noise function for organic movement +float noise(vec2 p) { + return fract(sin(dot(p, vec2(127.1, 311.7))) * 43758.5453); +} + +// Smooth noise for flowing effects +float smoothNoise(vec2 p) { + vec2 i = floor(p); + vec2 f = fract(p); + + float a = noise(i); + float b = noise(i + vec2(1.0, 0.0)); + float c = noise(i + vec2(0.0, 1.0)); + float d = noise(i + vec2(1.0, 1.0)); + + vec2 u = f * f * (3.0 - 2.0 * f); + + return mix(a, b, u.x) + (c - a) * u.y * (1.0 - u.x) + (d - b) * u.x * u.y; +} + +// Fractal noise for complex aurora patterns +float fractalNoise(vec2 p) { + float value = 0.0; + float amplitude = 0.5; + + for(int i = 0; i < 4; i++) { + value += amplitude * smoothNoise(p); + p *= 2.0; + amplitude *= 0.5; + } + + return value; +} + +void mainImage( out vec4 fragColor, in vec2 fragCoord ) +{ + // Normalize coordinates and center them + vec2 uv = fragCoord / iResolution.xy; + vec2 center = vec2(0.5, 0.5); + vec2 pos = uv - center; + + // Adjust aspect ratio + pos.x *= iResolution.x / iResolution.y; + + // Convert to polar coordinates + float angle = atan(pos.y, pos.x); + float dist = length(pos); + + // Normalize angle from -PI..PI to 0..1 + float normalizedAngle = (angle / 6.28318530718) + 0.5; + + // Time with speed control + float time = iTime * u_speed; + + // Create radial gradient for ring positioning (controls ring thickness) + float ringCenter = u_radius; // Distance from center where ring appears + float ringWidth = 0.15 * u_stretch; // Width of the ring + float radialGradient = 1.0 - abs(dist - ringCenter) / ringWidth; + radialGradient = clamp(radialGradient, 0.0, 1.0); + radialGradient = pow(radialGradient, 1.5); + + // Create seamless angular coordinate by using sin/cos for tiling + // This ensures the pattern wraps perfectly around the ring + float angularCoord = normalizedAngle * 6.28318530718 + time * 0.1 * 6.28318530718; + vec2 seamlessUV = vec2(cos(angularCoord), sin(angularCoord)); + + // Generate multiple aurora layers with different characteristics + // Using seamless UV coordinates to avoid seams + float aurora1 = fractalNoise(seamlessUV * u_frequency * 3.0 + vec2(time * 0.2, dist * 10.0)); + float aurora2 = fractalNoise(seamlessUV * u_frequency * 2.0 + vec2(time * 0.15 + 1000.0, dist * 8.0)); + float aurora3 = fractalNoise(seamlessUV * u_frequency * 4.0 + vec2(time * 0.25 + 2000.0, dist * 12.0)); + + // Add wave distortion for organic movement (radial waves) + float wave1 = sin(normalizedAngle * 8.0 * 6.28318530718 + time * 2.0) * 0.02; + float wave2 = sin(normalizedAngle * 12.0 * 6.28318530718 + time * 1.5) * 0.01; + + float distortedDist = dist + wave1 + wave2; + + // Apply radial positioning to aurora layers (creates the ring shape) + aurora1 *= smoothstep(ringCenter - ringWidth, ringCenter, distortedDist) * + smoothstep(ringCenter + ringWidth, ringCenter, distortedDist); + aurora2 *= smoothstep(ringCenter - ringWidth * 0.8, ringCenter, distortedDist) * + smoothstep(ringCenter + ringWidth * 0.8, ringCenter, distortedDist); + aurora3 *= smoothstep(ringCenter - ringWidth * 0.9, ringCenter, distortedDist) * + smoothstep(ringCenter + ringWidth * 0.9, ringCenter, distortedDist); + + // Combine aurora layers + float combinedAurora = (aurora1 * 0.6 + aurora2 * 0.8 + aurora3 * 0.4) * radialGradient; + + // Apply intensity scaling + combinedAurora *= u_intensity; + + // Create aurora color palette + vec3 color1 = vec3(0.0, 0.8, 0.4); // Green + vec3 color2 = vec3(0.2, 0.4, 1.0); // Blue + vec3 color3 = vec3(0.8, 0.2, 0.8); // Purple + vec3 color4 = vec3(0.0, 1.0, 0.8); // Cyan + + // Create seamless color transitions using sine waves for smooth wrapping + float colorPhase = normalizedAngle * 6.28318530718 * 2.0; // Two full cycles around ring + float colorMix = (sin(colorPhase) + 1.0) * 0.5; // 0 to 1 + float colorMix2 = (sin(colorPhase + 2.094395) + 1.0) * 0.5; // Offset by 120 degrees + float colorMix3 = (sin(colorPhase + 4.18879) + 1.0) * 0.5; // Offset by 240 degrees + + // Mix colors smoothly with seamless transitions + vec3 finalColor = mix(color1, color2, colorMix); + finalColor = mix(finalColor, color3, colorMix2 * 0.5); + finalColor = mix(finalColor, color4, colorMix3 * 0.3); + + // Apply vibrancy control + vec3 desaturated = vec3(dot(finalColor, vec3(0.299, 0.587, 0.114))); + finalColor = mix(desaturated, finalColor, u_vibrancy); + + // Apply aurora intensity + finalColor *= combinedAurora; + + // Add atmospheric glow around the ring + float ringGlow = exp(-abs(dist - ringCenter) * 8.0) * 0.15; + finalColor += finalColor * ringGlow; + + // Ensure colors stay in valid range + finalColor = clamp(finalColor, 0.0, 1.0); + + // Calculate alpha based on aurora intensity for transparency + float alpha = length(finalColor); + alpha = clamp(alpha, 0.0, 1.0); + + fragColor = vec4(finalColor, alpha); +} +`; + +export const AuroraShaders = forwardRef( + ( + { + className, + speed = 1.0, + intensity = 1.0, + vibrancy = 1.0, + frequency = 1.0, + stretch = 1.0, + radius = 0.3, + ...props + }, + ref + ) => { + console.log('radius', radius); + return ( +
+ +
+ ); + } +); + +AuroraShaders.displayName = 'AuroraShaders'; + +export default AuroraShaders; diff --git a/components/ui/shadcn-io/aurora-shaders/index copy.tsx b/components/ui/shadcn-io/aurora-shaders/index copy.tsx new file mode 100644 index 00000000..09043855 --- /dev/null +++ b/components/ui/shadcn-io/aurora-shaders/index copy.tsx @@ -0,0 +1,178 @@ +'use client'; + +import React, { forwardRef } from 'react'; +import { Shader } from 'react-shaders'; +import { cn } from '@/lib/utils'; + +export interface AuroraShadersProps extends React.HTMLAttributes { + /** + * Aurora wave speed + * @default 1.0 + */ + speed?: number; + + /** + * Light intensity and brightness + * @default 1.0 + */ + intensity?: number; + + /** + * Color vibrancy and saturation + * @default 1.0 + */ + vibrancy?: number; + + /** + * Wave frequency and complexity + * @default 1.0 + */ + frequency?: number; + + /** + * Vertical stretch of aurora bands + * @default 1.0 + */ + stretch?: number; +} + +const auroraShader = ` +// Noise function for organic movement +float noise(vec2 p) { + return fract(sin(dot(p, vec2(127.1, 311.7))) * 43758.5453); +} + +// Smooth noise for flowing effects +float smoothNoise(vec2 p) { + vec2 i = floor(p); + vec2 f = fract(p); + + float a = noise(i); + float b = noise(i + vec2(1.0, 0.0)); + float c = noise(i + vec2(0.0, 1.0)); + float d = noise(i + vec2(1.0, 1.0)); + + vec2 u = f * f * (3.0 - 2.0 * f); + + return mix(a, b, u.x) + (c - a) * u.y * (1.0 - u.x) + (d - b) * u.x * u.y; +} + +// Fractal noise for complex aurora patterns +float fractalNoise(vec2 p) { + float value = 0.0; + float amplitude = 0.5; + + for(int i = 0; i < 4; i++) { + value += amplitude * smoothNoise(p); + p *= 2.0; + amplitude *= 0.5; + } + + return value; +} + +void mainImage( out vec4 fragColor, in vec2 fragCoord ) +{ + // Normalize coordinates + vec2 uv = fragCoord / iResolution.xy; + + // Time with speed control + float time = iTime * u_speed; + + // Create vertical gradient for aurora positioning + float verticalGradient = 1.0 - abs(uv.y - 0.5) * 2.0; + verticalGradient = pow(verticalGradient, u_stretch); + + // Create flowing horizontal movement + vec2 flowUV = vec2(uv.x + time * 0.1, uv.y); + + // Generate multiple aurora layers with different characteristics + float aurora1 = fractalNoise(flowUV * u_frequency * 3.0 + vec2(time * 0.2, 0.0)); + float aurora2 = fractalNoise(flowUV * u_frequency * 2.0 + vec2(time * 0.15, 1000.0)); + float aurora3 = fractalNoise(flowUV * u_frequency * 4.0 + vec2(time * 0.25, 2000.0)); + + // Add wave distortion for organic movement + float wave1 = sin(uv.x * 8.0 + time * 2.0) * 0.1; + float wave2 = sin(uv.x * 12.0 + time * 1.5) * 0.05; + + float distortedY = uv.y + wave1 + wave2; + + // Apply vertical positioning to aurora layers + aurora1 *= smoothstep(0.3, 0.7, distortedY) * smoothstep(0.8, 0.6, distortedY); + aurora2 *= smoothstep(0.4, 0.6, distortedY) * smoothstep(0.7, 0.5, distortedY); + aurora3 *= smoothstep(0.35, 0.65, distortedY) * smoothstep(0.75, 0.55, distortedY); + + // Combine aurora layers + float combinedAurora = (aurora1 * 0.6 + aurora2 * 0.8 + aurora3 * 0.4) * verticalGradient; + + // Apply intensity scaling + combinedAurora *= u_intensity; + + // Create aurora color palette + vec3 color1 = vec3(0.0, 0.8, 0.4); // Green + vec3 color2 = vec3(0.2, 0.4, 1.0); // Blue + vec3 color3 = vec3(0.8, 0.2, 0.8); // Purple + vec3 color4 = vec3(0.0, 1.0, 0.8); // Cyan + + // Create color zones based on vertical position + float colorMix1 = smoothstep(0.2, 0.4, uv.y); + float colorMix2 = smoothstep(0.4, 0.6, uv.y); + float colorMix3 = smoothstep(0.6, 0.8, uv.y); + + // Mix colors for natural aurora appearance + vec3 finalColor = mix(color1, color2, colorMix1); + finalColor = mix(finalColor, color3, colorMix2); + finalColor = mix(finalColor, color4, colorMix3); + + // Apply vibrancy control + vec3 desaturated = vec3(dot(finalColor, vec3(0.299, 0.587, 0.114))); + finalColor = mix(desaturated, finalColor, u_vibrancy); + + // Apply aurora intensity + finalColor *= combinedAurora; + + // Add atmospheric glow at horizon + float horizonGlow = exp(-abs(uv.y - 0.5) * 8.0) * 0.1; + finalColor += finalColor * horizonGlow; + + // Ensure colors stay in valid range + finalColor = clamp(finalColor, 0.0, 1.0); + + fragColor = vec4(finalColor, 1.0); +} +`; + +export const AuroraShaders = forwardRef( + ( + { + className, + speed = 1.0, + intensity = 1.0, + vibrancy = 1.0, + frequency = 1.0, + stretch = 1.0, + ...props + }, + ref + ) => { + return ( +
+ +
+ ); + } +); + +AuroraShaders.displayName = 'AuroraShaders'; + +export default AuroraShaders; diff --git a/components/ui/shadcn-io/aurora-shaders/index.tsx b/components/ui/shadcn-io/aurora-shaders/index.tsx new file mode 100644 index 00000000..8683a401 --- /dev/null +++ b/components/ui/shadcn-io/aurora-shaders/index.tsx @@ -0,0 +1,261 @@ +'use client'; + +import React, { forwardRef } from 'react'; +import { Shader } from 'react-shaders'; +import { cn } from '@/lib/utils'; + +const auroraShaderSource = ` +const float TAU = 6.28318530718; + +// Cosine palette generator +vec3 pal(in float t, in vec3 a, in vec3 b, in vec3 c, in vec3 d) { + return a + b * cos(TAU * (c * t + d)); +} + +// Noise for dithering +vec2 randFibo(vec2 p) { + p = fract(p * vec2(443.897, 441.423)); + p += dot(p, p.yx + 19.19); + return fract((p.xx + p.yx) * p.xy); +} + +// Tonemap +vec3 Tonemap_Reinhard(vec3 x) { + x *= 4.0; + return x / (1.0 + x); +} + +// Luma for alpha +float luma(vec3 color) { + return dot(color, vec3(0.299, 0.587, 0.114)); +} + +// SDF shapes +float sdCircle(vec2 st, float r) { + return length(st) - r; +} + +float sdEllipse(vec2 st, float r) { + float a = length(st + vec2(0, r * 0.8)) - r; + float b = length(st + vec2(0, -r * 0.8)) - r; + return (a + b); +} + +float sdLine(vec2 p, float r) { + float halfLen = r * 2.0; + vec2 a = vec2(-halfLen, 0.0); + vec2 b = vec2(halfLen, 0.0); + vec2 pa = p - a; + vec2 ba = b - a; + float h = clamp(dot(pa, ba) / dot(ba, ba), 0.0, 1.0); + return length(pa - ba * h); +} + +float sdBox(vec2 p, float r) { + vec2 q = abs(p) - vec2(r); + return length(max(q, 0.0)) + min(max(q.x, q.y), 0.0) - r * mix(0.0, 0.3333, uAmplitude); +} + +float sdEquilateralTriangle(vec2 p, float r) { + const float k = sqrt(3.0); + p.x = abs(p.x) - r; + p.y = p.y + r / k; + if(p.x + k * p.y > 0.0) p = vec2(p.x - k * p.y, -k * p.x - p.y) / 2.0; + p.x -= clamp(p.x, -2.0 * r, 0.0); + return -length(p) * sign(p.y) - r * mix(0.0, 0.3333, uAmplitude); +} + +float getSdf(vec2 st) { + if(uShape == 1) return sdCircle(st, uScale); + else if(uShape == 2) return sdEllipse(st, uScale); + else if(uShape == 3) return sdLine(st, uScale); + else if(uShape == 4) return sdBox(st, uScale); + else if(uShape == 5) return sdEquilateralTriangle(st, uScale); + else return sdCircle(st, uScale); +} + +vec2 turb(vec2 pos, float t, float it) { + mat2 rot = mat2(0.6, -0.8, 0.8, 0.6); + float freq = mix(2.0, 15.0, uFrequency); + float amp = uAmplitude; + float xp = 1.4; + float time = t * 0.1 * uSpeed + uPhase; + + for(float i = 0.0; i < 4.0; i++) { + vec2 s = sin(freq * (pos * rot) + i * time + it); + pos += amp * rot[0] * s / freq; + rot *= mat2(0.6, -0.8, 0.8, 0.6); + amp *= mix(1.0, max(s.y, s.x), uVariance); + freq *= xp; + } + + return pos; +} + +const float ITERATIONS = 36.0; + +void mainImage(out vec4 fragColor, in vec2 fragCoord) { + vec2 uv = fragCoord / iResolution.xy; + + vec3 pp = vec3(0.0); + vec3 bloom = vec3(0.0); + float t = iTime * 0.5 + uPhase; + vec2 pos = uv - 0.5; + + vec2 prevPos = turb(pos, t, 0.0 - 1.0 / ITERATIONS); + float spacing = mix(1.0, TAU, uSpacing); + + for(float i = 1.0; i < ITERATIONS + 1.0; i++) { + float iter = i / ITERATIONS; + vec2 st = turb(pos, t, iter * spacing); + float d = abs(getSdf(st)); + float pd = distance(st, prevPos); + prevPos = st; + float dynamicBlur = exp2(pd * 2.0 * 1.4426950408889634) - 1.0; + float ds = smoothstep(0.0, uBlur * 0.05 + max(dynamicBlur * uSmoothing, 0.001), d); + + // Generate color using cosine palette + vec3 color = pal( + iter * mix(0.0, 3.0, uColorScale) + uColorPosition * 2.0, + vec3(0.5), + vec3(0.5), + vec3(1.0), + vec3(0.0, 0.33, 0.67) + ); + + float invd = 1.0 / max(d + dynamicBlur, 0.001); + pp += (ds - 1.0) * color; + bloom += clamp(invd, 0.0, 250.0) * color; + } + + pp *= 1.0 / ITERATIONS; + bloom = bloom / (bloom + 2e4); + + vec3 color = (-pp + bloom * 3.0 * uBloom); + color *= 1.2; + color += (randFibo(fragCoord).x - 0.5) / 255.0; + color = Tonemap_Reinhard(color); + + float alpha = luma(color) * uMix; + fragColor = vec4(color * uMix, alpha); +} +`; + +export interface AuroraShadersProps extends React.HTMLAttributes { + /** + * Aurora wave speed + * @default 1.0 + */ + speed?: number; + + /** + * Light intensity (bloom) + * @default 2.0 + */ + intensity?: number; + + /** + * Turbulence amplitude + * @default 0.5 + */ + amplitude?: number; + + /** + * Wave frequency and complexity + * @default 0.5 + */ + frequency?: number; + + /** + * Shape scale + * @default 0.3 + */ + scale?: number; + + /** + * Edge blur/softness + * @default 1.0 + */ + blur?: number; + + /** + * Shape type: 1=circle, 2=ellipse, 3=line, 4=box, 5=triangle + * @default 1 + */ + shape?: number; + + /** + * Color palette offset - shifts colors along the gradient (0-1) + * Lower values shift toward start colors, higher toward end colors + * @default 0.5 + * @example 0.0 - cool tones dominate + * @example 0.5 - balanced (default) + * @example 1.0 - warm tones dominate + */ + colorPosition?: number; + + /** + * Color variation across layers (0-1) + * Controls how much colors change between iterations + * @default 0.5 + * @example 0.0 - minimal color variation (more uniform) + * @example 0.5 - moderate variation (default) + * @example 1.0 - maximum variation (rainbow effect) + */ + colorScale?: number; + + /** + * Brightness of the aurora (0-1) + * @default 1.0 + */ + brightness?: number; +} + +export const AuroraShaders = forwardRef( + ( + { + className, + speed = 1.0, + intensity = 0.1, + amplitude = 0.5, + frequency = 0.5, + scale = 0.3, + blur = 1.0, + shape = 1, + colorPosition = 1.0, + colorScale = 1.0, + brightness = 1.0, + ...props + }, + ref + ) => { + return ( +
+ +
+ ); + } +); + +AuroraShaders.displayName = 'AuroraShaders'; + +export default AuroraShaders; diff --git a/components/ui/shadcn-io/cosmic-waves-shaders/index.tsx b/components/ui/shadcn-io/cosmic-waves-shaders/index.tsx new file mode 100644 index 00000000..14bd059f --- /dev/null +++ b/components/ui/shadcn-io/cosmic-waves-shaders/index.tsx @@ -0,0 +1,168 @@ +'use client'; + +import React, { forwardRef } from 'react'; +import { Shader } from 'react-shaders'; +import { cn } from '@/lib/utils'; + +export interface CosmicWavesShadersProps extends React.HTMLAttributes { + speed?: number; + amplitude?: number; + frequency?: number; + starDensity?: number; + colorShift?: number; +} + +const cosmicWavesFragment = ` + +// Hash function for pseudo-random values +float hash(vec2 p) { + return fract(sin(dot(p, vec2(127.1, 311.7))) * 43758.5453); +} + +// Smooth noise function +float noise(vec2 p) { + vec2 i = floor(p); + vec2 f = fract(p); + f = f * f * (3.0 - 2.0 * f); + + float a = hash(i); + float b = hash(i + vec2(1.0, 0.0)); + float c = hash(i + vec2(0.0, 1.0)); + float d = hash(i + vec2(1.0, 1.0)); + + return mix(mix(a, b, f.x), mix(c, d, f.x), f.y); +} + +// Fractal noise +float fbm(vec2 p) { + float value = 0.0; + float amplitude = 0.5; + for(int i = 0; i < 4; i++) { + value += amplitude * noise(p); + p *= 2.0; + amplitude *= 0.5; + } + return value; +} + +// Star field generation +float stars(vec2 p, float density) { + vec2 grid = floor(p * density); + vec2 local = fract(p * density); + + float h = hash(grid); + if(h > 0.95) { + float d = length(local - 0.5); + float star = exp(-d * 20.0); + return star * (0.5 + 0.5 * sin(iTime * 2.0 + h * 10.0)); + } + return 0.0; +} + +void mainImage( out vec4 fragColor, in vec2 fragCoord ) { + vec2 uv = fragCoord.xy / iResolution.xy; + vec2 p = uv * 2.0 - 1.0; + p.x *= iResolution.x / iResolution.y; + + float time = iTime * u_speed; + + // Create flowing wave patterns + vec2 wavePos = p * u_frequency; + wavePos.y += time * 0.3; + + // Multiple wave layers + float wave1 = sin(wavePos.x + cos(wavePos.y + time) * 0.5) * u_amplitude; + float wave2 = sin(wavePos.x * 1.3 - wavePos.y * 0.7 + time * 1.2) * u_amplitude * 0.7; + float wave3 = sin(wavePos.x * 0.8 + wavePos.y * 1.1 - time * 0.8) * u_amplitude * 0.5; + + // Combine waves + float waves = (wave1 + wave2 + wave3) * 0.3; + + // Add fractal noise for organic texture + vec2 noisePos = p * 1.5 + vec2(time * 0.1, time * 0.05); + float noiseValue = fbm(noisePos) * 0.4; + + // Combine waves and noise + float pattern = waves + noiseValue; + + // Create flowing cosmic gradient + float gradient = length(p) * 0.8; + gradient += pattern; + + // Color cycling through cosmic spectrum + vec3 color1 = vec3(0.1, 0.2, 0.8); // Deep blue + vec3 color2 = vec3(0.6, 0.1, 0.9); // Purple + vec3 color3 = vec3(0.1, 0.8, 0.9); // Cyan + vec3 color4 = vec3(0.9, 0.3, 0.6); // Pink + + // Color interpolation based on pattern and time + float colorTime = time * u_colorShift + pattern * 2.0; + vec3 finalColor; + + float t = fract(colorTime * 0.2); + if(t < 0.25) { + finalColor = mix(color1, color2, t * 4.0); + } else if(t < 0.5) { + finalColor = mix(color2, color3, (t - 0.25) * 4.0); + } else if(t < 0.75) { + finalColor = mix(color3, color4, (t - 0.5) * 4.0); + } else { + finalColor = mix(color4, color1, (t - 0.75) * 4.0); + } + + // Apply wave intensity + finalColor *= (0.5 + pattern * 0.8); + + // Add star field + float starField = stars(p + vec2(time * 0.02, time * 0.01), u_starDensity * 15.0); + starField += stars(p * 1.5 + vec2(-time * 0.015, time * 0.008), u_starDensity * 12.0); + + finalColor += vec3(starField * 0.8); + + // Add subtle glow effect + float glow = exp(-length(p) * 0.5) * 0.3; + finalColor += glow * vec3(0.2, 0.4, 0.8); + + // Vignette effect + float vignette = 1.0 - length(uv - 0.5) * 1.2; + vignette = smoothstep(0.0, 1.0, vignette); + + finalColor *= vignette; + + fragColor = vec4(finalColor, 1.0); +} +`; + +export const CosmicWavesShaders = forwardRef( + ( + { + speed = 1.0, + amplitude = 1.0, + frequency = 1.0, + starDensity = 1.0, + colorShift = 1.0, + className, + children, + ...props + }, + ref + ) => { + return ( +
+ +
+ ); + } +); + +CosmicWavesShaders.displayName = 'CosmicWavesShaders'; diff --git a/components/ui/shadcn-io/singularity-shaders/index.tsx b/components/ui/shadcn-io/singularity-shaders/index.tsx new file mode 100644 index 00000000..52a60ba8 --- /dev/null +++ b/components/ui/shadcn-io/singularity-shaders/index.tsx @@ -0,0 +1,78 @@ +'use client'; + +import React, { forwardRef } from 'react'; +import { Shader } from 'react-shaders'; +import { cn } from '@/lib/utils'; + +export interface SingularityShadersProps extends React.HTMLAttributes { + speed?: number; + intensity?: number; + size?: number; + waveStrength?: number; + colorShift?: number; +} + +const fragmentShader = ` +void mainImage(out vec4 O, vec2 F) +{ + float i = .2 * u_speed, a; + vec2 r = iResolution.xy, + p = ( F+F - r ) / r.y / (.7 * u_size), + d = vec2(-1,1), + b = p - i*d, + c = p * mat2(1, 1, d/(.1 + i/dot(b,b))), + v = c * mat2(cos(.5*log(a=dot(c,c)) + iTime*i*u_speed + vec4(0,33,11,0)))/i, + w = vec2(0.0); + + for(float j = 0.0; j < 9.0; j++) { + i++; + w += 1.0 + sin(v * u_waveStrength); + v += .7 * sin(v.yx * i + iTime * u_speed) / i + .5; + } + + i = length( sin(v/.3)*.4 + c*(3.+d) ); + + vec4 colorGrad = vec4(.6,-.4,-1,0) * u_colorShift; + + O = 1. - exp( -exp( c.x * colorGrad ) + / w.xyyx + / ( 2. + i*i/4. - i ) + / ( .5 + 1. / a ) + / ( .03 + abs( length(p)-.7 ) ) + * u_intensity + ); +} +`; + +export const SingularityShaders = forwardRef( + ( + { + className, + speed = 1.0, + intensity = 1.0, + size = 1.0, + waveStrength = 1.0, + colorShift = 1.0, + ...props + }, + ref + ) => { + return ( +
+ +
+ ); + } +); + +SingularityShaders.displayName = 'SingularityShaders'; diff --git a/package.json b/package.json index 0c43a95f..e66a98e3 100644 --- a/package.json +++ b/package.json @@ -29,6 +29,7 @@ "next-themes": "^0.4.6", "react": "^19.0.0", "react-dom": "^19.0.0", + "react-shaders": "^0.0.4", "sonner": "^2.0.3", "tailwind-merge": "^3.3.0" }, diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 2a20acd1..9eff56fc 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -62,6 +62,9 @@ importers: react-dom: specifier: ^19.0.0 version: 19.1.1(react@19.1.1) + react-shaders: + specifier: ^0.0.4 + version: 0.0.4(react-dom@19.1.1(react@19.1.1))(react@19.1.1) sonner: specifier: ^2.0.3 version: 2.0.7(react-dom@19.1.1(react@19.1.1))(react@19.1.1) @@ -2222,6 +2225,12 @@ packages: '@types/react': optional: true + react-shaders@0.0.4: + resolution: {integrity: sha512-uh42UkMeS4DXyoyYzx+SJryCbULJi6eOMZSLO8Xp1+ogn1WjkIwoP5ucjoYfYu3qouYw3vVIi/Amu0/xZ4/wsg==} + peerDependencies: + react: ^18.2.0 + react-dom: ^18.2.0 + react-style-singleton@2.2.3: resolution: {integrity: sha512-b6jSvxvVnyptAiLjbkWLE/lOnR4lfTtDAl+eUC7RZy+QQWc6wRzIV2CE6xBuMmDxc2qIihtDCZD5NPOFl7fRBQ==} engines: {node: '>=10'} @@ -4649,6 +4658,11 @@ snapshots: optionalDependencies: '@types/react': 19.1.12 + react-shaders@0.0.4(react-dom@19.1.1(react@19.1.1))(react@19.1.1): + dependencies: + react: 19.1.1 + react-dom: 19.1.1(react@19.1.1) + react-style-singleton@2.2.3(@types/react@19.1.12)(react@19.1.1): dependencies: get-nonce: 1.0.1 From 9f3993375bad6ef29a15be6d7bb02c5423cafafb Mon Sep 17 00:00:00 2001 From: Thomas Yuill Date: Tue, 28 Oct 2025 21:29:37 -0400 Subject: [PATCH 06/10] AudioShaderVisualizer --- app/ui/_components.tsx | 193 ++++-- components/app/tile-layout.tsx | 15 +- .../audio-bar-visualizer.tsx | 2 +- .../audio-shader-visualizer.tsx | 307 ++++++--- components/livekit/react-shader/logging.ts | 1 + .../react-shader/react-shader.original.tsx | 614 +++++++++++++++++ .../livekit/react-shader/react-shader.tsx | 649 ++++++++++++++++++ components/livekit/react-shader/texture.ts | 197 ++++++ components/livekit/react-shader/uniforms.ts | 145 ++++ .../aurora-shaders/aurora-shader.glsl | 111 +++ ...-shader.gl => aurora-shader.original.glsl} | 0 .../shadcn-io/aurora-shaders/index copy 2.tsx | 213 ------ .../shadcn-io/aurora-shaders/index copy.tsx | 178 ----- .../ui/shadcn-io/aurora-shaders/index.tsx | 93 ++- .../shadcn-io/cosmic-waves-shaders/index.tsx | 2 +- .../shadcn-io/singularity-shaders/index.tsx | 2 +- package.json | 1 - pnpm-lock.yaml | 14 - 18 files changed, 2097 insertions(+), 640 deletions(-) create mode 100644 components/livekit/react-shader/logging.ts create mode 100644 components/livekit/react-shader/react-shader.original.tsx create mode 100644 components/livekit/react-shader/react-shader.tsx create mode 100644 components/livekit/react-shader/texture.ts create mode 100644 components/livekit/react-shader/uniforms.ts create mode 100644 components/ui/shadcn-io/aurora-shaders/aurora-shader.glsl rename components/ui/shadcn-io/aurora-shaders/{aurora-shader.gl => aurora-shader.original.glsl} (100%) delete mode 100644 components/ui/shadcn-io/aurora-shaders/index copy 2.tsx delete mode 100644 components/ui/shadcn-io/aurora-shaders/index copy.tsx diff --git a/app/ui/_components.tsx b/app/ui/_components.tsx index 06b93c31..755d92d5 100644 --- a/app/ui/_components.tsx +++ b/app/ui/_components.tsx @@ -30,7 +30,10 @@ import { AudioRadialVisualizer, audioRadialVisualizerVariants, } from '@/components/livekit/audio-visualizer/audio-radial-visualizer/audio-radial-visualizer'; -import { AudioShaderVisualizer } from '@/components/livekit/audio-visualizer/audio-shader-visualizer/audio-shader-visualizer'; +import { + AudioShaderVisualizer, + audioShaderVisualizerVariants, +} from '@/components/livekit/audio-visualizer/audio-shader-visualizer/audio-shader-visualizer'; import { Button, buttonVariants } from '@/components/livekit/button'; import { ChatEntry } from '@/components/livekit/chat-entry'; import { @@ -52,6 +55,9 @@ type audioBarVisualizerVariantsSizeType = VariantProps['size']; +type audioShaderVisualizerVariantsSizeType = VariantProps< + typeof audioShaderVisualizerVariants +>['size']; export function useMicrophone() { const { startSession } = useSession(); @@ -563,93 +569,142 @@ export const COMPONENTS = { }, AudioShaderVisualizer: () => { - const [presetIndex, setPresetIndex] = useState(3); - - // speed - const [a, setA] = useState(50); - // // color scale - const [h, setH] = useState(0.1); - // // color position - const [i, setI] = useState(0.15); - // blur - const [f, setF] = useState(0.1); + const { startSession, endSession } = useSession(); + const { localParticipant } = useLocalParticipant(); + // shape - const [g, setG] = useState(1.0); + const [shape, setShape] = useState(1.0); + // color scale + const [colorScale, setColorScale] = useState(0.1); + // color position + const [colorPosition, setColorPosition] = useState(0.15); + + const sizes = ['icon', 'sm', 'md', 'lg', 'xl']; + const states = [ + 'disconnected', + 'connecting', + 'initializing', + 'listening', + 'thinking', + 'speaking', + ] as AgentState[]; + + const [size, setSize] = useState('lg'); + const [state, setState] = useState(states[0]); const { // state, audioTrack, } = useVoiceAssistant(); - useMicrophone(); + useEffect(() => { + if (state === 'speaking') { + startSession(); + localParticipant.setMicrophoneEnabled(true, undefined); + } else { + endSession(); + localParticipant.setMicrophoneEnabled(false, undefined); + } + }, [startSession, endSession, state, localParticipant]); const fields = [ - ['speed', a, setA, 0, 250, 10], - ['color position', i, setI, 0, 1, 0.01], - ['color scale', h, setH, 0, 1, 0.01], - ['blur', f, setF, 0, 2, 0.01], - ['shape', g, setG, 1, 5, 1], + ['color position', colorPosition, setColorPosition, 0, 1, 0.01], + ['color scale', colorScale, setColorScale, 0, 1, 0.01], ] as const; return ( -
+ +
+
+ + +
+ +
+ + +
+ +
+ + +
+
+ +
-
-
- Preset - -
+
- {fields.map(([name, value, setValue, min = 0.1, max = 10, step = 0.1]) => { - // Use 0-1 range for color phase channels - const isColorPhase = name.toString().startsWith('colorPhase'); - - return ( -
-
- {name} -
- {isColorPhase ? Number(value).toFixed(2) : String(value)} -
-
- setValue(parseFloat(e.target.value))} - className="w-full" - /> +
+ {fields.map(([name, value, setValue, min = 0.1, max = 10, step = 0.1]) => { + return ( +
+
+ {name} +
{String(value)}
- ); - })} -
+ setValue(parseFloat(e.target.value))} + className="w-full" + /> +
+ ); + })}
); diff --git a/components/app/tile-layout.tsx b/components/app/tile-layout.tsx index 9de67c97..38a7e481 100644 --- a/components/app/tile-layout.tsx +++ b/components/app/tile-layout.tsx @@ -8,7 +8,8 @@ import { useTracks, useVoiceAssistant, } from '@livekit/components-react'; -import { AudioBarVisualizer } from '@/components/livekit/audio-visualizer/audio-bar-visualizer/audio-bar-visualizer'; +// import { AudioBarVisualizer } from '@/components/livekit/audio-visualizer/audio-bar-visualizer/audio-bar-visualizer'; +import { AudioShaderVisualizer } from '@/components/livekit/audio-visualizer/audio-shader-visualizer/audio-shader-visualizer'; import { cn } from '@/lib/utils'; const MotionContainer = motion.create('div'); @@ -92,7 +93,7 @@ export function TileLayout({ chatOpen }: TileLayoutProps) { const videoHeight = agentVideoTrack?.publication.dimensions?.height ?? 0; return ( -
+
{/* Agent */} @@ -112,7 +113,7 @@ export function TileLayout({ chatOpen }: TileLayoutProps) { layoutId="agent" initial={{ opacity: 0, - scale: 0, + scale: chatOpen ? 1 : 6, }} animate={{ opacity: 1, @@ -127,11 +128,17 @@ export function TileLayout({ chatOpen }: TileLayoutProps) { chatOpen && 'border-input/50 drop-shadow-lg/10 delay-200' )} > - */} + )} diff --git a/components/livekit/audio-visualizer/audio-bar-visualizer/audio-bar-visualizer.tsx b/components/livekit/audio-visualizer/audio-bar-visualizer/audio-bar-visualizer.tsx index 8701c967..b7657097 100644 --- a/components/livekit/audio-visualizer/audio-bar-visualizer/audio-bar-visualizer.tsx +++ b/components/livekit/audio-visualizer/audio-bar-visualizer/audio-bar-visualizer.tsx @@ -84,7 +84,7 @@ export function AudioBarVisualizer({ case 'connecting': return 2000 / _barCount; case 'initializing': - return 500; + return 2000; case 'listening': return 500; case 'thinking': diff --git a/components/livekit/audio-visualizer/audio-shader-visualizer/audio-shader-visualizer.tsx b/components/livekit/audio-visualizer/audio-shader-visualizer/audio-shader-visualizer.tsx index dc716908..00b749b4 100644 --- a/components/livekit/audio-visualizer/audio-shader-visualizer/audio-shader-visualizer.tsx +++ b/components/livekit/audio-visualizer/audio-shader-visualizer/audio-shader-visualizer.tsx @@ -1,6 +1,16 @@ 'use client'; +import { useEffect, useRef, useState } from 'react'; +import { type VariantProps, cva } from 'class-variance-authority'; import { + type AnimationPlaybackControlsWithThen, + type ValueAnimationTransition, + animate, + useMotionValue, + useMotionValueEvent, +} from 'motion/react'; +import { + type AgentState, type TrackReference, type TrackReferenceOrPlaceholder, // useMultibandTrackVolume, @@ -9,121 +19,208 @@ import { import { AuroraShaders, type AuroraShadersProps } from '@/components/ui/shadcn-io/aurora-shaders'; import { cn } from '@/lib/utils'; -const PRESETS = [ - (volume: number) => ({ - amplitude: 0.3, - scale: 0.3 - 0.1 * (volume * 1.5), - frequency: 0.25, - brightness: 0.5 + 2.5 * volume, - }), - (volume: number) => ({ - amplitude: 0.2 + 1 * volume, - scale: 0.3 - 0.1 * (volume * 1.5), - frequency: 0.25 + 5 * volume, - brightness: 0.5 + 2.5 * volume, - }), - (volume: number) => ({ - amplitude: 0.5 + 0.1 * volume, - scale: 0.2 + 0.1 * (volume * 1.5), - frequency: 5 - 6 * volume, - brightness: 0.5 + 2.5 * volume, - }), - (volume: number) => ({ - amplitude: 0.5 + 0.1 * volume, - scale: 0.2 + 0.1 * volume * 0.5, - frequency: 1 - 1 * volume, - brightness: 0.5 + 2.5 * volume, - }), -]; +// const PRESETS = [ +// (volume: number) => ({ +// amplitude: 0.3, +// scale: 0.35 - 0.05 * volume, +// frequency: 0.25, +// brightness: 1.5 + 2.5 * volume, +// }), +// (volume: number) => ({ +// amplitude: 0.2 + 1 * volume, +// scale: 0.35 - 0.05 * volume, +// frequency: 0.25 + 5 * volume, +// brightness: 1.5 + 2.5 * volume, +// }), +// (volume: number) => ({ +// amplitude: 0.5 + 0.05 * volume, +// scale: 0.35 + 0.05 * volume, +// frequency: 2 - 2 * volume, +// brightness: 1.5 + 2.5 * volume, +// }), +// (volume: number) => ({ +// amplitude: 0.5 + 0.2 * volume, +// scale: 0.35 - 0.05 * volume, +// frequency: 1 - 1 * volume, +// brightness: 1.5 + 2.5 * volume, +// }), +// ]; + +export const audioShaderVisualizerVariants = cva(['aspect-square'], { + variants: { + size: { + icon: 'h-[24px] gap-[2px]', + sm: 'h-[56px] gap-[4px]', + md: 'h-[112px] gap-[8px]', + lg: 'h-[224px] gap-[16px]', + xl: 'h-[448px] gap-[32px]', + }, + }, + defaultVariants: { + size: 'md', + }, +}); + +interface AudioShaderVisualizerProps { + state?: AgentState; + audioTrack: TrackReferenceOrPlaceholder; +} export function AudioShaderVisualizer({ - speed = 1.0, - // intensity = 0, - blur = 0.2, + size = 'md', + state = 'speaking', shape = 1, - colorPosition = 0.5, - colorScale = 0.1, + colorScale = 0.05, + colorPosition = 0.18, audioTrack, - presetIndex = 0, className, -}: AuroraShadersProps & { presetIndex?: number; audioTrack?: TrackReferenceOrPlaceholder }) { - // const [volume] = useMultibandTrackVolume(audioTrack, { - // bands: 1, - // loPass: 100, - // hiPass: 150, - // }); +}: AudioShaderVisualizerProps & + AuroraShadersProps & + VariantProps) { + const [speed, setSpeed] = useState(10); + const [amplitude, setAmplitude] = useState(0.5); + const [frequency, setFrequency] = useState(1.0); + const [scale, setScale] = useState(0.2); + const [brightness, setBrightness] = useState(1.5); + + const amplitudeValue = useMotionValue(0.5); + const frequencyValue = useMotionValue(0.5); + const scaleValue = useMotionValue(0.3); + const brightnessValue = useMotionValue(0); + + const amplitudeControlsRef = useRef(null); + const frequencyControlsRef = useRef(null); + const scaleControlsRef = useRef(null); + const brightnessControlsRef = useRef(null); + + useMotionValueEvent(amplitudeValue, 'change', (value) => setAmplitude(value)); + useMotionValueEvent(frequencyValue, 'change', (value) => setFrequency(value)); + useMotionValueEvent(scaleValue, 'change', (value) => setScale(value)); + useMotionValueEvent(brightnessValue, 'change', (value) => setBrightness(value)); + const volume = useTrackVolume(audioTrack as TrackReference, { - fftSize: 2048, + fftSize: 512, smoothingTimeConstant: 0.5, }); - const { amplitude, scale, frequency, brightness } = PRESETS[presetIndex](volume); + useEffect(() => { + const DEFAULT_TRANSITION: ValueAnimationTransition = { duration: 0.5, ease: 'easeOut' }; - return ( -
- -
- ); -} - -// import { -// CosmicWavesShaders, -// type CosmicWavesShadersProps, -// } from '@/components/ui/shadcn-io/cosmic-waves-shaders'; + switch (state) { + case 'disconnected': + setSpeed(5); + scaleControlsRef.current = animate(scaleValue, 0.2, DEFAULT_TRANSITION); + amplitudeControlsRef.current = animate(amplitudeValue, 1.2, DEFAULT_TRANSITION); + frequencyControlsRef.current = animate(frequencyValue, 0.4, DEFAULT_TRANSITION); + brightnessControlsRef.current = animate(brightnessValue, 1.0, DEFAULT_TRANSITION); + return; + case 'connecting': + setSpeed(50); + scaleControlsRef.current = animate(scaleValue, 0.3, DEFAULT_TRANSITION); + amplitudeControlsRef.current = animate(amplitudeValue, 0.5, DEFAULT_TRANSITION); + frequencyControlsRef.current = animate(frequencyValue, 1, DEFAULT_TRANSITION); + brightnessControlsRef.current = animate(brightnessValue, [0.5, 2.5], { + duration: 1, + repeat: Infinity, + repeatType: 'mirror', + }); + return; + case 'initializing': + setSpeed(30); + scaleControlsRef.current = animate(scaleValue, 0.3, DEFAULT_TRANSITION); + amplitudeControlsRef.current = animate(amplitudeValue, 0.5, DEFAULT_TRANSITION); + frequencyControlsRef.current = animate(frequencyValue, 1, DEFAULT_TRANSITION); + brightnessControlsRef.current = animate(brightnessValue, [0.5, 2.5], { + duration: 0.2, + repeat: Infinity, + repeatType: 'mirror', + }); + return; + case 'listening': + setSpeed(20); + scaleControlsRef.current = animate(scaleValue, [0.3, 0.35], { + duration: 1.5, + repeat: Infinity, + repeatType: 'mirror', + }); + amplitudeControlsRef.current = animate(amplitudeValue, 0.5, DEFAULT_TRANSITION); + frequencyControlsRef.current = animate(frequencyValue, 1.0, DEFAULT_TRANSITION); + brightnessControlsRef.current = animate(brightnessValue, [1.5, 2.5], { + duration: 1.5, + repeat: Infinity, + repeatType: 'mirror', + }); + return; + case 'thinking': + setSpeed(50); + scaleControlsRef.current = animate(scaleValue, [0.35, 0.3], { + duration: 0.5, + repeat: Infinity, + repeatType: 'mirror', + }); + amplitudeControlsRef.current = animate(amplitudeValue, 0.5, { + ...DEFAULT_TRANSITION, + duration: 0.2, + }); + frequencyControlsRef.current = animate(frequencyValue, 2.5, { + ...DEFAULT_TRANSITION, + duration: 0.2, + }); + brightnessControlsRef.current = animate(brightnessValue, [0.5, 2.5], { + duration: 0.2, + repeat: Infinity, + repeatType: 'mirror', + }); + return; + case 'speaking': + setSpeed(50); + scaleControlsRef.current = animate(scaleValue, 0.35, DEFAULT_TRANSITION); + amplitudeControlsRef.current = animate(amplitudeValue, 0.5, DEFAULT_TRANSITION); + frequencyControlsRef.current = animate(frequencyValue, 1.0, DEFAULT_TRANSITION); + brightnessControlsRef.current = animate(brightnessValue, 0.5, DEFAULT_TRANSITION); + return; + } + }, [ + state, + shape, + colorScale, + scaleValue, + colorPosition, + amplitudeValue, + frequencyValue, + brightnessValue, + ]); -// export function AudioShaderVisualizer({ -// speed = 1.0, -// amplitude = 1.0, -// frequency = 1.0, -// starDensity = 1.0, -// colorShift = 1.0, -// }: CosmicWavesShadersProps) { -// return ( -//
-// -//
-// ); -// } + useEffect(() => { + if (state === 'speaking' && volume > 0) { + scaleControlsRef.current?.stop(); + amplitudeControlsRef.current?.stop(); + frequencyControlsRef.current?.stop(); + brightnessControlsRef.current?.stop(); -// import { -// SingularityShaders, -// type SingularityShadersProps, -// } from '@/components/ui/shadcn-io/singularity-shaders'; + scaleValue.set(0.3 - 0.05 * volume); + amplitudeValue.set(0.5 + 0.2 * volume); + frequencyValue.set(1 - 1 * volume); + brightnessValue.set(1.0 + 2.0 * volume); + } + }, [state, volume, scaleValue, amplitudeValue, frequencyValue, brightnessValue]); -// export function AudioShaderVisualizer({ -// speed = 1.0, -// intensity = 1.0, -// size = 1.0, -// waveStrength = 1.0, -// colorShift = 1.0, -// }: SingularityShadersProps) { -// return ( -//
-// -//
-// ); -// } + return ( + + ); +} diff --git a/components/livekit/react-shader/logging.ts b/components/livekit/react-shader/logging.ts new file mode 100644 index 00000000..aa742d8c --- /dev/null +++ b/components/livekit/react-shader/logging.ts @@ -0,0 +1 @@ +export const log = (text: string) => `react-shaders: ${text}`; diff --git a/components/livekit/react-shader/react-shader.original.tsx b/components/livekit/react-shader/react-shader.original.tsx new file mode 100644 index 00000000..aa709302 --- /dev/null +++ b/components/livekit/react-shader/react-shader.original.tsx @@ -0,0 +1,614 @@ +import { type CSSProperties, Component } from 'react'; +import { log } from './logging'; +import { + ClampToEdgeWrapping, + LinearFilter, + LinearMipMapLinearFilter, + LinearMipMapNearestFilter, + MirroredRepeatWrapping, + NearestFilter, + NearestMipMapLinearFilter, + NearestMipMapNearestFilter, + RepeatWrapping, + Texture, +} from './texture'; +import { + type UniformType, + type Vector2, + type Vector3, + type Vector4, + isMatrixType, + isVectorListType, + processUniform, + uniformTypeToGLSLType, +} from './uniforms'; + +export { + ClampToEdgeWrapping, + LinearFilter, + LinearMipMapLinearFilter, + LinearMipMapNearestFilter, + MirroredRepeatWrapping, + NearestFilter, + NearestMipMapLinearFilter, + NearestMipMapNearestFilter, + RepeatWrapping, +}; + +export type { Vector2, Vector3, Vector4 }; + +const PRECISIONS = ['lowp', 'mediump', 'highp']; +const FS_MAIN_SHADER = `\nvoid main(void){ + vec4 color = vec4(0.0,0.0,0.0,1.0); + mainImage( color, gl_FragCoord.xy ); + gl_FragColor = color; +}`; +const BASIC_FS = `void mainImage( out vec4 fragColor, in vec2 fragCoord ) { + vec2 uv = fragCoord/iResolution.xy; + vec3 col = 0.5 + 0.5*cos(iTime+uv.xyx+vec3(0,2,4)); + fragColor = vec4(col,1.0); +}`; +const BASIC_VS = `attribute vec3 aVertexPosition; +void main(void) { + gl_Position = vec4(aVertexPosition, 1.0); +}`; +const UNIFORM_TIME = 'iTime'; +const UNIFORM_TIMEDELTA = 'iTimeDelta'; +const UNIFORM_DATE = 'iDate'; +const UNIFORM_FRAME = 'iFrame'; +const UNIFORM_MOUSE = 'iMouse'; +const UNIFORM_RESOLUTION = 'iResolution'; +const UNIFORM_CHANNEL = 'iChannel'; +const UNIFORM_CHANNELRESOLUTION = 'iChannelResolution'; +const UNIFORM_DEVICEORIENTATION = 'iDeviceOrientation'; + +const latestPointerClientCoords = (e: MouseEvent | TouchEvent) => { + // @ts-expect-error TODO: Deal with this. + return [e.clientX || e.changedTouches[0].clientX, e.clientY || e.changedTouches[0].clientY]; +}; +const lerpVal = (v0: number, v1: number, t: number) => v0 * (1 - t) + v1 * t; +const insertStringAtIndex = (currentString: string, string: string, index: number) => + index > 0 + ? currentString.substring(0, index) + + string + + currentString.substring(index, currentString.length) + : string + currentString; + +type Uniform = { type: string; value: number[] | number }; +export type Uniforms = Record; +type TextureParams = { + url: string; + wrapS?: number; + wrapT?: number; + minFilter?: number; + magFilter?: number; + flipY?: number; +}; + +type Props = { + /** Fragment shader GLSL code. */ + fs: string; + + /** Vertex shader GLSL code. */ + vs?: string; + + /** + * Textures to be passed to the shader. Textures need to be squared or will be + * automatically resized. + * + * Options default to: + * + * ```js + * { + * minFilter: LinearMipMapLinearFilter, + * magFilter: LinearFilter, + * wrapS: RepeatWrapping, + * wrapT: RepeatWrapping, + * } + * ``` + * + * See [textures in the docs](https://rysana.com/docs/react-shaders#textures) + * for details. + */ + textures?: TextureParams[]; + + /** + * Custom uniforms to be passed to the shader. + * + * See [custom uniforms in the + * docs](https://rysana.com/docs/react-shaders#custom-uniforms) for details. + */ + uniforms?: Uniforms; + + /** + * Color used when clearing the canvas. + * + * See [the WebGL + * docs](https://developer.mozilla.org/en-US/docs/Web/API/WebGLRenderingContext/clearColor) + * for details. + */ + clearColor?: Vector4; + + /** + * GLSL precision qualifier. Defaults to `'highp'`. Balance between + * performance and quality. + */ + precision?: 'highp' | 'lowp' | 'mediump'; + + /** Custom inline style for canvas. */ + style?: CSSStyleDeclaration; + + /** Customize WebGL context attributes. See [the WebGL docs](https://developer.mozilla.org/en-US/docs/Web/API/WebGLRenderingContext/getContextAttributes) for details. */ + contextAttributes?: Record; + + /** Lerp value for `iMouse` built-in uniform. Must be between 0 and 1. */ + lerp?: number; + + /** Device pixel ratio. */ + devicePixelRatio?: number; + + /** + * Callback for when the textures are done loading. Useful if you want to do + * something like e.g. hide the canvas until textures are done loading. + */ + onDoneLoadingTextures?: () => void; + + /** Custom callback to handle errors. Defaults to `console.error`. */ + onError?: (error: string) => void; + + /** Custom callback to handle warnings. Defaults to `console.warn`. */ + onWarning?: (warning: string) => void; +}; +export class Shader extends Component { + uniforms: Record< + string, + { type: string; isNeeded: boolean; value?: number[] | number; arraySize?: string } + >; + constructor(props: Props) { + super(props); + this.uniforms = { + [UNIFORM_TIME]: { type: 'float', isNeeded: false, value: 0 }, + [UNIFORM_TIMEDELTA]: { type: 'float', isNeeded: false, value: 0 }, + [UNIFORM_DATE]: { type: 'vec4', isNeeded: false, value: [0, 0, 0, 0] }, + [UNIFORM_MOUSE]: { type: 'vec4', isNeeded: false, value: [0, 0, 0, 0] }, + [UNIFORM_RESOLUTION]: { type: 'vec2', isNeeded: false, value: [0, 0] }, + [UNIFORM_FRAME]: { type: 'int', isNeeded: false, value: 0 }, + [UNIFORM_DEVICEORIENTATION]: { type: 'vec4', isNeeded: false, value: [0, 0, 0, 0] }, + }; + } + static defaultProps = { + textures: [], + contextAttributes: {}, + devicePixelRatio: 1, + vs: BASIC_VS, + precision: 'highp', + onError: console.error, + onWarn: console.warn, + }; + componentDidMount = () => { + this.initWebGL(); + const { fs, vs, clearColor = [0, 0, 0, 1] } = this.props; + const { gl } = this; + if (gl && this.canvas) { + gl.clearColor(...clearColor); + gl.clearDepth(1.0); + gl.enable(gl.DEPTH_TEST); + gl.depthFunc(gl.LEQUAL); + gl.viewport(0, 0, this.canvas.width, this.canvas.height); + this.canvas.height = this.canvas.clientHeight; + this.canvas.width = this.canvas.clientWidth; + this.processCustomUniforms(); + this.processTextures(); + this.initShaders(this.preProcessFragment(fs || BASIC_FS), vs || BASIC_VS); + this.initBuffers(); + requestAnimationFrame(this.drawScene); + this.addEventListeners(); + this.onResize(); + } + }; + shouldComponentUpdate = () => false; + componentWillUnmount() { + const { gl } = this; + if (gl) { + gl.getExtension('WEBGL_lose_context')?.loseContext(); + gl.useProgram(null); + gl.deleteProgram(this.shaderProgram ?? null); + if (this.texturesArr.length > 0) { + for (const texture of this.texturesArr as Texture[]) { + gl.deleteTexture(texture._webglTexture); + } + } + this.shaderProgram = null; + } + this.removeEventListeners(); + cancelAnimationFrame(this.animFrameId ?? 0); + } + setupChannelRes = ({ width, height }: Texture, id: number) => { + const { devicePixelRatio = 1 } = this.props; + // @ts-expect-error TODO: Deal with this. + this.uniforms.iChannelResolution.value[id * 3] = width * devicePixelRatio; + // @ts-expect-error TODO: Deal with this. + this.uniforms.iChannelResolution.value[id * 3 + 1] = height * devicePixelRatio; + // @ts-expect-error TODO: Deal with this. + this.uniforms.iChannelResolution.value[id * 3 + 2] = 0; + // console.log(this.uniforms); + }; + initWebGL = () => { + const { contextAttributes } = this.props; + if (!this.canvas) return; + this.gl = (this.canvas.getContext('webgl', contextAttributes) || + this.canvas.getContext( + 'experimental-webgl', + contextAttributes + )) as WebGLRenderingContext | null; + this.gl?.getExtension('OES_standard_derivatives'); + this.gl?.getExtension('EXT_shader_texture_lod'); + }; + initBuffers = () => { + const { gl } = this; + this.squareVerticesBuffer = gl?.createBuffer(); + gl?.bindBuffer(gl.ARRAY_BUFFER, this.squareVerticesBuffer ?? null); + const vertices = [1.0, 1.0, 0.0, -1.0, 1.0, 0.0, 1.0, -1.0, 0.0, -1.0, -1.0, 0.0]; + gl?.bufferData(gl.ARRAY_BUFFER, new Float32Array(vertices), gl.STATIC_DRAW); + }; + addEventListeners = () => { + const options = { passive: true }; + if (this.uniforms.iMouse?.isNeeded && this.canvas) { + this.canvas.addEventListener('mousemove', this.mouseMove, options); + this.canvas.addEventListener('mouseout', this.mouseUp, options); + this.canvas.addEventListener('mouseup', this.mouseUp, options); + this.canvas.addEventListener('mousedown', this.mouseDown, options); + this.canvas.addEventListener('touchmove', this.mouseMove, options); + this.canvas.addEventListener('touchend', this.mouseUp, options); + this.canvas.addEventListener('touchstart', this.mouseDown, options); + } + if (this.uniforms.iDeviceOrientation?.isNeeded) { + window.addEventListener('deviceorientation', this.onDeviceOrientationChange, options); + } + if (this.canvas) { + this.resizeObserver = new ResizeObserver(this.onResize); + this.resizeObserver.observe(this.canvas); + } + }; + removeEventListeners = () => { + const options = { passive: true } as EventListenerOptions; + if (this.uniforms.iMouse?.isNeeded && this.canvas) { + this.canvas.removeEventListener('mousemove', this.mouseMove, options); + this.canvas.removeEventListener('mouseout', this.mouseUp, options); + this.canvas.removeEventListener('mouseup', this.mouseUp, options); + this.canvas.removeEventListener('mousedown', this.mouseDown, options); + this.canvas.removeEventListener('touchmove', this.mouseMove, options); + this.canvas.removeEventListener('touchend', this.mouseUp, options); + this.canvas.removeEventListener('touchstart', this.mouseDown, options); + } + if (this.uniforms.iDeviceOrientation?.isNeeded) { + window.removeEventListener('deviceorientation', this.onDeviceOrientationChange, options); + } + if (this.resizeObserver) { + this.resizeObserver.disconnect(); + } + }; + onDeviceOrientationChange = ({ alpha, beta, gamma }: DeviceOrientationEvent) => { + this.uniforms.iDeviceOrientation.value = [ + alpha ?? 0, + beta ?? 0, + gamma ?? 0, + window.orientation || 0, + ]; + }; + mouseDown = (e: MouseEvent | TouchEvent) => { + const [clientX, clientY] = latestPointerClientCoords(e); + const mouseX = clientX - (this.canvasPosition?.left ?? 0) - window.pageXOffset; + const mouseY = + (this.canvasPosition?.height ?? 0) - + clientY - + (this.canvasPosition?.top ?? 0) - + window.pageYOffset; + this.mousedown = true; + // @ts-expect-error TODO: Deal with this. + this.uniforms.iMouse.value[2] = mouseX; + // @ts-expect-error TODO: Deal with this. + this.uniforms.iMouse.value[3] = mouseY; + this.lastMouseArr[0] = mouseX; + this.lastMouseArr[1] = mouseY; + }; + mouseMove = (e: MouseEvent | TouchEvent) => { + this.canvasPosition = this.canvas?.getBoundingClientRect(); + const { lerp = 1 } = this.props; + const [clientX, clientY] = latestPointerClientCoords(e); + const mouseX = clientX - (this.canvasPosition?.left ?? 0); + const mouseY = (this.canvasPosition?.height ?? 0) - clientY - (this.canvasPosition?.top ?? 0); + if (lerp !== 1) { + this.lastMouseArr[0] = mouseX; + this.lastMouseArr[1] = mouseY; + } else { + // @ts-expect-error TODO: Deal with this. + this.uniforms.iMouse.value[0] = mouseX; + // @ts-expect-error TODO: Deal with this. + this.uniforms.iMouse.value[1] = mouseY; + } + }; + mouseUp = () => { + // @ts-expect-error TODO: Deal with this. + this.uniforms.iMouse.value[2] = 0; + // @ts-expect-error TODO: Deal with this. + this.uniforms.iMouse.value[3] = 0; + }; + onResize = () => { + const { gl } = this; + const { devicePixelRatio = 1 } = this.props; + if (!gl) return; + this.canvasPosition = this.canvas?.getBoundingClientRect(); + // Force pixel ratio to be one to avoid expensive calculus on retina display. + const realToCSSPixels = devicePixelRatio; + const displayWidth = Math.floor((this.canvasPosition?.width ?? 1) * realToCSSPixels); + const displayHeight = Math.floor((this.canvasPosition?.height ?? 1) * realToCSSPixels); + gl.canvas.width = displayWidth; + gl.canvas.height = displayHeight; + if (this.uniforms.iResolution?.isNeeded && this.shaderProgram) { + const rUniform = gl.getUniformLocation(this.shaderProgram, UNIFORM_RESOLUTION); + gl.uniform2fv(rUniform, [gl.canvas.width, gl.canvas.height]); + } + }; + drawScene = (timestamp: number) => { + const { gl } = this; + const { lerp = 1 } = this.props; + if (!gl) return; + gl.viewport(0, 0, gl.drawingBufferWidth, gl.drawingBufferHeight); + gl.clear(gl.COLOR_BUFFER_BIT | gl.DEPTH_BUFFER_BIT); + gl.bindBuffer(gl.ARRAY_BUFFER, this.squareVerticesBuffer ?? null); + gl.vertexAttribPointer(this.vertexPositionAttribute ?? 0, 3, gl.FLOAT, false, 0, 0); + this.setUniforms(timestamp); + gl.drawArrays(gl.TRIANGLE_STRIP, 0, 4); + if (this.uniforms.iMouse?.isNeeded && lerp !== 1) { + // @ts-expect-error TODO: Deal with this. + this.uniforms.iMouse.value[0] = lerpVal( + // @ts-expect-error TODO: Deal with this. + this.uniforms.iMouse.value[0], + this.lastMouseArr[0], + lerp + ); + // @ts-expect-error TODO: Deal with this. + this.uniforms.iMouse.value[1] = lerpVal( + // @ts-expect-error TODO: Deal with this. + this.uniforms.iMouse.value[1], + this.lastMouseArr[1], + lerp + ); + } + this.animFrameId = requestAnimationFrame(this.drawScene); + }; + createShader = (type: number, shaderCodeAsText: string) => { + const { gl } = this; + if (!gl) return null; + const shader = gl.createShader(type); + if (!shader) return null; + gl.shaderSource(shader, shaderCodeAsText); + gl.compileShader(shader); + if (!gl.getShaderParameter(shader, gl.COMPILE_STATUS)) { + this.props.onWarning?.(log(`Error compiling the shader:\n${shaderCodeAsText}`)); + const compilationLog = gl.getShaderInfoLog(shader); + gl.deleteShader(shader); + this.props.onError?.(log(`Shader compiler log: ${compilationLog}`)); + } + return shader; + }; + initShaders = (fs: string, vs: string) => { + const { gl } = this; + if (!gl) return; + // console.log(fs, vs); + const fragmentShader = this.createShader(gl.FRAGMENT_SHADER, fs); + const vertexShader = this.createShader(gl.VERTEX_SHADER, vs); + this.shaderProgram = gl.createProgram(); + if (!this.shaderProgram || !vertexShader || !fragmentShader) return; + gl.attachShader(this.shaderProgram, vertexShader); + gl.attachShader(this.shaderProgram, fragmentShader); + gl.linkProgram(this.shaderProgram); + if (!gl.getProgramParameter(this.shaderProgram, gl.LINK_STATUS)) { + this.props.onError?.( + log(`Unable to initialize the shader program: ${gl.getProgramInfoLog(this.shaderProgram)}`) + ); + return; + } + gl.useProgram(this.shaderProgram); + this.vertexPositionAttribute = gl.getAttribLocation(this.shaderProgram, 'aVertexPosition'); + gl.enableVertexAttribArray(this.vertexPositionAttribute); + }; + processCustomUniforms = () => { + const { uniforms } = this.props; + if (uniforms) { + for (const name of Object.keys(uniforms)) { + const uniform = this.props.uniforms?.[name]; + if (!uniform) return; + const { value, type } = uniform; + const glslType = uniformTypeToGLSLType(type); + if (!glslType) return; + const tempObject: { arraySize?: string } = {}; + if (isMatrixType(type, value)) { + const arrayLength = type.length; + const val = Number.parseInt(type.charAt(arrayLength - 3)); + const numberOfMatrices = Math.floor(value.length / (val * val)); + if (value.length > val * val) tempObject.arraySize = `[${numberOfMatrices}]`; + } else if (isVectorListType(type, value)) { + tempObject.arraySize = `[${Math.floor(value.length / Number.parseInt(type.charAt(0)))}]`; + } + this.uniforms[name] = { type: glslType, isNeeded: false, value, ...tempObject }; + } + } + }; + processTextures = () => { + const { gl } = this; + const { textures, onDoneLoadingTextures } = this.props; + if (!gl) return; + if (textures && textures.length > 0) { + this.uniforms[`${UNIFORM_CHANNELRESOLUTION}`] = { + type: 'vec3', + isNeeded: false, + arraySize: `[${textures.length}]`, + value: [], + }; + const texturePromisesArr = textures.map((texture: TextureParams, id: number) => { + // Dynamically add textures uniforms. + this.uniforms[`${UNIFORM_CHANNEL}${id}`] = { + type: 'sampler2D', + isNeeded: false, + }; + // Initialize array with 0s: + // @ts-expect-error TODO: Deal with this. + this.setupChannelRes(texture, id); + this.texturesArr[id] = new Texture(gl); + return ( + this.texturesArr[id] + // @ts-expect-error TODO: Deal with this. + ?.load(texture, id) + .then((t: Texture) => { + this.setupChannelRes(t, id); + }) + ); + }); + Promise.all(texturePromisesArr) + .then(() => { + if (onDoneLoadingTextures) onDoneLoadingTextures(); + }) + .catch((e) => { + this.props.onError?.(e); + if (onDoneLoadingTextures) onDoneLoadingTextures(); + }); + } else if (onDoneLoadingTextures) onDoneLoadingTextures(); + }; + preProcessFragment = (fragment: string) => { + const { precision, devicePixelRatio = 1 } = this.props; + const isValidPrecision = PRECISIONS.includes(precision ?? 'highp'); + const precisionString = `precision ${isValidPrecision ? precision : PRECISIONS[1]} float;\n`; + if (!isValidPrecision) { + this.props.onWarning?.( + log( + `wrong precision type ${precision}, please make sure to pass one of a valid precision lowp, mediump, highp, by default you shader precision will be set to highp.` + ) + ); + } + let fs = precisionString + .concat(`#define DPR ${devicePixelRatio.toFixed(1)}\n`) + .concat(fragment.replace(/texture\(/g, 'texture2D(')); + for (const uniform of Object.keys(this.uniforms)) { + if (fragment.includes(uniform)) { + const u = this.uniforms[uniform]; + if (!u) continue; + fs = insertStringAtIndex( + fs, + `uniform ${u.type} ${uniform}${u.arraySize || ''}; \n`, + fs.lastIndexOf(precisionString) + precisionString.length + ); + u.isNeeded = true; + } + } + const isShadertoy = fragment.includes('mainImage'); + if (isShadertoy) fs = fs.concat(FS_MAIN_SHADER); + return fs; + }; + setUniforms = (timestamp: number) => { + const { gl } = this; + if (!gl || !this.shaderProgram) return; + const delta = this.lastTime ? (timestamp - this.lastTime) / 1000 : 0; + this.lastTime = timestamp; + if (this.props.uniforms) { + for (const name of Object.keys(this.props.uniforms)) { + const currentUniform = this.props.uniforms?.[name]; + if (!currentUniform) return; + if (this.uniforms[name]?.isNeeded) { + if (!this.shaderProgram) return; + const customUniformLocation = gl.getUniformLocation(this.shaderProgram, name); + if (!customUniformLocation) return; + processUniform( + gl, + customUniformLocation, + currentUniform.type as UniformType, + currentUniform.value + ); + } + } + } + if (this.uniforms.iMouse?.isNeeded) { + const mouseUniform = gl.getUniformLocation(this.shaderProgram, UNIFORM_MOUSE); + gl.uniform4fv(mouseUniform, this.uniforms.iMouse.value as number[]); + } + if (this.uniforms.iChannelResolution?.isNeeded) { + const channelResUniform = gl.getUniformLocation( + this.shaderProgram, + UNIFORM_CHANNELRESOLUTION + ); + gl.uniform3fv(channelResUniform, this.uniforms.iChannelResolution.value as number[]); + } + if (this.uniforms.iDeviceOrientation?.isNeeded) { + const deviceOrientationUniform = gl.getUniformLocation( + this.shaderProgram, + UNIFORM_DEVICEORIENTATION + ); + gl.uniform4fv(deviceOrientationUniform, this.uniforms.iDeviceOrientation.value as number[]); + } + if (this.uniforms.iTime?.isNeeded) { + const timeUniform = gl.getUniformLocation(this.shaderProgram, UNIFORM_TIME); + gl.uniform1f(timeUniform, (this.timer += delta)); + } + if (this.uniforms.iTimeDelta?.isNeeded) { + const timeDeltaUniform = gl.getUniformLocation(this.shaderProgram, UNIFORM_TIMEDELTA); + gl.uniform1f(timeDeltaUniform, delta); + } + if (this.uniforms.iDate?.isNeeded) { + const d = new Date(); + const month = d.getMonth() + 1; + const day = d.getDate(); + const year = d.getFullYear(); + const time = + d.getHours() * 60 * 60 + d.getMinutes() * 60 + d.getSeconds() + d.getMilliseconds() * 0.001; + const dateUniform = gl.getUniformLocation(this.shaderProgram, UNIFORM_DATE); + gl.uniform4fv(dateUniform, [year, month, day, time]); + } + if (this.uniforms.iFrame?.isNeeded) { + const timeDeltaUniform = gl.getUniformLocation(this.shaderProgram, UNIFORM_FRAME); + gl.uniform1i(timeDeltaUniform, (this.uniforms.iFrame.value as number)++); + } + if (this.texturesArr.length > 0) { + for (let index = 0; index < this.texturesArr.length; index++) { + // TODO: Don't use this casting if possible: + const texture = this.texturesArr[index] as Texture | undefined; + if (!texture) return; + const { isVideo, _webglTexture, source, flipY, isLoaded } = texture; + if (!isLoaded || !_webglTexture || !source) return; + if (this.uniforms[`iChannel${index}`]?.isNeeded) { + if (!this.shaderProgram) return; + const iChannel = gl.getUniformLocation(this.shaderProgram, `iChannel${index}`); + // @ts-expect-error TODO: Fix. Can't index WebGL context with this dynamic value. + gl.activeTexture(gl[`TEXTURE${index}`]); + gl.bindTexture(gl.TEXTURE_2D, _webglTexture); + gl.uniform1i(iChannel, index); + if (isVideo) { + texture.updateTexture(_webglTexture, source as HTMLVideoElement, flipY); + } + } + } + } + }; + registerCanvas = (r: HTMLCanvasElement) => { + this.canvas = r; + }; + gl?: WebGLRenderingContext | null; + squareVerticesBuffer?: WebGLBuffer | null; + shaderProgram?: WebGLProgram | null; + vertexPositionAttribute?: number; + animFrameId?: number; + canvas?: HTMLCanvasElement; + mousedown = false; + canvasPosition?: DOMRect; + timer = 0; + lastMouseArr: number[] = [0, 0]; + texturesArr: WebGLTexture[] = []; + lastTime = 0; + resizeObserver?: ResizeObserver; + render = () => ( + + ); +} diff --git a/components/livekit/react-shader/react-shader.tsx b/components/livekit/react-shader/react-shader.tsx new file mode 100644 index 00000000..b5ea8b76 --- /dev/null +++ b/components/livekit/react-shader/react-shader.tsx @@ -0,0 +1,649 @@ +import { type CSSProperties, useEffect, useRef } from 'react'; +import { log } from './logging'; +import { + ClampToEdgeWrapping, + LinearFilter, + LinearMipMapLinearFilter, + LinearMipMapNearestFilter, + MirroredRepeatWrapping, + NearestFilter, + NearestMipMapLinearFilter, + NearestMipMapNearestFilter, + RepeatWrapping, + Texture, +} from './texture'; +import { + type UniformType, + type Vector2, + type Vector3, + type Vector4, + isMatrixType, + isVectorListType, + processUniform, + uniformTypeToGLSLType, +} from './uniforms'; + +export { + ClampToEdgeWrapping, + LinearFilter, + LinearMipMapLinearFilter, + LinearMipMapNearestFilter, + MirroredRepeatWrapping, + NearestFilter, + NearestMipMapLinearFilter, + NearestMipMapNearestFilter, + RepeatWrapping, +}; + +export type { Vector2, Vector3, Vector4 }; + +const PRECISIONS = ['lowp', 'mediump', 'highp']; +const FS_MAIN_SHADER = `\nvoid main(void){ + vec4 color = vec4(0.0,0.0,0.0,1.0); + mainImage( color, gl_FragCoord.xy ); + gl_FragColor = color; +}`; +const BASIC_FS = `void mainImage( out vec4 fragColor, in vec2 fragCoord ) { + vec2 uv = fragCoord/iResolution.xy; + vec3 col = 0.5 + 0.5*cos(iTime+uv.xyx+vec3(0,2,4)); + fragColor = vec4(col,1.0); +}`; +const BASIC_VS = `attribute vec3 aVertexPosition; +void main(void) { + gl_Position = vec4(aVertexPosition, 1.0); +}`; +const UNIFORM_TIME = 'iTime'; +const UNIFORM_TIMEDELTA = 'iTimeDelta'; +const UNIFORM_DATE = 'iDate'; +const UNIFORM_FRAME = 'iFrame'; +const UNIFORM_MOUSE = 'iMouse'; +const UNIFORM_RESOLUTION = 'iResolution'; +const UNIFORM_CHANNEL = 'iChannel'; +const UNIFORM_CHANNELRESOLUTION = 'iChannelResolution'; +const UNIFORM_DEVICEORIENTATION = 'iDeviceOrientation'; + +const latestPointerClientCoords = (e: MouseEvent | TouchEvent) => { + // @ts-expect-error TODO: Deal with this. + return [e.clientX || e.changedTouches[0].clientX, e.clientY || e.changedTouches[0].clientY]; +}; +const lerpVal = (v0: number, v1: number, t: number) => v0 * (1 - t) + v1 * t; +const insertStringAtIndex = (currentString: string, string: string, index: number) => + index > 0 + ? currentString.substring(0, index) + + string + + currentString.substring(index, currentString.length) + : string + currentString; + +type Uniform = { type: string; value: number[] | number }; +export type Uniforms = Record; +type TextureParams = { + url: string; + wrapS?: number; + wrapT?: number; + minFilter?: number; + magFilter?: number; + flipY?: number; +}; + +type Props = { + /** Fragment shader GLSL code. */ + fs: string; + + /** Vertex shader GLSL code. */ + vs?: string; + + /** + * Textures to be passed to the shader. Textures need to be squared or will be + * automatically resized. + * + * Options default to: + * + * ```js + * { + * minFilter: LinearMipMapLinearFilter, + * magFilter: LinearFilter, + * wrapS: RepeatWrapping, + * wrapT: RepeatWrapping, + * } + * ``` + * + * See [textures in the docs](https://rysana.com/docs/react-shaders#textures) + * for details. + */ + textures?: TextureParams[]; + + /** + * Custom uniforms to be passed to the shader. + * + * See [custom uniforms in the + * docs](https://rysana.com/docs/react-shaders#custom-uniforms) for details. + */ + uniforms?: Uniforms; + + /** + * Color used when clearing the canvas. + * + * See [the WebGL + * docs](https://developer.mozilla.org/en-US/docs/Web/API/WebGLRenderingContext/clearColor) + * for details. + */ + clearColor?: Vector4; + + /** + * GLSL precision qualifier. Defaults to `'highp'`. Balance between + * performance and quality. + */ + precision?: 'highp' | 'lowp' | 'mediump'; + + /** Custom inline style for canvas. */ + style?: CSSStyleDeclaration; + + /** Customize WebGL context attributes. See [the WebGL docs](https://developer.mozilla.org/en-US/docs/Web/API/WebGLRenderingContext/getContextAttributes) for details. */ + contextAttributes?: Record; + + /** Lerp value for `iMouse` built-in uniform. Must be between 0 and 1. */ + lerp?: number; + + /** Device pixel ratio. */ + devicePixelRatio?: number; + + /** + * Callback for when the textures are done loading. Useful if you want to do + * something like e.g. hide the canvas until textures are done loading. + */ + onDoneLoadingTextures?: () => void; + + /** Custom callback to handle errors. Defaults to `console.error`. */ + onError?: (error: string) => void; + + /** Custom callback to handle warnings. Defaults to `console.warn`. */ + onWarning?: (warning: string) => void; +}; +export const Shader = ({ + fs, + vs = BASIC_VS, + textures = [], + uniforms: propUniforms, + clearColor = [0, 0, 0, 1], + precision = 'highp', + style, + contextAttributes = {}, + lerp = 1, + devicePixelRatio = 1, + onDoneLoadingTextures, + onError = console.error, + onWarning = console.warn, +}: Props) => { + // Refs for WebGL state + const canvasRef = useRef(null); + const glRef = useRef(null); + const squareVerticesBufferRef = useRef(null); + const shaderProgramRef = useRef(null); + const vertexPositionAttributeRef = useRef(undefined); + const animFrameIdRef = useRef(undefined); + const mousedownRef = useRef(false); + const canvasPositionRef = useRef(undefined); + const timerRef = useRef(0); + const lastMouseArrRef = useRef([0, 0]); + const texturesArrRef = useRef([]); + const lastTimeRef = useRef(0); + const resizeObserverRef = useRef(undefined); + const uniformsRef = useRef< + Record< + string, + { type: string; isNeeded: boolean; value?: number[] | number; arraySize?: string } + > + >({ + [UNIFORM_TIME]: { type: 'float', isNeeded: false, value: 0 }, + [UNIFORM_TIMEDELTA]: { type: 'float', isNeeded: false, value: 0 }, + [UNIFORM_DATE]: { type: 'vec4', isNeeded: false, value: [0, 0, 0, 0] }, + [UNIFORM_MOUSE]: { type: 'vec4', isNeeded: false, value: [0, 0, 0, 0] }, + [UNIFORM_RESOLUTION]: { type: 'vec2', isNeeded: false, value: [0, 0] }, + [UNIFORM_FRAME]: { type: 'int', isNeeded: false, value: 0 }, + [UNIFORM_DEVICEORIENTATION]: { type: 'vec4', isNeeded: false, value: [0, 0, 0, 0] }, + }); + const propsUniformsRef = useRef(propUniforms); + + const setupChannelRes = ({ width, height }: Texture, id: number) => { + // @ts-expect-error TODO: Deal with this. + uniformsRef.current.iChannelResolution.value[id * 3] = width * devicePixelRatio; + // @ts-expect-error TODO: Deal with this. + uniformsRef.current.iChannelResolution.value[id * 3 + 1] = height * devicePixelRatio; + // @ts-expect-error TODO: Deal with this. + uniformsRef.current.iChannelResolution.value[id * 3 + 2] = 0; + }; + + const initWebGL = () => { + if (!canvasRef.current) return; + glRef.current = (canvasRef.current.getContext('webgl', contextAttributes) || + canvasRef.current.getContext( + 'experimental-webgl', + contextAttributes + )) as WebGLRenderingContext | null; + glRef.current?.getExtension('OES_standard_derivatives'); + glRef.current?.getExtension('EXT_shader_texture_lod'); + }; + + const initBuffers = () => { + const gl = glRef.current; + squareVerticesBufferRef.current = gl?.createBuffer() ?? null; + gl?.bindBuffer(gl.ARRAY_BUFFER, squareVerticesBufferRef.current); + const vertices = [1.0, 1.0, 0.0, -1.0, 1.0, 0.0, 1.0, -1.0, 0.0, -1.0, -1.0, 0.0]; + gl?.bufferData(gl.ARRAY_BUFFER, new Float32Array(vertices), gl.STATIC_DRAW); + }; + + const onDeviceOrientationChange = ({ alpha, beta, gamma }: DeviceOrientationEvent) => { + uniformsRef.current.iDeviceOrientation.value = [ + alpha ?? 0, + beta ?? 0, + gamma ?? 0, + window.orientation || 0, + ]; + }; + + const mouseDown = (e: MouseEvent | TouchEvent) => { + const [clientX, clientY] = latestPointerClientCoords(e); + const mouseX = clientX - (canvasPositionRef.current?.left ?? 0) - window.pageXOffset; + const mouseY = + (canvasPositionRef.current?.height ?? 0) - + clientY - + (canvasPositionRef.current?.top ?? 0) - + window.pageYOffset; + mousedownRef.current = true; + // @ts-expect-error TODO: Deal with this. + uniformsRef.current.iMouse.value[2] = mouseX; + // @ts-expect-error TODO: Deal with this. + uniformsRef.current.iMouse.value[3] = mouseY; + lastMouseArrRef.current[0] = mouseX; + lastMouseArrRef.current[1] = mouseY; + }; + + const mouseMove = (e: MouseEvent | TouchEvent) => { + canvasPositionRef.current = canvasRef.current?.getBoundingClientRect(); + const [clientX, clientY] = latestPointerClientCoords(e); + const mouseX = clientX - (canvasPositionRef.current?.left ?? 0); + const mouseY = + (canvasPositionRef.current?.height ?? 0) - clientY - (canvasPositionRef.current?.top ?? 0); + if (lerp !== 1) { + lastMouseArrRef.current[0] = mouseX; + lastMouseArrRef.current[1] = mouseY; + } else { + // @ts-expect-error TODO: Deal with this. + uniformsRef.current.iMouse.value[0] = mouseX; + // @ts-expect-error TODO: Deal with this. + uniformsRef.current.iMouse.value[1] = mouseY; + } + }; + + const mouseUp = () => { + // @ts-expect-error TODO: Deal with this. + uniformsRef.current.iMouse.value[2] = 0; + // @ts-expect-error TODO: Deal with this. + uniformsRef.current.iMouse.value[3] = 0; + }; + + const onResize = () => { + const gl = glRef.current; + if (!gl) return; + canvasPositionRef.current = canvasRef.current?.getBoundingClientRect(); + // Force pixel ratio to be one to avoid expensive calculus on retina display. + const realToCSSPixels = devicePixelRatio; + const displayWidth = Math.floor((canvasPositionRef.current?.width ?? 1) * realToCSSPixels); + const displayHeight = Math.floor((canvasPositionRef.current?.height ?? 1) * realToCSSPixels); + gl.canvas.width = displayWidth; + gl.canvas.height = displayHeight; + if (uniformsRef.current.iResolution?.isNeeded && shaderProgramRef.current) { + const rUniform = gl.getUniformLocation(shaderProgramRef.current, UNIFORM_RESOLUTION); + gl.uniform2fv(rUniform, [gl.canvas.width, gl.canvas.height]); + } + }; + + const createShader = (type: number, shaderCodeAsText: string) => { + const gl = glRef.current; + if (!gl) return null; + const shader = gl.createShader(type); + if (!shader) return null; + gl.shaderSource(shader, shaderCodeAsText); + gl.compileShader(shader); + if (!gl.getShaderParameter(shader, gl.COMPILE_STATUS)) { + onWarning?.(log(`Error compiling the shader:\n${shaderCodeAsText}`)); + const compilationLog = gl.getShaderInfoLog(shader); + gl.deleteShader(shader); + onError?.(log(`Shader compiler log: ${compilationLog}`)); + } + return shader; + }; + + const initShaders = (fragmentShader: string, vertexShader: string) => { + const gl = glRef.current; + if (!gl) return; + const fragmentShaderObj = createShader(gl.FRAGMENT_SHADER, fragmentShader); + const vertexShaderObj = createShader(gl.VERTEX_SHADER, vertexShader); + shaderProgramRef.current = gl.createProgram(); + if (!shaderProgramRef.current || !vertexShaderObj || !fragmentShaderObj) return; + gl.attachShader(shaderProgramRef.current, vertexShaderObj); + gl.attachShader(shaderProgramRef.current, fragmentShaderObj); + gl.linkProgram(shaderProgramRef.current); + if (!gl.getProgramParameter(shaderProgramRef.current, gl.LINK_STATUS)) { + onError?.( + log( + `Unable to initialize the shader program: ${gl.getProgramInfoLog(shaderProgramRef.current)}` + ) + ); + return; + } + gl.useProgram(shaderProgramRef.current); + vertexPositionAttributeRef.current = gl.getAttribLocation( + shaderProgramRef.current, + 'aVertexPosition' + ); + gl.enableVertexAttribArray(vertexPositionAttributeRef.current); + }; + + const processCustomUniforms = () => { + if (propUniforms) { + for (const name of Object.keys(propUniforms)) { + const uniform = propUniforms[name]; + if (!uniform) return; + const { value, type } = uniform; + const glslType = uniformTypeToGLSLType(type); + if (!glslType) return; + const tempObject: { arraySize?: string } = {}; + if (isMatrixType(type, value)) { + const arrayLength = type.length; + const val = Number.parseInt(type.charAt(arrayLength - 3)); + const numberOfMatrices = Math.floor(value.length / (val * val)); + if (value.length > val * val) tempObject.arraySize = `[${numberOfMatrices}]`; + } else if (isVectorListType(type, value)) { + tempObject.arraySize = `[${Math.floor(value.length / Number.parseInt(type.charAt(0)))}]`; + } + uniformsRef.current[name] = { type: glslType, isNeeded: false, value, ...tempObject }; + } + } + }; + + const processTextures = () => { + const gl = glRef.current; + if (!gl) return; + if (textures && textures.length > 0) { + uniformsRef.current[`${UNIFORM_CHANNELRESOLUTION}`] = { + type: 'vec3', + isNeeded: false, + arraySize: `[${textures.length}]`, + value: [], + }; + const texturePromisesArr = textures.map((texture: TextureParams, id: number) => { + // Dynamically add textures uniforms. + uniformsRef.current[`${UNIFORM_CHANNEL}${id}`] = { + type: 'sampler2D', + isNeeded: false, + }; + // Initialize array with 0s: + // @ts-expect-error TODO: Deal with this. + setupChannelRes(texture, id); + texturesArrRef.current[id] = new Texture(gl); + return ( + texturesArrRef.current[id] + // @ts-expect-error TODO: Deal with this. + ?.load(texture, id) + .then((t: Texture) => { + setupChannelRes(t, id); + }) + ); + }); + Promise.all(texturePromisesArr) + .then(() => { + if (onDoneLoadingTextures) onDoneLoadingTextures(); + }) + .catch((e) => { + onError?.(e); + if (onDoneLoadingTextures) onDoneLoadingTextures(); + }); + } else if (onDoneLoadingTextures) onDoneLoadingTextures(); + }; + + const preProcessFragment = (fragment: string) => { + const isValidPrecision = PRECISIONS.includes(precision ?? 'highp'); + const precisionString = `precision ${isValidPrecision ? precision : PRECISIONS[1]} float;\n`; + if (!isValidPrecision) { + onWarning?.( + log( + `wrong precision type ${precision}, please make sure to pass one of a valid precision lowp, mediump, highp, by default you shader precision will be set to highp.` + ) + ); + } + let fragmentShader = precisionString + .concat(`#define DPR ${devicePixelRatio.toFixed(1)}\n`) + .concat(fragment.replace(/texture\(/g, 'texture2D(')); + for (const uniform of Object.keys(uniformsRef.current)) { + if (fragment.includes(uniform)) { + const u = uniformsRef.current[uniform]; + if (!u) continue; + fragmentShader = insertStringAtIndex( + fragmentShader, + `uniform ${u.type} ${uniform}${u.arraySize || ''}; \n`, + fragmentShader.lastIndexOf(precisionString) + precisionString.length + ); + u.isNeeded = true; + } + } + const isShadertoy = fragment.includes('mainImage'); + if (isShadertoy) fragmentShader = fragmentShader.concat(FS_MAIN_SHADER); + return fragmentShader; + }; + + const setUniforms = (timestamp: number) => { + const gl = glRef.current; + if (!gl || !shaderProgramRef.current) return; + const delta = lastTimeRef.current ? (timestamp - lastTimeRef.current) / 1000 : 0; + lastTimeRef.current = timestamp; + const propUniforms = propsUniformsRef.current; + if (propUniforms) { + for (const name of Object.keys(propUniforms)) { + const currentUniform = propUniforms[name]; + if (!currentUniform) return; + if (uniformsRef.current[name]?.isNeeded) { + if (!shaderProgramRef.current) return; + const customUniformLocation = gl.getUniformLocation(shaderProgramRef.current, name); + if (!customUniformLocation) return; + processUniform( + gl, + customUniformLocation, + currentUniform.type as UniformType, + currentUniform.value + ); + } + } + } + if (uniformsRef.current.iMouse?.isNeeded) { + const mouseUniform = gl.getUniformLocation(shaderProgramRef.current, UNIFORM_MOUSE); + gl.uniform4fv(mouseUniform, uniformsRef.current.iMouse.value as number[]); + } + if (uniformsRef.current.iChannelResolution?.isNeeded) { + const channelResUniform = gl.getUniformLocation( + shaderProgramRef.current, + UNIFORM_CHANNELRESOLUTION + ); + gl.uniform3fv(channelResUniform, uniformsRef.current.iChannelResolution.value as number[]); + } + if (uniformsRef.current.iDeviceOrientation?.isNeeded) { + const deviceOrientationUniform = gl.getUniformLocation( + shaderProgramRef.current, + UNIFORM_DEVICEORIENTATION + ); + gl.uniform4fv( + deviceOrientationUniform, + uniformsRef.current.iDeviceOrientation.value as number[] + ); + } + if (uniformsRef.current.iTime?.isNeeded) { + const timeUniform = gl.getUniformLocation(shaderProgramRef.current, UNIFORM_TIME); + gl.uniform1f(timeUniform, (timerRef.current += delta)); + } + if (uniformsRef.current.iTimeDelta?.isNeeded) { + const timeDeltaUniform = gl.getUniformLocation(shaderProgramRef.current, UNIFORM_TIMEDELTA); + gl.uniform1f(timeDeltaUniform, delta); + } + if (uniformsRef.current.iDate?.isNeeded) { + const d = new Date(); + const month = d.getMonth() + 1; + const day = d.getDate(); + const year = d.getFullYear(); + const time = + d.getHours() * 60 * 60 + d.getMinutes() * 60 + d.getSeconds() + d.getMilliseconds() * 0.001; + const dateUniform = gl.getUniformLocation(shaderProgramRef.current, UNIFORM_DATE); + gl.uniform4fv(dateUniform, [year, month, day, time]); + } + if (uniformsRef.current.iFrame?.isNeeded) { + const timeDeltaUniform = gl.getUniformLocation(shaderProgramRef.current, UNIFORM_FRAME); + gl.uniform1i(timeDeltaUniform, (uniformsRef.current.iFrame.value as number)++); + } + if (texturesArrRef.current.length > 0) { + for (let index = 0; index < texturesArrRef.current.length; index++) { + // TODO: Don't use this casting if possible: + const texture = texturesArrRef.current[index] as Texture | undefined; + if (!texture) return; + const { isVideo, _webglTexture, source, flipY, isLoaded } = texture; + if (!isLoaded || !_webglTexture || !source) return; + if (uniformsRef.current[`iChannel${index}`]?.isNeeded) { + if (!shaderProgramRef.current) return; + const iChannel = gl.getUniformLocation(shaderProgramRef.current, `iChannel${index}`); + // @ts-expect-error TODO: Fix. Can't index WebGL context with this dynamic value. + gl.activeTexture(gl[`TEXTURE${index}`]); + gl.bindTexture(gl.TEXTURE_2D, _webglTexture); + gl.uniform1i(iChannel, index); + if (isVideo) { + texture.updateTexture(_webglTexture, source as HTMLVideoElement, flipY); + } + } + } + } + }; + + const drawScene = (timestamp: number) => { + const gl = glRef.current; + if (!gl) return; + gl.viewport(0, 0, gl.drawingBufferWidth, gl.drawingBufferHeight); + gl.clear(gl.COLOR_BUFFER_BIT | gl.DEPTH_BUFFER_BIT); + gl.bindBuffer(gl.ARRAY_BUFFER, squareVerticesBufferRef.current); + gl.vertexAttribPointer(vertexPositionAttributeRef.current ?? 0, 3, gl.FLOAT, false, 0, 0); + setUniforms(timestamp); + gl.drawArrays(gl.TRIANGLE_STRIP, 0, 4); + if (uniformsRef.current.iMouse?.isNeeded && lerp !== 1) { + // @ts-expect-error TODO: Deal with this. + uniformsRef.current.iMouse.value[0] = lerpVal( + // @ts-expect-error TODO: Deal with this. + uniformsRef.current.iMouse.value[0], + lastMouseArrRef.current[0], + lerp + ); + // @ts-expect-error TODO: Deal with this. + uniformsRef.current.iMouse.value[1] = lerpVal( + // @ts-expect-error TODO: Deal with this. + uniformsRef.current.iMouse.value[1], + lastMouseArrRef.current[1], + lerp + ); + } + animFrameIdRef.current = requestAnimationFrame(drawScene); + }; + + const addEventListeners = () => { + const options = { passive: true }; + if (uniformsRef.current.iMouse?.isNeeded && canvasRef.current) { + canvasRef.current.addEventListener('mousemove', mouseMove, options); + canvasRef.current.addEventListener('mouseout', mouseUp, options); + canvasRef.current.addEventListener('mouseup', mouseUp, options); + canvasRef.current.addEventListener('mousedown', mouseDown, options); + canvasRef.current.addEventListener('touchmove', mouseMove, options); + canvasRef.current.addEventListener('touchend', mouseUp, options); + canvasRef.current.addEventListener('touchstart', mouseDown, options); + } + if (uniformsRef.current.iDeviceOrientation?.isNeeded) { + window.addEventListener('deviceorientation', onDeviceOrientationChange, options); + } + if (canvasRef.current) { + resizeObserverRef.current = new ResizeObserver(onResize); + resizeObserverRef.current.observe(canvasRef.current); + window.addEventListener('resize', onResize, options); + } + }; + + const removeEventListeners = () => { + const options = { passive: true } as EventListenerOptions; + if (uniformsRef.current.iMouse?.isNeeded && canvasRef.current) { + canvasRef.current.removeEventListener('mousemove', mouseMove, options); + canvasRef.current.removeEventListener('mouseout', mouseUp, options); + canvasRef.current.removeEventListener('mouseup', mouseUp, options); + canvasRef.current.removeEventListener('mousedown', mouseDown, options); + canvasRef.current.removeEventListener('touchmove', mouseMove, options); + canvasRef.current.removeEventListener('touchend', mouseUp, options); + canvasRef.current.removeEventListener('touchstart', mouseDown, options); + } + if (uniformsRef.current.iDeviceOrientation?.isNeeded) { + window.removeEventListener('deviceorientation', onDeviceOrientationChange, options); + } + if (resizeObserverRef.current) { + resizeObserverRef.current.disconnect(); + window.removeEventListener('resize', onResize, options); + } + }; + + useEffect(() => { + propsUniformsRef.current = propUniforms; + }, [propUniforms]); + + // Main effect for initialization and cleanup + useEffect(() => { + const textures = texturesArrRef.current; + + function init() { + initWebGL(); + const gl = glRef.current; + if (gl && canvasRef.current) { + gl.clearColor(...clearColor); + gl.clearDepth(1.0); + gl.enable(gl.DEPTH_TEST); + gl.depthFunc(gl.LEQUAL); + gl.viewport(0, 0, canvasRef.current.width, canvasRef.current.height); + console.log('canvasRef.current', canvasRef.current); + console.log('canvasRef.current.width', canvasRef.current.width); + console.log('canvasRef.current.height', canvasRef.current.height); + canvasRef.current.height = canvasRef.current.clientHeight; + canvasRef.current.width = canvasRef.current.clientWidth; + processCustomUniforms(); + processTextures(); + initShaders(preProcessFragment(fs || BASIC_FS), vs || BASIC_VS); + initBuffers(); + requestAnimationFrame(drawScene); + addEventListeners(); + onResize(); + } + } + + requestAnimationFrame(init); + + // Cleanup function + return () => { + const gl = glRef.current; + if (gl) { + gl.getExtension('WEBGL_lose_context')?.loseContext(); + gl.useProgram(null); + gl.deleteProgram(shaderProgramRef.current ?? null); + if (textures.length > 0) { + for (const texture of textures as Texture[]) { + gl.deleteTexture(texture._webglTexture); + } + } + shaderProgramRef.current = null; + } + removeEventListeners(); + cancelAnimationFrame(animFrameIdRef.current ?? 0); + }; + // eslint-disable-next-line react-hooks/exhaustive-deps + }, []); // Empty dependency array to run only once on mount + + return ( + + ); +}; diff --git a/components/livekit/react-shader/texture.ts b/components/livekit/react-shader/texture.ts new file mode 100644 index 00000000..19282c17 --- /dev/null +++ b/components/livekit/react-shader/texture.ts @@ -0,0 +1,197 @@ +import { log } from './logging'; + +export const LinearFilter = 9729; +export const NearestFilter = 9728; +export const LinearMipMapLinearFilter = 9987; +export const NearestMipMapLinearFilter = 9986; +export const LinearMipMapNearestFilter = 9985; +export const NearestMipMapNearestFilter = 9984; +export const MirroredRepeatWrapping = 33648; +export const ClampToEdgeWrapping = 33071; +export const RepeatWrapping = 10497; + +export class Texture { + gl: WebGLRenderingContext; + url?: string; + wrapS?: number; + wrapT?: number; + minFilter?: number; + magFilter?: number; + source?: HTMLImageElement | HTMLVideoElement; + pow2canvas?: HTMLCanvasElement; + isLoaded = false; + isVideo = false; + flipY = -1; + width = 0; + height = 0; + _webglTexture: WebGLTexture | null = null; + constructor(gl: WebGLRenderingContext) { + this.gl = gl; + } + updateTexture = (texture: WebGLTexture, video: HTMLVideoElement, flipY: number) => { + const { gl } = this; + const level = 0; + const internalFormat = gl.RGBA; + const srcFormat = gl.RGBA; + const srcType = gl.UNSIGNED_BYTE; + gl.bindTexture(gl.TEXTURE_2D, texture); + gl.pixelStorei(gl.UNPACK_FLIP_Y_WEBGL, flipY); + gl.texImage2D(gl.TEXTURE_2D, level, internalFormat, srcFormat, srcType, video); + }; + setupVideo = (url: string) => { + const video = document.createElement('video'); + let playing = false; + let timeupdate = false; + video.autoplay = true; + video.muted = true; + video.loop = true; + video.crossOrigin = 'anonymous'; + const checkReady = () => { + if (playing && timeupdate) { + this.isLoaded = true; + } + }; + video.addEventListener( + 'playing', + () => { + playing = true; + this.width = video.videoWidth || 0; + this.height = video.videoHeight || 0; + checkReady(); + }, + true + ); + video.addEventListener( + 'timeupdate', + () => { + timeupdate = true; + checkReady(); + }, + true + ); + video.src = url; + // video.play(); // Not sure why this is here nor commented out. From STR. + return video; + }; + makePowerOf2 = (image: T): T => { + if ( + image instanceof HTMLImageElement || + image instanceof HTMLCanvasElement || + image instanceof ImageBitmap + ) { + if (this.pow2canvas === undefined) this.pow2canvas = document.createElement('canvas'); + this.pow2canvas.width = 2 ** Math.floor(Math.log(image.width) / Math.LN2); + this.pow2canvas.height = 2 ** Math.floor(Math.log(image.height) / Math.LN2); + const context = this.pow2canvas.getContext('2d'); + context?.drawImage(image, 0, 0, this.pow2canvas.width, this.pow2canvas.height); + console.warn( + log( + `Image is not power of two ${image.width} x ${image.height}. Resized to ${this.pow2canvas.width} x ${this.pow2canvas.height};` + ) + ); + return this.pow2canvas as T; + } + return image; + }; + load = async ( + textureArgs: Texture + // channelId: number // Not sure why this is here nor commented out. From STR. + ) => { + const { gl } = this; + const { url, wrapS, wrapT, minFilter, magFilter, flipY = -1 }: Texture = textureArgs; + if (!url) { + return Promise.reject( + new Error(log('Missing url, please make sure to pass the url of your texture { url: ... }')) + ); + } + const isImage = /(\.jpg|\.jpeg|\.png|\.gif|\.bmp)$/i.exec(url); + const isVideo = /(\.mp4|\.3gp|\.webm|\.ogv)$/i.exec(url); + if (isImage === null && isVideo === null) { + return Promise.reject( + new Error(log(`Please upload a video or an image with a valid format (url: ${url})`)) + ); + } + Object.assign(this, { url, wrapS, wrapT, minFilter, magFilter, flipY }); + const level = 0; + const internalFormat = gl.RGBA; + const width = 1; + const height = 1; + const border = 0; + const srcFormat = gl.RGBA; + const srcType = gl.UNSIGNED_BYTE; + const pixel = new Uint8Array([255, 255, 255, 0]); + const texture = gl.createTexture(); + gl.bindTexture(gl.TEXTURE_2D, texture); + gl.texImage2D( + gl.TEXTURE_2D, + level, + internalFormat, + width, + height, + border, + srcFormat, + srcType, + pixel + ); + if (isVideo) { + const video = this.setupVideo(url); + gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE); + gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE); + gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.LINEAR); + this._webglTexture = texture; + this.source = video; + this.isVideo = true; + return video.play().then(() => this); + } + async function loadImage() { + return new Promise((resolve, reject) => { + const image = new Image(); + image.crossOrigin = 'anonymous'; + image.onload = () => { + resolve(image); + }; + image.onerror = () => { + reject(new Error(log(`failed loading url: ${url}`))); + }; + image.src = url ?? ''; + }); + } + let image = (await loadImage()) as HTMLImageElement; + let isPowerOf2 = + (image.width & (image.width - 1)) === 0 && (image.height & (image.height - 1)) === 0; + if ( + (textureArgs.wrapS !== ClampToEdgeWrapping || + textureArgs.wrapT !== ClampToEdgeWrapping || + (textureArgs.minFilter !== NearestFilter && textureArgs.minFilter !== LinearFilter)) && + !isPowerOf2 + ) { + image = this.makePowerOf2(image); + isPowerOf2 = true; + } + gl.bindTexture(gl.TEXTURE_2D, texture); + gl.pixelStorei(gl.UNPACK_FLIP_Y_WEBGL, flipY); + gl.texImage2D(gl.TEXTURE_2D, level, internalFormat, srcFormat, srcType, image); + if ( + isPowerOf2 && + textureArgs.minFilter !== NearestFilter && + textureArgs.minFilter !== LinearFilter + ) { + gl.generateMipmap(gl.TEXTURE_2D); + } + gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, this.wrapS || RepeatWrapping); + gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, this.wrapT || RepeatWrapping); + gl.texParameteri( + gl.TEXTURE_2D, + gl.TEXTURE_MIN_FILTER, + this.minFilter || LinearMipMapLinearFilter + ); + gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, this.magFilter || LinearFilter); + this._webglTexture = texture; + this.source = image; + this.isVideo = false; + this.isLoaded = true; + this.width = image.width || 0; + this.height = image.height || 0; + return this; + }; +} diff --git a/components/livekit/react-shader/uniforms.ts b/components/livekit/react-shader/uniforms.ts new file mode 100644 index 00000000..39405a9a --- /dev/null +++ b/components/livekit/react-shader/uniforms.ts @@ -0,0 +1,145 @@ +import { log } from './logging'; + +export type Vector2 = [T, T]; +export type Vector3 = [T, T, T]; +export type Vector4 = [T, T, T, T]; +// biome-ignore format: +export type Matrix2 = [T, T, T, T]; +// biome-ignore format: +export type Matrix3 = [T, T, T, T, T, T, T, T, T]; +// biome-ignore format: +export type Matrix4 = [T, T, T, T, T, T, T, T, T, T, T, T, T, T, T, T]; +export type Uniforms = { + '1i': number; + '2i': Vector2; + '3i': Vector3; + '4i': Vector4; + '1f': number; + '2f': Vector2; + '3f': Vector3; + '4f': Vector4; + '1iv': Float32List; + '2iv': Float32List; + '3iv': Float32List; + '4iv': Float32List; + '1fv': Float32List; + '2fv': Float32List; + '3fv': Float32List; + '4fv': Float32List; + Matrix2fv: Float32List; + Matrix3fv: Float32List; + Matrix4fv: Float32List; +}; +export type UniformType = keyof Uniforms; + +export function isMatrixType(t: string, v: number[] | number): v is number[] { + return t.includes('Matrix') && Array.isArray(v); +} +export function isVectorListType(t: string, v: number[] | number): v is number[] { + return t.includes('v') && Array.isArray(v) && v.length > Number.parseInt(t.charAt(0)); +} +function isVectorType(t: string, v: number[] | number): v is Vector4 { + return !t.includes('v') && Array.isArray(v) && v.length > Number.parseInt(t.charAt(0)); +} +export const processUniform = ( + gl: WebGLRenderingContext, + location: WebGLUniformLocation, + t: T, + value: number | number[] +) => { + if (isVectorType(t, value)) { + switch (t) { + case '2f': + return gl.uniform2f(location, value[0], value[1]); + case '3f': + return gl.uniform3f(location, value[0], value[1], value[2]); + case '4f': + return gl.uniform4f(location, value[0], value[1], value[2], value[3]); + case '2i': + return gl.uniform2i(location, value[0], value[1]); + case '3i': + return gl.uniform3i(location, value[0], value[1], value[2]); + case '4i': + return gl.uniform4i(location, value[0], value[1], value[2], value[3]); + } + } + if (typeof value === 'number') { + switch (t) { + case '1i': + return gl.uniform1i(location, value); + default: + return gl.uniform1f(location, value); + } + } + switch (t) { + case '1iv': + return gl.uniform1iv(location, value); + case '2iv': + return gl.uniform2iv(location, value); + case '3iv': + return gl.uniform3iv(location, value); + case '4iv': + return gl.uniform4iv(location, value); + case '1fv': + return gl.uniform1fv(location, value); + case '2fv': + return gl.uniform2fv(location, value); + case '3fv': + return gl.uniform3fv(location, value); + case '4fv': + return gl.uniform4fv(location, value); + case 'Matrix2fv': + return gl.uniformMatrix2fv(location, false, value); + case 'Matrix3fv': + return gl.uniformMatrix3fv(location, false, value); + case 'Matrix4fv': + return gl.uniformMatrix4fv(location, false, value); + } +}; + +export const uniformTypeToGLSLType = (t: string) => { + switch (t) { + case '1f': + return 'float'; + case '2f': + return 'vec2'; + case '3f': + return 'vec3'; + case '4f': + return 'vec4'; + case '1i': + return 'int'; + case '2i': + return 'ivec2'; + case '3i': + return 'ivec3'; + case '4i': + return 'ivec4'; + case '1iv': + return 'int'; + case '2iv': + return 'ivec2'; + case '3iv': + return 'ivec3'; + case '4iv': + return 'ivec4'; + case '1fv': + return 'float'; + case '2fv': + return 'vec2'; + case '3fv': + return 'vec3'; + case '4fv': + return 'vec4'; + case 'Matrix2fv': + return 'mat2'; + case 'Matrix3fv': + return 'mat3'; + case 'Matrix4fv': + return 'mat4'; + default: + console.error( + log(`The uniform type "${t}" is not valid, please make sure your uniform type is valid`) + ); + } +}; diff --git a/components/ui/shadcn-io/aurora-shaders/aurora-shader.glsl b/components/ui/shadcn-io/aurora-shaders/aurora-shader.glsl new file mode 100644 index 00000000..9d089c6e --- /dev/null +++ b/components/ui/shadcn-io/aurora-shaders/aurora-shader.glsl @@ -0,0 +1,111 @@ +const float TAU = 6.28318530718; + +// Cosine palette generator +vec3 pal(in float t, in vec3 a, in vec3 b, in vec3 c, in vec3 d) { + return a + b * cos(TAU * (c * t + d)); +} + +// Noise for dithering +vec2 randFibo(vec2 p) { + p = fract(p * vec2(443.897, 441.423)); + p += dot(p, p.yx + 19.19); + return fract((p.xx + p.yx) * p.xy); +} + +// Tonemap +vec3 Tonemap_Reinhard(vec3 x) { + x *= 4.0; + return x / (1.0 + x); +} + +// Luma for alpha +float luma(vec3 color) { + return dot(color, vec3(0.299, 0.587, 0.114)); +} + +// SDF shapes +float sdCircle(vec2 st, float r) { + return length(st) - r; +} + +float sdLine(vec2 p, float r) { + float halfLen = r * 2.0; + vec2 a = vec2(-halfLen, 0.0); + vec2 b = vec2(halfLen, 0.0); + vec2 pa = p - a; + vec2 ba = b - a; + float h = clamp(dot(pa, ba) / dot(ba, ba), 0.0, 1.0); + return length(pa - ba * h); +} + +float getSdf(vec2 st) { + if(uShape == 1) return sdCircle(st, uScale); + else if(uShape == 2) return sdLine(st, uScale); + return sdCircle(st, uScale); // Default +} + +vec2 turb(vec2 pos, float t, float it) { + mat2 rot = mat2(0.6, -0.8, 0.8, 0.6); + float freq = mix(2.0, 15.0, uFrequency); + float amp = uAmplitude; + float xp = 1.4; + float time = t * 0.1 * uSpeed; + + for(float i = 0.0; i < 4.0; i++) { + vec2 s = sin(freq * (pos * rot) + i * time + it); + pos += amp * rot[0] * s / freq; + rot *= mat2(0.6, -0.8, 0.8, 0.6); + amp *= mix(1.0, max(s.y, s.x), uVariance); + freq *= xp; + } + + return pos; +} + +const float ITERATIONS = 36.0; + +void mainImage(out vec4 fragColor, in vec2 fragCoord) { + vec2 uv = fragCoord / iResolution.xy; + + vec3 pp = vec3(0.0); + vec3 bloom = vec3(0.0); + float t = iTime * 0.5; + vec2 pos = uv - 0.5; + + vec2 prevPos = turb(pos, t, 0.0 - 1.0 / ITERATIONS); + float spacing = mix(1.0, TAU, uSpacing); + + for(float i = 1.0; i < ITERATIONS + 1.0; i++) { + float iter = i / ITERATIONS; + vec2 st = turb(pos, t, iter * spacing); + float d = abs(getSdf(st)); + float pd = distance(st, prevPos); + prevPos = st; + float dynamicBlur = exp2(pd * 2.0 * 1.4426950408889634) - 1.0; + float ds = smoothstep(0.0, uBlur * 0.05 + max(dynamicBlur * uSmoothing, 0.001), d); + + // Generate color using cosine palette + vec3 color = pal( + iter * mix(0.0, 3.0, uColorScale) + uColorPosition * 2.0, + vec3(0.5), + vec3(0.5), + vec3(1.0), + vec3(0.0, 0.33, 0.67) + ); + + float invd = 1.0 / max(d + dynamicBlur, 0.001); + pp += (ds - 1.0) * color; + bloom += clamp(invd, 0.0, 250.0) * color; + } + + pp *= 1.0 / ITERATIONS; + bloom = bloom / (bloom + 2e4); + + vec3 color = (-pp + bloom * 3.0 * uBloom); + color *= 1.2; + color += (randFibo(fragCoord).x - 0.5) / 255.0; + color = Tonemap_Reinhard(color); + + float alpha = luma(color) * uMix; + fragColor = vec4(color * uMix, alpha); +} \ No newline at end of file diff --git a/components/ui/shadcn-io/aurora-shaders/aurora-shader.gl b/components/ui/shadcn-io/aurora-shaders/aurora-shader.original.glsl similarity index 100% rename from components/ui/shadcn-io/aurora-shaders/aurora-shader.gl rename to components/ui/shadcn-io/aurora-shaders/aurora-shader.original.glsl diff --git a/components/ui/shadcn-io/aurora-shaders/index copy 2.tsx b/components/ui/shadcn-io/aurora-shaders/index copy 2.tsx deleted file mode 100644 index c8f389f0..00000000 --- a/components/ui/shadcn-io/aurora-shaders/index copy 2.tsx +++ /dev/null @@ -1,213 +0,0 @@ -'use client'; - -import React, { forwardRef } from 'react'; -import { Shader } from 'react-shaders'; -import { cn } from '@/lib/utils'; - -export interface AuroraShadersProps extends React.HTMLAttributes { - /** - * Aurora wave speed - * @default 1.0 - */ - speed?: number; - - /** - * Light intensity and brightness - * @default 1.0 - */ - intensity?: number; - - /** - * Color vibrancy and saturation - * @default 1.0 - */ - vibrancy?: number; - - /** - * Wave frequency and complexity - * @default 1.0 - */ - frequency?: number; - - /** - * Vertical stretch of aurora bands (ring thickness) - * @default 1.0 - */ - stretch?: number; - - /** - * Ring radius from center - * @default 0.3 - */ - radius?: number; -} - -const auroraShader = ` -// Noise function for organic movement -float noise(vec2 p) { - return fract(sin(dot(p, vec2(127.1, 311.7))) * 43758.5453); -} - -// Smooth noise for flowing effects -float smoothNoise(vec2 p) { - vec2 i = floor(p); - vec2 f = fract(p); - - float a = noise(i); - float b = noise(i + vec2(1.0, 0.0)); - float c = noise(i + vec2(0.0, 1.0)); - float d = noise(i + vec2(1.0, 1.0)); - - vec2 u = f * f * (3.0 - 2.0 * f); - - return mix(a, b, u.x) + (c - a) * u.y * (1.0 - u.x) + (d - b) * u.x * u.y; -} - -// Fractal noise for complex aurora patterns -float fractalNoise(vec2 p) { - float value = 0.0; - float amplitude = 0.5; - - for(int i = 0; i < 4; i++) { - value += amplitude * smoothNoise(p); - p *= 2.0; - amplitude *= 0.5; - } - - return value; -} - -void mainImage( out vec4 fragColor, in vec2 fragCoord ) -{ - // Normalize coordinates and center them - vec2 uv = fragCoord / iResolution.xy; - vec2 center = vec2(0.5, 0.5); - vec2 pos = uv - center; - - // Adjust aspect ratio - pos.x *= iResolution.x / iResolution.y; - - // Convert to polar coordinates - float angle = atan(pos.y, pos.x); - float dist = length(pos); - - // Normalize angle from -PI..PI to 0..1 - float normalizedAngle = (angle / 6.28318530718) + 0.5; - - // Time with speed control - float time = iTime * u_speed; - - // Create radial gradient for ring positioning (controls ring thickness) - float ringCenter = u_radius; // Distance from center where ring appears - float ringWidth = 0.15 * u_stretch; // Width of the ring - float radialGradient = 1.0 - abs(dist - ringCenter) / ringWidth; - radialGradient = clamp(radialGradient, 0.0, 1.0); - radialGradient = pow(radialGradient, 1.5); - - // Create seamless angular coordinate by using sin/cos for tiling - // This ensures the pattern wraps perfectly around the ring - float angularCoord = normalizedAngle * 6.28318530718 + time * 0.1 * 6.28318530718; - vec2 seamlessUV = vec2(cos(angularCoord), sin(angularCoord)); - - // Generate multiple aurora layers with different characteristics - // Using seamless UV coordinates to avoid seams - float aurora1 = fractalNoise(seamlessUV * u_frequency * 3.0 + vec2(time * 0.2, dist * 10.0)); - float aurora2 = fractalNoise(seamlessUV * u_frequency * 2.0 + vec2(time * 0.15 + 1000.0, dist * 8.0)); - float aurora3 = fractalNoise(seamlessUV * u_frequency * 4.0 + vec2(time * 0.25 + 2000.0, dist * 12.0)); - - // Add wave distortion for organic movement (radial waves) - float wave1 = sin(normalizedAngle * 8.0 * 6.28318530718 + time * 2.0) * 0.02; - float wave2 = sin(normalizedAngle * 12.0 * 6.28318530718 + time * 1.5) * 0.01; - - float distortedDist = dist + wave1 + wave2; - - // Apply radial positioning to aurora layers (creates the ring shape) - aurora1 *= smoothstep(ringCenter - ringWidth, ringCenter, distortedDist) * - smoothstep(ringCenter + ringWidth, ringCenter, distortedDist); - aurora2 *= smoothstep(ringCenter - ringWidth * 0.8, ringCenter, distortedDist) * - smoothstep(ringCenter + ringWidth * 0.8, ringCenter, distortedDist); - aurora3 *= smoothstep(ringCenter - ringWidth * 0.9, ringCenter, distortedDist) * - smoothstep(ringCenter + ringWidth * 0.9, ringCenter, distortedDist); - - // Combine aurora layers - float combinedAurora = (aurora1 * 0.6 + aurora2 * 0.8 + aurora3 * 0.4) * radialGradient; - - // Apply intensity scaling - combinedAurora *= u_intensity; - - // Create aurora color palette - vec3 color1 = vec3(0.0, 0.8, 0.4); // Green - vec3 color2 = vec3(0.2, 0.4, 1.0); // Blue - vec3 color3 = vec3(0.8, 0.2, 0.8); // Purple - vec3 color4 = vec3(0.0, 1.0, 0.8); // Cyan - - // Create seamless color transitions using sine waves for smooth wrapping - float colorPhase = normalizedAngle * 6.28318530718 * 2.0; // Two full cycles around ring - float colorMix = (sin(colorPhase) + 1.0) * 0.5; // 0 to 1 - float colorMix2 = (sin(colorPhase + 2.094395) + 1.0) * 0.5; // Offset by 120 degrees - float colorMix3 = (sin(colorPhase + 4.18879) + 1.0) * 0.5; // Offset by 240 degrees - - // Mix colors smoothly with seamless transitions - vec3 finalColor = mix(color1, color2, colorMix); - finalColor = mix(finalColor, color3, colorMix2 * 0.5); - finalColor = mix(finalColor, color4, colorMix3 * 0.3); - - // Apply vibrancy control - vec3 desaturated = vec3(dot(finalColor, vec3(0.299, 0.587, 0.114))); - finalColor = mix(desaturated, finalColor, u_vibrancy); - - // Apply aurora intensity - finalColor *= combinedAurora; - - // Add atmospheric glow around the ring - float ringGlow = exp(-abs(dist - ringCenter) * 8.0) * 0.15; - finalColor += finalColor * ringGlow; - - // Ensure colors stay in valid range - finalColor = clamp(finalColor, 0.0, 1.0); - - // Calculate alpha based on aurora intensity for transparency - float alpha = length(finalColor); - alpha = clamp(alpha, 0.0, 1.0); - - fragColor = vec4(finalColor, alpha); -} -`; - -export const AuroraShaders = forwardRef( - ( - { - className, - speed = 1.0, - intensity = 1.0, - vibrancy = 1.0, - frequency = 1.0, - stretch = 1.0, - radius = 0.3, - ...props - }, - ref - ) => { - console.log('radius', radius); - return ( -
- -
- ); - } -); - -AuroraShaders.displayName = 'AuroraShaders'; - -export default AuroraShaders; diff --git a/components/ui/shadcn-io/aurora-shaders/index copy.tsx b/components/ui/shadcn-io/aurora-shaders/index copy.tsx deleted file mode 100644 index 09043855..00000000 --- a/components/ui/shadcn-io/aurora-shaders/index copy.tsx +++ /dev/null @@ -1,178 +0,0 @@ -'use client'; - -import React, { forwardRef } from 'react'; -import { Shader } from 'react-shaders'; -import { cn } from '@/lib/utils'; - -export interface AuroraShadersProps extends React.HTMLAttributes { - /** - * Aurora wave speed - * @default 1.0 - */ - speed?: number; - - /** - * Light intensity and brightness - * @default 1.0 - */ - intensity?: number; - - /** - * Color vibrancy and saturation - * @default 1.0 - */ - vibrancy?: number; - - /** - * Wave frequency and complexity - * @default 1.0 - */ - frequency?: number; - - /** - * Vertical stretch of aurora bands - * @default 1.0 - */ - stretch?: number; -} - -const auroraShader = ` -// Noise function for organic movement -float noise(vec2 p) { - return fract(sin(dot(p, vec2(127.1, 311.7))) * 43758.5453); -} - -// Smooth noise for flowing effects -float smoothNoise(vec2 p) { - vec2 i = floor(p); - vec2 f = fract(p); - - float a = noise(i); - float b = noise(i + vec2(1.0, 0.0)); - float c = noise(i + vec2(0.0, 1.0)); - float d = noise(i + vec2(1.0, 1.0)); - - vec2 u = f * f * (3.0 - 2.0 * f); - - return mix(a, b, u.x) + (c - a) * u.y * (1.0 - u.x) + (d - b) * u.x * u.y; -} - -// Fractal noise for complex aurora patterns -float fractalNoise(vec2 p) { - float value = 0.0; - float amplitude = 0.5; - - for(int i = 0; i < 4; i++) { - value += amplitude * smoothNoise(p); - p *= 2.0; - amplitude *= 0.5; - } - - return value; -} - -void mainImage( out vec4 fragColor, in vec2 fragCoord ) -{ - // Normalize coordinates - vec2 uv = fragCoord / iResolution.xy; - - // Time with speed control - float time = iTime * u_speed; - - // Create vertical gradient for aurora positioning - float verticalGradient = 1.0 - abs(uv.y - 0.5) * 2.0; - verticalGradient = pow(verticalGradient, u_stretch); - - // Create flowing horizontal movement - vec2 flowUV = vec2(uv.x + time * 0.1, uv.y); - - // Generate multiple aurora layers with different characteristics - float aurora1 = fractalNoise(flowUV * u_frequency * 3.0 + vec2(time * 0.2, 0.0)); - float aurora2 = fractalNoise(flowUV * u_frequency * 2.0 + vec2(time * 0.15, 1000.0)); - float aurora3 = fractalNoise(flowUV * u_frequency * 4.0 + vec2(time * 0.25, 2000.0)); - - // Add wave distortion for organic movement - float wave1 = sin(uv.x * 8.0 + time * 2.0) * 0.1; - float wave2 = sin(uv.x * 12.0 + time * 1.5) * 0.05; - - float distortedY = uv.y + wave1 + wave2; - - // Apply vertical positioning to aurora layers - aurora1 *= smoothstep(0.3, 0.7, distortedY) * smoothstep(0.8, 0.6, distortedY); - aurora2 *= smoothstep(0.4, 0.6, distortedY) * smoothstep(0.7, 0.5, distortedY); - aurora3 *= smoothstep(0.35, 0.65, distortedY) * smoothstep(0.75, 0.55, distortedY); - - // Combine aurora layers - float combinedAurora = (aurora1 * 0.6 + aurora2 * 0.8 + aurora3 * 0.4) * verticalGradient; - - // Apply intensity scaling - combinedAurora *= u_intensity; - - // Create aurora color palette - vec3 color1 = vec3(0.0, 0.8, 0.4); // Green - vec3 color2 = vec3(0.2, 0.4, 1.0); // Blue - vec3 color3 = vec3(0.8, 0.2, 0.8); // Purple - vec3 color4 = vec3(0.0, 1.0, 0.8); // Cyan - - // Create color zones based on vertical position - float colorMix1 = smoothstep(0.2, 0.4, uv.y); - float colorMix2 = smoothstep(0.4, 0.6, uv.y); - float colorMix3 = smoothstep(0.6, 0.8, uv.y); - - // Mix colors for natural aurora appearance - vec3 finalColor = mix(color1, color2, colorMix1); - finalColor = mix(finalColor, color3, colorMix2); - finalColor = mix(finalColor, color4, colorMix3); - - // Apply vibrancy control - vec3 desaturated = vec3(dot(finalColor, vec3(0.299, 0.587, 0.114))); - finalColor = mix(desaturated, finalColor, u_vibrancy); - - // Apply aurora intensity - finalColor *= combinedAurora; - - // Add atmospheric glow at horizon - float horizonGlow = exp(-abs(uv.y - 0.5) * 8.0) * 0.1; - finalColor += finalColor * horizonGlow; - - // Ensure colors stay in valid range - finalColor = clamp(finalColor, 0.0, 1.0); - - fragColor = vec4(finalColor, 1.0); -} -`; - -export const AuroraShaders = forwardRef( - ( - { - className, - speed = 1.0, - intensity = 1.0, - vibrancy = 1.0, - frequency = 1.0, - stretch = 1.0, - ...props - }, - ref - ) => { - return ( -
- -
- ); - } -); - -AuroraShaders.displayName = 'AuroraShaders'; - -export default AuroraShaders; diff --git a/components/ui/shadcn-io/aurora-shaders/index.tsx b/components/ui/shadcn-io/aurora-shaders/index.tsx index 8683a401..ec8fe97d 100644 --- a/components/ui/shadcn-io/aurora-shaders/index.tsx +++ b/components/ui/shadcn-io/aurora-shaders/index.tsx @@ -1,8 +1,7 @@ 'use client'; import React, { forwardRef } from 'react'; -import { Shader } from 'react-shaders'; -import { cn } from '@/lib/utils'; +import { Shader } from '../../../livekit/react-shader/react-shader'; const auroraShaderSource = ` const float TAU = 6.28318530718; @@ -35,12 +34,6 @@ float sdCircle(vec2 st, float r) { return length(st) - r; } -float sdEllipse(vec2 st, float r) { - float a = length(st + vec2(0, r * 0.8)) - r; - float b = length(st + vec2(0, -r * 0.8)) - r; - return (a + b); -} - float sdLine(vec2 p, float r) { float halfLen = r * 2.0; vec2 a = vec2(-halfLen, 0.0); @@ -51,27 +44,10 @@ float sdLine(vec2 p, float r) { return length(pa - ba * h); } -float sdBox(vec2 p, float r) { - vec2 q = abs(p) - vec2(r); - return length(max(q, 0.0)) + min(max(q.x, q.y), 0.0) - r * mix(0.0, 0.3333, uAmplitude); -} - -float sdEquilateralTriangle(vec2 p, float r) { - const float k = sqrt(3.0); - p.x = abs(p.x) - r; - p.y = p.y + r / k; - if(p.x + k * p.y > 0.0) p = vec2(p.x - k * p.y, -k * p.x - p.y) / 2.0; - p.x -= clamp(p.x, -2.0 * r, 0.0); - return -length(p) * sign(p.y) - r * mix(0.0, 0.3333, uAmplitude); -} - float getSdf(vec2 st) { - if(uShape == 1) return sdCircle(st, uScale); - else if(uShape == 2) return sdEllipse(st, uScale); - else if(uShape == 3) return sdLine(st, uScale); - else if(uShape == 4) return sdBox(st, uScale); - else if(uShape == 5) return sdEquilateralTriangle(st, uScale); - else return sdCircle(st, uScale); + if(uShape == 1.0) return sdCircle(st, uScale); + else if(uShape == 2.0) return sdLine(st, uScale); + return sdCircle(st, uScale); // Default } vec2 turb(vec2 pos, float t, float it) { @@ -79,7 +55,7 @@ vec2 turb(vec2 pos, float t, float it) { float freq = mix(2.0, 15.0, uFrequency); float amp = uAmplitude; float xp = 1.4; - float time = t * 0.1 * uSpeed + uPhase; + float time = t * 0.1 * uSpeed; for(float i = 0.0; i < 4.0; i++) { vec2 s = sin(freq * (pos * rot) + i * time + it); @@ -99,7 +75,7 @@ void mainImage(out vec4 fragColor, in vec2 fragCoord) { vec3 pp = vec3(0.0); vec3 bloom = vec3(0.0); - float t = iTime * 0.5 + uPhase; + float t = iTime * 0.5; vec2 pos = uv - 0.5; vec2 prevPos = turb(pos, t, 0.0 - 1.0 / ITERATIONS); @@ -138,8 +114,7 @@ void mainImage(out vec4 fragColor, in vec2 fragCoord) { float alpha = luma(color) * uMix; fragColor = vec4(color * uMix, alpha); -} -`; +}`; export interface AuroraShadersProps extends React.HTMLAttributes { /** @@ -148,12 +123,6 @@ export interface AuroraShadersProps extends React.HTMLAttributes */ speed?: number; - /** - * Light intensity (bloom) - * @default 2.0 - */ - intensity?: number; - /** * Turbulence amplitude * @default 0.5 @@ -173,16 +142,16 @@ export interface AuroraShadersProps extends React.HTMLAttributes scale?: number; /** - * Edge blur/softness - * @default 1.0 + * Shape type: 1=circle, 2=line + * @default 1 */ - blur?: number; + shape?: number; /** - * Shape type: 1=circle, 2=ellipse, 3=line, 4=box, 5=triangle - * @default 1 + * Edge blur/softness + * @default 1.0 */ - shape?: number; + blur?: number; /** * Color palette offset - shifts colors along the gradient (0-1) @@ -215,13 +184,12 @@ export const AuroraShaders = forwardRef( ( { className, + shape = 1.0, speed = 1.0, - intensity = 0.1, amplitude = 0.5, frequency = 0.5, - scale = 0.3, + scale = 0.2, blur = 1.0, - shape = 1, colorPosition = 1.0, colorScale = 1.0, brightness = 1.0, @@ -230,24 +198,43 @@ export const AuroraShaders = forwardRef( ref ) => { return ( -
+
{ + console.log('error', error); + }} + onWarning={(warning) => { + console.log('warning', warning); }} style={{ width: '100%', height: '100%' } as CSSStyleDeclaration} /> diff --git a/components/ui/shadcn-io/cosmic-waves-shaders/index.tsx b/components/ui/shadcn-io/cosmic-waves-shaders/index.tsx index 14bd059f..d9daa457 100644 --- a/components/ui/shadcn-io/cosmic-waves-shaders/index.tsx +++ b/components/ui/shadcn-io/cosmic-waves-shaders/index.tsx @@ -1,7 +1,7 @@ 'use client'; import React, { forwardRef } from 'react'; -import { Shader } from 'react-shaders'; +import { Shader } from '@/components/livekit/react-shader/react-shader'; import { cn } from '@/lib/utils'; export interface CosmicWavesShadersProps extends React.HTMLAttributes { diff --git a/components/ui/shadcn-io/singularity-shaders/index.tsx b/components/ui/shadcn-io/singularity-shaders/index.tsx index 52a60ba8..93a19105 100644 --- a/components/ui/shadcn-io/singularity-shaders/index.tsx +++ b/components/ui/shadcn-io/singularity-shaders/index.tsx @@ -1,7 +1,7 @@ 'use client'; import React, { forwardRef } from 'react'; -import { Shader } from 'react-shaders'; +import { Shader } from '@/components/livekit/react-shader/react-shader'; import { cn } from '@/lib/utils'; export interface SingularityShadersProps extends React.HTMLAttributes { diff --git a/package.json b/package.json index e66a98e3..0c43a95f 100644 --- a/package.json +++ b/package.json @@ -29,7 +29,6 @@ "next-themes": "^0.4.6", "react": "^19.0.0", "react-dom": "^19.0.0", - "react-shaders": "^0.0.4", "sonner": "^2.0.3", "tailwind-merge": "^3.3.0" }, diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 9eff56fc..2a20acd1 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -62,9 +62,6 @@ importers: react-dom: specifier: ^19.0.0 version: 19.1.1(react@19.1.1) - react-shaders: - specifier: ^0.0.4 - version: 0.0.4(react-dom@19.1.1(react@19.1.1))(react@19.1.1) sonner: specifier: ^2.0.3 version: 2.0.7(react-dom@19.1.1(react@19.1.1))(react@19.1.1) @@ -2225,12 +2222,6 @@ packages: '@types/react': optional: true - react-shaders@0.0.4: - resolution: {integrity: sha512-uh42UkMeS4DXyoyYzx+SJryCbULJi6eOMZSLO8Xp1+ogn1WjkIwoP5ucjoYfYu3qouYw3vVIi/Amu0/xZ4/wsg==} - peerDependencies: - react: ^18.2.0 - react-dom: ^18.2.0 - react-style-singleton@2.2.3: resolution: {integrity: sha512-b6jSvxvVnyptAiLjbkWLE/lOnR4lfTtDAl+eUC7RZy+QQWc6wRzIV2CE6xBuMmDxc2qIihtDCZD5NPOFl7fRBQ==} engines: {node: '>=10'} @@ -4658,11 +4649,6 @@ snapshots: optionalDependencies: '@types/react': 19.1.12 - react-shaders@0.0.4(react-dom@19.1.1(react@19.1.1))(react@19.1.1): - dependencies: - react: 19.1.1 - react-dom: 19.1.1(react@19.1.1) - react-style-singleton@2.2.3(@types/react@19.1.12)(react@19.1.1): dependencies: get-nonce: 1.0.1 From fd8bb32699253e81ee54c58cc3d79f40ebd59ca6 Mon Sep 17 00:00:00 2001 From: Thomas Yuill Date: Tue, 28 Oct 2025 21:29:37 -0400 Subject: [PATCH 07/10] AudioShaderVisualizer tweak jelly --- .../audio-shader-visualizer.tsx | 174 ++++++------------ .../ui/shadcn-io/aurora-shaders/index.tsx | 3 +- 2 files changed, 62 insertions(+), 115 deletions(-) diff --git a/components/livekit/audio-visualizer/audio-shader-visualizer/audio-shader-visualizer.tsx b/components/livekit/audio-visualizer/audio-shader-visualizer/audio-shader-visualizer.tsx index 00b749b4..272a11ad 100644 --- a/components/livekit/audio-visualizer/audio-shader-visualizer/audio-shader-visualizer.tsx +++ b/components/livekit/audio-visualizer/audio-shader-visualizer/audio-shader-visualizer.tsx @@ -1,9 +1,10 @@ 'use client'; -import { useEffect, useRef, useState } from 'react'; +import { useCallback, useEffect, useRef, useState } from 'react'; import { type VariantProps, cva } from 'class-variance-authority'; import { type AnimationPlaybackControlsWithThen, + type KeyframesTarget, type ValueAnimationTransition, animate, useMotionValue, @@ -19,32 +20,28 @@ import { import { AuroraShaders, type AuroraShadersProps } from '@/components/ui/shadcn-io/aurora-shaders'; import { cn } from '@/lib/utils'; -// const PRESETS = [ -// (volume: number) => ({ -// amplitude: 0.3, -// scale: 0.35 - 0.05 * volume, -// frequency: 0.25, -// brightness: 1.5 + 2.5 * volume, -// }), -// (volume: number) => ({ -// amplitude: 0.2 + 1 * volume, -// scale: 0.35 - 0.05 * volume, -// frequency: 0.25 + 5 * volume, -// brightness: 1.5 + 2.5 * volume, -// }), -// (volume: number) => ({ -// amplitude: 0.5 + 0.05 * volume, -// scale: 0.35 + 0.05 * volume, -// frequency: 2 - 2 * volume, -// brightness: 1.5 + 2.5 * volume, -// }), -// (volume: number) => ({ -// amplitude: 0.5 + 0.2 * volume, -// scale: 0.35 - 0.05 * volume, -// frequency: 1 - 1 * volume, -// brightness: 1.5 + 2.5 * volume, -// }), -// ]; +const DEFAULT_SPEED = 10; +const DEFAULT_AMPLITUDE = 2; +const DEFAULT_FREQUENCY = 0.5; +const DEFAULT_SCALE = 0.2; +const DEFAULT_BRIGHTNESS = 1.5; +const DEFAULT_TRANSITION: ValueAnimationTransition = { duration: 0.5, ease: 'easeOut' }; + +function useAnimatedValue(initialValue: T) { + const [value, setValue] = useState(initialValue); + const motionValue = useMotionValue(initialValue); + const controlsRef = useRef(null); + useMotionValueEvent(motionValue, 'change', (value) => setValue(value as T)); + + const animateFn = useCallback( + (targetValue: T | KeyframesTarget, transition: ValueAnimationTransition) => { + controlsRef.current = animate(motionValue, targetValue, transition); + }, + [motionValue] + ); + + return { value, controls: controlsRef, animate: animateFn }; +} export const audioShaderVisualizerVariants = cva(['aspect-square'], { variants: { @@ -77,133 +74,82 @@ export function AudioShaderVisualizer({ }: AudioShaderVisualizerProps & AuroraShadersProps & VariantProps) { - const [speed, setSpeed] = useState(10); - const [amplitude, setAmplitude] = useState(0.5); - const [frequency, setFrequency] = useState(1.0); - const [scale, setScale] = useState(0.2); - const [brightness, setBrightness] = useState(1.5); - - const amplitudeValue = useMotionValue(0.5); - const frequencyValue = useMotionValue(0.5); - const scaleValue = useMotionValue(0.3); - const brightnessValue = useMotionValue(0); - - const amplitudeControlsRef = useRef(null); - const frequencyControlsRef = useRef(null); - const scaleControlsRef = useRef(null); - const brightnessControlsRef = useRef(null); - - useMotionValueEvent(amplitudeValue, 'change', (value) => setAmplitude(value)); - useMotionValueEvent(frequencyValue, 'change', (value) => setFrequency(value)); - useMotionValueEvent(scaleValue, 'change', (value) => setScale(value)); - useMotionValueEvent(brightnessValue, 'change', (value) => setBrightness(value)); + const [speed, setSpeed] = useState(DEFAULT_SPEED); + const { value: amplitude, animate: animateAmplitude } = useAnimatedValue(DEFAULT_AMPLITUDE); + const { value: frequency, animate: animateFrequency } = useAnimatedValue(DEFAULT_FREQUENCY); + const { value: scale, animate: animateScale } = useAnimatedValue(DEFAULT_SCALE); + const { value: brightness, animate: animateBrightness } = useAnimatedValue(DEFAULT_BRIGHTNESS); const volume = useTrackVolume(audioTrack as TrackReference, { fftSize: 512, - smoothingTimeConstant: 0.5, + smoothingTimeConstant: 0.55, }); useEffect(() => { - const DEFAULT_TRANSITION: ValueAnimationTransition = { duration: 0.5, ease: 'easeOut' }; - switch (state) { case 'disconnected': setSpeed(5); - scaleControlsRef.current = animate(scaleValue, 0.2, DEFAULT_TRANSITION); - amplitudeControlsRef.current = animate(amplitudeValue, 1.2, DEFAULT_TRANSITION); - frequencyControlsRef.current = animate(frequencyValue, 0.4, DEFAULT_TRANSITION); - brightnessControlsRef.current = animate(brightnessValue, 1.0, DEFAULT_TRANSITION); + animateScale(0.2, DEFAULT_TRANSITION); + animateAmplitude(1.2, DEFAULT_TRANSITION); + animateFrequency(0.4, DEFAULT_TRANSITION); + animateBrightness(1.0, DEFAULT_TRANSITION); return; + case 'listening': case 'connecting': - setSpeed(50); - scaleControlsRef.current = animate(scaleValue, 0.3, DEFAULT_TRANSITION); - amplitudeControlsRef.current = animate(amplitudeValue, 0.5, DEFAULT_TRANSITION); - frequencyControlsRef.current = animate(frequencyValue, 1, DEFAULT_TRANSITION); - brightnessControlsRef.current = animate(brightnessValue, [0.5, 2.5], { - duration: 1, - repeat: Infinity, - repeatType: 'mirror', - }); + setSpeed(20); + animateScale(0.35, DEFAULT_TRANSITION); + animateAmplitude(1, DEFAULT_TRANSITION); + animateFrequency(0.7, DEFAULT_TRANSITION); + animateBrightness(2.0, DEFAULT_TRANSITION); return; case 'initializing': setSpeed(30); - scaleControlsRef.current = animate(scaleValue, 0.3, DEFAULT_TRANSITION); - amplitudeControlsRef.current = animate(amplitudeValue, 0.5, DEFAULT_TRANSITION); - frequencyControlsRef.current = animate(frequencyValue, 1, DEFAULT_TRANSITION); - brightnessControlsRef.current = animate(brightnessValue, [0.5, 2.5], { + animateScale(0.3, DEFAULT_TRANSITION); + animateAmplitude(0.5, DEFAULT_TRANSITION); + animateFrequency(1, DEFAULT_TRANSITION); + animateBrightness([0.5, 2.5], { duration: 0.2, repeat: Infinity, repeatType: 'mirror', }); return; - case 'listening': - setSpeed(20); - scaleControlsRef.current = animate(scaleValue, [0.3, 0.35], { - duration: 1.5, - repeat: Infinity, - repeatType: 'mirror', - }); - amplitudeControlsRef.current = animate(amplitudeValue, 0.5, DEFAULT_TRANSITION); - frequencyControlsRef.current = animate(frequencyValue, 1.0, DEFAULT_TRANSITION); - brightnessControlsRef.current = animate(brightnessValue, [1.5, 2.5], { - duration: 1.5, - repeat: Infinity, - repeatType: 'mirror', - }); - return; case 'thinking': - setSpeed(50); - scaleControlsRef.current = animate(scaleValue, [0.35, 0.3], { - duration: 0.5, + setSpeed(30); + animateScale([0.15, 0.13], { + duration: 0.3, repeat: Infinity, repeatType: 'mirror', }); - amplitudeControlsRef.current = animate(amplitudeValue, 0.5, { - ...DEFAULT_TRANSITION, - duration: 0.2, - }); - frequencyControlsRef.current = animate(frequencyValue, 2.5, { - ...DEFAULT_TRANSITION, - duration: 0.2, - }); - brightnessControlsRef.current = animate(brightnessValue, [0.5, 2.5], { - duration: 0.2, + animateAmplitude(1.0, DEFAULT_TRANSITION); + animateFrequency(3.0, DEFAULT_TRANSITION); + animateBrightness([1.5, 2.5], { + duration: 0.3, repeat: Infinity, repeatType: 'mirror', }); return; case 'speaking': setSpeed(50); - scaleControlsRef.current = animate(scaleValue, 0.35, DEFAULT_TRANSITION); - amplitudeControlsRef.current = animate(amplitudeValue, 0.5, DEFAULT_TRANSITION); - frequencyControlsRef.current = animate(frequencyValue, 1.0, DEFAULT_TRANSITION); - brightnessControlsRef.current = animate(brightnessValue, 0.5, DEFAULT_TRANSITION); return; } }, [ state, shape, colorScale, - scaleValue, - colorPosition, - amplitudeValue, - frequencyValue, - brightnessValue, + animateScale, + animateAmplitude, + animateFrequency, + animateBrightness, ]); useEffect(() => { if (state === 'speaking' && volume > 0) { - scaleControlsRef.current?.stop(); - amplitudeControlsRef.current?.stop(); - frequencyControlsRef.current?.stop(); - brightnessControlsRef.current?.stop(); - - scaleValue.set(0.3 - 0.05 * volume); - amplitudeValue.set(0.5 + 0.2 * volume); - frequencyValue.set(1 - 1 * volume); - brightnessValue.set(1.0 + 2.0 * volume); + animateScale(0.3 - 0.1 * volume, { duration: 0 }); + animateAmplitude(1.0 + 0.2 * volume, { duration: 0 }); + animateFrequency(0.7 - 0.3 * volume, { duration: 0 }); + animateBrightness(1.5 + 1.0 * volume, { duration: 0 }); } - }, [state, volume, scaleValue, amplitudeValue, frequencyValue, brightnessValue]); + }, [state, volume, animateScale, animateAmplitude, animateFrequency, animateBrightness]); return ( Date: Thu, 30 Oct 2025 16:41:03 -0400 Subject: [PATCH 08/10] AudioOscilloscopeVisualizer --- app/ui/_components.tsx | 101 ++++++++++- .../audio-oscilloscope-visualizer.tsx | 142 +++++++++++++++ .../oscilliscope-shaders.tsx | 168 ++++++++++++++++++ 3 files changed, 409 insertions(+), 2 deletions(-) create mode 100644 components/livekit/audio-visualizer/audio-oscilloscope-visualizer/audio-oscilloscope-visualizer.tsx create mode 100644 components/livekit/audio-visualizer/audio-oscilloscope-visualizer/oscilliscope-shaders.tsx diff --git a/app/ui/_components.tsx b/app/ui/_components.tsx index 755d92d5..ccd6a307 100644 --- a/app/ui/_components.tsx +++ b/app/ui/_components.tsx @@ -3,13 +3,13 @@ import { useEffect, useMemo, useState } from 'react'; import { type VariantProps } from 'class-variance-authority'; import { Track } from 'livekit-client'; -import { RoomAudioRenderer, StartAudio } from '@livekit/components-react'; +// import { RoomAudioRenderer, StartAudio } from '@livekit/components-react'; import { type AgentState, type TrackReference, type TrackReferenceOrPlaceholder, useLocalParticipant, - useVoiceAssistant, + // useVoiceAssistant, } from '@livekit/components-react'; import { MicrophoneIcon } from '@phosphor-icons/react/dist/ssr'; import { useSession } from '@/components/app/session-provider'; @@ -26,6 +26,7 @@ import { } from '@/components/livekit/audio-visualizer/audio-bar-visualizer/audio-bar-visualizer'; import { AudioGridVisualizer } from '@/components/livekit/audio-visualizer/audio-grid-visualizer/audio-grid-visualizer'; import { gridVariants } from '@/components/livekit/audio-visualizer/audio-grid-visualizer/demos'; +import { AudioOscilloscopeVisualizer } from '@/components/livekit/audio-visualizer/audio-oscilloscope-visualizer/audio-oscilloscope-visualizer'; import { AudioRadialVisualizer, audioRadialVisualizerVariants, @@ -710,6 +711,102 @@ export const COMPONENTS = { ); }, + AudioOscilloscopeVisualizer: () => { + // shape + const [shape, setShape] = useState(1.0); + + const sizes = ['icon', 'sm', 'md', 'lg', 'xl']; + const states = [ + 'disconnected', + 'connecting', + 'initializing', + 'listening', + 'thinking', + 'speaking', + ] as AgentState[]; + + const [size, setSize] = useState('lg'); + const [state, setState] = useState(states[0]); + + const { microphoneTrack, localParticipant } = useLocalParticipant(); + const micTrackRef = useMemo(() => { + return state === 'speaking' + ? ({ + participant: localParticipant, + source: Track.Source.Microphone, + publication: microphoneTrack, + } as TrackReference) + : undefined; + }, [state, localParticipant, microphoneTrack]); + + useMicrophone(); + + return ( + +
+
+ + +
+ +
+ + +
+
+ +
+ +
+ +
+ {states.map((stateType) => ( + + ))} +
+ + ); + }, + // Agent control bar AgentControlBar: () => { useMicrophone(); diff --git a/components/livekit/audio-visualizer/audio-oscilloscope-visualizer/audio-oscilloscope-visualizer.tsx b/components/livekit/audio-visualizer/audio-oscilloscope-visualizer/audio-oscilloscope-visualizer.tsx new file mode 100644 index 00000000..8d3bcbf4 --- /dev/null +++ b/components/livekit/audio-visualizer/audio-oscilloscope-visualizer/audio-oscilloscope-visualizer.tsx @@ -0,0 +1,142 @@ +'use client'; + +import { useCallback, useEffect, useMemo, useRef, useState } from 'react'; +import { type VariantProps, cva } from 'class-variance-authority'; +import { + type AnimationPlaybackControlsWithThen, + type ValueAnimationTransition, + animate, + useMotionValue, + useMotionValueEvent, +} from 'motion/react'; +import { + type AgentState, + type TrackReference, + type TrackReferenceOrPlaceholder, + // useMultibandTrackVolume, + useTrackVolume, +} from '@livekit/components-react'; +import { cn } from '@/lib/utils'; +import { OscilliscopeShaders, type OscilliscopeShadersProps } from './oscilliscope-shaders'; + +const DEFAULT_SPEED = 5; +const DEFAULT_AMPLITUDE = 0.025; +const DEFAULT_FREQUENCY = 10; +const DEFAULT_TRANSITION: ValueAnimationTransition = { duration: 0.2, ease: 'easeOut' }; + +function useAnimatedValue(initialValue: T) { + const [value, setValue] = useState(initialValue); + const motionValue = useMotionValue(initialValue); + const controlsRef = useRef(null); + useMotionValueEvent(motionValue, 'change', (value) => setValue(value as T)); + + const animateFn = useCallback( + (targetValue: T | T[], transition: ValueAnimationTransition) => { + controlsRef.current = animate(motionValue, targetValue, transition); + }, + [motionValue] + ); + + return { value, controls: controlsRef, animate: animateFn }; +} + +export const audioShaderVisualizerVariants = cva(['aspect-square'], { + variants: { + size: { + icon: 'h-[24px] gap-[2px]', + sm: 'h-[56px] gap-[4px]', + md: 'h-[112px] gap-[8px]', + lg: 'h-[224px] gap-[16px]', + xl: 'h-[448px] gap-[32px]', + }, + }, + defaultVariants: { + size: 'md', + }, +}); + +interface AudioOscilloscopeVisualizerProps { + speed?: number; + state?: AgentState; + audioTrack: TrackReferenceOrPlaceholder; + className?: string; +} + +export function AudioOscilloscopeVisualizer({ + size = 'lg', + state = 'speaking', + speed = DEFAULT_SPEED, + audioTrack, + className, +}: AudioOscilloscopeVisualizerProps & + OscilliscopeShadersProps & + VariantProps) { + const { value: amplitude, animate: animateAmplitude } = useAnimatedValue(DEFAULT_AMPLITUDE); + const { value: frequency, animate: animateFrequency } = useAnimatedValue(DEFAULT_FREQUENCY); + const { value: opacity, animate: animateOpacity } = useAnimatedValue(1.0); + + const volume = useTrackVolume(audioTrack as TrackReference, { + fftSize: 512, + smoothingTimeConstant: 0.55, + }); + + useEffect(() => { + switch (state) { + case 'disconnected': + animateAmplitude(0, DEFAULT_TRANSITION); + animateFrequency(0, DEFAULT_TRANSITION); + animateOpacity(1.0, DEFAULT_TRANSITION); + return; + case 'listening': + case 'connecting': + animateAmplitude(DEFAULT_AMPLITUDE, DEFAULT_TRANSITION); + animateFrequency(DEFAULT_FREQUENCY, DEFAULT_TRANSITION); + animateOpacity([1.0, 0.2], { + duration: 0.75, + repeat: Infinity, + repeatType: 'mirror', + }); + return; + case 'thinking': + case 'initializing': + animateAmplitude(DEFAULT_AMPLITUDE, DEFAULT_TRANSITION); + animateFrequency(DEFAULT_FREQUENCY, DEFAULT_TRANSITION); + animateOpacity([1.0, 0.2], { + duration: 0.2, + repeat: Infinity, + repeatType: 'mirror', + }); + return; + case 'speaking': + default: + animateAmplitude(DEFAULT_AMPLITUDE, DEFAULT_TRANSITION); + animateFrequency(DEFAULT_FREQUENCY, DEFAULT_TRANSITION); + animateOpacity(1.0, DEFAULT_TRANSITION); + return; + } + }, [state, animateAmplitude, animateFrequency, animateOpacity]); + + useEffect(() => { + if (state === 'speaking' && volume > 0) { + animateAmplitude(0.02 + 0.4 * volume, { duration: 0 }); + animateFrequency(20 + 60 * volume, { duration: 0 }); + } + }, [state, volume, animateAmplitude, animateFrequency]); + + return ( + + ); +} diff --git a/components/livekit/audio-visualizer/audio-oscilloscope-visualizer/oscilliscope-shaders.tsx b/components/livekit/audio-visualizer/audio-oscilloscope-visualizer/oscilliscope-shaders.tsx new file mode 100644 index 00000000..8b413716 --- /dev/null +++ b/components/livekit/audio-visualizer/audio-oscilloscope-visualizer/oscilliscope-shaders.tsx @@ -0,0 +1,168 @@ +'use client'; + +import React, { forwardRef } from 'react'; +import { Shader } from '@/components/livekit/react-shader/react-shader'; + +const oscilliscopeShaderSource = ` +const float TAU = 6.28318530718; + +// Cosine palette generator +vec3 pal(in float t, in vec3 a, in vec3 b, in vec3 c, in vec3 d) { + return a + b * cos(TAU * (c * t + d)); +} + +// Noise for dithering +vec2 randFibo(vec2 p) { + p = fract(p * vec2(443.897, 441.423)); + p += dot(p, p.yx + 19.19); + return fract((p.xx + p.yx) * p.xy); +} + +// Luma for alpha +float luma(vec3 color) { + return dot(color, vec3(0.299, 0.587, 0.114)); +} + +// Bell curve function for attenuation from center with rounded top +float bellCurve(float distanceFromCenter, float maxDistance) { + float normalizedDistance = distanceFromCenter / maxDistance; + // Use cosine with high power for smooth rounded top + return pow(cos(normalizedDistance * (3.14159265359 / 4.0)), 16.0); +} + +// Calculate the sine wave +float oscilloscopeWave(float x, float centerX, float time) { + float relativeX = x - centerX; + float maxDistance = centerX; + float distanceFromCenter = abs(relativeX); + + // Apply bell curve for amplitude attenuation + float bell = bellCurve(distanceFromCenter, maxDistance); + + // Calculate wave with uniforms and bell curve attenuation + float wave = sin(relativeX * uFrequency + time * uSpeed) * uAmplitude * bell; + + return wave; +} + +void mainImage(out vec4 fragColor, in vec2 fragCoord) { + vec2 uv = fragCoord / iResolution.xy; + vec2 pos = uv - 0.5; + + // Calculate center and positions + float centerX = 0.5; + float centerY = 0.5; + float x = uv.x; + float y = uv.y; + + // Find minimum distance to the wave by sampling nearby points + // This gives us consistent line width without high-frequency artifacts + const int NUM_SAMPLES = 50; // Must be const for GLSL loop + float minDist = 1000.0; + float sampleRange = 0.02; // Range to search for closest point + + for(int i = 0; i < NUM_SAMPLES; i++) { + float offset = (float(i) / float(NUM_SAMPLES - 1) - 0.5) * sampleRange; + float sampleX = x + offset; + float waveY = centerY + oscilloscopeWave(sampleX, centerX, iTime); + + // Calculate distance from current pixel to this point on the wave + vec2 wavePoint = vec2(sampleX, waveY); + vec2 currentPoint = vec2(x, y); + float dist = distance(currentPoint, wavePoint); + + minDist = min(minDist, dist); + } + + // Create solid line with anti-aliasing + float lineWidth = uLineWidth; + float smoothing = uSmoothing; + + // Solid line with smooth edges using minimum distance + float line = smoothstep(lineWidth + smoothing, lineWidth - smoothing, minDist); + + // Calculate color position based on x position for gradient effect + float colorPos = x; + vec3 color = pal( + colorPos * uColorScale + uColorPosition * 2.0, + vec3(0.5), + vec3(0.5), + vec3(1.0), + vec3(0.0, 0.33, 0.67) + ); + + // Apply line intensity + color *= line; + + // Add dithering for smoother gradients + // color += (randFibo(fragCoord).x - 0.5) / 255.0; + + // Calculate alpha based on line intensity + float alpha = line * uMix; + + fragColor = vec4(color * uMix, alpha); +}`; + +export interface OscilliscopeShadersProps extends React.HTMLAttributes { + className?: string; + speed?: number; + amplitude?: number; + frequency?: number; + colorScale?: number; + colorPosition?: number; + mix?: number; + lineWidth?: number; + smoothing?: number; +} + +export const OscilliscopeShaders = forwardRef( + ( + { + className, + speed = 10, + amplitude = 0.02, + frequency = 20.0, + colorScale = 0.12, + colorPosition = 0.18, + mix = 1.0, + lineWidth = 0.01, + smoothing = 0.0, + ...props + }, + ref + ) => { + const globalThis = typeof window !== 'undefined' ? window : global; + + console.log('OscilliscopeShaders rendering'); + + return ( +
+ { + console.error('Shader error:', error); + }} + onWarning={(warning) => { + console.warn('Shader warning:', warning); + }} + style={{ width: '100%', height: '100%' } as CSSStyleDeclaration} + /> +
+ ); + } +); + +OscilliscopeShaders.displayName = 'OscilliscopeShaders'; + +export default OscilliscopeShaders; From a87da109dde80e76bcc989acb00f601b685bb1ac Mon Sep 17 00:00:00 2001 From: Thomas Yuill Date: Thu, 30 Oct 2025 10:40:59 -0400 Subject: [PATCH 09/10] cleanup component ui --- app/ui/_components.tsx | 198 +++++++------- .../audio-shader-visualizer.tsx | 53 ++-- .../aurora-shaders.tsx | 250 ++++++++++++++++++ 3 files changed, 373 insertions(+), 128 deletions(-) create mode 100644 components/livekit/audio-visualizer/audio-shader-visualizer/aurora-shaders.tsx diff --git a/app/ui/_components.tsx b/app/ui/_components.tsx index ccd6a307..6f7f03c3 100644 --- a/app/ui/_components.tsx +++ b/app/ui/_components.tsx @@ -245,24 +245,6 @@ export const COMPONENTS = { return (
-
- - -
-
-
Original BarVisualizer
-
- -
+
+ + Original BarVisualizer + +
+ +
+
+
+ +
+ {states.map((stateType) => ( + + ))}
); @@ -363,24 +363,6 @@ export const COMPONENTS = { return (
-
- - -
-
+ +
+ {states.map((stateType) => ( + + ))} +
); }, @@ -476,24 +472,6 @@ export const COMPONENTS = { return (
-
- - -
-
-
- 
-            {JSON.stringify(demoOptions, null, 2)}
-
+ +
+ {states.map((stateType) => ( + + ))} +
+ +
+ Demo options +
+ 
+              {JSON.stringify(demoOptions, null, 2)}
+
+
); }, AudioShaderVisualizer: () => { - const { startSession, endSession } = useSession(); - const { localParticipant } = useLocalParticipant(); - // shape const [shape, setShape] = useState(1.0); // color scale @@ -593,20 +586,18 @@ export const COMPONENTS = { const [size, setSize] = useState('lg'); const [state, setState] = useState(states[0]); - const { - // state, - audioTrack, - } = useVoiceAssistant(); - - useEffect(() => { - if (state === 'speaking') { - startSession(); - localParticipant.setMicrophoneEnabled(true, undefined); - } else { - endSession(); - localParticipant.setMicrophoneEnabled(false, undefined); - } - }, [startSession, endSession, state, localParticipant]); + const { microphoneTrack, localParticipant } = useLocalParticipant(); + const micTrackRef = useMemo(() => { + return state === 'speaking' + ? ({ + participant: localParticipant, + source: Track.Source.Microphone, + publication: microphoneTrack, + } as TrackReference) + : undefined; + }, [state, localParticipant, microphoneTrack]); + + useMicrophone(); const fields = [ ['color position', colorPosition, setColorPosition, 0, 1, 0.01], @@ -615,28 +606,7 @@ export const COMPONENTS = { return ( - - -
-
- - -
-
+
+ {states.map((stateType) => ( + + ))} +
+
{fields.map(([name, value, setValue, min = 0.1, max = 10, step = 0.1]) => { return ( diff --git a/components/livekit/audio-visualizer/audio-shader-visualizer/audio-shader-visualizer.tsx b/components/livekit/audio-visualizer/audio-shader-visualizer/audio-shader-visualizer.tsx index 272a11ad..460a3341 100644 --- a/components/livekit/audio-visualizer/audio-shader-visualizer/audio-shader-visualizer.tsx +++ b/components/livekit/audio-visualizer/audio-shader-visualizer/audio-shader-visualizer.tsx @@ -17,8 +17,8 @@ import { // useMultibandTrackVolume, useTrackVolume, } from '@livekit/components-react'; -import { AuroraShaders, type AuroraShadersProps } from '@/components/ui/shadcn-io/aurora-shaders'; import { cn } from '@/lib/utils'; +import { AuroraShaders, type AuroraShadersProps } from './aurora-shaders'; const DEFAULT_SPEED = 10; const DEFAULT_AMPLITUDE = 2; @@ -26,6 +26,12 @@ const DEFAULT_FREQUENCY = 0.5; const DEFAULT_SCALE = 0.2; const DEFAULT_BRIGHTNESS = 1.5; const DEFAULT_TRANSITION: ValueAnimationTransition = { duration: 0.5, ease: 'easeOut' }; +const DEFAULT_PULSE_TRANSITION: ValueAnimationTransition = { + duration: 0.5, + ease: 'easeOut', + repeat: Infinity, + repeatType: 'mirror', +}; function useAnimatedValue(initialValue: T) { const [value, setValue] = useState(initialValue); @@ -40,7 +46,7 @@ function useAnimatedValue(initialValue: T) { [motionValue] ); - return { value, controls: controlsRef, animate: animateFn }; + return { value, motionValue, controls: controlsRef, animate: animateFn }; } export const audioShaderVisualizerVariants = cva(['aspect-square'], { @@ -75,9 +81,13 @@ export function AudioShaderVisualizer({ AuroraShadersProps & VariantProps) { const [speed, setSpeed] = useState(DEFAULT_SPEED); + const { + value: scale, + animate: animateScale, + motionValue: scaleMotionValue, + } = useAnimatedValue(DEFAULT_SCALE); const { value: amplitude, animate: animateAmplitude } = useAnimatedValue(DEFAULT_AMPLITUDE); const { value: frequency, animate: animateFrequency } = useAnimatedValue(DEFAULT_FREQUENCY); - const { value: scale, animate: animateScale } = useAnimatedValue(DEFAULT_SCALE); const { value: brightness, animate: animateBrightness } = useAnimatedValue(DEFAULT_BRIGHTNESS); const volume = useTrackVolume(audioTrack as TrackReference, { @@ -100,36 +110,29 @@ export function AudioShaderVisualizer({ animateScale(0.35, DEFAULT_TRANSITION); animateAmplitude(1, DEFAULT_TRANSITION); animateFrequency(0.7, DEFAULT_TRANSITION); - animateBrightness(2.0, DEFAULT_TRANSITION); + // animateBrightness(2.0, DEFAULT_TRANSITION); + animateBrightness([1.5, 2.0], DEFAULT_PULSE_TRANSITION); return; case 'initializing': setSpeed(30); animateScale(0.3, DEFAULT_TRANSITION); animateAmplitude(0.5, DEFAULT_TRANSITION); animateFrequency(1, DEFAULT_TRANSITION); - animateBrightness([0.5, 2.5], { - duration: 0.2, - repeat: Infinity, - repeatType: 'mirror', - }); + animateBrightness([0.5, 2.5], DEFAULT_PULSE_TRANSITION); return; case 'thinking': setSpeed(30); - animateScale([0.15, 0.13], { - duration: 0.3, - repeat: Infinity, - repeatType: 'mirror', - }); + animateScale(0.1, DEFAULT_TRANSITION); animateAmplitude(1.0, DEFAULT_TRANSITION); animateFrequency(3.0, DEFAULT_TRANSITION); - animateBrightness([1.5, 2.5], { - duration: 0.3, - repeat: Infinity, - repeatType: 'mirror', - }); + animateBrightness([1.0, 2.0], DEFAULT_PULSE_TRANSITION); return; case 'speaking': setSpeed(50); + animateScale(0.3, DEFAULT_TRANSITION); + animateAmplitude(1.0, DEFAULT_TRANSITION); + animateFrequency(0.7, DEFAULT_TRANSITION); + animateBrightness(1.5, DEFAULT_TRANSITION); return; } }, [ @@ -143,13 +146,21 @@ export function AudioShaderVisualizer({ ]); useEffect(() => { - if (state === 'speaking' && volume > 0) { + if (state === 'speaking' && volume > 0 && !scaleMotionValue.isAnimating()) { animateScale(0.3 - 0.1 * volume, { duration: 0 }); animateAmplitude(1.0 + 0.2 * volume, { duration: 0 }); animateFrequency(0.7 - 0.3 * volume, { duration: 0 }); animateBrightness(1.5 + 1.0 * volume, { duration: 0 }); } - }, [state, volume, animateScale, animateAmplitude, animateFrequency, animateBrightness]); + }, [ + state, + volume, + scaleMotionValue, + animateScale, + animateAmplitude, + animateFrequency, + animateBrightness, + ]); return ( { + /** + * Aurora wave speed + * @default 1.0 + */ + speed?: number; + + /** + * Turbulence amplitude + * @default 0.5 + */ + amplitude?: number; + + /** + * Wave frequency and complexity + * @default 0.5 + */ + frequency?: number; + + /** + * Shape scale + * @default 0.3 + */ + scale?: number; + + /** + * Shape type: 1=circle, 2=line + * @default 1 + */ + shape?: number; + + /** + * Edge blur/softness + * @default 1.0 + */ + blur?: number; + + /** + * Color palette offset - shifts colors along the gradient (0-1) + * Lower values shift toward start colors, higher toward end colors + * @default 0.5 + * @example 0.0 - cool tones dominate + * @example 0.5 - balanced (default) + * @example 1.0 - warm tones dominate + */ + colorPosition?: number; + + /** + * Color variation across layers (0-1) + * Controls how much colors change between iterations + * @default 0.5 + * @example 0.0 - minimal color variation (more uniform) + * @example 0.5 - moderate variation (default) + * @example 1.0 - maximum variation (rainbow effect) + */ + colorScale?: number; + + /** + * Brightness of the aurora (0-1) + * @default 1.0 + */ + brightness?: number; +} + +export const AuroraShaders = forwardRef( + ( + { + className, + shape = 1.0, + speed = 1.0, + amplitude = 0.5, + frequency = 0.5, + scale = 0.2, + blur = 1.0, + colorPosition = 1.0, + colorScale = 1.0, + brightness = 1.0, + ...props + }, + ref + ) => { + const globalThis = typeof window !== 'undefined' ? window : global; + return ( +
+ { + console.log('error', error); + }} + onWarning={(warning) => { + console.log('warning', warning); + }} + style={{ width: '100%', height: '100%' } as CSSStyleDeclaration} + /> +
+ ); + } +); + +AuroraShaders.displayName = 'AuroraShaders'; + +export default AuroraShaders; From ab3db1aff291e977b5bf8727e37ade9d2be46814 Mon Sep 17 00:00:00 2001 From: Thomas Yuill Date: Fri, 7 Nov 2025 09:45:44 -0500 Subject: [PATCH 10/10] --wip-- [skip ci] --- .../aurora-shaders.tsx | 49 +++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/components/livekit/audio-visualizer/audio-shader-visualizer/aurora-shaders.tsx b/components/livekit/audio-visualizer/audio-shader-visualizer/aurora-shaders.tsx index 57bf62fb..ccadd7f7 100644 --- a/components/livekit/audio-visualizer/audio-shader-visualizer/aurora-shaders.tsx +++ b/components/livekit/audio-visualizer/audio-shader-visualizer/aurora-shaders.tsx @@ -113,6 +113,55 @@ void mainImage(out vec4 fragColor, in vec2 fragCoord) { color += (randFibo(fragCoord).x - 0.5) / 255.0; color = Tonemap_Reinhard(color); + // // Add sphere in center + // float sphereRadius = uScale * 0.5; + // float distToCenter = length(pos); + + // if(distToCenter < sphereRadius) { + // // Generate sphere color using same palette as aurora + // vec3 sphereColor = pal( + // 0.5 * mix(0.0, 3.0, uColorScale) + uColorPosition * 2.0, + // vec3(0.5), + // vec3(0.5), + // vec3(1.0), + // vec3(0.0, 0.33, 0.67) + // ); + + // // Calculate 3D sphere normal for lighting + // float z = sqrt(max(0.0, sphereRadius * sphereRadius - distToCenter * distToCenter)); + // vec3 normal = normalize(vec3(pos.x, pos.y, z)); + + // // Light direction from above (slightly in front) + // vec3 lightDir = normalize(vec3(0.0, 0.3, 1.0)); + + // // View direction (looking straight at the sphere) + // vec3 viewDir = vec3(0.0, 1.0, 0.5); + + // // Diffuse lighting + // float diffuse = max(dot(normal, lightDir), 0.0); + + // // Specular lighting (Blinn-Phong) + // vec3 halfDir = normalize(lightDir + viewDir); + // float specular = pow(max(dot(normal, halfDir), 0.0), 32.0); + + // // Ambient lighting + // float ambient = 0.3; + + // // Combine lighting + // float lighting = ambient + diffuse * 0.7 + specular * 1.5; + // vec3 litSphereColor = sphereColor * lighting; + + // // Make sphere twice as bright as the aurora + // litSphereColor *= 2.0; + + // // Smooth edge for the sphere + // float sphereEdge = smoothstep(sphereRadius, sphereRadius - 0.02, distToCenter); + + // // Blend sphere with existing color (semitransparent) + // float sphereAlpha = 0.5 * sphereEdge; + // color = mix(color, litSphereColor, sphereAlpha); + } + float alpha = luma(color) * uMix; fragColor = vec4(color * uMix, alpha); }`;