From d8bf234b3d5f9d499f4cd652b3fa7c40d591e582 Mon Sep 17 00:00:00 2001 From: Daniel La Rocque Date: Fri, 17 Oct 2025 11:26:17 -0400 Subject: [PATCH 1/2] Add Live API video recording --- ai/ai-react-app/src/views/LiveView.module.css | 103 ++++++++- ai/ai-react-app/src/views/LiveView.tsx | 206 +++++++++++++----- 2 files changed, 258 insertions(+), 51 deletions(-) diff --git a/ai/ai-react-app/src/views/LiveView.module.css b/ai/ai-react-app/src/views/LiveView.module.css index 65396255c..971cb4dff 100644 --- a/ai/ai-react-app/src/views/LiveView.module.css +++ b/ai/ai-react-app/src/views/LiveView.module.css @@ -77,25 +77,126 @@ cursor: not-allowed; } + + .errorMessage { + background-color: var(--color-error-bg); + color: var(--color-error-text); + border: 1px solid var(--color-error-border); + padding: 10px 16px; + border-radius: 4px; + margin-top: 20px; + max-width: 500px; + white-space: pre-wrap; + +} + + + +.controlsContainer { + + display: flex; + + flex-direction: column; + + gap: 16px; + + margin-top: 20px; + + padding-top: 20px; + + border-top: 1px solid var(--color-border-primary); + + width: 100%; + + max-width: 500px; + + align-items: center; + +} + + + +.videoControls { + + display: flex; + + gap: 10px; + + align-items: center; + + justify-content: center; + + width: 100%; + } + + +.videoSourceSelect { + + padding: 12px 16px; + + background-color: var(--color-surface-primary); + + border: 1px solid var(--color-border-secondary); + + color: var(--color-text-primary); + + border-radius: 24px; + + font-size: 1rem; + + cursor: pointer; + + transition: border-color 0.15s ease; + +} + +.videoSourceSelect:hover { + + border-color: var(--brand-gray-50); + +} + +.videoSourceSelect:disabled { + + opacity: 0.5; + + cursor: not-allowed; + + background-color: var(--color-surface-tertiary); + +} + + + @keyframes pulse { + 0% { + box-shadow: 0 0 0 0 rgba(52, 168, 83, 0.7); + } + 70% { + box-shadow: 0 0 0 10px rgba(52, 168, 83, 0); + } + 100% { + box-shadow: 0 0 0 0 rgba(52, 168, 83, 0); + } -} \ No newline at end of file + +} diff --git a/ai/ai-react-app/src/views/LiveView.tsx b/ai/ai-react-app/src/views/LiveView.tsx index d328d94b3..9effb4fcc 100644 --- a/ai/ai-react-app/src/views/LiveView.tsx +++ b/ai/ai-react-app/src/views/LiveView.tsx @@ -4,7 +4,10 @@ import { AI, getLiveGenerativeModel, startAudioConversation, + startVideoRecording, AudioConversationController, + VideoRecordingController, + LiveSession, AIError, ResponseModality, } from "firebase/ai"; @@ -14,18 +17,27 @@ interface LiveViewProps { aiInstance: AI; } -type ConversationState = "idle" | "active" | "error"; +type SessionState = "idle" | "connecting" | "connected" | "error"; +type VideoSource = "camera" | "screen"; const LiveView: React.FC = ({ aiInstance }) => { - const [conversationState, setConversationState] = - useState("idle"); + const [sessionState, setSessionState] = useState("idle"); const [error, setError] = useState(null); - const [controller, setController] = + + const [liveSession, setLiveSession] = useState(null); + const [audioController, setAudioController] = useState(null); + const [videoController, setVideoController] = + useState(null); + + const [videoSource, setVideoSource] = useState("camera"); + + const isAudioRunning = audioController !== null; + const isVideoRunning = videoController !== null; - const handleStartConversation = useCallback(async () => { + const handleConnect = useCallback(async () => { setError(null); - setConversationState("active"); + setSessionState("connecting"); try { const modelName = LIVE_MODELS.get(aiInstance.backend.backendType)!; @@ -33,23 +45,17 @@ const LiveView: React.FC = ({ aiInstance }) => { const model = getLiveGenerativeModel(aiInstance, { model: modelName, generationConfig: { - responseModalities: [ResponseModality.AUDIO] - } + responseModalities: [ResponseModality.AUDIO], + }, }); console.log("[LiveView] Connecting to live session..."); - const liveSession = await model.connect(); - - console.log( - "[LiveView] Starting audio conversation. This will request microphone permissions.", - ); - - const newController = await startAudioConversation(liveSession); - - setController(newController); - console.log("[LiveView] Audio conversation started successfully."); + const session = await model.connect(); + setLiveSession(session); + setSessionState("connected"); + console.log("[LiveView] Live session connected successfully."); } catch (err: unknown) { - console.error("[LiveView] Failed to start conversation:", err); + console.error("[LiveView] Failed to connect to live session:", err); let errorMessage = "An unknown error occurred."; if (err instanceof AIError) { errorMessage = `Error (${err.code}): ${err.message}`; @@ -57,39 +63,104 @@ const LiveView: React.FC = ({ aiInstance }) => { errorMessage = err.message; } setError(errorMessage); - setConversationState("error"); - setController(null); // Ensure controller is cleared on error + setSessionState("error"); + setLiveSession(null); } }, [aiInstance]); - const handleStopConversation = useCallback(async () => { - if (!controller) return; + const handleDisconnect = useCallback(async () => { + console.log("[LiveView] Disconnecting live session..."); + if (audioController) { + await audioController.stop(); + setAudioController(null); + } + if (videoController) { + await videoController.stop(); + setVideoController(null); + } + // The liveSession does not have an explicit close() method in the public API. + // Resources are released when the controllers are stopped. + setLiveSession(null); + setSessionState("idle"); + console.log("[LiveView] Live session disconnected."); + }, [audioController, videoController]); + + const handleToggleAudio = useCallback(async () => { + if (!liveSession) return; + + if (isAudioRunning) { + console.log("[LiveView] Stopping audio conversation..."); + await audioController?.stop(); + setAudioController(null); + console.log("[LiveView] Audio conversation stopped."); + } else { + console.log( + "[LiveView] Starting audio conversation. This may request microphone permissions.", + ); + try { + const newController = await startAudioConversation(liveSession); + setAudioController(newController); + console.log("[LiveView] Audio conversation started successfully."); + } catch (err: unknown) { + console.error("[LiveView] Failed to start audio conversation:", err); + setError( + err instanceof Error + ? err.message + : "Failed to start audio conversation.", + ); + } + } + }, [liveSession, audioController, isAudioRunning]); - console.log("[LiveView] Stopping audio conversation..."); - await controller.stop(); - setController(null); - setConversationState("idle"); - console.log("[LiveView] Audio conversation stopped."); - }, [controller]); + const handleToggleVideo = useCallback(async () => { + if (!liveSession) return; - // Cleanup effect to stop the conversation if the component unmounts + if (isVideoRunning) { + console.log("[LiveView] Stopping video recording..."); + await videoController?.stop(); + setVideoController(null); + console.log("[LiveView] Video recording stopped."); + } else { + console.log( + `[LiveView] Starting video recording with source: ${videoSource}. This may request permissions.`, + ); + try { + const newController = await startVideoRecording(liveSession, { + videoSource, + }); + setVideoController(newController); + console.log("[LiveView] Video recording started successfully."); + } catch (err: unknown) { + console.error("[LiveView] Failed to start video recording:", err); + setError( + err instanceof Error + ? err.message + : "Failed to start video recording.", + ); + } + } + }, [liveSession, videoController, isVideoRunning, videoSource]); + + // Cleanup effect to disconnect if the component unmounts useEffect(() => { return () => { - if (controller) { + if (liveSession && liveSession.isClosed) { console.log( - "[LiveView] Component unmounting, stopping active conversation.", + "[LiveView] Component unmounting, disconnecting live session.", ); - controller.stop(); + handleDisconnect(); } }; - }, [controller]); + }, [liveSession, handleDisconnect]); const getStatusText = () => { - switch (conversationState) { + switch (sessionState) { case "idle": - return "Ready"; - case "active": - return "In Conversation"; + return "Idle"; + case "connecting": + return "Connecting..."; + case "connected": + return "Connected"; case "error": return "Error"; default: @@ -99,16 +170,16 @@ const LiveView: React.FC = ({ aiInstance }) => { return (
-

Live Conversation

+

Gemini Live

- Click the button below to start a real-time voice conversation with the - model. Your browser will ask for microphone permissions. + Connect to a live session, then start audio and/or video streams. + Ensure you grant microphone and camera/screen permissions when prompted.

Status: {getStatusText()} @@ -116,20 +187,55 @@ const LiveView: React.FC = ({ aiInstance }) => { + {sessionState === "connected" && ( +
+ {/* Audio Controls */} + + + {/* Video Controls */} +
+ + +
+
+ )} + {error &&
{error}
}
); From 05123a6d6fc8aec1ea3b776d0bb407f2c1f21081 Mon Sep 17 00:00:00 2001 From: Daniel La Rocque Date: Fri, 17 Oct 2025 14:16:48 -0400 Subject: [PATCH 2/2] Cleanup --- ai/ai-react-app/src/views/LiveView.module.css | 60 ------------------- 1 file changed, 60 deletions(-) diff --git a/ai/ai-react-app/src/views/LiveView.module.css b/ai/ai-react-app/src/views/LiveView.module.css index 971cb4dff..bb4f5612f 100644 --- a/ai/ai-react-app/src/views/LiveView.module.css +++ b/ai/ai-react-app/src/views/LiveView.module.css @@ -77,126 +77,66 @@ cursor: not-allowed; } - - .errorMessage { - background-color: var(--color-error-bg); - color: var(--color-error-text); - border: 1px solid var(--color-error-border); - padding: 10px 16px; - border-radius: 4px; - margin-top: 20px; - max-width: 500px; - white-space: pre-wrap; - } - - .controlsContainer { - display: flex; - flex-direction: column; - gap: 16px; - margin-top: 20px; - padding-top: 20px; - border-top: 1px solid var(--color-border-primary); - width: 100%; - max-width: 500px; - align-items: center; - } - - .videoControls { - display: flex; - gap: 10px; - align-items: center; - justify-content: center; - width: 100%; - } - - .videoSourceSelect { - padding: 12px 16px; - background-color: var(--color-surface-primary); - border: 1px solid var(--color-border-secondary); - color: var(--color-text-primary); - border-radius: 24px; - font-size: 1rem; - cursor: pointer; - transition: border-color 0.15s ease; - } .videoSourceSelect:hover { - border-color: var(--brand-gray-50); - } .videoSourceSelect:disabled { - opacity: 0.5; - cursor: not-allowed; - background-color: var(--color-surface-tertiary); - } - - @keyframes pulse { - 0% { - box-shadow: 0 0 0 0 rgba(52, 168, 83, 0.7); - } - 70% { - box-shadow: 0 0 0 10px rgba(52, 168, 83, 0); - } - 100% { - box-shadow: 0 0 0 0 rgba(52, 168, 83, 0); - } - }