feat: add local track state back in to useConversationWith

Author: 1egoman

packages/react/src/hooks/useConversationWith.ts

@@ -1,13 +1,14 @@
 import type TypedEventEmitter from 'typed-emitter';
-import { Room, RoomEvent, ConnectionState, TrackPublishOptions, Track } from 'livekit-client';
+import { Room, RoomEvent, ConnectionState, TrackPublishOptions, Track, LocalParticipant } from 'livekit-client';
 import { EventEmitter } from 'events';
 import { useCallback, useEffect, useMemo, useState } from 'react';
 
 import { ConnectionCredentials } from '../utils/ConnectionCredentialsProvider';
 import { useMaybeRoomContext } from '../context';
 import { RoomAgentDispatch, RoomConfiguration } from '@livekit/protocol';
 import { useAgent } from './useAgent';
-import { TrackReferencePlaceholder } from '@livekit/components-core';
+import { TrackReferenceOrPlaceholder, TrackReferencePlaceholder } from '@livekit/components-core';
+import { useLocalParticipant } from './useLocalParticipant';
 
 /** State representing the current connection status to the server hosted agent */
 // FIXME: maybe just make this ConnectionState?
@@ -75,18 +76,33 @@ type ConversationStateConnecting = ConversationStateCommon & {
   connectionState: "connecting";
   isConnected: false;
   isReconnecting: false;
+
+  local: {
+    camera: TrackReferencePlaceholder<Track.Source.Camera, LocalParticipant>;
+    microphone: TrackReferencePlaceholder<Track.Source.Microphone, LocalParticipant>;
+  };
 };
 
 type ConversationStateConnected = ConversationStateCommon & {
   connectionState: "connected" | "reconnecting" | "signalReconnecting";
   isConnected: true;
   isReconnecting: boolean;
+
+  local: {
+    camera: TrackReferenceOrPlaceholder<Track.Source.Camera, LocalParticipant>,
+    microphone: TrackReferenceOrPlaceholder<Track.Source.Microphone, LocalParticipant>,
+  };
 };
 
 type ConversationStateDisconnected = ConversationStateCommon & {
   connectionState: "disconnected";
   isConnected: false;
   isReconnecting: false;
+
+  local: {
+    camera: TrackReferencePlaceholder<Track.Source.Camera, LocalParticipant>;
+    microphone: TrackReferencePlaceholder<Track.Source.Microphone, LocalParticipant>;
+  };
 };
 
 type ConversationActions = {
@@ -166,6 +182,14 @@ export function useConversationWith(agentToDispatch: string | RoomAgentDispatch,
     };
   }, [room, emitter]);
 
+  const { localParticipant, cameraTrack, microphoneTrack } = useLocalParticipant({ room });
+  const localCamera = useMemo(() => {
+    return cameraTrack ? { source: Track.Source.Camera as const, participant: localParticipant, publication: microphoneTrack } : null;
+  }, [localParticipant, microphoneTrack]);
+  const localMicrophone = useMemo(() => {
+    return microphoneTrack ? { source: Track.Source.Microphone as const, participant: localParticipant, publication: microphoneTrack } : null;
+  }, [localParticipant, microphoneTrack]);
+
   const conversationState = useMemo((): ConversationStateConnecting | ConversationStateConnected | ConversationStateDisconnected => {
     const common: ConversationStateCommon = {
       [Symbol.toStringTag]: "AgentSessionInstance",
@@ -185,6 +209,11 @@ export function useConversationWith(agentToDispatch: string | RoomAgentDispatch,
 
           connectionState: 'connecting',
           ...generateDerivedConnectionStateValues('connecting'),
+
+          local: {
+            camera: { participant: localParticipant, source: Track.Source.Camera },
+            microphone: { participant: localParticipant, source: Track.Source.Microphone },
+          },
         };
 
       case ConnectionState.Connected:
@@ -195,6 +224,11 @@ export function useConversationWith(agentToDispatch: string | RoomAgentDispatch,
 
           connectionState: roomConnectionState,
           ...generateDerivedConnectionStateValues(roomConnectionState),
+
+          local: {
+            camera: localCamera ?? { participant: localParticipant, source: Track.Source.Camera },
+            microphone: localMicrophone ?? { participant: localParticipant, source: Track.Source.Microphone },
+          },
         };
 
       case ConnectionState.Disconnected:
@@ -203,9 +237,14 @@ export function useConversationWith(agentToDispatch: string | RoomAgentDispatch,
 
           connectionState: ConnectionState.Disconnected,
           ...generateDerivedConnectionStateValues(ConnectionState.Disconnected),
+
+          local: {
+            camera: { participant: localParticipant, source: Track.Source.Camera },
+            microphone: { participant: localParticipant, source: Track.Source.Microphone },
+          },
         };
     }
-  }, [options.credentials, room, emitter, roomConnectionState]);
+  }, [options.credentials, room, emitter, roomConnectionState, localParticipant]);
   useEffect(() => {
     emitter.emit(ConversationEvent.ConnectionStateChanged, conversationState.connectionState);
   }, [emitter, conversationState.connectionState]);