From ee84a7b208ca7bd362b744fcff0b75bc970a438f Mon Sep 17 00:00:00 2001 From: Scott Willeke Date: Thu, 20 Feb 2025 18:51:57 -0800 Subject: [PATCH 1/3] draft: conversation view of hydrated convo --- .../src/components/ConversationView.tsx | 143 ++++++++++++++++++ .../src/components/RealtimeSessionView.tsx | 51 ++++++- .../src/components/simpleConversation.ts | 37 +++++ .../src/pages/WebRTCExample.tsx | 10 ++ 4 files changed, 240 insertions(+), 1 deletion(-) create mode 100644 apps/browser-example/src/components/ConversationView.tsx create mode 100644 apps/browser-example/src/components/simpleConversation.ts diff --git a/apps/browser-example/src/components/ConversationView.tsx b/apps/browser-example/src/components/ConversationView.tsx new file mode 100644 index 0000000..22c5542 --- /dev/null +++ b/apps/browser-example/src/components/ConversationView.tsx @@ -0,0 +1,143 @@ +import { type ReactNode, useRef, useEffect } from "react" +import { debounce } from "lodash-es" +import type { + RealtimeConversationItem, + RealtimeConversationItemContent, +} from "@tsorta/browser/openai" +import { + DefinedRole, + RealtimeConversationItemSimple, + simplifyItem, +} from "./simpleConversation" + +const log = console + +export interface ConversationProps { + conversation: RealtimeConversationItemSimple[] +} + +export const ConversationView = ({ + conversation, +}: ConversationProps): ReactNode => { + return ( +
+ {conversation + .filter((item) => item.role !== "system") + .map((item, index, arr) => ( + + ))} +
+ ) +} + +const RoleBgColorMap: Record = { + user: "bg-primary-subtle", + assistant: "bg-secondary-subtle", + system: "bg-secondary", +} +const RoleTextColorMap: Record = { + user: "text-primary", + assistant: "text-secondary", + system: "text-secondary", +} +const RoleLabelMap: Record = { + user: "You", + assistant: "Assistant", + system: "System", +} + +interface ConversationItemProps { + item: RealtimeConversationItem + doScrollIntoView: boolean +} + +const ConversationItem = ({ + item, + doScrollIntoView, +}: ConversationItemProps): ReactNode => { + const simpleItem = simplifyItem(item) + const { id, role, content } = simpleItem + const alignClass = role === "user" ? "align-self-end" : "align-self-start" + + const divRef = useRef(null) + + useEffect(() => { + const executeScroll = () => { + // behavior: smooth works fine on FF+macOS, but not on Chrome+macOS, so we detect and force "instant" on chrome + const behavior = /Chrome/.test(navigator.userAgent) ? "instant" : "smooth" + divRef.current?.scrollIntoView({ + behavior, + block: "end", + inline: "nearest", + }) + } + + if (doScrollIntoView) { + debounce(executeScroll, 500)() + } + }, [item, doScrollIntoView, item.content, simpleItem.content.length]) + + return ( +
+
+ {RoleLabelMap[role]} +
+ +
+
+ {content.map((contentItem, index) => ( + + ))} +
+
+
+ ) +} + +interface ConversationItemContentProps { + content: RealtimeConversationItemContent + doScrollIntoView: boolean +} + +const ConversationItemContent = ({ + content, +}: ConversationItemContentProps): ReactNode => { + if (!["text", "input_text", "input_audio"].includes(content.type)) { + // NOTE: we find content.type="audio" coming in here in logging though it is not in the types! + if ((content.type as string) !== "audio") { + log.warn( + `Unexpected type for RealtimeConversationItemContent '${content.type}'. Will not be rendered: %o`, + content + ) + } + return null + } + + return ( +
+ {(content.type == "text" || content.type == "input_text") && ( + {content.text} + )} + {content.type == "input_audio" && ( + + {content.transcript ? content.transcript : "..."} + + )} +
+ ) +} diff --git a/apps/browser-example/src/components/RealtimeSessionView.tsx b/apps/browser-example/src/components/RealtimeSessionView.tsx index daedd29..8ac0644 100644 --- a/apps/browser-example/src/components/RealtimeSessionView.tsx +++ b/apps/browser-example/src/components/RealtimeSessionView.tsx @@ -3,6 +3,7 @@ import { BootstrapIcon } from "./BootstrapIcon" import { EventList } from "./EventList" import { useModal } from "../hooks/useModal" import { RealtimeSessionCreateRequest } from "@tsorta/browser/openai" +import { ConversationView } from "./ConversationView" type PartialSessionRequestWithModel = Partial & Pick, "model"> @@ -131,7 +132,55 @@ export function RealtimeSessionView({ -

Events:

+
    +
  • + +
    • +
  • + +
    • +
+
+
+ +
+
+
TODO
+ {/**/} +
+
) diff --git a/apps/browser-example/src/components/simpleConversation.ts b/apps/browser-example/src/components/simpleConversation.ts new file mode 100644 index 0000000..f30f07c --- /dev/null +++ b/apps/browser-example/src/components/simpleConversation.ts @@ -0,0 +1,37 @@ +import { + RealtimeConversationItem, + RealtimeConversationItemContent, +} from "@tsorta/browser/openai" + +/** Removes the possibility of `undefined` in @see RealtimeConversationItem.role. */ +export type DefinedRole = NonNullable + +/** Removes the possibility of `undefined` in @see RealtimeConversationItem.type. */ +type DefinedRealtimeConversationItemType = NonNullable< + RealtimeConversationItem["type"] +> + +export type RealtimeConversationItemSimple = Pick< + RealtimeConversationItem, + "id" +> & { + role: DefinedRole + type: DefinedRealtimeConversationItemType + content: RealtimeConversationItemContent[] +} + +export function simplifyItem( + item: RealtimeConversationItem +): RealtimeConversationItemSimple { + if (!item.role) { + throw new Error("Role missing in conversation item") + } + const role: DefinedRole = item.role + const id = item.id || "id-missing" + const type = item.type as DefinedRealtimeConversationItemType + // NOTE: There may be no contents on the initial creation while the model is still replying. + const content: RealtimeConversationItemContent[] = (item.content || + []) as RealtimeConversationItemContent[] + + return { id, type, role, content } +} diff --git a/apps/browser-example/src/pages/WebRTCExample.tsx b/apps/browser-example/src/pages/WebRTCExample.tsx index 435f1a8..9c598d1 100644 --- a/apps/browser-example/src/pages/WebRTCExample.tsx +++ b/apps/browser-example/src/pages/WebRTCExample.tsx @@ -5,6 +5,7 @@ import { } from "../components/RealtimeSessionView" import { RealtimeClient } from "@tsorta/browser/WebRTC" import { PageProps } from "./props" +import { RealtimeConversationItem } from "@tsorta/browser/openai" export function WebRTCExample({ apiKey, @@ -15,6 +16,9 @@ export function WebRTCExample({ const [client, setClient] = useState(undefined) const [events, setEvents] = useState([]) + const [conversation, setConversation] = useState( + [] + ) const startSession = useCallback( async function startSession({ @@ -52,6 +56,11 @@ export function WebRTCExample({ setEvents((events) => [...events, event.event]) }) + client.addEventListener("conversationChanged", (event) => { + console.debug("conversationChanged event:", event.conversation) + setConversation(event.conversation) + }) + await client.start() onSessionStatusChanged("recording") @@ -79,6 +88,7 @@ export function WebRTCExample({ by Scott Willeke.

+ Date: Thu, 20 Feb 2025 19:36:22 -0800 Subject: [PATCH 2/3] feat: shows the hydrated conversation from the client in WebRTC SDK --- .../src/components/ConversationView.tsx | 11 ++--- .../src/components/RealtimeSessionView.tsx | 49 +++++++++++++------ .../src/pages/WebRTCExample.tsx | 1 + 3 files changed, 39 insertions(+), 22 deletions(-) diff --git a/apps/browser-example/src/components/ConversationView.tsx b/apps/browser-example/src/components/ConversationView.tsx index 22c5542..0c88cb7 100644 --- a/apps/browser-example/src/components/ConversationView.tsx +++ b/apps/browser-example/src/components/ConversationView.tsx @@ -4,16 +4,12 @@ import type { RealtimeConversationItem, RealtimeConversationItemContent, } from "@tsorta/browser/openai" -import { - DefinedRole, - RealtimeConversationItemSimple, - simplifyItem, -} from "./simpleConversation" +import { DefinedRole, simplifyItem } from "./simpleConversation" const log = console export interface ConversationProps { - conversation: RealtimeConversationItemSimple[] + conversation: RealtimeConversationItem[] } export const ConversationView = ({ @@ -93,7 +89,8 @@ const ConversationItem = ({
{content.map((contentItem, index) => ( diff --git a/apps/browser-example/src/components/RealtimeSessionView.tsx b/apps/browser-example/src/components/RealtimeSessionView.tsx index 8ac0644..9941738 100644 --- a/apps/browser-example/src/components/RealtimeSessionView.tsx +++ b/apps/browser-example/src/components/RealtimeSessionView.tsx @@ -2,7 +2,10 @@ import { ReactNode, useState } from "react" import { BootstrapIcon } from "./BootstrapIcon" import { EventList } from "./EventList" import { useModal } from "../hooks/useModal" -import { RealtimeSessionCreateRequest } from "@tsorta/browser/openai" +import { + RealtimeConversationItem, + RealtimeSessionCreateRequest, +} from "@tsorta/browser/openai" import { ConversationView } from "./ConversationView" type PartialSessionRequestWithModel = Partial & @@ -16,6 +19,7 @@ interface RealtimeSessionViewProps { stopSession: () => Promise sessionStatus: "unavailable" | "stopped" | "recording" events: { type: string }[] + conversation?: RealtimeConversationItem[] } export function RealtimeSessionView({ @@ -23,6 +27,7 @@ export function RealtimeSessionView({ stopSession, sessionStatus, events, + conversation, }: RealtimeSessionViewProps): ReactNode { // TODO: allow user to select the model const model = "gpt-4o-realtime-preview-2024-12-17" @@ -31,6 +36,10 @@ export function RealtimeSessionView({ undefined ) + const [activeTab, setActiveTab] = useState<"events" | "conversation">( + "events" + ) + const modal = useModal({ title: "Edit Instructions", children: ( @@ -132,31 +141,31 @@ export function RealtimeSessionView({ -
    +
    • @@ -164,7 +173,9 @@ export function RealtimeSessionView({
    -
    TODO
    - {/**/} + {conversation && conversation.length > 0 ? ( + + ) : ( +
    + {conversation !== undefined + ? "Conversation data not yet available. Start a session and talk and they should appear." + : "Conversations not available in this SDK"} +
    + )}
    -
) } diff --git a/apps/browser-example/src/pages/WebRTCExample.tsx b/apps/browser-example/src/pages/WebRTCExample.tsx index 9c598d1..b704a7a 100644 --- a/apps/browser-example/src/pages/WebRTCExample.tsx +++ b/apps/browser-example/src/pages/WebRTCExample.tsx @@ -94,6 +94,7 @@ export function WebRTCExample({ stopSession={stopSession} sessionStatus={sessionStatus} events={events} + conversation={conversation} />
) From f73008e64c8d2168c76a7c70b1bb0c83980c4f7a Mon Sep 17 00:00:00 2001 From: Scott Willeke Date: Thu, 20 Feb 2025 19:40:43 -0800 Subject: [PATCH 3/3] chore: some cleanup and comments --- apps/browser-example/src/components/simpleConversation.ts | 6 ++++++ apps/browser-example/src/pages/WebRTCExample.tsx | 2 -- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/apps/browser-example/src/components/simpleConversation.ts b/apps/browser-example/src/components/simpleConversation.ts index f30f07c..120f66f 100644 --- a/apps/browser-example/src/components/simpleConversation.ts +++ b/apps/browser-example/src/components/simpleConversation.ts @@ -11,6 +11,9 @@ type DefinedRealtimeConversationItemType = NonNullable< RealtimeConversationItem["type"] > +/** + * A simplified form of @see RealtimeConversationItem for rendering. + */ export type RealtimeConversationItemSimple = Pick< RealtimeConversationItem, "id" @@ -20,6 +23,9 @@ export type RealtimeConversationItemSimple = Pick< content: RealtimeConversationItemContent[] } +/** + * Simplifies the @see RealtimeConversationItem for rendering purposes. Mostly removes the possibility of `undefined` values in places where they are unlikely (impossible) at render time. + */ export function simplifyItem( item: RealtimeConversationItem ): RealtimeConversationItemSimple { diff --git a/apps/browser-example/src/pages/WebRTCExample.tsx b/apps/browser-example/src/pages/WebRTCExample.tsx index b704a7a..b427100 100644 --- a/apps/browser-example/src/pages/WebRTCExample.tsx +++ b/apps/browser-example/src/pages/WebRTCExample.tsx @@ -52,12 +52,10 @@ export function WebRTCExample({ setClient(client) client.addEventListener("serverEvent", (event) => { - console.debug("serverEvent event:", event) setEvents((events) => [...events, event.event]) }) client.addEventListener("conversationChanged", (event) => { - console.debug("conversationChanged event:", event.conversation) setConversation(event.conversation) })