Skip to content

Commit 89e205b

Browse files
activescottactivescott
committed
feat: initial working WebRTC client for OpenAI Realtime API and demos
0 parents  commit 89e205b

35 files changed

+15085
-0
lines changed

‎.gitignore‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
.DS_Store
2+
node_modules/
3+
4+
# React Router
5+
.react-router/
6+
dist/
7+
tsconfig.tsbuildinfo

‎.vscode/settings.json‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
{
2+
"cSpell.words": [
3+
"tsorta"
4+
]
5+
}

‎README.md‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
# tsorta: TypeScript OpenAPI Realtime API
2+
3+
This repo is for an article I've written about how to use the OpenAI Realtime API with TypeScript + React and WebRTC.
4+
It is a working example that you can experience the Realtime API with yourself. The WebRTC client also does some heavy lifting around managing the conversation state and audio streams.
5+
6+
This repo also has an example that demonstrates the [OpenAI Official SDK support for WebSocket API](https://github.com/openai/openai-node/commit/a796d21f06307419f352da8b9943f6745ff4084f) that was released in beta on Jan 17, 2025. As of the time that I'm writing this, the official SDK doesn't provide support for managing the conversation state, audio streams, nor WebRTC which is all demonstrated in the example. See [apps/browser-example/src/pages/WebRTCExample.tsx](apps/browser-example/src/pages/WebRTCExample.tsx) for the complete example code.
7+
8+
This project also has a reusable package that you can use in your own projects in [packages/browser](packages/browser). I plan to publish this here soon for others. If you're interested in the package let me know and I'll get it pushed to npm!
9+
10+
More at https://scott.willeke.com/2025-01-31-typescript-client-for-openai-realtime-api
11+
12+
## References
13+
14+
- https://platform.openai.com/docs/guides/realtime-webrtc - guide
15+
- https://github.com/openai/openai-openapi - reference
16+
- https://reactrouter.com/home
17+
- https://www.typescriptlang.org/docs/handbook/project-references.html
18+
- https://github.com/openai/openai-node
19+
- https://webrtc.github.io/samples/
20+
- https://fly.io/docs/languages-and-frameworks/dockerfile/

‎apps/browser-example/.gitignore‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
# Logs
2+
logs
3+
*.log
4+
npm-debug.log*
5+
yarn-debug.log*
6+
yarn-error.log*
7+
pnpm-debug.log*
8+
lerna-debug.log*
9+
10+
node_modules
11+
dist
12+
dist-ssr
13+
*.local
14+
15+
# Editor directories and files
16+
.vscode/*
17+
!.vscode/extensions.json
18+
.idea
19+
.DS_Store
20+
*.suo
21+
*.ntvs*
22+
*.njsproj
23+
*.sln
24+
*.sw?

‎apps/browser-example/README.md‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
# OpenAI Official SDK Realtime Example
2+
3+
This is a working example of using OpenAI Realtime SDK using the tsorta unnoficial TypeScript OpenAI Realtime API package and official OpenAI SDK's recently added support for the Realtime API documented in [the readme here](https://github.com/openai/openai-node/tree/v4.81.0#realtime-api-beta) (added on 2025-01-17).
4+
5+
## For Posterity
6+
7+
Created with `npm create vite@6.0.11` and using the `react-ts` template preset as described at https://vite.dev/guide/

‎apps/browser-example/index.html‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
<!doctype html>
2+
<html lang="en">
3+
<head>
4+
<meta charset="UTF-8" />
5+
<link rel="icon" type="image/svg+xml" href="/vite.svg" />
6+
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
7+
<title>Vite + React + TS</title>
8+
</head>
9+
<body>
10+
<div id="root"></div>
11+
<script type="module" src="/src/main.tsx"></script>
12+
</body>
13+
</html>

‎apps/browser-example/package.json‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
{
2+
"name": "@tsorta/browser-example",
3+
"private": true,
4+
"version": "0.0.0",
5+
"type": "module",
6+
"scripts": {
7+
"dev": "vite",
8+
"build": "tsc -b && vite build",
9+
"clean": "rm -rf node_modules",
10+
"preview": "vite preview"
11+
},
12+
"dependencies": {
13+
"@tsorta/browser": "^1.0.0",
14+
"bootstrap": "^5.3.3",
15+
"bootstrap-icons": "^1.11.3",
16+
"openai": "^4.81.0",
17+
"react": "^18.3.1",
18+
"react-dom": "^18.3.1"
19+
},
20+
"devDependencies": {
21+
"@types/react": "^18.3.18",
22+
"@types/react-dom": "^18.3.5",
23+
"@vitejs/plugin-react": "^4.3.4",
24+
"globals": "^15.14.0",
25+
"saas": "^1.0.0",
26+
"sass-embedded": "^1.83.4",
27+
"typescript": "~5.6.2",
28+
"vite": "^6.0.5"
29+
}
30+
}

‎apps/browser-example/src/App.tsx‎

Lines changed: 109 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,109 @@
1+
import { ReactNode, useEffect, useState } from "react"
2+
import { BootstrapIcon } from "./components/BootstrapIcon"
3+
import { useKeyManager } from "./hooks/key"
4+
import { OfficialSDKWebSocketExample } from "./pages/OfficialSDKWebSocketExample"
5+
import { WebRTCExample } from "./pages/WebRTCExample"
6+
import { PageProps } from "./pages/props"
7+
8+
export function App() {
9+
const [events, setEvents] = useState<any[]>([])
10+
const { key, KeyModal, EnterKeyButton } = useKeyManager()
11+
const [sessionStatus, setSessionStatus] = useState<
12+
"unavailable" | "stopped" | "recording"
13+
>(key ? "stopped" : "unavailable")
14+
15+
useEffect(() => {
16+
if (key && sessionStatus === "unavailable") {
17+
setSessionStatus("stopped")
18+
}
19+
}, [key])
20+
21+
const onServerEvent = (event: any) =>
22+
setEvents((events) => [...events, event])
23+
24+
const [routes] = useState({
25+
WebRTC: {
26+
label: "WebRTC Example",
27+
page: (props: PageProps) => <WebRTCExample {...props} />,
28+
},
29+
"official-ws": {
30+
label: "Official SDK WebSocket Example",
31+
page: (props: PageProps) => {
32+
return <OfficialSDKWebSocketExample {...props} />
33+
},
34+
},
35+
})
36+
const [activeRoute, setActiveRoute] = useState<keyof typeof routes>("WebRTC")
37+
38+
return (
39+
<>
40+
<header>
41+
<nav className="navbar bg-body-tertiary">
42+
<div className="container-fluid">
43+
<a className="navbar-brand">TypeScript OpenAI Realtime Example</a>
44+
<ul className="nav nav-pills gap-2">
45+
{Object.keys(routes)
46+
.map((routeKey) => routeKey as keyof typeof routes)
47+
.map((routeKey) => ({ routeKey, ...routes[routeKey] }))
48+
.map((route) => (
49+
<LoadPageButton
50+
key={route.routeKey}
51+
route={route.routeKey}
52+
activeRoute={activeRoute}
53+
label={route.label}
54+
onNavigate={(route) => {
55+
setEvents([])
56+
setActiveRoute(route)
57+
}}
58+
/>
59+
))}
60+
</ul>
61+
<div className="spacer flex-grow-1"></div>
62+
{EnterKeyButton}
63+
</div>
64+
</nav>
65+
{KeyModal}
66+
</header>
67+
<main>
68+
{routes[activeRoute].page({
69+
apiKey: key,
70+
sessionStatus,
71+
onSessionStatusChanged: (status) => {
72+
if (status === "recording") {
73+
setEvents([])
74+
}
75+
setSessionStatus(status)
76+
},
77+
events,
78+
onServerEvent,
79+
})}
80+
</main>
81+
</>
82+
)
83+
}
84+
85+
function LoadPageButton<TRoute>({
86+
label,
87+
route,
88+
activeRoute,
89+
onNavigate,
90+
}: {
91+
label: string
92+
route: TRoute
93+
activeRoute: TRoute
94+
onNavigate: (route: TRoute) => void
95+
}): ReactNode {
96+
return (
97+
<li className={`nav-item`}>
98+
<a
99+
className={`nav-link ${route === activeRoute ? "active" : ""}`}
100+
type="button"
101+
onClick={() => onNavigate(route)}
102+
aria-current={route === activeRoute ? "page" : "false"}
103+
>
104+
<BootstrapIcon name="arrow" />
105+
{label}
106+
</a>
107+
</li>
108+
)
109+
}

‎apps/browser-example/src/components/BootstrapIcon.tsx‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
import { ReactNode } from "react"
2+
import svgPath from "bootstrap-icons/bootstrap-icons.svg"
3+
4+
export function BootstrapIcon({
5+
name,
6+
size,
7+
}: {
8+
name: string
9+
size?: 16 | 24 | 32 | 48
10+
}): ReactNode {
11+
size = size || 16
12+
return (
13+
<svg className="bi" width={size} height={size} fill="currentColor">
14+
<use xlinkHref={`${svgPath}#${name}`} />
15+
</svg>
16+
)
17+
}

‎apps/browser-example/src/components/EventList.tsx‎

Lines changed: 131 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,131 @@
1+
import { useState, useRef, useEffect } from "react"
2+
3+
export function EventList({ events }: { events: any[] }) {
4+
const [showFilter, setShowFilter] = useState(false)
5+
const [eventTypes, setEventTypes] = useState<string[]>([])
6+
const [selectedEventTypes, setSelectedEventTypes] = useState<string[]>([])
7+
8+
const lastEventRef = useRef<HTMLDivElement>(null)
9+
useEffect(() => {
10+
if (lastEventRef.current) {
11+
lastEventRef.current.scrollIntoView({ behavior: "smooth", block: "end" })
12+
}
13+
}, [events, lastEventRef, lastEventRef.current])
14+
15+
useEffect(() => {
16+
const existingTypes = eventTypes
17+
18+
const distinctTypes = events
19+
.map((event) => event.type)
20+
.filter((type, i, arr) => arr.indexOf(type) === i)
21+
22+
const newTypes = distinctTypes.filter(
23+
(type) => !existingTypes.includes(type)
24+
)
25+
26+
setEventTypes(distinctTypes)
27+
28+
if (selectedEventTypes.length === 0) {
29+
setSelectedEventTypes(distinctTypes)
30+
}
31+
if (newTypes.length > 0) {
32+
setSelectedEventTypes((selectedEventTypes) => [
33+
...selectedEventTypes,
34+
...newTypes,
35+
])
36+
}
37+
}, [events])
38+
39+
return (
40+
<div className="card my-2">
41+
<div className="card-header">
42+
<div className="dropdown">
43+
<button
44+
type="button"
45+
className="btn btn-primary dropdown-toggle"
46+
aria-expanded={showFilter}
47+
data-bs-auto-close="outside"
48+
onClick={() => setShowFilter(!showFilter)}
49+
>
50+
Filter
51+
</button>
52+
<span className="mx-2">Event Count: {events.length}</span>
53+
54+
<form
55+
className={`dropdown-menu p-4 ${showFilter ? "show" : ""}`}
56+
style={{
57+
/* > --bs-backdrop-zindex */
58+
zIndex: 2000,
59+
}}
60+
>
61+
<div className="mb-3 d-flex gap-2 justify-content-center">
62+
<a
63+
className="form-control btn btn-link"
64+
onClick={() => setSelectedEventTypes(eventTypes)}
65+
>
66+
All
67+
</a>
68+
<a
69+
className="form-control btn btn-link"
70+
onClick={() => setSelectedEventTypes([])}
71+
>
72+
None
73+
</a>
74+
</div>
75+
76+
<div className="mb-3">
77+
{eventTypes.map((eventType, i) => (
78+
<div className="form-check" key={i}>
79+
<input
80+
type="checkbox"
81+
className="form-check-input"
82+
id={`filter-${eventType}`}
83+
checked={selectedEventTypes.includes(eventType)}
84+
onChange={() => {
85+
setSelectedEventTypes((selectedEventTypes) =>
86+
selectedEventTypes.includes(eventType)
87+
? selectedEventTypes.filter((t) => t !== eventType)
88+
: [...selectedEventTypes, eventType]
89+
)
90+
}}
91+
/>
92+
<label
93+
className="form-check-label"
94+
htmlFor={`filter-${eventType}`}
95+
>
96+
{eventType}
97+
</label>
98+
</div>
99+
))}
100+
</div>
101+
</form>
102+
<div
103+
className={`modal-backdrop fade ${showFilter ? "show" : "d-none"}`}
104+
onClick={() => setShowFilter(!showFilter)}
105+
style={{ backgroundColor: "rgba(0, 0, 0, 0.01)" }}
106+
></div>
107+
</div>
108+
</div>
109+
<div
110+
className="card-body"
111+
style={{
112+
maxHeight: "80vh",
113+
overflowY: "auto",
114+
}}
115+
>
116+
{events
117+
.filter((event) => selectedEventTypes.includes(event.type))
118+
.map((event, i) => (
119+
<div
120+
ref={lastEventRef}
121+
key={i}
122+
className="alert alert-info"
123+
role="alert"
124+
>
125+
<pre>{JSON.stringify(event, null, 2)}</pre>
126+
</div>
127+
))}
128+
</div>
129+
</div>
130+
)
131+
}

0 commit comments

Comments
 (0)