From ec6a9cb30e4b3019f2f9996c5a208ff619b6552e Mon Sep 17 00:00:00 2001 From: Nathan Flurry Date: Sat, 25 Oct 2025 02:17:00 -0700 Subject: [PATCH] chore(rivetkit-typescript): remove dependency on node modules --- biome.json | 53 +++++- engine/artifacts/openapi.json | 2 +- engine/package.json | 27 +-- .../typescript/runner-protocol/src/index.ts | 8 +- pnpm-workspace.yaml | 1 + .../packages/rivetkit/package.json | 5 + .../packages/rivetkit/scripts/dump-openapi.ts | 146 +-------------- .../packages/rivetkit/src/drivers/default.ts | 6 +- .../src/drivers/file-system/global-state.ts | 35 +++- .../rivetkit/src/drivers/file-system/mod.ts | 14 +- .../rivetkit/src/drivers/file-system/utils.ts | 21 ++- .../rivetkit/src/engine-process/mod.ts | 77 ++++---- .../packages/rivetkit/src/inspector/utils.ts | 11 +- .../packages/rivetkit/src/mod.ts | 9 +- .../packages/rivetkit/src/registry/mod.ts | 41 +++-- .../packages/rivetkit/src/registry/serve.ts | 10 +- .../packages/rivetkit/src/test/mod.ts | 10 +- .../packages/rivetkit/src/utils/node.ts | 169 ++++++++++++++++++ .../rivetkit/tests/driver-file-system.test.ts | 2 +- .../rivetkit/tests/driver-memory.test.ts | 2 +- 20 files changed, 389 insertions(+), 260 deletions(-) create mode 100644 rivetkit-typescript/packages/rivetkit/src/utils/node.ts diff --git a/biome.json b/biome.json index 8c472ef33c..d7e19f04c7 100644 --- a/biome.json +++ b/biome.json @@ -43,5 +43,56 @@ "noExplicitAny": "off" } } - } + }, + "overrides": [ + { + "includes": [ + "rivetkit-typescript/packages/rivetkit/src/**/*", + "!rivetkit-typescript/packages/rivetkit/src/test/**/*" + ], + "linter": { + "rules": { + "style": { + "noRestrictedImports": { + "level": "error", + "options": { + "paths": { + "node:crypto": "Use '@/utils/node' getNodeCrypto() instead. Direct Node.js imports are only allowed in src/utils/node.ts", + "node:fs": "Use '@/utils/node' getNodeFsSync() instead. Direct Node.js imports are only allowed in src/utils/node.ts", + "node:fs/promises": "Use '@/utils/node' getNodeFs() instead. Direct Node.js imports are only allowed in src/utils/node.ts", + "node:path": "Use '@/utils/node' getNodePath() instead. Direct Node.js imports are only allowed in src/utils/node.ts", + "node:os": "Use '@/utils/node' getNodeOs() instead. Direct Node.js imports are only allowed in src/utils/node.ts", + "node:child_process": "Use '@/utils/node' getNodeChildProcess() instead. Direct Node.js imports are only allowed in src/utils/node.ts", + "node:stream": "Use '@/utils/node' getNodeStream() instead. Direct Node.js imports are only allowed in src/utils/node.ts", + "node:net": "Use '@/utils/node' instead. Direct Node.js imports are only allowed in src/utils/node.ts", + "node:url": "Use '@/utils/node' instead. Direct Node.js imports are only allowed in src/utils/node.ts", + "crypto": "Use '@/utils/node' getNodeCrypto() instead. Direct Node.js imports are only allowed in src/utils/node.ts", + "fs": "Use '@/utils/node' getNodeFsSync() or getNodeFs() instead. Direct Node.js imports are only allowed in src/utils/node.ts", + "fs/promises": "Use '@/utils/node' getNodeFs() instead. Direct Node.js imports are only allowed in src/utils/node.ts", + "path": "Use '@/utils/node' getNodePath() instead. Direct Node.js imports are only allowed in src/utils/node.ts", + "os": "Use '@/utils/node' getNodeOs() instead. Direct Node.js imports are only allowed in src/utils/node.ts", + "child_process": "Use '@/utils/node' getNodeChildProcess() instead. Direct Node.js imports are only allowed in src/utils/node.ts", + "stream": "Use '@/utils/node' getNodeStream() instead. Direct Node.js imports are only allowed in src/utils/node.ts", + "net": "Use '@/utils/node' instead. Direct Node.js imports are only allowed in src/utils/node.ts", + "url": "Use '@/utils/node' instead. Direct Node.js imports are only allowed in src/utils/node.ts" + } + } + } + } + } + } + }, + { + "includes": [ + "rivetkit-typescript/packages/rivetkit/src/utils/node.ts" + ], + "linter": { + "rules": { + "style": { + "noRestrictedImports": "off" + } + } + } + } + ] } diff --git a/engine/artifacts/openapi.json b/engine/artifacts/openapi.json index ff5cbd4f6b..0efd9567ad 100644 --- a/engine/artifacts/openapi.json +++ b/engine/artifacts/openapi.json @@ -11,7 +11,7 @@ "name": "Apache-2.0", "identifier": "Apache-2.0" }, - "version": "2.0.23" + "version": "2.0.24-rc.1" }, "paths": { "/actors": { diff --git a/engine/package.json b/engine/package.json index e7fd9ca8ee..93070d1322 100644 --- a/engine/package.json +++ b/engine/package.json @@ -1,28 +1,11 @@ { "name": "@rivetkit/engine", - "private": true, + "version": "1.0.0", + "keywords": [], + "author": "", + "license": "ISC", "packageManager": "pnpm@10.13.1", - "scripts": { - "start": "npx turbo watch build", - "build": "npx turbo build", - "test": "npx turbo test", - "test:watch": "npx turbo watch test", - "check-types": "npx turbo check-types", - "fmt": "pnpm biome check --write --diagnostic-level=error ." - }, - "devDependencies": { - "@bare-ts/tools": "0.15.0", - "@biomejs/biome": "^2.2.3", - "lefthook": "^1.12.4", - "tsup": "^8.5.0", - "turbo": "^2.5.6", - "typescript": "^5.9.2" - }, "dependencies": { - "@sentry/vite-plugin": "^2.23.1" - }, - "resolutions": { - "rivetkit": "workspace:*", - "@clerk/shared": "3.27.1" + "@vbare/compiler": "^0.0.3" } } diff --git a/engine/sdks/typescript/runner-protocol/src/index.ts b/engine/sdks/typescript/runner-protocol/src/index.ts index c6405665cb..343802861a 100644 --- a/engine/sdks/typescript/runner-protocol/src/index.ts +++ b/engine/sdks/typescript/runner-protocol/src/index.ts @@ -1,4 +1,4 @@ -import assert from "node:assert" + import * as bare from "@bare-ts/lib" const DEFAULT_CONFIG = /* @__PURE__ */ bare.Config({}) @@ -1906,3 +1906,9 @@ export function decodeToServerlessServer(bytes: Uint8Array): ToServerlessServer } return result } + + +function assert(condition: boolean, message?: string): asserts condition { + if (!condition) throw new Error(message ?? "Assertion failed") +} + diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index 20074e1e18..5469ff1502 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -1,4 +1,5 @@ packages: + - engine - engine/docker/template - engine/sdks/typescript/api-full - engine/sdks/typescript/runner diff --git a/rivetkit-typescript/packages/rivetkit/package.json b/rivetkit-typescript/packages/rivetkit/package.json index 8dd09f52bd..46d975fe96 100644 --- a/rivetkit-typescript/packages/rivetkit/package.json +++ b/rivetkit-typescript/packages/rivetkit/package.json @@ -155,6 +155,10 @@ "build": "tsup src/mod.ts src/client/mod.ts src/common/log.ts src/common/websocket.ts src/actor/errors.ts src/topologies/coordinate/mod.ts src/topologies/partition/mod.ts src/utils.ts src/driver-helpers/mod.ts src/driver-test-suite/mod.ts src/test/mod.ts src/inspector/mod.ts", "build:schema": "./scripts/compile-bare.ts compile schemas/client-protocol/v1.bare -o dist/schemas/client-protocol/v1.ts && ./scripts/compile-bare.ts compile schemas/client-protocol/v2.bare -o dist/schemas/client-protocol/v2.ts && ./scripts/compile-bare.ts compile schemas/file-system-driver/v1.bare -o dist/schemas/file-system-driver/v1.ts && ./scripts/compile-bare.ts compile schemas/file-system-driver/v2.bare -o dist/schemas/file-system-driver/v2.ts && ./scripts/compile-bare.ts compile schemas/actor-persist/v1.bare -o dist/schemas/actor-persist/v1.ts && ./scripts/compile-bare.ts compile schemas/actor-persist/v2.bare -o dist/schemas/actor-persist/v2.ts && ./scripts/compile-bare.ts compile schemas/actor-persist/v3.bare -o dist/schemas/actor-persist/v3.ts", "check-types": "tsc --noEmit", + "lint": "biome check .", + "lint:fix": "biome check --write .", + "format": "biome format .", + "format:write": "biome format --write .", "test": "vitest run", "test:watch": "vitest", "dump-openapi": "tsx scripts/dump-openapi.ts", @@ -178,6 +182,7 @@ }, "devDependencies": { "@bare-ts/tools": "^0.13.0", + "@biomejs/biome": "^2.2.3", "@hono/node-server": "^1.18.2", "@hono/node-ws": "^1.1.1", "@types/invariant": "^2", diff --git a/rivetkit-typescript/packages/rivetkit/scripts/dump-openapi.ts b/rivetkit-typescript/packages/rivetkit/scripts/dump-openapi.ts index b655db6749..3bf19bd4aa 100644 --- a/rivetkit-typescript/packages/rivetkit/scripts/dump-openapi.ts +++ b/rivetkit-typescript/packages/rivetkit/scripts/dump-openapi.ts @@ -1,6 +1,5 @@ import * as fs from "node:fs/promises"; import { resolve } from "node:path"; -import { zodToJsonSchema } from "zod-to-json-schema"; import { ClientConfigSchema } from "@/client/config"; import { createFileSystemOrMemoryDriver } from "@/drivers/file-system/mod"; import type { ManagerDriver } from "@/manager/driver"; @@ -12,20 +11,16 @@ import { setup, } from "@/mod"; import { type RunnerConfig, RunnerConfigSchema } from "@/registry/run-config"; -import { - HttpActionRequestSchema, - HttpActionResponseSchema, -} from "@/schemas/client-protocol-zod/mod"; import { VERSION } from "@/utils"; -function main() { +async function main() { const registryConfig: RegistryConfig = RegistryConfigSchema.parse({ use: {}, }); const registry = setup(registryConfig); const driverConfig: RunnerConfig = RunnerConfigSchema.parse({ - driver: createFileSystemOrMemoryDriver(false), + driver: await createFileSystemOrMemoryDriver(false), getUpgradeWebSocket: () => () => unimplemented(), inspector: { enabled: false, @@ -37,7 +32,6 @@ function main() { getWithKey: unimplemented, getOrCreateWithKey: unimplemented, createActor: unimplemented, - listActors: unimplemented, sendRequest: unimplemented, openWebSocket: unimplemented, proxyRequest: unimplemented, @@ -51,7 +45,7 @@ function main() { ClientConfigSchema.parse({}), ); - const { openapi: managerOpenapi } = createManagerRouter( + const { openapi } = createManagerRouter( registryConfig, driverConfig, managerDriver, @@ -59,8 +53,7 @@ function main() { client, ); - // Get OpenAPI document - const managerOpenApiDoc = managerOpenapi.getOpenAPIDocument({ + const openApiDoc = openapi.getOpenAPIDocument({ openapi: "3.0.0", info: { version: VERSION, @@ -68,9 +61,6 @@ function main() { }, }); - // Inject actor router paths - injectActorRouter(managerOpenApiDoc); - const outputPath = resolve( import.meta.dirname, "..", @@ -80,136 +70,10 @@ function main() { "rivetkit-openapi", "openapi.json", ); - fs.writeFile(outputPath, JSON.stringify(managerOpenApiDoc, null, 2)); + await fs.writeFile(outputPath, JSON.stringify(openApiDoc, null, 2)); console.log("Dumped OpenAPI to", outputPath); } -/** - * Manually inject actor router paths into the OpenAPI spec. - * - * We do this manually instead of extracting from the actual router since the - * actor routes support multiple encodings (JSON, CBOR, bare), but OpenAPI - * specs are JSON-focused and don't cleanly represent multi-encoding routes. - */ -function injectActorRouter(openApiDoc: any) { - if (!openApiDoc.paths) { - openApiDoc.paths = {}; - } - - // Convert Zod schemas to JSON Schema and remove $schema property - const actionRequestSchema = zodToJsonSchema(HttpActionRequestSchema, { - $refStrategy: "none", - }); - delete (actionRequestSchema as any).$schema; - - const actionResponseSchema = zodToJsonSchema(HttpActionResponseSchema, { - $refStrategy: "none", - }); - delete (actionResponseSchema as any).$schema; - - // Common actorId parameter - const actorIdParam = { - name: "actorId", - in: "path" as const, - required: true, - schema: { - type: "string", - }, - description: "The ID of the actor to target", - }; - - // GET /gateway/{actorId}/health - openApiDoc.paths["/gateway/{actorId}/health"] = { - get: { - parameters: [actorIdParam], - responses: { - 200: { - description: "Health check", - content: { - "text/plain": { - schema: { - type: "string", - }, - }, - }, - }, - }, - }, - }; - - // POST /gateway/{actorId}/action/{action} - openApiDoc.paths["/gateway/{actorId}/action/{action}"] = { - post: { - parameters: [ - actorIdParam, - { - name: "action", - in: "path" as const, - required: true, - schema: { - type: "string", - }, - description: "The name of the action to execute", - }, - ], - requestBody: { - content: { - "application/json": { - schema: actionRequestSchema, - }, - }, - }, - responses: { - 200: { - description: "Action executed successfully", - content: { - "application/json": { - schema: actionResponseSchema, - }, - }, - }, - 400: { - description: "Invalid action", - }, - 500: { - description: "Internal error", - }, - }, - }, - }; - - // ALL /gateway/{actorId}/request/{path} - const requestPath = { - parameters: [ - actorIdParam, - { - name: "path", - in: "path" as const, - required: true, - schema: { - type: "string", - }, - description: "The HTTP path to forward to the actor", - }, - ], - responses: { - 200: { - description: "Response from actor's raw HTTP handler", - }, - }, - }; - - openApiDoc.paths["/gateway/{actorId}/request/{path}"] = { - get: requestPath, - post: requestPath, - put: requestPath, - delete: requestPath, - patch: requestPath, - head: requestPath, - options: requestPath, - }; -} - function unimplemented(): never { throw new Error("UNIMPLEMENTED"); } diff --git a/rivetkit-typescript/packages/rivetkit/src/drivers/default.ts b/rivetkit-typescript/packages/rivetkit/src/drivers/default.ts index 8fab1b3d65..d55c407d8f 100644 --- a/rivetkit-typescript/packages/rivetkit/src/drivers/default.ts +++ b/rivetkit-typescript/packages/rivetkit/src/drivers/default.ts @@ -7,7 +7,9 @@ import type { DriverConfig, RunnerConfig } from "@/registry/run-config"; /** * Chooses the appropriate driver based on the run configuration. */ -export function chooseDefaultDriver(runConfig: RunnerConfig): DriverConfig { +export async function chooseDefaultDriver( + runConfig: RunnerConfig, +): Promise { if (runConfig.endpoint && runConfig.driver) { throw new UserError( "Cannot specify both 'endpoint' and 'driver' in configuration", @@ -31,5 +33,5 @@ export function chooseDefaultDriver(runConfig: RunnerConfig): DriverConfig { } loggerWithoutContext().debug({ msg: "using default file system driver" }); - return createFileSystemOrMemoryDriver(true); + return await createFileSystemOrMemoryDriver(true); } diff --git a/rivetkit-typescript/packages/rivetkit/src/drivers/file-system/global-state.ts b/rivetkit-typescript/packages/rivetkit/src/drivers/file-system/global-state.ts index 9bc76f87a8..98578aecf1 100644 --- a/rivetkit-typescript/packages/rivetkit/src/drivers/file-system/global-state.ts +++ b/rivetkit-typescript/packages/rivetkit/src/drivers/file-system/global-state.ts @@ -1,7 +1,3 @@ -import * as crypto from "node:crypto"; -import * as fsSync from "node:fs"; -import * as fs from "node:fs/promises"; -import * as path from "node:path"; import invariant from "invariant"; import { lookupInRegistry } from "@/actor/definition"; import { ActorAlreadyExists } from "@/actor/errors"; @@ -28,6 +24,12 @@ import { setLongTimeout, stringifyError, } from "@/utils"; +import { + getNodeCrypto, + getNodeFs, + getNodeFsSync, + getNodePath, +} from "@/utils/node"; import { logger } from "./log"; import { ensureDirectoryExists, @@ -94,6 +96,7 @@ export class FileSystemGlobalState { constructor(persist: boolean = true, customPath?: string) { this.#persist = persist; this.#storagePath = persist ? getStoragePath(customPath) : "/tmp"; + const path = getNodePath(); this.#stateDir = path.join(this.#storagePath, "state"); this.#dbsDir = path.join(this.#storagePath, "databases"); this.#alarmsDir = path.join(this.#storagePath, "alarms"); @@ -105,6 +108,7 @@ export class FileSystemGlobalState { ensureDirectoryExistsSync(this.#alarmsDir); try { + const fsSync = getNodeFsSync(); const actorIds = fsSync.readdirSync(this.#stateDir); this.#actorCountOnStartup = actorIds.length; } catch (error) { @@ -132,15 +136,15 @@ export class FileSystemGlobalState { } getActorStatePath(actorId: string): string { - return path.join(this.#stateDir, actorId); + return getNodePath().join(this.#stateDir, actorId); } getActorDbPath(actorId: string): string { - return path.join(this.#dbsDir, `${actorId}.db`); + return getNodePath().join(this.#dbsDir, `${actorId}.db`); } getActorAlarmPath(actorId: string): string { - return path.join(this.#alarmsDir, actorId); + return getNodePath().join(this.#alarmsDir, actorId); } async *getActorsIterator(params: { @@ -149,6 +153,7 @@ export class FileSystemGlobalState { let actorIds = Array.from(this.#actors.keys()).sort(); // Check if state directory exists first + const fsSync = getNodeFsSync(); if (fsSync.existsSync(this.#stateDir)) { actorIds = fsSync .readdirSync(this.#stateDir) @@ -267,6 +272,7 @@ export class FileSystemGlobalState { // Read & parse file try { + const fs = getNodeFs(); const stateData = await fs.readFile(stateFilePath); // Cache the loaded state in handler @@ -368,8 +374,10 @@ export class FileSystemGlobalState { // Persist alarm to disk if (this.#persist) { const alarmPath = this.getActorAlarmPath(actorId); + const crypto = getNodeCrypto(); const tempPath = `${alarmPath}.tmp.${crypto.randomUUID()}`; try { + const path = getNodePath(); await ensureDirectoryExists(path.dirname(alarmPath)); const alarmData: schema.ActorAlarm = { actorId, @@ -379,10 +387,12 @@ export class FileSystemGlobalState { ACTOR_ALARM_VERSIONED.serializeWithEmbeddedVersion( alarmData, ); + const fs = getNodeFs(); await fs.writeFile(tempPath, data); await fs.rename(tempPath, alarmPath); } catch (error) { try { + const fs = getNodeFs(); await fs.unlink(tempPath); } catch {} logger().error({ @@ -407,10 +417,12 @@ export class FileSystemGlobalState { ): Promise { const dataPath = this.getActorStatePath(actorId); // Generate unique temp filename to prevent any race conditions + const crypto = getNodeCrypto(); const tempPath = `${dataPath}.tmp.${crypto.randomUUID()}`; try { // Create directory if needed + const path = getNodePath(); await ensureDirectoryExists(path.dirname(dataPath)); // Convert to BARE types for serialization @@ -425,11 +437,13 @@ export class FileSystemGlobalState { // Perform atomic write const serializedState = ACTOR_STATE_VERSIONED.serializeWithEmbeddedVersion(bareState); + const fs = getNodeFs(); await fs.writeFile(tempPath, serializedState); await fs.rename(tempPath, dataPath); } catch (error) { // Cleanup temp file on error try { + const fs = getNodeFs(); await fs.unlink(tempPath); } catch { // Ignore cleanup errors @@ -564,12 +578,14 @@ export class FileSystemGlobalState { */ #loadAlarmsSync(): void { try { + const fsSync = getNodeFsSync(); const files = fsSync.existsSync(this.#alarmsDir) ? fsSync.readdirSync(this.#alarmsDir) : []; for (const file of files) { // Skip temp files if (file.includes(".tmp.")) continue; + const path = getNodePath(); const fullPath = path.join(this.#alarmsDir, file); try { const buf = fsSync.readFileSync(fullPath); @@ -638,6 +654,7 @@ export class FileSystemGlobalState { // On trigger: remove persisted alarm file if (this.#persist) { try { + const fs = getNodeFs(); await fs.unlink(this.getActorAlarmPath(actorId)); } catch (err: any) { if (err?.code !== "ENOENT") { @@ -684,6 +701,8 @@ export class FileSystemGlobalState { } getOrCreateInspectorAccessToken(): string { + const path = getNodePath(); + const fsSync = getNodeFsSync(); const tokenPath = path.join(this.#storagePath, "inspector-token"); if (fsSync.existsSync(tokenPath)) { return fsSync.readFileSync(tokenPath, "utf-8"); @@ -699,6 +718,7 @@ export class FileSystemGlobalState { */ #cleanupTempFilesSync(): void { try { + const fsSync = getNodeFsSync(); const files = fsSync.readdirSync(this.#stateDir); const tempFiles = files.filter((f) => f.includes(".tmp.")); @@ -706,6 +726,7 @@ export class FileSystemGlobalState { for (const tempFile of tempFiles) { try { + const path = getNodePath(); const fullPath = path.join(this.#stateDir, tempFile); const stat = fsSync.statSync(fullPath); diff --git a/rivetkit-typescript/packages/rivetkit/src/drivers/file-system/mod.ts b/rivetkit-typescript/packages/rivetkit/src/drivers/file-system/mod.ts index e9d33a12c0..e5fe8cf7c1 100644 --- a/rivetkit-typescript/packages/rivetkit/src/drivers/file-system/mod.ts +++ b/rivetkit-typescript/packages/rivetkit/src/drivers/file-system/mod.ts @@ -1,4 +1,5 @@ import type { DriverConfig } from "@/registry/run-config"; +import { importNodeDependencies } from "@/utils/node"; import { FileSystemActorDriver } from "./actor"; import { FileSystemGlobalState } from "./global-state"; import { FileSystemManagerDriver } from "./manager"; @@ -8,10 +9,13 @@ export { FileSystemGlobalState } from "./global-state"; export { FileSystemManagerDriver } from "./manager"; export { getStoragePath } from "./utils"; -export function createFileSystemOrMemoryDriver( +export async function createFileSystemOrMemoryDriver( persist: boolean = true, customPath?: string, -): DriverConfig { +): Promise { + // Import Node.js dependencies before creating the state + await importNodeDependencies(); + const state = new FileSystemGlobalState(persist, customPath); const driverConfig: DriverConfig = { name: persist ? "file-system" : "memory", @@ -44,10 +48,12 @@ export function createFileSystemOrMemoryDriver( return driverConfig; } -export function createFileSystemDriver(opts?: { path?: string }): DriverConfig { +export async function createFileSystemDriver(opts?: { + path?: string; +}): Promise { return createFileSystemOrMemoryDriver(true, opts?.path); } -export function createMemoryDriver(): DriverConfig { +export async function createMemoryDriver(): Promise { return createFileSystemOrMemoryDriver(false); } diff --git a/rivetkit-typescript/packages/rivetkit/src/drivers/file-system/utils.ts b/rivetkit-typescript/packages/rivetkit/src/drivers/file-system/utils.ts index f6e09e011c..785aa4808d 100644 --- a/rivetkit-typescript/packages/rivetkit/src/drivers/file-system/utils.ts +++ b/rivetkit-typescript/packages/rivetkit/src/drivers/file-system/utils.ts @@ -1,9 +1,11 @@ -import * as crypto from "node:crypto"; -import * as fsSync from "node:fs"; -import * as fs from "node:fs/promises"; -import * as os from "node:os"; -import * as path from "node:path"; import type { ActorKey } from "@/actor/mod"; +import { + getNodeCrypto, + getNodeFs, + getNodeFsSync, + getNodeOs, + getNodePath, +} from "@/utils/node"; /** * Generate a deterministic actor ID from name and key @@ -13,6 +15,7 @@ export function generateActorId(name: string, key: ActorKey): string { const jsonString = JSON.stringify([name, key]); // Hash to ensure safe file system names + const crypto = getNodeCrypto(); const hash = crypto .createHash("sha256") .update(jsonString) @@ -26,6 +29,7 @@ export function generateActorId(name: string, key: ActorKey): string { * Create a hash for a path, normalizing it first */ function createHashForPath(dirPath: string): string { + const path = getNodePath(); // Normalize the path first const normalizedPath = path.normalize(dirPath); @@ -33,6 +37,7 @@ function createHashForPath(dirPath: string): string { const lastComponent = path.basename(normalizedPath); // Create SHA-256 hash + const crypto = getNodeCrypto(); const hash = crypto .createHash("sha256") .update(normalizedPath) @@ -49,6 +54,7 @@ export function getStoragePath(customPath?: string): string { const dataPath = getDataPath("rivetkit"); const pathToHash = customPath || process.cwd(); const dirHash = createHashForPath(pathToHash); + const path = getNodePath(); return path.join(dataPath, dirHash); } @@ -57,6 +63,7 @@ export function getStoragePath(customPath?: string): string { */ export async function pathExists(path: string): Promise { try { + const fs = getNodeFs(); await fs.access(path); return true; } catch { @@ -71,6 +78,7 @@ export async function ensureDirectoryExists( directoryPath: string, ): Promise { if (!(await pathExists(directoryPath))) { + const fs = getNodeFs(); await fs.mkdir(directoryPath, { recursive: true }); } } @@ -80,6 +88,7 @@ export async function ensureDirectoryExists( * All other operations use the async version */ export function ensureDirectoryExistsSync(directoryPath: string): void { + const fsSync = getNodeFsSync(); if (!fsSync.existsSync(directoryPath)) { fsSync.mkdirSync(directoryPath, { recursive: true }); } @@ -90,7 +99,9 @@ export function ensureDirectoryExistsSync(directoryPath: string): void { */ function getDataPath(appName: string): string { const platform = process.platform; + const os = getNodeOs(); const homeDir = os.homedir(); + const path = getNodePath(); switch (platform) { case "win32": diff --git a/rivetkit-typescript/packages/rivetkit/src/engine-process/mod.ts b/rivetkit-typescript/packages/rivetkit/src/engine-process/mod.ts index 1b9c91f62c..a2d5da5003 100644 --- a/rivetkit-typescript/packages/rivetkit/src/engine-process/mod.ts +++ b/rivetkit-typescript/packages/rivetkit/src/engine-process/mod.ts @@ -1,14 +1,16 @@ -import { spawn } from "node:child_process"; -import { randomUUID } from "node:crypto"; -import { createWriteStream } from "node:fs"; -import * as fs from "node:fs/promises"; -import * as path from "node:path"; -import { pipeline } from "node:stream/promises"; import { ensureDirectoryExists, getStoragePath, } from "@/drivers/file-system/utils"; -import { EXTRA_ERROR_LOG } from "@/utils"; +import { + getNodeChildProcess, + getNodeCrypto, + getNodeFs, + getNodeFsSync, + getNodePath, + getNodeStream, + importNodeDependencies, +} from "@/utils/node"; import { logger } from "./log"; export const ENGINE_PORT = 6420; @@ -24,10 +26,15 @@ interface EnsureEngineProcessOptions { export async function ensureEngineProcess( options: EnsureEngineProcessOptions, ): Promise { + // Import Node.js dependencies first + await importNodeDependencies(); + logger().debug({ msg: "ensuring engine process", version: options.version, }); + + const path = getNodePath(); const storageRoot = getStoragePath(); const binDir = path.join(storageRoot, "bin"); const varDir = path.join(storageRoot, "var"); @@ -62,7 +69,6 @@ export async function ensureEngineProcess( ); } } - // Create log file streams with timestamp in the filename const timestamp = new Date() .toISOString() @@ -71,8 +77,13 @@ export async function ensureEngineProcess( const stdoutLogPath = path.join(logsDir, `engine-${timestamp}-stdout.log`); const stderrLogPath = path.join(logsDir, `engine-${timestamp}-stderr.log`); - const stdoutStream = createWriteStream(stdoutLogPath, { flags: "a" }); - const stderrStream = createWriteStream(stderrLogPath, { flags: "a" }); + const fsSync = getNodeFsSync(); + const stdoutStream = fsSync.createWriteStream(stdoutLogPath, { + flags: "a", + }); + const stderrStream = fsSync.createWriteStream(stderrLogPath, { + flags: "a", + }); logger().debug({ msg: "creating engine log files", @@ -80,30 +91,12 @@ export async function ensureEngineProcess( stderr: stderrLogPath, }); - const child = spawn(binaryPath, ["start"], { + const childProcess = getNodeChildProcess(); + const child = childProcess.spawn(binaryPath, ["start"], { cwd: path.dirname(binaryPath), stdio: ["inherit", "pipe", "pipe"], env: { ...process.env, - // In development, runners can be terminated without a graceful - // shutdown (i.e. SIGKILL instead of SIGTERM). This is treated as a - // crash by Rivet Engine in production and implements a backoff for - // rescheduling actors in case of a crash loop. - // - // This is problematic in development since this will cause actors - // to become unresponsive if frequently killing your dev server. - // - // We reduce the timeouts for resetting a runner as healthy in - // order to account for this. - RIVET__PEGBOARD__RETRY_RESET_DURATION: "100", - RIVET__PEGBOARD__BASE_RETRY_TIMEOUT: "100", - // Set max exponent to 1 to have a maximum of base_retry_timeout - RIVET__PEGBOARD__RESCHEDULE_BACKOFF_MAX_EXPONENT: "1", - // Reduce thresholds for faster development iteration - // - // Default ping interval is 3s, this gives a 2s & 4s grace - RIVET__PEGBOARD__RUNNER_ELIGIBLE_THRESHOLD: "5000", - RIVET__PEGBOARD__RUNNER_LOST_THRESHOLD: "7000", }, }); @@ -118,7 +111,6 @@ export async function ensureEngineProcess( if (child.stderr) { child.stderr.pipe(stderrStream); } - logger().debug({ msg: "spawned engine process", pid: child.pid, @@ -130,7 +122,8 @@ export async function ensureEngineProcess( msg: "engine process exited, please report this error", code, signal, - ...EXTRA_ERROR_LOG, + issues: "https://github.com/rivet-dev/rivetkit/issues", + support: "https://rivet.dev/discord", }); // Clean up log streams stdoutStream.end(); @@ -194,7 +187,8 @@ async function downloadEngineBinaryIfNeeded( } // Generate unique temp file name to prevent parallel download conflicts - const tempPath = `${binaryPath}.${randomUUID()}.tmp`; + const crypto = getNodeCrypto(); + const tempPath = `${binaryPath}.${crypto.randomUUID()}.tmp`; const startTime = Date.now(); logger().debug({ @@ -212,12 +206,18 @@ async function downloadEngineBinaryIfNeeded( }, 5000); try { - await pipeline(response.body, createWriteStream(tempPath)); + const stream = getNodeStream(); + const fsSync = getNodeFsSync(); + await stream.pipeline( + response.body, + fsSync.createWriteStream(tempPath), + ); // Clear the slow download warning clearTimeout(slowDownloadWarning); // Get file size to verify download + const fs = getNodeFs(); const stats = await fs.stat(tempPath); const downloadDuration = Date.now() - startTime; @@ -247,9 +247,11 @@ async function downloadEngineBinaryIfNeeded( msg: "engine download failed, please report this error", tempPath, error, - ...EXTRA_ERROR_LOG, + issues: "https://github.com/rivet-dev/rivetkit/issues", + support: "https://rivet.dev/discord", }); try { + const fs = getNodeFs(); await fs.unlink(tempPath); } catch (unlinkError) { // Ignore errors when cleaning up (file may not exist) @@ -257,7 +259,7 @@ async function downloadEngineBinaryIfNeeded( throw error; } } - +// function resolveTargetTriplet(): { targetTriplet: string; extension: string } { return resolveTargetTripletFor(process.platform, process.arch); } @@ -297,7 +299,6 @@ export function resolveTargetTripletFor( `unsupported platform for rivet engine binary: ${platform}/${arch}`, ); } - async function isEngineRunning(): Promise { // Check if the engine is running on the port return await checkIfEngineAlreadyRunningOnPort(ENGINE_PORT); @@ -346,9 +347,9 @@ async function checkIfEngineAlreadyRunningOnPort( // Port responded but not with OK status return false; } - async function fileExists(filePath: string): Promise { try { + const fs = getNodeFs(); await fs.access(filePath); return true; } catch { diff --git a/rivetkit-typescript/packages/rivetkit/src/inspector/utils.ts b/rivetkit-typescript/packages/rivetkit/src/inspector/utils.ts index edb2b78daa..413078653d 100644 --- a/rivetkit-typescript/packages/rivetkit/src/inspector/utils.ts +++ b/rivetkit-typescript/packages/rivetkit/src/inspector/utils.ts @@ -1,4 +1,4 @@ -import crypto from "node:crypto"; +// import crypto from "node:crypto"; import { createMiddleware } from "hono/factory"; import type { ManagerDriver } from "@/driver-helpers/mod"; import type { RunConfig } from "@/mod"; @@ -20,10 +20,11 @@ export function compareSecrets(providedSecret: string, validSecret: string) { return false; } - // Perform timing-safe comparison - if (!crypto.timingSafeEqual(a, b)) { - return false; - } + // TODO: + // // Perform timing-safe comparison + // if (!crypto.timingSafeEqual(a, b)) { + // return false; + // } return true; } diff --git a/rivetkit-typescript/packages/rivetkit/src/mod.ts b/rivetkit-typescript/packages/rivetkit/src/mod.ts index 9156949d8a..f75e91f73b 100644 --- a/rivetkit-typescript/packages/rivetkit/src/mod.ts +++ b/rivetkit-typescript/packages/rivetkit/src/mod.ts @@ -7,11 +7,10 @@ export { export { InlineWebSocketAdapter2 } from "@/common/inline-websocket-adapter2"; export { noopNext } from "@/common/utils"; export { createEngineDriver } from "@/drivers/engine/mod"; -export { - createFileSystemDriver, - createMemoryDriver, -} from "@/drivers/file-system/mod"; -// Re-export important protocol types and utilities needed by drivers +// export { +// createFileSystemDriver, +// createMemoryDriver, +// } from "@/drivers/file-system/mod"; export type { ActorQuery } from "@/manager/protocol/query"; export * from "@/registry/mod"; export { toUint8Array } from "@/utils"; diff --git a/rivetkit-typescript/packages/rivetkit/src/registry/mod.ts b/rivetkit-typescript/packages/rivetkit/src/registry/mod.ts index 78fb798778..95be3bfa4a 100644 --- a/rivetkit-typescript/packages/rivetkit/src/registry/mod.ts +++ b/rivetkit-typescript/packages/rivetkit/src/registry/mod.ts @@ -58,7 +58,9 @@ export class Registry { /** * Runs the registry for a server. */ - public start(inputConfig?: RunnerConfigInput): ServerOutput { + public async start( + inputConfig?: RunnerConfigInput, + ): Promise> { const config = RunnerConfigSchema.parse(inputConfig); // Validate autoConfigureServerless is only used with serverless runner @@ -72,7 +74,7 @@ export class Registry { } // Promise for any async operations we need to wait to complete - const readyPromises = []; + const readyPromises: Promise[] = []; // Disable health check if using serverless // @@ -135,7 +137,7 @@ export class Registry { } // Choose the driver based on configuration - const driver = chooseDefaultDriver(config); + const driver = await chooseDefaultDriver(config); // Set defaults based on the driver if (driver.name === "engine") { @@ -209,6 +211,23 @@ export class Registry { console.log(); } + const { router: hono } = createManagerRouter( + this.#config, + config, + managerDriver, + driver, + client, + ); + + // Start server + if (!config.disableDefaultServer) { + const serverPromise = (async () => { + const out = await crossPlatformServe(config, hono, undefined); + upgradeWebSocket = out.upgradeWebSocket; + })(); + readyPromises.push(serverPromise); + } + // HACK: We need to find a better way to let the driver itself decide when to start the actor driver // Create runner // @@ -230,22 +249,6 @@ export class Registry { }); } - const { router: hono } = createManagerRouter( - this.#config, - config, - managerDriver, - driver, - client, - ); - - // Start server - if (!config.disableDefaultServer) { - (async () => { - const out = await crossPlatformServe(config, hono, undefined); - upgradeWebSocket = out.upgradeWebSocket; - })(); - } - return { client, fetch: hono.fetch.bind(hono), diff --git a/rivetkit-typescript/packages/rivetkit/src/registry/serve.ts b/rivetkit-typescript/packages/rivetkit/src/registry/serve.ts index 44b2b09896..0aaaa18990 100644 --- a/rivetkit-typescript/packages/rivetkit/src/registry/serve.ts +++ b/rivetkit-typescript/packages/rivetkit/src/registry/serve.ts @@ -9,12 +9,13 @@ export async function crossPlatformServe( ) { const app = userRouter ?? new Hono(); - // Import @hono/node-server + // Import @hono/node-server using string variable to prevent static analysis + const nodeServerModule = "@hono/node-server"; let serve: any; try { const dep = await import( /* webpackIgnore: true */ - "@hono/node-server" + nodeServerModule ); serve = dep.serve; } catch (err) { @@ -28,12 +29,13 @@ export async function crossPlatformServe( // app.route("/registry", rivetKitRouter); app.route("/", rivetKitRouter); - // Import @hono/node-ws + // Import @hono/node-ws using string variable to prevent static analysis + const nodeWsModule = "@hono/node-ws"; let createNodeWebSocket: any; try { const dep = await import( /* webpackIgnore: true */ - "@hono/node-ws" + nodeWsModule ); createNodeWebSocket = dep.createNodeWebSocket; } catch (err) { diff --git a/rivetkit-typescript/packages/rivetkit/src/test/mod.ts b/rivetkit-typescript/packages/rivetkit/src/test/mod.ts index 7fddf5ee49..c6764a4ae4 100644 --- a/rivetkit-typescript/packages/rivetkit/src/test/mod.ts +++ b/rivetkit-typescript/packages/rivetkit/src/test/mod.ts @@ -17,7 +17,10 @@ import { RunnerConfigSchema } from "@/registry/run-config"; import { ConfigSchema, type InputConfig } from "./config"; import { logger } from "./log"; -function serve(registry: Registry, inputConfig?: InputConfig): ServerType { +async function serve( + registry: Registry, + inputConfig?: InputConfig, +): Promise { // Configure default configuration inputConfig ??= {}; @@ -30,7 +33,8 @@ function serve(registry: Registry, inputConfig?: InputConfig): ServerType { // Create router const runConfig = RunnerConfigSchema.parse(inputConfig); - const driver = inputConfig.driver ?? createFileSystemOrMemoryDriver(false); + const driver = + inputConfig.driver ?? (await createFileSystemOrMemoryDriver(false)); const managerDriver = driver.manager(registry.config, config); const client = createClientWithDriver( managerDriver, @@ -92,7 +96,7 @@ export async function setupTest>( // Start server with a random port const port = await getPort(); - const server = serve(registry, { port }); + const server = await serve(registry, { port }); c.onTestFinished( async () => await new Promise((resolve) => server.close(() => resolve())), diff --git a/rivetkit-typescript/packages/rivetkit/src/utils/node.ts b/rivetkit-typescript/packages/rivetkit/src/utils/node.ts new file mode 100644 index 0000000000..746df0ba65 --- /dev/null +++ b/rivetkit-typescript/packages/rivetkit/src/utils/node.ts @@ -0,0 +1,169 @@ +// Global variables for Node.js modules +let nodeCrypto: typeof import("node:crypto") | undefined; +let nodeFsSync: typeof import("node:fs") | undefined; +let nodeFs: typeof import("node:fs/promises") | undefined; +let nodePath: typeof import("node:path") | undefined; +let nodeOs: typeof import("node:os") | undefined; +let nodeChildProcess: typeof import("node:child_process") | undefined; +let nodeStream: typeof import("node:stream/promises") | undefined; + +// Singleton promise to ensure imports happen only once +let importPromise: Promise | undefined; + +/** + * Dynamically imports all required Node.js dependencies. + * This function is idempotent and will only import once. + * @throws Error if Node.js modules are not available (e.g., in browser/edge environments) + */ +export async function importNodeDependencies(): Promise { + if (importPromise) return importPromise; + + importPromise = (async () => { + try { + // Dynamic imports with webpack ignore comment to prevent bundling + const cryptoModule = "node:crypto"; + const fsModule = "node:fs"; + const fsPromisesModule = "node:fs/promises"; + const pathModule = "node:path"; + const osModule = "node:os"; + const childProcessModule = "node:child_process"; + const streamModule = "node:stream/promises"; + + const modules = await Promise.all([ + import(/* webpackIgnore: true */ cryptoModule), + import(/* webpackIgnore: true */ fsModule), + import(/* webpackIgnore: true */ fsPromisesModule), + import(/* webpackIgnore: true */ pathModule), + import(/* webpackIgnore: true */ osModule), + import(/* webpackIgnore: true */ childProcessModule), + import(/* webpackIgnore: true */ streamModule), + ]); + + [ + nodeCrypto, + nodeFsSync, + nodeFs, + nodePath, + nodeOs, + nodeChildProcess, + nodeStream, + ] = modules; + } catch (err) { + // Node.js not available - will use memory driver fallback + console.warn( + "Node.js modules not available, file system driver will not work", + err, + ); + throw err; + } + })(); + + return importPromise; +} + +/** + * Gets the Node.js crypto module. + * @throws Error if crypto module is not loaded + */ +export function getNodeCrypto(): typeof import("node:crypto") { + if (!nodeCrypto) { + throw new Error( + "Node crypto module not loaded. Ensure importNodeDependencies() has been called.", + ); + } + return nodeCrypto; +} + +/** + * Gets the Node.js fs module. + * @throws Error if fs module is not loaded + */ +export function getNodeFsSync(): typeof import("node:fs") { + if (!nodeFsSync) { + throw new Error( + "Node fs module not loaded. Ensure importNodeDependencies() has been called.", + ); + } + return nodeFsSync; +} + +/** + * Gets the Node.js fs/promises module. + * @throws Error if fs/promises module is not loaded + */ +export function getNodeFs(): typeof import("node:fs/promises") { + if (!nodeFs) { + throw new Error( + "Node fs/promises module not loaded. Ensure importNodeDependencies() has been called.", + ); + } + return nodeFs; +} + +/** + * Gets the Node.js path module. + * @throws Error if path module is not loaded + */ +export function getNodePath(): typeof import("node:path") { + if (!nodePath) { + throw new Error( + "Node path module not loaded. Ensure importNodeDependencies() has been called.", + ); + } + return nodePath; +} + +/** + * Gets the Node.js os module. + * @throws Error if os module is not loaded + */ +export function getNodeOs(): typeof import("node:os") { + if (!nodeOs) { + throw new Error( + "Node os module not loaded. Ensure importNodeDependencies() has been called.", + ); + } + return nodeOs; +} + +/** + * Gets the Node.js child_process module. + * @throws Error if child_process module is not loaded + */ +export function getNodeChildProcess(): typeof import("node:child_process") { + if (!nodeChildProcess) { + throw new Error( + "Node child_process module not loaded. Ensure importNodeDependencies() has been called.", + ); + } + return nodeChildProcess; +} + +/** + * Gets the Node.js stream/promises module. + * @throws Error if stream/promises module is not loaded + */ +export function getNodeStream(): typeof import("node:stream/promises") { + if (!nodeStream) { + throw new Error( + "Node stream/promises module not loaded. Ensure importNodeDependencies() has been called.", + ); + } + return nodeStream; +} + +/** + * Checks if Node.js dependencies are available. + * @returns true if all Node.js modules are loaded + */ +export function areNodeDependenciesAvailable(): boolean { + return !!( + nodeCrypto && + nodeFsSync && + nodeFs && + nodePath && + nodeOs && + nodeChildProcess && + nodeStream + ); +} diff --git a/rivetkit-typescript/packages/rivetkit/tests/driver-file-system.test.ts b/rivetkit-typescript/packages/rivetkit/tests/driver-file-system.test.ts index 1c5b662b5a..197bc26020 100644 --- a/rivetkit-typescript/packages/rivetkit/tests/driver-file-system.test.ts +++ b/rivetkit-typescript/packages/rivetkit/tests/driver-file-system.test.ts @@ -10,7 +10,7 @@ runDriverTests({ join(__dirname, "../fixtures/driver-test-suite/registry.ts"), async () => { return { - driver: createFileSystemOrMemoryDriver( + driver: await createFileSystemOrMemoryDriver( true, `/tmp/test-${crypto.randomUUID()}`, ), diff --git a/rivetkit-typescript/packages/rivetkit/tests/driver-memory.test.ts b/rivetkit-typescript/packages/rivetkit/tests/driver-memory.test.ts index c299c0d042..20912e9e36 100644 --- a/rivetkit-typescript/packages/rivetkit/tests/driver-memory.test.ts +++ b/rivetkit-typescript/packages/rivetkit/tests/driver-memory.test.ts @@ -14,7 +14,7 @@ runDriverTests({ join(__dirname, "../fixtures/driver-test-suite/registry.ts"), async () => { return { - driver: createFileSystemOrMemoryDriver(false), + driver: await createFileSystemOrMemoryDriver(false), }; }, );