From aa155a6cfcebaaf06dddc281664d2073ed1b2649 Mon Sep 17 00:00:00 2001 From: Ottomated Date: Mon, 20 Oct 2025 21:04:39 -0700 Subject: [PATCH 01/27] start --- .../kit/src/runtime/app/server/remote/form.js | 21 +++--------- .../client/remote-functions/form.svelte.js | 34 ++++++++++++------- packages/kit/src/runtime/form-utils.js | 29 +++++++++++++++- packages/kit/src/runtime/server/remote.js | 28 +++++++-------- packages/kit/src/types/internal.d.ts | 10 +++++- packages/kit/src/utils/http.js | 5 ++- .../remote/form/file-upload/+page.svelte | 8 +++++ .../remote/form/file-upload/form.remote.ts | 9 +++++ 8 files changed, 96 insertions(+), 48 deletions(-) create mode 100644 packages/kit/test/apps/basics/src/routes/remote/form/file-upload/+page.svelte create mode 100644 packages/kit/test/apps/basics/src/routes/remote/form/file-upload/form.remote.ts diff --git a/packages/kit/src/runtime/app/server/remote/form.js b/packages/kit/src/runtime/app/server/remote/form.js index 7bc7259185e9..b5945755403e 100644 --- a/packages/kit/src/runtime/app/server/remote/form.js +++ b/packages/kit/src/runtime/app/server/remote/form.js @@ -4,7 +4,6 @@ import { get_request_store } from '@sveltejs/kit/internal/server'; import { DEV } from 'esm-env'; import { - convert_formdata, create_field_proxy, set_nested_value, throw_on_old_property_access, @@ -104,19 +103,7 @@ export function form(validate_or_fn, maybe_fn) { type: 'form', name: '', id: '', - /** @param {FormData} form_data */ - fn: async (form_data) => { - const validate_only = form_data.get('sveltekit:validate_only') === 'true'; - - let data = maybe_fn ? convert_formdata(form_data) : undefined; - - if (data && data.id === undefined) { - const id = form_data.get('sveltekit:id'); - if (typeof id === 'string') { - data.id = JSON.parse(id); - } - } - + fn: async (data, meta) => { // TODO 3.0 remove this warning if (DEV && !data) { const error = () => { @@ -152,12 +139,12 @@ export function form(validate_or_fn, maybe_fn) { const { event, state } = get_request_store(); const validated = await schema?.['~standard'].validate(data); - if (validate_only) { + if (meta.validate_only) { return validated?.issues ?? []; } if (validated?.issues !== undefined) { - handle_issues(output, validated.issues, event.isRemoteRequest, form_data); + handle_issues(output, validated.issues, event.isRemoteRequest, data); } else { if (validated !== undefined) { data = validated.value; @@ -178,7 +165,7 @@ export function form(validate_or_fn, maybe_fn) { ); } catch (e) { if (e instanceof ValidationError) { - handle_issues(output, e.issues, event.isRemoteRequest, form_data); + handle_issues(output, e.issues, event.isRemoteRequest, data); } else { throw e; } diff --git a/packages/kit/src/runtime/client/remote-functions/form.svelte.js b/packages/kit/src/runtime/client/remote-functions/form.svelte.js index 02bbf9239720..40cb6aa02f3a 100644 --- a/packages/kit/src/runtime/client/remote-functions/form.svelte.js +++ b/packages/kit/src/runtime/client/remote-functions/form.svelte.js @@ -18,7 +18,9 @@ import { set_nested_value, throw_on_old_property_access, build_path_string, - normalize_issue + normalize_issue, + serialize_binary_form, + BINARY_FORM_CONTENT_TYPE } from '../../form-utils.js'; /** @@ -182,17 +184,16 @@ export function form(id) { try { await Promise.resolve(); - if (updates.length > 0) { - data.set('sveltekit:remote_refreshes', JSON.stringify(updates.map((u) => u._key))); - } - const response = await fetch(`${base}/${app_dir}/remote/${action_id}`, { method: 'POST', - body: data, headers: { - 'x-sveltekit-pathname': location.pathname, - 'x-sveltekit-search': location.search - } + 'Content-Type': BINARY_FORM_CONTENT_TYPE + }, + body: serialize_binary_form(data, { + remote_refreshes: updates.map((u) => u._key), + pathname: location.pathname, + search: location.search + }) }); if (!response.ok) { @@ -532,7 +533,9 @@ export function form(id) { /** @type {InternalRemoteFormIssue[]} */ let array = []; - const validated = await preflight_schema?.['~standard'].validate(convert(form_data)); + const data = convert(form_data); + + const validated = await preflight_schema?.['~standard'].validate(data); if (validate_id !== id) { return; @@ -541,11 +544,16 @@ export function form(id) { if (validated?.issues) { array = validated.issues.map((issue) => normalize_issue(issue, false)); } else if (!preflightOnly) { - form_data.set('sveltekit:validate_only', 'true'); - const response = await fetch(`${base}/${app_dir}/remote/${action_id}`, { method: 'POST', - body: form_data + headers: { + 'Content-Type': BINARY_FORM_CONTENT_TYPE + }, + body: serialize_binary_form(data, { + validate_only: true, + pathname: location.pathname, + search: location.search + }) }); const result = await response.json(); diff --git a/packages/kit/src/runtime/form-utils.js b/packages/kit/src/runtime/form-utils.js index e268ff80b2b8..bafdbc28ef6c 100644 --- a/packages/kit/src/runtime/form-utils.js +++ b/packages/kit/src/runtime/form-utils.js @@ -1,5 +1,5 @@ /** @import { RemoteForm } from '@sveltejs/kit' */ -/** @import { InternalRemoteFormIssue } from 'types' */ +/** @import { BinaryFormMeta, InternalRemoteFormIssue } from 'types' */ /** @import { StandardSchemaV1 } from '@standard-schema/spec' */ import { DEV } from 'esm-env'; @@ -64,6 +64,33 @@ export function convert_formdata(data) { return result; } +export const BINARY_FORM_CONTENT_TYPE = 'application/x-sveltekit-formdata'; + +/** + * + * @param {Record} data + * @param {BinaryFormMeta} meta + * @returns {ReadableStream>} + */ +export function serialize_binary_form(data, meta) { + return new ReadableStream({ + start(controller) {} + }); +} + +/** + * @param {Request} request + * @returns {Promise<{ data: Record; meta: BinaryFormMeta }>} + */ +export async function deserialize_binary_form(request) { + if (request.headers.get('content-type') !== BINARY_FORM_CONTENT_TYPE) { + const form_data = await request.formData(); + return { data: convert_formdata(form_data), meta: {} }; + } + + return { data: {}, meta: {} }; +} + const path_regex = /^[a-zA-Z_$]\w*(\.[a-zA-Z_$]\w*|\[\d+\])*$/; /** diff --git a/packages/kit/src/runtime/server/remote.js b/packages/kit/src/runtime/server/remote.js index 88ae0d41387c..e49f382e6385 100644 --- a/packages/kit/src/runtime/server/remote.js +++ b/packages/kit/src/runtime/server/remote.js @@ -12,6 +12,7 @@ import { normalize_error } from '../../utils/error.js'; import { check_incorrect_fail_use } from './page/actions.js'; import { DEV } from 'esm-env'; import { record_span } from '../telemetry/record_span.js'; +import { deserialize_binary_form } from '../form-utils.js'; /** @type {typeof handle_remote_call_internal} */ export async function handle_remote_call(event, state, options, manifest, id) { @@ -116,25 +117,24 @@ async function handle_remote_call_internal(event, state, options, manifest, id) ); } - const form_data = await event.request.formData(); - form_client_refreshes = /** @type {string[]} */ ( - JSON.parse(/** @type {string} */ (form_data.get('sveltekit:remote_refreshes')) ?? '[]') - ); - form_data.delete('sveltekit:remote_refreshes'); + const { data, meta } = await deserialize_binary_form(event.request); // If this is a keyed form instance (created via form.for(key)), add the key to the form data (unless already set) if (additional_args) { - form_data.set('sveltekit:id', decodeURIComponent(additional_args)); + meta.id = decodeURIComponent(additional_args); } const fn = info.fn; - const data = await with_request_store({ event, state }, () => fn(form_data)); + const result = await with_request_store({ event, state }, () => fn(data, meta)); return json( /** @type {RemoteFunctionResponse} */ ({ type: 'result', - result: stringify(data, transport), - refreshes: data.issues ? {} : await serialize_refreshes(form_client_refreshes) + result: stringify(result, transport), + refreshes: + result.issues || !meta.remote_refreshes + ? {} + : await serialize_refreshes(meta.remote_refreshes) }) ); } @@ -291,16 +291,14 @@ async function handle_remote_form_post_internal(event, state, manifest, id) { } try { - const form_data = await event.request.formData(); const fn = /** @type {RemoteInfo & { type: 'form' }} */ (/** @type {any} */ (form).__).fn; - // If this is a keyed form instance (created via form.for(key)), add the key to the form data (unless already set) - if (action_id && !form_data.has('id')) { - // The action_id is URL-encoded JSON, decode and parse it - form_data.set('sveltekit:id', decodeURIComponent(action_id)); + const { data, meta } = await deserialize_binary_form(event.request); + if (action_id && !data.id) { + meta.id = decodeURIComponent(action_id); } - await with_request_store({ event, state }, () => fn(form_data)); + await with_request_store({ event, state }, () => fn(data, meta)); // We don't want the data to appear on `let { form } = $props()`, which is why we're not returning it. // It is instead available on `myForm.result`, setting of which happens within the remote `form` function. diff --git a/packages/kit/src/types/internal.d.ts b/packages/kit/src/types/internal.d.ts index 84242df90417..6d9e30cb4723 100644 --- a/packages/kit/src/types/internal.d.ts +++ b/packages/kit/src/types/internal.d.ts @@ -552,6 +552,14 @@ export type ValidatedKitConfig = Omit, 'adapter'> & adapter?: Adapter; }; +export type BinaryFormMeta = { + id?: string; + remote_refreshes?: string[]; + validate_only?: boolean; + pathname?: string; + search?: string; +}; + export type RemoteInfo = | { type: 'query' | 'command'; @@ -572,7 +580,7 @@ export type RemoteInfo = type: 'form'; id: string; name: string; - fn: (data: FormData) => Promise; + fn: (body: Record, meta: BinaryFormMeta) => Promise; } | { type: 'prerender'; diff --git a/packages/kit/src/utils/http.js b/packages/kit/src/utils/http.js index bfd948d4cbf6..f9cb33c652ed 100644 --- a/packages/kit/src/utils/http.js +++ b/packages/kit/src/utils/http.js @@ -1,3 +1,5 @@ +import { BINARY_FORM_CONTENT_TYPE } from '../runtime/form-utils.js'; + /** * Given an Accept header and a list of possible content types, pick * the most suitable one to respond with @@ -74,6 +76,7 @@ export function is_form_content_type(request) { request, 'application/x-www-form-urlencoded', 'multipart/form-data', - 'text/plain' + 'text/plain', + BINARY_FORM_CONTENT_TYPE ); } diff --git a/packages/kit/test/apps/basics/src/routes/remote/form/file-upload/+page.svelte b/packages/kit/test/apps/basics/src/routes/remote/form/file-upload/+page.svelte new file mode 100644 index 000000000000..0afc25dd6550 --- /dev/null +++ b/packages/kit/test/apps/basics/src/routes/remote/form/file-upload/+page.svelte @@ -0,0 +1,8 @@ + + +
+ + +
diff --git a/packages/kit/test/apps/basics/src/routes/remote/form/file-upload/form.remote.ts b/packages/kit/test/apps/basics/src/routes/remote/form/file-upload/form.remote.ts new file mode 100644 index 000000000000..0bef7d0ff857 --- /dev/null +++ b/packages/kit/test/apps/basics/src/routes/remote/form/file-upload/form.remote.ts @@ -0,0 +1,9 @@ +import { form } from '$app/server'; +import * as v from 'valibot'; + +export const upload = form( + v.object({ + file: v.file() + }), + async () => {} +); From 3b67fe774b17e8bebfdae96f096c69715fb3cb15 Mon Sep 17 00:00:00 2001 From: Ottomated Date: Mon, 20 Oct 2025 21:36:44 -0700 Subject: [PATCH 02/27] pass in form_dat --- .../kit/src/runtime/app/server/remote/form.js | 15 ++++++--------- packages/kit/src/runtime/form-utils.js | 6 +++--- packages/kit/src/runtime/server/remote.js | 8 ++++---- packages/kit/src/types/internal.d.ts | 6 +++++- 4 files changed, 18 insertions(+), 17 deletions(-) diff --git a/packages/kit/src/runtime/app/server/remote/form.js b/packages/kit/src/runtime/app/server/remote/form.js index b5945755403e..b89944362449 100644 --- a/packages/kit/src/runtime/app/server/remote/form.js +++ b/packages/kit/src/runtime/app/server/remote/form.js @@ -103,7 +103,7 @@ export function form(validate_or_fn, maybe_fn) { type: 'form', name: '', id: '', - fn: async (data, meta) => { + fn: async (data, meta, form_data) => { // TODO 3.0 remove this warning if (DEV && !data) { const error = () => { @@ -144,7 +144,7 @@ export function form(validate_or_fn, maybe_fn) { } if (validated?.issues !== undefined) { - handle_issues(output, validated.issues, event.isRemoteRequest, data); + handle_issues(output, validated.issues, form_data); } else { if (validated !== undefined) { data = validated.value; @@ -165,7 +165,7 @@ export function form(validate_or_fn, maybe_fn) { ); } catch (e) { if (e instanceof ValidationError) { - handle_issues(output, e.issues, event.isRemoteRequest, data); + handle_issues(output, e.issues, form_data); } else { throw e; } @@ -284,15 +284,12 @@ export function form(validate_or_fn, maybe_fn) { /** * @param {{ issues?: InternalRemoteFormIssue[], input?: Record, result: any }} output * @param {readonly StandardSchemaV1.Issue[]} issues - * @param {boolean} is_remote_request - * @param {FormData} form_data + * @param {FormData | null} form_data - null if the form is progressively enhanced */ -function handle_issues(output, issues, is_remote_request, form_data) { +function handle_issues(output, issues, form_data) { output.issues = issues.map((issue) => normalize_issue(issue, true)); - // if it was a progressively-enhanced submission, we don't need - // to return the input — it's already there - if (!is_remote_request) { + if (form_data) { output.input = {}; for (let key of form_data.keys()) { diff --git a/packages/kit/src/runtime/form-utils.js b/packages/kit/src/runtime/form-utils.js index bafdbc28ef6c..88829b7c9c7a 100644 --- a/packages/kit/src/runtime/form-utils.js +++ b/packages/kit/src/runtime/form-utils.js @@ -80,15 +80,15 @@ export function serialize_binary_form(data, meta) { /** * @param {Request} request - * @returns {Promise<{ data: Record; meta: BinaryFormMeta }>} + * @returns {Promise<{ data: Record; meta: BinaryFormMeta; form_data: FormData | null }>} */ export async function deserialize_binary_form(request) { if (request.headers.get('content-type') !== BINARY_FORM_CONTENT_TYPE) { const form_data = await request.formData(); - return { data: convert_formdata(form_data), meta: {} }; + return { data: convert_formdata(form_data), meta: {}, form_data }; } - return { data: {}, meta: {} }; + return { data: {}, meta: {}, form_data: null }; } const path_regex = /^[a-zA-Z_$]\w*(\.[a-zA-Z_$]\w*|\[\d+\])*$/; diff --git a/packages/kit/src/runtime/server/remote.js b/packages/kit/src/runtime/server/remote.js index e49f382e6385..abc2d1904cf6 100644 --- a/packages/kit/src/runtime/server/remote.js +++ b/packages/kit/src/runtime/server/remote.js @@ -117,7 +117,7 @@ async function handle_remote_call_internal(event, state, options, manifest, id) ); } - const { data, meta } = await deserialize_binary_form(event.request); + const { data, meta, form_data } = await deserialize_binary_form(event.request); // If this is a keyed form instance (created via form.for(key)), add the key to the form data (unless already set) if (additional_args) { @@ -125,7 +125,7 @@ async function handle_remote_call_internal(event, state, options, manifest, id) } const fn = info.fn; - const result = await with_request_store({ event, state }, () => fn(data, meta)); + const result = await with_request_store({ event, state }, () => fn(data, meta, form_data)); return json( /** @type {RemoteFunctionResponse} */ ({ @@ -293,12 +293,12 @@ async function handle_remote_form_post_internal(event, state, manifest, id) { try { const fn = /** @type {RemoteInfo & { type: 'form' }} */ (/** @type {any} */ (form).__).fn; - const { data, meta } = await deserialize_binary_form(event.request); + const { data, meta, form_data } = await deserialize_binary_form(event.request); if (action_id && !data.id) { meta.id = decodeURIComponent(action_id); } - await with_request_store({ event, state }, () => fn(data, meta)); + await with_request_store({ event, state }, () => fn(data, meta, form_data)); // We don't want the data to appear on `let { form } = $props()`, which is why we're not returning it. // It is instead available on `myForm.result`, setting of which happens within the remote `form` function. diff --git a/packages/kit/src/types/internal.d.ts b/packages/kit/src/types/internal.d.ts index 6d9e30cb4723..d9abcc2d53d6 100644 --- a/packages/kit/src/types/internal.d.ts +++ b/packages/kit/src/types/internal.d.ts @@ -580,7 +580,11 @@ export type RemoteInfo = type: 'form'; id: string; name: string; - fn: (body: Record, meta: BinaryFormMeta) => Promise; + fn: ( + body: Record, + meta: BinaryFormMeta, + form_data: FormData | null + ) => Promise; } | { type: 'prerender'; From 75005a70df79e0de9d671744517c7bc73b78e75c Mon Sep 17 00:00:00 2001 From: Ottomated Date: Mon, 20 Oct 2025 22:39:22 -0700 Subject: [PATCH 03/27] serialization --- .../client/remote-functions/form.svelte.js | 2 +- packages/kit/src/runtime/form-utils.js | 49 +++++++++++++++++-- 2 files changed, 47 insertions(+), 4 deletions(-) diff --git a/packages/kit/src/runtime/client/remote-functions/form.svelte.js b/packages/kit/src/runtime/client/remote-functions/form.svelte.js index 40cb6aa02f3a..3d7a116a7cc6 100644 --- a/packages/kit/src/runtime/client/remote-functions/form.svelte.js +++ b/packages/kit/src/runtime/client/remote-functions/form.svelte.js @@ -189,7 +189,7 @@ export function form(id) { headers: { 'Content-Type': BINARY_FORM_CONTENT_TYPE }, - body: serialize_binary_form(data, { + body: serialize_binary_form(convert(data), { remote_refreshes: updates.map((u) => u._key), pathname: location.pathname, search: location.search diff --git a/packages/kit/src/runtime/form-utils.js b/packages/kit/src/runtime/form-utils.js index 88829b7c9c7a..471040d885d0 100644 --- a/packages/kit/src/runtime/form-utils.js +++ b/packages/kit/src/runtime/form-utils.js @@ -3,6 +3,7 @@ /** @import { StandardSchemaV1 } from '@standard-schema/spec' */ import { DEV } from 'esm-env'; +import * as devalue from 'devalue'; /** * Sets a value in a nested object using a path string, mutating the original object @@ -70,12 +71,50 @@ export const BINARY_FORM_CONTENT_TYPE = 'application/x-sveltekit-formdata'; * * @param {Record} data * @param {BinaryFormMeta} meta - * @returns {ReadableStream>} + * @returns {Blob} */ export function serialize_binary_form(data, meta) { - return new ReadableStream({ - start(controller) {} + /** @type {BlobPart[]} */ + const blob_parts = []; + + /** @type {File[]} */ + const files = []; + + const encoded_header = devalue.stringify([data, meta], { + File: (file) => { + if (!(file instanceof File)) return; + files.push(file); + return { + name: file.name, + type: file.type, + i: files.length - 1 + }; + } }); + const length_buffer = new Uint8Array(4); + const length_view = new DataView(length_buffer.buffer); + + length_view.setUint32(0, encoded_header.length, true); + blob_parts.push(length_buffer.slice()); + blob_parts.push(encoded_header); + + length_view.setUint32(0, files.length, true); + blob_parts.push(length_buffer); + + if (files.length === 0) { + return new Blob(blob_parts); + } + + const size_buffer = new Uint8Array(16); + const size_view = new DataView(size_buffer.buffer); + + for (const file of files) { + // Use a u64 so we aren't limited to 4GB files + size_view.setBigUint64(0, BigInt(file.size), true); + blob_parts.push(size_buffer.slice()); + blob_parts.push(file); + } + return new Blob(blob_parts); } /** @@ -87,7 +126,11 @@ export async function deserialize_binary_form(request) { const form_data = await request.formData(); return { data: convert_formdata(form_data), meta: {}, form_data }; } + if (!request.body) { + return { data: {}, meta: {}, form_data: null }; + } + const reader = request.body.getReader(); return { data: {}, meta: {}, form_data: null }; } From 7c60494248d02c85b07a9a06c3c3b555baab4e19 Mon Sep 17 00:00:00 2001 From: Ottomated Date: Tue, 21 Oct 2025 00:20:06 -0700 Subject: [PATCH 04/27] start deserializer --- packages/kit/src/runtime/form-utils.js | 176 +++++++++++++++--- .../remote/form/file-upload/+page.svelte | 1 + .../remote/form/file-upload/form.remote.ts | 7 +- 3 files changed, 159 insertions(+), 25 deletions(-) diff --git a/packages/kit/src/runtime/form-utils.js b/packages/kit/src/runtime/form-utils.js index 471040d885d0..29b20173134e 100644 --- a/packages/kit/src/runtime/form-utils.js +++ b/packages/kit/src/runtime/form-utils.js @@ -4,6 +4,7 @@ import { DEV } from 'esm-env'; import * as devalue from 'devalue'; +import { text_decoder } from './utils.js'; /** * Sets a value in a nested object using a path string, mutating the original object @@ -66,29 +67,31 @@ export function convert_formdata(data) { } export const BINARY_FORM_CONTENT_TYPE = 'application/x-sveltekit-formdata'; +const BINARY_FORM_VERSION = 0; /** - * + * The binary format is as follows: + * - 1 byte: Format version + * - 4 bytes: Length of the header (u32) + * - 4 bytes: Number of files (u32) + * - header: devalue.stringify([data, meta]) + * - N files * @param {Record} data * @param {BinaryFormMeta} meta * @returns {Blob} */ export function serialize_binary_form(data, meta) { - /** @type {BlobPart[]} */ - const blob_parts = []; + /** @type {Array} */ + const blob_parts = [new Uint8Array([BINARY_FORM_VERSION])]; - /** @type {File[]} */ + /** @type {Array} */ const files = []; const encoded_header = devalue.stringify([data, meta], { File: (file) => { if (!(file instanceof File)) return; files.push(file); - return { - name: file.name, - type: file.type, - i: files.length - 1 - }; + return [file.name, file.type, file.size, file.lastModified, files.length - 1]; } }); const length_buffer = new Uint8Array(4); @@ -96,24 +99,13 @@ export function serialize_binary_form(data, meta) { length_view.setUint32(0, encoded_header.length, true); blob_parts.push(length_buffer.slice()); - blob_parts.push(encoded_header); length_view.setUint32(0, files.length, true); blob_parts.push(length_buffer); - if (files.length === 0) { - return new Blob(blob_parts); - } - - const size_buffer = new Uint8Array(16); - const size_view = new DataView(size_buffer.buffer); + blob_parts.push(encoded_header); - for (const file of files) { - // Use a u64 so we aren't limited to 4GB files - size_view.setBigUint64(0, BigInt(file.size), true); - blob_parts.push(size_buffer.slice()); - blob_parts.push(file); - } + blob_parts.push(...files); return new Blob(blob_parts); } @@ -127,11 +119,147 @@ export async function deserialize_binary_form(request) { return { data: convert_formdata(form_data), meta: {}, form_data }; } if (!request.body) { - return { data: {}, meta: {}, form_data: null }; + throw new Error('Could not deserialize binary form: no body'); } const reader = request.body.getReader(); - return { data: {}, meta: {}, form_data: null }; + + const first_chunk = await reader.read(); + if (first_chunk.done) { + throw new Error('Could not deserialize binary form: empty body'); + } + if (first_chunk.value.byteLength < 1 + 4 + 4) { + throw new Error('Could not deserialize binary form: first chunk was too small'); + } + const version = first_chunk.value[0]; + if (version !== BINARY_FORM_VERSION) { + throw new Error( + `Could not deserialize binary form: got version ${version}, expected version ${BINARY_FORM_VERSION}` + ); + } + const start_view = new DataView(first_chunk.value.buffer); + const header_length = start_view.getUint32(1, true); + const file_count = start_view.getUint32(5, true); + + // Read the header + const header_buffer = new Uint8Array(header_length); + header_buffer.set(first_chunk.value.subarray(9, header_length + 9)); + let received_length = first_chunk.value.byteLength - 9; + /** @type {Array} */ + let file_data; + if (received_length >= header_length) { + file_data = [first_chunk.value.subarray(header_length)]; + } else { + while (true) { + const chunk = await reader.read(); + if (chunk.done) { + throw new Error('Could not deserialize binary form: incomplete header'); + } + const header_chunk = chunk.value.subarray(0, header_length - received_length); + header_buffer.set(header_chunk, received_length); + + received_length += chunk.value.byteLength; + + if (received_length >= header_length) { + file_data = [chunk.value.subarray(header_length)]; + break; + } + } + } + + /** @type {Array} */ + const file_sizes = new Array(file_count); + /** @type {Array<{start: number, end: number}> | null} */ + let file_offsets = null; + /** @type {Array>} */ + const file_buffers = new Array(file_count); + + const [data, meta] = devalue.parse(text_decoder.decode(header_buffer), { + File: ([name, type, size, last_modified, index]) => { + file_sizes[index] = size; + console.log(file_sizes); + return new Proxy( + new LazyFile(name, type, size, last_modified, async () => { + if (file_buffers[index]) return file_buffers[index]; + if (file_offsets === null) { + file_offsets = new Array(file_count); + let start = 0; + for (let i = 0; i < file_count; i++) { + const end = start + file_sizes[i]; + file_offsets[i] = { start, end }; + start = end; + } + } + const { start, end } = file_offsets[index]; + const buffer = new Uint8Array(end - start); + // let offset = 0; + // while (offset < buffer.byteLength) { + // TODO: + // - find the element from `file_data` that contains start + offset + // - if it doesn't exist, read from the request body until we get it, and cache results in `file_data` + // - copy subarray into `buffer` + // } + return buffer; + }), + { + getPrototypeOf() { + // Trick validators into thinking this is a normal File + return File.prototype; + } + } + ); + } + }); + console.log(data); + + return { data, meta, form_data: null }; +} + +class LazyFile { + /** @type {() => Promise} */ + getter; + /** + * @param {string} name + * @param {string} type + * @param {number} size + * @param {number} last_modified + * @param {() => Promise} getter + */ + constructor(name, type, size, last_modified, getter) { + this.name = name; + this.type = type; + this.size = size; + this.lastModified = last_modified; + this.webkitRelativePath = ''; + this.getter = getter; + } + arrayBuffer() { + return this.getter(); + } + async bytes() { + return new Uint8Array(await this.arrayBuffer()); + } + /** + * @param {number=} start + * @param {number=} end + * @param {string=} contentType + */ + slice(start, end, contentType) { + return new LazyFile(this.name, contentType ?? '', this.size, this.lastModified, () => + this.getter().then((buffer) => buffer.slice(start, end)) + ); + } + stream() { + return new ReadableStream({ + start: async (controller) => { + controller.enqueue(await this.arrayBuffer()); + controller.close(); + } + }); + } + async text() { + return text_decoder.decode(await this.arrayBuffer()); + } } const path_regex = /^[a-zA-Z_$]\w*(\.[a-zA-Z_$]\w*|\[\d+\])*$/; diff --git a/packages/kit/test/apps/basics/src/routes/remote/form/file-upload/+page.svelte b/packages/kit/test/apps/basics/src/routes/remote/form/file-upload/+page.svelte index 0afc25dd6550..d45aebd5be64 100644 --- a/packages/kit/test/apps/basics/src/routes/remote/form/file-upload/+page.svelte +++ b/packages/kit/test/apps/basics/src/routes/remote/form/file-upload/+page.svelte @@ -3,6 +3,7 @@
+
diff --git a/packages/kit/test/apps/basics/src/routes/remote/form/file-upload/form.remote.ts b/packages/kit/test/apps/basics/src/routes/remote/form/file-upload/form.remote.ts index 0bef7d0ff857..daa95214aa64 100644 --- a/packages/kit/test/apps/basics/src/routes/remote/form/file-upload/form.remote.ts +++ b/packages/kit/test/apps/basics/src/routes/remote/form/file-upload/form.remote.ts @@ -3,7 +3,12 @@ import * as v from 'valibot'; export const upload = form( v.object({ + text: v.string(), file: v.file() }), - async () => {} + async (data) => { + console.log(data.text); + console.log(data.file); + console.log(await data.file.text()); + } ); From 8a62a3ce54b91683cb179c1de9046ed706c9ab25 Mon Sep 17 00:00:00 2001 From: Ottomated Date: Tue, 21 Oct 2025 20:38:25 -0700 Subject: [PATCH 05/27] finished? deserializer --- packages/kit/src/runtime/form-utils.js | 279 ++++++++++++------ .../remote/form/file-upload/form.remote.ts | 4 +- 2 files changed, 190 insertions(+), 93 deletions(-) diff --git a/packages/kit/src/runtime/form-utils.js b/packages/kit/src/runtime/form-utils.js index 29b20173134e..5dd3c520e09e 100644 --- a/packages/kit/src/runtime/form-utils.js +++ b/packages/kit/src/runtime/form-utils.js @@ -124,117 +124,203 @@ export async function deserialize_binary_form(request) { const reader = request.body.getReader(); - const first_chunk = await reader.read(); - if (first_chunk.done) { - throw new Error('Could not deserialize binary form: empty body'); - } - if (first_chunk.value.byteLength < 1 + 4 + 4) { - throw new Error('Could not deserialize binary form: first chunk was too small'); - } - const version = first_chunk.value[0]; - if (version !== BINARY_FORM_VERSION) { - throw new Error( - `Could not deserialize binary form: got version ${version}, expected version ${BINARY_FORM_VERSION}` - ); - } - const start_view = new DataView(first_chunk.value.buffer); - const header_length = start_view.getUint32(1, true); - const file_count = start_view.getUint32(5, true); - - // Read the header - const header_buffer = new Uint8Array(header_length); - header_buffer.set(first_chunk.value.subarray(9, header_length + 9)); - let received_length = first_chunk.value.byteLength - 9; - /** @type {Array} */ - let file_data; - if (received_length >= header_length) { - file_data = [first_chunk.value.subarray(header_length)]; - } else { - while (true) { - const chunk = await reader.read(); - if (chunk.done) { - throw new Error('Could not deserialize binary form: incomplete header'); - } - const header_chunk = chunk.value.subarray(0, header_length - received_length); - header_buffer.set(header_chunk, received_length); + /** @type {Array>} */ + const chunks = []; - received_length += chunk.value.byteLength; + /** + * @param {number} index + * @returns {Promise | null>} + */ + async function get_chunk(index) { + if (chunks[index]) return chunks[index]; + let i = chunks.length; + while (i <= index) { + const chunk = await reader.read(); + if (chunk.done) return null; + chunks[i] = chunk.value; + i++; + } + return chunks[index]; + } - if (received_length >= header_length) { - file_data = [chunk.value.subarray(header_length)]; + /** + * @param {number} offset + * @param {number} length + * @returns {Promise} + */ + async function get_buffer(offset, length) { + /** @type {Uint8Array} */ + let start_chunk; + let chunk_start = 0; + /** @type {number} */ + let chunk_index; + for (chunk_index = 0; ; chunk_index++) { + const chunk = await get_chunk(chunk_index); + if (!chunk) return null; + + const chunk_end = chunk_start + chunk.byteLength; + // If this chunk contains the target offset + if (offset >= chunk_start && offset < chunk_end) { + start_chunk = chunk; break; } + chunk_start = chunk_end; + } + // If the buffer is completely contained in one chunk, do a subarray + if (offset + length <= chunk_start + start_chunk.byteLength) { + return start_chunk.subarray(offset - chunk_start, offset + length - chunk_start); + } + // Otherwise, copy the data into a new buffer + const buffer = new Uint8Array(length); + buffer.set(start_chunk.subarray(offset - chunk_start)); + let cursor = start_chunk.byteLength - offset + chunk_start; + while (cursor < length) { + chunk_index++; + let chunk = await get_chunk(chunk_index); + if (!chunk) return null; + if (chunk.byteLength > length - cursor) { + chunk = chunk.subarray(0, length - cursor); + } + buffer.set(chunk, cursor); + cursor += chunk.byteLength; } + + return buffer; } - /** @type {Array} */ - const file_sizes = new Array(file_count); - /** @type {Array<{start: number, end: number}> | null} */ - let file_offsets = null; - /** @type {Array>} */ - const file_buffers = new Array(file_count); + const header = await get_buffer(0, 1 + 4 + 4); + if (!header) throw new Error('Could not deserialize binary form: too short'); - const [data, meta] = devalue.parse(text_decoder.decode(header_buffer), { + if (header[0] !== BINARY_FORM_VERSION) { + throw new Error( + `Could not deserialize binary form: got version ${header[0]}, expected version ${BINARY_FORM_VERSION}` + ); + } + const header_view = new DataView(header.buffer); + const data_length = header_view.getUint32(1, true); + const file_count = header_view.getUint32(5, true); + + // Read the form data + const data_buffer = await get_buffer(1 + 4 + 4, data_length); + if (!data_buffer) throw new Error('Could not deserialize binary form: data too short'); + + /** @type {Array} */ + const files = new Array(file_count); + + const [data, meta] = devalue.parse(text_decoder.decode(data_buffer), { File: ([name, type, size, last_modified, index]) => { - file_sizes[index] = size; - console.log(file_sizes); - return new Proxy( - new LazyFile(name, type, size, last_modified, async () => { - if (file_buffers[index]) return file_buffers[index]; - if (file_offsets === null) { - file_offsets = new Array(file_count); - let start = 0; - for (let i = 0; i < file_count; i++) { - const end = start + file_sizes[i]; - file_offsets[i] = { start, end }; - start = end; - } - } - const { start, end } = file_offsets[index]; - const buffer = new Uint8Array(end - start); - // let offset = 0; - // while (offset < buffer.byteLength) { - // TODO: - // - find the element from `file_data` that contains start + offset - // - if it doesn't exist, read from the request body until we get it, and cache results in `file_data` - // - copy subarray into `buffer` - // } - return buffer; - }), - { - getPrototypeOf() { - // Trick validators into thinking this is a normal File - return File.prototype; - } + const file = new LazyFile(name, type, size, last_modified); + files[index] = file; + return new Proxy(file, { + getPrototypeOf() { + // Trick validators into thinking this is a normal File + return File.prototype; } - ); + }); } }); - console.log(data); + + let offset = 1 + 4 + 4 + data_length; + for (const file of files) { + const start = offset; + const end = start + file.size; + file._setup_internal(get_chunk, start); + offset = end; + } return { data, meta, form_data: null }; } +/** @implements {File} */ class LazyFile { - /** @type {() => Promise} */ - getter; + /** @type {ReadableStream>} */ + // @ts-expect-error no equivalent to !: + #stream; + /** @type {(index: number) => Promise | null>} */ + // @ts-expect-error no equivalent to !: + #get_chunk; + /** @type {number} */ + // @ts-expect-error no equivalent to !: + #offset; + /** + * @param {(index: number) => Promise | null>} get_chunk + * @param {number} offset + */ + _setup_internal(get_chunk, offset) { + if (this.#stream) throw new TypeError('_setup_internal called twice'); + let cursor = 0; + let chunk_index = 0; + this.#stream = new ReadableStream({ + start: async (controller) => { + let chunk_start = 0; + let start_chunk = null; + for (chunk_index = 0; ; chunk_index++) { + const chunk = await get_chunk(chunk_index); + if (!chunk) return null; + + const chunk_end = chunk_start + chunk.byteLength; + // If this chunk contains the target offset + if (offset >= chunk_start && offset < chunk_end) { + start_chunk = chunk; + break; + } + chunk_start = chunk_end; + } + // If the buffer is completely contained in one chunk, do a subarray + if (offset + this.size <= chunk_start + start_chunk.byteLength) { + controller.enqueue( + start_chunk.subarray(offset - chunk_start, offset + this.size - chunk_start) + ); + controller.close(); + } else { + controller.enqueue(start_chunk.subarray(offset - chunk_start)); + cursor = start_chunk.byteLength - offset + chunk_start; + } + }, + pull: async (controller) => { + chunk_index++; + let chunk = await get_chunk(chunk_index); + if (!chunk) { + controller.error('Could not deserialize binary form: incomplete data'); + return; + } + if (chunk.byteLength > this.size - cursor) { + chunk = chunk.subarray(0, this.size - cursor); + } + controller.enqueue(chunk); + cursor += chunk.byteLength; + if (cursor >= this.size) { + controller.close(); + } + } + }); + this.#get_chunk = get_chunk; + this.#offset = offset; + } /** * @param {string} name * @param {string} type * @param {number} size * @param {number} last_modified - * @param {() => Promise} getter */ - constructor(name, type, size, last_modified, getter) { + constructor(name, type, size, last_modified) { this.name = name; this.type = type; this.size = size; this.lastModified = last_modified; this.webkitRelativePath = ''; - this.getter = getter; + // TODO - hacky, required for private members to be accessed on proxy + this.arrayBuffer = this.arrayBuffer.bind(this); + this.bytes = this.bytes.bind(this); + this.slice = this.slice.bind(this); + this.stream = this.stream.bind(this); + this.text = this.text.bind(this); } - arrayBuffer() { - return this.getter(); + /** @type {ArrayBuffer | undefined} */ + #buffer; + async arrayBuffer() { + this.#buffer ??= await new Response(this.#stream).arrayBuffer(); + return this.#buffer; } async bytes() { return new Uint8Array(await this.arrayBuffer()); @@ -244,18 +330,27 @@ class LazyFile { * @param {number=} end * @param {string=} contentType */ - slice(start, end, contentType) { - return new LazyFile(this.name, contentType ?? '', this.size, this.lastModified, () => - this.getter().then((buffer) => buffer.slice(start, end)) - ); + slice(start = 0, end = this.size, contentType = this.type) { + // https://github.com/nodejs/node/blob/a5f3cd8cb5ba9e7911d93c5fd3ebc6d781220dd8/lib/internal/blob.js#L240 + if (start < 0) { + start = Math.max(this.size + start, 0); + } else { + start = Math.min(start, this.size); + } + + if (end < 0) { + end = Math.max(this.size + end, 0); + } else { + end = Math.min(end, this.size); + } + const size = Math.max(end - start, 0); + const file = new LazyFile(this.name, contentType, size, this.lastModified); + + file._setup_internal(this.#get_chunk, this.#offset + start); + return file; } stream() { - return new ReadableStream({ - start: async (controller) => { - controller.enqueue(await this.arrayBuffer()); - controller.close(); - } - }); + return this.#stream; } async text() { return text_decoder.decode(await this.arrayBuffer()); diff --git a/packages/kit/test/apps/basics/src/routes/remote/form/file-upload/form.remote.ts b/packages/kit/test/apps/basics/src/routes/remote/form/file-upload/form.remote.ts index daa95214aa64..071897ab1eb4 100644 --- a/packages/kit/test/apps/basics/src/routes/remote/form/file-upload/form.remote.ts +++ b/packages/kit/test/apps/basics/src/routes/remote/form/file-upload/form.remote.ts @@ -9,6 +9,8 @@ export const upload = form( async (data) => { console.log(data.text); console.log(data.file); - console.log(await data.file.text()); + for await (const chunk of data.file.stream()) { + console.log(chunk); + } } ); From 28d6e901fa5db142698f259b44307160a76c74c5 Mon Sep 17 00:00:00 2001 From: Ottomated Date: Tue, 21 Oct 2025 21:24:03 -0700 Subject: [PATCH 06/27] upload progress via XHR --- .../client/remote-functions/form.svelte.js | 72 ++++++++++++++----- packages/kit/src/runtime/form-utils.js | 17 ++++- .../remote/form/file-upload/+page.svelte | 5 +- .../remote/form/file-upload/form.remote.ts | 14 ++-- 4 files changed, 82 insertions(+), 26 deletions(-) diff --git a/packages/kit/src/runtime/client/remote-functions/form.svelte.js b/packages/kit/src/runtime/client/remote-functions/form.svelte.js index 3d7a116a7cc6..04338699b8c4 100644 --- a/packages/kit/src/runtime/client/remote-functions/form.svelte.js +++ b/packages/kit/src/runtime/client/remote-functions/form.svelte.js @@ -184,25 +184,63 @@ export function form(id) { try { await Promise.resolve(); - const response = await fetch(`${base}/${app_dir}/remote/${action_id}`, { - method: 'POST', - headers: { - 'Content-Type': BINARY_FORM_CONTENT_TYPE - }, - body: serialize_binary_form(convert(data), { - remote_refreshes: updates.map((u) => u._key), - pathname: location.pathname, - search: location.search - }) + const { file_offsets, blob } = serialize_binary_form(convert(data), { + remote_refreshes: updates.map((u) => u._key), + pathname: location.pathname, + search: location.search }); - if (!response.ok) { - // We only end up here in case of a network error or if the server has an internal error - // (which shouldn't happen because we handle errors on the server and always send a 200 response) - throw new Error('Failed to execute remote function'); - } + // TODO - check this - does it block the event loop? + // TODO - extract XHR to a function and reuse for validate_only + const response = await new Promise((resolve, reject) => { + const xhr = new XMLHttpRequest(); + xhr.addEventListener('error', () => { + reject(new Error('Failed to execute remote function')); + }); + xhr.addEventListener('readystatechange', () => { + if (xhr.readyState === 2 /* HEADERS_RECEIVED */) { + if (xhr.status !== 200) { + reject(new Error('Failed to execute remote function')); + } + } else if (xhr.readyState === 4 /* DONE */) { + resolve(xhr.responseText); + } + }); + if (file_offsets.length) { + xhr.upload.addEventListener('progress', (ev) => { + for (const file of file_offsets) { + const progress = (ev.loaded - file.start) / file.size; + if (progress < 0 || progress > 1) continue; + + console.log(`File ${file.name}: ${(progress * 100).toFixed(2)}%`); + } + }); + } + xhr.open('POST', `${base}/${app_dir}/remote/${action_id}`); + xhr.setRequestHeader('Content-Type', BINARY_FORM_CONTENT_TYPE); + xhr.send(blob); + }); - const form_result = /** @type { RemoteFunctionResponse} */ (await response.json()); + // const response = await fetch(`${base}/${app_dir}/remote/${action_id}`, { + // method: 'POST', + // headers: { + // 'Content-Type': BINARY_FORM_CONTENT_TYPE + // }, + // body: serialize_binary_form(convert(data), { + // remote_refreshes: updates.map((u) => u._key), + // pathname: location.pathname, + // search: location.search + // }) + // }); + + // if (!response.ok) { + // // We only end up here in case of a network error or if the server has an internal error + // // (which shouldn't happen because we handle errors on the server and always send a 200 response) + // throw new Error('Failed to execute remote function'); + // } + + // const form_result = /** @type { RemoteFunctionResponse} */ (await response.json()); + const form_result = /** @type { RemoteFunctionResponse} */ (JSON.parse(response)); if (form_result.type === 'result') { ({ issues: raw_issues = [], result } = devalue.parse(form_result.result, app.decoders)); @@ -553,7 +591,7 @@ export function form(id) { validate_only: true, pathname: location.pathname, search: location.search - }) + }).blob }); const result = await response.json(); diff --git a/packages/kit/src/runtime/form-utils.js b/packages/kit/src/runtime/form-utils.js index 5dd3c520e09e..4b79a90d21df 100644 --- a/packages/kit/src/runtime/form-utils.js +++ b/packages/kit/src/runtime/form-utils.js @@ -78,7 +78,6 @@ const BINARY_FORM_VERSION = 0; * - N files * @param {Record} data * @param {BinaryFormMeta} meta - * @returns {Blob} */ export function serialize_binary_form(data, meta) { /** @type {Array} */ @@ -105,8 +104,18 @@ export function serialize_binary_form(data, meta) { blob_parts.push(encoded_header); - blob_parts.push(...files); - return new Blob(blob_parts); + /** @type {Array<{ start: number, size: number, name: string }>} */ + const file_offsets = []; + let start = 1 + 4 + 4 + encoded_header.length; + for (const file of files) { + blob_parts.push(file); + file_offsets.push({ start, size: file.size, name: file.name }); + start += file.size; + } + return { + blob: new Blob(blob_parts), + file_offsets + }; } /** @@ -135,6 +144,7 @@ export async function deserialize_binary_form(request) { if (chunks[index]) return chunks[index]; let i = chunks.length; while (i <= index) { + // TODO - this breaks when two chunks are read at once :( const chunk = await reader.read(); if (chunk.done) return null; chunks[i] = chunk.value; @@ -282,6 +292,7 @@ class LazyFile { let chunk = await get_chunk(chunk_index); if (!chunk) { controller.error('Could not deserialize binary form: incomplete data'); + controller.close(); return; } if (chunk.byteLength > this.size - cursor) { diff --git a/packages/kit/test/apps/basics/src/routes/remote/form/file-upload/+page.svelte b/packages/kit/test/apps/basics/src/routes/remote/form/file-upload/+page.svelte index d45aebd5be64..67503c3693e7 100644 --- a/packages/kit/test/apps/basics/src/routes/remote/form/file-upload/+page.svelte +++ b/packages/kit/test/apps/basics/src/routes/remote/form/file-upload/+page.svelte @@ -4,6 +4,9 @@
- +

File 1:

+ +

File 2:

+
diff --git a/packages/kit/test/apps/basics/src/routes/remote/form/file-upload/form.remote.ts b/packages/kit/test/apps/basics/src/routes/remote/form/file-upload/form.remote.ts index 071897ab1eb4..5ef57e146646 100644 --- a/packages/kit/test/apps/basics/src/routes/remote/form/file-upload/form.remote.ts +++ b/packages/kit/test/apps/basics/src/routes/remote/form/file-upload/form.remote.ts @@ -4,13 +4,17 @@ import * as v from 'valibot'; export const upload = form( v.object({ text: v.string(), - file: v.file() + file1: v.file(), + file2: v.file() }), async (data) => { console.log(data.text); - console.log(data.file); - for await (const chunk of data.file.stream()) { - console.log(chunk); - } + console.log(data.file1); + console.log(data.file2); + console.log(await data.file1.text()); + console.log(await data.file2.text()); + // for await (const chunk of data.file1.stream()) { + // console.log('file 1', chunk.length); + // } } ); From ed58a94b65dfccfb1ee7b402fcaf43cc62d63151 Mon Sep 17 00:00:00 2001 From: Ottomated Date: Wed, 22 Oct 2025 14:57:30 -0700 Subject: [PATCH 07/27] simplify file offsets, sort small files first --- .../client/remote-functions/form.svelte.js | 22 +-- packages/kit/src/runtime/form-utils.js | 139 ++++++++++-------- 2 files changed, 89 insertions(+), 72 deletions(-) diff --git a/packages/kit/src/runtime/client/remote-functions/form.svelte.js b/packages/kit/src/runtime/client/remote-functions/form.svelte.js index 04338699b8c4..f7e9cb3ae915 100644 --- a/packages/kit/src/runtime/client/remote-functions/form.svelte.js +++ b/packages/kit/src/runtime/client/remote-functions/form.svelte.js @@ -184,7 +184,7 @@ export function form(id) { try { await Promise.resolve(); - const { file_offsets, blob } = serialize_binary_form(convert(data), { + const { blob } = serialize_binary_form(convert(data), { remote_refreshes: updates.map((u) => u._key), pathname: location.pathname, search: location.search @@ -206,16 +206,16 @@ export function form(id) { resolve(xhr.responseText); } }); - if (file_offsets.length) { - xhr.upload.addEventListener('progress', (ev) => { - for (const file of file_offsets) { - const progress = (ev.loaded - file.start) / file.size; - if (progress < 0 || progress > 1) continue; - - console.log(`File ${file.name}: ${(progress * 100).toFixed(2)}%`); - } - }); - } + // if (file_offsets.length) { + // xhr.upload.addEventListener('progress', (ev) => { + // for (const file of file_offsets) { + // const progress = (ev.loaded - file.start) / file.size; + // if (progress < 0 || progress > 1) continue; + + // console.log(`File ${file.name}: ${(progress * 100).toFixed(2)}%`); + // } + // }); + // } xhr.open('POST', `${base}/${app_dir}/remote/${action_id}`); xhr.setRequestHeader('Content-Type', BINARY_FORM_CONTENT_TYPE); xhr.send(blob); diff --git a/packages/kit/src/runtime/form-utils.js b/packages/kit/src/runtime/form-utils.js index 4b79a90d21df..65b59308ed3d 100644 --- a/packages/kit/src/runtime/form-utils.js +++ b/packages/kit/src/runtime/form-utils.js @@ -73,9 +73,10 @@ const BINARY_FORM_VERSION = 0; * The binary format is as follows: * - 1 byte: Format version * - 4 bytes: Length of the header (u32) - * - 4 bytes: Number of files (u32) + * - 4 bytes: Length of the file offset table (u32) * - header: devalue.stringify([data, meta]) - * - N files + * - file offset table: JSON.stringify([offset1, offset2, ...]) (offsets start from the end of the table) + * - file1, file2, ... * @param {Record} data * @param {BinaryFormMeta} meta */ @@ -83,38 +84,48 @@ export function serialize_binary_form(data, meta) { /** @type {Array} */ const blob_parts = [new Uint8Array([BINARY_FORM_VERSION])]; - /** @type {Array} */ + /** @type {Array<[file: File, index: number]>} */ const files = []; const encoded_header = devalue.stringify([data, meta], { File: (file) => { if (!(file instanceof File)) return; - files.push(file); + + files.push([file, files.length]); return [file.name, file.type, file.size, file.lastModified, files.length - 1]; } }); + + // Sort small files to the front + files.sort(([a], [b]) => a.size - b.size); + + /** @type {Array} */ + const file_offsets = new Array(files.length); + let start = 0; + for (const [file, index] of files) { + file_offsets[index] = start; + start += file.size; + } + const encoded_file_offsets = JSON.stringify(file_offsets); + const length_buffer = new Uint8Array(4); const length_view = new DataView(length_buffer.buffer); length_view.setUint32(0, encoded_header.length, true); blob_parts.push(length_buffer.slice()); - length_view.setUint32(0, files.length, true); + length_view.setUint32(0, encoded_file_offsets.length, true); blob_parts.push(length_buffer); blob_parts.push(encoded_header); + blob_parts.push(encoded_file_offsets); - /** @type {Array<{ start: number, size: number, name: string }>} */ - const file_offsets = []; - let start = 1 + 4 + 4 + encoded_header.length; - for (const file of files) { + for (const [file] of files) { blob_parts.push(file); - file_offsets.push({ start, size: file.size, name: file.name }); - start += file.size; } + return { - blob: new Blob(blob_parts), - file_offsets + blob: new Blob(blob_parts) }; } @@ -133,21 +144,19 @@ export async function deserialize_binary_form(request) { const reader = request.body.getReader(); - /** @type {Array>} */ + /** @type {Array | undefined>>} */ const chunks = []; /** * @param {number} index - * @returns {Promise | null>} + * @returns {Promise | undefined>} */ async function get_chunk(index) { - if (chunks[index]) return chunks[index]; + if (index in chunks) return chunks[index]; + let i = chunks.length; while (i <= index) { - // TODO - this breaks when two chunks are read at once :( - const chunk = await reader.read(); - if (chunk.done) return null; - chunks[i] = chunk.value; + chunks[i] = reader.read().then((chunk) => chunk.value); i++; } return chunks[index]; @@ -208,56 +217,70 @@ export async function deserialize_binary_form(request) { } const header_view = new DataView(header.buffer); const data_length = header_view.getUint32(1, true); - const file_count = header_view.getUint32(5, true); + const file_offsets_length = header_view.getUint32(5, true); // Read the form data const data_buffer = await get_buffer(1 + 4 + 4, data_length); if (!data_buffer) throw new Error('Could not deserialize binary form: data too short'); - /** @type {Array} */ - const files = new Array(file_count); + // Read the file offset table + const file_offsets_buffer = await get_buffer(1 + 4 + 4 + data_length, file_offsets_length); + if (!file_offsets_buffer) + throw new Error('Could not deserialize binary form: file offset table too short'); + + const file_offsets = /** @type {Array} */ ( + JSON.parse(text_decoder.decode(file_offsets_buffer)) + ); + + const files_start_offset = 1 + 4 + 4 + data_length + file_offsets_length; const [data, meta] = devalue.parse(text_decoder.decode(data_buffer), { File: ([name, type, size, last_modified, index]) => { - const file = new LazyFile(name, type, size, last_modified); - files[index] = file; - return new Proxy(file, { - getPrototypeOf() { - // Trick validators into thinking this is a normal File - return File.prototype; + return new Proxy( + new LazyFile( + name, + type, + size, + last_modified, + get_chunk, + files_start_offset + file_offsets[index] + ), + { + getPrototypeOf() { + // Trick validators into thinking this is a normal File + return File.prototype; + } } - }); + ); } }); - let offset = 1 + 4 + 4 + data_length; - for (const file of files) { - const start = offset; - const end = start + file.size; - file._setup_internal(get_chunk, start); - offset = end; - } - return { data, meta, form_data: null }; } /** @implements {File} */ class LazyFile { /** @type {ReadableStream>} */ - // @ts-expect-error no equivalent to !: #stream; - /** @type {(index: number) => Promise | null>} */ - // @ts-expect-error no equivalent to !: + /** @type {(index: number) => Promise | undefined>} */ #get_chunk; /** @type {number} */ - // @ts-expect-error no equivalent to !: #offset; /** - * @param {(index: number) => Promise | null>} get_chunk + * @param {string} name + * @param {string} type + * @param {number} size + * @param {number} last_modified + * @param {(index: number) => Promise | undefined>} get_chunk * @param {number} offset */ - _setup_internal(get_chunk, offset) { - if (this.#stream) throw new TypeError('_setup_internal called twice'); + constructor(name, type, size, last_modified, get_chunk, offset) { + this.name = name; + this.type = type; + this.size = size; + this.lastModified = last_modified; + this.webkitRelativePath = ''; + let cursor = 0; let chunk_index = 0; this.#stream = new ReadableStream({ @@ -291,7 +314,7 @@ class LazyFile { chunk_index++; let chunk = await get_chunk(chunk_index); if (!chunk) { - controller.error('Could not deserialize binary form: incomplete data'); + controller.error('Could not deserialize binary form: incomplete file data'); controller.close(); return; } @@ -307,19 +330,7 @@ class LazyFile { }); this.#get_chunk = get_chunk; this.#offset = offset; - } - /** - * @param {string} name - * @param {string} type - * @param {number} size - * @param {number} last_modified - */ - constructor(name, type, size, last_modified) { - this.name = name; - this.type = type; - this.size = size; - this.lastModified = last_modified; - this.webkitRelativePath = ''; + // TODO - hacky, required for private members to be accessed on proxy this.arrayBuffer = this.arrayBuffer.bind(this); this.bytes = this.bytes.bind(this); @@ -355,9 +366,15 @@ class LazyFile { end = Math.min(end, this.size); } const size = Math.max(end - start, 0); - const file = new LazyFile(this.name, contentType, size, this.lastModified); + const file = new LazyFile( + this.name, + contentType, + size, + this.lastModified, + this.#get_chunk, + this.#offset + start + ); - file._setup_internal(this.#get_chunk, this.#offset + start); return file; } stream() { From e604470d4fb040c53a705bcb1eac8ac2607522e1 Mon Sep 17 00:00:00 2001 From: Ottomated Date: Wed, 22 Oct 2025 15:02:48 -0700 Subject: [PATCH 08/27] don't cache stream --- packages/kit/src/runtime/form-utils.js | 100 ++++++++++++------------- 1 file changed, 48 insertions(+), 52 deletions(-) diff --git a/packages/kit/src/runtime/form-utils.js b/packages/kit/src/runtime/form-utils.js index 65b59308ed3d..88f99db16c15 100644 --- a/packages/kit/src/runtime/form-utils.js +++ b/packages/kit/src/runtime/form-utils.js @@ -260,8 +260,6 @@ export async function deserialize_binary_form(request) { /** @implements {File} */ class LazyFile { - /** @type {ReadableStream>} */ - #stream; /** @type {(index: number) => Promise | undefined>} */ #get_chunk; /** @type {number} */ @@ -280,54 +278,6 @@ class LazyFile { this.size = size; this.lastModified = last_modified; this.webkitRelativePath = ''; - - let cursor = 0; - let chunk_index = 0; - this.#stream = new ReadableStream({ - start: async (controller) => { - let chunk_start = 0; - let start_chunk = null; - for (chunk_index = 0; ; chunk_index++) { - const chunk = await get_chunk(chunk_index); - if (!chunk) return null; - - const chunk_end = chunk_start + chunk.byteLength; - // If this chunk contains the target offset - if (offset >= chunk_start && offset < chunk_end) { - start_chunk = chunk; - break; - } - chunk_start = chunk_end; - } - // If the buffer is completely contained in one chunk, do a subarray - if (offset + this.size <= chunk_start + start_chunk.byteLength) { - controller.enqueue( - start_chunk.subarray(offset - chunk_start, offset + this.size - chunk_start) - ); - controller.close(); - } else { - controller.enqueue(start_chunk.subarray(offset - chunk_start)); - cursor = start_chunk.byteLength - offset + chunk_start; - } - }, - pull: async (controller) => { - chunk_index++; - let chunk = await get_chunk(chunk_index); - if (!chunk) { - controller.error('Could not deserialize binary form: incomplete file data'); - controller.close(); - return; - } - if (chunk.byteLength > this.size - cursor) { - chunk = chunk.subarray(0, this.size - cursor); - } - controller.enqueue(chunk); - cursor += chunk.byteLength; - if (cursor >= this.size) { - controller.close(); - } - } - }); this.#get_chunk = get_chunk; this.#offset = offset; @@ -341,7 +291,7 @@ class LazyFile { /** @type {ArrayBuffer | undefined} */ #buffer; async arrayBuffer() { - this.#buffer ??= await new Response(this.#stream).arrayBuffer(); + this.#buffer ??= await new Response(this.stream()).arrayBuffer(); return this.#buffer; } async bytes() { @@ -378,7 +328,53 @@ class LazyFile { return file; } stream() { - return this.#stream; + let cursor = 0; + let chunk_index = 0; + return new ReadableStream({ + start: async (controller) => { + let chunk_start = 0; + let start_chunk = null; + for (chunk_index = 0; ; chunk_index++) { + const chunk = await this.#get_chunk(chunk_index); + if (!chunk) return null; + + const chunk_end = chunk_start + chunk.byteLength; + // If this chunk contains the target offset + if (this.#offset >= chunk_start && this.#offset < chunk_end) { + start_chunk = chunk; + break; + } + chunk_start = chunk_end; + } + // If the buffer is completely contained in one chunk, do a subarray + if (this.#offset + this.size <= chunk_start + start_chunk.byteLength) { + controller.enqueue( + start_chunk.subarray(this.#offset - chunk_start, this.#offset + this.size - chunk_start) + ); + controller.close(); + } else { + controller.enqueue(start_chunk.subarray(this.#offset - chunk_start)); + cursor = start_chunk.byteLength - this.#offset + chunk_start; + } + }, + pull: async (controller) => { + chunk_index++; + let chunk = await this.#get_chunk(chunk_index); + if (!chunk) { + controller.error('Could not deserialize binary form: incomplete file data'); + controller.close(); + return; + } + if (chunk.byteLength > this.size - cursor) { + chunk = chunk.subarray(0, this.size - cursor); + } + controller.enqueue(chunk); + cursor += chunk.byteLength; + if (cursor >= this.size) { + controller.close(); + } + } + }); } async text() { return text_decoder.decode(await this.arrayBuffer()); From ecda6eaf9b3d0ddb43087c94739b1dc3c6f49a7a Mon Sep 17 00:00:00 2001 From: Ottomated Date: Wed, 22 Oct 2025 15:10:45 -0700 Subject: [PATCH 09/27] fix scoped ids --- .../src/runtime/client/remote-functions/form.svelte.js | 6 ------ packages/kit/src/runtime/form-utils.js | 4 ---- packages/kit/src/runtime/server/remote.js | 8 ++++---- packages/kit/src/types/internal.d.ts | 1 - 4 files changed, 4 insertions(+), 15 deletions(-) diff --git a/packages/kit/src/runtime/client/remote-functions/form.svelte.js b/packages/kit/src/runtime/client/remote-functions/form.svelte.js index f7e9cb3ae915..7f712d786526 100644 --- a/packages/kit/src/runtime/client/remote-functions/form.svelte.js +++ b/packages/kit/src/runtime/client/remote-functions/form.svelte.js @@ -683,12 +683,6 @@ function clone(element) { */ function validate_form_data(form_data, enctype) { for (const key of form_data.keys()) { - if (key.startsWith('sveltekit:')) { - throw new Error( - 'FormData keys starting with `sveltekit:` are reserved for internal use and should not be set manually' - ); - } - if (/^\$[.[]?/.test(key)) { throw new Error( '`$` is used to collect all FormData validation issues and cannot be used as the `name` of a form control' diff --git a/packages/kit/src/runtime/form-utils.js b/packages/kit/src/runtime/form-utils.js index 88f99db16c15..5302b94fc3e7 100644 --- a/packages/kit/src/runtime/form-utils.js +++ b/packages/kit/src/runtime/form-utils.js @@ -33,10 +33,6 @@ export function convert_formdata(data) { const result = {}; for (let key of data.keys()) { - if (key.startsWith('sveltekit:')) { - continue; - } - const is_array = key.endsWith('[]'); /** @type {any[]} */ let values = data.getAll(key); diff --git a/packages/kit/src/runtime/server/remote.js b/packages/kit/src/runtime/server/remote.js index abc2d1904cf6..f7a960a49ff5 100644 --- a/packages/kit/src/runtime/server/remote.js +++ b/packages/kit/src/runtime/server/remote.js @@ -120,8 +120,8 @@ async function handle_remote_call_internal(event, state, options, manifest, id) const { data, meta, form_data } = await deserialize_binary_form(event.request); // If this is a keyed form instance (created via form.for(key)), add the key to the form data (unless already set) - if (additional_args) { - meta.id = decodeURIComponent(additional_args); + if (additional_args && !('id' in data)) { + data.id = JSON.parse(decodeURIComponent(additional_args)); } const fn = info.fn; @@ -294,8 +294,8 @@ async function handle_remote_form_post_internal(event, state, manifest, id) { const fn = /** @type {RemoteInfo & { type: 'form' }} */ (/** @type {any} */ (form).__).fn; const { data, meta, form_data } = await deserialize_binary_form(event.request); - if (action_id && !data.id) { - meta.id = decodeURIComponent(action_id); + if (action_id && !('id' in data)) { + data.id = JSON.parse(decodeURIComponent(action_id)); } await with_request_store({ event, state }, () => fn(data, meta, form_data)); diff --git a/packages/kit/src/types/internal.d.ts b/packages/kit/src/types/internal.d.ts index d9abcc2d53d6..a632917174cd 100644 --- a/packages/kit/src/types/internal.d.ts +++ b/packages/kit/src/types/internal.d.ts @@ -553,7 +553,6 @@ export type ValidatedKitConfig = Omit, 'adapter'> & }; export type BinaryFormMeta = { - id?: string; remote_refreshes?: string[]; validate_only?: boolean; pathname?: string; From 238dd9acbd0e07e928e19f21a217d1440cd21c27 Mon Sep 17 00:00:00 2001 From: Ottomated Date: Wed, 22 Oct 2025 16:19:40 -0700 Subject: [PATCH 10/27] tests --- packages/kit/src/runtime/form-utils.spec.js | 88 ++++++++++++++++++- .../remote/form/file-upload/+page.svelte | 2 + .../remote/form/file-upload/form.remote.ts | 9 +- packages/kit/test/apps/basics/test/test.js | 23 +++++ 4 files changed, 113 insertions(+), 9 deletions(-) diff --git a/packages/kit/src/runtime/form-utils.spec.js b/packages/kit/src/runtime/form-utils.spec.js index 77b7e355f4a8..20a6bd0ad33a 100644 --- a/packages/kit/src/runtime/form-utils.spec.js +++ b/packages/kit/src/runtime/form-utils.spec.js @@ -1,5 +1,12 @@ import { describe, expect, test } from 'vitest'; -import { convert_formdata, split_path } from './form-utils.js'; +import { + BINARY_FORM_CONTENT_TYPE, + convert_formdata, + deserialize_binary_form, + serialize_binary_form, + split_path +} from './form-utils.js'; +import { text_decoder } from './utils.js'; describe('split_path', () => { const good = [ @@ -90,3 +97,82 @@ describe('convert_formdata', () => { }); } }); + +describe('binary form serializer', () => { + test.each([ + { + data: {}, + meta: {} + }, + { + data: { foo: 'foo', nested: { prop: 'prop' } }, + meta: { pathname: '/foo', validate_only: true } + } + ])('simple', async (input) => { + const { blob } = serialize_binary_form(input.data, input.meta); + const res = await deserialize_binary_form( + new Request('http://test', { + method: 'POST', + body: blob, + headers: { + 'Content-Type': BINARY_FORM_CONTENT_TYPE + } + }) + ); + expect(res.form_data).toBeNull(); + expect(res.data).toEqual(input.data); + expect(res.meta).toEqual(input.meta ?? {}); + }); + test('file uploads', async () => { + const { blob } = serialize_binary_form( + { + small: new File(['a'], 'a.txt', { type: 'text/plain' }), + large: new File([new Uint8Array(1024).fill('a'.charCodeAt(0))], 'large.txt', { + type: 'text/plain', + lastModified: 100 + }) + }, + {} + ); + // Split the stream into 1 byte chunks to make sure all the chunking deserialization works + const stream = blob.stream().pipeThrough( + new TransformStream({ + transform(chunk, controller) { + for (const byte of chunk) { + controller.enqueue(new Uint8Array([byte])); + } + } + }) + ); + const res = await deserialize_binary_form( + new Request('http://test', { + method: 'POST', + body: stream, + // @ts-expect-error duplex required in node + duplex: 'half', + headers: { + 'Content-Type': BINARY_FORM_CONTENT_TYPE + } + }) + ); + const { small, large } = res.data; + expect(small.name).toBe('a.txt'); + expect(small.type).toBe('text/plain'); + expect(small.size).toBe(1); + expect(await small.text()).toBe('a'); + + expect(large.name).toBe('large.txt'); + expect(large.type).toBe('text/plain'); + expect(large.size).toBe(1024); + expect(large.lastModified).toBe(100); + const buffer = new Uint8Array(large.size); + let cursor = 0; + for await (const chunk of large.stream()) { + buffer.set(chunk, cursor); + cursor += chunk.byteLength; + } + expect(buffer).toEqual(new Uint8Array(1024).fill('a'.charCodeAt(0))); + // text should be callable after stream is consumed + expect(await large.text()).toBe('a'.repeat(1024)); + }); +}); diff --git a/packages/kit/test/apps/basics/src/routes/remote/form/file-upload/+page.svelte b/packages/kit/test/apps/basics/src/routes/remote/form/file-upload/+page.svelte index 67503c3693e7..388dfcf26cf5 100644 --- a/packages/kit/test/apps/basics/src/routes/remote/form/file-upload/+page.svelte +++ b/packages/kit/test/apps/basics/src/routes/remote/form/file-upload/+page.svelte @@ -10,3 +10,5 @@ + +
{JSON.stringify(upload.result)}
diff --git a/packages/kit/test/apps/basics/src/routes/remote/form/file-upload/form.remote.ts b/packages/kit/test/apps/basics/src/routes/remote/form/file-upload/form.remote.ts index 5ef57e146646..be0e0c014090 100644 --- a/packages/kit/test/apps/basics/src/routes/remote/form/file-upload/form.remote.ts +++ b/packages/kit/test/apps/basics/src/routes/remote/form/file-upload/form.remote.ts @@ -8,13 +8,6 @@ export const upload = form( file2: v.file() }), async (data) => { - console.log(data.text); - console.log(data.file1); - console.log(data.file2); - console.log(await data.file1.text()); - console.log(await data.file2.text()); - // for await (const chunk of data.file1.stream()) { - // console.log('file 1', chunk.length); - // } + return { text: data.text, file1: await data.file1.text(), file2: await data.file2.text() }; } ); diff --git a/packages/kit/test/apps/basics/test/test.js b/packages/kit/test/apps/basics/test/test.js index 9b57a59b79d7..f5b76d97237a 100644 --- a/packages/kit/test/apps/basics/test/test.js +++ b/packages/kit/test/apps/basics/test/test.js @@ -2018,6 +2018,29 @@ test.describe('remote functions', () => { await page.fill('input', 'hello'); await expect(page.locator('select')).toHaveValue('one'); }); + test('file uploads work', async ({ page }) => { + await page.goto('/remote/form/file-upload'); + + await page.locator('input[name="file1"]').setInputFiles({ + name: 'a.txt', + mimeType: 'text/plain', + buffer: Buffer.from('a') + }); + await page.locator('input[name="file2"]').setInputFiles({ + name: 'b.txt', + mimeType: 'text/plain', + buffer: Buffer.from('b') + }); + await page.locator('button').click(); + + await expect(page.locator('pre')).toHaveText( + JSON.stringify({ + text: 'Hello world', + file1: 'a', + file2: 'b' + }) + ); + }); }); test.describe('params prop', () => { From 5d2c8a530919a141451d7fa83413810dd14ceff4 Mon Sep 17 00:00:00 2001 From: Ottomated Date: Wed, 22 Oct 2025 16:22:23 -0700 Subject: [PATCH 11/27] re-add comment --- packages/kit/src/runtime/app/server/remote/form.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/kit/src/runtime/app/server/remote/form.js b/packages/kit/src/runtime/app/server/remote/form.js index b89944362449..910ecb42c80b 100644 --- a/packages/kit/src/runtime/app/server/remote/form.js +++ b/packages/kit/src/runtime/app/server/remote/form.js @@ -289,6 +289,8 @@ export function form(validate_or_fn, maybe_fn) { function handle_issues(output, issues, form_data) { output.issues = issues.map((issue) => normalize_issue(issue, true)); + // if it was a progressively-enhanced submission, we don't need + // to return the input — it's already there if (form_data) { output.input = {}; From cd106a2f342c987c1b0461f3ca1abdcaf54c4e37 Mon Sep 17 00:00:00 2001 From: Ottomated Date: Wed, 22 Oct 2025 16:27:59 -0700 Subject: [PATCH 12/27] move location & pathname back to headers --- .../runtime/client/remote-functions/form.svelte.js | 14 +++++++------- packages/kit/src/runtime/form-utils.spec.js | 1 - packages/kit/src/types/internal.d.ts | 2 -- 3 files changed, 7 insertions(+), 10 deletions(-) diff --git a/packages/kit/src/runtime/client/remote-functions/form.svelte.js b/packages/kit/src/runtime/client/remote-functions/form.svelte.js index 7f712d786526..76f9afb27f3c 100644 --- a/packages/kit/src/runtime/client/remote-functions/form.svelte.js +++ b/packages/kit/src/runtime/client/remote-functions/form.svelte.js @@ -185,9 +185,7 @@ export function form(id) { await Promise.resolve(); const { blob } = serialize_binary_form(convert(data), { - remote_refreshes: updates.map((u) => u._key), - pathname: location.pathname, - search: location.search + remote_refreshes: updates.map((u) => u._key) }); // TODO - check this - does it block the event loop? @@ -218,6 +216,8 @@ export function form(id) { // } xhr.open('POST', `${base}/${app_dir}/remote/${action_id}`); xhr.setRequestHeader('Content-Type', BINARY_FORM_CONTENT_TYPE); + xhr.setRequestHeader('x-sveltekit-pathname', location.pathname); + xhr.setRequestHeader('x-sveltekit-search', location.search); xhr.send(blob); }); @@ -585,12 +585,12 @@ export function form(id) { const response = await fetch(`${base}/${app_dir}/remote/${action_id}`, { method: 'POST', headers: { - 'Content-Type': BINARY_FORM_CONTENT_TYPE + 'Content-Type': BINARY_FORM_CONTENT_TYPE, + 'x-sveltekit-pathname': location.pathname, + 'x-sveltekit-search': location.search }, body: serialize_binary_form(data, { - validate_only: true, - pathname: location.pathname, - search: location.search + validate_only: true }).blob }); diff --git a/packages/kit/src/runtime/form-utils.spec.js b/packages/kit/src/runtime/form-utils.spec.js index 20a6bd0ad33a..683097d05ad0 100644 --- a/packages/kit/src/runtime/form-utils.spec.js +++ b/packages/kit/src/runtime/form-utils.spec.js @@ -6,7 +6,6 @@ import { serialize_binary_form, split_path } from './form-utils.js'; -import { text_decoder } from './utils.js'; describe('split_path', () => { const good = [ diff --git a/packages/kit/src/types/internal.d.ts b/packages/kit/src/types/internal.d.ts index a632917174cd..6384201af551 100644 --- a/packages/kit/src/types/internal.d.ts +++ b/packages/kit/src/types/internal.d.ts @@ -555,8 +555,6 @@ export type ValidatedKitConfig = Omit, 'adapter'> & export type BinaryFormMeta = { remote_refreshes?: string[]; validate_only?: boolean; - pathname?: string; - search?: string; }; export type RemoteInfo = From b4d41f730facabbae37405021c9a206e94e7b723 Mon Sep 17 00:00:00 2001 From: Ottomated Date: Wed, 22 Oct 2025 16:48:22 -0700 Subject: [PATCH 13/27] skip test on node 18 --- packages/kit/src/runtime/form-utils.js | 4 ++++ packages/kit/src/runtime/form-utils.spec.js | 6 ++++++ 2 files changed, 10 insertions(+) diff --git a/packages/kit/src/runtime/form-utils.js b/packages/kit/src/runtime/form-utils.js index 5302b94fc3e7..12434e92f0a0 100644 --- a/packages/kit/src/runtime/form-utils.js +++ b/packages/kit/src/runtime/form-utils.js @@ -83,6 +83,10 @@ export function serialize_binary_form(data, meta) { /** @type {Array<[file: File, index: number]>} */ const files = []; + if (!meta.remote_refreshes?.length) { + delete meta.remote_refreshes; + } + const encoded_header = devalue.stringify([data, meta], { File: (file) => { if (!(file instanceof File)) return; diff --git a/packages/kit/src/runtime/form-utils.spec.js b/packages/kit/src/runtime/form-utils.spec.js index 683097d05ad0..ccb9c0b6aae9 100644 --- a/packages/kit/src/runtime/form-utils.spec.js +++ b/packages/kit/src/runtime/form-utils.spec.js @@ -123,6 +123,12 @@ describe('binary form serializer', () => { expect(res.meta).toEqual(input.meta ?? {}); }); test('file uploads', async () => { + // eslint-disable-next-line n/prefer-global/process + const major = process.versions.node.split('.', 2).map((str) => +str)[0]; + if (major < 20) { + // TODO: remove after dropping support for Node < 20 + return; + } const { blob } = serialize_binary_form( { small: new File(['a'], 'a.txt', { type: 'text/plain' }), From 2284b9fe1703abdd787484b82756a98a251539a6 Mon Sep 17 00:00:00 2001 From: Ottomated Date: Wed, 22 Oct 2025 16:58:26 -0700 Subject: [PATCH 14/27] changeset --- .changeset/new-rivers-run.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/new-rivers-run.md diff --git a/.changeset/new-rivers-run.md b/.changeset/new-rivers-run.md new file mode 100644 index 000000000000..257e21a14651 --- /dev/null +++ b/.changeset/new-rivers-run.md @@ -0,0 +1,5 @@ +--- +'@sveltejs/kit': minor +--- + +feat: File uploads inside `form` remote functions are now streamed - form data can be accessed before large files finish uploading. From bcd016b140431ce31c0e456faba7ef3f263e0bc4 Mon Sep 17 00:00:00 2001 From: Ottomated Date: Wed, 22 Oct 2025 17:31:10 -0700 Subject: [PATCH 15/27] polyfill file for node 18 test --- packages/kit/src/runtime/form-utils.spec.js | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/packages/kit/src/runtime/form-utils.spec.js b/packages/kit/src/runtime/form-utils.spec.js index ccb9c0b6aae9..25b589baa705 100644 --- a/packages/kit/src/runtime/form-utils.spec.js +++ b/packages/kit/src/runtime/form-utils.spec.js @@ -1,4 +1,4 @@ -import { describe, expect, test } from 'vitest'; +import { beforeAll, describe, expect, test } from 'vitest'; import { BINARY_FORM_CONTENT_TYPE, convert_formdata, @@ -6,6 +6,7 @@ import { serialize_binary_form, split_path } from './form-utils.js'; +import buffer from 'node:buffer'; describe('split_path', () => { const good = [ @@ -98,6 +99,13 @@ describe('convert_formdata', () => { }); describe('binary form serializer', () => { + beforeAll(() => { + // TODO: remove after dropping support for Node 18 + if (!('File' in globalThis)) { + // @ts-ignore + globalThis.File = buffer.File; + } + }); test.each([ { data: {}, @@ -123,12 +131,6 @@ describe('binary form serializer', () => { expect(res.meta).toEqual(input.meta ?? {}); }); test('file uploads', async () => { - // eslint-disable-next-line n/prefer-global/process - const major = process.versions.node.split('.', 2).map((str) => +str)[0]; - if (major < 20) { - // TODO: remove after dropping support for Node < 20 - return; - } const { blob } = serialize_binary_form( { small: new File(['a'], 'a.txt', { type: 'text/plain' }), From d6e684d5cbb9a7a7a6c3153ffeb4507729149471 Mon Sep 17 00:00:00 2001 From: Ottomated Date: Wed, 22 Oct 2025 18:25:23 -0700 Subject: [PATCH 16/27] fix refreshes --- packages/kit/src/runtime/server/remote.js | 29 +++++++++++------------ 1 file changed, 14 insertions(+), 15 deletions(-) diff --git a/packages/kit/src/runtime/server/remote.js b/packages/kit/src/runtime/server/remote.js index f7a960a49ff5..f68a94460ac9 100644 --- a/packages/kit/src/runtime/server/remote.js +++ b/packages/kit/src/runtime/server/remote.js @@ -131,10 +131,7 @@ async function handle_remote_call_internal(event, state, options, manifest, id) /** @type {RemoteFunctionResponse} */ ({ type: 'result', result: stringify(result, transport), - refreshes: - result.issues || !meta.remote_refreshes - ? {} - : await serialize_refreshes(meta.remote_refreshes) + refreshes: result.issues ? undefined : await serialize_refreshes(meta.remote_refreshes) }) ); } @@ -178,7 +175,7 @@ async function handle_remote_call_internal(event, state, options, manifest, id) /** @type {RemoteFunctionResponse} */ ({ type: 'redirect', location: error.location, - refreshes: await serialize_refreshes(form_client_refreshes ?? []) + refreshes: await serialize_refreshes(form_client_refreshes) }) ); } @@ -204,24 +201,26 @@ async function handle_remote_call_internal(event, state, options, manifest, id) } /** - * @param {string[]} client_refreshes + * @param {string[]=} client_refreshes */ async function serialize_refreshes(client_refreshes) { const refreshes = state.refreshes ?? {}; - for (const key of client_refreshes) { - if (refreshes[key] !== undefined) continue; + if (client_refreshes) { + for (const key of client_refreshes) { + if (refreshes[key] !== undefined) continue; - const [hash, name, payload] = key.split('/'); + const [hash, name, payload] = key.split('/'); - const loader = manifest._.remotes[hash]; - const fn = (await loader?.())?.default?.[name]; + const loader = manifest._.remotes[hash]; + const fn = (await loader?.())?.default?.[name]; - if (!fn) error(400, 'Bad Request'); + if (!fn) error(400, 'Bad Request'); - refreshes[key] = with_request_store({ event, state }, () => - fn(parse_remote_arg(payload, transport)) - ); + refreshes[key] = with_request_store({ event, state }, () => + fn(parse_remote_arg(payload, transport)) + ); + } } if (Object.keys(refreshes).length === 0) { From c31ff7c2c9e775f366f2e5178a8afa3a6ecf59c1 Mon Sep 17 00:00:00 2001 From: Ottomated Date: Wed, 22 Oct 2025 18:55:15 -0700 Subject: [PATCH 17/27] optimize file offset table --- packages/kit/src/runtime/form-utils.js | 62 +++++++++++++++----------- 1 file changed, 35 insertions(+), 27 deletions(-) diff --git a/packages/kit/src/runtime/form-utils.js b/packages/kit/src/runtime/form-utils.js index 12434e92f0a0..03aee5819d0b 100644 --- a/packages/kit/src/runtime/form-utils.js +++ b/packages/kit/src/runtime/form-utils.js @@ -69,9 +69,9 @@ const BINARY_FORM_VERSION = 0; * The binary format is as follows: * - 1 byte: Format version * - 4 bytes: Length of the header (u32) - * - 4 bytes: Length of the file offset table (u32) + * - 2 bytes: Length of the file offset table (u16) * - header: devalue.stringify([data, meta]) - * - file offset table: JSON.stringify([offset1, offset2, ...]) (offsets start from the end of the table) + * - file offset table: JSON.stringify([offset1, offset2, ...]) (empty if not files) (offsets start from the end of the table) * - file1, file2, ... * @param {Record} data * @param {BinaryFormMeta} meta @@ -96,17 +96,20 @@ export function serialize_binary_form(data, meta) { } }); - // Sort small files to the front - files.sort(([a], [b]) => a.size - b.size); - - /** @type {Array} */ - const file_offsets = new Array(files.length); - let start = 0; - for (const [file, index] of files) { - file_offsets[index] = start; - start += file.size; + let encoded_file_offsets = ''; + if (files.length) { + // Sort small files to the front + files.sort(([a], [b]) => a.size - b.size); + + /** @type {Array} */ + const file_offsets = new Array(files.length); + let start = 0; + for (const [file, index] of files) { + file_offsets[index] = start; + start += file.size; + } + encoded_file_offsets = JSON.stringify(file_offsets); } - const encoded_file_offsets = JSON.stringify(file_offsets); const length_buffer = new Uint8Array(4); const length_view = new DataView(length_buffer.buffer); @@ -114,8 +117,8 @@ export function serialize_binary_form(data, meta) { length_view.setUint32(0, encoded_header.length, true); blob_parts.push(length_buffer.slice()); - length_view.setUint32(0, encoded_file_offsets.length, true); - blob_parts.push(length_buffer); + length_view.setUint16(0, encoded_file_offsets.length, true); + blob_parts.push(length_buffer.slice(0, 2)); blob_parts.push(encoded_header); blob_parts.push(encoded_file_offsets); @@ -207,7 +210,7 @@ export async function deserialize_binary_form(request) { return buffer; } - const header = await get_buffer(0, 1 + 4 + 4); + const header = await get_buffer(0, 1 + 4 + 2); if (!header) throw new Error('Could not deserialize binary form: too short'); if (header[0] !== BINARY_FORM_VERSION) { @@ -217,22 +220,27 @@ export async function deserialize_binary_form(request) { } const header_view = new DataView(header.buffer); const data_length = header_view.getUint32(1, true); - const file_offsets_length = header_view.getUint32(5, true); + const file_offsets_length = header_view.getUint16(5, true); // Read the form data - const data_buffer = await get_buffer(1 + 4 + 4, data_length); + const data_buffer = await get_buffer(1 + 4 + 2, data_length); if (!data_buffer) throw new Error('Could not deserialize binary form: data too short'); - // Read the file offset table - const file_offsets_buffer = await get_buffer(1 + 4 + 4 + data_length, file_offsets_length); - if (!file_offsets_buffer) - throw new Error('Could not deserialize binary form: file offset table too short'); - - const file_offsets = /** @type {Array} */ ( - JSON.parse(text_decoder.decode(file_offsets_buffer)) - ); - - const files_start_offset = 1 + 4 + 4 + data_length + file_offsets_length; + /** @type {Array} */ + let file_offsets; + /** @type {number} */ + let files_start_offset; + if (file_offsets_length > 0) { + // Read the file offset table + const file_offsets_buffer = await get_buffer(1 + 4 + 2 + data_length, file_offsets_length); + if (!file_offsets_buffer) + throw new Error('Could not deserialize binary form: file offset table too short'); + + file_offsets = /** @type {Array} */ ( + JSON.parse(text_decoder.decode(file_offsets_buffer)) + ); + files_start_offset = 1 + 4 + 2 + data_length + file_offsets_length; + } const [data, meta] = devalue.parse(text_decoder.decode(data_buffer), { File: ([name, type, size, last_modified, index]) => { From 9e4853cbb3eac4324f89c7d362882c35435197c9 Mon Sep 17 00:00:00 2001 From: Ottomated Date: Wed, 22 Oct 2025 18:55:37 -0700 Subject: [PATCH 18/27] typo --- packages/kit/src/runtime/form-utils.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/kit/src/runtime/form-utils.js b/packages/kit/src/runtime/form-utils.js index 03aee5819d0b..51375cedb9e3 100644 --- a/packages/kit/src/runtime/form-utils.js +++ b/packages/kit/src/runtime/form-utils.js @@ -71,7 +71,7 @@ const BINARY_FORM_VERSION = 0; * - 4 bytes: Length of the header (u32) * - 2 bytes: Length of the file offset table (u16) * - header: devalue.stringify([data, meta]) - * - file offset table: JSON.stringify([offset1, offset2, ...]) (empty if not files) (offsets start from the end of the table) + * - file offset table: JSON.stringify([offset1, offset2, ...]) (empty if no files) (offsets start from the end of the table) * - file1, file2, ... * @param {Record} data * @param {BinaryFormMeta} meta From 86ec52af15629821e6a63af39a820abad3390e62 Mon Sep 17 00:00:00 2001 From: Ottomated Date: Thu, 23 Oct 2025 12:35:21 -0700 Subject: [PATCH 19/27] add lazyfile tests --- packages/kit/src/runtime/form-utils.spec.js | 31 +++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/packages/kit/src/runtime/form-utils.spec.js b/packages/kit/src/runtime/form-utils.spec.js index 25b589baa705..88409435cbe0 100644 --- a/packages/kit/src/runtime/form-utils.spec.js +++ b/packages/kit/src/runtime/form-utils.spec.js @@ -7,6 +7,7 @@ import { split_path } from './form-utils.js'; import buffer from 'node:buffer'; +import { text_encoder } from './utils.js'; describe('split_path', () => { const good = [ @@ -182,4 +183,34 @@ describe('binary form serializer', () => { // text should be callable after stream is consumed expect(await large.text()).toBe('a'.repeat(1024)); }); + test('LazyFile methods', async () => { + const { blob } = serialize_binary_form( + { + file: new File(['Hello World'], 'a.txt') + }, + {} + ); + const res = await deserialize_binary_form( + new Request('http://test', { + method: 'POST', + body: blob, + headers: { + 'Content-Type': BINARY_FORM_CONTENT_TYPE + } + }) + ); + /** @type {File} */ + const file = res.data.file; + const expected = text_encoder.encode('Hello World'); + expect(await file.text()).toBe('Hello World'); + expect(await file.arrayBuffer()).toEqual(expected.buffer); + expect(await file.bytes()).toEqual(expected); + expect(await new Response(file.stream()).arrayBuffer()).toEqual(expected.buffer); + const ello_slice = file.slice(1, 5, 'test/content-type'); + expect(ello_slice.type).toBe('test/content-type'); + expect(await ello_slice.text()).toBe('ello'); + const world_slice = file.slice(-5); + expect(await world_slice.text()).toBe('World'); + expect(world_slice.type).toBe(file.type); + }); }); From aea26e0637d6bd602662aab14827630b0116206f Mon Sep 17 00:00:00 2001 From: Ottomated Date: Fri, 31 Oct 2025 17:50:51 -0700 Subject: [PATCH 20/27] avoid double-sending form keys --- .../kit/src/runtime/client/remote-functions/form.svelte.js | 6 ++++-- packages/kit/src/runtime/server/remote.js | 1 + 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/packages/kit/src/runtime/client/remote-functions/form.svelte.js b/packages/kit/src/runtime/client/remote-functions/form.svelte.js index 76f9afb27f3c..f4d132c0418c 100644 --- a/packages/kit/src/runtime/client/remote-functions/form.svelte.js +++ b/packages/kit/src/runtime/client/remote-functions/form.svelte.js @@ -57,6 +57,7 @@ export function form(id) { /** @param {string | number | boolean} [key] */ function create_instance(key) { + const action_id_without_key = id; const action_id = id + (key != undefined ? `/${JSON.stringify(key)}` : ''); const action = '?/remote=' + encodeURIComponent(action_id); @@ -214,7 +215,8 @@ export function form(id) { // } // }); // } - xhr.open('POST', `${base}/${app_dir}/remote/${action_id}`); + // Use `action_id_without_key` here because the id is included in the body via `convert(data)` + xhr.open('POST', `${base}/${app_dir}/remote/${action_id_without_key}`); xhr.setRequestHeader('Content-Type', BINARY_FORM_CONTENT_TYPE); xhr.setRequestHeader('x-sveltekit-pathname', location.pathname); xhr.setRequestHeader('x-sveltekit-search', location.search); @@ -582,7 +584,7 @@ export function form(id) { if (validated?.issues) { array = validated.issues.map((issue) => normalize_issue(issue, false)); } else if (!preflightOnly) { - const response = await fetch(`${base}/${app_dir}/remote/${action_id}`, { + const response = await fetch(`${base}/${app_dir}/remote/${action_id_without_key}`, { method: 'POST', headers: { 'Content-Type': BINARY_FORM_CONTENT_TYPE, diff --git a/packages/kit/src/runtime/server/remote.js b/packages/kit/src/runtime/server/remote.js index f68a94460ac9..d52b4be8d87b 100644 --- a/packages/kit/src/runtime/server/remote.js +++ b/packages/kit/src/runtime/server/remote.js @@ -120,6 +120,7 @@ async function handle_remote_call_internal(event, state, options, manifest, id) const { data, meta, form_data } = await deserialize_binary_form(event.request); // If this is a keyed form instance (created via form.for(key)), add the key to the form data (unless already set) + // Note that additional_args will only be set if the form is not enhanced, as enhanced forms transfer the key inside `data`. if (additional_args && !('id' in data)) { data.id = JSON.parse(decodeURIComponent(additional_args)); } From ca9c53c109eae4ea74ec24dee1f4c5a6a1e8a951 Mon Sep 17 00:00:00 2001 From: Ottomated Date: Sat, 1 Nov 2025 20:41:59 -0700 Subject: [PATCH 21/27] remove xhr for next PR --- .../client/remote-functions/form.svelte.js | 67 +++++-------------- 1 file changed, 15 insertions(+), 52 deletions(-) diff --git a/packages/kit/src/runtime/client/remote-functions/form.svelte.js b/packages/kit/src/runtime/client/remote-functions/form.svelte.js index f4d132c0418c..230721a71c4f 100644 --- a/packages/kit/src/runtime/client/remote-functions/form.svelte.js +++ b/packages/kit/src/runtime/client/remote-functions/form.svelte.js @@ -189,60 +189,23 @@ export function form(id) { remote_refreshes: updates.map((u) => u._key) }); - // TODO - check this - does it block the event loop? - // TODO - extract XHR to a function and reuse for validate_only - const response = await new Promise((resolve, reject) => { - const xhr = new XMLHttpRequest(); - xhr.addEventListener('error', () => { - reject(new Error('Failed to execute remote function')); - }); - xhr.addEventListener('readystatechange', () => { - if (xhr.readyState === 2 /* HEADERS_RECEIVED */) { - if (xhr.status !== 200) { - reject(new Error('Failed to execute remote function')); - } - } else if (xhr.readyState === 4 /* DONE */) { - resolve(xhr.responseText); - } - }); - // if (file_offsets.length) { - // xhr.upload.addEventListener('progress', (ev) => { - // for (const file of file_offsets) { - // const progress = (ev.loaded - file.start) / file.size; - // if (progress < 0 || progress > 1) continue; - - // console.log(`File ${file.name}: ${(progress * 100).toFixed(2)}%`); - // } - // }); - // } - // Use `action_id_without_key` here because the id is included in the body via `convert(data)` - xhr.open('POST', `${base}/${app_dir}/remote/${action_id_without_key}`); - xhr.setRequestHeader('Content-Type', BINARY_FORM_CONTENT_TYPE); - xhr.setRequestHeader('x-sveltekit-pathname', location.pathname); - xhr.setRequestHeader('x-sveltekit-search', location.search); - xhr.send(blob); + const response = await fetch(`${base}/${app_dir}/remote/${action_id_without_key}`, { + method: 'POST', + headers: { + 'Content-Type': BINARY_FORM_CONTENT_TYPE, + 'x-sveltekit-pathname': location.pathname, + 'x-sveltekit-search': location.search + }, + body: blob }); - // const response = await fetch(`${base}/${app_dir}/remote/${action_id}`, { - // method: 'POST', - // headers: { - // 'Content-Type': BINARY_FORM_CONTENT_TYPE - // }, - // body: serialize_binary_form(convert(data), { - // remote_refreshes: updates.map((u) => u._key), - // pathname: location.pathname, - // search: location.search - // }) - // }); - - // if (!response.ok) { - // // We only end up here in case of a network error or if the server has an internal error - // // (which shouldn't happen because we handle errors on the server and always send a 200 response) - // throw new Error('Failed to execute remote function'); - // } - - // const form_result = /** @type { RemoteFunctionResponse} */ (await response.json()); - const form_result = /** @type { RemoteFunctionResponse} */ (JSON.parse(response)); + if (!response.ok) { + // We only end up here in case of a network error or if the server has an internal error + // (which shouldn't happen because we handle errors on the server and always send a 200 response) + throw new Error('Failed to execute remote function'); + } + + const form_result = /** @type { RemoteFunctionResponse} */ (await response.json()); if (form_result.type === 'result') { ({ issues: raw_issues = [], result } = devalue.parse(form_result.result, app.decoders)); From 0c1157c300fd79feb5f5c7e0db11ab9b7eed06d8 Mon Sep 17 00:00:00 2001 From: Ottomated Date: Sun, 2 Nov 2025 00:17:28 -0700 Subject: [PATCH 22/27] initial upload progress --- packages/kit/src/exports/public.d.ts | 7 +- .../kit/src/runtime/app/server/remote/form.js | 3 +- .../client/remote-functions/form.svelte.js | 78 +++++++++++++----- packages/kit/src/runtime/form-utils.js | 79 ++++++++++++++++--- .../remote/form/file-upload/+page.svelte | 8 +- .../remote/form/file-upload/form.remote.ts | 10 ++- packages/kit/types/index.d.ts | 7 +- 7 files changed, 154 insertions(+), 38 deletions(-) diff --git a/packages/kit/src/exports/public.d.ts b/packages/kit/src/exports/public.d.ts index 3516689dce67..fb87dbed168a 100644 --- a/packages/kit/src/exports/public.d.ts +++ b/packages/kit/src/exports/public.d.ts @@ -1880,7 +1880,12 @@ type RemoteFormFieldMethods = { set(input: T): T; /** Validation issues, if any */ issues(): RemoteFormIssue[] | undefined; -}; +} & (T extends File + ? { + /** Current upload progress, from 0 to 1 */ + progress(): number; + } + : object); export type RemoteFormFieldValue = string | string[] | number | boolean | File | File[]; diff --git a/packages/kit/src/runtime/app/server/remote/form.js b/packages/kit/src/runtime/app/server/remote/form.js index 248409e3aef4..d6b3f73342a5 100644 --- a/packages/kit/src/runtime/app/server/remote/form.js +++ b/packages/kit/src/runtime/app/server/remote/form.js @@ -213,7 +213,8 @@ export function form(validate_or_fn, maybe_fn) { (get_cache(__)[''] ??= {}).input = input; }, - () => issues + () => issues, + () => 0 /* upload progress is always 0 on the server */ ); } }); diff --git a/packages/kit/src/runtime/client/remote-functions/form.svelte.js b/packages/kit/src/runtime/client/remote-functions/form.svelte.js index 230721a71c4f..2a90678bec77 100644 --- a/packages/kit/src/runtime/client/remote-functions/form.svelte.js +++ b/packages/kit/src/runtime/client/remote-functions/form.svelte.js @@ -20,7 +20,9 @@ import { build_path_string, normalize_issue, serialize_binary_form, - BINARY_FORM_CONTENT_TYPE + BINARY_FORM_CONTENT_TYPE, + deep_get, + get_file_paths } from '../../form-utils.js'; /** @@ -66,6 +68,11 @@ export function form(id) { */ let input = $state({}); + /** + * @type {Record} + */ + let upload_progress = $state({}); + /** @type {InternalRemoteFormIssue[]} */ let raw_issues = $state.raw([]); @@ -160,10 +167,10 @@ export function form(id) { } /** - * @param {FormData} data + * @param {FormData} form_data * @returns {Promise & { updates: (...args: any[]) => any }} */ - function submit(data) { + function submit(form_data) { // Store a reference to the current instance and increment the usage count for the duration // of the request. This ensures that the instance is not deleted in case of an optimistic update // (e.g. when deleting an item in a list) that fails and wants to surface an error to the user afterwards. @@ -185,27 +192,55 @@ export function form(id) { try { await Promise.resolve(); - const { blob } = serialize_binary_form(convert(data), { + const data = convert(form_data); + + const { blob, file_offsets } = serialize_binary_form(data, { remote_refreshes: updates.map((u) => u._key) }); - const response = await fetch(`${base}/${app_dir}/remote/${action_id_without_key}`, { - method: 'POST', - headers: { - 'Content-Type': BINARY_FORM_CONTENT_TYPE, - 'x-sveltekit-pathname': location.pathname, - 'x-sveltekit-search': location.search - }, - body: blob + /** @type {string} */ + const response_text = await new Promise((resolve, reject) => { + const xhr = new XMLHttpRequest(); + xhr.addEventListener('readystatechange', () => { + switch (xhr.readyState) { + case 2 /* HEADERS_RECEIVED */: + if (xhr.status !== 200) { + // We only end up here if the server has an internal error + // (which shouldn't happen because we handle errors on the server and always send a 200 response) + reject(new Error('Failed to execute remote function')); + } + break; + case 4 /* DONE */: + if (xhr.status !== 200) { + reject(new Error('Failed to execute remote function')); + break; + } + resolve(xhr.responseText); + break; + } + }); + if (file_offsets) { + const file_paths = get_file_paths(data); + xhr.upload.addEventListener('progress', (ev) => { + for (const file of file_offsets) { + let progress = (ev.loaded - file.start) / file.file.size; + if (progress <= 0) continue; + if (progress > 1) progress = 1; + const path = file_paths.get(file.file); + if (!path) continue; + deep_set(upload_progress, path, progress); + } + }); + } + // Use `action_id_without_key` here because the id is included in the body via `convert(data)` + xhr.open('POST', `${base}/${app_dir}/remote/${action_id_without_key}`); + xhr.setRequestHeader('Content-Type', BINARY_FORM_CONTENT_TYPE); + xhr.setRequestHeader('x-sveltekit-pathname', location.pathname); + xhr.setRequestHeader('x-sveltekit-search', location.search); + xhr.send(blob); }); - if (!response.ok) { - // We only end up here in case of a network error or if the server has an internal error - // (which shouldn't happen because we handle errors on the server and always send a 200 response) - throw new Error('Failed to execute remote function'); - } - - const form_result = /** @type { RemoteFunctionResponse} */ (await response.json()); + const form_result = /** @type { RemoteFunctionResponse} */ (JSON.parse(response_text)); if (form_result.type === 'result') { ({ issues: raw_issues = [], result } = devalue.parse(form_result.result, app.decoders)); @@ -374,6 +409,7 @@ export function form(id) { if (file) { set_nested_value(input, name, file); + set_nested_value(upload_progress, name, 0); } else { // Remove the property by setting to undefined and clean up const path_parts = name.split(/\.|\[|\]/).filter(Boolean); @@ -403,6 +439,7 @@ export function form(id) { await tick(); input = convert_formdata(new FormData(form)); + upload_progress = {}; }); return () => { @@ -505,7 +542,8 @@ export function form(id) { touched[key] = true; } }, - () => issues + () => issues, + (path) => deep_get(upload_progress, path) ?? 0 ) }, result: { diff --git a/packages/kit/src/runtime/form-utils.js b/packages/kit/src/runtime/form-utils.js index 884a8e94627e..3776e25e28f7 100644 --- a/packages/kit/src/runtime/form-utils.js +++ b/packages/kit/src/runtime/form-utils.js @@ -97,12 +97,13 @@ export function serialize_binary_form(data, meta) { }); let encoded_file_offsets = ''; + /** @type {Array | undefined} */ + let file_offsets; if (files.length) { // Sort small files to the front files.sort(([a], [b]) => a.size - b.size); - /** @type {Array} */ - const file_offsets = new Array(files.length); + file_offsets = new Array(files.length); let start = 0; for (const [file, index] of files) { file_offsets[index] = start; @@ -127,8 +128,13 @@ export function serialize_binary_form(data, meta) { blob_parts.push(file); } + const file_offset_start = 1 + 4 + 2 + encoded_header.length + encoded_file_offsets.length; return { - blob: new Blob(blob_parts) + blob: new Blob(blob_parts), + file_offsets: file_offsets?.map((o, i) => ({ + start: o + file_offset_start, + file: files[i][0] + })) }; } @@ -415,6 +421,27 @@ function check_prototype_pollution(key) { } } +/** + * Finds the paths to every File in an object + * @param {unknown} object + * @param {Map} paths + * @param {string[]} path + */ +export function get_file_paths(object, paths = new Map(), path = []) { + if (Array.isArray(object)) { + for (let i = 0; i < object.length; i++) { + get_file_paths(object[i], paths, [...path, i.toString()]); + } + } else if (object instanceof File) { + paths.set(object, path); + } else if (typeof object === 'object' && object !== null) { + for (const [key, value] of Object.entries(object)) { + get_file_paths(value, paths, [...path, key]); + } + } + return paths; +} + /** * Sets a value in a nested object using an array of keys, mutating the original object. * @param {Record} object @@ -532,10 +559,18 @@ export function deep_get(object, path) { * @param {() => Record} get_input - Function to get current input data * @param {(path: (string | number)[], value: any) => void} set_input - Function to set input data * @param {() => Record} get_issues - Function to get current issues + * @param {(path: (string | number)[]) => number} get_progress - Function to get upload progress of a file * @param {(string | number)[]} path - Current access path * @returns {any} Proxy object with name(), value(), and issues() methods */ -export function create_field_proxy(target, get_input, set_input, get_issues, path = []) { +export function create_field_proxy( + target, + get_input, + set_input, + get_issues, + get_progress, + path = [] +) { const get_value = () => { return deep_get(get_input(), path); }; @@ -546,7 +581,7 @@ export function create_field_proxy(target, get_input, set_input, get_issues, pat // Handle array access like jobs[0] if (/^\d+$/.test(prop)) { - return create_field_proxy({}, get_input, set_input, get_issues, [ + return create_field_proxy({}, get_input, set_input, get_issues, get_progress, [ ...path, parseInt(prop, 10) ]); @@ -559,11 +594,17 @@ export function create_field_proxy(target, get_input, set_input, get_issues, pat set_input(path, newValue); return newValue; }; - return create_field_proxy(set_func, get_input, set_input, get_issues, [...path, prop]); + return create_field_proxy(set_func, get_input, set_input, get_issues, get_progress, [ + ...path, + prop + ]); } if (prop === 'value') { - return create_field_proxy(get_value, get_input, set_input, get_issues, [...path, prop]); + return create_field_proxy(get_value, get_input, set_input, get_issues, get_progress, [ + ...path, + prop + ]); } if (prop === 'issues' || prop === 'allIssues') { @@ -585,7 +626,19 @@ export function create_field_proxy(target, get_input, set_input, get_issues, pat })); }; - return create_field_proxy(issues_func, get_input, set_input, get_issues, [...path, prop]); + return create_field_proxy(issues_func, get_input, set_input, get_issues, get_progress, [ + ...path, + prop + ]); + } + + if (prop === 'progress') { + const progress_func = () => get_progress(path); + + return create_field_proxy(progress_func, get_input, set_input, get_issues, get_progress, [ + ...path, + prop + ]); } if (prop === 'as') { @@ -734,11 +787,17 @@ export function create_field_proxy(target, get_input, set_input, get_issues, pat }); }; - return create_field_proxy(as_func, get_input, set_input, get_issues, [...path, 'as']); + return create_field_proxy(as_func, get_input, set_input, get_issues, get_progress, [ + ...path, + 'as' + ]); } // Handle property access (nested fields) - return create_field_proxy({}, get_input, set_input, get_issues, [...path, prop]); + return create_field_proxy({}, get_input, set_input, get_issues, get_progress, [ + ...path, + prop + ]); } }); } diff --git a/packages/kit/test/apps/basics/src/routes/remote/form/file-upload/+page.svelte b/packages/kit/test/apps/basics/src/routes/remote/form/file-upload/+page.svelte index 388dfcf26cf5..2db4665cca0e 100644 --- a/packages/kit/test/apps/basics/src/routes/remote/form/file-upload/+page.svelte +++ b/packages/kit/test/apps/basics/src/routes/remote/form/file-upload/+page.svelte @@ -4,10 +4,12 @@
-

File 1:

+

File 1: (progress: {upload.fields.file1.progress()})

-

File 2:

- +

File 2: (progress: {upload.fields.deep.files[0].progress()})

+ +

File 3: (progress: {upload.fields.deep.files[1].progress()})

+
diff --git a/packages/kit/test/apps/basics/src/routes/remote/form/file-upload/form.remote.ts b/packages/kit/test/apps/basics/src/routes/remote/form/file-upload/form.remote.ts index be0e0c014090..06400cd385dc 100644 --- a/packages/kit/test/apps/basics/src/routes/remote/form/file-upload/form.remote.ts +++ b/packages/kit/test/apps/basics/src/routes/remote/form/file-upload/form.remote.ts @@ -5,9 +5,15 @@ export const upload = form( v.object({ text: v.string(), file1: v.file(), - file2: v.file() + deep: v.object({ + files: v.array(v.file()) + }) }), async (data) => { - return { text: data.text, file1: await data.file1.text(), file2: await data.file2.text() }; + return { + text: data.text, + file1: await data.file1.text(), + files: await Promise.all(data.deep.files.map((f) => f.text())) + }; } ); diff --git a/packages/kit/types/index.d.ts b/packages/kit/types/index.d.ts index 0ddf7ca08844..39737a20ada9 100644 --- a/packages/kit/types/index.d.ts +++ b/packages/kit/types/index.d.ts @@ -1856,7 +1856,12 @@ declare module '@sveltejs/kit' { set(input: T): T; /** Validation issues, if any */ issues(): RemoteFormIssue[] | undefined; - }; + } & (T extends File + ? { + /** Current upload progress, from 0 to 1 */ + progress(): number; + } + : object); export type RemoteFormFieldValue = string | string[] | number | boolean | File | File[]; From eae94ee3c593acc40f240213c21f6ccb36f2ea56 Mon Sep 17 00:00:00 2001 From: Ottomated Date: Sun, 2 Nov 2025 00:38:51 -0700 Subject: [PATCH 23/27] fix requests stalling if files aren't read --- packages/kit/src/runtime/form-utils.js | 9 +++++++ .../remote/form/file-upload/+page.svelte | 6 +++++ .../remote/form/file-upload/form.remote.ts | 16 +++++++++++-- packages/kit/test/apps/basics/test/test.js | 24 +++++++++++++++++++ 4 files changed, 53 insertions(+), 2 deletions(-) diff --git a/packages/kit/src/runtime/form-utils.js b/packages/kit/src/runtime/form-utils.js index 884a8e94627e..a6a49e0f741c 100644 --- a/packages/kit/src/runtime/form-utils.js +++ b/packages/kit/src/runtime/form-utils.js @@ -263,6 +263,15 @@ export async function deserialize_binary_form(request) { } }); + // Read the request body asyncronously so it doesn't stall + void (async () => { + let has_more = true; + while (has_more) { + const chunk = await get_chunk(chunks.length); + has_more = !!chunk; + } + })(); + return { data, meta, form_data: null }; } diff --git a/packages/kit/test/apps/basics/src/routes/remote/form/file-upload/+page.svelte b/packages/kit/test/apps/basics/src/routes/remote/form/file-upload/+page.svelte index 388dfcf26cf5..850fa70a14db 100644 --- a/packages/kit/test/apps/basics/src/routes/remote/form/file-upload/+page.svelte +++ b/packages/kit/test/apps/basics/src/routes/remote/form/file-upload/+page.svelte @@ -8,6 +8,12 @@

File 2:

+ +
+
diff --git a/packages/kit/test/apps/basics/src/routes/remote/form/file-upload/form.remote.ts b/packages/kit/test/apps/basics/src/routes/remote/form/file-upload/form.remote.ts index be0e0c014090..0282c8ff9264 100644 --- a/packages/kit/test/apps/basics/src/routes/remote/form/file-upload/form.remote.ts +++ b/packages/kit/test/apps/basics/src/routes/remote/form/file-upload/form.remote.ts @@ -5,9 +5,21 @@ export const upload = form( v.object({ text: v.string(), file1: v.file(), - file2: v.file() + file2: v.file(), + read_files: v.optional(v.boolean()) }), async (data) => { - return { text: data.text, file1: await data.file1.text(), file2: await data.file2.text() }; + if (!data.read_files) { + return { + text: data.text, + file1: data.file1.size, + file2: data.file2.size + }; + } + return { + text: data.text, + file1: await data.file1.text(), + file2: await data.file2.text() + }; } ); diff --git a/packages/kit/test/apps/basics/test/test.js b/packages/kit/test/apps/basics/test/test.js index ed9c9e16b387..5a8d4303ceed 100644 --- a/packages/kit/test/apps/basics/test/test.js +++ b/packages/kit/test/apps/basics/test/test.js @@ -2038,6 +2038,7 @@ test.describe('remote functions', () => { mimeType: 'text/plain', buffer: Buffer.from('b') }); + await page.locator('input[type="checkbox"]').check(); await page.locator('button').click(); await expect(page.locator('pre')).toHaveText( @@ -2048,6 +2049,29 @@ test.describe('remote functions', () => { }) ); }); + test('large file uploads work', async ({ page }) => { + await page.goto('/remote/form/file-upload'); + + await page.locator('input[name="file1"]').setInputFiles({ + name: 'a.txt', + mimeType: 'text/plain', + buffer: Buffer.alloc(1024 * 1024 * 10) + }); + await page.locator('input[name="file2"]').setInputFiles({ + name: 'b.txt', + mimeType: 'text/plain', + buffer: Buffer.from('b') + }); + await page.locator('button').click(); + + await expect(page.locator('pre')).toHaveText( + JSON.stringify({ + text: 'Hello world', + file1: 1024 * 1024 * 10, + file2: 1 + }) + ); + }); }); test.describe('params prop', () => { From 774503854a4b4609fab838a208c74f5ecb2b26d1 Mon Sep 17 00:00:00 2001 From: Ottomated Date: Sun, 2 Nov 2025 00:59:59 -0700 Subject: [PATCH 24/27] add test --- .../remote/form/file-upload/+page.svelte | 2 +- .../remote/form/file-upload/form.remote.ts | 2 +- packages/kit/test/apps/basics/test/test.js | 57 +++++++++++++++++-- 3 files changed, 55 insertions(+), 6 deletions(-) diff --git a/packages/kit/test/apps/basics/src/routes/remote/form/file-upload/+page.svelte b/packages/kit/test/apps/basics/src/routes/remote/form/file-upload/+page.svelte index 3553d6e8f0c1..db601b1fc2b5 100644 --- a/packages/kit/test/apps/basics/src/routes/remote/form/file-upload/+page.svelte +++ b/packages/kit/test/apps/basics/src/routes/remote/form/file-upload/+page.svelte @@ -4,7 +4,7 @@
-

File 1: (progress: {upload.fields.file1.progress()})

+

File 1: (progress: {upload.fields.file1.progress()})

File 2: (progress: {upload.fields.deep.files[0].progress()})

diff --git a/packages/kit/test/apps/basics/src/routes/remote/form/file-upload/form.remote.ts b/packages/kit/test/apps/basics/src/routes/remote/form/file-upload/form.remote.ts index 6e7959bc13b4..9c36d1a75a77 100644 --- a/packages/kit/test/apps/basics/src/routes/remote/form/file-upload/form.remote.ts +++ b/packages/kit/test/apps/basics/src/routes/remote/form/file-upload/form.remote.ts @@ -15,7 +15,7 @@ export const upload = form( return { text: data.text, file1: data.file1.size, - file2: data.deep.files.map((f) => f.size) + files: data.deep.files.map((f) => f.size) }; } return { diff --git a/packages/kit/test/apps/basics/test/test.js b/packages/kit/test/apps/basics/test/test.js index 5a8d4303ceed..4d2582255437 100644 --- a/packages/kit/test/apps/basics/test/test.js +++ b/packages/kit/test/apps/basics/test/test.js @@ -2033,11 +2033,16 @@ test.describe('remote functions', () => { mimeType: 'text/plain', buffer: Buffer.from('a') }); - await page.locator('input[name="file2"]').setInputFiles({ + await page.locator('input[name="deep.files[0]"]').setInputFiles({ name: 'b.txt', mimeType: 'text/plain', buffer: Buffer.from('b') }); + await page.locator('input[name="deep.files[1]"]').setInputFiles({ + name: 'c.txt', + mimeType: 'text/plain', + buffer: Buffer.from('c') + }); await page.locator('input[type="checkbox"]').check(); await page.locator('button').click(); @@ -2045,7 +2050,7 @@ test.describe('remote functions', () => { JSON.stringify({ text: 'Hello world', file1: 'a', - file2: 'b' + files: ['b', 'c'] }) ); }); @@ -2057,21 +2062,65 @@ test.describe('remote functions', () => { mimeType: 'text/plain', buffer: Buffer.alloc(1024 * 1024 * 10) }); - await page.locator('input[name="file2"]').setInputFiles({ + await page.locator('input[name="deep.files[0]"]').setInputFiles({ name: 'b.txt', mimeType: 'text/plain', buffer: Buffer.from('b') }); + await page.locator('input[name="deep.files[1]"]').setInputFiles({ + name: 'c.txt', + mimeType: 'text/plain', + buffer: Buffer.from('c') + }); await page.locator('button').click(); await expect(page.locator('pre')).toHaveText( JSON.stringify({ text: 'Hello world', file1: 1024 * 1024 * 10, - file2: 1 + files: [1, 1] }) ); }); + test('file upload progress works', async ({ page, context, javaScriptEnabled }) => { + if (!javaScriptEnabled) return; + await page.goto('/remote/form/file-upload'); + const cdp = await context.newCDPSession(page); + await cdp.send('Network.emulateNetworkConditions', { + offline: false, + latency: 0, + downloadThroughput: -1, + uploadThroughput: 1024 * 1024 * 5 // throttle so it'll take 2 seconds to upload + }); + try { + const progress = page.locator('#progress1'); + expect(progress).toHaveText('0'); + await page.locator('input[name="file1"]').setInputFiles({ + name: 'a.txt', + mimeType: 'text/plain', + buffer: Buffer.alloc(1024 * 1024 * 10) + }); + await page.locator('input[name="deep.files[0]"]').setInputFiles({ + name: 'b.txt', + mimeType: 'text/plain', + buffer: Buffer.from('b') + }); + await page.locator('input[name="deep.files[1]"]').setInputFiles({ + name: 'c.txt', + mimeType: 'text/plain', + buffer: Buffer.from('c') + }); + await page.locator('button').click(); + await expect(progress).not.toHaveText('0'); + } finally { + await cdp.send('Network.emulateNetworkConditions', { + offline: false, + latency: 0, + downloadThroughput: -1, + uploadThroughput: -1 + }); + } + }); }); test.describe('params prop', () => { From 2a08865e1b844f717358919638be3086833dce4b Mon Sep 17 00:00:00 2001 From: Ottomated Date: Sun, 2 Nov 2025 01:01:43 -0700 Subject: [PATCH 25/27] changeset --- .changeset/eager-news-serve.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/eager-news-serve.md diff --git a/.changeset/eager-news-serve.md b/.changeset/eager-news-serve.md new file mode 100644 index 000000000000..979b52bc4db1 --- /dev/null +++ b/.changeset/eager-news-serve.md @@ -0,0 +1,5 @@ +--- +'@sveltejs/kit': minor +--- + +feat: File upload progress is available on `myForm.fields.someFile.progress()` From 7df0c6b8adb75c2758c8af15a553e5c82b6d85ed Mon Sep 17 00:00:00 2001 From: ottomated <31470743+ottomated@users.noreply.github.com> Date: Mon, 10 Nov 2025 13:56:12 -0800 Subject: [PATCH 26/27] Update .changeset/eager-news-serve.md Co-authored-by: Ben McCann <322311+benmccann@users.noreply.github.com> --- .changeset/eager-news-serve.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.changeset/eager-news-serve.md b/.changeset/eager-news-serve.md index 979b52bc4db1..e3c94394b504 100644 --- a/.changeset/eager-news-serve.md +++ b/.changeset/eager-news-serve.md @@ -2,4 +2,4 @@ '@sveltejs/kit': minor --- -feat: File upload progress is available on `myForm.fields.someFile.progress()` +feat: file upload progress is available via `myForm.fields.someFile.progress()` From bf29572581bd9072f8c660ee499c67ddbf2114f2 Mon Sep 17 00:00:00 2001 From: ottomated <31470743+ottomated@users.noreply.github.com> Date: Mon, 10 Nov 2025 13:57:15 -0800 Subject: [PATCH 27/27] Update new-rivers-run.md --- .changeset/new-rivers-run.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.changeset/new-rivers-run.md b/.changeset/new-rivers-run.md index 257e21a14651..c8f873bce988 100644 --- a/.changeset/new-rivers-run.md +++ b/.changeset/new-rivers-run.md @@ -2,4 +2,4 @@ '@sveltejs/kit': minor --- -feat: File uploads inside `form` remote functions are now streamed - form data can be accessed before large files finish uploading. +feat: stream file uploads inside `form` remote functions allowing form data to be accessed before large files finish uploading