From cbbe87624090966043433861a772492a565c4d62 Mon Sep 17 00:00:00 2001
From: Julian Scheid <julians37@gmail.com>
Date: Mon, 13 Oct 2025 23:27:49 +0200
Subject: [PATCH] Fix JS_WriteObject2 to check all malloc failures

Add dbuf_error() check at the end of JS_WriteObject2() to detect
buffer allocation failures during BJSON encoding.

Without this check, if malloc/realloc fails at certain points during
encoding, the error flag is set in the DynBuf but never
checked. JS_WriteObject2() returns the incomplete buffer as if
encoding succeeded, causing data corruption.

Here we ensure that when dbuf.error is set, an OutOfMemory exception
is thrown and the function returns NULL.

Co-Authored-By: Claude <noreply@anthropic.com>
---
 api-test.c | 57 ++++++++++++++++++++++++++++++++++++++++++++++++++++++
 quickjs.c  |  4 ++++
 2 files changed, 61 insertions(+)

diff --git a/api-test.c b/api-test.c
index 8cbf8fac5..47118e3ce 100644
--- a/api-test.c
+++ b/api-test.c
@@ -648,6 +648,62 @@ static void slice_string_tocstring(void)
     JS_FreeRuntime(rt);
 }
 
+static void js_writeobject_oom(void)
+{
+    // Test JS_WriteObject2 OOM handling by sweeping memory limits.
+    // This verifies that JS_WriteObject returns NULL (not corrupted data)
+    // when buffer allocation fails during serialization.
+    for (size_t limit_kb = 10; limit_kb <= 200; limit_kb += 1) {
+        JSRuntime *rt = JS_NewRuntime();
+        JSContext *ctx = JS_NewContext(rt);
+
+        size_t limit = limit_kb * 1024;
+        JS_SetMemoryLimit(rt, limit);
+
+        // Create object that needs ~30KB when serialized
+        static const char code[] = "({ data: new Array(3000).fill('test') })";
+        JSValue obj = JS_Eval(ctx, code, strlen(code), "<test>", JS_EVAL_TYPE_GLOBAL);
+
+        if (JS_IsException(obj)) {
+            // Can't even create the object at this limit - skip
+            JS_FreeValue(ctx, obj);
+            JS_FreeContext(ctx);
+            JS_FreeRuntime(rt);
+            continue;
+        }
+
+        // Try to serialize
+        size_t out_len = 0;
+        uint8_t *buf = JS_WriteObject(ctx, &out_len, obj, JS_WRITE_OBJ_REFERENCE);
+
+        if (buf != NULL) {
+            // Remove memory limit before decoding to ensure we're testing
+            // for data corruption, not OOM during decode
+            JS_SetMemoryLimit(rt, -1);
+
+            // Try to decode to check for corruption
+            JSValue decoded = JS_ReadObject(ctx, buf, out_len, JS_READ_OBJ_REFERENCE);
+            assert(!JS_IsException(decoded));
+
+            // Verify decoded data is complete
+            JSValue data_prop = JS_GetPropertyStr(ctx, decoded, "data");
+            assert(JS_IsArray(data_prop));
+            JSValue len_val = JS_GetPropertyStr(ctx, data_prop, "length");
+            int32_t array_len = 0;
+            JS_ToInt32(ctx, &array_len, len_val);
+            assert(array_len == 3000);
+            JS_FreeValue(ctx, len_val);
+            JS_FreeValue(ctx, data_prop);
+            JS_FreeValue(ctx, decoded);
+            js_free(ctx, buf);
+        }
+
+        JS_FreeValue(ctx, obj);
+        JS_FreeContext(ctx);
+        JS_FreeRuntime(rt);
+    }
+}
+
 int main(void)
 {
     sync_call();
@@ -663,5 +719,6 @@ int main(void)
     new_errors();
     global_object_prototype();
     slice_string_tocstring();
+    js_writeobject_oom();
     return 0;
 }
diff --git a/quickjs.c b/quickjs.c
index 793a13e9f..2685fff31 100644
--- a/quickjs.c
+++ b/quickjs.c
@@ -35856,6 +35856,10 @@ uint8_t *JS_WriteObject2(JSContext *ctx, size_t *psize, JSValueConst obj,
         goto fail;
     if (JS_WriteObjectAtoms(s))
         goto fail;
+    if (dbuf_error(&s->dbuf)) {
+        JS_ThrowOutOfMemory(ctx);
+        goto fail;
+    }
     js_object_list_end(ctx, &s->object_list);
     js_free(ctx, s->atom_to_idx);
     js_free(ctx, s->idx_to_atom);