extmod/modmarshal: Add new marshal module.

This commit implements a small subset of the CPython `marshal` module.  It
implements `marshal.dumps()` and `marshal.loads()`, but only supports
(un)marshalling code objects at this stage.  The semantics match CPython,
except that the actual marshalled bytes is not compatible with CPython's
marshalled bytes.

The module is enabled at the everything level (only on the unix coverage
build at this stage).

Signed-off-by: Damien George <damien@micropython.org>

Author: dpgeorge

extmod/extmod.cmake

@@ -24,6 +24,7 @@ set(MICROPY_SOURCE_EXTMOD
     ${MICROPY_EXTMOD_DIR}/modframebuf.c
     ${MICROPY_EXTMOD_DIR}/modlwip.c
     ${MICROPY_EXTMOD_DIR}/modmachine.c
+    ${MICROPY_EXTMOD_DIR}/modmarshal.c
     ${MICROPY_EXTMOD_DIR}/modnetwork.c
     ${MICROPY_EXTMOD_DIR}/modonewire.c
     ${MICROPY_EXTMOD_DIR}/modasyncio.c

extmod/extmod.mk

@@ -29,6 +29,7 @@ SRC_EXTMOD_C += \
 	extmod/modjson.c \
 	extmod/modlwip.c \
 	extmod/modmachine.c \
+	extmod/modmarshal.c \
 	extmod/modnetwork.c \
 	extmod/modonewire.c \
 	extmod/modopenamp.c \

extmod/modmarshal.c

@@ -0,0 +1,88 @@
+/*
+ * This file is part of the MicroPython project, http://micropython.org/
+ *
+ * The MIT License (MIT)
+ *
+ * Copyright (c) 2025 Damien P. George
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+ * THE SOFTWARE.
+ *
+ */
+
+#include "py/objcode.h"
+#include "py/objfun.h"
+#include "py/persistentcode.h"
+#include "py/runtime.h"
+
+#if MICROPY_PY_MARSHAL
+
+static mp_obj_t marshal_dumps(mp_obj_t value_in) {
+    if (mp_obj_is_type(value_in, &mp_type_code)) {
+        mp_obj_code_t *code = MP_OBJ_TO_PTR(value_in);
+        const void *proto_fun = mp_code_get_proto_fun(code);
+        const uint8_t *bytecode;
+        if (mp_proto_fun_is_bytecode(proto_fun)) {
+            bytecode = proto_fun;
+        } else {
+            const mp_raw_code_t *rc = proto_fun;
+            if (!(rc->kind == MP_CODE_BYTECODE && rc->children == NULL)) {
+                mp_raise_ValueError(MP_ERROR_TEXT("function must be bytecode with no children"));
+            }
+            bytecode = rc->fun_data;
+        }
+        return mp_raw_code_save_fun_to_bytes(mp_code_get_constants(code), bytecode);
+    } else {
+        mp_raise_ValueError(MP_ERROR_TEXT("unmarshallable object"));
+    }
+}
+static MP_DEFINE_CONST_FUN_OBJ_1(marshal_dumps_obj, marshal_dumps);
+
+static mp_obj_t marshal_loads(mp_obj_t data_in) {
+    mp_buffer_info_t bufinfo;
+    mp_get_buffer_raise(data_in, &bufinfo, MP_BUFFER_READ);
+    mp_module_context_t ctx;
+    ctx.module.globals = mp_globals_get();
+    mp_compiled_module_t cm = { .context = &ctx };
+    mp_raw_code_load_mem(bufinfo.buf, bufinfo.len, &cm);
+    #if MICROPY_PY_BUILTINS_CODE <= MICROPY_PY_BUILTINS_CODE_BASIC
+    return mp_obj_new_code(ctx.constants, cm.rc);
+    #else
+    mp_module_context_t *ctx_ptr = m_new_obj(mp_module_context_t);
+    *ctx_ptr = ctx;
+    return mp_obj_new_code(ctx_ptr, cm.rc, true);
+    #endif
+}
+static MP_DEFINE_CONST_FUN_OBJ_1(marshal_loads_obj, marshal_loads);
+
+static const mp_rom_map_elem_t mod_marshal_globals_table[] = {
+    { MP_ROM_QSTR(MP_QSTR___name__), MP_ROM_QSTR(MP_QSTR_marshal) },
+    { MP_ROM_QSTR(MP_QSTR_dumps), MP_ROM_PTR(&marshal_dumps_obj) },
+    { MP_ROM_QSTR(MP_QSTR_loads), MP_ROM_PTR(&marshal_loads_obj) },
+};
+
+static MP_DEFINE_CONST_DICT(mod_marshal_globals, mod_marshal_globals_table);
+
+const mp_obj_module_t mp_module_marshal = {
+    .base = { &mp_type_module },
+    .globals = (mp_obj_dict_t *)&mod_marshal_globals,
+};
+
+MP_REGISTER_MODULE(MP_QSTR_marshal, mp_module_marshal);
+
+#endif // MICROPY_PY_MARSHAL

ports/windows/msvc/sources.props

@@ -15,6 +15,7 @@
     <PyExtModSource Include="$(PyBaseDir)extmod\modheapq.c" />
     <PyExtModSource Include="$(PyBaseDir)extmod\modjson.c" />
     <PyExtModSource Include="$(PyBaseDir)extmod\modmachine.c" />
+    <PyExtModSource Include="$(PyBaseDir)extmod\modmarshal.c" />
     <PyExtModSource Include="$(PyBaseDir)extmod\modos.c" />
     <PyExtModSource Include="$(PyBaseDir)extmod\modrandom.c" />
     <PyExtModSource Include="$(PyBaseDir)extmod\modre.c" />

py/mpconfig.h

@@ -344,7 +344,7 @@
 
 // Whether to support converting functions to persistent code (bytes)
 #ifndef MICROPY_PERSISTENT_CODE_SAVE_FUN
-#define MICROPY_PERSISTENT_CODE_SAVE_FUN (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_EVERYTHING)
+#define MICROPY_PERSISTENT_CODE_SAVE_FUN (MICROPY_PY_MARSHAL)
 #endif
 
 // Whether generated code can persist independently of the VM/runtime instance
@@ -1382,6 +1382,11 @@ typedef double mp_float_t;
 #define MICROPY_PY_COLLECTIONS_NAMEDTUPLE__ASDICT (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_EVERYTHING)
 #endif
 
+// Whether to provide "marshal" module
+#ifndef MICROPY_PY_MARSHAL
+#define MICROPY_PY_MARSHAL (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_EVERYTHING)
+#endif
+
 // Whether to provide "math" module
 #ifndef MICROPY_PY_MATH
 #define MICROPY_PY_MATH (MICROPY_CONFIG_ROM_LEVEL_AT_LEAST_CORE_FEATURES)

tests/extmod/marshal_basic.py

@@ -0,0 +1,38 @@
+# Test the marshal module, basic functionality.
+
+try:
+    import marshal
+
+    (lambda: 0).__code__
+except (AttributeError, ImportError):
+    print("SKIP")
+    raise SystemExit
+
+ftype = type(lambda: 0)
+
+# Test basic dumps and loads.
+print(ftype(marshal.loads(marshal.dumps((lambda: a).__code__)), {"a": 4})())
+
+# Test dumps of a result from compile().
+ftype(marshal.loads(marshal.dumps(compile("print(a)", "", "exec"))), {"print": print, "a": 5})()
+
+# Test marshalling a function with arguments.
+print(ftype(marshal.loads(marshal.dumps((lambda x, y: x + y).__code__)), {})(1, 2))
+
+# Test marshalling a function with default arguments.
+print(ftype(marshal.loads(marshal.dumps((lambda x=0: x).__code__)), {})("arg"))
+
+# Test marshalling a function containing constant objects (a tuple).
+print(ftype(marshal.loads(marshal.dumps((lambda: (None, ...)).__code__)), {})())
+
+# Test instantiating multiple code's with different globals dicts.
+code = marshal.loads(marshal.dumps((lambda: a).__code__))
+f1 = ftype(code, {"a": 1})
+f2 = ftype(code, {"a": 2})
+print(f1(), f2())
+
+# Test unmarshallable object.
+try:
+    marshal.dumps(type)
+except ValueError:
+    print("ValueError")

tests/extmod/marshal_micropython.py

@@ -0,0 +1,21 @@
+# Test the marshal module, MicroPython-specific functionality.
+
+try:
+    import marshal
+except ImportError:
+    print("SKIP")
+    raise SystemExit
+
+import unittest
+
+
+class Test(unittest.TestCase):
+    def test_function_with_children(self):
+        # Can't marshal a function with children (in this case the module has a child function f).
+        code = compile("def f(): pass", "", "exec")
+        with self.assertRaises(ValueError):
+            marshal.dumps(code)
+
+
+if __name__ == "__main__":
+    unittest.main()

tests/extmod/marshal_stress.py

@@ -0,0 +1,122 @@
+# Test the marshal module, stressing edge cases.
+
+try:
+    import marshal
+
+    (lambda: 0).__code__
+except (AttributeError, ImportError):
+    print("SKIP")
+    raise SystemExit
+
+ftype = type(lambda: 0)
+
+# Test a large function.
+
+
+def large_function(arg0, arg1, arg2, arg3):
+    # Arguments.
+    print(arg0, arg1, arg2, arg3)
+
+    # Positive medium-sized integer (still a small-int though).
+    print(1234)
+
+    # Negative small-ish integer.
+    print(-20)
+
+    # More than 64 constant objects.
+    x = (0,)
+    x = (1,)
+    x = (2,)
+    x = (3,)
+    x = (4,)
+    x = (5,)
+    x = (6,)
+    x = (7,)
+    x = (8,)
+    x = (9,)
+    x = (10,)
+    x = (11,)
+    x = (12,)
+    x = (13,)
+    x = (14,)
+    x = (15,)
+    x = (16,)
+    x = (17,)
+    x = (18,)
+    x = (19,)
+    x = (20,)
+    x = (21,)
+    x = (22,)
+    x = (23,)
+    x = (24,)
+    x = (25,)
+    x = (26,)
+    x = (27,)
+    x = (28,)
+    x = (29,)
+    x = (30,)
+    x = (31,)
+    x = (32,)
+    x = (33,)
+    x = (34,)
+    x = (35,)
+    x = (36,)
+    x = (37,)
+    x = (38,)
+    x = (39,)
+    x = (40,)
+    x = (41,)
+    x = (42,)
+    x = (43,)
+    x = (44,)
+    x = (45,)
+    x = (46,)
+    x = (47,)
+    x = (48,)
+    x = (49,)
+    x = (50,)
+    x = (51,)
+    x = (52,)
+    x = (53,)
+    x = (54,)
+    x = (55,)
+    x = (56,)
+    x = (57,)
+    x = (58,)
+    x = (59,)
+    x = (60,)
+    x = (61,)
+    x = (62,)
+    x = (63,)
+    x = (64,)
+
+    # Small jump.
+    x = 0
+    while x < 2:
+        print("loop", x)
+        x += 1
+
+    # Large jump.
+    x = 0
+    while x < 2:
+        try:
+            try:
+                try:
+                    print
+                except Exception as e:
+                    print
+                finally:
+                    print
+            except Exception as e:
+                print
+            finally:
+                print
+        except Exception as e:
+            print
+        finally:
+            print("loop", x)
+        x += 1
+
+
+code = marshal.dumps(large_function.__code__)
+ftype(marshal.loads(code), {"print": print})(0, 1, 2, 3)

tests/ports/unix/extra_coverage.py.exp

@@ -56,13 +56,13 @@ cmath           collections     cppexample      cryptolib
 deflate         errno           example_package
 ffi             framebuf        gc              hashlib
 heapq           io              json            machine
-math            os              platform        random
-re              select          socket          struct
-sys             termios         time            tls
-uctypes         vfs             websocket
+marshal         math            os              platform
+random          re              select          socket
+struct          sys             termios         time
+tls             uctypes         vfs             websocket
 me
 
-micropython     machine         math
+micropython     machine         marshal         math
 
 argv            atexit          byteorder       exc_info
 executable      exit            getsizeof       implementation