[GR-69672] Optimize frozen module footprint

PullRequest: graalpython/4003

Author: msimacek

graalpython/com.oracle.graal.python.frozen/freeze_modules.py

@@ -44,17 +44,20 @@
     ('stdlib - startup, without site (python -S)', [
         'abc',
         'codecs',
-        '<encodings.*>',
+        # GraalPy change: don't freeze all the encodings, we have our own intrinsified encodings
+        '<encodings>',
+        'encodings.aliases',
         'io',
         ]),
     ('stdlib - startup, with site', [
         '_py_abc',
         '_weakrefset',
         'types',
         'enum',
-        'sre_constants',
-        'sre_parse',
-        'sre_compile',
+        # GraalPy change: don't freeze these, they are deprecated, CPython probably just forgot to remove them from here
+        # 'sre_constants',
+        # 'sre_parse',
+        # 'sre_compile',
         'operator',
         'keyword',
         'heapq',
@@ -553,7 +556,8 @@ def lower_camel_case(str):
 def freeze_module(src):
     with open(src.pyfile, "r", encoding="utf-8") as src_file, open(src.frozenfile, "wb") as binary_file:
         code_obj = compile(src_file.read(), f"<frozen {src.id}>", "exec")
-        marshal.dump(code_obj, binary_file)
+        # GraalPy note: we don't use marshal here, we just dump the co_code which implicitly marshals the code unit
+        binary_file.write(code_obj.co_code)
 
 
 def write_frozen_modules_map(out_file, modules):

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/PythonLanguage.java

@@ -569,43 +569,7 @@ protected CallTarget parse(ParsingRequest request) {
         if (MIME_TYPE_BYTECODE.equals(source.getMimeType())) {
             byte[] bytes = source.getBytes().toByteArray();
             CodeUnit code = MarshalModuleBuiltins.deserializeCodeUnit(null, context, bytes);
-            boolean internal = shouldMarkSourceInternal(context);
-            // The original file path should be passed as the name
-            String name = source.getName();
-            if (name != null && !name.isEmpty()) {
-                Source textSource = tryLoadSource(context, code, internal, name);
-                if (textSource == null) {
-                    if (name.startsWith(FROZEN_FILENAME_PREFIX) && name.endsWith(FROZEN_FILENAME_SUFFIX)) {
-                        String id = name.substring(FROZEN_FILENAME_PREFIX.length(), name.length() - FROZEN_FILENAME_SUFFIX.length());
-                        String fs = context.getEnv().getFileNameSeparator();
-                        String path = context.getStdlibHome() + fs + id.replace(".", fs) + J_PY_EXTENSION;
-                        textSource = tryLoadSource(context, code, internal, path);
-                        if (textSource == null) {
-                            path = context.getStdlibHome() + fs + id.replace(".", fs) + fs + "__init__.py";
-                            textSource = tryLoadSource(context, code, internal, path);
-                        }
-                    }
-                }
-                if (textSource != null) {
-                    source = textSource;
-                }
-            }
-            if (internal && !source.isInternal()) {
-                source = Source.newBuilder(source).internal(true).build();
-            }
-            RootNode rootNode = null;
-
-            if (PythonOptions.ENABLE_BYTECODE_DSL_INTERPRETER) {
-                if (source.hasBytes()) {
-                    // Force a character-based source so that source sections work as expected.
-                    source = Source.newBuilder(source).content(Source.CONTENT_NONE).build();
-                }
-                rootNode = ((BytecodeDSLCodeUnit) code).createRootNode(context, source);
-            } else {
-                rootNode = PBytecodeRootNode.create(this, (BytecodeCodeUnit) code, source);
-            }
-
-            return PythonUtils.getOrCreateCallTarget(rootNode);
+            return callTargetFromBytecode(context, source, code);
         }
 
         String mime = source.getMimeType();
@@ -631,6 +595,46 @@ protected CallTarget parse(ParsingRequest request) {
         return parse(context, source, type, false, optimize, false, null, FutureFeature.fromFlags(flags));
     }
 
+    public RootCallTarget callTargetFromBytecode(PythonContext context, Source source, CodeUnit code) {
+        boolean internal = shouldMarkSourceInternal(context);
+        // The original file path should be passed as the name
+        String name = source.getName();
+        if (name != null && !name.isEmpty()) {
+            Source textSource = tryLoadSource(context, code, internal, name);
+            if (textSource == null) {
+                if (name.startsWith(FROZEN_FILENAME_PREFIX) && name.endsWith(FROZEN_FILENAME_SUFFIX)) {
+                    String id = name.substring(FROZEN_FILENAME_PREFIX.length(), name.length() - FROZEN_FILENAME_SUFFIX.length());
+                    String fs = context.getEnv().getFileNameSeparator();
+                    String path = context.getStdlibHome() + fs + id.replace(".", fs) + J_PY_EXTENSION;
+                    textSource = tryLoadSource(context, code, internal, path);
+                    if (textSource == null) {
+                        path = context.getStdlibHome() + fs + id.replace(".", fs) + fs + "__init__.py";
+                        textSource = tryLoadSource(context, code, internal, path);
+                    }
+                }
+            }
+            if (textSource != null) {
+                source = textSource;
+            }
+        }
+        if (internal && !source.isInternal()) {
+            source = Source.newBuilder(source).internal(true).build();
+        }
+        RootNode rootNode;
+
+        if (PythonOptions.ENABLE_BYTECODE_DSL_INTERPRETER) {
+            if (source.hasBytes()) {
+                // Force a character-based source so that source sections work as expected.
+                source = Source.newBuilder(source).content(Source.CONTENT_NONE).build();
+            }
+            rootNode = ((BytecodeDSLCodeUnit) code).createRootNode(context, source);
+        } else {
+            rootNode = PBytecodeRootNode.create(this, (BytecodeCodeUnit) code, source);
+        }
+
+        return PythonUtils.getOrCreateCallTarget(rootNode);
+    }
+
     private static Source tryLoadSource(PythonContext context, CodeUnit code, boolean internal, String path) {
         try {
             return Source.newBuilder(PythonLanguage.ID, context.getEnv().getPublicTruffleFile(path)).name(code.name.toJavaStringUncached()).internal(internal).build();