[GR-47698] Support GraalPy build with GraalOS musl toolchain.

PullRequest: graalpython/3939

Author: timfel

graalpython/python-liblzma/src/lzma.c

@@ -50,7 +50,7 @@
 #include <limits.h>
 #include <stdlib.h>
 #include <string.h>
-
+#include <sys/types.h>
 
 #include <lzma.h>
 
@@ -196,7 +196,7 @@ enum {
     FORMAT_RAW,
 };
 
-// The ref_count is important for `copy` as we 
+// The ref_count is important for `copy` as we
 // share off heap storage between native objects.
 typedef struct
 {
@@ -239,7 +239,7 @@ void get_macros(int* formats, int* checks, uint64_t* filters, int* mfs, int* mod
     checks[CHECK_SHA256_INDEX] = LZMA_CHECK_SHA256;
     checks[CHECK_ID_MAX_INDEX] = LZMA_CHECK_ID_MAX;
     checks[CHECK_UNKNOWN_INDEX] = LZMA_CHECK_UNKNOWN;
-    
+
     filters[FILTER_LZMA1_INDEX] = LZMA_FILTER_LZMA1;
     filters[FILTER_LZMA2_INDEX] = LZMA_FILTER_LZMA2;
     filters[FILTER_DELTA_INDEX] = LZMA_FILTER_DELTA;
@@ -249,7 +249,7 @@ void get_macros(int* formats, int* checks, uint64_t* filters, int* mfs, int* mod
     filters[FILTER_ARM_INDEX] = LZMA_FILTER_ARM;
     filters[FILTER_ARMTHUMB_INDEX] = LZMA_FILTER_ARMTHUMB;
     filters[FILTER_SPARC_INDEX] = LZMA_FILTER_SPARC;
-    
+
     mfs[MF_HC3_INDEX] = LZMA_MF_HC3;
     mfs[MF_HC4_INDEX] = LZMA_MF_HC4;
     mfs[MF_BT2_INDEX] = LZMA_MF_BT2;
@@ -258,7 +258,7 @@ void get_macros(int* formats, int* checks, uint64_t* filters, int* mfs, int* mod
 
     modes[MODE_FAST_INDEX] = LZMA_MODE_FAST;
     modes[MODE_NORMAL_INDEX] = LZMA_MODE_NORMAL;
-    
+
     preset[PRESET_DEFAULT_INDEX] = LZMA_PRESET_DEFAULT;
     preset[PRESET_EXTREME_INDEX] = LZMA_PRESET_EXTREME;
 }
@@ -267,7 +267,7 @@ static void* LZMA_Malloc(void* ctx, size_t items, size_t size)
 {
     void *m = malloc((size_t)items * (size_t)size);
     LOG_FINER("malloc(address: %p, items: %u, size: %u)\n", m, items, size);
-    return m; 
+    return m;
 }
 
 static void LZMA_Free(void* ctx, void *ptr)
@@ -524,7 +524,7 @@ int lzma_set_filter_spec_lzma(lzmast_stream *lzmast, int fidx, int64_t* opts) {
     initFilters(lzmast);
     lzma_options_lzma *options;
     lzmast->filters[fidx].id = (uint64_t) opts[ID_INDEX];
-    
+
     options = (lzma_options_lzma *)calloc(1, sizeof *options);
     if (options == NULL) {
         return LZMA_MEM_ERROR;
@@ -538,31 +538,31 @@ int lzma_set_filter_spec_lzma(lzmast_stream *lzmast, int fidx, int64_t* opts) {
     }
 
     if (opts[LC_INDEX] != -1) {
-        options->lc = (uint32_t) opts[LC_INDEX]; 
+        options->lc = (uint32_t) opts[LC_INDEX];
     }
 
     if (opts[LP_INDEX] != -1) {
-        options->lp = (uint32_t) opts[LP_INDEX]; 
+        options->lp = (uint32_t) opts[LP_INDEX];
     }
 
     if (opts[PB_INDEX] != -1) {
-        options->pb = (uint32_t) opts[PB_INDEX]; 
+        options->pb = (uint32_t) opts[PB_INDEX];
     }
 
     if (opts[MODE_INDEX] != -1) {
-        options->mode = (lzma_mode) opts[MODE_INDEX]; 
+        options->mode = (lzma_mode) opts[MODE_INDEX];
     }
 
     if (opts[NICE_LEN_INDEX] != -1) {
-        options->nice_len = (uint32_t) opts[NICE_LEN_INDEX]; 
+        options->nice_len = (uint32_t) opts[NICE_LEN_INDEX];
     }
 
     if (opts[MF_INDEX] != -1) {
-        options->mf = (lzma_match_finder) opts[MF_INDEX]; 
+        options->mf = (lzma_match_finder) opts[MF_INDEX];
     }
 
     if (opts[DEPTH_INDEX] != -1) {
-        options->depth = (uint32_t) opts[DEPTH_INDEX]; 
+        options->depth = (uint32_t) opts[DEPTH_INDEX];
     }
 
     lzmast->filters[fidx].options = options;
@@ -662,7 +662,7 @@ int lzma_decode_filter_spec(int64_t filter_id, Byte* encoded_props, int len, int
     lzma_filter filter;
 
     filter.id = (lzma_vli) filter_id;
-    
+
     lzret = lzma_properties_decode(&filter, NULL, encoded_props, len);
     if (!isOK(lzret)) {
         return lzret;
@@ -879,8 +879,8 @@ int lzma_lzma_alone_decoder(lzmast_stream *lzmast, uint64_t memlimit) {
    d->lzs.avail_in are updated to reflect the consumed input. */
 
 // nfi_function: name('decompress') map('lzmast_stream*', 'POINTER')
-int lzma_decompress(lzmast_stream *lzmast, 
-                Byte *input_buffer, ssize_t offset, 
+int lzma_decompress(lzmast_stream *lzmast,
+                Byte *input_buffer, ssize_t offset,
                 ssize_t max_length,
                 ssize_t bufsize, size_t lzs_avail_in) {
     LOG_INFO("lzma_decompress(%p, %p, %zd, %zd, %zd, %zd)\n", lzmast, input_buffer, offset, bufsize, lzs_avail_in);
@@ -919,11 +919,11 @@ int lzma_decompress(lzmast_stream *lzmast,
         if (!isOK(lzret)) {
             clear_output(lzmast);
             return lzret;
-        } 
+        }
         if (lzret == LZMA_GET_CHECK || lzret == LZMA_NO_CHECK) {
             lzmast->check = lzma_get_check(&lzmast->lzs);
             lzret = LZMA_OK;
-        } 
+        }
         if (lzret == LZMA_STREAM_END) {
             // lzmast->eof = 1;
             // we'll return LZMA_STREAM_END to the java side and process EOF.

mx.graalpython/mx_graalpython.py

@@ -267,9 +267,15 @@ def libpythonvm_build_args():
     build_args = []
     build_args += bytecode_dsl_build_args()
 
+    if graalos := ("musl" in mx_subst.path_substitutions.substitute("<multitarget_libc_selection>")):
+        build_args += ['-H:+GraalOS']
+    else:
+        build_args += ["-Dpolyglot.image-build-time.PreinitializeContexts=python"]
+
     if (
-            mx_sdk_vm_ng.is_nativeimage_ee()
-            and mx.is_linux()
+            mx.is_linux()
+            and not graalos
+            and mx_sdk_vm_ng.is_nativeimage_ee()
             and not os.environ.get('NATIVE_IMAGE_AUXILIARY_ENGINE_CACHE')
             and not _is_overridden_native_image_arg("--gc")
     ):
@@ -430,8 +436,7 @@ def full_python(args, env=None):
         args.insert(0, '--python.WithJavaStacktrace=1')
 
     if "--hosted" in args[:2]:
-        args.remove("--hosted")
-        return python(args)
+        return do_run_python(args)
 
     if '--vm.da' not in args:
         args.insert(0, '--vm.ea')
@@ -467,15 +472,13 @@ def handle_debug_arg(args):
                     f"--vm.agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=127.0.0.1:{mx._opts.java_dbg_port}")
 
 
-def python(args, **kwargs):
-    """run a Python program or shell"""
-    if not any(arg.startswith('--python.WithJavaStacktrace') for arg in args):
-        args.insert(0, '--python.WithJavaStacktrace=1')
-
-    do_run_python(args, **kwargs)
+def do_run_python(args, extra_vm_args=None, env=None, jdk=None, extra_dists=None, cp_prefix=None, cp_suffix=None, main_class=GRAALPYTHON_MAIN_CLASS, minimal=False, **kwargs):
 
+    if "--hosted" in args[:2]:
+        args.remove("--hosted")
+        if not any(arg.startswith('--python.WithJavaStacktrace') for arg in args):
+            args.insert(0, '--python.WithJavaStacktrace=1')
 
-def do_run_python(args, extra_vm_args=None, env=None, jdk=None, extra_dists=None, cp_prefix=None, cp_suffix=None, main_class=GRAALPYTHON_MAIN_CLASS, minimal=False, **kwargs):
     if not any(arg.startswith("--python.CAPI") for arg in args):
         capi_home = _get_capi_home()
         args.insert(0, "--python.CAPI=%s" % capi_home)
@@ -495,7 +498,7 @@ def do_run_python(args, extra_vm_args=None, env=None, jdk=None, extra_dists=None
 
     if minimal:
         x = [x for x in SUITE.dists if x.name == "GRAALPYTHON"][0]
-        dists = [dep for dep in x.deps if dep.isJavaProject() or dep.isJARDistribution()]
+        dists = [dep for dep in x.deps if dep.isJavaProject() or dep.isJARDistribution() and dep.exists()]
         # Hack: what we should just do is + ['GRAALPYTHON_VERSIONS_MAIN'] and let MX figure out
         # the class-path and other VM arguments necessary for it. However, due to a bug in MX,
         # LayoutDirDistribution causes an exception if passed to mx.get_runtime_jvm_args,
@@ -2322,16 +2325,17 @@ def run(self, args, env=None, cwd=None, **kwargs):
             args.insert(0, "-q")
 
         args[:0] = [
-            f"--python.PyCachePrefix={pycache_dir}",
-            "--python.DisableFrozenModules",
+            "--PosixModuleBackend=java",
+            "--CompressionModulesBackend=java",
+            f"--PyCachePrefix={pycache_dir}",
+            "--DisableFrozenModules",
             "-B",
             "-S"
         ]
         mx_util.ensure_dir_exists(cwd)
 
         env = env.copy() if env else os.environ.copy()
         env.update(cast(GraalpythonProject, self.subject).getBuildEnv())
-        args.insert(0, '--PosixModuleBackend=java')
         jdk = mx.get_jdk()  # Don't get JVMCI, it might not have finished building by this point
         rc = do_run_python(args, jdk=jdk, env=env, cwd=cwd, minimal=True, out=self.PrefixingOutput(self.subject.name, mx.log), err=self.PrefixingOutput(self.subject.name, mx.log_error), **kwargs)