[GR-21590] Update Python imports.

PullRequest: graalpython/3790

Author: timfel

ci.jsonnet

@@ -1 +1 @@
-{ "overlay": "2ffa5d51a35c1e6d5b1bad7038dbaf30b0118422" }
+{ "overlay": "1f06979c2fc643ef5d92c903790114a063ce486f" }

mx.graalpython/mx_graalpython.py

@@ -704,7 +704,9 @@ def graalvm_jdk():
     if graal_jdk_home:
         graal_jdk_home = os.path.abspath(glob.glob(graal_jdk_home)[0])
         if sys.platform == "darwin":
-            graal_jdk_home = os.path.join(graal_jdk_home, 'Contents', 'Home')
+            jdk_home_subdir = os.path.join(graal_jdk_home, 'Contents', 'Home')
+            if os.path.exists(jdk_home_subdir):
+                graal_jdk_home = jdk_home_subdir
         mx.log("Using Graal from GRAAL_JDK_HOME: " + graal_jdk_home)
 
         # Try to verify that we're getting what we expect:

mx.graalpython/mx_graalpython_benchmark.py

@@ -23,6 +23,7 @@
 # OF THE POSSIBILITY OF SUCH DAMAGE.
 from __future__ import print_function
 
+import itertools
 import functools
 import statistics
 import sys
@@ -54,14 +55,7 @@
 # constants
 #
 # ----------------------------------------------------------------------------------------------------------------------
-ENV_PYPY_HOME = "PYPY_HOME"
-ENV_PYTHON3_HOME = "PYTHON3_HOME"
-ENV_VIRTUAL_ENV = "VIRTUAL_ENV"
-ENV_JYTHON_JAR = "JYTHON_JAR"
 VM_NAME_GRAALPYTHON = "graalpython"
-VM_NAME_CPYTHON = "cpython"
-VM_NAME_PYPY = "pypy"
-VM_NAME_JYTHON = "jython"
 VM_NAME_GRAALPYTHON_SVM = "graalpython-svm"
 GROUP_GRAAL = "Graal"
 SUBGROUP_GRAAL_PYTHON = "graalpython"
@@ -129,12 +123,11 @@ def _handle_var(key_value):
 #
 # ----------------------------------------------------------------------------------------------------------------------
 class AbstractPythonVm(OutputCapturingVm, ABC):
-    def __init__(self, name, config_name, options=None, env=None):
+    def __init__(self, name, config_name, options=None):
         super().__init__()
         self._name = name
         self._config_name = config_name
         self._options = options
-        self._env = env
 
     @property
     def options(self):
@@ -153,21 +146,21 @@ def interpreter(self):
     def post_process_command_line_args(self, args):
         return args
 
-    def run_vm(self, args, out=None, err=None, cwd=None, nonZeroIsFatal=False):
+    def run_vm(self, args, out=None, err=None, cwd=None, nonZeroIsFatal=False, env=None):
         mx.logv(shlex.join([self.interpreter] + args))
         return mx.run(
             [self.interpreter] + args,
             out=out,
             err=err,
             cwd=cwd,
             nonZeroIsFatal=nonZeroIsFatal,
-            env=self._env
+            env=env,
         )
 
 
 class AbstractPythonIterationsControlVm(AbstractPythonVm):
-    def __init__(self, name, config_name, options=None, env=None, iterations=None):
-        super().__init__(name, config_name, options=options, env=env)
+    def __init__(self, name, config_name, options=None, iterations=None):
+        super().__init__(name, config_name, options=options)
         try:
             self._iterations = int(iterations)
         except:
@@ -194,10 +187,8 @@ def run_vm(self, args, *splat, **kwargs):
 
 
 class CPythonVm(AbstractPythonIterationsControlVm):
-    PYTHON_INTERPRETER = "python3"
-
-    def __init__(self, config_name, options=None, env=None, virtualenv=None, iterations=0):
-        super().__init__(VM_NAME_CPYTHON, config_name, options=options, env=env, iterations=iterations)
+    def __init__(self, config_name, options=None, virtualenv=None, iterations=0):
+        super().__init__("cpython", config_name, options=options, iterations=iterations)
         self._virtualenv = virtualenv
 
     def override_iterations(self, requested_iterations):
@@ -206,41 +197,74 @@ def override_iterations(self, requested_iterations):
 
     @property
     def interpreter(self):
-        venv = self._virtualenv if self._virtualenv else mx.get_env(ENV_VIRTUAL_ENV)
-        if venv:
-            mx.log(f"CPythonVM virtualenv={venv}")
-            return os.path.join(venv, 'bin', CPythonVm.PYTHON_INTERPRETER)
-        home = mx.get_env(ENV_PYTHON3_HOME)
-        if home:
-            mx.log(f"CPythonVM python3 home={home}")
-            return os.path.join(home, CPythonVm.PYTHON_INTERPRETER)
-        return CPythonVm.PYTHON_INTERPRETER
+        candidates_pre = [
+            self._virtualenv,
+            mx.get_env("VIRTUAL_ENV"),
+            mx.get_env("PYTHON3_HOME"),
+        ]
+        candidates_suf = [
+            join("bin", "python3"),
+            join("bin", "python"),
+            "python3",
+            "python",
+            sys.executable,
+        ]
+        for p, s in itertools.product(candidates_pre, candidates_suf):
+            if os.path.exists(exe := os.path.join(p or "", s)):
+                mx.log(f"CPython VM {exe=}")
+                return exe
+        assert False, "sys.executable should really exist"
 
+    def run_vm(self, args, *splat, **kwargs):
+        for idx, arg in enumerate(args):
+            if "--vm.Xmx" in arg:
+                mx.warn(f"Ignoring {arg}, cannot restrict memory on CPython.")
+                args = args[:idx] + args[idx + 1 :]
+                break
+        return super().run_vm(args, *splat, **kwargs)
 
-class PyPyVm(AbstractPythonIterationsControlVm):
-    PYPY_INTERPRETER = "pypy3"
 
-    def __init__(self, config_name, options=None, env=None, virtualenv=None, iterations=0):
-        super().__init__(VM_NAME_PYPY, config_name, options=options, env=env, iterations=iterations)
+class PyPyVm(AbstractPythonIterationsControlVm):
+    def __init__(self, config_name, options=None, virtualenv=None, iterations=0):
+        super().__init__("pypy", config_name, options=options, iterations=iterations)
 
     def override_iterations(self, requested_iterations):
         # PyPy warms up much faster, half should be enough
         return int(requested_iterations / 2)
 
     @property
     def interpreter(self):
-        home = mx.get_env(ENV_PYPY_HOME)
-        if not home:
+        if home := mx.get_env("PYPY_HOME"):
+            exe = join(home, "bin", "pypy3")
+        else:
             try:
-                return subprocess.check_output("which %s" % PyPyVm.PYPY_INTERPRETER, shell=True).decode().strip()
+                exe = subprocess.check_output("which pypy3", shell=True).decode().strip()
             except OSError:
-                mx.abort("{} is not set!".format(ENV_PYPY_HOME))
-        return join(home, 'bin', PyPyVm.PYPY_INTERPRETER)
+                mx.abort("PYPY_HOME is not set!")
+        mx.log(f"PyPy {exe=}")
+        return exe
+
+    def run_vm(self, args, *splat, env=None, **kwargs):
+        env = env or os.environ.copy()
+        xmxArg = re.compile("--vm.Xmx([0-9]+)([kKgGmM])")
+        pypyGcMax = "8GB"
+        for idx, arg in enumerate(args):
+            if m := xmxArg.search(arg):
+                args = args[:idx] + args[idx + 1 :]
+                pypyGcMax = f"{m.group(1)}{m.group(2).upper()}B"
+                mx.log(f"Setting PYPY_GC_MAX={pypyGcMax} via {arg}")
+                break
+        else:
+            mx.log(
+                f"Setting PYPY_GC_MAX={pypyGcMax}, use --vm.Xmx argument to override it"
+            )
+        env["PYPY_GC_MAX"] = pypyGcMax
+        return super().run_vm(args, *splat, env=env, **kwargs)
 
 
 class GraalPythonVm(AbstractPythonIterationsControlVm):
-    def __init__(self, config_name, options=None, env=None, virtualenv=None, iterations=None, extra_polyglot_args=None):
-        super().__init__(VM_NAME_GRAALPYTHON, config_name, options=options, env=env, iterations=iterations)
+    def __init__(self, config_name, options=None, virtualenv=None, iterations=None, extra_polyglot_args=None):
+        super().__init__(VM_NAME_GRAALPYTHON, config_name, options=options, iterations=iterations)
         self._extra_polyglot_args = extra_polyglot_args or []
 
     @property
@@ -273,7 +297,7 @@ def extract_vm_info(self, args=None):
         out_version = subprocess.check_output([self.interpreter, '--version'], universal_newlines=True)
         # The benchmark data goes back a ways, we modify the reported dims for
         # continuity with the historical queries
-        graalvm_version_match = re.search(r"\(([^\)]+ ((?:\d+\.?)+))\)", out_version)
+        graalvm_version_match = re.search(r"\(([^\)]+ ((?:\d+\.?)+)).*\)", out_version)
         if not graalvm_version_match:
             mx.log(f"Using {out_version} as platform version string input")
             graalvm_version_match = [out_version, out_version, out_version]
@@ -311,15 +335,14 @@ def get_extra_polyglot_args(self):
 
 class GraalPythonJavaDriverVm(GuestVm):
     def __init__(self, config_name=CONFIGURATION_DEFAULT, cp_suffix=None, distributions=None, cp_prefix=None,
-                 host_vm=None, extra_vm_args=None, extra_polyglot_args=None, env=None):
+                 host_vm=None, extra_vm_args=None, extra_polyglot_args=None):
         super().__init__(host_vm=host_vm)
         self._config_name = config_name
         self._distributions = distributions or ['GRAALPYTHON_BENCH']
         self._cp_suffix = cp_suffix
         self._cp_prefix = cp_prefix
         self._extra_vm_args = extra_vm_args
         self._extra_polyglot_args = extra_polyglot_args if isinstance(extra_polyglot_args, list) else []
-        self._env = env
 
     def name(self):
         return VM_NAME_GRAALPYTHON
@@ -341,30 +364,30 @@ def get_classpath(self):
     def with_host_vm(self, host_vm):
         return self.__class__(config_name=self._config_name, distributions=self._distributions,
                               cp_suffix=self._cp_suffix, cp_prefix=self._cp_prefix, host_vm=host_vm,
-                              extra_vm_args=self._extra_vm_args, extra_polyglot_args=self._extra_polyglot_args,
-                              env=self._env)
+                              extra_vm_args=self._extra_vm_args, extra_polyglot_args=self._extra_polyglot_args)
 
     def launcher_class(self):
         return 'com.oracle.graal.python.benchmarks.JavaBenchmarkDriver'
 
     def run(self, cwd, args):
         extra_polyglot_args = self.get_extra_polyglot_args()
         host_vm = self.host_vm()
-        with environ(self._env or {}):
-            cp = self.get_classpath()
-            jhm = mx.dependency("mx:JMH_1_21")
-            cp_deps = [
-                mx.distribution('GRAALPYTHON_BENCH', fatalIfMissing=True),
-                jhm,
-                mx.dependency("sdk:LAUNCHER_COMMON")
-            ] + jhm.deps
-            cp += [x.classpath_repr() for x in cp_deps]
-            java_args = ['-cp', ':'.join(cp)] + [self.launcher_class()]
-            out = mx.TeeOutputCapture(mx.OutputCapture())
-            code = host_vm.run_java(java_args + extra_polyglot_args + args, cwd=cwd, out=out, err=out)
-            out = out.underlying.data
-            dims = host_vm.dimensions(cwd, args, code, out)
-            return code, out, dims
+        cp = self.get_classpath()
+        jhm = mx.dependency("mx:JMH_1_21")
+        cp_deps = mx.classpath_entries(filter(None, [
+            mx.distribution('GRAALPYTHON_BENCH', fatalIfMissing=True),
+            mx.distribution('TRUFFLE_RUNTIME', fatalIfMissing=True),
+            mx.distribution('TRUFFLE_ENTERPRISE', fatalIfMissing=False),
+            jhm,
+            mx.dependency("sdk:LAUNCHER_COMMON")
+        ] + jhm.deps))
+        cp += [x.classpath_repr() for x in cp_deps]
+        java_args = ['-cp', ':'.join(cp)] + [self.launcher_class()]
+        out = mx.TeeOutputCapture(mx.OutputCapture())
+        code = host_vm.run_java(java_args + extra_polyglot_args + args, cwd=cwd, out=out, err=out)
+        out = out.underlying.data
+        dims = host_vm.dimensions(cwd, args, code, out)
+        return code, out, dims
 
     def get_extra_polyglot_args(self):
         return ["--experimental-options", "--python.MaxNativeMemory=%s" % (2**34), *self._extra_polyglot_args]
@@ -381,7 +404,7 @@ def get_extra_polyglot_args(self):
 
 class PythonBaseBenchmarkSuite(VmBenchmarkSuite, AveragingBenchmarkMixin):
     def __init__(self, name, benchmarks):
-        super(PythonBaseBenchmarkSuite, self).__init__()
+        super().__init__()
         self._checkup = 'GRAALPYTHON_BENCHMARKS_CHECKUP' in os.environ
         self._name = name
         self._benchmarks = benchmarks
@@ -578,14 +601,7 @@ def rules(self, output, benchmarks, bm_suite_args):
         ]
 
     def runAndReturnStdOut(self, benchmarks, bmSuiteArgs):
-        ret_code, out, dims = super(PythonBaseBenchmarkSuite, self).runAndReturnStdOut(benchmarks, bmSuiteArgs)
-
-        # host-vm rewrite rules
-        if dims['host-vm'] not in ('graalvm-ce', 'graalvm-ee'):
-            if mx.suite('graal-enterprise', fatalIfMissing=False):
-                dims['host-vm'] = 'graalvm-ee'
-            else:
-                dims['host-vm'] = 'graalvm-ce'
+        ret_code, out, dims = super().runAndReturnStdOut(benchmarks, bmSuiteArgs)
 
         self.post_run_graph(benchmarks[0], dims['host-vm-config'], dims['guest-vm-config'])
 
@@ -598,7 +614,7 @@ def run(self, benchmarks, bm_suite_args):
         if '--checkup' in bm_suite_args:
             self._checkup = True
             bm_suite_args.remove('--checkup')
-        results = super(PythonBaseBenchmarkSuite, self).run(benchmarks, bm_suite_args)
+        results = super().run(benchmarks, bm_suite_args)
         self.addAverageAcrossLatestResults(results)
         return results
 
@@ -710,7 +726,7 @@ def checkup(self, out):
 
 class PythonBenchmarkSuite(PythonBaseBenchmarkSuite):
     def __init__(self, name, bench_path, benchmarks, python_path=None):
-        super(PythonBenchmarkSuite, self).__init__(name, benchmarks)
+        super().__init__(name, benchmarks)
         self._python_path = python_path
         self._harness_path = HARNESS_PATH
         self._harness_path = join(SUITE.dir, self._harness_path)
@@ -778,7 +794,7 @@ def get_benchmark_suites(cls, benchmarks):
 
 class PythonJavaEmbeddingBenchmarkSuite(PythonBaseBenchmarkSuite):
     def __init__(self, name, bench_path, benchmarks):
-        super(PythonJavaEmbeddingBenchmarkSuite, self).__init__(name, benchmarks)
+        super().__init__(name, benchmarks)
         self._bench_path = bench_path
 
     def get_vm_registry(self):
@@ -1001,6 +1017,9 @@ def register_vms(suite, sandboxed_options):
     # Other Python VMs:
     python_vm_registry.add_vm(CPythonVm(config_name=CONFIGURATION_DEFAULT), suite)
     python_vm_registry.add_vm(PyPyVm(config_name=CONFIGURATION_DEFAULT), suite)
+    # For continuity with old datapoints, provide CPython and PyPy with launcher config_name
+    python_vm_registry.add_vm(CPythonVm(config_name="launcher"), suite)
+    python_vm_registry.add_vm(PyPyVm(config_name="launcher"), suite)
 
     graalpy_vms = []