[GR-40848] [GR-40714] Fix detection of the executable name and pyvenv.cfg support.

PullRequest: graalpython/2443

Author: steve-s

ci.jsonnet

@@ -1 +1 @@
-{ "overlay": "bbd123566faf600786d8ea6dc6a16639f9f97bce" }
+{ "overlay": "f31469651b749c69dc177d1392d1c2dffa4e0b04" }

graalpython/com.oracle.graal.python.shell/src/com/oracle/graal/python/shell/GraalPythonMain.java

@@ -25,10 +25,12 @@
  */
 package com.oracle.graal.python.shell;
 
+import java.io.BufferedReader;
 import java.io.EOFException;
 import java.io.File;
 import java.io.FileDescriptor;
 import java.io.FileOutputStream;
+import java.io.IOError;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.OutputStream;
@@ -73,8 +75,7 @@ public static void main(String[] args) {
 
     private static final String LANGUAGE_ID = "python";
 
-    // provided by GraalVM bash launchers, ignored in native image mode
-    protected static final String J_BASH_LAUNCHER_EXEC_NAME = System.getProperty("org.graalvm.launcher.executablename");
+    private static final String J_PYENVCFG = "pyvenv.cfg";
 
     private static long startupWallClockTime = -1;
     private static long startupNanoTime = -1;
@@ -101,6 +102,7 @@ public static void main(String[] args) {
     private boolean dontWriteBytecode = false;
     private String warnOptions = null;
     private String checkHashPycsMode = "default";
+    private String execName;
 
     boolean useASTInterpreter = false;
 
@@ -398,18 +400,16 @@ private static void print(String string) {
         System.err.println(string);
     }
 
-    private static String getLauncherExecName() {
-        if (ImageInfo.inImageCode()) {
-            String binPathName = null;
-            if (ProcessProperties.getArgumentVectorBlockSize() > 0) {
-                binPathName = calculateProgramFullPath(ProcessProperties.getArgumentVectorProgramName());
-            }
-            if (binPathName != null) {
-                return binPathName;
-            }
-            return ProcessProperties.getExecutableName();
+    protected String getLauncherExecName() {
+        if (execName != null) {
+            return execName;
+        }
+        execName = getProgramName();
+        if (execName == null) {
+            return null;
         }
-        return GraalPythonMain.J_BASH_LAUNCHER_EXEC_NAME;
+        execName = calculateProgramFullPath(execName);
+        return execName;
     }
 
     /**
@@ -578,10 +578,21 @@ protected void launch(Builder contextBuilder) {
         if (executable != null) {
             contextBuilder.option("python.ExecutableList", executable);
         } else {
-            contextBuilder.option("python.Executable", getExecutable());
+            executable = getExecutable();
+            contextBuilder.option("python.Executable", executable);
             // The unlikely separator is used because options need to be
             // strings. See PythonOptions.getExecutableList()
             contextBuilder.option("python.ExecutableList", String.join("🏆", getExecutableList()));
+            // We try locating and loading options from pyvenv.cfg according to PEP405 as long as
+            // the user did not explicitly pass some options that would be otherwise loaded from
+            // pyvenv.cfg. Notable usage of this feature is GraalPython venvs which generate a
+            // launcher script that passes those options explicitly without relying on pyvenv.cfg
+            boolean tryVenvCfg = !hasContextOptionSetViaCommandLine("SysPrefix") &&
+                            !hasContextOptionSetViaCommandLine("PythonHome") &&
+                            System.getenv("GRAAL_PYTHONHOME") == null;
+            if (tryVenvCfg) {
+                findAndApplyVenvCfg(contextBuilder, executable);
+            }
         }
 
         // setting this to make sure our TopLevelExceptionHandler calls the excepthook
@@ -676,6 +687,50 @@ protected void launch(Builder contextBuilder) {
         System.exit(rc);
     }
 
+    private void findAndApplyVenvCfg(Builder contextBuilder, String executable) {
+        Path binDir = Paths.get(executable).getParent();
+        if (binDir == null) {
+            return;
+        }
+        Path venvCfg = binDir.resolve(J_PYENVCFG);
+        if (!Files.exists(venvCfg)) {
+            Path binParent = binDir.getParent();
+            if (binParent == null) {
+                return;
+            }
+            venvCfg = binParent.resolve(J_PYENVCFG);
+            if (!Files.exists(venvCfg)) {
+                return;
+            }
+        }
+        try (BufferedReader reader = Files.newBufferedReader(venvCfg)) {
+            String line;
+            while ((line = reader.readLine()) != null) {
+                String[] parts = line.split("=", 2);
+                if (parts.length != 2) {
+                    continue;
+                }
+                String name = parts[0].trim();
+                if (name.equals("home")) {
+                    contextBuilder.option("python.PythonHome", parts[1].trim());
+                    String sysPrefix = null;
+                    try {
+                        sysPrefix = venvCfg.getParent().toAbsolutePath().toString();
+                    } catch (IOError | NullPointerException ex) {
+                        // NullPointerException covers the possible null result of getParent()
+                        warn("Could not set the sys.prefix according to the pyvenv.cfg file.");
+                    }
+                    if (sysPrefix != null) {
+                        contextBuilder.option("python.SysPrefix", sysPrefix);
+                    }
+                    break;
+                }
+            }
+        } catch (IOException ex) {
+            throw abort("Could not read the pyvenv.cfg file.", 66);
+        }
+    }
+
     private static boolean matchesPythonOption(String arg, String key) {
         assert !key.startsWith("python.");
         return arg.startsWith("--python." + key) || arg.startsWith("--" + key);
@@ -698,6 +753,18 @@ private String getContextOptionIfSetViaCommandLine(String key) {
         return null;
     }
 
+    private boolean hasContextOptionSetViaCommandLine(String key) {
+        if (System.getProperty("polyglot.python." + key) != null) {
+            return System.getProperty("polyglot.python." + key) != null;
+        }
+        for (String f : givenArguments) {
+            if (matchesPythonOption(f, key)) {
+                return true;
+            }
+        }
+        return false;
+    }
+
     private static void printFileNotFoundException(NoSuchFileException e) {
         String reason = e.getReason();
         if (reason == null) {

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/GraalPythonModuleBuiltins.java

@@ -499,6 +499,92 @@ public Object doIt(PFunction func,
         }
     }
 
+    @Builtin(name = "get_toolchain_tools_for_venv")
+    @TypeSystemReference(PythonArithmeticTypes.class)
+    @GenerateNodeFactory
+    public abstract static class GetToolchainToolsForVenv extends PythonBuiltinNode {
+        private static final class Tool {
+            final String name;
+            final boolean isVariableName;
+            final Object[] targets;
+
+            public Tool(String name, boolean isVariableName, Object[] targets) {
+                this.name = name;
+                this.isVariableName = isVariableName;
+                this.targets = targets;
+            }
+
+            static Tool forVariable(String name, Object... targets) {
+                return new Tool(name, true, targets);
+            }
+
+            static Tool forBinary(String name, Object... targets) {
+                return new Tool(name, true, targets);
+            }
+        }
+
+        static final Tool[] tools = new Tool[]{
+                        Tool.forVariable("AR", tsLiteral("ar")),
+                        Tool.forVariable("RANLIB", tsLiteral("ranlib")),
+                        Tool.forVariable("NM", tsLiteral("nm")),
+                        Tool.forVariable("LD", tsLiteral("ld.lld"), tsLiteral("ld"), tsLiteral("lld")),
+                        Tool.forVariable("CC", tsLiteral("clang"), tsLiteral("cc")),
+                        Tool.forVariable("CXX", tsLiteral("clang++"), tsLiteral("c++")),
+                        Tool.forBinary("llvm-as", tsLiteral("as")),
+                        Tool.forBinary("clang-cl", tsLiteral("cl")),
+                        Tool.forBinary("clang-cpp", tsLiteral("cpp")),
+        };
+
+        @Specialization
+        @TruffleBoundary
+        protected Object getToolPath() {
+            Env env = getContext().getEnv();
+            LanguageInfo llvmInfo = env.getInternalLanguages().get(J_LLVM_LANGUAGE);
+            Toolchain toolchain = env.lookup(llvmInfo, Toolchain.class);
+            List<TruffleFile> toolchainPaths = toolchain.getPaths("PATH");
+            EconomicMapStorage storage = EconomicMapStorage.create(tools.length);
+            for (Tool tool : tools) {
+                String path = null;
+                if (tool.isVariableName) {
+                    TruffleFile toolPath = toolchain.getToolPath(tool.name);
+                    if (toolPath != null) {
+                        path = toolPath.getAbsoluteFile().getPath();
+                    }
+                } else {
+                    for (TruffleFile toolchainPath : toolchainPaths) {
+                        LOGGER.finest(() -> " Testing path " + toolchainPath.getPath() + " for tool " + tool.name);
+                        TruffleFile pathToTest = toolchainPath.resolve(tool.name);
+                        if (pathToTest.exists()) {
+                            path = pathToTest.getAbsoluteFile().getPath();
+                            break;
+                        }
+                    }
+                }
+                if (path != null) {
+                    storage.putUncached(toTruffleStringUncached(path), factory().createTuple(tool.targets));
+                } else {
+                    LOGGER.info("Could not locate tool " + tool.name);
+                }
+            }
+            return factory().createDict(storage);
+        }
+    }
+
+    /*
+     * Internal check used in tests only to check that we are running through managed launcher.
+     */
+    @Builtin(name = "is_managed_launcher")
+    @TypeSystemReference(PythonArithmeticTypes.class)
+    @GenerateNodeFactory
+    public abstract static class IsManagedLauncher extends PythonBuiltinNode {
+        @Specialization
+        @TruffleBoundary
+        protected boolean isManaged() {
+            // The best approximation for now
+            return !getContext().getEnv().isNativeAccessAllowed() && getContext().getOption(PythonOptions.RunViaLauncher);
+        }
+    }
+
     @Builtin(name = "get_toolchain_tool_path", minNumOfPositionalArgs = 1)
     @TypeSystemReference(PythonArithmeticTypes.class)
     @GenerateNodeFactory

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/common/ObjectHashMap.java

@@ -833,7 +833,7 @@ private boolean keysEqual(int[] originalIndices, ThreadState state, int index, O
      * smaller than the old size if there were many dummy entries. The rehashing also removes the
      * dummy entries.
      */
-    @TruffleBoundary(allowInlining = true)
+    @TruffleBoundary
     private void rehashAndPut(Object newKey, long newKeyHash, Object newValue) {
         int requiredIndicesSize = usedHashes * GROWTH_RATE;
         // We need the hash table of this size, in order to accommodate "requiredIndicesSize" items
@@ -867,7 +867,7 @@ private void rehashAndPut(Object newKey, long newKeyHash, Object newValue) {
         insertNewKey(localIndices, newKey, newKeyHash, newValue);
     }
 
-    @TruffleBoundary(allowInlining = true)
+    @TruffleBoundary
     private void compact() {
         // shuffle[X] will tell us by how much value X found in 'indices' should be shuffled to left
         int[] shuffle = new int[hashes.length];

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/PythonContext.java

@@ -1551,20 +1551,22 @@ public void initializeHomeAndPrefixPaths(Env newEnv, String languageHome) {
                         "\n\tCAPI: {5}" +
                         "\n\tJNI library: {6}", languageHome, sysPrefix, basePrefix, coreHome, stdLibHome, capiHome, jniHome));
 
-        String envHome = null;
-        try {
-            envHome = System.getenv("GRAAL_PYTHONHOME");
-        } catch (SecurityException e) {
+        String pythonHome = newEnv.getOptions().get(PythonOptions.PythonHome);
+        if (pythonHome.isEmpty()) {
+            try {
+                pythonHome = System.getenv("GRAAL_PYTHONHOME");
+            } catch (SecurityException e) {
+            }
         }
 
         final TruffleFile home;
-        if (languageHome != null && envHome == null) {
+        if (languageHome != null && pythonHome == null) {
             home = newEnv.getInternalTruffleFile(languageHome);
-        } else if (envHome != null) {
+        } else if (pythonHome != null) {
             boolean envHomeIsDirectory = false;
             TruffleFile envHomeFile = null;
             try {
-                envHomeFile = newEnv.getInternalTruffleFile(envHome);
+                envHomeFile = newEnv.getInternalTruffleFile(pythonHome);
                 envHomeIsDirectory = envHomeFile.isDirectory();
             } catch (SecurityException e) {
             }

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/PythonOptions.java

@@ -92,6 +92,10 @@ private PythonOptions() {
         // no instances
     }
 
+    @Option(category = OptionCategory.EXPERT, help = "Set the home of Python. Equivalent of GRAAL_PYTHONHOME env variable. " +
+                    "Determines default values for the CoreHome, StdLibHome, SysBasePrefix, SysPrefix.", usageSyntax = "<path>", stability = OptionStability.EXPERIMENTAL) //
+    public static final OptionKey<String> PythonHome = new OptionKey<>("");
+
     @Option(category = OptionCategory.USER, help = "Set the location of sys.prefix. Overrides any environment variables or Java options.", usageSyntax = "<path>", stability = OptionStability.STABLE) //
     public static final OptionKey<TruffleString> SysPrefix = new OptionKey<>(T_EMPTY_STRING, TS_OPTION_TYPE);
 

graalpython/lib-python/3/venv/__init__.py

@@ -187,38 +187,12 @@ def create_if_needed(d):
 
     def _install_compilers(self, context):
         """Puts the Graal LLVM compiler tools on the path"""
-
-        # Table of well-known LLVM tools that must be queried by a variable name.
-        llvm_tools = {
-            "AR": ("ar",),
-            "RANLIB": ("ranlib",),
-            "NM": ("nm",),
-            "LD": ("ld.lld", "ld", "lld"),
-            "CC": ("clang", "cc"),
-            "CXX": ("clang++", "c++"),
-        }
-        # Table of additional LLVM tools to use if they are available.
-        _llvm_bins = {
-            "llvm-as": ("as",),
-            "clang-cl": ("cl",),
-            "clang-cpp": ("cpp",),
-        }
         bin_dir = os.path.join(context.env_dir, context.bin_name)
-        def create_symlinks(table, resolver):
-            for tool_var in table:
-                tool_path = resolver(tool_var)
-                if os.path.exists(tool_path):
-                    for name in table[tool_var]:
-                        dest = os.path.join(bin_dir, name)
-                        if not os.path.exists(dest):
-                            os.symlink(tool_path, dest)
-
-        create_symlinks(llvm_tools, __graalpython__.get_toolchain_tool_path)
-        # NOTE: function 'get_toolcahin_paths' returns a tuple
-        llvm_path = __graalpython__.get_toolchain_paths("PATH")
-        if llvm_path and llvm_path[0]:
-            create_symlinks(_llvm_bins, lambda binary_name: os.path.join(llvm_path[0], binary_name))
-
+        for (tool_path, names) in __graalpython__.get_toolchain_tools_for_venv().items():
+            for name in names:
+                dest = os.path.join(bin_dir, name)
+                if not os.path.exists(dest):
+                    os.symlink(tool_path, dest)
 
     def _patch_shebang(self, context):
         # Truffle change: we need to patch the pip/pip3 (and maybe other)

mx.graalpython/suite.py

@@ -44,23 +44,23 @@
             },
             {
                 "name": "tools",
-                "version": "9a0570863be7d236519bceda4b05ee3e00a6c384",
+                "version": "98fb946635802e01b8c892e4ecee44c48fb7ffa3",
                 "subdir": True,
                 "urls": [
                     {"url": "https://github.com/oracle/graal", "kind": "git"},
                 ],
             },
             {
                 "name": "sulong",
-                "version": "9a0570863be7d236519bceda4b05ee3e00a6c384",
+                "version": "98fb946635802e01b8c892e4ecee44c48fb7ffa3",
                 "subdir": True,
                 "urls": [
                     {"url": "https://github.com/oracle/graal", "kind": "git"},
                 ]
             },
             {
                 "name": "regex",
-                "version": "9a0570863be7d236519bceda4b05ee3e00a6c384",
+                "version": "98fb946635802e01b8c892e4ecee44c48fb7ffa3",
                 "subdir": True,
                 "urls": [
                     {"url": "https://github.com/oracle/graal", "kind": "git"},