[GR-65718] Fix torch on 3.12

PullRequest: graalpython/3834

Author: msimacek

graalpython/com.oracle.graal.python.cext/src/gcmodule.c

@@ -43,6 +43,11 @@
 static inline int
 call_traverse(traverseproc traverse, PyObject *op, visitproc visit, void *arg)
 {
+    if (points_to_py_handle_space(op)) {
+        // Ignore managed handles. Some builtin types don't (yet) create native objects when subclassed in native,
+        // so they can end up with native traverse. Example is pybind11_static_property that subclasses builtin property
+        return 0;
+    }
     if (!traverse) {
         PyTruffle_Log(PY_TRUFFLE_LOG_FINE,
                       "type '%.100s' is a GC type but tp_traverse is NULL",

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/PosixModuleBuiltins.java

@@ -25,6 +25,7 @@
  */
 package com.oracle.graal.python.builtins.modules;
 
+import static com.oracle.graal.python.nodes.BuiltinNames.T_ENVIRON;
 import static com.oracle.graal.python.nodes.BuiltinNames.T_NT;
 import static com.oracle.graal.python.nodes.BuiltinNames.T_POSIX;
 import static com.oracle.graal.python.nodes.StringLiterals.T_DOT;
@@ -252,7 +253,7 @@ public void initialize(Python3Core core) {
         }
         PythonLanguage language = core.getLanguage();
         addBuiltinConstant("_have_functions", PFactory.createList(language, haveFunctions.toArray()));
-        addBuiltinConstant("environ", PFactory.createDict(language));
+        addBuiltinConstant(T_ENVIRON, PFactory.createDict(language));
 
         LinkedHashMap<String, Object> sysconfigNames = new LinkedHashMap<>();
         for (IntConstant name : PosixConstants.sysconfigNames) {
@@ -313,16 +314,12 @@ public void postInitialize(Python3Core core) {
                 // we don't want subprocesses to pick it up
                 continue;
             }
-            Object key, val;
-            if (PythonOS.getPythonOS() == PythonOS.PLATFORM_WIN32) {
-                if (entry.getKey().startsWith("=")) {
-                    // Hidden variable, shouldn't be visible to python
-                    continue;
-                }
-                key = toTruffleStringUncached(entry.getKey());
-            } else {
-                key = PFactory.createBytes(language, entry.getKey().getBytes());
+            if (PythonOS.getPythonOS() == PythonOS.PLATFORM_WIN32 && entry.getKey().startsWith("=")) {
+                // Hidden variable, shouldn't be visible to python
+                continue;
             }
+            Object key = toEnv(language, entry.getKey());
+            Object val = toEnv(language, entry.getValue());
             if (pyenvLauncherKey.equals(entry.getKey())) {
                 // On Mac, the CPython launcher uses this env variable to specify the real Python
                 // executable. It will be honored by packages like "site". So, if it is set, we
@@ -334,31 +331,24 @@ public void postInitialize(Python3Core core) {
                     posixLib.setenv(posixSupport, k, v, true);
                 } catch (PosixException ignored) {
                 }
-                if (PythonOS.getPythonOS() == PythonOS.PLATFORM_WIN32) {
-                    val = value;
-                } else {
-                    val = PFactory.createBytes(language, value.toJavaStringUncached().getBytes());
-                }
-            } else {
-                if (PythonOS.getPythonOS() == PythonOS.PLATFORM_WIN32) {
-                    val = toTruffleStringUncached(entry.getValue());
-                } else {
-                    val = PFactory.createBytes(language, (entry.getValue().getBytes()));
-                }
+                val = toEnv(language, value);
             }
             environ.setItem(key, val);
         }
         if (PythonOS.getPythonOS() == PythonOS.PLATFORM_WIN32) {
             // XXX: Until we fix pip
-            environ.setItem(toTruffleStringUncached("PIP_NO_CACHE_DIR"), toTruffleStringUncached("0"));
+            environ.setItem(toEnv(language, "PIP_NO_CACHE_DIR"), toEnv(language, "0"));
         }
+        // XXX: Until a pyo3 version that doesn't have a different maximum version for GraalPy than
+        // CPython gets widespread
+        environ.setItem(toEnv(language, "UNSAFE_PYO3_SKIP_VERSION_CHECK"), toEnv(language, "1"));
         PythonModule posix;
         if (PythonOS.getPythonOS() == PythonOS.PLATFORM_WIN32) {
             posix = core.lookupBuiltinModule(T_NT);
         } else {
             posix = core.lookupBuiltinModule(T_POSIX);
         }
-        Object environAttr = posix.getAttribute(tsLiteral("environ"));
+        Object environAttr = posix.getAttribute(T_ENVIRON);
         ((PDict) environAttr).setDictStorage(environ.getDictStorage());
 
         if (posixLib.getBackend(posixSupport).toJavaStringUncached().equals("java")) {
@@ -370,6 +360,22 @@ public void postInitialize(Python3Core core) {
         }
     }
 
+    private static Object toEnv(PythonLanguage language, String value) {
+        if (PythonOS.getPythonOS() == PythonOS.PLATFORM_WIN32) {
+            return toTruffleStringUncached(value);
+        } else {
+            return PFactory.createBytes(language, value.getBytes());
+        }
+    }
+
+    private static Object toEnv(PythonLanguage language, TruffleString value) {
+        if (PythonOS.getPythonOS() == PythonOS.PLATFORM_WIN32) {
+            return value;
+        } else {
+            return PFactory.createBytes(language, value.toJavaStringUncached().getBytes());
+        }
+    }
+
     @Builtin(name = "putenv", minNumOfPositionalArgs = 2, parameterNames = {"name", "value"})
     @ArgumentClinic(name = "name", conversionClass = FsConverterNode.class)
     @ArgumentClinic(name = "value", conversionClass = FsConverterNode.class)

graalpython/lib-graalpython/patches/torch-2.4.1.patch

@@ -645,6 +645,46 @@ index 7c9b4be00..b8edbcfda 100644
  }
 
  ExtraState* init_and_set_extra_state(PyCodeObject* code) {
+diff --git a/torch/csrc/dynamo/guards.cpp b/torch/csrc/dynamo/guards.cpp
+index b7fde50a9..712e3613b 100644
+--- a/torch/csrc/dynamo/guards.cpp
++++ b/torch/csrc/dynamo/guards.cpp
+@@ -26,7 +26,7 @@
+ // https://github.com/python/cpython/blob/9afc6d102d16080535325f645849cd84eb04d57d/Objects/tupleobject.c#L1058-L1062
+ // To handle this, we manually copy the struct here and manually cast it to this
+ // new struct. From 3.12, the struct is included in the header file.
+-#if IS_PYTHON_3_12_PLUS
++#if 0 // GraalPy change
+ 
+ #define Py_BUILD_CORE
+ // Bring _PyTupleIterObject from the header file
+@@ -596,7 +596,7 @@ static PyObject* check_obj_id(PyObject* dummy, PyObject* args) {
+   }
+ }
+ 
+-#if IS_PYTHON_3_12_PLUS
++#if 0 // GraalPy change
+ 
+ static std::unordered_map<PyObject*, uint64_t> dict_version_map;
+ static int dict_version_watcher_id;
+@@ -617,7 +617,7 @@ static int dict_version_watch_callback(
+ #endif
+ 
+ static uint64_t get_dict_version_unchecked(PyObject* dict) {
+-#if IS_PYTHON_3_12_PLUS
++#if 0 // GraalPy change
+ 
+   if (PyDict_Watch(dict_version_watcher_id, dict)) {
+     throw std::runtime_error("failed to add version watcher to dict!");
+@@ -4013,7 +4013,7 @@ PyObject* torch_c_dynamo_guards_init() {
+       "install_no_tensor_aliasing_guard", install_no_tensor_aliasing_guard);
+ 
+ // initialize dict_version_map watcher for 3.12
+-#if IS_PYTHON_3_12_PLUS
++#if 0 // GraalPy change
+ 
+   dict_version_watcher_id = PyDict_AddWatcher(dict_version_watch_callback);
+   if (dict_version_watcher_id == -1) {
 diff --git a/torch/csrc/jit/python/python_tracer.cpp b/torch/csrc/jit/python/python_tracer.cpp
 index 92e6e2d3a..4d2ec0bfe 100644
 --- a/torch/csrc/jit/python/python_tracer.cpp

graalpython/lib-graalpython/patches/torch-2.7.0.patch

@@ -750,6 +750,46 @@ index b839fb26f..b7853029f 100644
  }
 
  #else
+diff --git a/torch/csrc/dynamo/guards.cpp b/torch/csrc/dynamo/guards.cpp
+index 60fb339d1..28d07734a 100644
+--- a/torch/csrc/dynamo/guards.cpp
++++ b/torch/csrc/dynamo/guards.cpp
+@@ -42,7 +42,7 @@
+ // To handle the older python versions, we manually copy the struct here and
+ // manually cast it to this new struct. For newer versions, the struct is
+ // included in the header file.
+-#if IS_PYTHON_3_12_PLUS
++#if 0 // GraalPy change
+ 
+ #define Py_BUILD_CORE
+ #include <internal/pycore_range.h> // _PyRangeIterObject
+@@ -707,7 +707,7 @@ static PyObject* check_obj_id(PyObject* dummy, PyObject* args) {
+   }
+ }
+ 
+-#if IS_PYTHON_3_12_PLUS
++#if 0 // GraalPy change
+ 
+ static std::unordered_map<PyObject*, uint64_t> dict_version_map;
+ static int dict_version_watcher_id;
+@@ -728,7 +728,7 @@ static int dict_version_watch_callback(
+ #endif
+ 
+ static uint64_t get_dict_version_unchecked(PyObject* dict) {
+-#if IS_PYTHON_3_12_PLUS
++#if 0 // GraalPy change
+ 
+   if (PyDict_Watch(dict_version_watcher_id, dict)) {
+     throw std::runtime_error("failed to add version watcher to dict!");
+@@ -6090,7 +6090,7 @@ PyObject* torch_c_dynamo_guards_init() {
+   py_m.def("profile_guard_manager", profile_guard_manager);
+ 
+ // initialize dict_version_map watcher for 3.12
+-#if IS_PYTHON_3_12_PLUS
++#if 0 // GraalPy change
+ 
+   dict_version_watcher_id = PyDict_AddWatcher(dict_version_watch_callback);
+   if (dict_version_watcher_id == -1) {
 diff --git a/torch/csrc/jit/python/python_tracer.cpp b/torch/csrc/jit/python/python_tracer.cpp
 index 876186743..041348257 100644
 --- a/torch/csrc/jit/python/python_tracer.cpp

graalpython/lib-python/3/test/test_subprocess.py

@@ -832,7 +832,8 @@ def is_env_var_to_ignore(n):
                     n == 'PWD' or n == 'SHLVL' or # Graalpython bash launcher
                     n.startswith('JAVA_MAIN_CLASS') or # JVM on MacOS
                     n == 'LD_LIBRARY_PATH' or n == 'DYLD_LIBRARY_PATH' or # Added by graalpy launcher
-                    n == 'LC_CTYPE') # Locale coercion triggered
+                    n == 'LC_CTYPE', # Locale coercion triggered
+                    n == 'UNSAFE_PYO3_SKIP_VERSION_CHECK') # GraalPy change
 
         with subprocess.Popen([sys.executable, "-c",
                                'import os; print(list(os.environ.keys()))'],