From 0f7151cabf73ff2afbd0e156c6a01007212c51f0 Mon Sep 17 00:00:00 2001 From: Tim Felgentreff Date: Sun, 19 Jan 2025 12:27:45 +0100 Subject: [PATCH 01/16] A little more warning about multi-context C extensions --- docs/user/Native-Extensions.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/user/Native-Extensions.md b/docs/user/Native-Extensions.md index 5746d1372a..ecbeb12830 100644 --- a/docs/user/Native-Extensions.md +++ b/docs/user/Native-Extensions.md @@ -50,5 +50,6 @@ The implementation also relies on `venv` to work, even if you are not using exte To support creating multiple GraalPy contexts that access native modules within the same JVM or Native Image, we need to isolate them from each other. The current strategy for this is to copy the libraries and modify them such that the dynamic library loader of the operating system will isolate them for us. To do this, all GraalPy contexts in the same process (not just those in the same engine!) must set the `python.IsolateNativeModules` option to `true`. +You should test your applications thoroughly if you want to use this feature, as there are many possiblities for native code to sidestep the library isolation through other process-wide global state. For more details on this, see [our implementation details](https://github.com/oracle/graalpython/blob/master/docs/contributor/IMPLEMENTATION_DETAILS.md#c-extension-copying). From 9535e621b644238053b35ad76c14ff25a124b526 Mon Sep 17 00:00:00 2001 From: Tim Felgentreff Date: Mon, 20 Jan 2025 20:49:47 +0100 Subject: [PATCH 02/16] return a better bases tuple for foreign types --- .../foreign/ForeignAbstractClassBuiltins.java | 26 ++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/foreign/ForeignAbstractClassBuiltins.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/foreign/ForeignAbstractClassBuiltins.java index ad90c12136..f4542e8005 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/foreign/ForeignAbstractClassBuiltins.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/foreign/ForeignAbstractClassBuiltins.java @@ -36,20 +36,25 @@ import com.oracle.graal.python.builtins.CoreFunctions; import com.oracle.graal.python.builtins.PythonBuiltinClassType; import com.oracle.graal.python.builtins.PythonBuiltins; +import com.oracle.graal.python.builtins.objects.common.SequenceStorageNodes; +import com.oracle.graal.python.lib.PyObjectGetIter; import com.oracle.graal.python.nodes.function.PythonBuiltinBaseNode; import com.oracle.graal.python.nodes.function.builtins.PythonBinaryBuiltinNode; import com.oracle.graal.python.nodes.function.builtins.PythonUnaryBuiltinNode; import com.oracle.graal.python.runtime.GilNode; import com.oracle.graal.python.runtime.object.PFactory; +import com.oracle.graal.python.runtime.sequence.storage.SequenceStorage; import com.oracle.truffle.api.CompilerDirectives; import com.oracle.truffle.api.dsl.Bind; import com.oracle.truffle.api.dsl.Cached; import com.oracle.truffle.api.dsl.GenerateNodeFactory; import com.oracle.truffle.api.dsl.NodeFactory; import com.oracle.truffle.api.dsl.Specialization; +import com.oracle.truffle.api.frame.VirtualFrame; import com.oracle.truffle.api.interop.InteropLibrary; import com.oracle.truffle.api.interop.UnsupportedMessageException; import com.oracle.truffle.api.library.CachedLibrary; +import com.oracle.truffle.api.nodes.Node; /* * NOTE: We are not using IndirectCallContext here in this file @@ -67,10 +72,25 @@ protected List> getNodeFa @Builtin(name = J___BASES__, minNumOfPositionalArgs = 1, isGetter = true, isSetter = false) @GenerateNodeFactory abstract static class BasesNode extends PythonUnaryBuiltinNode { - @Specialization - static Object getBases(Object self, + @Specialization(limit = "2") + static Object getBases(VirtualFrame frame, Object self, + @Bind Node inliningTarget, + @CachedLibrary("self") InteropLibrary lib, + @Cached PyObjectGetIter getIter, + @Cached SequenceStorageNodes.CreateStorageFromIteratorNode createStorageFromIteratorNode, @Bind PythonLanguage language) { - return PFactory.createEmptyTuple(language); + if (lib.hasMetaParents(self)) { + try { + Object parents = lib.getMetaParents(self); + Object iterObj = getIter.execute(frame, inliningTarget, parents); + SequenceStorage storage = createStorageFromIteratorNode.execute(frame, iterObj); + return PFactory.createTuple(language, storage); + } catch (UnsupportedMessageException e) { + throw CompilerDirectives.shouldNotReachHere(); + } + } else { + return PFactory.createEmptyTuple(language); + } } } From 7760b1e5c335cfeb5bf7e0b9b2323284f11b59dd Mon Sep 17 00:00:00 2001 From: Tim Felgentreff Date: Mon, 20 Jan 2025 20:50:01 +0100 Subject: [PATCH 03/16] fix identity comparison for foreign types --- .../src/tests/test_interop.py | 7 +++ .../builtins/objects/type/TypeNodes.java | 48 +++++++++++++++++-- 2 files changed, 51 insertions(+), 4 deletions(-) diff --git a/graalpython/com.oracle.graal.python.test/src/tests/test_interop.py b/graalpython/com.oracle.graal.python.test/src/tests/test_interop.py index 63e26b2579..ed0c87c309 100644 --- a/graalpython/com.oracle.graal.python.test/src/tests/test_interop.py +++ b/graalpython/com.oracle.graal.python.test/src/tests/test_interop.py @@ -720,6 +720,13 @@ def test_super(self): super(list, l).remove(0) # ArrayList#remove(int index) assert l == [6] + def test_issubclass_isinstance(self): + from java.util import ArrayList, List + assert issubclass(ArrayList, List) + assert issubclass(ArrayList, ArrayList) + assert isinstance(ArrayList(), List) + assert isinstance(ArrayList(), ArrayList) + def test_java_array(self): import java il = java.type("int[]")(20) diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/type/TypeNodes.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/type/TypeNodes.java index deb731b8e3..2a63cdfcb2 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/type/TypeNodes.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/type/TypeNodes.java @@ -109,6 +109,7 @@ import com.oracle.graal.python.builtins.modules.WeakRefModuleBuiltinsFactory; import com.oracle.graal.python.builtins.modules.cext.PythonCextTypeBuiltins.GraalPyPrivate_Type_AddMember; import com.oracle.graal.python.builtins.objects.PNone; +import com.oracle.graal.python.builtins.objects.PythonAbstractObject; import com.oracle.graal.python.builtins.objects.cell.PCell; import com.oracle.graal.python.builtins.objects.cext.PythonAbstractNativeObject; import com.oracle.graal.python.builtins.objects.cext.PythonNativeClass; @@ -222,6 +223,7 @@ import com.oracle.truffle.api.CompilerDirectives.ValueType; import com.oracle.truffle.api.HostCompilerDirectives.InliningCutoff; import com.oracle.truffle.api.RootCallTarget; +import com.oracle.truffle.api.TruffleLanguage.Env; import com.oracle.truffle.api.dsl.Bind; import com.oracle.truffle.api.dsl.Cached; import com.oracle.truffle.api.dsl.Cached.Exclusive; @@ -237,6 +239,8 @@ import com.oracle.truffle.api.frame.Frame; import com.oracle.truffle.api.frame.VirtualFrame; import com.oracle.truffle.api.interop.InteropLibrary; +import com.oracle.truffle.api.interop.UnknownIdentifierException; +import com.oracle.truffle.api.interop.UnsupportedMessageException; import com.oracle.truffle.api.library.CachedLibrary; import com.oracle.truffle.api.nodes.ControlFlowException; import com.oracle.truffle.api.nodes.Node; @@ -1482,23 +1486,23 @@ static boolean doManaged(PythonManagedClass left, PythonManagedClass right) { } @Specialization - static boolean doManaged(PythonBuiltinClassType left, PythonBuiltinClassType right) { + static boolean doTypeType(PythonBuiltinClassType left, PythonBuiltinClassType right) { return left == right; } @Specialization - static boolean doManaged(PythonBuiltinClassType left, PythonBuiltinClass right) { + static boolean doTypeClass(PythonBuiltinClassType left, PythonBuiltinClass right) { return left == right.getType(); } @Specialization - static boolean doManaged(PythonBuiltinClass left, PythonBuiltinClassType right) { + static boolean doClassType(PythonBuiltinClass left, PythonBuiltinClassType right) { return left.getType() == right; } @Specialization @InliningCutoff - static boolean doNativeSingleContext(PythonAbstractNativeObject left, PythonAbstractNativeObject right, + static boolean doNative(PythonAbstractNativeObject left, PythonAbstractNativeObject right, @CachedLibrary(limit = "1") InteropLibrary lib) { if (left == right) { return true; @@ -1509,6 +1513,42 @@ static boolean doNativeSingleContext(PythonAbstractNativeObject left, PythonAbst return lib.isIdentical(left.getPtr(), right.getPtr(), lib); } + @Specialization(guards = {"!isAnyPythonObject(left)", "!isAnyPythonObject(right)"}) + @InliningCutoff + static boolean doOther(Object left, Object right, + @Bind PythonContext context, + @CachedLibrary(limit = "2") InteropLibrary lib) { + if (left == right) { + return true; + } + if (lib.isMetaObject(left) && lib.isMetaObject(right)) { + // *sigh*... Host classes have split personality with a "static" and a "class" + // side, and that affects identity comparisons. And they report their "class" sides + // as bases, but importing from Java gives you the "static" side. + Env env = context.getEnv(); + if (env.isHostObject(left) && env.isHostObject(right)) { + // the activation of isMemberReadable and later readMember serves as branch + // profile + boolean leftIsStatic = lib.isMemberReadable(left, "class"); + if (leftIsStatic != lib.isMemberReadable(right, "class")) { + try { + if (leftIsStatic) { + left = lib.readMember(left, "class"); + } else { + right = lib.readMember(right, "class"); + } + } catch (UnsupportedMessageException | UnknownIdentifierException e) { + throw CompilerDirectives.shouldNotReachHere(e); + } + } + } + if (lib.isIdentical(left, right, lib)) { + return true; + } + } + return false; + } + @Fallback static boolean doOther(@SuppressWarnings("unused") Object left, @SuppressWarnings("unused") Object right) { return false; From e8cc70170f931cf35554394fe63a4bb5214ffb0e Mon Sep 17 00:00:00 2001 From: Tim Felgentreff Date: Mon, 20 Jan 2025 21:01:03 +0100 Subject: [PATCH 04/16] New style subclassing from Java --- .../builtins/modules/BuiltinFunctions.java | 10 +- .../lib-graalpython/__graalpython__.py | 114 +++++++++++++++++- 2 files changed, 118 insertions(+), 6 deletions(-) diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/BuiltinFunctions.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/BuiltinFunctions.java index eda0903d5f..ae15bcfc23 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/BuiltinFunctions.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/BuiltinFunctions.java @@ -2434,22 +2434,22 @@ public abstract static class BuildClassNode extends PythonVarargsBuiltinNode { // No BoundaryCallContext: calls only internal well-behaved Python code @TruffleBoundary - private static Object buildJavaClass(Object namespace, TruffleString name, Object base) { + private static Object buildJavaClass(Object namespace, TruffleString name, Object base, PKeyword[] keywords) { // uncached PythonContext get, since this code path is slow in any case Object module = PythonContext.get(null).lookupBuiltinModule(T___GRAALPYTHON__); Object buildFunction = PyObjectLookupAttr.executeUncached(module, T_BUILD_JAVA_CLASS); - return CallNode.executeUncached(buildFunction, namespace, name, base); + return CallNode.executeUncached(buildFunction, new Object[]{namespace, name, base}, keywords); } @InliningCutoff private static Object buildJavaClass(VirtualFrame frame, Node inliningTarget, PythonLanguage language, PFunction function, Object[] arguments, - CallDispatchers.FunctionCachedInvokeNode invokeBody, + PKeyword[] keywords, CallDispatchers.FunctionCachedInvokeNode invokeBody, TruffleString name) { PDict ns = PFactory.createDict(language, new DynamicObjectStorage(language)); Object[] args = PArguments.create(0); PArguments.setSpecialArgument(args, ns); invokeBody.execute(frame, inliningTarget, function, args); - return buildJavaClass(ns, name, arguments[1]); + return buildJavaClass(ns, name, arguments[1], keywords); } @Specialization @@ -2492,7 +2492,7 @@ protected Object doItNonFunction(VirtualFrame frame, Object function, Object[] a if (arguments.length == 2 && env.isHostObject(arguments[1]) && env.asHostObject(arguments[1]) instanceof Class) { // we want to subclass a Java class - return buildJavaClass(frame, inliningTarget, language, (PFunction) function, arguments, invokeBody, name); + return buildJavaClass(frame, inliningTarget, language, (PFunction) function, arguments, keywords, invokeBody, name); } class InitializeBuildClass { diff --git a/graalpython/lib-graalpython/__graalpython__.py b/graalpython/lib-graalpython/__graalpython__.py index 1f9cf47c7f..382788edc6 100644 --- a/graalpython/lib-graalpython/__graalpython__.py +++ b/graalpython/lib-graalpython/__graalpython__.py @@ -142,7 +142,17 @@ def import_current_as_named_module_with_delegate(module, module_name, delegate_n @builtin -def build_java_class(module, ns, name, base): +def build_java_class(module, ns, name, base, old_style=True): + if not old_style: + return build_new_style_java_class(ns, name, base) + import warnings + warnings.warn("Subclassing Java classes is going to change " + "to a new instance layout that is hopefully " + "more intuitive. Pass the keyword old_style=False " + "to your class definition to try the new style. " + "The new style will become the default in the next " + "release and the old style will be removed soon after.", DeprecationWarning, 1) + ns['__super__'] = None # place where store the original java class when instance is created ExtenderClass = type("PythonJavaExtenderClass", (object, ), ns) HostAdapter = __graalpython__.extend(base) @@ -158,3 +168,105 @@ def factory (cls, *args): resultClass.__new__ = classmethod(factory) return resultClass + + +@builtin +def build_new_style_java_class(module, ns, name, base): + import polyglot + + # First, generate the Java subclass using the Truffle API. Instances of + # this class is what we want to generate. + JavaClass = __graalpython__.extend(base) + + # Second, generate the delegate object class. Code calling from Java will + # end up delegating methods to an instance of this type and the Java object + # will use this delegate instance to manage dynamic attributes. + # + # The __init__ function would not do what the user thinks, so we take it + # out and call it explicitly in the factory below. The `self` passed into + # those Python-defined methods is the delegate instance, but that would be + # confusing for users. So we wrap all methods to get to the Java instance + # and pass that one as `self`. + delegate_namespace = dict(**ns) + delegate_namespace["__java_init__"] = delegate_namespace.pop("__init__", lambda self, *a, **kw: None) + + def python_to_java_decorator(fun): + return lambda self, *args, **kwds: fun(self.__this__, *args, **kwds) + + for n, v in delegate_namespace.items(): + if type(v) == type(python_to_java_decorator): + delegate_namespace[n] = python_to_java_decorator(v) + DelegateClass = type(f"PythonDelegateClassFor{base}", (object,), delegate_namespace) + DelegateClass.__qualname__ = DelegateClass.__name__ + + # Third, generate the class used to inject into the MRO of the generated + # Java subclass. Code calling from Python will go through this class for + # lookup. + # + # The `self` passed into those Python-defined methods will be the Java + # instance. We add `__getattr__`, `__setattr__`, and `__delattr__` + # implementations to look to the Python delegate object when the Java-side + # lookup fails. For convenience, we also allow retrieving static fields + # from Java. + mro_namespace = dict(**ns) + + def java_getattr(self, name): + if name == "super": + return __graalpython__.super(self) + sentinel = object() + result = getattr(self.this, name, sentinel) + if result is sentinel: + return getattr(self.getClass().static, name) + else: + return result + + mro_namespace['__getattr__'] = java_getattr + mro_namespace['__setattr__'] = lambda self, name, value: setattr(self.this, name, value) + mro_namespace['__delattr__'] = lambda self, name: delattr(self.this, name) + + @classmethod + def factory(cls, *args, **kwds): + # create the delegate object + delegate = DelegateClass() + # create the Java object (remove the class argument and add the delegate instance) + java_object = polyglot.__new__(JavaClass, *(args[1:] + (delegate, ))) + delegate.__this__ = java_object + # call the __init__ function on the delegate object now that the Java instance is available + delegate.__java_init__(*args[1:], **kwds) + return java_object + + mro_namespace['__constructor__'] = factory + if '__new__' not in mro_namespace: + mro_namespace['__new__'] = classmethod(lambda cls, *args, **kwds: cls.__constructor__(*args, **kwds)) + MroClass = type(f"PythonMROMixinFor{base}", (object,), mro_namespace) + MroClass.__qualname__ = MroClass.__name__ + polyglot.register_interop_type(JavaClass, MroClass) + + # Finally, generate a factory that implements the factory and type checking + # methods and denies inheriting again + class FactoryMeta(type): + @property + def __bases__(self): + return (JavaClass,) + + def __instancecheck__(cls, obj): + return isinstance(obj, JavaClass) + + def __subclasscheck__(cls, derived): + return cls is derived or issubclass(derived, JavaClass) + + def __new__(mcls, name, bases, namespace): + if bases: + raise NotImplementedError("Grandchildren of Java classes are not supported") + return type.__new__(mcls, name, bases, namespace) + + class FactoryClass(metaclass=FactoryMeta): + @classmethod + def __new__(cls, *args, **kwds): + return MroClass.__new__(*args, **kwds) + + FactoryClass.__name__ = ns['__qualname__'].rsplit(".", 1)[-1] + FactoryClass.__qualname__ = ns['__qualname__'] + FactoryClass.__module__ = ns['__module__'] + + return FactoryClass From 4ee74358f74f060066193b1a105a824509bb7d12 Mon Sep 17 00:00:00 2001 From: Tim Felgentreff Date: Tue, 16 Sep 2025 10:59:55 +0200 Subject: [PATCH 05/16] Fix style --- .../com/oracle/graal/python/builtins/objects/type/TypeNodes.java | 1 - 1 file changed, 1 deletion(-) diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/type/TypeNodes.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/type/TypeNodes.java index 2a63cdfcb2..d1fa110ee6 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/type/TypeNodes.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/type/TypeNodes.java @@ -109,7 +109,6 @@ import com.oracle.graal.python.builtins.modules.WeakRefModuleBuiltinsFactory; import com.oracle.graal.python.builtins.modules.cext.PythonCextTypeBuiltins.GraalPyPrivate_Type_AddMember; import com.oracle.graal.python.builtins.objects.PNone; -import com.oracle.graal.python.builtins.objects.PythonAbstractObject; import com.oracle.graal.python.builtins.objects.cell.PCell; import com.oracle.graal.python.builtins.objects.cext.PythonAbstractNativeObject; import com.oracle.graal.python.builtins.objects.cext.PythonNativeClass; From e747684343535bdc3c13711682ad08a775c08e1a Mon Sep 17 00:00:00 2001 From: Benoit Daloze Date: Wed, 5 Nov 2025 13:48:29 +0000 Subject: [PATCH 06/16] Chain cause for debugging --- .../builtins/objects/foreign/ForeignAbstractClassBuiltins.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/foreign/ForeignAbstractClassBuiltins.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/foreign/ForeignAbstractClassBuiltins.java index f4542e8005..f951adf58a 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/foreign/ForeignAbstractClassBuiltins.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/foreign/ForeignAbstractClassBuiltins.java @@ -86,7 +86,7 @@ static Object getBases(VirtualFrame frame, Object self, SequenceStorage storage = createStorageFromIteratorNode.execute(frame, iterObj); return PFactory.createTuple(language, storage); } catch (UnsupportedMessageException e) { - throw CompilerDirectives.shouldNotReachHere(); + throw CompilerDirectives.shouldNotReachHere(e); } } else { return PFactory.createEmptyTuple(language); From 3e253875a079a8c1b9832e872771e3797a22cd88 Mon Sep 17 00:00:00 2001 From: Benoit Daloze Date: Wed, 5 Nov 2025 13:50:33 +0000 Subject: [PATCH 07/16] Make check a bit clearer --- .../oracle/graal/python/builtins/objects/type/TypeNodes.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/type/TypeNodes.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/type/TypeNodes.java index d1fa110ee6..82e54abdf2 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/type/TypeNodes.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/type/TypeNodes.java @@ -1529,7 +1529,8 @@ static boolean doOther(Object left, Object right, // the activation of isMemberReadable and later readMember serves as branch // profile boolean leftIsStatic = lib.isMemberReadable(left, "class"); - if (leftIsStatic != lib.isMemberReadable(right, "class")) { + boolean rightIsStatic = lib.isMemberReadable(right, "class"); + if (leftIsStatic != rightIsStatic) { try { if (leftIsStatic) { left = lib.readMember(left, "class"); From 5f8faec9bab35acb3666bf14fa7f77af6ce7617e Mon Sep 17 00:00:00 2001 From: Benoit Daloze Date: Wed, 5 Nov 2025 13:51:40 +0000 Subject: [PATCH 08/16] Add assertion that we are on the correct side of the class' split personality --- .../com/oracle/graal/python/builtins/objects/type/TypeNodes.java | 1 + 1 file changed, 1 insertion(+) diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/type/TypeNodes.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/type/TypeNodes.java index 82e54abdf2..faf720c0a6 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/type/TypeNodes.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/type/TypeNodes.java @@ -1535,6 +1535,7 @@ static boolean doOther(Object left, Object right, if (leftIsStatic) { left = lib.readMember(left, "class"); } else { + assert rightIsStatic; right = lib.readMember(right, "class"); } } catch (UnsupportedMessageException | UnknownIdentifierException e) { From 4a26a6e50cfdc2544fa15a49aa5c31db7dcd3567 Mon Sep 17 00:00:00 2001 From: Benoit Daloze Date: Wed, 5 Nov 2025 15:01:49 +0100 Subject: [PATCH 09/16] Tweak precision of type check for new style Java subclasses --- .../graal/python/builtins/objects/type/TypeNodes.java | 8 ++++---- graalpython/lib-graalpython/__graalpython__.py | 6 +++--- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/type/TypeNodes.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/type/TypeNodes.java index faf720c0a6..9da09488f9 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/type/TypeNodes.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/type/TypeNodes.java @@ -1512,15 +1512,15 @@ static boolean doNative(PythonAbstractNativeObject left, PythonAbstractNativeObj return lib.isIdentical(left.getPtr(), right.getPtr(), lib); } - @Specialization(guards = {"!isAnyPythonObject(left)", "!isAnyPythonObject(right)"}) + @Specialization(guards = {"isForeignObject(left)", "isForeignObject(right)"}) @InliningCutoff static boolean doOther(Object left, Object right, @Bind PythonContext context, @CachedLibrary(limit = "2") InteropLibrary lib) { - if (left == right) { - return true; - } if (lib.isMetaObject(left) && lib.isMetaObject(right)) { + if (left == right) { + return true; + } // *sigh*... Host classes have split personality with a "static" and a "class" // side, and that affects identity comparisons. And they report their "class" sides // as bases, but importing from Java gives you the "static" side. diff --git a/graalpython/lib-graalpython/__graalpython__.py b/graalpython/lib-graalpython/__graalpython__.py index 382788edc6..395ae140da 100644 --- a/graalpython/lib-graalpython/__graalpython__.py +++ b/graalpython/lib-graalpython/__graalpython__.py @@ -142,13 +142,13 @@ def import_current_as_named_module_with_delegate(module, module_name, delegate_n @builtin -def build_java_class(module, ns, name, base, old_style=True): - if not old_style: +def build_java_class(module, ns, name, base, new_style=False): + if new_style: return build_new_style_java_class(ns, name, base) import warnings warnings.warn("Subclassing Java classes is going to change " "to a new instance layout that is hopefully " - "more intuitive. Pass the keyword old_style=False " + "more intuitive. Pass the keyword new_style=True " "to your class definition to try the new style. " "The new style will become the default in the next " "release and the old style will be removed soon after.", DeprecationWarning, 1) From 9b2b685648771574e10bbf8aab7758d6f61a06a5 Mon Sep 17 00:00:00 2001 From: Tim Felgentreff Date: Fri, 11 Jul 2025 11:01:03 +0200 Subject: [PATCH 10/16] Add script to build gensim with appropriate gcc-toolset --- scripts/wheelbuilder/darwin/gensim.sh | 42 ++++++++++++++++++++++++ scripts/wheelbuilder/linux/gensim.sh | 47 +++++++++++++++++++++++++++ scripts/wheelbuilder/linux/numpy.sh | 5 +++ 3 files changed, 94 insertions(+) create mode 100755 scripts/wheelbuilder/darwin/gensim.sh create mode 100755 scripts/wheelbuilder/linux/gensim.sh diff --git a/scripts/wheelbuilder/darwin/gensim.sh b/scripts/wheelbuilder/darwin/gensim.sh new file mode 100755 index 0000000000..80409d25c4 --- /dev/null +++ b/scripts/wheelbuilder/darwin/gensim.sh @@ -0,0 +1,42 @@ +# Copyright (c) 2024, 2025, Oracle and/or its affiliates. All rights reserved. +# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. +# +# The Universal Permissive License (UPL), Version 1.0 +# +# Subject to the condition set forth below, permission is hereby granted to any +# person obtaining a copy of this software, associated documentation and/or +# data (collectively the "Software"), free of charge and under any and all +# copyright rights in the Software, and any and all patent rights owned or +# freely licensable by each licensor hereunder covering either (i) the +# unmodified Software as contributed to or provided by such licensor, or (ii) +# the Larger Works (as defined below), to deal in both +# +# (a) the Software, and +# +# (b) any piece of software and/or hardware listed in the lrgrwrks.txt file if +# one is included with the Software each a "Larger Work" to which the Software +# is contributed by such licensors), +# +# without restriction, including without limitation the rights to copy, create +# derivative works of, display, perform, and distribute the Software and make, +# use, sell, offer for sale, import, export, have made, and have sold the +# Software and the Larger Work(s), and to sublicense the foregoing rights on +# either these or other terms. +# +# This license is subject to the following condition: +# +# The above copyright notice and either this complete permission notice or at a +# minimum a reference to the UPL must be included in all copies or substantial +# portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +if [ -n "$GITHUB_RUN_ID" ]; then + brew install gcc openblas +fi diff --git a/scripts/wheelbuilder/linux/gensim.sh b/scripts/wheelbuilder/linux/gensim.sh new file mode 100755 index 0000000000..d977ffc283 --- /dev/null +++ b/scripts/wheelbuilder/linux/gensim.sh @@ -0,0 +1,47 @@ +# Copyright (c) 2024, 2025, Oracle and/or its affiliates. All rights reserved. +# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. +# +# The Universal Permissive License (UPL), Version 1.0 +# +# Subject to the condition set forth below, permission is hereby granted to any +# person obtaining a copy of this software, associated documentation and/or +# data (collectively the "Software"), free of charge and under any and all +# copyright rights in the Software, and any and all patent rights owned or +# freely licensable by each licensor hereunder covering either (i) the +# unmodified Software as contributed to or provided by such licensor, or (ii) +# the Larger Works (as defined below), to deal in both +# +# (a) the Software, and +# +# (b) any piece of software and/or hardware listed in the lrgrwrks.txt file if +# one is included with the Software each a "Larger Work" to which the Software +# is contributed by such licensors), +# +# without restriction, including without limitation the rights to copy, create +# derivative works of, display, perform, and distribute the Software and make, +# use, sell, offer for sale, import, export, have made, and have sold the +# Software and the Larger Work(s), and to sublicense the foregoing rights on +# either these or other terms. +# +# This license is subject to the following condition: +# +# The above copyright notice and either this complete permission notice or at a +# minimum a reference to the UPL must be included in all copies or substantial +# portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +if [ -n "$GITHUB_RUN_ID" ]; then + dnf install -y gcc-toolset-12-gcc-gfortran gcc-toolset-12-gcc-c++ openblas-devel + if [ -n "$1" ]; then + scl enable gcc-toolset-12 "pip wheel \"gensim==$1\"" + else + scl enable gcc-toolset-12 "pip wheel gensim" + fi +fi diff --git a/scripts/wheelbuilder/linux/numpy.sh b/scripts/wheelbuilder/linux/numpy.sh index 8cd8c11077..40a64fa1ca 100755 --- a/scripts/wheelbuilder/linux/numpy.sh +++ b/scripts/wheelbuilder/linux/numpy.sh @@ -39,4 +39,9 @@ if [ -n "$GITHUB_RUN_ID" ]; then dnf install -y gcc-toolset-12-gcc-gfortran openblas-devel + if [ -n "$1" ]; then + scl enable gcc-toolset-12 "pip wheel \"numpy==$1\"" + else + scl enable gcc-toolset-12 "pip wheel numpy" + fi fi From 7e6673feaf74cc4f1f4c3624d9b6f20da799c5c6 Mon Sep 17 00:00:00 2001 From: Tim Felgentreff Date: Fri, 11 Jul 2025 15:17:19 +0200 Subject: [PATCH 11/16] use ld_classic and no openblas for scipy builds on darwin --- scripts/wheelbuilder/darwin/scipy.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/wheelbuilder/darwin/scipy.sh b/scripts/wheelbuilder/darwin/scipy.sh index 5b6e8e24a8..5be933a8df 100755 --- a/scripts/wheelbuilder/darwin/scipy.sh +++ b/scripts/wheelbuilder/darwin/scipy.sh @@ -1,4 +1,4 @@ -# Copyright (c) 2024, Oracle and/or its affiliates. All rights reserved. +# Copyright (c) 2024, 2025, Oracle and/or its affiliates. All rights reserved. # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. # # The Universal Permissive License (UPL), Version 1.0 @@ -38,10 +38,10 @@ # SOFTWARE. if [ -n "$GITHUB_RUN_ID" ]; then - brew install gcc openblas pkg-config - export PKG_CONFIG_PATH=/opt/homebrew/opt/openblas/lib/pkgconfig + brew install gcc pkg-config fi -export FFLAGS=-fallow-argument-mismatch +export FFLAGS="-fallow-argument-mismatch" +export LDFLAGS="-Wl,-ld_classic" if [ -n "$1" ]; then pip wheel "scipy==$1" else From e1a02884cf0edca9ce32d159c09ff07a36038f08 Mon Sep 17 00:00:00 2001 From: Ivo Horak Date: Wed, 16 Jul 2025 12:43:43 +0200 Subject: [PATCH 12/16] Adjusting darwin scipy script to use GNU gcc and lld --- scripts/wheelbuilder/darwin/scipy.sh | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/scripts/wheelbuilder/darwin/scipy.sh b/scripts/wheelbuilder/darwin/scipy.sh index 5be933a8df..6185dd4615 100755 --- a/scripts/wheelbuilder/darwin/scipy.sh +++ b/scripts/wheelbuilder/darwin/scipy.sh @@ -38,10 +38,17 @@ # SOFTWARE. if [ -n "$GITHUB_RUN_ID" ]; then - brew install gcc pkg-config + brew install lld gcc pkg-config openblas fi -export FFLAGS="-fallow-argument-mismatch" -export LDFLAGS="-Wl,-ld_classic" + +export CC="gcc-15" +export CXX="g++-15" +export CXXFLAGS="-fuse-ld=lld" +export CFLAGS="-fuse-ld=lld" +export FFLAGS="-fuse-ld=lld" +export LDFLAGS="-Wl,-platform_version,macos,15.0.0,15.0.0" +export PKG_CONFIG_PATH="/opt/homebrew/opt/openblas/lib/pkgconfig" + if [ -n "$1" ]; then pip wheel "scipy==$1" else From db7a7bd135ca20e8de9826c1255ec584c611b94c Mon Sep 17 00:00:00 2001 From: Ivo Horak Date: Fri, 18 Jul 2025 15:51:01 +0200 Subject: [PATCH 13/16] Adding MacOS numpy 2.0.2 build script --- scripts/wheelbuilder/darwin/numpy.2.0.2.sh | 45 ++++++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100644 scripts/wheelbuilder/darwin/numpy.2.0.2.sh diff --git a/scripts/wheelbuilder/darwin/numpy.2.0.2.sh b/scripts/wheelbuilder/darwin/numpy.2.0.2.sh new file mode 100644 index 0000000000..bb49721f30 --- /dev/null +++ b/scripts/wheelbuilder/darwin/numpy.2.0.2.sh @@ -0,0 +1,45 @@ +# Copyright (c) 2024, Oracle and/or its affiliates. All rights reserved. +# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. +# +# The Universal Permissive License (UPL), Version 1.0 +# +# Subject to the condition set forth below, permission is hereby granted to any +# person obtaining a copy of this software, associated documentation and/or +# data (collectively the "Software"), free of charge and under any and all +# copyright rights in the Software, and any and all patent rights owned or +# freely licensable by each licensor hereunder covering either (i) the +# unmodified Software as contributed to or provided by such licensor, or (ii) +# the Larger Works (as defined below), to deal in both +# +# (a) the Software, and +# +# (b) any piece of software and/or hardware listed in the lrgrwrks.txt file if +# one is included with the Software each a "Larger Work" to which the Software +# is contributed by such licensors), +# +# without restriction, including without limitation the rights to copy, create +# derivative works of, display, perform, and distribute the Software and make, +# use, sell, offer for sale, import, export, have made, and have sold the +# Software and the Larger Work(s), and to sublicense the foregoing rights on +# either these or other terms. +# +# This license is subject to the following condition: +# +# The above copyright notice and either this complete permission notice or at a +# minimum a reference to the UPL must be included in all copies or substantial +# portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +if [ -n "$GITHUB_RUN_ID" ]; then + brew install gcc openblas +fi + +export CC="gcc-15" +export CXX="g++-15" \ No newline at end of file From 8d88d07958c2eae0ac90fe51698a37723a4cd476 Mon Sep 17 00:00:00 2001 From: Tim Felgentreff Date: Fri, 7 Nov 2025 14:06:01 +0100 Subject: [PATCH 14/16] Adapt new_style Java subclassing code to support super() and multiple levels of Python subclasses under Java --- CHANGELOG.md | 1 + .../src/tests/test_interop.py | 62 +++++++ .../lib-graalpython/__graalpython__.py | 163 +++++++++--------- 3 files changed, 148 insertions(+), 78 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1aed048653..08c77c63d2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ language runtime. The main focus is on user-observable behavior of the engine. * Allocation reporting via Truffle has been removed. Python object sizes were never reported correctly, so the data was misleading and there was a non-neglible overhead for object allocations even when reporting was inactive. * Better `readline` support via JLine. Autocompletion and history now works in `pdb` * Remove the intrinsified _ctypes module in favor of the native CPython version. This makes GraalPy's ctypes implementation more compatible and reduces the memory footprint of using ctypes. +* Add a new, more natural style of subclassing Java classes from Python. Multiple levels of inheritance are supported, and `super()` calls both in the constructor override via `__new__` as well as in Java method overrides work as expected. ## Version 25.0.1 * Allow users to keep going on unsupported JDK/OS/ARCH combinations at their own risk by opting out of early failure using `-Dtruffle.UseFallbackRuntime=true`, `-Dpolyglot.engine.userResourceCache=/set/to/a/writeable/dir`, `-Dpolyglot.engine.allowUnsupportedPlatform=true`, and `-Dpolyglot.python.UnsupportedPlatformEmulates=[linux|macos|windows]` and `-Dorg.graalvm.python.resources.exclude=native.files`. diff --git a/graalpython/com.oracle.graal.python.test/src/tests/test_interop.py b/graalpython/com.oracle.graal.python.test/src/tests/test_interop.py index ed0c87c309..66f9e8625a 100644 --- a/graalpython/com.oracle.graal.python.test/src/tests/test_interop.py +++ b/graalpython/com.oracle.graal.python.test/src/tests/test_interop.py @@ -1416,6 +1416,68 @@ def test_foreign_repl(self): assert repr(Integer).startswith(' Date: Fri, 7 Nov 2025 14:22:14 +0100 Subject: [PATCH 15/16] Fix copyrights --- CHANGELOG.md | 2 +- scripts/wheelbuilder/darwin/numpy.2.0.2.sh | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 08c77c63d2..0fb44b8089 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,7 +8,7 @@ language runtime. The main focus is on user-observable behavior of the engine. * Allocation reporting via Truffle has been removed. Python object sizes were never reported correctly, so the data was misleading and there was a non-neglible overhead for object allocations even when reporting was inactive. * Better `readline` support via JLine. Autocompletion and history now works in `pdb` * Remove the intrinsified _ctypes module in favor of the native CPython version. This makes GraalPy's ctypes implementation more compatible and reduces the memory footprint of using ctypes. -* Add a new, more natural style of subclassing Java classes from Python. Multiple levels of inheritance are supported, and `super()` calls both in the constructor override via `__new__` as well as in Java method overrides work as expected. +* Add a new, more natural style of subclassing Java classes from Python by passing the `new_style=True` keyword. Multiple levels of inheritance are supported, and `super()` calls both in the constructor override via `__new__` as well as in Java method overrides work as expected. ## Version 25.0.1 * Allow users to keep going on unsupported JDK/OS/ARCH combinations at their own risk by opting out of early failure using `-Dtruffle.UseFallbackRuntime=true`, `-Dpolyglot.engine.userResourceCache=/set/to/a/writeable/dir`, `-Dpolyglot.engine.allowUnsupportedPlatform=true`, and `-Dpolyglot.python.UnsupportedPlatformEmulates=[linux|macos|windows]` and `-Dorg.graalvm.python.resources.exclude=native.files`. diff --git a/scripts/wheelbuilder/darwin/numpy.2.0.2.sh b/scripts/wheelbuilder/darwin/numpy.2.0.2.sh index bb49721f30..0e9b3baec4 100644 --- a/scripts/wheelbuilder/darwin/numpy.2.0.2.sh +++ b/scripts/wheelbuilder/darwin/numpy.2.0.2.sh @@ -1,4 +1,4 @@ -# Copyright (c) 2024, Oracle and/or its affiliates. All rights reserved. +# Copyright (c) 2024, 2025, Oracle and/or its affiliates. All rights reserved. # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. # # The Universal Permissive License (UPL), Version 1.0 @@ -42,4 +42,4 @@ if [ -n "$GITHUB_RUN_ID" ]; then fi export CC="gcc-15" -export CXX="g++-15" \ No newline at end of file +export CXX="g++-15" From 2c3046e833bad9596c7318c14a39554067f2ec32 Mon Sep 17 00:00:00 2001 From: Tim Felgentreff Date: Fri, 7 Nov 2025 15:13:56 +0100 Subject: [PATCH 16/16] Add missing call to `__init__`, since Java subclasses' `__new__` returns a non-subinstance --- .../com.oracle.graal.python.test/src/tests/test_interop.py | 2 +- graalpython/lib-graalpython/__graalpython__.py | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/graalpython/com.oracle.graal.python.test/src/tests/test_interop.py b/graalpython/com.oracle.graal.python.test/src/tests/test_interop.py index 66f9e8625a..48cdf8b582 100644 --- a/graalpython/com.oracle.graal.python.test/src/tests/test_interop.py +++ b/graalpython/com.oracle.graal.python.test/src/tests/test_interop.py @@ -1423,7 +1423,7 @@ class PythonLevel(Level, new_style=True): def __new__(cls, name="default name", level=2): return super().__new__(cls, name, level) - def __init__(self): + def __init__(self, *args, **kwarg): self.misc_value = 42 def getName(self): diff --git a/graalpython/lib-graalpython/__graalpython__.py b/graalpython/lib-graalpython/__graalpython__.py index a2b282c308..c88326028b 100644 --- a/graalpython/lib-graalpython/__graalpython__.py +++ b/graalpython/lib-graalpython/__graalpython__.py @@ -204,7 +204,9 @@ def __delattr__(self, name): polyglot.register_interop_type(JavaClass, MroClass, allow_method_overwrites=True) # A class to make sure that the returned Python class can be used for - # issubclass and isinstance checks with the Java instances + # issubclass and isinstance checks with the Java instances, and to wrap all + # methods in created subclasses to provide the proper `self` (the Java + # instance) and still make `super()` work. class JavaSubclassMeta(type): @property def __bases__(self): @@ -274,6 +276,7 @@ def __new__(cls, *args, **kwds): delegate = object.__new__(cls) java_object = polyglot.__new__(JavaClass, *(args + (delegate,))) delegate.__this__ = java_object + delegate.__init__(*args, **kwds) return java_object return type(name, (DelegateSuperclass,), ns)