[GR-65639][GR-67183][GR-59702] Multiple smaller fixes for release.

PullRequest: graalpython/3909

Author: timfel

ci.jsonnet

@@ -303,7 +303,6 @@
         "python-unittest-retagger": ut_retagger + platform_spec(no_jobs) + platform_spec({
             "linux:amd64:jdk-latest"     : weekly    + t("20:00:00"),
             "linux:aarch64:jdk-latest"   : weekly    + t("20:00:00"),
-            "darwin:amd64:jdk-latest"    : weekly    + t("20:00:00"),
             "darwin:aarch64:jdk-latest"  : weekly    + t("20:00:00"),
             "windows:amd64:jdk-latest"   : weekly    + t("20:00:00"),
         }),

docs/site/01-docs.md

@@ -8,6 +8,13 @@ permalink: docs/
 {% gfm_docs ../user/Python-Runtime.md %}
 {% gfm_docs ../user/Performance.md %}
 {% gfm_docs ../user/Python-on-JVM.md %}
+
+<h3 id="python-context-options">
+<a href="#python-context-options" class="anchor-link">Python Context Options</a>
+</h3>
+Below are the options you can set on contexts for GraalPy.
+{% python_options ../../graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/PythonOptions.java %}
+
 {% gfm_docs ../user/Native-Images-with-Python.md %}
 {% gfm_docs ../user/Python-Standalone-Applications.md %}
 {% gfm_docs ../user/Interoperability.md %}

docs/site/_plugins/python_options_plugin.rb

@@ -0,0 +1,76 @@
+require 'cgi'
+require 'kramdown'
+require 'kramdown-parser-gfm'
+
+module GraalPy
+  class PythonOptions < Liquid::Tag
+    def initialize(tag_name, text, tokens)
+      super
+      @text = text.strip
+    end
+
+    def render(context)
+      content = File.read(@text)
+      options = {}
+
+      regexp = /
+        ^(\s*@EngineOption\s*)?                                  # Optional @EngineOption
+        \s*@Option\s*\(                                          # @Option(
+        (.*?)                                                    # non-greedy match content
+        \)\s*(?:\/\/)?\s*                                        # )
+        public\s+static\s+final\s+OptionKey<[^>]+>\s+(\w+)       # field name (e.g. InitialLocale)
+      /mx
+
+      help_regexp = /
+          help\s*=\s*((?:"[^"]*"\s*(?:\+\s*"[^"]*"\s*)*))        # help = "..."+ "...",
+      /mx
+
+      stability_regexp = /stability\s*=\s*OptionStability\.(\w+)/
+
+      category_regexp = /category\s*=\s*OptionCategory\.(\w+)/
+
+      usage_regexp = /usageSyntax\s*=\s*((?:"[^"]*"\s*(?:\+\s*"[^"]*"\s*)*))/
+
+      content.scan(regexp) do |engine_option, content, field|
+        engine_option = !!engine_option
+
+        help = content.match(help_regexp)
+        help = help[1].scan(/"([^"]*)"/).flatten.join(" ").gsub(/\s+/, " ").strip if help
+
+        stable = content.match(stability_regexp)
+        stable = stable[1] == "STABLE" if stable
+
+        category = content.match(category_regexp)
+        category = category[1] if category
+
+        usage = content.match(usage_regexp)
+        usage = usage[1].scan(/"([^"]*)"/).flatten.join(" ").gsub(/\s+/, " ").strip if usage
+
+        if help && !help.empty?
+          help << "." unless help.end_with? "."
+          help = CGI::escapeHTML help
+          help << " Accepts: <code>#{CGI::escapeHTML usage}</code>" if usage and !usage.empty?
+          options[field] = { help: help, category: category, stable: stable, engine_option: engine_option }
+        end
+      end
+
+      doc_content = options.select do |name, opt|
+        opt[:stable] && ["USER", "EXPERT"].include?(opt[:category])
+      end.sort_by do |name, opt|
+        [opt[:category] == "USER" ? 0 : 1, name.downcase]
+      end.map do |name, opt|
+        <<~HTML
+        <p>
+          <strong>#{name}</strong>
+          #{' (Has to be the same for all Contexts in an Engine)' if opt[:engine_option]}
+        </p>
+        <p>
+        &nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;#{opt[:help]}
+        </p>
+        HTML
+      end.join("\n")
+    end
+  end
+end
+
+Liquid::Template.register_tag('python_options', GraalPy::PythonOptions)

docs/user/Troubleshooting.md

@@ -1,41 +1,47 @@
 # GraalPy Troubleshooting
 
-[[TOC]]
-
 ## GraalPy Embedding
 
-#### VirtualFileSystem cannot load a file
+### VirtualFileSystem cannot load a file
+
 There are situations where a file cannot be loaded even though it is part of the Virtual Filesystem resources.
-GraalPy tries to prevent such situations by automatically [extracting](Embedding-Build-Tools.md#extracting-files-from-virtual-filesystem) 
+GraalPy tries to prevent such situations by automatically [extracting](Embedding-Build-Tools.md#extracting-files-from-virtual-filesystem)
 some well known files to the real filesystem, but if you see an error like:
+
+```bash
+ImportError: cannot load /graalpy_vfs/venv/lib/python3.11/site-packages/_cffi_backend.graalpy250dev09433ef706-311-native-aarch64-darwin.so:
+NFIUnsatisfiedLinkError: dlopen(/graalpy_vfs/venv/lib/python3.11/site-packages/_cffi_backend.graalpy250dev09433ef706-311-native-aarch64-darwin.so, 0x0002):
 ```
-ImportError: cannot load /graalpy_vfs/venv/lib/python3.11/site-packages/_cffi_backend.graalpy250dev09433ef706-311-native-aarch64-darwin.so: 
-NFIUnsatisfiedLinkError: dlopen(/graalpy_vfs/venv/lib/python3.11/site-packages/_cffi_backend.graalpy250dev09433ef706-311-native-aarch64-darwin.so, 0x0002): 
-```
+
 then the default behavior did not work as intended.
 
 Depending on how you [deploy Python resources](Embedding-Build-Tools.md#deployment) in your application, you can try one of the following:
 - if you need to package resources within your Jar or Native Image executable:
-  - if the problematic file is not one of the following types: `.so`, `.dylib`, `.pyd`, `.dll`, or `.ttf`, which are extracted to 
-  the real filesystem by default, you can simply attempt to add it to the extraction filter using: 
-  ```java 
+  - if the problematic file is not one of the following types: `.so`, `.dylib`, `.pyd`, `.dll`, or `.ttf`, which are extracted to
+  the real filesystem by default, you can simply attempt to add it to the extraction filter using:
+
+  ```java
   VirtualFileSystem.Builder.extractFilter(filter);
   ```
+
   - if the previous does not help, it is also possible to extract all Python resources into a user-defined directory before creating a GraalPy
     context, and then configure the context to use that directory:
+
   ```java
   // extract the Python resources from the jar or native image into a given directory
   GraalPyResources.extractVirtualFileSystemResources(VirtualFileSystem.create(), externalResourceDirectoryPath);
   // create a GraalPy context configured with an external Python resource directory
   Context context = GraalPyResources.contextBuilder(externalResourceDirectoryPath).build();
   ```
-- or if you're able to ship resources in a separate directory, you have to set the `externalDirectory` tag in  
-  [Maven](Embedding-Build-Tools.md#graalpy-maven-plugin) or `externalDirectory` field in [Gradle](Embedding-Build-Tools.md#graalpy-gradle-plugin) 
-  and also configure the GraalPy context to use that directory as well: 
+- or if you're able to ship resources in a separate directory, you have to set the `externalDirectory` tag in
+  [Maven](Embedding-Build-Tools.md#graalpy-maven-plugin) or `externalDirectory` field in [Gradle](Embedding-Build-Tools.md#graalpy-gradle-plugin)
+  and also configure the GraalPy context to use that directory as well:
+
   ```java
   // create a Context configured with an external Python resource directory
   Context context = GraalPyResources.contextBuilder(externalResourceDirectoryPath).build();
   ```
+
   Please **note**, that if switching from Virtual FileSystem to an external directory, also all **user files** from the previous
   Virtual FileSystem resource root have to be moved into that directory as well.
 
@@ -44,35 +50,36 @@ For more details about the Python resources in GraalPy Embedding please refer to
 For more details about GraalPy context and Virtual FileSystem configuration please refer to [GraalPyResources](https://github.com/oracle/graalpython/blob/master/graalpython/org.graalvm.python.embedding/src/org/graalvm/python/embedding/GraalPyResources.java) and
 [VirtualFileSystem](https://github.com/oracle/graalpython/blob/master/graalpython/org.graalvm.python.embedding/src/org/graalvm/python/embedding/VirtualFileSystem.java) javadoc.
 
-#### Issues with GraalPy 'java' posix backend
-The Virtual FileSystem is built on the Truffle filesystem and relies on the GraalPy Java POSIX backend. 
-Unfortunately, certain Python packages bypass Python's I/O and directly access files through their 
+### Issues with GraalPy 'java' posix backend
+
+The Virtual FileSystem is built on the Truffle filesystem and relies on the GraalPy Java POSIX backend.
+Unfortunately, certain Python packages bypass Python's I/O and directly access files through their
 native extensions. If you encounter an error like:
 ```
 NotImplementedError: 'PyObject_AsFileDescriptor' not supported when using 'java' posix backend
 ```
 then you have to override the default `java` GraalPy backend option byt setting the `native` POSIX backend
-and completely omit the Virtual FileSystem at runtime. 
+and completely omit the Virtual FileSystem at runtime.
 
-Depending on how you [deploy Python resources](Embedding-Build-Tools.md#deployment) in your application, 
+Depending on how you [deploy Python resources](Embedding-Build-Tools.md#deployment) in your application,
 you can do one of the following:
 
 - if you need to package Python resources within your Jar or Native Image executable, then:
   ```java
   // extract the Python resources from the jar or native image into a given directory
   GraalPyResources.extractVirtualFileSystemResources(VirtualFileSystem.create(), externalResourceDirectoryPath);
-  // create a Context.Builder configured with an external python resource directory 
+  // create a Context.Builder configured with an external python resource directory
   Builder builder = GraalPyResources.contextBuilder(externalResourceDirectoryPath);
   // override the python.PosixModuleBackend option with "native"
   builder.option("python.PosixModuleBackend", "native");
   // create a context
   Context context = builder.build();
   ```
-- or if you're able to ship Python resources in a separate directory, you have to set the `externalDirectory` tag in  
+- or if you're able to ship Python resources in a separate directory, you have to set the `externalDirectory` tag in
   [Maven](Embedding-Build-Tools.md#graalpy-maven-plugin) or `externalDirectory` field in [Gradle](Embedding-Build-Tools.md#graalpy-gradle-plugin)
   and configure the GraalPy context accordingly:
   ```java
-  // create a Context.Builder configured with an external python resource directory 
+  // create a Context.Builder configured with an external python resource directory
   Builder builder = GraalPyResources.contextBuilder(externalResourceDirectoryPath);
   // override the python.PosixModuleBackend option with "native"
   builder.option("python.PosixModuleBackend", "native");
@@ -87,55 +94,65 @@ For more details about the Python resources in GraalPy Embedding please refer to
 For more details about GraalPy context and Virtual FileSystem configuration please refer to [GraalPyResources](https://github.com/oracle/graalpython/blob/master/graalpython/org.graalvm.python.embedding/src/org/graalvm/python/embedding/GraalPyResources.java) and
 [VirtualFileSystem](https://github.com/oracle/graalpython/blob/master/graalpython/org.graalvm.python.embedding/src/org/graalvm/python/embedding/VirtualFileSystem.java) javadoc.
 
-### Maven and Gradle applications
+### ImportError reports "unknown location"
 
-####  ImportError reports "unknown location"
-A possible cause of an `ImportError` ending with `(unknown location)` when running a GraalPy Java Embedding project might be 
+A possible cause of an `ImportError` ending with `(unknown location)` when running a GraalPy Java Embedding project might be
 that an embedded Python package was built for a different operating system. If you see an error like the following:
-```
+
+```bash
 ImportError: cannot import name 'exceptions' from 'cryptography.hazmat.bindings._rust' (unknown location)
-``` 
+```
+
 You probably need to rebuild your project on the correct operating system before running it.
 
-#### GraalVM JDK Compatibility
-To enable runtime compilation when running GraalPy from a Maven or Gradle application, 
-you must use versions of GraalPy and the Polyglot API dependencies that are compatible 
+### GraalVM JDK Compatibility
+
+To enable runtime compilation when running GraalPy from a Maven or Gradle application,
+you must use versions of GraalPy and the Polyglot API dependencies that are compatible
 with the specified GraalVM JDK version. If you see errors like the following:
 
-```
+```bash
 Your Java runtime '23.0.1+11-jvmci-b01' with compiler version '24.1.1' is incompatible with polyglot version '24.1.0'.
 ```
-You need to keep the versions of your GraalPy and Polyglot dependencies according to the error message, 
-so either upgrade the version of your JDK or your polyglot and GraalPy dependencies.  
 
-Note, this can also apply to cases when your dependencies are transitively pulled in by another artifact, 
+You need to keep the versions of your GraalPy and Polyglot dependencies according to the error message,
+so either upgrade the version of your JDK or your polyglot and GraalPy dependencies.
+
+Note, this can also apply to cases when your dependencies are transitively pulled in by another artifact,
 e.g. micronaut-graalpy.
 
-#### The following artifacts could not be resolved: org.graalvm.python:python-language-enterprise
+### The following artifacts could not be resolved: org.graalvm.python:python-language-enterprise
+
 `python-language-enterprise` was discontinued, use `org.graalvm.polyglot:python` instead.
 
-#### Issues with Meson build system when installing Python packages on OSX with Maven or Gradle GraalPy plugin
+### Issues with Meson build system when installing Python packages on OSX with Maven or Gradle GraalPy plugin
+
 Errors like the following:
-```
+
+```bash
 ../meson.build:1:0: ERROR: Failed running 'cython', binary or interpreter not executable.
 ```
 
-could be caused by the GraalPy launcher used internally by the Maven or Gradle GraalPy plugin 
-for installing Python packages. Currently, there is no straightforward solution. However, 
-a workaround is to set the Java system property graalpy.vfs.venvLauncher to a launcher from 
+could be caused by the GraalPy launcher used internally by the Maven or Gradle GraalPy plugin
+for installing Python packages. Currently, there is no straightforward solution. However,
+a workaround is to set the Java system property graalpy.vfs.venvLauncher to a launcher from
 a downloaded [GraalPy](https://github.com/oracle/graalpython/releases/) distribution with a version
 matching the GraalPy Maven artifacts version.
 
 e.g.
-```
+
+```bash
 mvn package -Dgraalpy.vfs.venvLauncher={graalpy_directroy}/Contents/Home/bin/graalpy
 ```
 
-#### No language and polyglot implementation was found on the module-path.
+### No language and polyglot implementation was found on the module-path.
+
 If you see an error like:
+
+```bash
+java.lang.IllegalStateException: No language and polyglot implementation was found on the module-path. Make sure at last one language is added to the module-path.
 ```
-java.lang.IllegalStateException: No language and polyglot implementation was found on the module-path. Make sure at last one language is added to the module-path. 
-```
-you are probably missing the python langauge dependency in your Maven of Gradle configuration file. 
+
+you are probably missing the python langauge dependency in your Maven of Gradle configuration file.
 You need to add either `org.graalvm.polyglot:python` or `org.graalvm.polyglot:python-community` to your dependencies.
 

graalpython/com.oracle.graal.python.cext/include/cpython/floatobject.h

@@ -1,4 +1,4 @@
-/* Copyright (c) 2023, 2024, Oracle and/or its affiliates.
+/* Copyright (c) 2023, 2025, Oracle and/or its affiliates.
  * Copyright (C) 1996-2023 Python Software Foundation
  *
  * Licensed under the PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2
@@ -12,6 +12,12 @@ typedef struct {
     double ob_fval;
 } PyFloatObject;
 
+// GraalPy addition
+typedef struct {
+    GraalPyObject ob_base;
+    double ob_fval;
+} GraalPyFloatObject;
+
 #define _PyFloat_CAST(op) \
     (assert(PyFloat_Check(op)), _Py_CAST(PyFloatObject*, op))
 
@@ -21,7 +27,11 @@ static inline double PyFloat_AS_DOUBLE(PyObject *op) {
 #if 0 // GraalPy change
     return _PyFloat_CAST(op)->ob_fval;
 #else // GraalPy change
-    return PyFloat_AsDouble(op);
+    if (points_to_py_handle_space(op)) {
+        return ((GraalPyFloatObject*) pointer_to_stub(op))->ob_fval;
+    } else {
+        return _PyFloat_CAST(op)->ob_fval;
+    }
 #endif // GraalPy change
 }
 #define PyFloat_AS_DOUBLE(op) PyFloat_AS_DOUBLE(_PyObject_CAST(op))

graalpython/com.oracle.graal.python.cext/include/object.h

@@ -1036,8 +1036,20 @@ static inline int PyType_CheckExact(PyObject *op) {
 #  define PyType_CheckExact(op) PyType_CheckExact(_PyObject_CAST(op))
 #endif
 
+// GraalPy additions
 PyAPI_FUNC(void) _PyTruffle_DebugTrace(void);
 
+typedef struct {
+    PyObject_HEAD
+    int32_t handle_table_index;
+} GraalPyObject;
+
+typedef struct {
+    GraalPyObject ob_base;
+    Py_ssize_t ob_size;
+    PyObject **ob_item;
+} GraalPyVarObject;
+
 #ifdef __cplusplus
 }
 #endif

graalpython/com.oracle.graal.python.cext/src/capi.h

@@ -112,22 +112,6 @@ typedef struct {
     int getter_doc;
 } propertyobject;
 
-typedef struct {
-    PyObject_HEAD
-    int32_t handle_table_index;
-} GraalPyObject;
-
-typedef struct {
-    GraalPyObject ob_base;
-    Py_ssize_t ob_size;
-    PyObject **ob_item;
-} GraalPyVarObject;
-
-typedef struct {
-    GraalPyObject ob_base;
-    double ob_fval;
-} GraalPyFloatObject;
-
 /* GraalPy GC support */
 
 typedef struct _cycle_node {