[GR-65490] Add sqlite3 compile time options similar to CPython.

PullRequest: graalpython/3825

Author: timfel

CHANGELOG.md

@@ -8,8 +8,14 @@ language runtime. The main focus is on user-observable behavior of the engine.
 * `GRAALPY_VERSION_NUM` C macro now inlcudes the release level and serial number at the end to conform to the `hexversion` format. This shouldn't break any existing comparisons.
 * `dir(foreign_object)` now returns both foreign methods and Python methods (it used to return only foreign methods).
 * Support `__name__`, `__doc__`, `__text_signature__` fields on foreign executables to serve as their proper counterparts on the Python side. This is useful to, for example, use Java functional interfaces in lieu of Python functions for things like LangChain's `@tool` annotation that want to inspect the underlying function.
-* Remove support for running C extensions as LLVM bitcode. This also removes the related options `python.UseSystemToolchain` and `python.NativeModules`.
+* Remove support for running C extensions as LLVM bitcode. This also removes the related options `python.UseSystemToolchain` and `python.NativeModules`. Isolation of native code when embedding GraalPy into Java projects is now provided via the GraalVM enterprise edition polyglot isolate feature, which can launch in a separate external sub-process by setting the `--engine.IsolateMode=external` option.
 * Remove built-in HPy module. HPy can now be installed and used from the upstream sources.
+* Update Python standard library and core to 3.12.8.
+* Implement `faulthandler.dump_traceback_later` to better support testing frameworks that implement resilience to crashes.
+* Fix various issues affecting cibuildwheels on Windows, to make it easier for Python projects to provide native extensions for GraalPy on all supported platforms.
+* Add support for sharing Arrow arrays and tables between Java, PyArrow, and Pandas to avoid data copying when embedding those libraries into a Java project.
+* Enable FTS3, FTS4, FTS5, RTREE, and math function features in the bundled sqlite3 library.
+* Add support patches for Torch 2.7.0, PyGObject 3.52.3, xmlschema 4.0.0, lxml < 5.4.0, SciPy 1.15, jq 1.8.0, NumPy < 2.3, ormsgpack < 1.9.1, pandas 2.2.3, PyArrow 19.0, PyMuPDF 1.25.4.
 
 ## Version 24.2.0
 * Updated developer metadata of Maven artifacts.

graalpython/com.oracle.graal.python.cext/CMakeLists.txt

@@ -80,14 +80,14 @@ if (MSVC)
     /wd4710 # 'fprintf': function not inlined
     /wd4706 # assignment within conditional expression
     /wd4774 # 'sprintf': format string expected in argument 2 is not a string literal
+    /wd4191 # unsafe conversion from 'PyObject *(__cdecl *)(MatchObject *,PyObject *const *,Py_ssize_t)' to 'void (__cdecl *)(void)'
+    /wd4574 # sqlite, expat: 'SQLITE_ATOMIC_INTRINSICS' is defined to be '0': did you mean to use '#if SQLITE_ATOMIC_INTRINSICS'?
+    /wd4701 # potentially uninitialized local variable used
 
     # Some that I'm not so happy about
     /wd4232 # sre: nonstandard extension used: 'ml_meth': address of dllimport 'Py_GenericAlias' is not static, identity not guaranteed
-    /wd4191 # sre and sqlite: 'type cast': like unsafe conversion from 'PyObject *(__cdecl *)(MatchObject *,PyObject *const *,Py_ssize_t)' to 'void (__cdecl *)(void)'
     /wd4918 # sre: invalid character in pragma optimization list
-    /wd4701 # cpython_unicodedata: potentially uninitialized local variable 'rc' used
     /wd4703 # unicodeobject.c:potentially uninitialized local pointer variable used
-    /wd4574 # sqlite: 'SQLITE_ATOMIC_INTRINSICS' is defined to be '0': did you mean to use '#if SQLITE_ATOMIC_INTRINSICS'?
     /wd4310 # xmlparse: cast truncates constant value
     /wd4777 # format string '%zd' requires an argument of type 'unsigned __int64'
   )
@@ -238,7 +238,19 @@ set(SQLITE3_SRC
     "${SRC_DIR}/modules/_sqlite/util.c"
 )
 native_module("_sqlite3" TRUE "${SQLITE3_SRC}")
+# This combines the flags CPython uses on macOS and Windows, on Linux systems
+# it usually links against the system sqlite3 so we do not really know. See
+# https://github.com/python/cpython/issues/88017 for some reasons
 target_include_directories("_sqlite3" PUBLIC "${SRC_DIR}/modules/_sqlite/sqlite")
+target_compile_definitions("_sqlite3" PRIVATE
+    SQLITE_ENABLE_MATH_FUNCTIONS
+    SQLITE_ENABLE_FTS5
+    SQLITE_ENABLE_FTS4
+    SQLITE_ENABLE_FTS3_PARENTHESIS
+    SQLITE_ENABLE_RTREE
+    SQLITE_OMIT_AUTOINIT
+    SQLITE_TCL=0
+)
 
 set(LIBHACL_HEADERS
     # "${SRC_DIR}/modules/_hacl/include/krml/FStar_UInt128_Verified.h"
@@ -299,13 +311,19 @@ set(TESTCAPI_SRC
     "${SRC_DIR}/modules/_testcapi.c"
 )
 
-if(NOT WIN32)
-    native_module("_testcapi" FALSE "${TESTCAPI_SRC}")
-    simple_native_module("_testbuffer")
-    simple_native_module("_testmultiphase")
-    simple_native_module("_testsinglephase")
-    simple_native_module("_ctypes_test")
+native_module("_testcapi" FALSE "${TESTCAPI_SRC}")
+if(WIN32)
+    target_compile_options("_testcapi" PRIVATE /wd4296 /wd4130)
+endif()
+simple_native_module("_testbuffer")
+if(WIN32)
+    target_compile_options("_testbuffer" PRIVATE /wd4090)
+endif()
+simple_native_module("_testmultiphase")
+simple_native_module("_testsinglephase")
+simple_native_module("_ctypes_test")
 
+if(NOT WIN32)
     ###################### BZIP2 ########################
     if(DEFINED LIBBZ2_BUILD_FILE)
         include("${LIBBZ2_BUILD_FILE}")

graalpython/com.oracle.graal.python.frozen/freeze_modules.py

@@ -607,6 +607,7 @@ def write_frozen_module_file(file, modules):
         stat_result = os.stat(file)
         atime, mtime = stat_result.st_atime, stat_result.st_mtime
     else:
+        linesep = os.linesep
         content = None
     os.makedirs(os.path.dirname(file), exist_ok=True)
     with open(file, "w", encoding="utf-8", newline=linesep) as out_file:

graalpython/com.oracle.graal.python.pegparser.generator/CMakeLists.txt

@@ -37,10 +37,7 @@ add_custom_target(grammar ALL
     DEPENDS "${PARSER_OUTPUT}")
 add_custom_command(
     OUTPUT "${PARSER_OUTPUT}"
-    COMMAND ${PYTHON_EXE} "${CMAKE_CURRENT_LIST_DIR}/main_parser_gen.py" "${GRAMMAR}" "${TOKENS}" "${PARSER_OUTPUT}"
-    COMMAND ${CMAKE_COMMAND} -E copy_if_different
-            ${PARSER_OUTPUT}
-	    ${PARSER_TARGET}
+    COMMAND ${PYTHON_EXE} "${CMAKE_CURRENT_LIST_DIR}/main_parser_gen.py" "${GRAMMAR}" "${TOKENS}" "${PARSER_TARGET}"
     DEPENDS "${CMAKE_CURRENT_LIST_DIR}/main_parser_gen.py" "${GRAMMAR}" "${TOKENS}" "${PARSER_TARGET}" ${PEGEN_FILES} ${PEGJAVA_FILES})
 
 add_custom_target(asdl ALL 

graalpython/com.oracle.graal.python.pegparser.generator/asdl/java_file.py

@@ -131,7 +131,24 @@ def define(self, line: str, *annotations: str):
 def create(out_dir_base: str, java_package: str, java_class_name: str):
     sst_dir = os.path.join(out_dir_base, *java_package.split('.'))
     os.makedirs(sst_dir, exist_ok=True)
-    with open(os.path.join(sst_dir, java_class_name + '.java'), 'w') as f:
+    filename = os.path.join(sst_dir, java_class_name + '.java')
+
+    # Determine which line separator to use
+    if os.path.exists(filename):
+        with open(filename, "r", encoding="utf-8", newline=os.linesep) as f:
+            content = f.read()
+        if os.linesep != "\n":
+            if content.replace(os.linesep, "\n") == content:
+                # Windows file has Unix line endings
+                linesep = "\n"
+            else:
+                linesep = os.linesep
+        else:
+            linesep = "\n"
+    else:
+        linesep = os.linesep
+
+    with open(filename, 'w', encoding="utf-8", newline=linesep) as f:
         emitter = Emitter(f)
         f.write(HEADER)
         f.write(f'package {java_package};\n')

graalpython/com.oracle.graal.python.pegparser.generator/main_parser_gen.py

@@ -1,5 +1,5 @@
 #!/usr/bin/env python3
-# Copyright (c) 2021, 2022, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2021, 2025, Oracle and/or its affiliates. All rights reserved.
 # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 #
 # The Universal Permissive License (UPL), Version 1.0
@@ -69,10 +69,37 @@ def main():
     with open(args.tokens_file, "r") as tok_file:
         all_tokens, exact_tokens, non_exact_tokens = generate_token_definitions(tok_file)
 
-    with open(args.output_file, "w") as file:
+    # Determine which line separator to use
+    if os.path.exists(args.output_file):
+        stat_result = os.stat(args.output_file)
+        atime, mtime = stat_result.st_atime, stat_result.st_mtime
+        with open(args.output_file, "r", encoding="utf-8", newline=os.linesep) as f:
+            content = f.read()
+        if os.linesep != "\n":
+            unix_content = content.replace(os.linesep, "\n")
+            if unix_content == content:
+                # Windows file has Unix line endings
+                linesep = "\n"
+                content = unix_content
+            else:
+                linesep = os.linesep
+        else:
+            linesep = "\n"
+    else:
+        content = None
+        linesep = os.linesep
+
+    with open(args.output_file, "w", encoding="utf-8", newline=linesep) as file:
         gen = JavaParserGenerator(grammar, all_tokens, exact_tokens, non_exact_tokens, file, debug=args.debug)
         gen.generate(os.path.basename(args.grammar_file))
 
+    with open(args.output_file, "r", encoding="utf-8", newline=linesep) as file:
+        new_content = file.read()
+
+    if content == new_content:
+        print(f"{args.output_file} not modified")
+        os.utime(args.output_file, (atime, mtime))
+
 
 if __name__ == '__main__':
     main()

graalpython/com.oracle.graal.python.processor/src/com/oracle/graal/python/processor/CApiBuiltinsProcessor.java

@@ -748,6 +748,10 @@ private static Path resolvePath(Path path) {
                     "_PyCFunction_SetModule", "_PyCFunction_SetMethodDef",
                     "PyCode_GetFileName", "_PyArray_Resize", "_PyArray_Data",
                     "_PyErr_Occurred", "_PyNamespace_New", "_Py_GetErrorHandler",
+                    // Not actually additional, only defined on Windows.
+                    // TODO: fix generated CAPIFunctions.txt
+                    "PyUnicode_AsMBCSString", "PyUnicode_EncodeCodePage", "PyUnicode_DecodeMBCS",
+                    "PyUnicode_DecodeCodePageStateful", "PyUnicode_DecodeMBCSStateful",
     };
 
     public String resolveArgDescriptor(String sig) {

graalpython/com.oracle.graal.python.test/src/tests/test_sqlite3.py

@@ -1,4 +1,4 @@
-# Copyright (c) 2023, 2024, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2023, 2025, Oracle and/or its affiliates. All rights reserved.
 # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 #
 # The Universal Permissive License (UPL), Version 1.0
@@ -49,3 +49,21 @@ def test_basic_functionality():
     rows = conn.execute("select sqlite_version()")
     assert len(next(rows)[0]) >= 5
     conn.close()
+
+
+def test_fts5_works():
+    # we explicitly enable those features below, but on CPython they might not
+    # be available if using some system libsqlite that doesn't have them
+    import sqlite3
+    conn = sqlite3.connect(':memory:')
+    try:
+        conn.execute("CREATE VIRTUAL TABLE IF NOT EXISTS your_table USING fts5(column1, column2)")
+        conn.execute("CREATE VIRTUAL TABLE IF NOT EXISTS your_table USING fts4(column1, column2)")
+        conn.execute("CREATE VIRTUAL TABLE IF NOT EXISTS your_table USING fts3(column1, column2)")
+        conn.execute("CREATE VIRTUAL TABLE IF NOT EXISTS your_table USING rtree(column1, column2)")
+        sqpi = next(conn.execute("SELECT pi()"))[0]
+        assert 3.14 == float(f'{sqpi:.2f}'), sqpi
+    except sqlite3.OperationalError:
+        import sys
+        assert sys.implementation.name != "graalpy"
+    conn.close()