Add support for Python 3.11 and make it the default while building hostpython3 and python3 (#2850)

* python: update to 3.11.5

* scipy: update to 1.11.3

* add `pybind11` to `ci/constants.py` in BROKEN_RECIPES_PYTHON3

Author: T-Dynamos

ci/constants.py

@@ -39,7 +39,8 @@ class TargetPython(Enum):
     'boost',
     # libtorrent gives errors (requires boost. Also, see issue #2809, to start with)
     'libtorrent',
-
+    # pybind11 build fails on macos
+    'pybind11',
 ])
 
 BROKEN_RECIPES = {

pythonforandroid/bootstraps/common/build/src/main/java/org/kivy/android/PythonUtil.java

@@ -55,6 +55,7 @@ protected static ArrayList<String> getLibraries(File libsDir) {
         libsList.add("python3.8");
         libsList.add("python3.9");
         libsList.add("python3.10");
+        libsList.add("python3.11");
         libsList.add("main");
         return libsList;
     }
@@ -74,7 +75,7 @@ public static void loadLibraries(File filesDir, File libsDir) {
                 // load, and it has failed, give a more
                 // general error
                 Log.v(TAG, "Library loading error: " + e.getMessage());
-                if (lib.startsWith("python3.10") && !foundPython) {
+                if (lib.startsWith("python3.11") && !foundPython) {
                     throw new RuntimeException("Could not load any libpythonXXX.so");
                 } else if (lib.startsWith("python")) {
                     continue;

pythonforandroid/recipes/cython/__init__.py

@@ -3,7 +3,7 @@
 
 class CythonRecipe(CompiledComponentsPythonRecipe):
 
-    version = '0.29.28'
+    version = '0.29.36'
     url = 'https://github.com/cython/cython/archive/{version}.tar.gz'
     site_packages_name = 'cython'
     depends = ['setuptools']

pythonforandroid/recipes/hostpython3/__init__.py

@@ -35,7 +35,7 @@ class HostPython3Recipe(Recipe):
         :class:`~pythonforandroid.python.HostPythonRecipe`
     '''
 
-    version = '3.10.10'
+    version = '3.11.5'
     name = 'hostpython3'
 
     build_subdir = 'native-build'

pythonforandroid/recipes/pybind11/__init__.py

@@ -4,7 +4,7 @@
 
 class Pybind11Recipe(PythonRecipe):
 
-    version = '2.9.0'
+    version = '2.11.1'
     url = 'https://github.com/pybind/pybind11/archive/refs/tags/v{version}.zip'
     depends = ['setuptools']
     call_hostpython_via_targetpython = False

pythonforandroid/recipes/python3/__init__.py

@@ -2,7 +2,6 @@
 import sh
 import subprocess
 
-from multiprocessing import cpu_count
 from os import environ, utime
 from os.path import dirname, exists, join
 from pathlib import Path
@@ -56,7 +55,7 @@ class Python3Recipe(TargetPythonRecipe):
         :class:`~pythonforandroid.python.GuestPythonRecipe`
     '''
 
-    version = '3.10.10'
+    version = '3.11.5'
     url = 'https://www.python.org/ftp/python/{version}/Python-{version}.tgz'
     name = 'python3'
 
@@ -71,15 +70,17 @@ class Python3Recipe(TargetPythonRecipe):
         # Python 3.8.1 & 3.9.X
         ('patches/py3.8.1.patch', version_starts_with("3.8")),
         ('patches/py3.8.1.patch', version_starts_with("3.9")),
-        ('patches/py3.8.1.patch', version_starts_with("3.10"))
+        ('patches/py3.8.1.patch', version_starts_with("3.10")),
+        ('patches/cpython-311-ctypes-find-library.patch', version_starts_with("3.11")),
     ]
 
     if shutil.which('lld') is not None:
-        patches = patches + [
+        patches += [
             ("patches/py3.7.1_fix_cortex_a8.patch", version_starts_with("3.7")),
             ("patches/py3.8.1_fix_cortex_a8.patch", version_starts_with("3.8")),
             ("patches/py3.8.1_fix_cortex_a8.patch", version_starts_with("3.9")),
-            ("patches/py3.8.1_fix_cortex_a8.patch", version_starts_with("3.10"))
+            ("patches/py3.8.1_fix_cortex_a8.patch", version_starts_with("3.10")),
+            ("patches/py3.8.1_fix_cortex_a8.patch", version_starts_with("3.11")),
         ]
 
     depends = ['hostpython3', 'sqlite3', 'openssl', 'libffi']
@@ -101,7 +102,12 @@ class Python3Recipe(TargetPythonRecipe):
         'ac_cv_header_sys_eventfd_h=no',
         '--prefix={prefix}',
         '--exec-prefix={exec_prefix}',
-        '--enable-loadable-sqlite-extensions')
+        '--enable-loadable-sqlite-extensions'
+    )
+
+    if version_starts_with("3.11"):
+        configure_args += ('--with-build-python={python_host_bin}',)
+
     '''The configure arguments needed to build the python recipe. Those are
     used in method :meth:`build_arch` (if not overwritten like python3's
     recipe does).
@@ -323,12 +329,19 @@ def build_arch(self, arch):
                     *(' '.join(self.configure_args).format(
                                     android_host=env['HOSTARCH'],
                                     android_build=android_build,
+                                    python_host_bin=join(self.get_recipe(
+                                        'host' + self.name, self.ctx
+                                    ).get_path_to_python(), "python3"),
                                     prefix=sys_prefix,
                                     exec_prefix=sys_exec_prefix)).split(' '),
                     _env=env)
 
+            # Python build does not seem to play well with make -j option from Python 3.11 and onwards
+            # Before losing some time, please check issue
+            # https://github.com/python/cpython/issues/101295 , as the root cause looks similar
             shprint(
-                sh.make, 'all', '-j', str(cpu_count()),
+                sh.make,
+                'all',
                 'INSTSONAME={lib_name}'.format(lib_name=self._libpython),
                 _env=env
             )

pythonforandroid/recipes/python3/patches/cpython-311-ctypes-find-library.patch

@@ -0,0 +1,19 @@
+--- Python-3.11.5/Lib/ctypes/util.py	2023-08-24 17:39:18.000000000 +0530
++++ Python-3.11.5.mod/Lib/ctypes/util.py	2023-11-18 22:12:17.356160615 +0530
+@@ -4,7 +4,15 @@
+ import sys
+ 
+ # find_library(name) returns the pathname of a library, or None.
+-if os.name == "nt":
++
++# This patch overrides the find_library to look in the right places on
++# Android
++if True:
++    from android._ctypes_library_finder import find_library as _find_lib
++    def find_library(name):
++        return _find_lib(name)
++
++elif os.name == "nt":
+ 
+     def _get_build_version():
+         """Return the version of MSVC that was used to build Python.

pythonforandroid/recipes/scipy/__init__.py

@@ -1,8 +1,10 @@
-from pythonforandroid.recipe import CompiledComponentsPythonRecipe, Recipe
 from multiprocessing import cpu_count
 from os.path import join
 from os import environ
-from pythonforandroid.util import build_platform
+import sh
+from pythonforandroid.logger import shprint
+from pythonforandroid.recipe import CompiledComponentsPythonRecipe, Recipe
+from pythonforandroid.util import build_platform, current_directory
 
 
 def arch_to_toolchain(arch):
@@ -13,12 +15,14 @@ def arch_to_toolchain(arch):
 
 class ScipyRecipe(CompiledComponentsPythonRecipe):
 
-    version = '1.8.1'
-    url = f'https://github.com/scipy/scipy/releases/download/v{version}/scipy-{version}.zip'
+    version = 'maintenance/1.11.x'
+    url = 'git+https://github.com/scipy/scipy.git'
+    git_commit = 'b430bf54b5064465983813e2cfef3fcb86c3df07'  # version 1.11.3
     site_packages_name = 'scipy'
     depends = ['setuptools', 'cython', 'numpy', 'lapack', 'pybind11']
     call_hostpython_via_targetpython = False
     need_stl_shared = True
+    patches = ["setup.py.patch"]
 
     def build_compiled_components(self, arch):
         self.setup_extra_args = ['-j', str(cpu_count())]
@@ -30,6 +34,12 @@ def rebuild_compiled_components(self, arch, env):
         super().rebuild_compiled_components(arch, env)
         self.setup_extra_args = []
 
+    def download_file(self, url, target, cwd=None):
+        super().download_file(url, target, cwd=cwd)
+        with current_directory(target):
+            shprint(sh.git, 'fetch', '--unshallow')
+            shprint(sh.git, 'checkout', self.git_commit)
+
     def get_recipe_env(self, arch):
         env = super().get_recipe_env(arch)
         arch_env = arch.get_env()