Merge pull request #2615 from mzakharo/scipy

use LEGACY_NDK option to build lapack/scipy with a separate NDK

Author: AndreMiras

pythonforandroid/recipes/lapack/__init__.py

@@ -9,6 +9,10 @@
 from multiprocessing import cpu_count
 from os.path import join
 import sh
+from os import environ
+from pythonforandroid.util import build_platform
+
+arch_to_sysroot = {'armeabi': 'arm', 'armeabi-v7a': 'arm', 'arm64-v8a': 'arm64'}
 
 
 class LapackRecipe(Recipe):
@@ -18,12 +22,20 @@ class LapackRecipe(Recipe):
     url = 'https://github.com/Reference-LAPACK/lapack/archive/{version}.tar.gz'
     libdir = 'build/install/lib'
     built_libraries = {'libblas.so': libdir, 'liblapack.so': libdir, 'libcblas.so': libdir}
-    need_stl_shared = True
 
     def get_recipe_env(self, arch):
         env = super().get_recipe_env(arch)
-        sysroot = f"{self.ctx.ndk_dir}/platforms/{env['NDK_API']}/{arch.platform_dir}"
-        FC = f"{env['TOOLCHAIN_PREFIX']}-gfortran"  # FIXME
+
+        ndk_dir = environ.get("LEGACY_NDK")
+        if ndk_dir is None:
+            raise BuildInterruptingException("Please set the environment variable 'LEGACY_NDK' to point to a NDK location with gcc/gfortran support (last tested NDK version was 'r19c')")
+
+        GCC_VER = '4.9'
+        HOST = build_platform
+
+        sysroot_suffix = arch_to_sysroot.get(arch.arch, arch.arch)
+        sysroot = f"{ndk_dir}/platforms/{env['NDK_API']}/arch-{sysroot_suffix}"
+        FC = f"{ndk_dir}/toolchains/{arch.command_prefix}-{GCC_VER}/prebuilt/{HOST}/bin/{arch.command_prefix}-gfortran"
         env['FC'] = f'{FC} --sysroot={sysroot}'
         if sh.which(FC) is None:
             raise BuildInterruptingException(f"{FC} not found. See https://github.com/mzakharo/android-gfortran")
@@ -37,12 +49,13 @@ def build_arch(self, arch):
         ensure_dir(build_target)
         with current_directory(build_target):
             env = self.get_recipe_env(arch)
+            ndk_dir = environ["LEGACY_NDK"]
             shprint(sh.rm, '-rf', 'CMakeFiles/', 'CMakeCache.txt', _env=env)
             shprint(sh.cmake, source_dir,
                     '-DCMAKE_SYSTEM_NAME=Android',
                     '-DCMAKE_POSITION_INDEPENDENT_CODE=1',
                     '-DCMAKE_ANDROID_ARCH_ABI={arch}'.format(arch=arch.arch),
-                    '-DCMAKE_ANDROID_NDK=' + self.ctx.ndk_dir,
+                    '-DCMAKE_ANDROID_NDK=' + ndk_dir,
                     '-DCMAKE_BUILD_TYPE=Release',
                     '-DCMAKE_INSTALL_PREFIX={}'.format(install_target),
                     '-DANDROID_ABI={arch}'.format(arch=arch.arch),

pythonforandroid/recipes/numpy/__init__.py

@@ -19,6 +19,7 @@ class NumpyRecipe(CompiledComponentsPythonRecipe):
     patches = [
         join("patches", "remove-default-paths.patch"),
         join("patches", "add_libm_explicitly_to_build.patch"),
+        join("patches", "fix-missing-threads.h.patch"),
     ]
 
     def get_recipe_env(self, arch=None, with_flags_in_cc=True):

pythonforandroid/recipes/numpy/patches/fix-missing-threads.h.patch

@@ -0,0 +1,26 @@
+From: Mikhail Zakharov <mzakharo@gmail.com>
+Date: Sun, 5 Jun 2022 11:14:23 -0400
+Subject: [PATCH] fix missing threads.h
+
+---
+ numpy/f2py/cfuncs.py | 3 +--
+ 1 file changed, 1 insertion(+), 2 deletions(-)
+
+
+diff --git a/numpy/f2py/cfuncs.py b/numpy/f2py/cfuncs.py
+index bdd27ad..39cb470 100644
+--- a/numpy/f2py/cfuncs.py
++++ b/numpy/f2py/cfuncs.py
+@@ -586,8 +586,7 @@ cppmacros["F2PY_THREAD_LOCAL_DECL"] = """\
+    so `!defined(__STDC_NO_THREADS__)` may give false positive for the existence
+    of `threads.h` when using an older release of glibc 2.12
+    See gh-19437 for details on OpenBSD */
+-#include <threads.h>
+-#define F2PY_THREAD_LOCAL_DECL thread_local
++#define F2PY_THREAD_LOCAL_DECL __thread
+ #elif defined(__GNUC__) \\
+       && (__GNUC__ > 4 || (__GNUC__ == 4 && (__GNUC_MINOR__ >= 4)))
+ #define F2PY_THREAD_LOCAL_DECL __thread
+-- 
+2.25.1
+

pythonforandroid/recipes/pybind11/__init__.py

@@ -1,11 +1,14 @@
-from pythonforandroid.recipe import Recipe
+from pythonforandroid.recipe import PythonRecipe
 from os.path import join
 
 
-class Pybind11Recipe(Recipe):
+class Pybind11Recipe(PythonRecipe):
 
     version = '2.9.0'
     url = 'https://github.com/pybind/pybind11/archive/refs/tags/v{version}.zip'
+    depends = ['setuptools']
+    call_hostpython_via_targetpython = False
+    install_in_hostpython = True
 
     def get_include_dir(self, arch):
         return join(self.get_build_dir(arch.arch), 'include')

pythonforandroid/recipes/scipy/__init__.py

@@ -1,15 +1,20 @@
 from pythonforandroid.recipe import CompiledComponentsPythonRecipe, Recipe
 from multiprocessing import cpu_count
 from os.path import join
+from os import environ
+from pythonforandroid.util import build_platform
+
+arch_to_sysroot = {'armeabi': 'arm', 'armeabi-v7a': 'arm', 'arm64-v8a': 'arm64'}
 
 
 class ScipyRecipe(CompiledComponentsPythonRecipe):
 
-    version = '1.5.4'
+    version = '1.8.1'
     url = f'https://github.com/scipy/scipy/releases/download/v{version}/scipy-{version}.zip'
     site_packages_name = 'scipy'
-    depends = ['setuptools', 'cython', 'numpy', 'lapack']
+    depends = ['setuptools', 'cython', 'numpy', 'lapack', 'pybind11']
     call_hostpython_via_targetpython = False
+    need_stl_shared = True
 
     def build_compiled_components(self, arch):
         self.setup_extra_args = ['-j', str(cpu_count())]
@@ -24,27 +29,42 @@ def rebuild_compiled_components(self, arch, env):
     def get_recipe_env(self, arch):
         env = super().get_recipe_env(arch)
 
+        ndk_dir = environ["LEGACY_NDK"]
         GCC_VER = '4.9'
-        HOST = 'linux-x86_64'
-        LIB = 'lib64' if '64' in arch.arch else 'lib'
+        HOST = build_platform
+        suffix = '64' if '64' in arch.arch else ''
 
-        lapack_dir = join(Recipe.get_recipe('lapack', self.ctx).get_build_dir(arch.arch), 'build', 'install')
-        sysroot = f"{self.ctx.ndk_dir}/platforms/{env['NDK_API']}/{arch.platform_dir}"
-        sysroot_include = f'{self.ctx.ndk_dir}/toolchains/llvm/prebuilt/{HOST}/sysroot/usr/include'
-        prefix = ""  # FIXME
-        libgfortran = f'{self.ctx.ndk_dir}/toolchains/{prefix}-{GCC_VER}/prebuilt/{HOST}/{prefix}/{LIB}'
-        numpylib = self.ctx.get_python_install_dir(arch.arch) + '/numpy/core/lib'
+        prefix = arch.command_prefix
+        sysroot_suffix = arch_to_sysroot.get(arch.arch, arch.arch)
+        sysroot = f"{ndk_dir}/platforms/{env['NDK_API']}/arch-{sysroot_suffix}"
+        sysroot_include = f'{ndk_dir}/toolchains/llvm/prebuilt/{HOST}/sysroot/usr/include'
+        CLANG_BIN = f'{ndk_dir}/toolchains/llvm/prebuilt/{HOST}/bin/'
+        GCC = f'{ndk_dir}/toolchains/{prefix}-{GCC_VER}/prebuilt/{HOST}'
+        libgfortran = f'{GCC}/{prefix}/lib{suffix}'
+
+        numpylib = self.ctx.get_python_install_dir(arch.arch) + '/numpy'
         LDSHARED_opts = env['LDSHARED'].split('clang')[1]
+        arch_cflags = ' '.join(arch.arch_cflags)
 
+        # TODO: add pythran support
+        env['SCIPY_USE_PYTHRAN'] = '0'
+
+        lapack_dir = join(Recipe.get_recipe('lapack', self.ctx).get_build_dir(arch.arch), 'build', 'install')
         env['LAPACK'] = f'{lapack_dir}/lib'
         env['BLAS'] = env['LAPACK']
-        env['F90'] = f'{prefix}-gfortran'
-        env['CXX'] += f' -Wl,-l{self.stl_lib_name} -Wl,-L{self.get_stl_lib_dir(arch)}'
-        env['CPPFLAGS'] += f' --sysroot={sysroot} -I{sysroot_include}/c++/v1 -I{sysroot_include}'
-        env['LDSHARED'] = 'clang'
-        env['LDFLAGS'] += f' {LDSHARED_opts} --sysroot={sysroot} -L{libgfortran} -L{numpylib}'
-        env['LDFLAGS'] += f' -L{self.ctx.ndk_dir}/sources/cxx-stl/llvm-libc++/libs/{arch.arch}/'
 
+        # compilers
+        env['F77'] = f'{GCC}/bin/{prefix}-gfortran'
+        env['F90'] = f'{GCC}/bin/{prefix}-gfortran'
+        env['CC'] = f'{CLANG_BIN}/clang -target {arch.target} {arch_cflags}'
+        env['CXX'] = f'{CLANG_BIN}/clang++ -target {arch.target} {arch_cflags}'
+        env['LDSHARED'] = f'{CLANG_BIN}/clang'
+
+        # flags
+        env['CPPFLAGS'] = f'-DANDROID -I{sysroot_include}/{prefix} --sysroot={sysroot} -I{sysroot_include}/c++/v1 -I{sysroot_include}'
+        env['LDFLAGS'] += f' {LDSHARED_opts} --sysroot={sysroot} -L{libgfortran} -L{numpylib}/core/lib -L{numpylib}/random/lib'
+        env['LDFLAGS'] += f' -l{self.stl_lib_name} '
+        env['LDFLAGS'] += f' -L{ndk_dir}/sources/cxx-stl/llvm-libc++/libs/{arch.arch}/'  # for arm32 - unwind
         return env