Added pythonforandroid.androidndk.AndroidNDK + some changes needed in order to support build on Apple Silicon macs. (#2586)

* Added pythonforandroid.androidndk.AndroidNDK + some changes needed in order to support build on Apple Silicon macs.

* Enhance tests on host_tag

* Update pythonforandroid/androidndk.py

Co-authored-by: Andre Miras <AndreMiras@users.noreply.github.com>

Co-authored-by: Andre Miras <AndreMiras@users.noreply.github.com>

Author: misl6

.github/workflows/push.yml

@@ -109,11 +109,17 @@ jobs:
             target: testapps-with-numpy
           - name: webview
             target: testapps-webview
+        include:
+          - runs_on: macos-latest
+            ndk_version: '23b'
+          - runs_on: apple-silicon-m1
+            ndk_version: '24'
     env:
       ANDROID_HOME: ${HOME}/.android
       ANDROID_SDK_ROOT: ${HOME}/.android/android-sdk
       ANDROID_SDK_HOME: ${HOME}/.android/android-sdk
       ANDROID_NDK_HOME: ${HOME}/.android/android-ndk
+      ANDROID_NDK_VERSION: ${{ matrix.ndk_version }}
     steps:
       - name: Checkout python-for-android
         uses: actions/checkout@v2
@@ -127,7 +133,7 @@ jobs:
           source ci/osx_ci.sh
           arm64_set_path_and_python_version 3.9.7
           python3 pythonforandroid/prerequisites.py
-      - name: Install dependencies
+      - name: Install dependencies (Legacy)
         run: |
           source ci/osx_ci.sh
           arm64_set_path_and_python_version 3.9.7
@@ -241,11 +247,17 @@ jobs:
             target: testapps-with-numpy-aab
           - name: webview
             target: testapps-webview-aab
+        include:
+          - runs_on: macos-latest
+            ndk_version: '23b'
+          - runs_on: apple-silicon-m1
+            ndk_version: '24'
     env:
       ANDROID_HOME: ${HOME}/.android
       ANDROID_SDK_ROOT: ${HOME}/.android/android-sdk
       ANDROID_SDK_HOME: ${HOME}/.android/android-sdk
       ANDROID_NDK_HOME: ${HOME}/.android/android-ndk
+      ANDROID_NDK_VERSION: ${{ matrix.ndk_version }}
     steps:
       - name: Checkout python-for-android
         uses: actions/checkout@v2
@@ -259,7 +271,7 @@ jobs:
           source ci/osx_ci.sh
           arm64_set_path_and_python_version 3.9.7
           python3 pythonforandroid/prerequisites.py
-      - name: Install dependencies
+      - name: Install dependencies (Legacy)
         run: |
           source ci/osx_ci.sh
           arm64_set_path_and_python_version 3.9.7
@@ -318,12 +330,18 @@ jobs:
       matrix:
         android_arch: ["arm64-v8a", "armeabi-v7a", "x86_64", "x86"]
         runs_on: [macos-latest, apple-silicon-m1]
+        include:
+          - runs_on: macos-latest
+            ndk_version: '23b'
+          - runs_on: apple-silicon-m1
+            ndk_version: '24'
     env:
       ANDROID_HOME: ${HOME}/.android
       ANDROID_SDK_ROOT: ${HOME}/.android/android-sdk
       ANDROID_SDK_HOME: ${HOME}/.android/android-sdk
       ANDROID_NDK_HOME: ${HOME}/.android/android-ndk
       REBUILD_UPDATED_RECIPES_EXTRA_ARGS: --arch=${{ matrix.android_arch }}
+      ANDROID_NDK_VERSION: ${{ matrix.ndk_version }}
     steps:
       - name: Checkout python-for-android
         uses: actions/checkout@v2
@@ -339,7 +357,7 @@ jobs:
           source ci/osx_ci.sh
           arm64_set_path_and_python_version 3.9.7
           python3 pythonforandroid/prerequisites.py
-      - name: Install dependencies
+      - name: Install dependencies (Legacy)
         run: |
           source ci/osx_ci.sh
           arm64_set_path_and_python_version 3.9.7

pythonforandroid/androidndk.py

@@ -0,0 +1,83 @@
+import sys
+import os
+
+
+class AndroidNDK:
+    """
+    This class is used to get the current NDK information.
+    """
+
+    ndk_dir = ""
+
+    def __init__(self, ndk_dir):
+        self.ndk_dir = ndk_dir
+
+    @property
+    def host_tag(self):
+        """
+        Returns the host tag for the current system.
+        Note: The host tag is ``darwin-x86_64`` even on Apple Silicon macs.
+        """
+        return f"{sys.platform}-x86_64"
+
+    @property
+    def llvm_prebuilt_dir(self):
+        return os.path.join(
+            self.ndk_dir, "toolchains", "llvm", "prebuilt", self.host_tag
+        )
+
+    @property
+    def llvm_bin_dir(self):
+        return os.path.join(self.llvm_prebuilt_dir, "bin")
+
+    @property
+    def clang(self):
+        return os.path.join(self.llvm_bin_dir, "clang")
+
+    @property
+    def clang_cxx(self):
+        return os.path.join(self.llvm_bin_dir, "clang++")
+
+    @property
+    def llvm_binutils_prefix(self):
+        return os.path.join(self.llvm_bin_dir, "llvm-")
+
+    @property
+    def llvm_ar(self):
+        return f"{self.llvm_binutils_prefix}ar"
+
+    @property
+    def llvm_ranlib(self):
+        return f"{self.llvm_binutils_prefix}ranlib"
+
+    @property
+    def llvm_objcopy(self):
+        return f"{self.llvm_binutils_prefix}objcopy"
+
+    @property
+    def llvm_objdump(self):
+        return f"{self.llvm_binutils_prefix}objdump"
+
+    @property
+    def llvm_readelf(self):
+        return f"{self.llvm_binutils_prefix}readelf"
+
+    @property
+    def llvm_strip(self):
+        return f"{self.llvm_binutils_prefix}strip"
+
+    @property
+    def sysroot(self):
+        return os.path.join(self.llvm_prebuilt_dir, "sysroot")
+
+    @property
+    def sysroot_include_dir(self):
+        return os.path.join(self.sysroot, "usr", "include")
+
+    @property
+    def sysroot_lib_dir(self):
+        return os.path.join(self.sysroot, "usr", "lib")
+
+    @property
+    def libcxx_include_dir(self):
+        return os.path.join(self.sysroot_include_dir, "c++", "v1")

pythonforandroid/archs.py

@@ -25,7 +25,7 @@ class Arch:
 
     common_cppflags = [
         '-DANDROID',
-        '-I{ctx.ndk_sysroot}/usr/include',
+        '-I{ctx.ndk.sysroot_include_dir}',
         '-I{python_includes}',
     ]
 
@@ -54,7 +54,11 @@ def __str__(self):
 
     @property
     def ndk_lib_dir(self):
-        return join(self.ctx.ndk_sysroot, 'usr', 'lib', self.command_prefix, str(self.ctx.ndk_api))
+        return join(self.ctx.ndk.sysroot_lib_dir, self.command_prefix)
+
+    @property
+    def ndk_lib_dir_versioned(self):
+        return join(self.ndk_lib_dir, str(self.ctx.ndk_api))
 
     @property
     def include_dirs(self):
@@ -74,18 +78,6 @@ def target(self):
             triplet=self.command_prefix, ndk_api=self.ctx.ndk_api
         )
 
-    @property
-    def clang_path(self):
-        """Full path of the clang compiler"""
-        return join(
-            self.ctx.ndk_dir,
-            'toolchains',
-            'llvm',
-            'prebuilt',
-            build_platform,
-            'bin',
-        )
-
     @property
     def clang_exe(self):
         """Full path of the clang compiler depending on the android's ndk
@@ -112,7 +104,7 @@ def get_clang_exe(self, with_target=False, plus_plus=False):
             )
         if plus_plus:
             compiler += '++'
-        return join(self.clang_path, compiler)
+        return join(self.ctx.ndk.llvm_bin_dir, compiler)
 
     def get_env(self, with_flags_in_cc=True):
         env = {}
@@ -195,11 +187,11 @@ def get_env(self, with_flags_in_cc=True):
                 ccache=ccache)
 
         # Android's LLVM binutils
-        env['AR'] = f'{self.clang_path}/llvm-ar'
-        env['RANLIB'] = f'{self.clang_path}/llvm-ranlib'
-        env['STRIP'] = f'{self.clang_path}/llvm-strip --strip-unneeded'
-        env['READELF'] = f'{self.clang_path}/llvm-readelf'
-        env['OBJCOPY'] = f'{self.clang_path}/llvm-objcopy'
+        env['AR'] = self.ctx.ndk.llvm_ar
+        env['RANLIB'] = self.ctx.ndk.llvm_ranlib
+        env['STRIP'] = f'{self.ctx.ndk.llvm_strip} --strip-unneeded'
+        env['READELF'] = self.ctx.ndk.llvm_readelf
+        env['OBJCOPY'] = self.ctx.ndk.llvm_objcopy
 
         env['MAKE'] = 'make -j{}'.format(str(cpu_count()))
 

pythonforandroid/build.py

@@ -5,7 +5,6 @@
 import copy
 import os
 import glob
-import sys
 import re
 import sh
 import shutil
@@ -23,20 +22,7 @@
 from pythonforandroid.recommendations import (
     check_ndk_version, check_target_api, check_ndk_api,
     RECOMMENDED_NDK_API, RECOMMENDED_TARGET_API)
-from pythonforandroid.util import build_platform
-
-
-def get_ndk_standalone(ndk_dir):
-    return join(ndk_dir, 'toolchains', 'llvm', 'prebuilt', build_platform)
-
-
-def get_ndk_sysroot(ndk_dir):
-    sysroot = join(get_ndk_standalone(ndk_dir), 'sysroot')
-    sysroot_exists = True
-    if not exists(sysroot):
-        warning("sysroot doesn't exist: {}".format(sysroot))
-        sysroot_exists = False
-    return sysroot, sysroot_exists
+from pythonforandroid.androidndk import AndroidNDK
 
 
 def get_targets(sdk_dir):
@@ -101,9 +87,7 @@ class Context:
 
     ccache = None  # whether to use ccache
 
-    ndk_standalone = None
-    ndk_sysroot = None
-    ndk_include_dir = None  # usr/include
+    ndk = None
 
     bootstrap = None
     bootstrap_build_dir = None
@@ -330,7 +314,6 @@ def prepare_build_environment(self,
         if ndk_dir is None:
             raise BuildInterruptingException('Android NDK dir was not specified')
         self.ndk_dir = realpath(ndk_dir)
-
         check_ndk_version(ndk_dir)
 
         ndk_api = None
@@ -350,6 +333,8 @@ def prepare_build_environment(self,
 
         check_ndk_api(ndk_api, self.android_api)
 
+        self.ndk = AndroidNDK(self.ndk_dir)
+
         # path to some tools
         self.ccache = sh.which("ccache")
         if not self.ccache:
@@ -364,17 +349,9 @@ def prepare_build_environment(self,
                     ' a python 3 target (which is the default)'
                     ' then THINGS WILL BREAK.')
 
-        py_platform = sys.platform
-        if py_platform in ['linux2', 'linux3']:
-            py_platform = 'linux'
-
-        self.ndk_standalone = get_ndk_standalone(self.ndk_dir)
-        self.ndk_sysroot, ndk_sysroot_exists = get_ndk_sysroot(self.ndk_dir)
-        self.ndk_include_dir = join(self.ndk_sysroot, 'usr', 'include')
-
         self.env["PATH"] = ":".join(
             [
-                f"{self.ndk_dir}/toolchains/llvm/prebuilt/{py_platform}-x86_64/bin",
+                self.ndk.llvm_bin_dir,
                 self.ndk_dir,
                 f"{self.sdk_dir}/tools",
                 environ.get("PATH"),

pythonforandroid/recipe.py

@@ -135,24 +135,9 @@ class Recipe(with_metaclass(RecipeMeta)):
         starting from NDK r18 the `gnustl_shared` lib has been deprecated.
     '''
 
-    stl_lib_source = '{ctx.ndk_dir}/sources/cxx-stl/llvm-libc++'
-    '''
-    The source directory of the selected stl lib, defined in property
-    `stl_lib_name`
-    '''
-
-    @property
-    def stl_include_dir(self):
-        return join(self.stl_lib_source.format(ctx=self.ctx), 'include')
-
-    def get_stl_lib_dir(self, arch):
-        return join(
-            self.stl_lib_source.format(ctx=self.ctx), 'libs', arch.arch
-        )
-
     def get_stl_library(self, arch):
         return join(
-            self.get_stl_lib_dir(arch),
+            arch.ndk_lib_dir,
             'lib{name}.so'.format(name=self.stl_lib_name),
         )
 
@@ -510,14 +495,14 @@ def get_recipe_env(self, arch=None, with_flags_in_cc=True):
 
         if self.need_stl_shared:
             env['CPPFLAGS'] = env.get('CPPFLAGS', '')
-            env['CPPFLAGS'] += ' -I{}'.format(self.stl_include_dir)
+            env['CPPFLAGS'] += ' -I{}'.format(self.ctx.ndk.libcxx_include_dir)
 
             env['CXXFLAGS'] = env['CFLAGS'] + ' -frtti -fexceptions'
 
             if with_flags_in_cc:
                 env['CXX'] += ' -frtti -fexceptions'
 
-            env['LDFLAGS'] += ' -L{}'.format(self.get_stl_lib_dir(arch))
+            env['LDFLAGS'] += ' -L{}'.format(arch.ndk_lib_dir)
             env['LIBS'] = env.get('LIBS', '') + " -l{}".format(
                 self.stl_lib_name
             )

pythonforandroid/recipes/Pillow/__init__.py

@@ -35,9 +35,6 @@ class PillowRecipe(CompiledComponentsPythonRecipe):
     def get_recipe_env(self, arch=None, with_flags_in_cc=True):
         env = super().get_recipe_env(arch, with_flags_in_cc)
 
-        ndk_lib_dir = arch.ndk_lib_dir
-        ndk_include_dir = self.ctx.ndk_include_dir
-
         png = self.get_recipe('png', self.ctx)
         png_lib_dir = join(png.get_build_dir(arch.arch), '.libs')
         png_inc_dir = png.get_build_dir(arch)
@@ -71,7 +68,7 @@ def get_recipe_env(self, arch=None, with_flags_in_cc=True):
         cflags += f' -I{jpeg_inc_dir}'
         if build_with_webp_support:
             cflags += f' -I{join(webp_install, "include")}'
-        cflags += f' -I{ndk_include_dir}'
+        cflags += f' -I{self.ctx.ndk.sysroot_include_dir}'
 
         # Link the basic Pillow libraries...no need to add webp's libraries
         # since it seems that the linkage is properly made without it :)
@@ -84,7 +81,7 @@ def get_recipe_env(self, arch=None, with_flags_in_cc=True):
         env['LDFLAGS'] += f' -L{jpeg_lib_dir}'
         if build_with_webp_support:
             env['LDFLAGS'] += f' -L{join(webp_install, "lib")}'
-        env['LDFLAGS'] += f' -L{ndk_lib_dir}'
+        env['LDFLAGS'] += f' -L{arch.ndk_lib_dir_versioned}'
         if cflags not in env['CFLAGS']:
             env['CFLAGS'] += cflags + " -lm"
         return env