NDK 23 + Gradle 7 support (#2550)

* Adds NDK 23 and Gradle 7 support

* Updates gradle and android gradle plugin version + avoid using a deprecated property

* Cleanup + fix some broken recipes

* Cleanup + icu now seems that is not needing stl_shared anymore

* Updates docs and MIN_NDK_VERSION (MAX_NDK_VERSION could be 24, but needs to be tested)

Author: misl6

ci/constants.py

@@ -32,6 +32,8 @@ class TargetPython(Enum):
     'zope_interface',
     # Requires zope_interface, which is broken.
     'twisted',
+    # genericndkbuild is incompatible with sdl2 (which is build by default when targeting sdl2 bootstrap)
+    'genericndkbuild',
 ])
 
 BROKEN_RECIPES = {

ci/makefiles/android.mk

@@ -1,7 +1,7 @@
 # Downloads and installs the Android SDK depending on supplied platform: darwin or linux
 
 # Those android NDK/SDK variables can be override when running the file
-ANDROID_NDK_VERSION ?= 19b
+ANDROID_NDK_VERSION ?= 23b
 ANDROID_SDK_TOOLS_VERSION ?= 6514223
 ANDROID_SDK_BUILD_TOOLS_VERSION ?= 29.0.3
 ANDROID_HOME ?= $(HOME)/.android
@@ -22,7 +22,7 @@ ANDROID_SDK_TOOLS_DL_URL=https://dl.google.com/android/repository/$(ANDROID_SDK_
 
 ANDROID_NDK_HOME=$(ANDROID_HOME)/android-ndk
 ANDROID_NDK_FOLDER=$(ANDROID_HOME)/android-ndk-r$(ANDROID_NDK_VERSION)
-ANDROID_NDK_ARCHIVE=android-ndk-r$(ANDROID_NDK_VERSION)-$(TARGET_OS)-x86_64.zip
+ANDROID_NDK_ARCHIVE=android-ndk-r$(ANDROID_NDK_VERSION)-$(TARGET_OS).zip
 ANDROID_NDK_DL_URL=https://dl.google.com/android/repository/$(ANDROID_NDK_ARCHIVE)
 
 $(info Target install OS is          : $(target_os))
@@ -59,7 +59,7 @@ extract_android_sdk:
 extract_android_ndk:
 	mkdir -p $(ANDROID_NDK_FOLDER) \
 	&& unzip -q $(ANDROID_NDK_ARCHIVE) -d $(ANDROID_HOME) \
-	&& ln -sfn $(ANDROID_NDK_FOLDER) $(ANDROID_NDK_HOME) \
+	&& mv $(ANDROID_NDK_FOLDER) $(ANDROID_NDK_HOME) \
 	&& rm -f $(ANDROID_NDK_ARCHIVE)
 
 # updates Android SDK, install Android API, Build Tools and accept licenses

doc/source/quickstart.rst

@@ -120,7 +120,7 @@ named ``tools``, and you will need to run extra commands to install
 the SDK packages needed. 
 
 For Android NDK, note that modern releases will only work on a 64-bit
-operating system. **The minimal, and recommended, NDK version to use is r19b:**
+operating system. **The minimal, and recommended, NDK version to use is r23b:**
 
  - `Go to ndk downloads page <https://developer.android.com/ndk/downloads/>`_
  - Windows users should create a virtual machine with an GNU Linux os
@@ -154,7 +154,7 @@ variables necessary for building on android::
 
     # Adjust the paths!
     export ANDROIDSDK="$HOME/Documents/android-sdk-27"
-    export ANDROIDNDK="$HOME/Documents/android-ndk-r19b"
+    export ANDROIDNDK="$HOME/Documents/android-ndk-r23b"
     export ANDROIDAPI="27"  # Target API version of your application
     export NDKAPI="21"  # Minimum supported API version of your application
     export ANDROIDNDKVER="r10e"  # Version of the NDK you installed

pythonforandroid/archs.py

@@ -1,19 +1,14 @@
 from distutils.spawn import find_executable
 from os import environ
-from os.path import join, split, exists
+from os.path import join
 from multiprocessing import cpu_count
-from glob import glob
 
-from pythonforandroid.logger import warning
 from pythonforandroid.recipe import Recipe
 from pythonforandroid.util import BuildInterruptingException, build_platform
 
 
 class Arch:
 
-    toolchain_prefix = None
-    '''The prefix for the toolchain dir in the NDK.'''
-
     command_prefix = None
     '''The prefix for NDK commands such as gcc.'''
 
@@ -30,8 +25,7 @@ class Arch:
 
     common_cppflags = [
         '-DANDROID',
-        '-D__ANDROID_API__={ctx.ndk_api}',
-        '-I{ctx.ndk_sysroot}/usr/include/{command_prefix}',
+        '-I{ctx.ndk_sysroot}/usr/include',
         '-I{python_includes}',
     ]
 
@@ -62,20 +56,6 @@ def __str__(self):
     def ndk_lib_dir(self):
         return join(self.ctx.ndk_sysroot, 'usr', 'lib', self.command_prefix, str(self.ctx.ndk_api))
 
-    @property
-    def ndk_platform(self):
-        warning("ndk_platform is deprecated and should be avoided in new recipes")
-        ndk_platform = join(
-            self.ctx.ndk_dir,
-            'platforms',
-            'android-{}'.format(self.ctx.ndk_api),
-            self.platform_dir)
-        if not exists(ndk_platform):
-            BuildInterruptingException(
-                "The requested platform folder doesn't exist. If you're building on ndk >= r22, and seeing this error, one of the required recipe is using a removed feature."
-            )
-        return ndk_platform
-
     @property
     def include_dirs(self):
         return [
@@ -97,13 +77,10 @@ def target(self):
     @property
     def clang_path(self):
         """Full path of the clang compiler"""
-        llvm_dirname = split(
-            glob(join(self.ctx.ndk_dir, 'toolchains', 'llvm*'))[-1]
-        )[-1]
         return join(
             self.ctx.ndk_dir,
             'toolchains',
-            llvm_dirname,
+            'llvm',
             'prebuilt',
             build_platform,
             'bin',
@@ -190,12 +167,10 @@ def get_env(self, with_flags_in_cc=True):
             )
 
         # Compiler: `CC` and `CXX` (and make sure that the compiler exists)
-        environ['PATH'] = '{clang_path}:{path}'.format(
-            clang_path=self.clang_path, path=environ['PATH']
-        )
-        cc = find_executable(self.clang_exe, path=environ['PATH'])
+        env['PATH'] = self.ctx.env['PATH']
+        cc = find_executable(self.clang_exe, path=env['PATH'])
         if cc is None:
-            print('Searching path are: {!r}'.format(environ['PATH']))
+            print('Searching path are: {!r}'.format(env['PATH']))
             raise BuildInterruptingException(
                 'Couldn\'t find executable for CC. This indicates a '
                 'problem locating the {} executable in the Android '
@@ -219,21 +194,18 @@ def get_env(self, with_flags_in_cc=True):
                 execxx=self.clang_exe_cxx,
                 ccache=ccache)
 
-        # Android's binaries
-        command_prefix = self.command_prefix
-        env['AR'] = '{}-ar'.format(command_prefix)
-        env['RANLIB'] = '{}-ranlib'.format(command_prefix)
-        env['STRIP'] = '{}-strip --strip-unneeded'.format(command_prefix)
+        # Android's LLVM binutils
+        env['AR'] = f'{self.clang_path}/llvm-ar'
+        env['RANLIB'] = f'{self.clang_path}/llvm-ranlib'
+        env['STRIP'] = f'{self.clang_path}/llvm-strip --strip-unneeded'
+        env['READELF'] = f'{self.clang_path}/llvm-readelf'
+        env['OBJCOPY'] = f'{self.clang_path}/llvm-objcopy'
+
         env['MAKE'] = 'make -j{}'.format(str(cpu_count()))
-        env['READELF'] = '{}-readelf'.format(command_prefix)
-        env['NM'] = '{}-nm'.format(command_prefix)
-        env['LD'] = '{}-ld'.format(command_prefix)
 
         # Android's arch/toolchain
         env['ARCH'] = self.arch
         env['NDK_API'] = 'android-{}'.format(str(self.ctx.ndk_api))
-        env['TOOLCHAIN_PREFIX'] = self.toolchain_prefix
-        env['TOOLCHAIN_VERSION'] = self.ctx.toolchain_version
 
         # Custom linker options
         env['LDSHARED'] = env['CC'] + ' ' + ' '.join(self.common_ldshared)
@@ -251,8 +223,6 @@ def get_env(self, with_flags_in_cc=True):
             ),
         )
 
-        env['PATH'] = environ['PATH']
-
         # for reproducible builds
         if 'SOURCE_DATE_EPOCH' in environ:
             for k in 'LC_ALL TZ SOURCE_DATE_EPOCH PYTHONHASHSEED BUILD_DATE BUILD_TIME'.split():
@@ -264,9 +234,7 @@ def get_env(self, with_flags_in_cc=True):
 
 class ArchARM(Arch):
     arch = "armeabi"
-    toolchain_prefix = 'arm-linux-androideabi'
     command_prefix = 'arm-linux-androideabi'
-    platform_dir = 'arch-arm'
 
     @property
     def target(self):
@@ -290,12 +258,9 @@ class ArchARMv7_a(ArchARM):
 
 class Archx86(Arch):
     arch = 'x86'
-    toolchain_prefix = 'x86'
     command_prefix = 'i686-linux-android'
-    platform_dir = 'arch-x86'
     arch_cflags = [
         '-march=i686',
-        '-mtune=intel',
         '-mssse3',
         '-mfpmath=sse',
         '-m32',
@@ -304,26 +269,22 @@ class Archx86(Arch):
 
 class Archx86_64(Arch):
     arch = 'x86_64'
-    toolchain_prefix = 'x86_64'
     command_prefix = 'x86_64-linux-android'
-    platform_dir = 'arch-x86_64'
     arch_cflags = [
         '-march=x86-64',
         '-msse4.2',
         '-mpopcnt',
         '-m64',
-        '-mtune=intel',
         '-fPIC',
     ]
 
 
 class ArchAarch_64(Arch):
     arch = 'arm64-v8a'
-    toolchain_prefix = 'aarch64-linux-android'
     command_prefix = 'aarch64-linux-android'
-    platform_dir = 'arch-arm64'
     arch_cflags = [
         '-march=armv8-a',
+        '-fPIC'
         # '-I' + join(dirname(__file__), 'includes', 'arm64-v8a'),
     ]
 

pythonforandroid/bootstraps/common/build/gradle/wrapper/gradle-wrapper.properties

@@ -3,4 +3,4 @@ distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
 zipStoreBase=GRADLE_USER_HOME
 zipStorePath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-6.4.1-all.zip
+distributionUrl=https\://services.gradle.org/distributions/gradle-7.4.1-all.zip

pythonforandroid/bootstraps/common/build/templates/build.tmpl.gradle

@@ -5,7 +5,7 @@ buildscript {
        jcenter()
     }
     dependencies {
-        classpath 'com.android.tools.build:gradle:3.5.4'
+        classpath 'com.android.tools.build:gradle:7.1.2'
     }
 }
 
@@ -41,6 +41,9 @@ android {
 
 
 	packagingOptions {
+        jniLibs {
+            useLegacyPackaging = true
+        }
         {% if debug_build -%}
         doNotStrip '**/*.so'
         {% else %}

pythonforandroid/bootstraps/common/build/templates/gradle.tmpl.properties

@@ -1,5 +1,4 @@
 {% if args.enable_androidx %}
 android.useAndroidX=true
 android.enableJetifier=true
-{% endif %}
-android.bundle.enableUncompressedNativeLibs=false
+{% endif %}

pythonforandroid/bootstraps/sdl2/build/templates/AndroidManifest.tmpl.xml

@@ -62,7 +62,7 @@
                  {{ args.extra_manifest_application_arguments }}
                  android:theme="{{args.android_apptheme}}{% if not args.window %}.Fullscreen{% endif %}"
                  android:hardwareAccelerated="true"
-                 >
+                 android:extractNativeLibs="true" >
         {% for l in args.android_used_libs %}
         <uses-library android:name="{{ l }}" />
         {% endfor %}

pythonforandroid/bootstraps/service_only/build/templates/AndroidManifest.tmpl.xml

@@ -50,7 +50,8 @@
                  android:allowBackup="{{ args.allow_backup }}"
                  {% if args.backup_rules %}android:fullBackupContent="@xml/{{ args.backup_rules }}"{% endif %}
                  android:theme="{{args.android_apptheme}}{% if not args.window %}.Fullscreen{% endif %}"
-                 android:hardwareAccelerated="true" >
+                 android:hardwareAccelerated="true"
+                 android:extractNativeLibs="true" >
         {% for l in args.android_used_libs %}
         <uses-library android:name="{{ l }}" />
         {% endfor %}

pythonforandroid/bootstraps/webview/build/templates/AndroidManifest.tmpl.xml

@@ -52,6 +52,7 @@
                  android:theme="{{args.android_apptheme}}{% if not args.window %}.Fullscreen{% endif %}"
                  android:hardwareAccelerated="true"
                  android:usesCleartextTraffic="true"
+                 android:extractNativeLibs="true"
                  {% if debug %}android:debuggable="true"{% endif %}
                  >
         {% for l in args.android_used_libs %}