More fixes.

Author: Alexhuszagh

docker/Dockerfile.aarch64-linux-android

@@ -17,13 +17,16 @@ ARG ANDROID_NDK=r21d
 ARG ANDROID_SDK=28
 ARG ANDROID_VERSION=9.0.0_r1
 ARG ANDROID_SYSTEM_COMPLETE=0
+ARG PYTHON_TMPDIR=/tmp/android
 
 COPY android-ndk.sh /
 RUN /android-ndk.sh arm64
 ENV PATH=$PATH:/android-ndk/bin
 
 # TODO(ahuszagh) Restore...
-COPY android-system.sh remove_android_tests.py /
+COPY android-system.sh /
+RUN mkdir -p $PYTHON_TMPDIR
+COPY android $PYTHON_TMPDIR
 #RUN /android-system.sh arm64
 #
 #COPY android-symlink.sh /

docker/android-system.sh

@@ -64,7 +64,6 @@ main() {
         python \
         python3 \
         xz-utils
-    purge_list+=(default-jre)
 
     curl --retry 3 -sSfL https://storage.googleapis.com/git-repo-downloads/repo -O
     chmod +x repo
@@ -128,30 +127,40 @@ main() {
     popd
 
     rm -rf "${td}"
+    rm -rf "${PYTHON_TMPDIR}"
     rm "${0}"
 }
 
+# java isn't required for the build, but the build expects to
+# find a java compiler. the supported android versions are:
+# https://source.android.com/docs/setup/start/older-versions
+#   Android 7: OpenJDK-8
 fake_java() {
+    local java_type=
     local java_version=
-    local icedtea_version=
+    local jre_info=
+    local build_info=
+
     case "${MAJOR_VERSION}" in
         5|6)
+            java_type=java
             java_version=1.7.0
-            icedtea_version=2.6.9
+            jre_info="IcedTea 2.6.9"
+            build_info="build 24.131-b00, mixed mode"
             ;;
         *)
-            java_version=1.8.0
-            icedtea_version=2.6.9
+            java_type=openjdk
+            java_version=1.8.0_342
+            jre_info="build 1.8.0_342-8u342-b07-0ubuntu1~20.04-b07"
+            build_info="build 25.342-b07, mixed mode"
             ;;
     esac
 
-    # fake java and javac, it is not necessary for what we build, but the build
-    # script asks for it
     cat << EOF > /usr/bin/java
 #!/usr/bin/env bash
-echo "java version \"${java_version}\""
-echo "OpenJDK Runtime Environment (IcedTea ${icedtea_version})"
-echo "OpenJDK 64-Bit Server VM (build 24.131-b00, mixed mode)"
+echo "${java_type} version \"${java_version}\""
+echo "OpenJDK Runtime Environment (${jre_info})"
+echo "OpenJDK 64-Bit Server VM (${build_info})"
 EOF
 
     cat << EOF > /usr/bin/javac
@@ -164,7 +173,7 @@ EOF
 
     # more faking
     export ANDROID_JAVA_HOME=/tmp
-    mkdir /tmp/lib/
+    mkdir -p /tmp/lib/
     touch /tmp/lib/tools.jar
 }
 
@@ -225,6 +234,7 @@ android_repo_v5() {
     sync external/libselinux
     sync external/mksh
     sync external/openssl
+    sync external/pcre
     sync external/stlport
     sync prebuilts/clang/linux-x86/host/3.5
     sync system/core
@@ -262,9 +272,11 @@ android_repo_v6() {
     sync external/compiler-rt
     sync external/libcxx
     sync external/libcxxabi
+    sync external/libselinux
     sync external/elfutils
     sync external/jemalloc
     sync external/mksh
+    sync external/pcre
     sync external/safe-iop
     sync external/zlib
     sync libnativehelper
@@ -300,6 +312,7 @@ android_repo_v6() {
 
 # tested on 7.0.0_r36 (SDK 24)
 # tested on 7.1.2_r39 (SDK 25, not supported)
+#   API level 25, requires for Android 7.1, is not provided in NDKs
 android_repo_v7() {
     sync bionic
     sync build
@@ -321,6 +334,7 @@ android_repo_v7() {
     sync prebuilts/clang/host/linux-x86
     sync prebuilts/gcc/linux-x86/host/x86_64-linux-glibc2.15-4.8
     sync prebuilts/misc
+    sync prebuilts/ndk
     sync prebuilts/ninja/linux-x86
     sync system/core
 
@@ -349,12 +363,26 @@ android_repo_v7() {
     # the unittests from it is a bit of work.
     rm bionic/tools/relocation_packer/Android.mk
 
-    ANDROID_MAJOR=7 python3 /remove_android_tests.py
+    remove_tests
 }
 
 # TODO: tested on 8.0.0_r51 (SDK 26)
-# TODO: tested on 8.1.0_r81 (SDK 2278.0.0_r516)
+#   Currently failing due to libLLVM/libclang issues
+# TODO: tested on 8.1.0_r81 (SDK 27)
 android_repo_v8() {
+    # external/compiler-rt/lib/sanitizer_common/tests/Android.bp:37:1
+    # name: "san_test-Nolibc"
+    #
+    # libnativehelper/Android.bp
+    # subdirs = ["tests"]
+    #
+    # system/core/bootstat/Android.bp
+    # whole_static_libs: ["libgtest_prod"],
+    #
+    # external/compiler-rt/lib/asan/Android.bp
+    # name: "libasan_noinst_test",
+    #
+
     sync bionic
     sync build
     sync build/make
@@ -363,34 +391,42 @@ android_repo_v8() {
     sync external/boringssl
     sync external/clang
     sync external/compiler-rt
+    sync external/elfutils
     sync external/jemalloc
     sync external/libcxx
     sync external/libcxxabi
-    sync external/libnl
+    # TODO(ahuszagh) Remove?
+    #sync external/libnl
+    sync external/libevent
     sync external/libunwind
     sync external/libunwind_llvm
     sync external/llvm
     sync external/lzma
     sync external/mksh
     sync external/pcre
+    sync external/safe-iop
     sync external/selinux
     sync external/zlib
-    sync hardware/interfaces
-    sync hardware/libhardware
-    sync frameworks/native
+    # TODO(ahuszagh) Remove?
+    #sync hardware/interfaces
+    #sync hardware/libhardware
+    #sync frameworks/native
     sync libnativehelper
     sync prebuilts/build-tools
     sync prebuilts/clang/host/linux-x86
+    # TODO(ahuszagh) clang-tools isn't available till 9.0
     sync prebuilts/gcc/linux-x86/host/x86_64-linux-glibc2.15-4.8
     sync prebuilts/go/linux-x86
-    # needed for libnativehelper_compat_libc++
     sync prebuilts/ndk
     sync system/core
-    sync system/libfmq
-    sync system/libhidl
-    sync system/libhwbinder
-    sync system/media
-    sync system/tools/hidl
+    sync toolchain/binutils
+
+    # TODO(ahuszagh) See if I can remove all of these
+    #sync system/libfmq
+    #sync system/libhidl
+    #sync system/libhwbinder
+    #sync system/media
+    #sync system/tools/hidl
     # TODO(ahuszagh) Android 8 is a mess
 
     # avoid build tests
@@ -399,26 +435,21 @@ android_repo_v8() {
     rm bionic/tests/Android.bp
     rm bionic/benchmarks/Android.bp
     rm bionic/tests/libs/Android.bp
-    rm hardware/interfaces/tests/Android.bp
-    rm system/tools/hidl/test/Android.bp
+    # TODO(ahuszagh) Can probably remove this
+#    rm hardware/interfaces/tests/Android.bp
+#    rm system/tools/hidl/test/Android.bp
     # we don't need the relocation packer, and removing
     # the unittests from it is a bit of work.
     rm bionic/tools/relocation_packer/Android.bp
     rm bionic/tools/relocation_packer/Android.mk
+    rm system/core/libgrallocusage/Android.bp
+    rm system/core/libmemtrack/Android.bp
+    rm system/core/libsysutils/Android.bp
+
+    # avoid java dependencies
+    rm external/lzma/Java/Tukaani/Android.mk
 
-    # extra utilities we don't need
-    rm frameworks/native/libs/vr/Android.bp
-    rm frameworks/native/services/Android.bp
-    rm frameworks/native/services/*/Android.bp
-    rm hardware/interfaces/automotive/Android.bp
-    rm hardware/interfaces/camera/Android.bp
-    rm system/libhidl/transport/Android.bp
-    rm system/media/alsa_utils/Android.bp
-    rm system/media/audio_route/Android.bp
-    rm system/media/audio_utils/tests/Android.bp
-    # TODO(ahuszagh) Need to remove the tests here
-
-    ANDROID_MAJOR=8 python3 /remove_android_tests.py
+    remove_tests
 }
 
 # tested on 9.0.0_r1 (SDK 28)
@@ -482,7 +513,7 @@ android_repo_v9() {
     rm bionic/tests/headers/Android.bp
     rm bionic/tests/headers/posix/Android.bp
 
-    ANDROID_MAJOR=9 python3 /remove_android_tests.py
+    remove_tests
 }
 
 # tested on 10.0.0_r47 (SDK 29)
@@ -553,7 +584,19 @@ android_repo_v10() {
     rm bionic/tests/headers/Android.bp
     rm bionic/tests/headers/posix/Android.bp
 
-    ANDROID_MAJOR=10 python3 /remove_android_tests.py
+    remove_tests
+}
+
+remove_tests() {
+    install_packages python3-pip
+
+    export PYTHONPATH="${PYTHON_TMPDIR}/lib/python3.8/site-packages/":"${PYTHONPATH}"
+    mkdir -p "${PYTHON_TMPDIR}"
+    python3 -m pip install sly==0.4.0 --prefix "${PYTHON_TMPDIR}"
+
+    # TODO(ahuszagh) Need to change this
+    # Need a custom entrypoint
+    ANDROID_MAJOR="${MAJOR_VERSION}" python3 "${PYTHON_TMPDIR}/remove_tests.py"
 }
 
 sync() {

docker/android/android/__init__.py

@@ -0,0 +1,11 @@
+import sys
+
+# we run this script once every build, and we'd rather
+# have much smaller image sizes, so copying without
+# any bytecode is a better idea.
+sys.dont_write_bytecode = True
+
+__all__ = [
+    "make",
+    "soong",
+]

docker/android/android/__pycache__/__init__.cpython-310.pyc

@@ -0,0 +1,66 @@
+'''
+    make
+    ====
+
+    utilities to process makefiles
+'''
+
+import re
+
+from . import util
+
+class Section:
+    def __init__(self, title, contents):
+        self.title = title
+        self.contents = contents
+
+    def is_test(self):
+        return self.title is not None and 'test' in self.title.lower()
+
+    def is_benchmark(self):
+        return self.title is not None and 'benchmark' in self.title.lower()
+
+    def is_dev(self):
+        return self.is_test() or self.is_benchmark()
+
+    def __repr__(self):
+        return self.contents
+
+class Makefile:
+    def __init__(self, sections):
+        self.sections = sections
+
+    @staticmethod
+    def parse(path):
+        with open(path, 'r') as f:
+            contents = f.read()
+
+        sep1 = r'#\s+=+'
+        sep2 = r'#\s+-+'
+        comment = r'[A-Za-z0-9._ -]+'
+        pat1 = fr'(?:{sep1}\n)?#\s+({comment})\n{sep1}'
+        pat2 = fr'(?:{sep2}\n)?#\s+({comment})\n{sep2}'
+        pattern = fr'(?:{pat1})|(?:{pat2})'
+
+        sections = []
+        matches = list(re.finditer(pattern, contents))
+        if len(matches) == 0:
+            sections.append(Section(None, contents))
+        else:
+            first = matches[0]
+            last = matches[-1]
+            if first.start() != 0:
+                sections.append(Section(None, contents[:first.start()]))
+            for (prev, nxt) in util.windows(matches, 2):
+                title = prev.group(1) or prev.group(2)
+                sections.append(Section(title, contents[prev.start():nxt.start()]))
+            title = last.group(1) or prev.group(2)
+            sections.append(Section(title, contents[last.start():]))
+
+        return Makefile(sections)
+
+    def filter_dev(self):
+        return Makefile([i for i in self.sections if not i.is_dev()])
+
+    def __repr__(self):
+        return ''.join([str(i) for i in self.sections])