Auto merge of #41863 - malbarbo:update-android-builder, r=alexcrichton

ci: Update android ndk and sdk

Make install-sdk.sh and install-ndk.sh more generic so future updates can be made directly on Dockerfile. Update ndk to r13b, which will be necessary to make host builds for android (in the future). Update sdk to r25.2.5 (maybe some emulator performance improvement).

Author: bors

src/ci/docker/arm-android/Dockerfile

@@ -1,7 +1,6 @@
 FROM ubuntu:16.04
 
-RUN dpkg --add-architecture i386 && \
-    apt-get update && \
+RUN apt-get update && \
     apt-get install -y --no-install-recommends \
   g++ \
   make \
@@ -12,35 +11,54 @@ RUN dpkg --add-architecture i386 && \
   git \
   cmake \
   unzip \
-  expect \
-  openjdk-9-jre-headless \
   sudo \
-  libstdc++6:i386 \
   xz-utils \
   libssl-dev \
   pkg-config
 
-WORKDIR /android/
-ENV PATH=$PATH:/android/ndk-arm-9/bin:/android/sdk/tools:/android/sdk/platform-tools
-
-COPY install-ndk.sh install-sdk.sh accept-licenses.sh /android/
-RUN sh /android/install-ndk.sh
-RUN sh /android/install-sdk.sh
-
 RUN curl -OL https://github.com/Yelp/dumb-init/releases/download/v1.2.0/dumb-init_1.2.0_amd64.deb && \
     dpkg -i dumb-init_*.deb && \
     rm dumb-init_*.deb
 
-COPY start-emulator.sh /android/
+RUN curl -o /usr/local/bin/sccache \
+      https://s3.amazonaws.com/rust-lang-ci/rust-ci-mirror/2017-04-29-sccache-x86_64-unknown-linux-musl && \
+    chmod +x /usr/local/bin/sccache
 
-ENTRYPOINT ["/usr/bin/dumb-init", "--", "/android/start-emulator.sh"]
+# Install NDK
+COPY install-ndk.sh /tmp
+RUN . /tmp/install-ndk.sh && \
+    download_ndk android-ndk-r13b-linux-x86_64.zip && \
+    make_standalone_toolchain arm 9 && \
+    remove_ndk
 
-RUN curl -o /usr/local/bin/sccache \
-      https://s3.amazonaws.com/rust-lang-ci/rust-ci-mirror/2017-02-24-sccache-x86_64-unknown-linux-gnu && \
-      chmod +x /usr/local/bin/sccache
+# Install SDK
+RUN dpkg --add-architecture i386 && \
+    apt-get update && \
+    apt-get install -y --no-install-recommends \
+  openjdk-9-jre-headless \
+  tzdata \
+  libstdc++6:i386 \
+  libgl1-mesa-glx \
+  libpulse0
+
+COPY install-sdk.sh /tmp
+RUN . /tmp/install-sdk.sh && \
+    download_sdk tools_r25.2.5-linux.zip && \
+    download_sysimage armeabi-v7a 18 && \
+    create_avd armeabi-v7a 18
+
+# Setup env
+ENV PATH=$PATH:/android/sdk/tools
+ENV PATH=$PATH:/android/sdk/platform-tools
+
+ENV TARGETS=arm-linux-androideabi
 
 ENV RUST_CONFIGURE_ARGS \
-      --target=arm-linux-androideabi \
-      --arm-linux-androideabi-ndk=/android/ndk-arm-9
+      --target=$TARGETS \
+      --arm-linux-androideabi-ndk=/android/ndk/arm-9
+
+ENV SCRIPT python2.7 ../x.py test --target $TARGETS --verbose
 
-ENV SCRIPT python2.7 ../x.py test --target arm-linux-androideabi
+# Entrypoint
+COPY start-emulator.sh /android/
+ENTRYPOINT ["/usr/bin/dumb-init", "--", "/android/start-emulator.sh"]

src/ci/docker/arm-android/accept-licenses.sh

@@ -11,23 +11,25 @@
 
 set -ex
 
-cpgdb() {
-  cp android-ndk-r11c/prebuilt/linux-x86_64/bin/gdb /android/$1/bin/$2-gdb
-  cp android-ndk-r11c/prebuilt/linux-x86_64/bin/gdb-orig /android/$1/bin/gdb-orig
-  cp -r android-ndk-r11c/prebuilt/linux-x86_64/share /android/$1/share
+URL=https://dl.google.com/android/repository
+
+download_ndk() {
+    mkdir -p /android/ndk
+    cd /android/ndk
+    curl -O $URL/$1
+    unzip -q $1
+    rm $1
+    mv android-ndk-* ndk
 }
 
-# Prep the Android NDK
-#
-# See https://github.com/servo/servo/wiki/Building-for-Android
-curl -O https://dl.google.com/android/repository/android-ndk-r11c-linux-x86_64.zip
-unzip -q android-ndk-r11c-linux-x86_64.zip
-bash android-ndk-r11c/build/tools/make-standalone-toolchain.sh \
-        --platform=android-9 \
-        --toolchain=arm-linux-androideabi-4.9 \
-        --install-dir=/android/ndk-arm-9 \
-        --ndk-dir=/android/android-ndk-r11c \
-        --arch=arm
-cpgdb ndk-arm-9 arm-linux-androideabi
+make_standalone_toolchain() {
+    # See https://developer.android.com/ndk/guides/standalone_toolchain.html
+    python2.7 /android/ndk/ndk/build/tools/make_standalone_toolchain.py \
+        --install-dir /android/ndk/$1-$2 \
+        --arch $1 \
+        --api $2
+}
 
-rm -rf ./android-ndk-r11c-linux-x86_64.zip ./android-ndk-r11c
+remove_ndk() {
+    rm -rf /android/ndk/ndk
+}

src/ci/docker/arm-android/install-ndk.sh

@@ -1,5 +1,5 @@
 #!/bin/sh
-# Copyright 2016 The Rust Project Developers. See the COPYRIGHT
+# Copyright 2017 The Rust Project Developers. See the COPYRIGHT
 # file at the top-level directory of this distribution and at
 # http://rust-lang.org/COPYRIGHT.
 #
@@ -11,23 +11,39 @@
 
 set -ex
 
-# Prep the SDK and emulator
-#
-# Note that the update process requires that we accept a bunch of licenses, and
-# we can't just pipe `yes` into it for some reason, so we take the same strategy
-# located in https://github.com/appunite/docker by just wrapping it in a script
-# which apparently magically accepts the licenses.
+URL=https://dl.google.com/android/repository
+
+download_sdk() {
+    mkdir -p /android/sdk
+    cd /android/sdk
+    curl -O $URL/$1
+    unzip -q $1
+    rm -rf $1
+}
+
+download_sysimage() {
+    # See https://developer.android.com/studio/tools/help/android.html
+    abi=$1
+    api=$2
+
+    filter="platform-tools,android-$api"
+    filter="$filter,sys-img-$abi-android-$api"
 
-mkdir sdk
-curl https://dl.google.com/android/android-sdk_r24.4-linux.tgz | \
-    tar xzf - -C sdk --strip-components=1
+    # Keep printing yes to accept the licenses
+    while true; do echo yes; sleep 10; done | \
+        /android/sdk/tools/android update sdk -a --no-ui \
+            --filter "$filter"
+}
 
-filter="platform-tools,android-18"
-filter="$filter,sys-img-armeabi-v7a-android-18"
+create_avd() {
+    # See https://developer.android.com/studio/tools/help/android.html
+    abi=$1
+    api=$2
 
-./accept-licenses.sh "android - update sdk -a --no-ui --filter $filter"
+    echo no | \
+        /android/sdk/tools/android create avd \
+            --name $abi-$api \
+            --target android-$api \
+            --abi $abi
+}
 
-echo "no" | android create avd \
-                --name arm-18 \
-                --target android-18 \
-                --abi armeabi-v7a

src/ci/docker/arm-android/install-sdk.sh

@@ -14,5 +14,12 @@ set -ex
 # Setting SHELL to a file instead on a symlink helps android
 # emulator identify the system
 export SHELL=/bin/bash
-nohup nohup emulator @arm-18 -no-window -partition-size 2047 0<&- &>/dev/null &
+
+# Using the default qemu2 engine makes time::tests::since_epoch fails because
+# the emulator date is set to unix epoch (in armeabi-v7a-18 image). Using
+# classic engine the emulator starts with the current date and the tests run
+# fine. If another image is used, this need to be evaluated again.
+nohup nohup emulator @armeabi-v7a-18 \
+    -engine classic -no-window -partition-size 2047 0<&- &>/dev/null &
+
 exec "$@"

src/ci/docker/arm-android/start-emulator.sh

@@ -1,7 +1,6 @@
 FROM ubuntu:16.04
 
-RUN dpkg --add-architecture i386 && \
-    apt-get update && \
+RUN apt-get update && \
     apt-get install -y --no-install-recommends \
   g++ \
   make \
@@ -12,28 +11,30 @@ RUN dpkg --add-architecture i386 && \
   git \
   cmake \
   unzip \
-  expect \
-  openjdk-9-jre \
   sudo \
-  libstdc++6:i386 \
   xz-utils \
   libssl-dev \
   pkg-config
 
-WORKDIR /android/
-ENV PATH=$PATH:/android/ndk-arm-9/bin:/android/sdk/tools:/android/sdk/platform-tools
-
-COPY install-ndk.sh /android/
-RUN sh /android/install-ndk.sh
-
 RUN curl -OL https://github.com/Yelp/dumb-init/releases/download/v1.2.0/dumb-init_1.2.0_amd64.deb && \
     dpkg -i dumb-init_*.deb && \
     rm dumb-init_*.deb
-ENTRYPOINT ["/usr/bin/dumb-init", "--"]
 
 RUN curl -o /usr/local/bin/sccache \
       https://s3.amazonaws.com/rust-lang-ci/rust-ci-mirror/2017-04-29-sccache-x86_64-unknown-linux-musl && \
-      chmod +x /usr/local/bin/sccache
+    chmod +x /usr/local/bin/sccache
+
+ENTRYPOINT ["/usr/bin/dumb-init", "--"]
+
+# Install NDK
+COPY install-ndk.sh /tmp
+RUN . /tmp/install-ndk.sh && \
+    download_ndk android-ndk-r13b-linux-x86_64.zip && \
+    make_standalone_toolchain arm 9 && \
+    make_standalone_toolchain x86 9 && \
+    make_standalone_toolchain arm64 21 && \
+    make_standalone_toolchain x86_64 21 && \
+    remove_ndk
 
 ENV TARGETS=arm-linux-androideabi
 ENV TARGETS=$TARGETS,armv7-linux-androideabi
@@ -44,10 +45,10 @@ ENV TARGETS=$TARGETS,x86_64-linux-android
 ENV RUST_CONFIGURE_ARGS \
       --target=$TARGETS \
       --enable-extended \
-      --arm-linux-androideabi-ndk=/android/ndk-arm-9 \
-      --armv7-linux-androideabi-ndk=/android/ndk-arm-9 \
-      --i686-linux-android-ndk=/android/ndk-x86-9 \
-      --aarch64-linux-android-ndk=/android/ndk-arm64-21 \
-      --x86_64-linux-android-ndk=/android/ndk-x86_64-21
+      --arm-linux-androideabi-ndk=/android/ndk/arm-9 \
+      --armv7-linux-androideabi-ndk=/android/ndk/arm-9 \
+      --i686-linux-android-ndk=/android/ndk/x86-9 \
+      --aarch64-linux-android-ndk=/android/ndk/arm64-21 \
+      --x86_64-linux-android-ndk=/android/ndk/x86_64-21
 
 ENV SCRIPT python2.7 ../x.py dist --target $TARGETS

src/ci/docker/dist-android/Dockerfile

@@ -11,34 +11,25 @@
 
 set -ex
 
-# Prep the Android NDK
-#
-# See https://github.com/servo/servo/wiki/Building-for-Android
-curl -O https://dl.google.com/android/repository/android-ndk-r11c-linux-x86_64.zip
-unzip -q android-ndk-r11c-linux-x86_64.zip
-bash android-ndk-r11c/build/tools/make-standalone-toolchain.sh \
-        --platform=android-9 \
-        --toolchain=arm-linux-androideabi-4.9 \
-        --install-dir=/android/ndk-arm-9 \
-        --ndk-dir=/android/android-ndk-r11c \
-        --arch=arm
-bash android-ndk-r11c/build/tools/make-standalone-toolchain.sh \
-        --platform=android-21 \
-        --toolchain=aarch64-linux-android-4.9 \
-        --install-dir=/android/ndk-arm64-21 \
-        --ndk-dir=/android/android-ndk-r11c \
-        --arch=arm64
-bash android-ndk-r11c/build/tools/make-standalone-toolchain.sh \
-        --platform=android-9 \
-        --toolchain=x86-4.9 \
-        --install-dir=/android/ndk-x86-9 \
-        --ndk-dir=/android/android-ndk-r11c \
-        --arch=x86
-bash android-ndk-r11c/build/tools/make-standalone-toolchain.sh \
-        --platform=android-21 \
-        --toolchain=x86_64-4.9 \
-        --install-dir=/android/ndk-x86_64-21 \
-        --ndk-dir=/android/android-ndk-r11c \
-        --arch=x86_64
+URL=https://dl.google.com/android/repository
+
+download_ndk() {
+    mkdir -p /android/ndk
+    cd /android/ndk
+    curl -O $URL/$1
+    unzip -q $1
+    rm $1
+    mv android-ndk-* ndk
+}
+
+make_standalone_toolchain() {
+    # See https://developer.android.com/ndk/guides/standalone_toolchain.html
+    python2.7 /android/ndk/ndk/build/tools/make_standalone_toolchain.py \
+        --install-dir /android/ndk/$1-$2 \
+        --arch $1 \
+        --api $2
+}
 
-rm -rf ./android-ndk-r11c-linux-x86_64.zip ./android-ndk-r11c
+remove_ndk() {
+    rm -rf /android/ndk/ndk
+}

src/ci/docker/dist-android/install-ndk.sh

@@ -570,7 +570,7 @@ actual:\n\
                          format!("-command={}", debugger_script.to_str().unwrap())];
 
                 let mut gdb_path = tool_path;
-                gdb_path.push_str(&format!("/bin/{}-gdb", self.config.target));
+                gdb_path.push_str("/bin/gdb");
                 let procsrv::Result {
                     out,
                     err,