Auto merge of #1152 - palfrey:retry-curl, r=alexcrichton

bors · bors · commit d4b45f51ea75 · 2018-11-26T22:20:01.000Z
Retry all curl operations As the docker pulls from #1148 appear mostly fixed, but we're seeing occasional curl timeouts still (e.g. https://travis-ci.org/rust-lang/libc/jobs/459715534, https://travis-ci.org/rust-lang/libc/jobs/459611627), so this should improve things at least a bit.
diff --git a/ci/android-install-ndk.sh b/ci/android-install-ndk.sh
@@ -11,7 +11,7 @@
 
 set -ex
 
-curl -O https://dl.google.com/android/repository/android-ndk-r15b-linux-x86_64.zip
+curl --retry 5 -O https://dl.google.com/android/repository/android-ndk-r15b-linux-x86_64.zip
 unzip -q android-ndk-r15b-linux-x86_64.zip
 
 case "$1" in
diff --git a/ci/android-install-sdk.sh b/ci/android-install-sdk.sh
@@ -19,7 +19,7 @@ set -ex
 # which apparently magically accepts the licenses.
 
 mkdir sdk
-curl https://dl.google.com/android/repository/sdk-tools-linux-3859397.zip -O
+curl --retry 5 https://dl.google.com/android/repository/sdk-tools-linux-3859397.zip -O
 unzip -d sdk sdk-tools-linux-3859397.zip
 
 case "$1" in
diff --git a/ci/android-sysimage.sh b/ci/android-sysimage.sh
@@ -24,7 +24,7 @@ main() {
     apt-get install --no-install-recommends e2tools
 
     pushd "${td}"
-    curl -O "${URL}/${name}"
+    curl --retry 5 -O "${URL}/${name}"
     unzip -q "${name}"
 
     local system
diff --git a/ci/docker/aarch64-unknown-linux-musl/Dockerfile b/ci/docker/aarch64-unknown-linux-musl/Dockerfile
@@ -3,7 +3,7 @@ FROM ubuntu:17.10
 RUN apt-get update && apt-get install -y --no-install-recommends \
   gcc make libc6-dev git curl ca-certificates \
   gcc-aarch64-linux-gnu qemu-user
-RUN curl https://www.musl-libc.org/releases/musl-1.1.19.tar.gz | \
+RUN curl --retry 5 https://www.musl-libc.org/releases/musl-1.1.19.tar.gz | \
     tar xzf - && \
     cd musl-1.1.19 && \
     CC=aarch64-linux-gnu-gcc \
@@ -12,7 +12,7 @@ RUN curl https://www.musl-libc.org/releases/musl-1.1.19.tar.gz | \
     cd .. && \
     rm -rf musl-1.1.19
 # Install linux kernel headers sanitized for use with musl
-RUN curl -L  https://github.com/sabotage-linux/kernel-headers/archive/v3.12.6-6.tar.gz | \
+RUN curl --retry 5 -L https://github.com/sabotage-linux/kernel-headers/archive/v3.12.6-6.tar.gz | \
     tar xzf - && \
     cd kernel-headers-3.12.6-6 && \
     make ARCH=arm64 prefix=/musl-aarch64 install -j4 && \
diff --git a/ci/docker/arm-unknown-linux-musleabihf/Dockerfile b/ci/docker/arm-unknown-linux-musleabihf/Dockerfile
@@ -4,15 +4,15 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
   gcc make libc6-dev git curl ca-certificates \
   gcc-arm-linux-gnueabihf qemu-user
 
-RUN curl https://www.musl-libc.org/releases/musl-1.1.19.tar.gz | tar xzf -
+RUN curl --retry 5 https://www.musl-libc.org/releases/musl-1.1.19.tar.gz | tar xzf -
 WORKDIR /musl-1.1.19
 RUN CC=arm-linux-gnueabihf-gcc \
     CFLAGS="-march=armv6 -marm" \
     ./configure --prefix=/musl-arm --enable-wrapper=yes
 RUN make install -j4
 
 # Install linux kernel headers sanitized for use with musl
-RUN curl -L  https://github.com/sabotage-linux/kernel-headers/archive/v3.12.6-6.tar.gz | \
+RUN curl --retry 5 -L https://github.com/sabotage-linux/kernel-headers/archive/v3.12.6-6.tar.gz | \
     tar xzf - && \
     cd kernel-headers-3.12.6-6 && \
     make ARCH=arm prefix=/musl-arm install -j4 && \
diff --git a/ci/docker/i686-unknown-linux-musl/Dockerfile b/ci/docker/i686-unknown-linux-musl/Dockerfile
@@ -12,15 +12,15 @@ RUN apt-get install -y --no-install-recommends \
 #   since otherwise the script will fail to find a compiler.
 # * We manually unset CROSS_COMPILE when running make; otherwise the makefile
 #   will call the non-existent binary 'i686-ar'.
-RUN curl https://www.musl-libc.org/releases/musl-1.1.19.tar.gz | \
+RUN curl --retry 5 https://www.musl-libc.org/releases/musl-1.1.19.tar.gz | \
     tar xzf - && \
     cd musl-1.1.19 && \
     CC=gcc CFLAGS=-m32 ./configure --prefix=/musl-i686 --disable-shared --target=i686 && \
     make CROSS_COMPILE= install -j4 && \
     cd .. && \
     rm -rf musl-1.1.19
 # Install linux kernel headers sanitized for use with musl
-RUN curl -L  https://github.com/sabotage-linux/kernel-headers/archive/v3.12.6-6.tar.gz | \
+RUN curl --retry 5 -L https://github.com/sabotage-linux/kernel-headers/archive/v3.12.6-6.tar.gz | \
     tar xzf - && \
     cd kernel-headers-3.12.6-6 && \
     make ARCH=i386 prefix=/musl-i686 install -j4 && \
diff --git a/ci/docker/mips-unknown-linux-musl/Dockerfile b/ci/docker/mips-unknown-linux-musl/Dockerfile
@@ -8,7 +8,7 @@ RUN mkdir /toolchain
 
 # Note that this originally came from:
 # https://downloads.openwrt.org/snapshots/trunk/ar71xx/generic/OpenWrt-SDK-ar71xx-generic_gcc-5.3.0_musl-1.1.15.Linux-x86_64.tar.bz2
-RUN curl -L https://s3-us-west-1.amazonaws.com/rust-lang-ci2/libc/OpenWrt-SDK-ar71xx-generic_gcc-5.3.0_musl-1.1.15.Linux-x86_64.tar.bz2 | \
+RUN curl --retry 5 -L https://s3-us-west-1.amazonaws.com/rust-lang-ci2/libc/OpenWrt-SDK-ar71xx-generic_gcc-5.3.0_musl-1.1.15.Linux-x86_64.tar.bz2 | \
       tar xjf - -C /toolchain --strip-components=1
 
 ENV PATH=$PATH:/rust/bin:/toolchain/staging_dir/toolchain-mips_34kc_gcc-5.3.0_musl-1.1.15/bin \
diff --git a/ci/docker/mipsel-unknown-linux-musl/Dockerfile b/ci/docker/mipsel-unknown-linux-musl/Dockerfile
@@ -8,7 +8,7 @@ RUN mkdir /toolchain
 
 # Note that this originally came from:
 # https://downloads.openwrt.org/snapshots/trunk/malta/generic/OpenWrt-Toolchain-malta-le_gcc-5.3.0_musl-1.1.15.Linux-x86_64.tar.bz2
-RUN curl -L https://s3-us-west-1.amazonaws.com/rust-lang-ci2/libc/OpenWrt-Toolchain-malta-le_gcc-5.3.0_musl-1.1.15.Linux-x86_64.tar.bz2 | \
+RUN curl --retry 5 -L https://s3-us-west-1.amazonaws.com/rust-lang-ci2/libc/OpenWrt-Toolchain-malta-le_gcc-5.3.0_musl-1.1.15.Linux-x86_64.tar.bz2 | \
       tar xjf - -C /toolchain --strip-components=2
 
 ENV PATH=$PATH:/rust/bin:/toolchain/bin \
diff --git a/ci/docker/x86_64-unknown-linux-musl/Dockerfile b/ci/docker/x86_64-unknown-linux-musl/Dockerfile
@@ -3,15 +3,15 @@ FROM ubuntu:17.10
 RUN apt-get update
 RUN apt-get install -y --no-install-recommends \
   gcc make libc6-dev git curl ca-certificates
-RUN curl https://www.musl-libc.org/releases/musl-1.1.19.tar.gz | \
+RUN curl --retry 5 https://www.musl-libc.org/releases/musl-1.1.19.tar.gz | \
     tar xzf - && \
     cd musl-1.1.19 && \
     ./configure --prefix=/musl-x86_64 && \
     make install -j4 && \
     cd .. && \
     rm -rf musl-1.1.19
 # Install linux kernel headers sanitized for use with musl
-RUN curl -L  https://github.com/sabotage-linux/kernel-headers/archive/v3.12.6-6.tar.gz | \
+RUN curl --retry 5 -L https://github.com/sabotage-linux/kernel-headers/archive/v3.12.6-6.tar.gz | \
     tar xzf - && \
     cd kernel-headers-3.12.6-6 && \
     make ARCH=x86_64 prefix=/musl-x86_64 install -j4 && \
diff --git a/ci/emscripten.sh b/ci/emscripten.sh
@@ -29,7 +29,7 @@ exit 1
 }
 
 cd /
-curl -L https://s3.amazonaws.com/mozilla-games/emscripten/releases/emsdk-portable.tar.gz | \
+curl --retry 5 -L https://s3.amazonaws.com/mozilla-games/emscripten/releases/emsdk-portable.tar.gz | \
     tar -xz
 
 cd /emsdk-portable
@@ -51,6 +51,6 @@ chmod a+rxw -R /emsdk-portable
 
 # node 8 is required to run wasm
 cd /
-curl -L https://nodejs.org/dist/v8.0.0/node-v8.0.0-linux-x64.tar.xz | \
+curl --retry 5 -L https://nodejs.org/dist/v8.0.0/node-v8.0.0-linux-x64.tar.xz | \
     tar -xJ
 
diff --git a/ci/linux-s390x.sh b/ci/linux-s390x.sh
@@ -5,9 +5,9 @@ set -ex
 mkdir -m 777 /qemu
 cd /qemu
 
-curl -LO https://github.com/qemu/qemu/raw/master/pc-bios/s390-ccw.img
-curl -LO http://ftp.debian.org/debian/dists/testing/main/installer-s390x/20170828/images/generic/kernel.debian
-curl -LO http://ftp.debian.org/debian/dists/testing/main/installer-s390x/20170828/images/generic/initrd.debian
+curl --retry 5 -LO https://github.com/qemu/qemu/raw/master/pc-bios/s390-ccw.img
+curl --retry 5 -LO http://ftp.debian.org/debian/dists/testing/main/installer-s390x/20170828/images/generic/kernel.debian
+curl --retry 5 -LO http://ftp.debian.org/debian/dists/testing/main/installer-s390x/20170828/images/generic/initrd.debian
 
 mv kernel.debian kernel
 mv initrd.debian initrd.gz
diff --git a/ci/linux-sparc64.sh b/ci/linux-sparc64.sh
@@ -5,7 +5,7 @@ set -ex
 mkdir -m 777 /qemu
 cd /qemu
 
-curl -LO https://cdimage.debian.org/cdimage/ports/9.0/sparc64/iso-cd/debian-9.0-sparc64-NETINST-1.iso
+curl --retry 5 -LO https://cdimage.debian.org/cdimage/ports/9.0/sparc64/iso-cd/debian-9.0-sparc64-NETINST-1.iso
 7z e debian-9.0-sparc64-NETINST-1.iso boot/initrd.gz
 7z e debian-9.0-sparc64-NETINST-1.iso boot/sparc64
 mv sparc64 kernel
diff --git a/ci/run.sh b/ci/run.sh
@@ -21,21 +21,21 @@ if [ "$QEMU" != "" ]; then
     # image is .gz : download and uncompress it
     qemufile="$(echo "${QEMU%.gz}" | sed 's/\//__/g')"
     if [ ! -f "${tmpdir}/${qemufile}" ]; then
-      curl "https://s3-us-west-1.amazonaws.com/rust-lang-ci2/libc/${QEMU}" | \
+      curl --retry 5 "https://s3-us-west-1.amazonaws.com/rust-lang-ci2/libc/${QEMU}" | \
           gunzip -d > "${tmpdir}/${qemufile}"
     fi
   elif [ -z "${QEMU#*.xz}" ]; then
     # image is .xz : download and uncompress it
     qemufile="$(echo "${QEMU%.xz}" | sed 's/\//__/g')"
     if [ ! -f "${tmpdir}/${qemufile}" ]; then
-      curl "https://s3-us-west-1.amazonaws.com/rust-lang-ci2/libc/${QEMU}" | \
+      curl --retry 5 "https://s3-us-west-1.amazonaws.com/rust-lang-ci2/libc/${QEMU}" | \
           unxz > "${tmpdir}/${qemufile}"
     fi
   else
     # plain qcow2 image: just download it
     qemufile="$(echo "${QEMU}" | sed 's/\//__/g')"
     if [ ! -f "${tmpdir}/${qemufile}" ]; then
-      curl "https://s3-us-west-1.amazonaws.com/rust-lang-ci2/libc/${QEMU}" \
+      curl --retry 5 "https://s3-us-west-1.amazonaws.com/rust-lang-ci2/libc/${QEMU}" \
         > "${tmpdir}/${qemufile}"
     fi
   fi
