travis: Make more network requests retryable

This commit attempts to move more network operations to being retryable through
various operations. For example git submodule updates, downloading snapshots,
etc, are now all in retryable steps.

Hopefully this commit can cut down on the number of network failures we've been
seeing!

Author: alexcrichton

.travis.yml

@@ -45,7 +45,7 @@ matrix:
       os: osx
       osx_image: xcode8.2
       install: &osx_install_sccache >
-        curl -o /usr/local/bin/sccache https://s3.amazonaws.com/rust-lang-ci/rust-ci-mirror/2017-02-24-sccache-x86_64-apple-darwin &&
+        travis_retry curl -o /usr/local/bin/sccache https://s3.amazonaws.com/rust-lang-ci/rust-ci-mirror/2017-02-24-sccache-x86_64-apple-darwin &&
           chmod +x /usr/local/bin/sccache
     - env: >
         RUST_CHECK_TARGET=check
@@ -63,7 +63,7 @@ matrix:
       os: osx
       osx_image: xcode8.2
       install: >
-        curl -o /usr/local/bin/sccache https://s3.amazonaws.com/rust-lang-ci/rust-ci-mirror/2017-02-24-sccache-x86_64-apple-darwin &&
+        travis_retry curl -o /usr/local/bin/sccache https://s3.amazonaws.com/rust-lang-ci/rust-ci-mirror/2017-02-24-sccache-x86_64-apple-darwin &&
           chmod +x /usr/local/bin/sccache &&
           brew uninstall --ignore-dependencies openssl &&
           brew install openssl --universal --without-test

appveyor.yml

@@ -103,7 +103,7 @@ install:
   # /usr/bin/python2.7.exe is not. To ensure we use the right interpreter we
   # move `C:\Python27` ahead in PATH and then also make sure the `python2.7.exe`
   # file exists in there (which it doesn't by default).
-  - if defined MINGW_URL appveyor DownloadFile %MINGW_URL%/%MINGW_ARCHIVE%
+  - if defined MINGW_URL appveyor-retry appveyor DownloadFile %MINGW_URL%/%MINGW_ARCHIVE%
   - if defined MINGW_URL 7z x -y %MINGW_ARCHIVE% > nul
   - if defined MINGW_URL set PATH=%CD%\%MINGW_DIR%\bin;C:\msys64\usr\bin;%PATH%
 
@@ -115,12 +115,12 @@ install:
   - set PATH=C:\Python27;%PATH%
 
   # Download and install sccache
-  - appveyor DownloadFile https://s3.amazonaws.com/rust-lang-ci/rust-ci-mirror/2017-02-24-sccache-x86_64-pc-windows-msvc
+  - appveyor-retry appveyor DownloadFile https://s3.amazonaws.com/rust-lang-ci/rust-ci-mirror/2017-02-24-sccache-x86_64-pc-windows-msvc
   - mv 2017-02-24-sccache-x86_64-pc-windows-msvc sccache
   - set PATH=%PATH%;%CD%
 
   # Install InnoSetup to get `iscc` used to produce installers
-  - choco install -y InnoSetup
+  - appveyor-retry choco install -y InnoSetup
   - set PATH="C:\Program Files (x86)\Inno Setup 5";%PATH%
 
   # Help debug some handle issues on AppVeyor
@@ -131,7 +131,7 @@ install:
   - handle.exe -accepteula -help
 
 test_script:
-  - git submodule update --init
+  - appveyor-retry sh -c 'git submodule deinit -f . && git submodule update --init'
   - set SRC=.
   - set NO_CCACHE=1
   - sh src/ci/run.sh

src/bootstrap/mk/Makefile.in

@@ -71,6 +71,8 @@ install:
 	$(Q)$(BOOTSTRAP) dist --install $(BOOTSTRAP_ARGS)
 tidy:
 	$(Q)$(BOOTSTRAP) test src/tools/tidy $(BOOTSTRAP_ARGS)
+prepare:
+	$(Q)$(BOOTSTRAP) build nonexistent/path/to/trigger/cargo/metadata
 
 check-stage2-T-arm-linux-androideabi-H-x86_64-unknown-linux-gnu:
 	$(Q)$(BOOTSTRAP) test --target arm-linux-androideabi

src/ci/docker/run.sh

@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/bin/bash
 # Copyright 2016 The Rust Project Developers. See the COPYRIGHT
 # file at the top-level directory of this distribution and at
 # http://rust-lang.org/COPYRIGHT.
@@ -19,7 +19,9 @@ ci_dir="`dirname $docker_dir`"
 src_dir="`dirname $ci_dir`"
 root_dir="`dirname $src_dir`"
 
-docker \
+source "$ci_dir/shared.sh"
+
+retry docker \
   build \
   --rm \
   -t rust-ci \

src/ci/run.sh

@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/bin/bash
 # Copyright 2016 The Rust Project Developers. See the COPYRIGHT
 # file at the top-level directory of this distribution and at
 # http://rust-lang.org/COPYRIGHT.
@@ -20,6 +20,9 @@ if [ "$NO_CHANGE_USER" = "" ]; then
   fi
 fi
 
+ci_dir=`cd $(dirname $0) && pwd`
+source "$ci_dir/shared.sh"
+
 RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --enable-sccache"
 RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --enable-quiet-tests"
 RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --disable-manage-submodules"
@@ -54,16 +57,17 @@ else
   fi
 fi
 
-set -ex
-
 $SRC/configure $RUST_CONFIGURE_ARGS
+retry make prepare
 
 if [ "$TRAVIS_OS_NAME" = "osx" ]; then
     ncpus=$(sysctl -n hw.ncpu)
 else
     ncpus=$(grep processor /proc/cpuinfo | wc -l)
 fi
 
+set -x
+
 if [ ! -z "$SCRIPT" ]; then
   sh -x -c "$SCRIPT"
 else

src/ci/shared.sh

@@ -0,0 +1,29 @@
+#!/bin/bash
+# Copyright 2016 The Rust Project Developers. See the COPYRIGHT
+# file at the top-level directory of this distribution and at
+# http://rust-lang.org/COPYRIGHT.
+#
+# Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+# http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+# <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+# option. This file may not be copied, modified, or distributed
+# except according to those terms.
+
+# See http://unix.stackexchange.com/questions/82598
+function retry {
+  local n=1
+  local max=5
+  local delay=15
+  while true; do
+    echo "Attempting:" "$@"
+    "$@" && break || {
+      if [[ $n -lt $max ]]; then
+        ((n++))
+        echo "Command failed. Attempt $n/$max:"
+      else
+        echo "The command has failed after $n attempts."
+        exit 1
+      fi
+    }
+  done
+}