build: improve script and travis config

tesuji · tesuji · commit b869eeb2a451 · 2019-10-22T10:14:39.000+07:00
* fix a diff failure on windows See https://travis-ci.com/rust-lang/rust-clippy/jobs/245971932#L1625 for an example. * use cmp instead of diff > /dev/null * clone single branch instead of clone then checking out * do not decrypt key if have no diff change
diff --git a/.github/deploy.sh b/.github/deploy.sh
@@ -13,11 +13,7 @@ SSH_REPO=${REPO/https:\/\/github.com\//git@github.com:}
 SHA=$(git rev-parse --verify HEAD)
 
 # Clone the existing gh-pages for this repo into out/
-(
-    git clone "$REPO" out
-    cd out
-    git checkout $TARGET_BRANCH
-)
+git clone --quiet --single-branch --branch "$TARGET_BRANCH" "$REPO" out
 
 echo "Removing the current docs for master"
 rm -rf out/master/ || exit 0
@@ -27,35 +23,44 @@ mkdir out/master/
 cp util/gh-pages/index.html out/master
 python ./util/export.py out/master/lints.json
 
-if [ -n "$TRAVIS_TAG" ]; then
+if [[ -n "$TRAVIS_TAG" ]]; then
     echo "Save the doc for the current tag ($TRAVIS_TAG) and point current/ to it"
     cp -r out/master "out/$TRAVIS_TAG"
     rm -f out/current
     ln -s "$TRAVIS_TAG" out/current
 fi
 
 # Generate version index that is shown as root index page
-(
-    cp util/gh-pages/versions.html out/index.html
-
-    cd out
-    python -c '\
-        import os, json;\
-        print json.dumps([\
-            dir for dir in os.listdir(".")\
-            if not dir.startswith(".") and os.path.isdir(dir)\
-        ])' > versions.json
-)
+cp util/gh-pages/versions.html out/index.html
+pushd out
+
+cat <<-EOF | python - > versions.json
+import os, json
+print json.dumps([
+    dir for dir in os.listdir(".") if not dir.startswith(".") and os.path.isdir(dir)
+])
+EOF
+popd
 
 # Pull requests and commits to other branches shouldn't try to deploy, just build to verify
-if [ "$TRAVIS_PULL_REQUEST" != "false" ] || [ "$TRAVIS_BRANCH" != "$SOURCE_BRANCH" ]; then
+if [[ "$TRAVIS_PULL_REQUEST" != "false" ]] || [[ "$TRAVIS_BRANCH" != "$SOURCE_BRANCH" ]]; then
     # Tags should deploy
-    if [ -z "$TRAVIS_TAG" ]; then
+    if [[ -z "$TRAVIS_TAG" ]]; then
         echo "Generated, won't push"
         exit 0
     fi
 fi
 
+# Now let's go have some fun with the cloned repo
+cd out
+git config user.name "Travis CI"
+git config user.email "travis@ci.invalid"
+
+if git diff --exit-code --quiet; then
+    echo "No changes to the output on this push; exiting."
+    exit 0
+fi
+
 # Get the deploy key by using Travis's stored variables to decrypt deploy_key.enc
 ENCRYPTION_LABEL=e3a2d77100be
 ENCRYPTED_KEY_VAR="encrypted_${ENCRYPTION_LABEL}_key"
@@ -64,19 +69,9 @@ ENCRYPTED_KEY=${!ENCRYPTED_KEY_VAR}
 ENCRYPTED_IV=${!ENCRYPTED_IV_VAR}
 openssl aes-256-cbc -K "$ENCRYPTED_KEY" -iv "$ENCRYPTED_IV" -in .github/deploy_key.enc -out .github/deploy_key -d
 chmod 600 .github/deploy_key
-eval $(ssh-agent -s)
+eval "$(ssh-agent -s)"
 ssh-add .github/deploy_key
 
-# Now let's go have some fun with the cloned repo
-cd out
-git config user.name "Travis CI"
-git config user.email "travis@ci.invalid"
-
-if [ -z "$(git diff --exit-code)" ]; then
-    echo "No changes to the output on this push; exiting."
-    exit 0
-fi
-
 git add .
 git commit -m "Automatic deploy to GitHub Pages: ${SHA}"
 
diff --git a/.travis.yml b/.travis.yml
@@ -1,11 +1,6 @@
 dist: xenial
 language: bash
 
-os:
-  - linux
-  - osx
-  - windows
-
 branches:
   # Don't build these branches
   except:
@@ -34,13 +29,12 @@ install:
       if ! rustup component add rustfmt; then
         cargo install -Z install-upgrade --git https://github.com/rust-lang/rustfmt --bin rustfmt
       fi
-      if [ "$TRAVIS_OS_NAME" == "linux" ]; then
+      if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then
         . $HOME/.nvm/nvm.sh
         nvm install stable
         nvm use stable
         npm install remark-cli remark-lint
-      fi
-      if [ "$TRAVIS_OS_NAME" == "windows" ]; then
+      elif [[ "$TRAVIS_OS_NAME" == "windows" ]]; then
         choco install windows-sdk-10.1
       fi
     fi
@@ -96,50 +90,45 @@ matrix:
   allow_failures:
     - os: windows
       env: CARGO_INCREMENTAL=0 BASE_TESTS=true OS_WINDOWS=true
-  # prevent these jobs with default env vars
-  exclude:
-    - os: linux
-    - os: osx
-    - os: windows
 
 before_script:
   - |
-    if [ "$TRAVIS_BRANCH" == "auto" ] || [ "$TRAVIS_BRANCH" == "try" ]; then
-      pr=$(echo $TRAVIS_COMMIT_MESSAGE | grep -o "#[0-9]*" | head -1 | sed 's/^#//g')
-      output=$(curl -H "Authorization: token $GITHUB_API_TOKEN" -s "https://api.github.com/repos/rust-lang/rust-clippy/pulls/$pr" | \
+    if [[ "$TRAVIS_BRANCH" == "auto" ]] || [[ "$TRAVIS_BRANCH" == "try" ]]; then
+      PR=$(echo "$TRAVIS_COMMIT_MESSAGE" | grep -o "#[0-9]*" | head -1 | sed 's/^#//g')
+      output=$(curl -H "Authorization: token $GITHUB_API_TOKEN" -s "https://api.github.com/repos/rust-lang/rust-clippy/pulls/$PR" | \
         python -c "import sys, json; print(json.load(sys.stdin)['body'])" | \
         grep "^changelog: " | \
         sed "s/changelog: //g")
-      if [ -z "$output" ]; then
+      if [[ -z "$output" ]]; then
         echo "ERROR: PR body must contain 'changelog: ...'"
         exit 1
-      elif [ "$output" = "none" ]; then
+      elif [[ "$output" = "none" ]]; then
         echo "WARNING: changelog is 'none'"
       fi
     fi
   - |
     rm rust-toolchain
     ./setup-toolchain.sh
   - |
-    if [ "$TRAVIS_OS_NAME" == "windows" ]; then
+    if [[ "$TRAVIS_OS_NAME" == "windows" ]]; then
       export PATH=$PATH:$(rustc --print sysroot)/bin
     else
       export LD_LIBRARY_PATH=$(rustc --print sysroot)/lib
     fi
 
 script:
   - |
-    if [ -z ${INTEGRATION} ]; then
-      travis_wait 30 ./ci/base-tests.sh && sleep 5
-    else
+    if [[ -n ${INTEGRATION} ]]; then
       ./ci/integration-tests.sh && sleep 5
+    else
+      travis_wait 30 ./ci/base-tests.sh && sleep 5
     fi
 
 after_success:
   - |
-    if [ "$TRAVIS_OS_NAME" == "linux" ]; then
+    if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then
       set -e
-      if [ -z ${INTEGRATION} ]; then
+      if [[ -z ${INTEGRATION} ]]; then
         ./.github/deploy.sh
       else
         echo "Not deploying, because we're in an integration test run"
diff --git a/ci/base-tests.sh b/ci/base-tests.sh
@@ -1,16 +1,18 @@
+#!/usr/bin/env bash
 set -ex
 
 echo "Running clippy base tests"
 
 PATH=$PATH:./node_modules/.bin
-if [ "$TRAVIS_OS_NAME" == "linux" ]; then
-  remark -f *.md -f doc/*.md > /dev/null
+if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then
+  remark -f ./*.md -f doc/*.md > /dev/null
 fi
 # build clippy in debug mode and run tests
 cargo build --features "debugging deny-warnings"
 cargo test --features "debugging deny-warnings"
 # for faster build, share target dir between subcrates
-export CARGO_TARGET_DIR=`pwd`/target/
+CARGO_TARGET_DIR=$(pwd)/target/
+export CARGO_TARGET_DIR
 (cd clippy_lints && cargo test)
 (cd rustc_tools_util && cargo test)
 (cd clippy_dev && cargo test)
@@ -29,26 +31,27 @@ export CARGO_TARGET_DIR=`pwd`/target/
 (
   # Check sysroot handling
   sysroot=$(./target/debug/clippy-driver --print sysroot)
-  test $sysroot = $(rustc --print sysroot)
+  test "$sysroot" = "$(rustc --print sysroot)"
 
-  if [ -z $OS_WINDOWS ]; then
+  if [[ -z "$OS_WINDOWS" ]]; then
     desired_sysroot=/tmp
   else
     desired_sysroot=C:/tmp
   fi
   sysroot=$(./target/debug/clippy-driver --sysroot $desired_sysroot --print sysroot)
-  test $sysroot = $desired_sysroot
+  test "$sysroot" = $desired_sysroot
 
   sysroot=$(SYSROOT=$desired_sysroot ./target/debug/clippy-driver --print sysroot)
-  test $sysroot = $desired_sysroot
+  test "$sysroot" = $desired_sysroot
 
   # Make sure this isn't set - clippy-driver should cope without it
   unset CARGO_MANIFEST_DIR
 
   # Run a lint and make sure it produces the expected output. It's also expected to exit with code 1
-  # XXX How to match the clippy invocation in compile-test.rs?
+  # FIXME: How to match the clippy invocation in compile-test.rs?
   ! ./target/debug/clippy-driver -Dwarnings -Aunused -Zui-testing --emit metadata --crate-type bin tests/ui/cstring.rs 2> cstring.stderr
-  diff <(sed -e 's,tests/ui,$DIR,' -e '/= help/d' cstring.stderr) tests/ui/cstring.stderr
+  sed -e 's,tests/ui,$DIR,' -e '/= help/d' cstring.stderr > normalized.stderr
+  diff normalized.stderr tests/ui/cstring.stderr
 
   # TODO: CLIPPY_CONF_DIR / CARGO_MANIFEST_DIR
 )
diff --git a/tests/ui-toml/update-all-references.sh b/tests/ui-toml/update-all-references.sh
@@ -13,6 +13,6 @@ if [[ "$1" == "--help" || "$1" == "-h" ]]; then
 fi
 
 BUILD_DIR=$PWD/target/debug/test_build_base
-MY_DIR=$(dirname $0)
-cd $MY_DIR
-find . -name '*.rs' | xargs ./update-references.sh $BUILD_DIR
+MY_DIR=$(dirname "$0")
+cd "$MY_DIR" || exit
+find . -name '*.rs' -exec ./update-references.sh "$BUILD_DIR" {} +
diff --git a/tests/ui-toml/update-references.sh b/tests/ui-toml/update-references.sh
@@ -16,7 +16,7 @@ if [[ "$1" == "--help" || "$1" == "-h" || "$1" == "" || "$2" == "" ]]; then
     echo "   $0 ../../../build/x86_64-apple-darwin/test/ui *.rs */*.rs"
 fi
 
-MYDIR=$(dirname $0)
+MYDIR=$(dirname "$0")
 
 BUILD_DIR="$1"
 shift
@@ -25,16 +25,14 @@ while [[ "$1" != "" ]]; do
     STDERR_NAME="${1/%.rs/.stderr}"
     STDOUT_NAME="${1/%.rs/.stdout}"
     shift
-    if [ -f $BUILD_DIR/$STDOUT_NAME ] && \
-           ! (diff $BUILD_DIR/$STDOUT_NAME $MYDIR/$STDOUT_NAME >& /dev/null); then
-        echo updating $MYDIR/$STDOUT_NAME
-        cp $BUILD_DIR/$STDOUT_NAME $MYDIR/$STDOUT_NAME
+    if [[ -f "$BUILD_DIR"/"$STDOUT_NAME" ]] && \
+           ! (cmp -s -- "$BUILD_DIR"/"$STDOUT_NAME" "$MYDIR"/"$STDOUT_NAME"); then
+        echo updating "$MYDIR"/"$STDOUT_NAME"
+        cp "$BUILD_DIR"/"$STDOUT_NAME" "$MYDIR"/"$STDOUT_NAME"
     fi
-    if [ -f $BUILD_DIR/$STDERR_NAME ] && \
-           ! (diff $BUILD_DIR/$STDERR_NAME $MYDIR/$STDERR_NAME >& /dev/null); then
-        echo updating $MYDIR/$STDERR_NAME
-        cp $BUILD_DIR/$STDERR_NAME $MYDIR/$STDERR_NAME
+    if [[ -f "$BUILD_DIR"/"$STDERR_NAME" ]] && \
+           ! (cmp -s -- "$BUILD_DIR"/"$STDERR_NAME" "$MYDIR"/"$STDERR_NAME"); then
+        echo updating "$MYDIR"/"$STDERR_NAME"
+        cp "$BUILD_DIR"/"$STDERR_NAME" "$MYDIR"/"$STDERR_NAME"
     fi
 done
-
-
diff --git a/tests/ui/update-all-references.sh b/tests/ui/update-all-references.sh
@@ -13,6 +13,6 @@ if [[ "$1" == "--help" || "$1" == "-h" ]]; then
 fi
 
 BUILD_DIR=$PWD/target/debug/test_build_base
-MY_DIR=$(dirname $0)
-cd $MY_DIR
-find . -name '*.rs' | xargs ./update-references.sh $BUILD_DIR
+MY_DIR=$(dirname "$0")
+cd "$MY_DIR" || exit
+find . -name '*.rs' -exec ./update-references.sh "$BUILD_DIR" {} +
diff --git a/tests/ui/update-references.sh b/tests/ui/update-references.sh
@@ -16,7 +16,7 @@ if [[ "$1" == "--help" || "$1" == "-h" || "$1" == "" || "$2" == "" ]]; then
     echo "   $0 ../../../build/x86_64-apple-darwin/test/ui *.rs */*.rs"
 fi
 
-MYDIR=$(dirname $0)
+MYDIR=$(dirname "$0")
 
 BUILD_DIR="$1"
 shift
@@ -26,21 +26,19 @@ while [[ "$1" != "" ]]; do
     STDOUT_NAME="${1/%.rs/.stdout}"
     FIXED_NAME="${1/%.rs/.fixed}"
     shift
-    if [ -f $BUILD_DIR/$STDOUT_NAME ] && \
-           ! (diff $BUILD_DIR/$STDOUT_NAME $MYDIR/$STDOUT_NAME >& /dev/null); then
-        echo updating $MYDIR/$STDOUT_NAME
-        cp $BUILD_DIR/$STDOUT_NAME $MYDIR/$STDOUT_NAME
+    if [[ -f "$BUILD_DIR"/"$STDOUT_NAME" ]] && \
+           ! (cmp -s -- "$BUILD_DIR"/"$STDOUT_NAME" "$MYDIR"/"$STDOUT_NAME"); then
+        echo updating "$MYDIR"/"$STDOUT_NAME"
+        cp "$BUILD_DIR"/"$STDOUT_NAME" "$MYDIR"/"$STDOUT_NAME"
     fi
-    if [ -f $BUILD_DIR/$STDERR_NAME ] && \
-           ! (diff $BUILD_DIR/$STDERR_NAME $MYDIR/$STDERR_NAME >& /dev/null); then
-        echo updating $MYDIR/$STDERR_NAME
-        cp $BUILD_DIR/$STDERR_NAME $MYDIR/$STDERR_NAME
+    if [[ -f "$BUILD_DIR"/"$STDERR_NAME" ]] && \
+           ! (cmp -s -- "$BUILD_DIR"/"$STDERR_NAME" "$MYDIR"/"$STDERR_NAME"); then
+        echo updating "$MYDIR"/"$STDERR_NAME"
+        cp "$BUILD_DIR"/"$STDERR_NAME" "$MYDIR"/"$STDERR_NAME"
     fi
-    if [ -f $BUILD_DIR/$FIXED_NAME ] && \
-           ! (diff $BUILD_DIR/$FIXED_NAME $MYDIR/$FIXED_NAME >& /dev/null); then
-        echo updating $MYDIR/$FIXED_NAME
-        cp $BUILD_DIR/$FIXED_NAME $MYDIR/$FIXED_NAME
+    if [[ -f "$BUILD_DIR"/"$FIXED_NAME" ]] && \
+           ! (cmp -s -- "$BUILD_DIR"/"$FIXED_NAME" "$MYDIR"/"$FIXED_NAME"); then
+        echo updating "$MYDIR"/"$FIXED_NAME"
+        cp "$BUILD_DIR"/"$FIXED_NAME" "$MYDIR"/"$FIXED_NAME"
     fi
 done
-
-
diff --git a/util/fetch_prs_between.sh b/util/fetch_prs_between.sh
@@ -1,4 +1,4 @@
-#!/bin/sh
+#!/bin/bash
 
 # Fetches the merge commits between two git commits and prints the PR URL
 # together with the full commit message
@@ -12,14 +12,15 @@ last=$2
 IFS='
 '
 for pr in $(git log --oneline --grep "Merge #" --grep "Merge pull request" --grep "Auto merge of" --grep "Rollup merge of" "$first...$last" | sort -rn | uniq); do
-  id=$(echo $pr | rg -o '#[0-9]{3,5}' | cut -c 2-)
-  commit=$(echo $pr | cut -d' ' -f 1)
-  message=$(git --no-pager show --pretty=medium $commit)
-  if [ ! -z $(echo "$message" | rg "^[\s]{4}changelog: [nN]one\.*$") ]; then
+  id=$(echo "$pr" | rg -o '#[0-9]{3,5}' | cut -c 2-)
+  commit=$(echo "$pr" | cut -d' ' -f 1)
+  message=$(git --no-pager show --pretty=medium "$commit")
+  if [[ -n $(echo "$message" | rg "^[\s]{4}changelog: [nN]one\.*$") ]]; then
     continue
   fi
 
   echo "URL: https://github.com/rust-lang/rust-clippy/pull/$id"
   echo "$message"
-  echo "---------------------------------------------------------\n"
+  echo "---------------------------------------------------------"
+  echo
 done
