Auto merge of rust-lang#62690 - alexcrichton:azure-update, r=pietroalbini

azure: Prepare configuration for 4-core machines

This commit updates some of our assorted Azure/CI configuration to
prepare for some 4-core machines coming online. We're still in the
process of performance testing them to get final numbers, but some
changes are worth landing ahead of this. The updates here are:

* Use `C:/` instead of `D:/` for submodule checkout since it should have
  plenty of space and the 4-core machines won't have `D:/`

* Update `lzma-sys` to 0.1.14 which has support for VS2019, where 0.1.10
  doesn't.

* Update `src/ci/docker/run.sh` to work when it itself is running inside
  of a docker container (see the comment in the file for more info)

* Print step timings on the `try` branch in addition to the `auto`
  branch in. The logs there should be seen by similarly many humans (not
  many) and can be useful for performance analysis after a `try` build
  runs.

* Install the WIX and InnoSetup tools manually on Windows instead of
  relying on pre-installed copies on the VM. This gives us more control
  over what's being used on the Azure cloud right now (we control the
  version) and in the 4-core machines these won't be pre-installed. Note
  that on AppVeyor we actually already were installing InnoSetup, we
  just didn't carry that over on Azure!

Author: bors

.azure-pipelines/steps/install-windows-build-deps.yml

@@ -1,4 +1,29 @@
 steps:
+# We use the WIX toolset to create combined installers for Windows, and these
+# binaries are downloaded from
+# https://github.com/wixtoolset/wix3 originally
+- bash: |
+    set -e
+    curl -O https://rust-lang-ci2.s3-us-west-1.amazonaws.com/rust-ci-mirror/wix311-binaries.zip
+    echo "##vso[task.setvariable variable=WIX]`pwd`/wix"
+    mkdir -p wix/bin
+    cd wix/bin
+    7z x ../../wix311-binaries.zip
+  displayName: Install wix
+  condition: and(succeeded(), eq(variables['Agent.OS'], 'Windows_NT'))
+
+# We use InnoSetup and its `iscc` program to also create combined installers.
+# Honestly at this point WIX above and `iscc` are just holdovers from
+# oh-so-long-ago and are required for creating installers on Windows. I think
+# one is MSI installers and one is EXE, but they're not used so frequently at
+# this point anyway so perhaps it's a wash!
+- script: |
+    powershell -Command "$ProgressPreference = 'SilentlyContinue'; iwr -outf is-install.exe https://rust-lang-ci2.s3.amazonaws.com/rust-ci-mirror/2017-08-22-is.exe"
+    is-install.exe /VERYSILENT /SUPPRESSMSGBOXES /NORESTART /SP-
+    echo ##vso[task.prependpath]C:\Program Files (x86)\Inno Setup 5
+  displayName: Install InnoSetup
+  condition: and(succeeded(), eq(variables['Agent.OS'], 'Windows_NT'))
+
 # We've had issues with the default drive in use running out of space during a
 # build, and it looks like the `C:` drive has more space than the default `D:`
 # drive. We should probably confirm this with the azure pipelines team at some

.azure-pipelines/steps/run.yml

@@ -111,8 +111,8 @@ steps:
   condition: and(succeeded(), not(variables.SKIP_JOB), ne(variables['Agent.OS'], 'Windows_NT'))
   displayName: Check out submodules (Unix)
 - script: |
-    if not exist D:\cache\rustsrc\NUL mkdir D:\cache\rustsrc
-    sh src/ci/init_repo.sh . /d/cache/rustsrc
+    if not exist C:\cache\rustsrc\NUL mkdir C:\cache\rustsrc
+    sh src/ci/init_repo.sh . /c/cache/rustsrc
   condition: and(succeeded(), not(variables.SKIP_JOB), eq(variables['Agent.OS'], 'Windows_NT'))
   displayName: Check out submodules (Windows)
 

Cargo.lock

@@ -1547,11 +1547,10 @@ dependencies = [
 
 [[package]]
 name = "lzma-sys"
-version = "0.1.10"
+version = "0.1.14"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 dependencies = [
  "cc 1.0.35 (registry+https://github.com/rust-lang/crates.io-index)",
- "filetime 0.2.4 (registry+https://github.com/rust-lang/crates.io-index)",
  "libc 0.2.54 (registry+https://github.com/rust-lang/crates.io-index)",
  "pkg-config 0.3.14 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
@@ -4282,7 +4281,7 @@ name = "xz2"
 version = "0.1.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 dependencies = [
- "lzma-sys 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)",
+ "lzma-sys 0.1.14 (registry+https://github.com/rust-lang/crates.io-index)",
 ]
 
 [[package]]
@@ -4437,7 +4436,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 "checksum log_settings 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "19af41f0565d7c19b2058153ad0b42d4d5ce89ec4dbf06ed6741114a8b63e7cd"
 "checksum lsp-codec 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "169d737ad89cf8ddd82d1804d9122f54568c49377665157277cc90d747b1d31a"
 "checksum lsp-types 0.57.2 (registry+https://github.com/rust-lang/crates.io-index)" = "b62b77309737b1e262b3bbf37ff8faa740562c633b14702afe9be85dbcb6f88a"
-"checksum lzma-sys 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)" = "d1eaa027402541975218bb0eec67d6b0412f6233af96e0d096d31dbdfd22e614"
+"checksum lzma-sys 0.1.14 (registry+https://github.com/rust-lang/crates.io-index)" = "16b5c59c57cc4d39e7999f50431aa312ea78af7c93b23fbb0c3567bd672e7f35"
 "checksum mac 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "c41e0c4fef86961ac6d6f8a82609f55f31b05e4fce149ac5710e439df7619ba4"
 "checksum macro-utils 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "f2c4deaccc2ead6a28c16c0ba82f07d52b6475397415ce40876e559b0b0ea510"
 "checksum maplit 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "08cbb6b4fef96b6d77bfc40ec491b1690c779e77b05cd9f07f787ed376fd4c43"

src/ci/docker/run.sh

@@ -125,26 +125,60 @@ fi
 # goes ahead and sets it for all builders.
 args="$args --privileged"
 
-exec docker \
+# Things get a little weird if this script is already running in a docker
+# container. If we're already in a docker container then we assume it's set up
+# to do docker-in-docker where we have access to a working `docker` command.
+#
+# If this is the case (we check via the presence of `/.dockerenv`)
+# then we can't actually use the `--volume` argument. Typically we use
+# `--volume` to efficiently share the build and source directory between this
+# script and the container we're about to spawn. If we're inside docker already
+# though the `--volume` argument maps the *host's* folder to the container we're
+# about to spawn, when in fact we want the folder in this container itself. To
+# work around this we use a recipe cribbed from
+# https://circleci.com/docs/2.0/building-docker-images/#mounting-folders to
+# create a temporary container with a volume. We then copy the entire source
+# directory into this container, and then use that copy in the container we're
+# about to spawn. Finally after the build finishes we re-extract the object
+# directory.
+#
+# Note that none of this is necessary if we're *not* in a docker-in-docker
+# scenario. If this script is run on a bare metal host then we share a bunch of
+# data directories to share as much data as possible. Note that we also use
+# `LOCAL_USER_ID` (recognized in `src/ci/run.sh`) to ensure that files are all
+# read/written as the same user as the bare-metal user.
+if [ -f /.dockerenv ]; then
+  docker create -v /checkout --name checkout alpine:3.4 /bin/true
+  docker cp . checkout:/checkout
+  args="$args --volumes-from checkout"
+else
+  args="$args --volume $root_dir:/checkout:ro"
+  args="$args --volume $objdir:/checkout/obj"
+  args="$args --volume $HOME/.cargo:/cargo"
+  args="$args --volume $HOME/rustsrc:$HOME/rustsrc"
+  args="$args --env LOCAL_USER_ID=`id -u`"
+fi
+
+docker \
   run \
-  --volume "$root_dir:/checkout:ro" \
-  --volume "$objdir:/checkout/obj" \
   --workdir /checkout/obj \
   --env SRC=/checkout \
   $args \
   --env CARGO_HOME=/cargo \
   --env DEPLOY \
   --env DEPLOY_ALT \
-  --env LOCAL_USER_ID=`id -u` \
   --env CI \
   --env TF_BUILD \
   --env BUILD_SOURCEBRANCHNAME \
   --env TOOLSTATE_REPO_ACCESS_TOKEN \
   --env TOOLSTATE_REPO \
   --env CI_JOB_NAME="${CI_JOB_NAME-$IMAGE}" \
-  --volume "$HOME/.cargo:/cargo" \
-  --volume "$HOME/rustsrc:$HOME/rustsrc" \
   --init \
   --rm \
   rust-ci \
   /checkout/src/ci/run.sh
+
+if [ -f /.dockerenv ]; then
+  rm -rf $objdir
+  docker cp checkout:/checkout/obj $objdir
+fi

src/ci/run.sh

@@ -25,7 +25,7 @@ source "$ci_dir/shared.sh"
 
 branch_name=$(getCIBranch)
 
-if [ ! isCI ] || [ "$branch_name" = "auto" ]; then
+if [ ! isCI ] || [ "$branch_name" = "auto" ] || [ "$branch_name" = "try" ]; then
     RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --set build.print-step-timings --enable-verbose-tests"
 fi