ci: extract uploading artifacts into a script

Author: pietroalbini

src/ci/azure-pipelines/steps/run.yml

@@ -201,37 +201,21 @@ steps:
   condition: and(succeeded(), not(variables.SKIP_JOB))
   displayName: Run build
 
-# If we're a deploy builder, use the `aws` command to publish everything to our
-# bucket.
-- bash: |
-    set -e
-    source src/ci/shared.sh
-    if [ "$AGENT_OS" = "Linux" ]; then
-        rm -rf obj/build/dist/doc
-        upload_dir=obj/build/dist
-    else
-        rm -rf build/dist/doc
-        upload_dir=build/dist
-    fi
-    ls -la $upload_dir
-    deploy_dir=rustc-builds
-    if [ "$DEPLOY_ALT" == "1" ]; then
-        deploy_dir=rustc-builds-alt
-    fi
-    retry aws s3 cp --no-progress --recursive --acl public-read ./$upload_dir s3://$DEPLOY_BUCKET/$deploy_dir/$BUILD_SOURCEVERSION
+- bash: src/ci/scripts/upload-artifacts.sh
   env:
     AWS_ACCESS_KEY_ID: $(UPLOAD_AWS_ACCESS_KEY_ID)
     AWS_SECRET_ACCESS_KEY: $(UPLOAD_AWS_SECRET_ACCESS_KEY)
-  condition: and(succeeded(), not(variables.SKIP_JOB), or(eq(variables.DEPLOY, '1'), eq(variables.DEPLOY_ALT, '1')))
   displayName: Upload artifacts
-
-# Upload CPU usage statistics that we've been gathering this whole time. Always
-# execute this step in case we want to inspect failed builds, but don't let
-# errors here ever fail the build since this is just informational.
-- bash: aws s3 cp --acl public-read cpu-usage.csv s3://$DEPLOY_BUCKET/rustc-builds/$BUILD_SOURCEVERSION/cpu-$CI_JOB_NAME.csv
-  env:
-    AWS_ACCESS_KEY_ID: $(UPLOAD_AWS_ACCESS_KEY_ID)
-    AWS_SECRET_ACCESS_KEY: $(UPLOAD_AWS_SECRET_ACCESS_KEY)
-  condition: variables['UPLOAD_AWS_SECRET_ACCESS_KEY']
-  continueOnError: true
-  displayName: Upload CPU usage statistics
+  # Adding a condition on DEPLOY=1 or DEPLOY_ALT=1 is not needed as all deploy
+  # builders *should* have the AWS credentials available. Still, explicitly
+  # adding the condition is helpful as this way CI will not silently skip
+  # deploying artifacts from a dist builder if the variables are misconfigured,
+  # erroring about invalid credentials instead.
+  condition: |
+    and(
+      succeeded(), not(variables.SKIP_JOB),
+      or(
+        variables.UPLOAD_AWS_SECRET_ACCESS_KEY,
+        eq(variables.DEPLOY, '1'), eq(variables.DEPLOY_ALT, '1')
+      )
+    )

src/ci/scripts/upload-artifacts.sh

@@ -0,0 +1,36 @@
+#!/bin/bash
+# Upload all the artifacts to our S3 bucket. All the files inside ${upload_dir}
+# will be uploaded to the deploy bucket and eventually signed and released in
+# static.rust-lang.org.
+
+set -euo pipefail
+IFS=$'\n\t'
+
+source "$(cd "$(dirname "$0")" && pwd)/../shared.sh"
+
+upload_dir="$(mktemp -d)"
+
+# Release tarballs produced by a dist builder.
+if [[ "${DEPLOY-0}" = "1" ]] || [[ "${DEPLOY_ALT-0}" = "1" ]]; then
+    dist_dir=build/dist
+    if isLinux; then
+        dist_dir=obj/build/dist
+    fi
+    rm -rf "${dist_dir}/doc"
+    cp -r "${dist_dir}"/* "${upload_dir}"
+fi
+
+# CPU usage statistics.
+cp cpu-usage.csv "${upload_dir}/cpu-${CI_JOB_NAME}.csv"
+
+echo "Files that will be uploaded:"
+ls -lah "${upload_dir}"
+echo
+
+deploy_dir="rustc-builds"
+if [[ "${DEPLOY_ALT-0}" = "1" ]]; then
+    deploy_dir="rustc-builds-alt"
+fi
+deploy_url="s3://${DEPLOY_BUCKET}/${deploy_dir}/$(ciCommit)"
+
+retry aws s3 cp --no-progress --recursive --acl public-read "${upload_dir}" "${deploy_url}"

src/ci/shared.sh

@@ -46,6 +46,10 @@ function getCIBranch {
   echo "$BUILD_SOURCEBRANCHNAME"
 }
 
+function ciCommit {
+  echo "${BUILD_SOURCEVERSION}"
+}
+
 function ciCommandAddPath {
     if [[ $# -ne 1 ]]; then
         echo "usage: $0 <path>"