Prevent Windows GHA CI disk exhausation (#468)

DellaBitta · web-flow · commit 7ae9569eb9aa · 2021-06-15T17:02:56.000-04:00
Windows BoringSSL builds are now larger due to #454. The checkout & build partition **(D:)** hits capacity when attempting to install the Google Cloud SDK, which is what we use to upload the final integraion_test build artifacts to storage. This change moves the arhives destined for GCS to the **C:** drive, then installs the Cloud SDK after they're off of **D:**. The migration of the integration tests only occurs on Windows, for now. Some notes: - We can’t set the build_testapps.py’s --output_directory flag to directly output to the **C:** directory because python’s relpath operations fail when working across different drives. - The **github.workspace** env var can be changed to override check-out & installation paths, but the [GHA team notes potential unknown side effects](https://github.community/t/is-it-possible-to-change-github-workspace/136094/3). Therefore I didn’t test this. - We couldn't use **mv** the results to **C:** as Firestore's build results include symlinks. Using **tar** to ferry the files instead.
diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml
@@ -253,20 +253,20 @@ jobs:
         with:
           ref: ${{needs.prepare_matrix.outputs.github_ref}}
           submodules: true
-      - name: Set env vars (ubuntu)
+      - name: Set env vars (Linux)
         if: startsWith(matrix.os, 'ubuntu')
         run: echo "VCPKG_TRIPLET=x64-linux" >> $GITHUB_ENV
-      - name: Set env vars (macos)
+      - name: Set env vars (macOS)
         if: startsWith(matrix.os, 'macos')
         run: echo "VCPKG_TRIPLET=x64-osx" >> $GITHUB_ENV
-      - name: Set env vars (windows)
+      - name: Set env vars (Windows)
         shell: bash
         if: startsWith(matrix.os, 'windows')
         run: echo "VCPKG_TRIPLET=x64-windows-static" >> $GITHUB_ENV
-      - name: Set env vars(all)
+      - name: Set env vars (all)
         shell: bash
         run: echo "VCPKG_RESPONSE_FILE=external/vcpkg_${{ env.VCPKG_TRIPLET }}_response_file.txt" >> $GITHUB_ENV
-      - name: Add msbuild to PATH (windows)
+      - name: Add msbuild to PATH (Windows)
         if: startsWith(matrix.os, 'windows')
         uses: microsoft/setup-msbuild@v1.0.2
       - name: Cache vcpkg C++ dependencies
@@ -293,12 +293,12 @@ jobs:
         run: |
           pip install -r scripts/gha/requirements.txt
           python scripts/gha/restore_secrets.py --passphrase "${{ secrets.TEST_SECRET }}"
-      - name: Install OpenSSL (windows)
+      - name: Install OpenSSL (Windows)
         if: matrix.ssl_variant == 'openssl' &&
             startsWith(matrix.os, 'windows')
         run: |
           choco install openssl -r
-      - name: Install OpenSSL (MacOS)
+      - name: Install OpenSSL (macOS)
         if: matrix.ssl_variant == 'openssl' &&
             startsWith(matrix.os, 'macos')
         run: |
@@ -341,7 +341,7 @@ jobs:
             --artifact_name "desktop-${{ matrix.os }}-${{ matrix.ssl_variant }}" \
             --noadd_timestamp \
             --short_output_paths \
-            ${additional_flags[*]} 
+            ${additional_flags[*]}
       - name: Upload Desktop integration tests artifact
         uses: actions/upload-artifact@v2.2.2
         if: ${{ !cancelled() }}
@@ -356,6 +356,39 @@ jobs:
           name: log-artifact
           path: build-results-desktop-${{ matrix.os }}-${{ matrix.ssl_variant }}*
           retention-days: ${{ env.artifactRetentionDays }}
+      - name: Cleanup Local Copies of Uploaded Artifacts
+        shell: bash
+        run: |
+          rm -rf testapps-desktop-${{ matrix.os }}-${{ matrix.ssl_variant }}
+      - name: Configure GCS Upload Directory (Linux, macOS)
+        if: (startsWith(matrix.os, 'ubuntu') || startsWith(matrix.os, 'macos')) && !cancelled()
+        shell: bash
+        run: echo "GCS_UPLOAD_DIR=${{ github.workspace }}" >> $GITHUB_ENV
+      - name: Configure GCS Upload Directory (Windows)
+        # For Windows move the uploads to C: to create enough space for the
+        # Cloud SDK installation on D:.
+        if: startsWith(matrix.os, 'windows') && !cancelled()
+        shell: bash
+        run: echo "GCS_UPLOAD_DIR=/c/fpl-upload" >> $GITHUB_ENV
+      - name: Move GCS Artifacts to GCS Upload Dir
+        # Copy the artifacts and also the upload scripts since python's relpath
+        # produces errors when called across two different drives. Use tar to
+        # move the files as it neatly deepcopies firestore's symlinks.
+        if:  ${{ (env.GCS_UPLOAD_DIR != github.workspace) && !cancelled() }}
+        shell: bash
+        run: |
+          set -x
+          mkdir -p ${{env.GCS_UPLOAD_DIR}}/scripts/gha/integration_testing
+          mkdir -p ${{env.GCS_UPLOAD_DIR}}/scripts/gha-encrypted
+          # The ta directory contains the integration testapps to archive.
+          tar -chf ${{env.GCS_UPLOAD_DIR}}/ta.tar ta
+          rm -rf ta
+          tar xf ${{env.GCS_UPLOAD_DIR}}/ta.tar -C ${{env.GCS_UPLOAD_DIR}}
+          rm ${{env.GCS_UPLOAD_DIR}}/ta.tar
+          cp scripts/gha/gcs_uploader.py  ${{env.GCS_UPLOAD_DIR}}/scripts/gha-encrypted/
+          cp scripts/gha/integration_testing/gcs.py ${{env.GCS_UPLOAD_DIR}}/scripts/gha/integration_testing/
+          cp scripts/gha/gcs_uploader.py ${{env.GCS_UPLOAD_DIR}}/scripts/gha/
+          cp scripts/gha-encrypted/gcs_key_file.json ${{env.GCS_UPLOAD_DIR}}//scripts/gha-encrypted/
       - name: Set CLOUDSDK_PYTHON (Windows)
         shell: bash
         if: startsWith(matrix.os, 'windows') && !cancelled()
@@ -364,8 +397,12 @@ jobs:
         if: ${{ !cancelled() }}
         uses: google-github-actions/setup-gcloud@master
       - name: Upload Desktop Artifacts to GCS
+        shell: bash
         if: ${{ !cancelled() }}
-        run: python scripts/gha/gcs_uploader.py --testapp_dir ta --key_file scripts/gha-encrypted/gcs_key_file.json
+        run: |
+          pushd ${{env.GCS_UPLOAD_DIR}}
+          python scripts/gha/gcs_uploader.py --testapp_dir ta --key_file scripts/gha-encrypted/gcs_key_file.json
+          popd
       - name: Add failure label
         # We can mark a failure as soon as any one test fails.
         if: ${{ needs.check_trigger.outputs.should_update_labels && failure() && !cancelled() }}
@@ -431,20 +468,20 @@ jobs:
         with:
           ref: ${{needs.prepare_matrix.outputs.github_ref}}
           submodules: true
-      - name: Set env vars (ubuntu)
+      - name: Set env vars (Linux)
         if: startsWith(matrix.os, 'ubuntu')
         run: echo "VCPKG_TRIPLET=x64-linux" >> $GITHUB_ENV
-      - name: Set env vars (macos)
+      - name: Set env vars (macOS)
         if: startsWith(matrix.os, 'macos')
         run: echo "VCPKG_TRIPLET=x64-osx" >> $GITHUB_ENV
-      - name: Set env vars (windows)
+      - name: Set env vars (Windows)
         shell: bash
-        if: startsWith(matrix.os, 'windows')
+        if: startsWith(matrix.os, 'Windows')
         run: echo "VCPKG_TRIPLET=x64-windows-static" >> $GITHUB_ENV
-      - name: Set env vars(all)
+      - name: Set env vars (all)
         shell: bash
         run: echo "VCPKG_RESPONSE_FILE=external/vcpkg_${{ env.VCPKG_TRIPLET }}_response_file.txt" >> $GITHUB_ENV
-      - name: Add msbuild to PATH (windows)
+      - name: Add msbuild to PATH (Windows)
         if: startsWith(matrix.os, 'windows')
         uses: microsoft/setup-msbuild@v1.0.2
       - name: Cache NDK
@@ -575,9 +612,9 @@ jobs:
         with:
           ref: ${{needs.prepare_matrix.outputs.github_ref}}
           submodules: true
-      - name: Set env vars (macos)
+      - name: Set env vars (macOS)
         run: echo "VCPKG_TRIPLET=x64-osx" >> $GITHUB_ENV
-      - name: Set env vars(all)
+      - name: Set env vars (all)
         shell: bash
         run: echo "VCPKG_RESPONSE_FILE=external/vcpkg_${{ env.VCPKG_TRIPLET }}_response_file.txt" >> $GITHUB_ENV
       - name: Setup python
