Merge pull request #11364 from github/redsun82/swift-ci-cache

Swift: cache more aggressively in CI

Author: erik-krogh

.github/workflows/swift.yml

@@ -8,42 +8,30 @@ on:
       - "*.bazel*"
       - .github/workflows/swift.yml
       - .github/actions/fetch-codeql/action.yml
+      - .github/actions/cache-query-compilation/action.yml
       - codeql-workspace.yml
       - .pre-commit-config.yaml
       - "!**/*.md"
       - "!**/*.qhelp"
     branches:
       - main
+      - rc/*
+  push:
+    paths:
+      - "swift/**"
+      - "misc/bazel/**"
+      - "*.bazel*"
+      - .github/workflows/swift.yml
+      - .github/actions/fetch-codeql/action.yml
+      - .github/actions/cache-query-compilation/action.yml
+      - codeql-workspace.yml
+      - "!**/*.md"
+      - "!**/*.qhelp"
+    branches:
+      - main
+      - rc/*
 
 jobs:
-  changes:
-    runs-on: ubuntu-latest
-    outputs:
-      codegen: ${{ steps.filter.outputs.codegen }}
-      ql: ${{ steps.filter.outputs.ql }}
-    steps:
-      - uses: dorny/paths-filter@4512585405083f25c027a35db413c2b3b9006d50
-        id: filter
-        with:
-          filters: |
-            codegen:
-              - '.github/workflows/swift.yml'
-              - "misc/bazel/**"
-              - "*.bazel*"
-              - 'swift/actions/setup-env/**'
-              - '.pre-commit-config.yaml'
-              - 'swift/codegen/**'
-              - 'swift/schema.py'
-              - 'swift/**/*.dbscheme'
-              - 'swift/ql/lib/codeql/swift/elements.qll'
-              - 'swift/ql/lib/codeql/swift/elements/**'
-              - 'swift/ql/lib/codeql/swift/generated/**'
-              - 'swift/ql/test/extractor-tests/generated/**'
-              - 'swift/ql/.generated.list'
-            ql:
-              - 'github/workflows/swift.yml'
-              - 'swift/**/*.ql'
-              - 'swift/**/*.qll'
   # not using a matrix as you cannot depend on a specific job in a matrix, and we want to start linux checks
   # without waiting for the macOS build
   build-and-test-macos:
@@ -54,46 +42,41 @@ jobs:
       - uses: ./swift/actions/run-quick-tests
       - uses: ./swift/actions/print-unextracted
   build-and-test-linux:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest-xl
     steps:
       - uses: actions/checkout@v3
       - uses: ./swift/actions/create-extractor-pack
       - uses: ./swift/actions/run-quick-tests
       - uses: ./swift/actions/print-unextracted
   qltests-linux:
     needs: build-and-test-linux
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-latest-xl
     steps:
       - uses: actions/checkout@v3
       - uses: ./swift/actions/run-ql-tests
   qltests-macos:
+    if : ${{ github.event_name == 'pull_request' }}
     needs: build-and-test-macos
     runs-on: macos-12-xl
-    strategy:
-      fail-fast: false
-      matrix:
-        slice: ["1/2", "2/2"]
     steps:
       - uses: actions/checkout@v3
       - uses: ./swift/actions/run-ql-tests
-        with:
-          flags: --slice ${{ matrix.slice }}
   integration-tests-linux:
     needs: build-and-test-linux
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-latest-xl
     steps:
       - uses: actions/checkout@v3
       - uses: ./swift/actions/run-integration-tests
   integration-tests-macos:
+    if : ${{ github.event_name == 'pull_request' }}
     needs: build-and-test-macos
     runs-on: macos-12-xl
     steps:
       - uses: actions/checkout@v3
       - uses: ./swift/actions/run-integration-tests
   codegen:
+    if : ${{ github.event_name == 'pull_request' }}
     runs-on: ubuntu-latest
-    needs: changes
-    if: ${{ needs.changes.outputs.codegen == 'true' }}
     steps:
       - uses: actions/checkout@v3
       - uses: ./swift/actions/setup-env
@@ -114,6 +97,7 @@ jobs:
           name: swift-generated-cpp-files
           path: generated-cpp-files/**
   database-upgrade-scripts:
+    if : ${{ github.event_name == 'pull_request' }}
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3

swift/actions/run-integration-tests/action.yml

@@ -17,7 +17,11 @@ runs:
       with:
         swift-version: "${{steps.get_swift_version.outputs.version}}"
     - uses: ./.github/actions/fetch-codeql
+    - id: query-cache
+      uses: ./.github/actions/cache-query-compilation
+      with:
+        key: swift-qltest
     - name: Run integration tests
       shell: bash
       run: |
-        python swift/integration-tests/runner.py
+        python swift/integration-tests/runner.py --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}"

swift/actions/run-ql-tests/action.yml

@@ -10,18 +10,23 @@ runs:
   steps:
     - uses: ./swift/actions/share-extractor-pack
     - uses: ./.github/actions/fetch-codeql
+    - id: query-cache
+      uses: ./.github/actions/cache-query-compilation
+      with:
+        key: swift-qltest
     - name: Run QL tests
       shell: bash
       run: |
         codeql test run \
           --threads=0 \
-          --ram 5000 \
+          --ram 52000 \
           --search-path "${{ github.workspace }}/swift/extractor-pack" \
           --check-databases \
           --check-unused-labels \
           --check-repeated-labels \
           --check-redefined-labels \
           --check-use-before-definition \
+          --compilation-cache "${{ steps.query-cache.outputs.cache-dir }}" \
           ${{ inputs.flags }} \
           swift/ql/test
       env:

swift/integration-tests/runner.py

@@ -25,6 +25,7 @@ def options():
     p.add_argument("--check-databases", action="store_true")
     p.add_argument("--learn", action="store_true")
     p.add_argument("--threads", "-j", type=int, default=0)
+    p.add_argument("--compilation-cache")
     return p.parse_args()
 
 
@@ -65,6 +66,8 @@ def main(opts):
             cmd.append("--no-check-databases")
         if opts.learn:
             cmd.append("--learn")
+        if opts.compilation_cache:
+            cmd.append(f'--compilation-cache="{opts.compilation_cache}"')
         cmd.extend(str(t.parent) for t in succesful_db_creation)
         ql_test_success = subprocess.run(cmd).returncode == 0