Merge branch 'main' into p--ruby-kernel-open-addition

Author: p-

.github/ISSUE_TEMPLATE/ql--false-positive.md

@@ -1,5 +1,5 @@
 ---
-name: CodeQL False positive
+name: CodeQL false positive
 about: Report CodeQL alerts that you think should not have been detected (not applicable, not exploitable, etc.)
 title: False positive
 labels: false-positive

.github/workflows/go-tests-other-os.yml

@@ -0,0 +1,81 @@
+name: "Go: Run Tests - Other OS"
+on:
+  pull_request:
+    paths:
+      - "go/**"
+      - "!go/ql/**" # don't run other-os if only ql/ files changed
+      - .github/workflows/go-tests-other-os.yml
+      - .github/actions/fetch-codeql/action.yml
+      - .github/actions/cache-query-compilation/action.yml
+      - codeql-workspace.yml
+jobs:
+  test-mac:
+    name: Test MacOS
+    runs-on: macos-latest
+    steps:
+      - name: Set up Go 1.19
+        uses: actions/setup-go@v3
+        with:
+          go-version: 1.19
+        id: go
+
+      - name: Check out code
+        uses: actions/checkout@v2
+
+      - name: Set up CodeQL CLI
+        uses: ./.github/actions/fetch-codeql
+
+      - name: Enable problem matchers in repository
+        shell: bash
+        run: 'find .github/problem-matchers -name \*.json -exec echo "::add-matcher::{}" \;'
+
+      - name: Build
+        run: |
+          cd go
+          make
+
+      - name: Cache compilation cache
+        id: query-cache
+        uses: ./.github/actions/cache-query-compilation
+        with:
+          key: go-qltest
+      - name: Test
+        run: |
+          cd go
+          make test cache="${{ steps.query-cache.outputs.cache-dir }}"
+
+  test-win:
+    name: Test Windows
+    runs-on: windows-latest-xl
+    steps:
+      - name: Set up Go 1.19
+        uses: actions/setup-go@v3
+        with:
+          go-version: 1.19
+        id: go
+
+      - name: Check out code
+        uses: actions/checkout@v2
+
+      - name: Set up CodeQL CLI
+        uses: ./.github/actions/fetch-codeql
+
+      - name: Enable problem matchers in repository
+        shell: bash
+        run: 'find .github/problem-matchers -name \*.json -exec echo "::add-matcher::{}" \;'
+
+      - name: Build
+        run: |
+          cd go
+          make
+
+      - name: Cache compilation cache
+        id: query-cache
+        uses: ./.github/actions/cache-query-compilation
+        with:
+          key: go-qltest
+
+      - name: Test
+        run: |
+          cd go
+          make test cache="${{ steps.query-cache.outputs.cache-dir }}"

.github/workflows/go-tests.yml

@@ -1,15 +1,26 @@
 name: "Go: Run Tests"
 on:
+  push:
+    paths:
+      - "go/**"
+      - .github/workflows/go-tests.yml
+      - .github/actions/fetch-codeql/action.yml
+      - .github/actions/cache-query-compilation/action.yml
+      - codeql-workspace.yml
+    branches:
+      - main
+      - "rc/*"
   pull_request:
     paths:
       - "go/**"
       - .github/workflows/go-tests.yml
       - .github/actions/fetch-codeql/action.yml
+      - .github/actions/cache-query-compilation/action.yml
       - codeql-workspace.yml
 jobs:
   test-linux:
     name: Test Linux (Ubuntu)
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-latest-xl
     steps:
       - name: Set up Go 1.19
         uses: actions/setup-go@v3
@@ -32,7 +43,7 @@ jobs:
           cd go
           make
 
-      - name: Check that all QL and Go code is autoformatted
+      - name: Check that all Go code is autoformatted
         run: |
           cd go
           make check-formatting
@@ -48,67 +59,13 @@ jobs:
           name: qhelp-markdown
           path: go/qhelp-out/**/*.md
 
-      - name: Test
-        run: |
-          cd go
-          make test
-
-  test-mac:
-    name: Test MacOS
-    runs-on: macos-latest
-    steps:
-      - name: Set up Go 1.19
-        uses: actions/setup-go@v3
-        with:
-          go-version: 1.19
-        id: go
-
-      - name: Check out code
-        uses: actions/checkout@v2
-
-      - name: Set up CodeQL CLI
-        uses: ./.github/actions/fetch-codeql
-
-      - name: Enable problem matchers in repository
-        shell: bash
-        run: 'find .github/problem-matchers -name \*.json -exec echo "::add-matcher::{}" \;'
-
-      - name: Build
-        run: |
-          cd go
-          make
-
-      - name: Test
-        run: |
-          cd go
-          make test
-
-  test-win:
-    name: Test Windows
-    runs-on: windows-2019
-    steps:
-      - name: Set up Go 1.19
-        uses: actions/setup-go@v3
+      - name: Cache compilation cache
+        id: query-cache
+        uses: ./.github/actions/cache-query-compilation
         with:
-          go-version: 1.19
-        id: go
-
-      - name: Check out code
-        uses: actions/checkout@v2
-
-      - name: Set up CodeQL CLI
-        uses: ./.github/actions/fetch-codeql
-
-      - name: Enable problem matchers in repository
-        shell: bash
-        run: 'find .github/problem-matchers -name \*.json -exec echo "::add-matcher::{}" \;'
-
-      - name: Build
-        run: |
-          cd go
-          make
-
+          key: go-qltest
+    
       - name: Test
         run: |
           cd go
-          make test
+          make test cache="${{ steps.query-cache.outputs.cache-dir }}"

.github/workflows/swift.yml

@@ -8,92 +8,76 @@ on:
       - "*.bazel*"
       - .github/workflows/swift.yml
       - .github/actions/fetch-codeql/action.yml
+      - .github/actions/cache-query-compilation/action.yml
       - codeql-workspace.yml
       - .pre-commit-config.yaml
       - "!**/*.md"
       - "!**/*.qhelp"
     branches:
       - main
+      - rc/*
+  push:
+    paths:
+      - "swift/**"
+      - "misc/bazel/**"
+      - "*.bazel*"
+      - .github/workflows/swift.yml
+      - .github/actions/fetch-codeql/action.yml
+      - .github/actions/cache-query-compilation/action.yml
+      - codeql-workspace.yml
+      - "!**/*.md"
+      - "!**/*.qhelp"
+    branches:
+      - main
+      - rc/*
 
 jobs:
-  changes:
-    runs-on: ubuntu-latest
-    outputs:
-      codegen: ${{ steps.filter.outputs.codegen }}
-      ql: ${{ steps.filter.outputs.ql }}
-    steps:
-      - uses: dorny/paths-filter@4512585405083f25c027a35db413c2b3b9006d50
-        id: filter
-        with:
-          filters: |
-            codegen:
-              - '.github/workflows/swift.yml'
-              - "misc/bazel/**"
-              - "*.bazel*"
-              - 'swift/actions/setup-env/**'
-              - '.pre-commit-config.yaml'
-              - 'swift/codegen/**'
-              - 'swift/schema.py'
-              - 'swift/**/*.dbscheme'
-              - 'swift/ql/lib/codeql/swift/elements.qll'
-              - 'swift/ql/lib/codeql/swift/elements/**'
-              - 'swift/ql/lib/codeql/swift/generated/**'
-              - 'swift/ql/test/extractor-tests/generated/**'
-              - 'swift/ql/.generated.list'
-            ql:
-              - 'github/workflows/swift.yml'
-              - 'swift/**/*.ql'
-              - 'swift/**/*.qll'
   # not using a matrix as you cannot depend on a specific job in a matrix, and we want to start linux checks
   # without waiting for the macOS build
   build-and-test-macos:
+    if: ${{ github.event_name == 'pull_request' }}
     runs-on: macos-12-xl
     steps:
       - uses: actions/checkout@v3
       - uses: ./swift/actions/create-extractor-pack
       - uses: ./swift/actions/run-quick-tests
       - uses: ./swift/actions/print-unextracted
   build-and-test-linux:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest-xl
     steps:
       - uses: actions/checkout@v3
       - uses: ./swift/actions/create-extractor-pack
       - uses: ./swift/actions/run-quick-tests
       - uses: ./swift/actions/print-unextracted
   qltests-linux:
     needs: build-and-test-linux
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-latest-xl
     steps:
       - uses: actions/checkout@v3
       - uses: ./swift/actions/run-ql-tests
   qltests-macos:
+    if : ${{ github.event_name == 'pull_request' }}
     needs: build-and-test-macos
     runs-on: macos-12-xl
-    strategy:
-      fail-fast: false
-      matrix:
-        slice: ["1/2", "2/2"]
     steps:
       - uses: actions/checkout@v3
       - uses: ./swift/actions/run-ql-tests
-        with:
-          flags: --slice ${{ matrix.slice }}
   integration-tests-linux:
     needs: build-and-test-linux
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-latest-xl
     steps:
       - uses: actions/checkout@v3
       - uses: ./swift/actions/run-integration-tests
   integration-tests-macos:
+    if : ${{ github.event_name == 'pull_request' }}
     needs: build-and-test-macos
     runs-on: macos-12-xl
     steps:
       - uses: actions/checkout@v3
       - uses: ./swift/actions/run-integration-tests
   codegen:
+    if : ${{ github.event_name == 'pull_request' }}
     runs-on: ubuntu-latest
-    needs: changes
-    if: ${{ needs.changes.outputs.codegen == 'true' }}
     steps:
       - uses: actions/checkout@v3
       - uses: ./swift/actions/setup-env
@@ -114,6 +98,7 @@ jobs:
           name: swift-generated-cpp-files
           path: generated-cpp-files/**
   database-upgrade-scripts:
+    if : ${{ github.event_name == 'pull_request' }}
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3

codeql-workspace.yml

@@ -25,6 +25,7 @@ provide:
   - "misc/suite-helpers/qlpack.yml"
   - "ruby/extractor-pack/codeql-extractor.yml"
   - "swift/extractor-pack/codeql-extractor.yml"
+  - "swift/integration-tests/qlpack.yml"
   - "ql/extractor-pack/codeql-extractor.yml"
 
 versionPolicies:

go/Makefile

@@ -34,7 +34,6 @@ autoformat:
 	find . -path '**/vendor' -prune -or -type f -iname '*.go' ! -empty -print0 | xargs -0 grep -L "//\s*autoformat-ignore" | xargs gofmt -w
 
 check-formatting:
-	find ql -iregex '.*\.qll?' -print0 | xargs -0 codeql query format --check-only
 	test -z "$$(find . -path '**/vendor' -prune -or -type f -iname '*.go' ! -empty -print0 | xargs -0 grep -L "//\s*autoformat-ignore" | xargs gofmt -l)"
 
 install-deps:
@@ -117,9 +116,9 @@ ql/lib/go.dbscheme.stats: ql/lib/go.dbscheme build/stats/src.stamp extractor
 	codeql dataset measure -o $@ build/stats/database/db-go
 
 test: all build/testdb/check-upgrade-path
-	codeql test run ql/test --search-path build/codeql-extractor-go --consistency-queries ql/test/consistency
+	codeql test run -j0 ql/test --search-path build/codeql-extractor-go --consistency-queries ql/test/consistency --compilation-cache=$(cache)
   #	use GOOS=linux because GOOS=darwin GOARCH=386 is no longer supported
-	env GOOS=linux GOARCH=386 codeql$(EXE) test run ql/test/query-tests/Security/CWE-681 --search-path build/codeql-extractor-go --consistency-queries ql/test/consistency
+	env GOOS=linux GOARCH=386 codeql$(EXE) test run -j0 ql/test/query-tests/Security/CWE-681 --search-path build/codeql-extractor-go --consistency-queries ql/test/consistency --compilation-cache=$(cache)
 	cd extractor; go test -mod=vendor ./... | grep -vF "[no test files]"
 	bash extractor-smoke-test/test.sh || (echo "Extractor smoke test FAILED"; exit 1)
 

java/kotlin-extractor/gradle.properties

@@ -1,5 +1,5 @@
 kotlin.code.style=official
-kotlinVersion=1.7.0
+kotlinVersion=1.7.21
 
 GROUP=com.github.codeql
 VERSION_NAME=0.0.1

java/kotlin-extractor/kotlin_plugin_versions.py

@@ -22,10 +22,10 @@ def version_string_to_tuple(version):
     return tuple([int(m.group(i)) for i in range(1, 4)] + [m.group(4)])
 
 # Version number used by CI. It needs to be one of the versions in many_versions.
-ci_version = '1.7.0'
+ci_version = '1.7.20'
 
 # Version numbers in the list need to be in semantically increasing order
-many_versions = [ '1.4.32', '1.5.0', '1.5.10', '1.5.20', '1.5.30', '1.6.0', '1.6.20', '1.7.0', '1.7.20-Beta' ]
+many_versions = [ '1.4.32', '1.5.0', '1.5.10', '1.5.20', '1.5.30', '1.6.0', '1.6.20', '1.7.0', '1.7.20' ]
 
 many_versions_tuples = [version_string_to_tuple(v) for v in many_versions]