.github/workflows/codeql.yml: Add CodeQL workflow

makubacki · mergify[bot] · commit 1384ce443ded · 2023-11-07T03:19:26.000Z
Adds a workflow to run CodeQL against all packages built in
.pytool/CISettings.py. The following is done:

1. Determine which packages to build against. Those that support
   are managed by .pytool/CISettings.py will be selected.

For each package:

2. Determine how to interact with the package. Such as whether
   `stuart_ci_setup` or `stuart_setup` should be used.
3. Perform supported Stuart steps for setup and update.
4. Discover the CodeQL plugin directory in the repo.
5. Attempt to load the CodeQL CLI specific to the host OS from a
   GitHub cache.
6. Perform the build.
7. Clean up some files after build to improve robustness.
8. Upload the CodeQL results (generated SARIF file) to GitHub Code
   Scanning. The results will be associated with the trigger of the
   workflow.

After each step that can upload logs such as the setup, update, and
build steps the logs are uploaded as an artifact to the workflow run.
This allows easy debugging in case there's an error in the step.

The SARIF file is also uploaded to the workflow run so it can be
downloaded and analyzed.

Cc: Sean Brogan &lt;sean.brogan@microsoft.com&gt;
Cc: Michael D Kinney &lt;michael.d.kinney@intel.com&gt;
Signed-off-by: Michael Kubacki &lt;michael.kubacki@microsoft.com&gt;
Reviewed-by: Sean Brogan &lt;sean.brogan@microsoft.com&gt;
Acked-by: Laszlo Ersek &lt;lersek@redhat.com&gt;
Acked-by: Michael D Kinney &lt;michael.d.kinney@intel.com&gt;
diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml
@@ -0,0 +1,338 @@
+# This workflow runs CodeQL against the repository.
+#
+# Results are uploaded to GitHub Code Scanning.
+#
+# Due to a known issue with the CodeQL extractor when building the edk2
+# codebase on Linux systems, only Windows agents are used for build with
+# the VS toolchain.
+#
+# Copyright (c) Microsoft Corporation.
+# SPDX-License-Identifier: BSD-2-Clause-Patent
+
+name: "CodeQL"
+
+on:
+  push:
+    branches:
+      - master
+  pull_request:
+    branches:
+      - master
+    paths-ignore:
+      - '!**.c'
+      - '!**.h'
+
+jobs:
+  analyze:
+    name: Analyze
+    runs-on: windows-2019
+    permissions:
+      actions: read
+      contents: read
+      security-events: write
+
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - Package: "ArmPkg"
+            ArchList: "IA32,X64"
+          - Package: "CryptoPkg"
+            ArchList: "IA32"
+          - Package: "CryptoPkg"
+            ArchList: "X64"
+          - Package: "DynamicTablesPkg"
+            ArchList: "IA32,X64"
+          - Package: "FatPkg"
+            ArchList: "IA32,X64"
+          - Package: "FmpDevicePkg"
+            ArchList: "IA32,X64"
+          - Package: "IntelFsp2Pkg"
+            ArchList: "IA32,X64"
+          - Package: "IntelFsp2WrapperPkg"
+            ArchList: "IA32,X64"
+          - Package: "MdeModulePkg"
+            ArchList: "IA32"
+          - Package: "MdeModulePkg"
+            ArchList: "X64"
+          - Package: "MdePkg"
+            ArchList: "IA32,X64"
+          - Package: "PcAtChipsetPkg"
+            ArchList: "IA32,X64"
+          - Package: "PrmPkg"
+            ArchList: "IA32,X64"
+          - Package: "SecurityPkg"
+            ArchList: "IA32,X64"
+          - Package: "ShellPkg"
+            ArchList: "IA32,X64"
+          - Package: "SourceLevelDebugPkg"
+            ArchList: "IA32,X64"
+          - Package: "StandaloneMmPkg"
+            ArchList: "IA32,X64"
+          - Package: "UefiCpuPkg"
+            ArchList: "IA32,X64"
+          - Package: "UnitTestFrameworkPkg"
+            ArchList: "IA32,X64"
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v4
+
+    - name: Install Python
+      uses: actions/setup-python@v4
+      with:
+        python-version: '3.11'
+        cache: 'pip'
+        cache-dependency-path: 'pip-requirements.txt'
+
+    - name: Use Git Long Paths on Windows
+      if: runner.os == 'Windows'
+      shell: pwsh
+      run: |
+        git config --system core.longpaths true
+
+    - name: Install/Upgrade pip Modules
+      run: pip install -r pip-requirements.txt --upgrade requests
+
+    - name: Determine CI Settings File Supported Operations
+      id: get_ci_file_operations
+      shell: python
+      run: |
+        import importlib
+        import os
+        import sys
+        from pathlib import Path
+        from edk2toolext.invocables.edk2_ci_setup import CiSetupSettingsManager
+        from edk2toolext.invocables.edk2_setup import SetupSettingsManager
+
+        # Find the repo CI Settings file
+        ci_settings_file = list(Path(os.environ['GITHUB_WORKSPACE']).rglob('.pytool/CISettings.py'))
+
+        # Note: At this point, submodules have not been pulled, only one CI Settings file should exist
+        if len(ci_settings_file) != 1 or not ci_settings_file[0].is_file():
+            print("::error title=Workspace Error!::Failed to find CI Settings file!")
+            sys.exit(1)
+
+        ci_settings_file = ci_settings_file[0]
+
+        # Try Finding the Settings class in the file
+        module_name = 'ci_settings'
+
+        spec = importlib.util.spec_from_file_location(module_name, ci_settings_file)
+        module = importlib.util.module_from_spec(spec)
+        spec.loader.exec_module(module)
+
+        try:
+            settings = getattr(module, 'Settings')
+        except AttributeError:
+            print("::error title=Workspace Error!::Failed to find Settings class in CI Settings file!")
+            sys.exit(1)
+
+        # Determine Which Operations Are Supported by the Settings Class
+        ci_setup_supported = issubclass(settings, CiSetupSettingsManager)
+        setup_supported = issubclass(settings, SetupSettingsManager)
+
+        with open(os.environ['GITHUB_OUTPUT'], 'a') as fh:
+            print(f'ci_setup_supported={str(ci_setup_supported).lower()}', file=fh)
+            print(f'setup_supported={str(setup_supported).lower()}', file=fh)
+
+    - name: Setup
+      if: steps.get_ci_file_operations.outputs.setup_supported == 'true'
+      run: stuart_setup -c .pytool/CISettings.py -t DEBUG -a ${{ matrix.ArchList }} TOOL_CHAIN_TAG=VS2019
+
+    - name: Upload Setup Log As An Artifact
+      uses: actions/upload-artifact@v3
+      if: (success() || failure()) && steps.get_ci_file_operations.outputs.setup_supported == 'true'
+      with:
+        name: ${{ matrix.Package }}-Logs
+        path: |
+          **/SETUPLOG.txt
+          retention-days: 7
+        if-no-files-found: ignore
+
+    - name: CI Setup
+      if: steps.get_ci_file_operations.outputs.ci_setup_supported == 'true'
+      run: stuart_ci_setup -c .pytool/CISettings.py -t DEBUG -a ${{ matrix.ArchList }} TOOL_CHAIN_TAG=VS2019
+
+    - name: Upload CI Setup Log As An Artifact
+      uses: actions/upload-artifact@v3
+      if: (success() || failure()) && steps.get_ci_file_operations.outputs.ci_setup_supported == 'true'
+      with:
+        name: ${{ matrix.Package }}-Logs
+        path: |
+          **/CISETUP.txt
+          retention-days: 7
+        if-no-files-found: ignore
+
+    - name: Update
+      run: stuart_update -c .pytool/CISettings.py -t DEBUG -a ${{ matrix.ArchList }} TOOL_CHAIN_TAG=VS2019
+
+    - name: Upload Update Log As An Artifact
+      uses: actions/upload-artifact@v3
+      if: success() || failure()
+      with:
+        name: ${{ matrix.Package }}-Logs
+        path: |
+          **/UPDATE_LOG.txt
+        retention-days: 7
+        if-no-files-found: ignore
+
+    - name: Build Tools From Source
+      run: python BaseTools/Edk2ToolsBuild.py -t VS2019
+
+    - name: Find CodeQL Plugin Directory
+      id: find_dir
+      shell: python
+      run: |
+        import os
+        import sys
+        from pathlib import Path
+
+        # Find the plugin directory that contains the CodeQL plugin
+        plugin_dir = list(Path(os.environ['GITHUB_WORKSPACE']).rglob('BaseTools/Plugin/CodeQL'))
+
+        # This should only be found once
+        if len(plugin_dir) == 1:
+            plugin_dir = str(plugin_dir[0])
+
+            with open(os.environ['GITHUB_OUTPUT'], 'a') as fh:
+                print(f'codeql_plugin_dir={plugin_dir}', file=fh)
+        else:
+            print("::error title=Workspace Error!::Failed to find CodeQL plugin directory!")
+            sys.exit(1)
+
+    - name: Get CodeQL CLI Cache Data
+      id: cache_key_gen
+      env:
+        CODEQL_PLUGIN_DIR: ${{ steps.find_dir.outputs.codeql_plugin_dir }}
+      shell: python
+      run: |
+        import os
+        import yaml
+
+        codeql_cli_ext_dep_name = 'codeqlcli_windows_ext_dep'
+        codeql_plugin_file = os.path.join(os.environ['CODEQL_PLUGIN_DIR'], codeql_cli_ext_dep_name + '.yaml')
+
+        with open (codeql_plugin_file) as pf:
+            codeql_cli_ext_dep = yaml.safe_load(pf)
+
+            cache_key_name = codeql_cli_ext_dep['name']
+            cache_key_version = codeql_cli_ext_dep['version']
+            cache_key = f'{cache_key_name}-{cache_key_version}'
+
+            codeql_plugin_cli_ext_dep_dir = os.path.join(os.environ['CODEQL_PLUGIN_DIR'], codeql_cli_ext_dep['name'].strip() + '_extdep')
+
+            with open(os.environ['GITHUB_OUTPUT'], 'a') as fh:
+                print(f'codeql_cli_cache_key={cache_key}', file=fh)
+                print(f'codeql_cli_ext_dep_dir={codeql_plugin_cli_ext_dep_dir}', file=fh)
+
+    - name: Attempt to Load CodeQL CLI From Cache
+      id: codeqlcli_cache
+      uses: actions/cache@v3
+      with:
+        path: ${{ steps.cache_key_gen.outputs.codeql_cli_ext_dep_dir }}
+        key: ${{ steps.cache_key_gen.outputs.codeql_cli_cache_key }}
+
+    - name: Download CodeQL CLI
+      if: steps.codeqlcli_cache.outputs.cache-hit != 'true'
+      run: stuart_update -c .pytool/CISettings.py -t DEBUG -a ${{ matrix.ArchList }} TOOL_CHAIN_TAG=VS2019 --codeql
+
+    - name: Remove CI Plugins Irrelevant to CodeQL
+      shell: python
+      env:
+        CODEQL_PLUGIN_DIR: ${{ steps.find_dir.outputs.codeql_plugin_dir }}
+      run: |
+        import os
+        import shutil
+        from pathlib import Path
+
+        # Only these two plugins are needed for CodeQL
+        plugins_to_keep = ['CompilerPlugin']
+
+        plugin_dir = Path('.pytool/Plugin').absolute()
+        if plugin_dir.is_dir():
+            for dir in plugin_dir.iterdir():
+                if str(dir.stem) not in plugins_to_keep:
+                    shutil.rmtree(str(dir.absolute()), ignore_errors=True)
+
+    - name: CI Build
+      env:
+        STUART_CODEQL_PATH: ${{ steps.cache_key_gen.outputs.codeql_cli_ext_dep_dir }}
+      run: stuart_ci_build -c .pytool/CISettings.py -t DEBUG -p ${{ matrix.Package }} -a ${{ matrix.ArchList }} TOOL_CHAIN_TAG=VS2019 --codeql
+
+    - name: Build Cleanup
+      id: build_cleanup
+      shell: python
+      run: |
+        import os
+        import shutil
+        from pathlib import Path
+
+        dirs_to_delete = ['ia32', 'x64', 'arm', 'aarch64']
+
+        def delete_dirs(path: Path):
+            if path.exists() and path.is_dir():
+                if path.name.lower() in dirs_to_delete:
+                    print(f'Removed {str(path)}')
+                    shutil.rmtree(path)
+                    return
+
+                for child_dir in path.iterdir():
+                    delete_dirs(child_dir)
+
+        build_path = Path(os.environ['GITHUB_WORKSPACE'], 'Build')
+        delete_dirs(build_path)
+
+    - name: Upload Build Logs As An Artifact
+      uses: actions/upload-artifact@v3
+      if: success() || failure()
+      with:
+        name: ${{ matrix.Package }}-Logs
+        path: |
+          **/BUILD_REPORT.TXT
+          **/OVERRIDELOG.TXT
+          **/BUILDLOG_*.md
+          **/BUILDLOG_*.txt
+          **/CI_*.md
+          **/CI_*.txt
+        retention-days: 7
+        if-no-files-found: ignore
+
+    - name: Prepare Env Data for CodeQL Upload
+      id: env_data
+      env:
+        PACKAGE_NAME: ${{ matrix.Package }}
+      shell: python
+      run: |
+        import os
+
+        package = os.environ['PACKAGE_NAME'].strip().lower()
+        directory_name = 'codeql-analysis-' + package + '-debug'
+        file_name = 'codeql-db-' + package + '-debug-0.sarif'
+        sarif_path = os.path.join('Build', directory_name, file_name)
+
+        with open(os.environ['GITHUB_OUTPUT'], 'a') as fh:
+            if os.path.isfile(sarif_path):
+                print(f'upload_sarif_file=true', file=fh)
+                print(f'sarif_file_path={sarif_path}', file=fh)
+            else:
+                print(f'upload_sarif_file=false', file=fh)
+
+    - name: Upload CodeQL Results (SARIF) As An Artifact
+      uses: actions/upload-artifact@v3
+      if: steps.env_data.outputs.upload_sarif_file == 'true'
+      with:
+        name: ${{ matrix.Package }}-CodeQL-SARIF
+        path: ${{ steps.env_data.outputs.sarif_file_path }}
+        retention-days: 14
+        if-no-files-found: warn
+
+    - name: Upload CodeQL Results (SARIF) To GitHub Code Scanning
+      uses: github/codeql-action/upload-sarif@v2
+      if: steps.env_data.outputs.upload_sarif_file == 'true'
+      with:
+        # Path to SARIF file relative to the root of the repository.
+        sarif_file: ${{ steps.env_data.outputs.sarif_file_path }}
+        # Optional category for the results. Used to differentiate multiple results for one commit.
+        # Each package is a separate category.
+        category: ${{ matrix.Package }}
