Upgrade CodeQL dependencies now updates qlpack.yml files

The appropriate version of the `codeql/cpp-all` pack is identified
by querying the qlpack.yml of the tag for the CodeQL version on
github/codeql. This is then applied to all relevant qlpack.yml
files in the repo, then codeql pack upgrade is used to update the
lock files.

Author: lcartey

scripts/upgrade-codeql-dependencies/requirements.txt

@@ -4,3 +4,4 @@ idna==3.4
 requests==2.31.0
 semantic-version==2.10.0
 urllib3==1.26.18
+pyyaml==6.0.1

scripts/upgrade-codeql-dependencies/upgrade-codeql-dependencies.py

@@ -1,18 +1,23 @@
 import json
 import requests
-from typing import Optional, Dict, List
+from typing import Optional, Dict, List, Tuple
 from semantic_version import Version
 from pathlib import Path
+import yaml
 
 SCRIPT_PATH = Path(__file__)
-SUPPORTED_VERSIONS_PATH = SCRIPT_PATH.parent.parent.parent / "supported_codeql_configs.json"
+CODING_STANDARDS_ROOT = SCRIPT_PATH.parent.parent.parent
+SUPPORTED_VERSIONS_PATH = CODING_STANDARDS_ROOT / "supported_codeql_configs.json"
 
-def get_compatible_stdlib(version: Version) -> Optional[str]:
+def get_compatible_stdlib(version: Version) -> Optional[Tuple[str, str]]:
     tag = f"codeql-cli/v{version}"
     response = requests.get(f"https://raw.githubusercontent.com/github/codeql/{tag}/cpp/ql/lib/qlpack.yml")
 
     if response.status_code == 200:
-        return tag
+        # Parse the qlpack.yml returned in the response as a yaml file to read the version property
+        qlpack = yaml.safe_load(response.text)
+        if qlpack is not None and "version" in qlpack:
+            return (tag, qlpack["version"])
     return None
 
 def get_compatible_bundle(version: Version, token: str) -> Optional[str]:
@@ -30,15 +35,17 @@ def get_compatible_bundle(version: Version, token: str) -> Optional[str]:
 def main(cli_version : str, github_token: str) -> None:
     try:
         parsed_cli_version = Version(cli_version)
-        compatible_stdlib = get_compatible_stdlib(parsed_cli_version)
-        if compatible_stdlib is None:
+        compatible_stdlib_return = get_compatible_stdlib(parsed_cli_version)
+        if compatible_stdlib_return is None:
             print(f"Unable to find compatible standard library for: {parsed_cli_version}")
             exit(1)
         compatible_bundle = get_compatible_bundle(parsed_cli_version, github_token)
         if compatible_bundle is None:
             print(f"Unable to find compatible bundle for: {parsed_cli_version}")
             exit(1)
 
+        compatible_stdlib_tag, compatible_stdlib_version = compatible_stdlib_return
+
         with SUPPORTED_VERSIONS_PATH.open("r") as f:
             supported_versions = json.load(f)
 
@@ -49,10 +56,36 @@ def main(cli_version : str, github_token: str) -> None:
         supported_env = supported_envs[0]
         supported_env["codeql_cli"] = str(parsed_cli_version)
         supported_env["codeql_cli_bundle"] = compatible_bundle
-        supported_env["codeql_standard_library"] = compatible_stdlib
+        supported_env["codeql_standard_library"] = compatible_stdlib_tag
 
         with SUPPORTED_VERSIONS_PATH.open("w") as f:
             json.dump(supported_versions, f, indent=2)
+
+        # Find every qlpack.yml file in the repository
+        qlpack_files = list(CODING_STANDARDS_ROOT.rglob("qlpack.yml"))
+        # Filter out any files that are in a hidden directory
+        qlpack_files = [f for f in qlpack_files if not any(part for part in f.parts if part.startswith("."))]
+
+        # Update the "codeql/cpp-all" entries in the "dependencies" property in every qlpack.yml file
+        updated_qlpacks = []
+        for qlpack_file in qlpack_files:
+            with qlpack_file.open("r") as f:
+                qlpack = yaml.safe_load(f)
+            print("Updating dependencies in " + str(qlpack_file))
+            if "codeql/cpp-all" in qlpack["dependencies"]:
+                qlpack["dependencies"]["codeql/cpp-all"] = compatible_stdlib_version
+                with qlpack_file.open("w") as f:
+                    yaml.safe_dump(qlpack, f)
+                updated_qlpacks.append(qlpack_file.parent)
+
+        # Call CodeQL to update the lock files by running codeql pack upgrade
+        # Note: we need to do this after updating all the qlpack files,
+        #       otherwise we may get dependency resolution errors
+        for qlpack in updated_qlpacks:
+            print("Updating lock files for " + str(qlpack))
+            os.system(f"codeql pack upgrade {qlpack}")
+
+
     except ValueError as e:
         print(e)
         exit(1)