Release v2.0.0

Author: rvermeulen

.codeqlmanifest.json

@@ -1 +1 @@
-{ "provide": [ "codeql_modules/*/.codeqlmanifest.json", "cpp/.codeqlmanifest.json"] }
+{ "provide": [ "codeql_modules/*/.codeqlmanifest.json", "cpp/.codeqlmanifest.json", "c/.codeqlmanifest.json"] }

.github/actions/install-codeql/action.yml

@@ -90,7 +90,7 @@ runs:
 
         if [ "$CODEQL_STDLIB_VERSION" != "latest" ]
         then
-          pushd codeql-stdlib
+          push codeql-stdlib
           echo "::debug::Switching to revision $CODEQL_STDLIB_VERSION"
           git checkout $CODEQL_STDLIB_VERSION
           popd

.github/workflows/code-scanning-pack-gen.yml

@@ -5,6 +5,7 @@ on:
     branches:
       - main
       - "rc/**"
+
   push:
     branches:
       - main
@@ -69,14 +70,17 @@ jobs:
         run: |
           PATH=$PATH:$CODEQL_HOME/codeql
           pip install -r scripts/requirements.txt
-          find rule_packages/ -name '*.json' -exec basename {} .json \; | xargs --max-procs "$XARGS_MAX_PROCS" --max-args 1 python3 scripts/generate_rules/generate_package_files.py -a
+          find rule_packages/cpp -name '*.json' -exec basename {} .json \; | xargs --max-procs "$XARGS_MAX_PROCS" --max-args 1 python3 scripts/generate_rules/generate_package_files.py -a cpp
+          find rule_packages/c -name '*.json' -exec basename {} .json \; | xargs --max-procs "$XARGS_MAX_PROCS" --max-args 1 python3 scripts/generate_rules/generate_package_files.py -a c
 
           echo "Generating help markdown file for cert"
           $CODEQL_LATEST_HOME/codeql/codeql generate query-help -vvv --format=markdown -o cpp/cert/src/ cpp/cert/src/rules
+          $CODEQL_LATEST_HOME/codeql/codeql generate query-help -vvv --format=markdown -o c/cert/src/ c/cert/src/rules
 
           codeql query compile --search-path cpp --threads 0 cpp
+          codeql query compile --search-path c --search-path cpp --threads 0 c
           cd ..
-          zip -r codeql-coding-standards/code-scanning-cpp-query-pack-anon.zip codeql-coding-standards/cpp/ codeql-coding-standards/.codeqlmanifest.json codeql-coding-standards/supported_codeql_configs.json codeql-coding-standards/scripts/deviations codeql-coding-standards/scripts/reports
+          zip -r codeql-coding-standards/code-scanning-cpp-query-pack-anon.zip codeql-coding-standards/c/ codeql-coding-standards/cpp/ codeql-coding-standards/.codeqlmanifest.json codeql-coding-standards/supported_codeql_configs.json codeql-coding-standards/scripts/deviations codeql-coding-standards/scripts/reports
 
       - name: Upload GHAS Query Pack
         uses: actions/upload-artifact@v2
@@ -91,10 +95,10 @@ jobs:
           PATH=$PATH:$CODEQL_HOME/codeql
           mkdir -p lgtm-cpp-query-pack
           function copy_queries_for_pack {
-            for q in $(codeql resolve queries cpp/$1/src/codeql-suites/$1-default.qls)
+            for q in $(codeql resolve queries $2/$1/src/codeql-suites/$1-default.qls)
             do
               copy_from_root_name="${q%.*}"
-              copy_to_root_name=$(realpath --relative-to "./cpp/$1/src/" "$copy_from_root_name")
+              copy_to_root_name=$(realpath --relative-to "./$2/$1/src/" "$copy_from_root_name")
               query_dir=$(dirname "lgtm-cpp-query-pack/$copy_to_root_name")
               mkdir -p "$query_dir"
               # Copy each selected ql file, plus the related files (qhelp, qhelp implementation)
@@ -103,10 +107,18 @@ jobs:
               cp "$copy_from_root_name.qhelp" "lgtm-cpp-query-pack/$copy_to_root_name.qhelp"
             done
           }
-          echo "Copying autosar-default queries"
-          copy_queries_for_pack "autosar"
-          echo "Copying cert-default queries"
-          copy_queries_for_pack "cert"
+          
+          echo "Copying autosar-default queries (CPP)"
+          copy_queries_for_pack "autosar" "cpp"
+          echo "Copying cert-default queries (CPP)"
+          copy_queries_for_pack "cert" "cpp"
+
+          echo "Copying misra-default queries (C)"
+          copy_queries_for_pack "misra" "c"
+          echo "Copying cert-default queries (C)"
+          copy_queries_for_pack "cert" "c"
+
+
           # Now copy all the .qll files
           lgtm_pack_dir=$(realpath lgtm-cpp-query-pack)
           for query_pack in autosar cert common
@@ -122,6 +134,22 @@ jobs:
             done
             popd
           done
+
+
+          for query_pack in misra cert common
+          do
+            echo "Copying $query_pack qll files"
+            pushd c/$query_pack/src
+            for query_library in $(find . -name \*.qll)
+            do
+              qll_dir=$(dirname "$lgtm_pack_dir/$query_library")
+              echo "Making $qll_dir"
+              mkdir -p "$qll_dir"
+              cp "$query_library" "$lgtm_pack_dir/$query_library"
+            done
+            popd
+          done
+
           cd lgtm-cpp-query-pack
           zip -9 -r ../lgtm-cpp-query-pack-anon.zip *
 
@@ -174,9 +202,19 @@ jobs:
             echo "Generating help markdown file for $s"
             $CODEQL_LATEST_HOME/codeql/codeql generate query-help -vvv --format=markdown -o cpp/$s/src/ cpp/$s/src/rules
           done
+
+          for s in "misra" "cert"
+          do
+            echo "Generating help markdown file for $s"
+            $CODEQL_LATEST_HOME/codeql/codeql generate query-help -vvv --format=markdown -o c/$s/src/ c/$s/src/rules
+          done
+
+
           codeql query compile --search-path cpp --threads 0 cpp
+          codeql query compile --search-path c --search-path cpp --threads 0 c
+
           cd ..
-          zip -r codeql-coding-standards/code-scanning-cpp-query-pack.zip codeql-coding-standards/cpp/ codeql-coding-standards/.codeqlmanifest.json codeql-coding-standards/supported_codeql_configs.json codeql-coding-standards/scripts/deviations codeql-coding-standards/scripts/reports
+          zip -r codeql-coding-standards/code-scanning-cpp-query-pack.zip codeql-coding-standards/c/ codeql-coding-standards/cpp/ codeql-coding-standards/.codeqlmanifest.json codeql-coding-standards/supported_codeql_configs.json codeql-coding-standards/scripts/deviations codeql-coding-standards/scripts/reports
 
       - name: Upload GHAS Query Pack
         uses: actions/upload-artifact@v2
@@ -191,18 +229,25 @@ jobs:
           PATH=$PATH:$CODEQL_HOME/codeql
           mkdir -p lgtm-cpp-query-pack
           function copy_queries_for_pack {
-            for rule_dir in $(codeql resolve queries cpp/$1/src/codeql-suites/$1-default.qls | xargs -L1 dirname | uniq)
+            for rule_dir in $(codeql resolve queries $2/$1/src/codeql-suites/$1-default.qls | xargs -L1 dirname | uniq)
             do
-              copy_to_root="lgtm-cpp-query-pack/$(realpath --relative-to "./cpp/$1/src/" "$rule_dir")"
+              copy_to_root="lgtm-cpp-query-pack/$(realpath --relative-to "./$2/$1/src/" "$rule_dir")"
               mkdir -p "$copy_to_root"
               # Copy each selected ql file, plus the related files (qhelp, qhelp implementation)
-              find "$rule_dir" -name '*.ql' -o -name '*.qhelp' -o -name '*.cpp' -o -name '*.png' -exec cp -n {} "$copy_to_root" \;
+              find "$rule_dir" -name '*.ql' -o -name '*.qhelp' -o -name '*.c' -name '*.cpp' -o -name '*.png' -exec cp -n {} "$copy_to_root" \;
             done
           }
-          echo "Copying autosar-default queries"
-          copy_queries_for_pack "autosar"
-          echo "Copying cert-default queries"
-          copy_queries_for_pack "cert"
+          echo "Copying autosar-default queries (CPP)"
+          copy_queries_for_pack "autosar" "cpp"
+          echo "Copying cert-default queries (CPP)"
+          copy_queries_for_pack "cert" "cpp"
+
+          echo "Copying misra-default queries (C)"
+          copy_queries_for_pack "misra" "c"
+          echo "Copying cert-default queries (C)"
+          copy_queries_for_pack "cert" "c"
+
+
           # Now copy all the .qll files
           lgtm_pack_dir=$(realpath lgtm-cpp-query-pack)
           for query_pack in autosar cert common
@@ -218,6 +263,21 @@ jobs:
             done
             popd
           done
+
+          for query_pack in misra cert common
+          do
+            echo "Copying $query_pack qll files"
+            pushd c/$query_pack/src
+            for query_library in $(find . -name \*.qll)
+            do
+              qll_dir=$(dirname "$lgtm_pack_dir/$query_library")
+              echo "Making $qll_dir"
+              mkdir -p "$qll_dir"
+              cp "$query_library" "$lgtm_pack_dir/$query_library"
+            done
+            popd
+          done
+
           cd lgtm-cpp-query-pack
           zip -9 -r ../lgtm-cpp-query-pack.zip *
 

.github/workflows/codeql_unit_tests.yml

@@ -23,11 +23,11 @@ jobs:
       - name: Export unit test matrix
         id: export-unit-test-matrix
         run: |
+          echo "Merging Result:"
+          python scripts/create_language_matrix.py
           echo "::set-output name=matrix::$(
-            jq --compact-output \
-            '.supported_environment | map([.+{os: "ubuntu-20.04-xl", codeql_standard_library_ident : .codeql_standard_library | sub("\/"; "_")}]) | flatten | {include: .}' \
-            supported_codeql_configs.json
-          )"
+            python scripts/create_language_matrix.py | \
+            jq --compact-output 'map([.+{os: "ubuntu-20.04-xl", codeql_standard_library_ident : .codeql_standard_library | sub("\/"; "_")}]) | flatten | {include: .}')"
 
   run-test-suites:
     name: Run unit tests
@@ -37,7 +37,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix: ${{ fromJSON(needs.prepare-unit-test-matrix.outputs.matrix) }}
-
+        
     steps:
       - name: Checkout repository
         uses: actions/checkout@v2
@@ -99,8 +99,8 @@ jobs:
           codeql_home = os.environ['CODEQL_HOME']
           codeql_bin = os.path.join(codeql_home, 'codeql', 'codeql')
 
-          cpp_root = Path(workspace, 'cpp')
-          test_roots = list(map(str, cpp_root.glob('*/test')))
+          language_root = Path(workspace, '${{ matrix.language }}')
+          test_roots = list(map(str, language_root.glob('*/test')))
           for test_root in test_roots:
             print(f"Executing tests found (recursively) in the directory '{test_root}'")
           files_to_close = []
@@ -109,11 +109,11 @@ jobs:
             num_slices = 8
             procs = []
             for slice in range(1, num_slices+1):
-              test_report_path = os.path.join(runner_temp, f"test_report_{runner_os}_{cli_version}_{stdlib_ref_ident}_slice_{slice}_of_{num_slices}.json")
+              test_report_path = os.path.join(runner_temp, "${{ matrix.language }}", f"test_report_{runner_os}_{cli_version}_{stdlib_ref_ident}_slice_{slice}_of_{num_slices}.json")
               os.makedirs(os.path.dirname(test_report_path), exist_ok=True)
               test_report_file = open(test_report_path, 'w')
               files_to_close.append(test_report_file)
-              procs.append(subprocess.Popen([codeql_bin, "test", "run", "--failing-exitcode=122", f"--slice={slice}/{num_slices}", "--ram=2048", "--format=json", f'--search-path={cpp_root}', *test_roots], stdout=test_report_file, stderr=subprocess.PIPE))
+              procs.append(subprocess.Popen([codeql_bin, "test", "run", "--failing-exitcode=122", f"--slice={slice}/{num_slices}", "--ram=2048", "--format=json", f'--search-path={language_root}', *test_roots], stdout=test_report_file, stderr=subprocess.PIPE))
             for p in procs:
               p.wait()
               if p.returncode != 0:
@@ -135,7 +135,7 @@ jobs:
         with:
           name: test-results-${{ runner.os }}-${{ matrix.codeql_cli }}-${{ matrix.codeql_standard_library_ident }}
           path: |
-            ${{ runner.temp }}/test_report_${{ runner.os }}_${{ matrix.codeql_cli }}_${{ matrix.codeql_standard_library_ident }}_slice_*.json
+            ${{ runner.temp }}/${{ matrix.language }}/test_report_${{ runner.os }}_${{ matrix.codeql_cli }}_${{ matrix.codeql_standard_library_ident }}_slice_*.json
           if-no-files-found: error
 
   validate-test-results:

.github/workflows/create-compiler-validation-report-x86.yml

@@ -5,14 +5,8 @@ on:
     inputs:
       release_version_tag:
         description: |
-          The tag for the new draft release, e.g. v0.5.1 (exact string, including the v).
+          The tag for the new draft release, e.g. v0.5.1.
         required: true
-
-      previous_release_version_tag:
-        description: |
-          The tag preceding this release, e.g. v0.5.0 (exact string, including the v).
-        required: true
-
       codeql_analysis_threads:
         description: |
           Number of threads to evaluate queries
@@ -52,6 +46,6 @@ jobs:
 
       - name: Create draft release
         run: |
-          scripts/release/create_draft_release.sh ${GITHUB_REF#refs/heads/} "$RELEASE_VERSION_TAG" "$PREVIOUS_RELEASE_VERSION_TAG"
+          scripts/release/create_draft_release.sh ${GITHUB_REF#refs/heads/} "$RELEASE_VERSION_TAG"
         env:
           GITHUB_TOKEN: ${{ github.token }}

.github/workflows/create-draft-release.yml

@@ -42,7 +42,9 @@ jobs:
           unzip -q codeql-linux64.zip
 
       - name: Update CodeQL formatting based on new CLI version
-        run: find cpp -name '*.ql' -or -name '*.qll' | xargs --max-procs "$XARGS_MAX_PROCS" --max-args 1 codeql/codeql query format --in-place
+        run: |
+          find cpp -name '*.ql' -or -name '*.qll' | xargs --max-procs "$XARGS_MAX_PROCS" --max-args 1 codeql/codeql query format --in-place
+          find c -name '*.ql' -or -name '*.qll' | xargs --max-procs "$XARGS_MAX_PROCS" --max-args 1 codeql/codeql query format --in-place
 
       - name: Create Pull Request
         uses: peter-evans/create-pull-request@v3