Merge pull request #220 from advanced-security/ci-updates

feat: Enhance extractor pack workflow with compile and bundle jobs

Author: GeekMasher

.github/workflows/publish.yml

@@ -39,6 +39,65 @@ jobs:
             echo "release=true" >> "$GITHUB_OUTPUT"
           fi
 
+  compile:
+    name: "Compile Extractor Pack for ${{ matrix.os }}"
+    needs: [release-check]
+
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        # TODO: Add windows-latest
+        os: [ubuntu-latest, macos-latest]
+
+    if: ${{ needs.release-check.outputs.release == 'true' }}
+    steps:
+      - name: "Checkout"
+        uses: actions/checkout@v5
+        with:
+          submodules: true
+
+      - name: "Set up Rust"
+        uses: dtolnay/rust-toolchain@nightly
+        if: ${{ matrix.os != 'windows-latest' }}
+
+      - name: "Build Extractor"
+        if: ${{ matrix.os != 'windows-latest' }}
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: ./scripts/create-extractor-pack.sh
+
+      - name: "Upload bundle artifact"
+        uses: actions/upload-artifact@v4
+        with:
+          name: "extractor-bundle-${{ matrix.os }}"
+          path: "./extractor-pack"
+
+  bundle:
+    name: "Bundle Extractor Pack"
+    runs-on: ubuntu-latest
+    needs: [compile]
+    if: ${{ needs.release-check.outputs.release == 'true' }}
+
+    steps:
+      - name: "Checkout"
+        uses: actions/checkout@v5
+        with:
+          submodules: true
+
+      - name: "Downloadd all artifacts"
+        uses: actions/download-artifact@v4
+        with:
+          path: "./extractor-pack"
+          merge-multiple: true
+
+      - name: "Publish Extractor Pack"
+        if: github.ref == 'refs/heads/main'
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          EXTRACTOR_NAME: "iac"
+        run: |
+          ./scripts/publish-extractor-pack.sh
 
   queries:
     runs-on: ubuntu-latest
@@ -61,32 +120,24 @@ jobs:
       - name: "Check and Publish CodeQL Packs"
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          PACKS: ${{ matrix.packs }}
+          ORG: ${{ github.repository_owner }}
         run: |
-          PACK_PATH=ql/${{ matrix.packs }}/qlpack.yml
+          set -e
+          
+          PACK_PATH="ql/${PACKS}/qlpack.yml"
+          echo "[+] Pack Path :: $PACK_PATH"
+
           CURRENT_VERSION=$(grep version $PACK_PATH | awk '{print $2}')
           PACK_FULLNAME=$(cat $PACK_PATH | grep "name:" | awk '{print $2}')
           PACK_NAME=$(echo $PACK_FULLNAME | awk -F '/' '{print $2}')
+          echo "[+] Pack Name :: $PACK_NAME ($PACK_FULLNAME)"
 
-          PUBLISHED_VERSION=$(gh api /orgs/advanced-security/packages/container/$PACK_NAME/versions --jq '.[0].metadata.container.tags[0]')
+          PUBLISHED_VERSION=$(gh api /orgs/${ORG}/packages/container/$PACK_NAME/versions --jq '.[0].metadata.container.tags[0]')
           echo "Packs :: ${CURRENT_VERSION} -> ${PUBLISHED_VERSION}"
 
           if [ "$PUBLISHED_VERSION" != "$CURRENT_VERSION" ]; then
             gh extension install github/gh-codeql
-            gh codeql pack install "ql/${{ matrix.packs }}"
-            gh codeql pack publish "ql/${{ matrix.packs }}"
+            gh codeql pack install "ql/${PACKS}"
+            gh codeql pack publish "ql/${PACKS}"
           fi
-
-  compile:
-    runs-on: ubuntu-latest
-    needs: [release-check]
-    if: ${{ needs.release-check.outputs.release == 'true' }}
-
-    steps:
-      - uses: actions/checkout@v5
-
-      - name: "Publish Extractor Pack"
-        if: github.ref == 'refs/heads/main'
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        run: |
-          ./scripts/publish-extractor-pack.sh

codeql-extractor.yml

@@ -6,11 +6,9 @@ legacy_qltest_extraction: true
 github_api_languages:
   - HCL
   - Docker
-  - Bicep
 scc_languages:
   - HCL
   - Docker
-  - Bicep
 
 # File types
 file_types:
@@ -26,8 +24,3 @@ file_types:
     extensions:
       - .Dockerfile
       - .Containerfile
-
-  - name: bicep
-    display_name: Bicep
-    extensions:
-      - .bicep

ql/lib/codeql/hcl/Providers.qll

@@ -0,0 +1,8 @@
+import providers.Alicloud
+import providers.AWS
+import providers.Azure
+import providers.GCP
+import providers.GitHub
+import providers.Helm
+import providers.Kubernetes
+import providers.OCI

ql/lib/codeql/hcl/Security.qll

@@ -0,0 +1 @@
+import security.HardcodedPasswords

ql/lib/hcl.qll

@@ -6,9 +6,6 @@ import codeql.hcl.Constants
 import codeql.hcl.Locals
 import codeql.hcl.Terraform
 // providers
-import codeql.hcl.providers.AWS
-import codeql.hcl.providers.Azure
-import codeql.hcl.providers.GCP
-import codeql.hcl.providers.Helm
+import codeql.hcl.Providers
 // security
-import codeql.hcl.security.HardcodedPasswords
+import codeql.hcl.Security

scripts/create-extractor-pack.sh

@@ -34,8 +34,3 @@ mkdir -p extractor-pack
 cp -r codeql-extractor.yml downgrades tools ql/lib/iac.dbscheme ql/lib/iac.dbscheme.stats extractor-pack/
 mkdir -p extractor-pack/tools/${platform}
 cp target/release/codeql-extractor-iac extractor-pack/tools/${platform}/extractor
-
-# pushd ql/lib
-# $CODEQL_BINARY pack installl .
-# $CODEQL_BINARY pack create --output=$HOME/.codeql/packages .
-# popd

scripts/publish-extractor-pack.sh

@@ -1,24 +1,51 @@
 #!/bin/bash
+set -eux
+
+DRY_RUN=${DRY_RUN:-false}
 
 # Extractor info
 EXTRACTOR_INFO="./codeql-extractor.yml"
-EXTRACTOR_NAME="iac"
+EXTRACTOR_NAME="${EXTRACTOR_NAME:-$(basename $(dirname $EXTRACTOR_INFO))}"
 EXTRACTOR_VERSION=$(grep version $EXTRACTOR_INFO | awk '{print $2}')
 
 LATEST_RELEASE=$(gh release list | head -n 1 | awk '{print $1}' | sed 's/v//')
 
+if which codeql >/dev/null 2>&1; then
+  CODEQL_BINARY="codeql"
+elif gh codeql version >/dev/null 2>&1; then
+  CODEQL_BINARY="gh codeql"
+else
+  gh extension install github/gh-codeql
+  CODEQL_BINARY="gh codeql"
+fi
+
 echo "[+] ${EXTRACTOR_NAME} (${EXTRACTOR_VERSION})"
 echo "[+] Last release: ${LATEST_RELEASE}"
 
 if [ "$LATEST_RELEASE" != "$EXTRACTOR_VERSION" ]; then
     echo "[+] New Extractor version being released"
 
-    # create extractor pack
-    ./scripts/create-extractor-pack.sh
+    # Check extracrtor-pack
+    if [ ! -d "extractor-pack" ]; then
+        echo "[+] No extractor-pack found"
+        exit 1
+    fi
+
+    echo "[+] Install pack dependencies"
+    $CODEQL_BINARY pack install "./ql/lib"
+
+    echo "[+] Add queries to extractor-pack"
+    $CODEQL_BINARY pack create --output="./extractor-pack/qlpacks" "./ql/lib"
+    $CODEQL_BINARY pack create --output="./extractor-pack/qlpacks" "./ql/src"
 
     # bundle extractor
     tar czf extractor-$EXTRACTOR_NAME.tar.gz extractor-pack
 
+    if [ "$DRY_RUN" = "true" ]; then
+        echo "[+] Dry run - skipping release"
+        exit 0
+    fi
+
     export GH_TOKEN=$GITHUB_TOKEN
 
     # create release