From e17f9401ca64647afaa770483ae07ec22c896dcf Mon Sep 17 00:00:00 2001 From: Dilan Bhalla Date: Sun, 2 Nov 2025 21:16:03 -0800 Subject: [PATCH] iac windows extraction --- .github/workflows/build.yml | 31 +++++++++++++-- .github/workflows/publish.yml | 15 ++++--- scripts/create-extractor-pack.ps1 | 66 +++++++++++++++++++++++++------ scripts/install-extractor.ps1 | 26 ++++++++++++ scripts/run-tests.ps1 | 57 ++++++++++++++++++++++++++ tools/index-files.cmd | 1 + tools/qltest.cmd | 4 +- 7 files changed, 179 insertions(+), 21 deletions(-) create mode 100644 scripts/install-extractor.ps1 create mode 100644 scripts/run-tests.ps1 diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index ffaae6f..b6f075e 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -12,9 +12,10 @@ permissions: jobs: tests: - runs-on: ubuntu-latest + runs-on: ${{ matrix.os }} strategy: matrix: + os: [ubuntu-latest, windows-latest] test-folders: ["library-tests", "queries-tests"] steps: - name: "Checkout" @@ -53,8 +54,8 @@ jobs: - uses: dtolnay/rust-toolchain@4305c38b25d97ef35a8ad1f985ccf2d2242004f2 # stable if: steps.extractor-changes.outputs.src == 'true' - - name: "Build Extractor" - if: steps.extractor-changes.outputs.src == 'true' + - name: "Build Extractor (Linux/macOS)" + if: steps.extractor-changes.outputs.src == 'true' && matrix.os != 'windows-latest' env: GH_TOKEN: ${{ github.token }} run: | @@ -66,12 +67,34 @@ jobs: gh codeql resolve languages --format=json --search-path ./extractor-pack - - name: "Run Tests" + - name: "Build Extractor (Windows)" + if: steps.extractor-changes.outputs.src == 'true' && matrix.os == 'windows-latest' + env: + GH_TOKEN: ${{ github.token }} + shell: pwsh + run: | + gh extension install github/gh-codeql + gh codeql set-version latest + + .\scripts\create-extractor-pack.ps1 + + gh codeql resolve languages --format=json --search-path .\extractor-pack + + - name: "Run Tests (Linux/macOS)" + if: matrix.os != 'windows-latest' env: GH_TOKEN: ${{ github.token }} run: | ./scripts/run-tests.sh "ql/test/${{ matrix.test-folders }}" + - name: "Run Tests (Windows)" + if: matrix.os == 'windows-latest' + env: + GH_TOKEN: ${{ github.token }} + shell: pwsh + run: | + .\scripts\run-tests.ps1 "ql/test/${{ matrix.test-folders }}" + # scanning: # runs-on: ubuntu-latest # needs: [tests] diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 2012c2f..7962bea 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -47,8 +47,7 @@ jobs: strategy: fail-fast: false matrix: - # TODO: Add windows-latest - os: [ubuntu-latest, macos-latest] + os: [ubuntu-latest, macos-latest, windows-latest] if: ${{ needs.release-check.outputs.release == 'true' }} steps: @@ -59,14 +58,20 @@ jobs: - name: "Set up Rust" uses: dtolnay/rust-toolchain@nightly - if: ${{ matrix.os != 'windows-latest' }} - - name: "Build Extractor" + - name: "Build Extractor (Linux/macOS)" if: ${{ matrix.os != 'windows-latest' }} env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: ./scripts/create-extractor-pack.sh + - name: "Build Extractor (Windows)" + if: ${{ matrix.os == 'windows-latest' }} + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + shell: pwsh + run: .\scripts\create-extractor-pack.ps1 + - name: "Upload bundle artifact" uses: actions/upload-artifact@v4 with: @@ -85,7 +90,7 @@ jobs: with: submodules: true - - name: "Downloadd all artifacts" + - name: "Download all artifacts" uses: actions/download-artifact@v4 with: path: "./extractor-pack" diff --git a/scripts/create-extractor-pack.ps1 b/scripts/create-extractor-pack.ps1 index 65b4dfc..fba204d 100644 --- a/scripts/create-extractor-pack.ps1 +++ b/scripts/create-extractor-pack.ps1 @@ -1,14 +1,58 @@ -cargo build --release +$ErrorActionPreference = "Stop" + +# Set platform +$platform = "win64" -cargo run --release -p ql-generator -- --dbscheme ql/src/ql.dbscheme --library ql/src/codeql_ql/ast/internal/TreeSitter.qll -codeql query format -i ql\src\codeql_ql\ast\internal\TreeSitter.qll +# Check for CodeQL binary +if (Get-Command "codeql" -ErrorAction SilentlyContinue) { + $CODEQL_BINARY = "codeql" +} +elseif (Get-Command "gh" -ErrorAction SilentlyContinue) { + try { + gh codeql version 2>&1 | Out-Null + $CODEQL_BINARY = "gh codeql" + } + catch { + Write-Host "Installing gh-codeql extension..." + gh extension install github/gh-codeql + $CODEQL_BINARY = "gh codeql" + } +} +else { + Write-Error "Neither 'codeql' nor 'gh' command found" + exit 1 +} +Write-Host "Creating extractor pack..." if (Test-Path -Path extractor-pack) { - rm -Recurse -Force extractor-pack -} -mkdir extractor-pack | Out-Null -cp codeql-extractor.yml, ql\src\ql.dbscheme, ql\src\ql.dbscheme.stats extractor-pack -cp -Recurse tools extractor-pack -mkdir extractor-pack\tools\win64 | Out-Null -cp target\release\ql-extractor.exe extractor-pack\tools\win64\extractor.exe -cp target\release\ql-autobuilder.exe extractor-pack\tools\win64\autobuilder.exe + Remove-Item -Recurse -Force extractor-pack +} +if (Test-Path -Path target) { + Remove-Item -Recurse -Force target +} + +Write-Host "Update submodules..." +git submodule update --init --recursive + +Write-Host "Building extractor..." +cargo build --release + +Write-Host "Generating TreeSitter library..." +cargo run --release --bin codeql-extractor-iac -- generate --dbscheme ql/lib/iac.dbscheme --library ql/lib/codeql/iac/ast/internal/TreeSitter.qll + +Write-Host "Formatting generated library..." +if ($CODEQL_BINARY -eq "gh codeql") { + gh codeql query format -i ql/lib/codeql/iac/ast/internal/TreeSitter.qll +} +else { + codeql query format -i ql/lib/codeql/iac/ast/internal/TreeSitter.qll +} + +New-Item -ItemType Directory -Path extractor-pack | Out-Null +Copy-Item -Path codeql-extractor.yml, ql/lib/iac.dbscheme, ql/lib/iac.dbscheme.stats -Destination extractor-pack/ +Copy-Item -Recurse -Path downgrades, tools -Destination extractor-pack/ + +New-Item -ItemType Directory -Path "extractor-pack/tools/$platform" -Force | Out-Null +Copy-Item -Path "target/release/codeql-extractor-iac.exe" -Destination "extractor-pack/tools/$platform/extractor.exe" + +Write-Host "Extractor pack created successfully!" diff --git a/scripts/install-extractor.ps1 b/scripts/install-extractor.ps1 new file mode 100644 index 0000000..d192284 --- /dev/null +++ b/scripts/install-extractor.ps1 @@ -0,0 +1,26 @@ +param( + [string]$ExtractorName = "iac", + [string]$ExtractorLocations = "$env:USERPROFILE\.codeql\extractors" +) + +$ErrorActionPreference = "Stop" + +Write-Host "Creating extractor directory..." +if (!(Test-Path $ExtractorLocations)) { + New-Item -ItemType Directory -Path $ExtractorLocations -Force | Out-Null +} + +Write-Host "Checking latest release..." +gh release list -L 1 -R "advanced-security/codeql-extractor-$ExtractorName" + +Write-Host "Downloading extractor pack..." +gh release download ` + -R "advanced-security/codeql-extractor-$ExtractorName" ` + -D "$ExtractorLocations" ` + --clobber ` + --pattern 'extractor-*.tar.gz' + +Write-Host "Extracting extractor pack..." +tar -zxf "$ExtractorLocations/extractor-$ExtractorName.tar.gz" --directory "$ExtractorLocations" + +Write-Host "Installation complete! Extractor installed to: $ExtractorLocations" \ No newline at end of file diff --git a/scripts/run-tests.ps1 b/scripts/run-tests.ps1 new file mode 100644 index 0000000..8bacc7f --- /dev/null +++ b/scripts/run-tests.ps1 @@ -0,0 +1,57 @@ +param( + [string]$TestsDir = "ql/test" +) + +$ErrorActionPreference = "Stop" + +# Check for CodeQL binary +if (Get-Command "codeql" -ErrorAction SilentlyContinue) { + $CODEQL_BINARY = "codeql" +} +elseif (Get-Command "gh" -ErrorAction SilentlyContinue) { + try { + gh codeql version 2>&1 | Out-Null + $CODEQL_BINARY = "gh codeql" + } + catch { + Write-Host "Installing gh-codeql extension..." + gh extension install github/gh-codeql + $CODEQL_BINARY = "gh codeql" + } +} +else { + Write-Error "Neither 'codeql' nor 'gh' command found" + exit 1 +} + +Write-Host "Installing ql/test pack dependencies..." +if ($CODEQL_BINARY -eq "gh codeql") { + gh codeql pack install ql/test +} +else { + codeql pack install ql/test +} + +Write-Host "Running tests in $TestsDir" + +if ($CODEQL_BINARY -eq "gh codeql") { + gh codeql test run ` + -j 0 ` + --check-databases --check-unused-labels --check-repeated-labels --check-redefined-labels --check-use-before-definition ` + --search-path ./extractor-pack ` + "$TestsDir" +} +else { + codeql test run ` + -j 0 ` + --check-databases --check-unused-labels --check-repeated-labels --check-redefined-labels --check-use-before-definition ` + --search-path ./extractor-pack ` + "$TestsDir" +} + +if ($LASTEXITCODE -ne 0) { + Write-Error "Tests failed with exit code $LASTEXITCODE" + exit $LASTEXITCODE +} + +Write-Host "All tests passed!" \ No newline at end of file diff --git a/tools/index-files.cmd b/tools/index-files.cmd index ab05465..56a743b 100755 --- a/tools/index-files.cmd +++ b/tools/index-files.cmd @@ -1,6 +1,7 @@ @echo off type NUL && "%CODEQL_EXTRACTOR_IAC_ROOT%\tools\win64\extractor.exe" ^ + extract ^ --file-list "%1" ^ --source-archive-dir "%CODEQL_EXTRACTOR_IAC_SOURCE_ARCHIVE_DIR%" ^ --output-dir "%CODEQL_EXTRACTOR_IAC_TRAP_DIR%" diff --git a/tools/qltest.cmd b/tools/qltest.cmd index c960b58..233cb68 100755 --- a/tools/qltest.cmd +++ b/tools/qltest.cmd @@ -4,8 +4,10 @@ type NUL && "%CODEQL_DIST%\codeql.exe" database index-files ^ --prune=**/*.testproj ^ --include-extension=.hcl ^ --include-extension=.tf ^ + --include-extension=.tfvars ^ + --include-extension=.bicep ^ --size-limit=5m ^ - --language=hcl ^ + --language=iac ^ --working-dir=. ^ "%CODEQL_EXTRACTOR_IAC_WIP_DATABASE%"