wrk

Author: jsinglet

scripts/matrix_testing/CreateMatrixTestReport.ps1

@@ -297,8 +297,7 @@ $jobRows = $queriesToCheck | ForEach-Object -ThrottleLimit $NumThreads -Parallel
         "RULE"              = $CurrentRuleName;
         "QUERY"             = $CurrentQueryName;
         "COMPILE_PASS"      = $false;
-        "EXTRACTOR_PASS"    = $false;
-        "EXTRACTOR_ERRORS"  = "";
+        "COMPILE_ERROR_OUTPUT"    = "";
         "TEST_PASS"         = $false ;
         "TEST_DIFFERENCE"   = "";
     }
@@ -323,32 +322,19 @@ $jobRows = $queriesToCheck | ForEach-Object -ThrottleLimit $NumThreads -Parallel
     }
     catch {
         Write-Host -ForegroundColor ([ConsoleColor]4) "FAILED"
+        $row["COMPILE_ERROR_OUTPUT"] = $_
 
         return $row # although it is unlikely to succeed with the next rule skipping to the next rule
                     # ensures all of the rules will be reported in the
                     # output. 
     }
 
     $row["COMPILE_PASS"] = $true
-    Write-Host "Validating extractor results..." -NoNewline
-
-    try {
-         $diagnostics = Execute-QueryAndDecodeAsJson -DatabasePath $db -QueryPath $diagnostic_query
-   }catch {
-        Write-Host -ForegroundColor ([ConsoleColor]4) $_Exception.Message
-        return $row
-   }
-
-    if ( $diagnostics.'#select'.tuples.Length -eq 0 ) {
-        $row["EXTRACTOR_PASS"] = $true
-        Write-Host -ForegroundColor ([ConsoleColor]2) "OK"
-    } else {
-        Write-Host -ForegroundColor ([ConsoleColor]4) "FAILED"
-        $row["EXTRACTOR_ERRORS"] = $diagnostics | ConvertTo-Json -Depth 100
-    }
-
+    
     Write-Host "Checking expected output..."
 
+    # Dragons below 🐉🐉🐉
+    #  
     # Note this technique uses so-called "wizard" settings to make it possible
     # to compare hand compiled databases using qltest. The relative paths and
     # other options are required to be set as below (especially the detail about
@@ -388,7 +374,6 @@ $jobRows = $queriesToCheck | ForEach-Object -ThrottleLimit $NumThreads -Parallel
     Write-Host "Standard Out Buffered to: $stdOut"
     Write-Host "Standard Error Buffered to: $stdErr"
 
-
     $procDetails = Start-Process -FilePath "codeql" -PassThru -NoNewWindow -Wait -ArgumentList "test run $qlRefFile --dataset=`"$datasetRelPath`"" -RedirectStandardOutput $stdOut -RedirectStandardError $stdErr
 
     if (-Not $procDetails.ExitCode -eq 0) {

scripts/matrix_testing/NewDatabaseForRule.ps1

@@ -39,10 +39,14 @@ function New-Database-For-Rule {
 
     Write-Host "codeql database create -l cpp -s $RuleTestDir --command='$BUILD_COMMAND' $DB_PATH"
 
-    $procDetails = Start-Process -FilePath "codeql" -PassThru -NoNewWindow -Wait -ArgumentList "database create -l cpp -s $RuleTestDir --command=`"$BUILD_COMMAND`" $DB_PATH"
+    $stdOut = Join-Path ([System.IO.Path]::GetTempPath()) ([System.Guid]::NewGuid())
+    
+    $procDetails = Start-Process -FilePath "codeql" -RedirectStandardOutput $stdOut -PassThru -NoNewWindow -Wait -ArgumentList "database create -l cpp -s $RuleTestDir --command=`"$BUILD_COMMAND`" $DB_PATH"
+
+    Get-Content $stdOut | Out-String | Write-Host 
 
     if (-Not $procDetails.ExitCode -eq 0) {
-        throw "Database creation failed."
+        throw   Get-Content $stdOut | Out-String 
     }
 
     return $DB_PATH