Merge branch 'main' into codeql/upgrade-to-v2.23.6

Author: data-douser

extractors/cds/tools/cds-extractor.ts

@@ -3,17 +3,16 @@ import { join } from 'path';
 import { sync as globSync } from 'glob';
 
 import { orchestrateCompilation } from './src/cds/compiler';
-import { buildCdsProjectDependencyGraph } from './src/cds/parser';
-import { runJavaScriptExtractor } from './src/codeql';
+import { buildCdsProjectDependencyGraph, type CdsDependencyGraph } from './src/cds/parser';
+import { handleEarlyExit, runJavaScriptExtractionWithMarker } from './src/codeql';
 import {
   addCompilationDiagnostic,
   addDependencyGraphDiagnostic,
   addDependencyInstallationDiagnostic,
   addEnvironmentSetupDiagnostic,
-  addJavaScriptExtractorDiagnostic,
   addNoCdsProjectsDiagnostic,
 } from './src/diagnostics';
-import { configureLgtmIndexFilters, setupAndValidateEnvironment } from './src/environment';
+import { setupAndValidateEnvironment } from './src/environment';
 import {
   cdsExtractorLog,
   generateStatusReport,
@@ -27,31 +26,28 @@ import {
 import { cacheInstallDependencies } from './src/packageManager';
 import { validateArguments } from './src/utils';
 
-// Validate the script arguments.
+// ============================================================================
+// Main Extraction Flow
+// ============================================================================
+
+// Validate arguments
 const validationResult = validateArguments(process.argv);
 if (!validationResult.isValid) {
   console.warn(validationResult.usageMessage);
-  // For invalid arguments, we can't proceed but we also can't add diagnostics since we don't have
-  // the necessary context (sourceRoot, codeqlExePath). Log the issue and exit gracefully.
   console.log(
     `CDS extractor terminated due to invalid arguments: ${validationResult.usageMessage}`,
   );
   console.log(`Completed run of the cds-extractor.js script for the CDS extractor.`);
-  process.exit(0); // Use exit code 0 to not fail the overall JavaScript extractor
+  process.exit(0);
 }
 
-// Get the validated and sanitized arguments.
 const { sourceRoot } = validationResult.args!;
 
-// Initialize the unified logging system with the source root directory.
+// Initialize logging
 setSourceRootDirectory(sourceRoot);
-
-// Log the start of the CDS extractor session as a whole.
 logExtractorStart(sourceRoot);
 
-// Setup the environment and validate all requirements first, before changing
-// directory back to the "sourceRoot" directory. This ensures we can properly locate
-// the CodeQL tools.
+// Setup and validate environment
 logPerformanceTrackingStart('Environment Setup');
 const {
   success: envSetupSuccess,
@@ -64,25 +60,18 @@ logPerformanceTrackingStop('Environment Setup');
 
 if (!envSetupSuccess) {
   const codeqlExe = platformInfo.isWindows ? 'codeql.exe' : 'codeql';
-  const errorMessage = `'${codeqlExe} database index-files --language cds' terminated early due to: ${errorMessages.join(
-    ', ',
-  )}.`;
-
+  const errorMessage = `'${codeqlExe} database index-files --language cds' terminated early due to: ${errorMessages.join(', ')}.`;
   cdsExtractorLog('warn', errorMessage);
 
-  // Add diagnostic for environment setup failure if we have a codeqlExePath
   if (codeqlExePath) {
     addEnvironmentSetupDiagnostic(sourceRoot, errorMessage, codeqlExePath);
   }
 
-  // Continue with a warning instead of exiting - let JavaScript extractor proceed
   logExtractorStop(
     false,
     'Warning: Environment setup failed, continuing with limited functionality',
   );
 } else {
-  // Force this script, and any process it spawns, to use the project (source) root
-  // directory as the current working directory.
   process.chdir(sourceRoot);
 }
 
@@ -91,17 +80,9 @@ cdsExtractorLog(
   `CodeQL CDS extractor using autobuild mode for scan of project source root directory '${sourceRoot}'.`,
 );
 
+// Build CDS project dependency graph
 cdsExtractorLog('info', 'Building CDS project dependency graph...');
-
-// Build the CDS project `dependencyGraph` as the foundation for the extraction process.
-// This graph will contain all discovered CDS projects, their dependencies, the `.cds`
-// files discovered within each project, the expected `.cds.json` files for each project
-// and the compilation status of such `.cds.json` files.
-//
-// The `dependencyGraph` will be updated as CDS extractor phases progress, allowing for
-// a single data structure to be used for planning, execution, retries (i.e. error handling),
-// debugging, and final reporting.
-let dependencyGraph;
+let dependencyGraph: CdsDependencyGraph;
 
 try {
   logPerformanceTrackingStart('Dependency Graph Build');
@@ -113,7 +94,7 @@ try {
     `${dependencyGraph.projects.size} projects, ${dependencyGraph.statusSummary.totalCdsFiles} CDS files`,
   );
 
-  // Log details about discovered projects for debugging
+  // Log project details
   if (dependencyGraph.projects.size > 0) {
     for (const [projectDir, project] of dependencyGraph.projects.entries()) {
       cdsExtractorLog(
@@ -122,11 +103,12 @@ try {
       );
     }
   } else {
+    // No CDS projects detected - try direct file search as diagnostic
     cdsExtractorLog(
       'error',
       'No CDS projects were detected. This is an unrecoverable error as there is nothing to scan.',
     );
-    // Let's also try to find CDS files directly as a backup check
+
     try {
       const allCdsFiles = Array.from(
         new Set([
@@ -139,6 +121,7 @@ try {
         'info',
         `Direct search found ${allCdsFiles.length} CDS files in the source tree.`,
       );
+
       if (allCdsFiles.length > 0) {
         cdsExtractorLog(
           'info',
@@ -158,160 +141,96 @@ try {
       cdsExtractorLog('warn', `Could not perform direct CDS file search: ${String(globError)}`);
     }
 
-    // Add diagnostic warning for no CDS projects detected
     const warningMessage =
       'No CDS projects were detected. This may be expected if the source does not contain CAP/CDS projects.';
     if (codeqlExePath) {
       addNoCdsProjectsDiagnostic(sourceRoot, warningMessage, codeqlExePath);
     }
 
-    // Continue instead of exiting - let JavaScript extractor proceed with non-CDS files
     logExtractorStop(false, 'Warning: No CDS projects detected, skipping CDS-specific processing');
-
-    // Skip the rest of CDS processing and go directly to JavaScript extraction
-    configureLgtmIndexFilters();
-
-    // Run CodeQL's JavaScript extractor to process any remaining files
-    logPerformanceTrackingStart('JavaScript Extraction');
-    const extractorResult = runJavaScriptExtractor(
+    handleEarlyExit(
       sourceRoot,
-      autobuildScriptPath || '', // Use empty string if autobuildScriptPath is undefined
+      autobuildScriptPath || '',
       codeqlExePath,
+      'JavaScript extraction completed (CDS processing was skipped)',
     );
-    logPerformanceTrackingStop('JavaScript Extraction');
-
-    if (!extractorResult.success && extractorResult.error) {
-      cdsExtractorLog('error', `Error running JavaScript extractor: ${extractorResult.error}`);
-      if (codeqlExePath) {
-        addJavaScriptExtractorDiagnostic(
-          sourceRoot,
-          extractorResult.error,
-          codeqlExePath,
-          sourceRoot,
-        );
-      }
-      logExtractorStop(false, 'JavaScript extractor failed');
-    } else {
-      logExtractorStop(true, 'JavaScript extraction completed (CDS processing was skipped)');
-    }
-
-    console.log(`Completed run of the cds-extractor.js script for the CDS extractor.`);
-    process.exit(0); // Graceful exit to skip the rest of the processing
   }
 } catch (error) {
   const errorMessage = `Failed to build CDS dependency graph: ${String(error)}`;
   cdsExtractorLog('error', errorMessage);
 
-  // Add diagnostic for dependency graph build failure
   if (codeqlExePath) {
     addDependencyGraphDiagnostic(sourceRoot, errorMessage, codeqlExePath);
   }
 
-  // Continue with a warning instead of exiting - let JavaScript extractor proceed with non-CDS files
   logExtractorStop(
     false,
     'Warning: Dependency graph build failed, skipping CDS-specific processing',
   );
-
-  // Skip the rest of CDS processing and go directly to JavaScript extraction
-  configureLgtmIndexFilters();
-
-  // Run CodeQL's JavaScript extractor to process any remaining files
-  logPerformanceTrackingStart('JavaScript Extraction');
-  const extractorResult = runJavaScriptExtractor(
+  handleEarlyExit(
     sourceRoot,
-    autobuildScriptPath || '', // Use empty string if autobuildScriptPath is undefined
+    autobuildScriptPath || '',
     codeqlExePath,
+    'JavaScript extraction completed (CDS processing was skipped)',
   );
-  logPerformanceTrackingStop('JavaScript Extraction');
-
-  if (!extractorResult.success && extractorResult.error) {
-    cdsExtractorLog('error', `Error running JavaScript extractor: ${extractorResult.error}`);
-    if (codeqlExePath) {
-      addJavaScriptExtractorDiagnostic(
-        sourceRoot,
-        extractorResult.error,
-        codeqlExePath,
-        sourceRoot,
-      );
-    }
-    logExtractorStop(false, 'JavaScript extractor failed');
-  } else {
-    logExtractorStop(true, 'JavaScript extraction completed (CDS processing was skipped)');
-  }
-
-  console.log(`Completed run of the cds-extractor.js script for the CDS extractor.`);
-  process.exit(0); // Graceful exit to skip the rest of the processing
 }
 
+// Install dependencies
 logPerformanceTrackingStart('Dependency Installation');
 const projectCacheDirMap = cacheInstallDependencies(dependencyGraph, sourceRoot, codeqlExePath);
 logPerformanceTrackingStop('Dependency Installation');
 
-// Check if dependency installation resulted in any usable project mappings
 if (projectCacheDirMap.size === 0) {
   cdsExtractorLog(
     'error',
     'No project cache directory mappings were created. This indicates that dependency installation failed for all discovered projects.',
   );
 
-  // This is a critical error if we have projects but no cache mappings
   if (dependencyGraph.projects.size > 0) {
     const errorMessage = `Found ${dependencyGraph.projects.size} CDS projects but failed to install dependencies for any of them. Cannot proceed with compilation.`;
     cdsExtractorLog('error', errorMessage);
 
-    // Add diagnostic for dependency installation failure
     if (codeqlExePath) {
       addDependencyInstallationDiagnostic(sourceRoot, errorMessage, codeqlExePath);
     }
 
-    // Continue with a warning instead of exiting - let JavaScript extractor proceed
     logExtractorStop(
       false,
       'Warning: Dependency installation failed for all projects, continuing with limited functionality',
     );
   }
 
-  // If we have no projects and no cache mappings, this should have been caught earlier
   cdsExtractorLog(
     'warn',
     'No projects and no cache mappings - this should have been detected earlier.',
   );
 }
 
+// Collect all CDS files to process
 const cdsFilePathsToProcess: string[] = [];
-
-// Use the dependency graph to collect all `.cds` files from each project.
-// We want to "extract" all `.cds` files from all projects so that we have a copy
-// of each `.cds` source file in the CodeQL database.
 for (const project of dependencyGraph.projects.values()) {
   cdsFilePathsToProcess.push(...project.cdsFiles);
 }
 
-// TODO : Improve logging / debugging of dependencyGraph.statusSummary. Just log the JSON?
 cdsExtractorLog(
   'info',
   `Found ${cdsFilePathsToProcess.length} total CDS files, ${dependencyGraph.statusSummary.totalCdsFiles} CDS files in dependency graph`,
 );
 
+// Compile CDS files
 logPerformanceTrackingStart('CDS Compilation');
 try {
-  // Use the new orchestrated compilation approach (autobuild mode, no debug)
   orchestrateCompilation(dependencyGraph, projectCacheDirMap, codeqlExePath);
 
-  // Handle compilation failures for normal mode
   if (!dependencyGraph.statusSummary.overallSuccess) {
     cdsExtractorLog(
       'error',
       `Compilation completed with failures: ${dependencyGraph.statusSummary.failedCompilations} failed out of ${dependencyGraph.statusSummary.totalCompilationTasks} total tasks`,
     );
 
-    // Add diagnostics for critical errors
     for (const error of dependencyGraph.errors.critical) {
       cdsExtractorLog('error', `Critical error in ${error.phase}: ${error.message}`);
     }
-
-    // Don't exit with error - let the JavaScript extractor run on whatever was compiled
   }
 
   logPerformanceTrackingStop('CDS Compilation');
@@ -320,67 +239,32 @@ try {
   logPerformanceTrackingStop('CDS Compilation');
   cdsExtractorLog('error', `Compilation orchestration failed: ${String(error)}`);
 
-  // Add diagnostic for the overall failure
   if (cdsFilePathsToProcess.length > 0) {
     addCompilationDiagnostic(
-      cdsFilePathsToProcess[0], // Use first file as representative
+      cdsFilePathsToProcess[0],
       `Compilation orchestration failed: ${String(error)}`,
       codeqlExePath,
       sourceRoot,
     );
   }
 }
 
-// Configure the "LGTM" index filters for proper extraction.
-configureLgtmIndexFilters();
-
-// Run CodeQL's JavaScript extractor to process the .cds source files and
-// the compiled .cds.json files.
-logPerformanceTrackingStart('JavaScript Extraction');
-const extractionStartTime = Date.now();
-const extractorResult = runJavaScriptExtractor(sourceRoot, autobuildScriptPath, codeqlExePath);
-const extractionEndTime = Date.now();
-logPerformanceTrackingStop('JavaScript Extraction');
-
-// Update the dependency graph's performance metrics with the extraction duration
-dependencyGraph.statusSummary.performance.extractionDurationMs =
-  extractionEndTime - extractionStartTime;
-
-// Calculate total duration by summing all phases
-const totalDuration =
-  dependencyGraph.statusSummary.performance.parsingDurationMs +
-  dependencyGraph.statusSummary.performance.compilationDurationMs +
-  dependencyGraph.statusSummary.performance.extractionDurationMs;
-dependencyGraph.statusSummary.performance.totalDurationMs = totalDuration;
-
-if (!extractorResult.success && extractorResult.error) {
-  cdsExtractorLog('error', `Error running JavaScript extractor: ${extractorResult.error}`);
-
-  // Add diagnostic for JavaScript extractor failure
-  if (codeqlExePath && dependencyGraph.projects.size > 0) {
-    // Use the first CDS file as a representative file for the diagnostic
-    const firstProject = Array.from(dependencyGraph.projects.values())[0];
-    const representativeFile = firstProject.cdsFiles[0] || sourceRoot;
-    addJavaScriptExtractorDiagnostic(
-      representativeFile,
-      extractorResult.error,
-      codeqlExePath,
-      sourceRoot,
-    );
-  }
+// Run JavaScript extraction with marker file handling
+const extractionSuccess = runJavaScriptExtractionWithMarker(
+  sourceRoot,
+  autobuildScriptPath,
+  codeqlExePath,
+  dependencyGraph,
+);
 
-  logExtractorStop(false, 'JavaScript extractor failed');
-} else {
-  logExtractorStop(true, 'CDS extraction completed successfully');
-}
+logExtractorStop(
+  extractionSuccess,
+  extractionSuccess ? 'CDS extraction completed successfully' : 'JavaScript extractor failed',
+);
 
 cdsExtractorLog(
   'info',
   'CDS Extractor Status Report : Final...\n' + generateStatusReport(dependencyGraph),
 );
 
-// Use the `cds-extractor.js` name in the log message as that is the name of the script
-// that is actually run by the `codeql database index-files` command. This TypeScript
-// file is where the code/logic is edited/implemented, but the runnable script is
-// generated by the TypeScript compiler and is named `cds-extractor.js`.
 console.log(`Completed run of the cds-extractor.js script for the CDS extractor.`);