CDS extractor cleanup checkpoint 3

Author: data-douser

extractors/cds/tools/cds-extractor.ts

@@ -9,6 +9,7 @@ import { addCompilationDiagnostic } from './src/diagnostics';
 import { configureLgtmIndexFilters, setupAndValidateEnvironment } from './src/environment';
 import {
   cdsExtractorLog,
+  generateStatusReport,
   logExtractorStart,
   logExtractorStop,
   logPerformanceMilestone,
@@ -85,7 +86,7 @@ let dependencyGraph;
 
 try {
   logPerformanceTrackingStart('Dependency Graph Build');
-  dependencyGraph = buildCdsProjectDependencyGraph(sourceRoot, __dirname);
+  dependencyGraph = buildCdsProjectDependencyGraph(sourceRoot);
   logPerformanceTrackingStop('Dependency Graph Build');
 
   logPerformanceMilestone(
@@ -234,18 +235,36 @@ configureLgtmIndexFilters();
 // Run CodeQL's JavaScript extractor to process the .cds source files and
 // the compiled .cds.json files.
 logPerformanceTrackingStart('JavaScript Extraction');
+const extractionStartTime = Date.now();
 const extractorResult = runJavaScriptExtractor(sourceRoot, autobuildScriptPath, codeqlExePath);
+const extractionEndTime = Date.now();
 logPerformanceTrackingStop('JavaScript Extraction');
 
+// Update the dependency graph's performance metrics with the extraction duration
+dependencyGraph.statusSummary.performance.extractionDurationMs =
+  extractionEndTime - extractionStartTime;
+
+// Calculate total duration by summing all phases
+const totalDuration =
+  dependencyGraph.statusSummary.performance.parsingDurationMs +
+  dependencyGraph.statusSummary.performance.compilationDurationMs +
+  dependencyGraph.statusSummary.performance.extractionDurationMs;
+dependencyGraph.statusSummary.performance.totalDurationMs = totalDuration;
+
 if (!extractorResult.success && extractorResult.error) {
   cdsExtractorLog('error', `Error running JavaScript extractor: ${extractorResult.error}`);
   logExtractorStop(false, 'JavaScript extractor failed');
 } else {
   logExtractorStop(true, 'CDS extraction completed successfully');
 }
 
+cdsExtractorLog(
+  'info',
+  'CDS Extractor Status Report : Final...\n' + generateStatusReport(dependencyGraph),
+);
+
 // Use the `cds-extractor.js` name in the log message as that is the name of the script
 // that is actually run by the `codeql database index-files` command. This TypeScript
 // file is where the code/logic is edited/implemented, but the runnable script is
 // generated by the TypeScript compiler and is named `cds-extractor.js`.
-console.log(`Completed run of cds-extractor.js script for CDS extractor.`);
+console.log(`Completed run of the cds-extractor.js script for the CDS extractor.`);

extractors/cds/tools/src/cds/compiler/command.ts

@@ -8,21 +8,7 @@ import { fileExists } from '../../filesystem';
 import { cdsExtractorLog } from '../../logging';
 
 /**
- * Interface for detailed CDS command analysis results
- */
-export interface CdsCommandAnalysis {
-  selectedCommand: string;
-  selectedVersion?: string;
-  availableCommands: Array<{
-    strategy: string;
-    command: string;
-    version?: string;
-    error?: string;
-  }>;
-}
-
-/**
- * Cache for CDS command test results to avoid repeated testing
+ * Cache for CDS command test results to avoid running the same CLI commands repeatedly.
  */
 interface CdsCommandCache {
   /** Map of command strings to their test results */
@@ -145,94 +131,6 @@ function getBestCdsCommand(cacheDir: string | undefined, sourceRoot: string): st
   return 'npx -y --package @sap/cds-dk cds';
 }
 
-/**
- * Get detailed command analysis for debug purposes
- * This replaces the old performComprehensiveAnalysis function
- */
-export function getCommandAnalysisForDebug(
-  cacheDir: string | undefined,
-  sourceRoot: string,
-): CdsCommandAnalysis {
-  try {
-    initializeCdsCommandCache(sourceRoot);
-
-    const availableCommands: CdsCommandAnalysis['availableCommands'] = [];
-    let selectedCommand: string | null = null;
-    let selectedVersion: string | undefined;
-
-    // Test cache directory if provided
-    if (cacheDir) {
-      const localCdsBin = join(cacheDir, 'node_modules', '.bin', 'cds');
-      const result = testCdsCommand(localCdsBin, sourceRoot);
-
-      availableCommands.push({
-        strategy: 'provided-cache',
-        command: localCdsBin,
-        version: result.version,
-        error: result.error,
-      });
-
-      if (result.works && !selectedCommand) {
-        selectedCommand = localCdsBin;
-        selectedVersion = result.version;
-      }
-    }
-
-    // Test all available cache directories
-    for (const availableCacheDir of cdsCommandCache.availableCacheDirs) {
-      const localCdsBin = join(availableCacheDir, 'node_modules', '.bin', 'cds');
-      const result = testCdsCommand(localCdsBin, sourceRoot);
-
-      availableCommands.push({
-        strategy: 'cached-local',
-        command: localCdsBin,
-        version: result.version,
-        error: result.error,
-      });
-
-      if (result.works && !selectedCommand) {
-        selectedCommand = localCdsBin;
-        selectedVersion = result.version;
-      }
-    }
-
-    // Test global commands
-    const globalCommands = [
-      { command: 'cds', strategy: 'global-cds' },
-      { command: 'npx -y --package @sap/cds-dk cds', strategy: 'npx-cds-dk' },
-      { command: 'npx -y --package @sap/cds cds', strategy: 'npx-cds' },
-    ];
-
-    for (const { command, strategy } of globalCommands) {
-      const result = testCdsCommand(command, sourceRoot);
-
-      availableCommands.push({
-        strategy,
-        command,
-        version: result.version,
-        error: result.error,
-      });
-
-      if (result.works && !selectedCommand) {
-        selectedCommand = command;
-        selectedVersion = result.version;
-      }
-    }
-
-    if (!selectedCommand) {
-      throw new Error(`No working CDS command found. Tested ${availableCommands.length} options.`);
-    }
-
-    return {
-      selectedCommand,
-      selectedVersion,
-      availableCommands,
-    };
-  } catch (error) {
-    throw new Error(`Failed to analyze CDS commands: ${String(error)}`);
-  }
-}
-
 /**
  * Initialize the CDS command cache by testing global commands
  * @param sourceRoot The source root directory

extractors/cds/tools/src/cds/compiler/graph.ts

@@ -2,12 +2,10 @@ import { determineCdsCommand } from './command';
 import { compileCdsToJson } from './compile';
 import { CompilationAttempt, CompilationTask, CompilationConfig } from './types';
 import { addCompilationDiagnostic } from '../../diagnostics';
-import { cdsExtractorLog } from '../../logging';
+import { cdsExtractorLog, generateStatusReport } from '../../logging';
 import { CdsDependencyGraph, CdsProject } from '../parser/types';
 
-/**
- * Attempt compilation with a specific command and configuration
- */
+/** Attempt compilation with a specific command and configuration. */
 function attemptCompilation(
   task: CompilationTask,
   cdsCommand: string,
@@ -92,7 +90,6 @@ function createCompilationTask(
   expectedOutputFiles: string[],
   projectDir: string,
   useProjectLevelCompilation: boolean,
-  priority: number = 0,
 ): CompilationTask {
   return {
     id: `${type}_${projectDir}_${Date.now()}_${Math.random().toString(36).substr(2, 9)}`,
@@ -103,7 +100,6 @@ function createCompilationTask(
     projectDir,
     attempts: [],
     useProjectLevelCompilation,
-    priority,
     dependencies: [],
   };
 }
@@ -171,18 +167,18 @@ function executeCompilationTask(
 }
 
 /**
- * Execute all compilation tasks for the dependency graph
+ * Executes all compilation tasks for the provided {@link CdsDependencyGraph}.
+ * Uses the provided `codeqlExePath` to run the CodeQL CLI, as needed, for
+ * generating diagnositic warnings and/or errors for problems encountered while
+ * running the CodeQL CDS extractor.
  */
-export function executeCompilationTasks(
-  dependencyGraph: CdsDependencyGraph,
-  codeqlExePath: string,
-): void {
+function executeCompilationTasks(dependencyGraph: CdsDependencyGraph, codeqlExePath: string): void {
   cdsExtractorLog('info', 'Starting compilation execution for all projects...');
 
   dependencyGraph.currentPhase = 'compiling';
   const compilationStartTime = new Date();
 
-  // Collect all tasks and sort by priority
+  // Collect all compilation tasks from all projects.
   const allTasks: Array<{ task: CompilationTask; project: CdsProject }> = [];
 
   for (const project of dependencyGraph.projects.values()) {
@@ -191,10 +187,9 @@ export function executeCompilationTasks(
     }
   }
 
-  // Sort by priority (higher priority first)
-  allTasks.sort((a, b) => b.task.priority - a.task.priority);
-
-  // Execute tasks sequentially (could be parallelized in the future)
+  // Execute compilation tasks sequentially. There is room for optimization in the future.
+  // For now, we keep it simple to ensure consistent debug information collection.
+  cdsExtractorLog('info', `Executing ${allTasks.length} compilation task(s)...`);
   for (const { task, project } of allTasks) {
     try {
       executeCompilationTask(task, project, dependencyGraph, codeqlExePath);
@@ -239,79 +234,26 @@ export function executeCompilationTasks(
 }
 
 /**
- * Generate a comprehensive status report for the dependency graph
- * Supports both normal execution and debug modes
- */
-export function generateStatusReport(dependencyGraph: CdsDependencyGraph): string {
-  const summary = dependencyGraph.statusSummary;
-  const lines: string[] = [];
-
-  lines.push('='.repeat(80));
-  lines.push(`CDS EXTRACTOR STATUS REPORT`);
-  lines.push('='.repeat(80));
-  lines.push('');
-
-  // Overall summary
-  lines.push('OVERALL SUMMARY:');
-  lines.push(`  Status: ${summary.overallSuccess ? 'SUCCESS' : 'FAILED'}`);
-  lines.push(`  Current Phase: ${dependencyGraph.currentPhase.toUpperCase()}`);
-  lines.push(`  Projects: ${summary.totalProjects}`);
-  lines.push(`  CDS Files: ${summary.totalCdsFiles}`);
-  lines.push(`  JSON Files Generated: ${summary.jsonFilesGenerated}`);
-  lines.push('');
-
-  // Compilation summary
-  lines.push('COMPILATION SUMMARY:');
-  lines.push(`  Total Tasks: ${summary.totalCompilationTasks}`);
-  lines.push(`  Successful: ${summary.successfulCompilations}`);
-  lines.push(`  Failed: ${summary.failedCompilations}`);
-  lines.push(`  Skipped: ${summary.skippedCompilations}`);
-  lines.push('');
-
-  // Performance metrics
-  lines.push('PERFORMANCE:');
-  lines.push(`  Total Duration: ${summary.performance.totalDurationMs}ms`);
-  lines.push(`  Parsing: ${summary.performance.parsingDurationMs}ms`);
-  lines.push(`  Compilation: ${summary.performance.compilationDurationMs}ms`);
-  lines.push(`  Extraction: ${summary.performance.extractionDurationMs}ms`);
-  lines.push('');
-
-  // Errors and warnings
-  if (summary.criticalErrors.length > 0) {
-    lines.push('CRITICAL ERRORS:');
-    for (const error of summary.criticalErrors) {
-      lines.push(`  - ${error}`);
-    }
-    lines.push('');
-  }
-
-  if (summary.warnings.length > 0) {
-    lines.push('WARNINGS:');
-    for (const warning of summary.warnings) {
-      lines.push(`  - ${warning}`);
-    }
-    lines.push('');
-  }
-
-  lines.push('='.repeat(80));
-
-  return lines.join('\n');
-}
-
-/**
- * Main compilation orchestration function to replace the big for loop in cds-extractor.ts
- * Now supports consistent debug information collection
+ * Orchestrates the compilation process for CDS files based on a dependency graph.
+ *
+ * This function coordinates the planning and execution of compilation tasks,
+ * tracks the compilation status, and generates a post-compilation report.
+ *
+ * @param dependencyGraph - The {@link CdsDependencyGraph} representing the CDS projects,
+ * project dependencies, expected compilation tasks, and their statuses.
+ * @param projectCacheDirMap - A map from project identifiers to their cache directory paths.
+ * @param codeqlExePath - The path to the CodeQL executable. Used for generating diagnostic
+ * messages as part of the broader CodeQL (JavaScript) extraction process.
+ * @throws Will rethrow any errors encountered during compilation, after logging them.
  */
 export function orchestrateCompilation(
   dependencyGraph: CdsDependencyGraph,
   projectCacheDirMap: Map<string, string>,
   codeqlExePath: string,
 ): void {
   try {
-    // Plan compilation tasks
     planCompilationTasks(dependencyGraph, projectCacheDirMap);
 
-    // Execute compilation tasks
     executeCompilationTasks(dependencyGraph, codeqlExePath);
 
     // Update overall status
@@ -322,9 +264,9 @@ export function orchestrateCompilation(
     dependencyGraph.statusSummary.overallSuccess = !hasFailures;
     dependencyGraph.currentPhase = hasFailures ? 'failed' : 'completed';
 
-    // Generate and log status report
+    // Generate and log a "Post-Compilation" status report, aka before the JavaScript extractor runs.
     const statusReport = generateStatusReport(dependencyGraph);
-    cdsExtractorLog('info', 'Final Status Report:\n' + statusReport);
+    cdsExtractorLog('info', 'CDS Extractor Status Report : Post-Compilation...\n' + statusReport);
   } catch (error) {
     const errorMessage = `Compilation orchestration failed: ${String(error)}`;
     cdsExtractorLog('error', errorMessage);
@@ -343,10 +285,8 @@ export function orchestrateCompilation(
   }
 }
 
-/**
- * Plan compilation tasks for all projects in the dependency graph
- */
-export function planCompilationTasks(
+/** Plan compilation tasks for all projects in the dependency graph. */
+function planCompilationTasks(
   dependencyGraph: CdsDependencyGraph,
   projectCacheDirMap: Map<string, string>,
 ): void {
@@ -379,7 +319,6 @@ export function planCompilationTasks(
           project.expectedOutputFiles,
           projectDir,
           true,
-          10, // Higher priority for project-level compilation
         );
         project.compilationTasks = [task];
       } else {
@@ -393,7 +332,6 @@ export function planCompilationTasks(
             [expectedOutput],
             projectDir,
             false,
-            5, // Lower priority for individual files
           );
           tasks.push(task);
         }

extractors/cds/tools/src/cds/compiler/index.ts

@@ -1,18 +1,5 @@
-export { determineCdsCommand, getCommandAnalysisForDebug, resetCdsCommandCache } from './command';
-export type { CdsCommandAnalysis } from './command';
+export { determineCdsCommand, resetCdsCommandCache } from './command';
 export { compileCdsToJson } from './compile';
-export {
-  executeCompilationTasks,
-  generateStatusReport,
-  orchestrateCompilation,
-  planCompilationTasks,
-} from './graph';
+export { orchestrateCompilation } from './graph';
 export { findProjectForCdsFile } from './project';
-export type {
-  CdsCompilationResult,
-  CompilationAttempt,
-  CompilationStatus,
-  CompilationTask,
-  CompilationConfig,
-} from './types';
 export { getCdsVersion } from './version';