Fixes CDS extractor project-aware file detection

Fixes detection of .cds file in CDS projects by ensuring that
"node_modules" subdirectories are explicitly ignored and "srv" and "db"
subdirectories are explicitly included.

Migrates some logic from cds-extractor.ts (entrypoint) script to
testable functions under extractors/cds/tools/src/ directory.

Adds and improves unit tests related to code changes from this commit.

Author: data-douser

extractors/cds/tools/cds-extractor.ts

@@ -8,7 +8,7 @@ import { CdsProjectMapWithDebugSignals } from './src/cds/parser/types';
 import { runJavaScriptExtractor } from './src/codeql';
 import { addCompilationDiagnostic } from './src/diagnostics';
 import { configureLgtmIndexFilters, setupAndValidateEnvironment } from './src/environment';
-import { getCdsFilePathsToProcess } from './src/filesystem';
+import { handleIndexFilesMode } from './src/indexFiles';
 import { installDependencies } from './src/packageManager';
 import { RunMode, validateArguments } from './src/utils';
 
@@ -60,21 +60,6 @@ console.log(
   `INFO: CodeQL CDS extractor using run mode '${runMode}' for scan of project source root directory '${sourceRoot}'.`,
 );
 
-// Only process response file for INDEX_FILES mode
-let cdsFilePathsToProcess: string[] = [];
-if (runMode === (RunMode.INDEX_FILES as string)) {
-  // Validate response file and get the full paths of CDS files to process.
-  const filePathsResult = getCdsFilePathsToProcess(responseFile, platformInfo);
-  if (!filePathsResult.success) {
-    console.warn(filePathsResult.errorMessage);
-    // Exit with an error if unable to get a list of `.cds` file paths to process.
-    process.exit(1);
-  }
-
-  // Get the validated list of CDS files to process
-  cdsFilePathsToProcess = filePathsResult.cdsFilePaths;
-}
-
 // Using the new project-aware approach to find CDS projects and their dependencies
 console.log('Detecting CDS projects and analyzing their structure...');
 
@@ -98,6 +83,29 @@ if (typedProjectMap.__debugParserSuccess) {
 console.log('Installing required CDS compiler versions using cached approach...');
 const projectCacheDirMap = installDependencies(projectMap, sourceRoot, codeqlExePath);
 
+// Handle run mode specific logic
+let cdsFilePathsToProcess: string[] = [];
+
+if (runMode === (RunMode.INDEX_FILES as string)) {
+  // Use the dedicated function for INDEX_FILES mode
+  const indexFilesResult = handleIndexFilesMode(projectMap, sourceRoot, responseFile, platformInfo);
+
+  if (!indexFilesResult.validationResult.success) {
+    console.warn(indexFilesResult.validationResult.errorMessage);
+    // Exit with an error if unable to read the response file
+    process.exit(1);
+  }
+
+  cdsFilePathsToProcess = indexFilesResult.cdsFilePathsToProcess;
+} else {
+  console.log('Extracting CDS files from discovered projects...');
+
+  // Extract all CDS files from the discovered projects for other run modes
+  for (const [, project] of projectMap.entries()) {
+    cdsFilePathsToProcess.push(...project.cdsFiles);
+  }
+}
+
 console.log('Processing CDS files to JSON ...');
 
 // Compile each `.cds` file to create a `.cds.json` file.

extractors/cds/tools/package.json

@@ -9,7 +9,7 @@
     "clean": "rm -rf out coverage",
     "prebuild": "npm run clean",
     "lint": "eslint --ext .ts src/",
-    "lint:fix": "eslint --ext .ts --fix cds-extractor.ts src/",
+    "lint:fix": "eslint --ext .ts --fix cds-extractor.ts src/ test/src/",
     "format": "prettier --write 'src/**/*.ts'",
     "test": "jest",
     "test:watch": "jest --watch",

extractors/cds/tools/src/cds/parser/functions.ts

@@ -197,8 +197,11 @@ export function determineCdsFilesForProjectDir(
   }
 
   try {
-    // Use glob to find all .cds files under the project directory
-    const cdsFiles = sync(join(projectDir, '**/*.cds'), { nodir: true });
+    // Use glob to find all .cds files under the project directory, excluding node_modules
+    const cdsFiles = sync(join(projectDir, '**/*.cds'), {
+      nodir: true,
+      ignore: ['**/node_modules/**'],
+    });
 
     // Convert absolute paths to paths relative to sourceRootDir
     return cdsFiles.map(file => relative(sourceRootDir, file));
@@ -223,8 +226,11 @@ export function determineCdsProjectsUnderSourceDir(sourceRootDir: string): strin
   const projectDirs: string[] = [];
   const processedDirectories = new Set<string>();
 
-  // Find all package.json files under the source directory
-  const packageJsonFiles = sync(join(sourceRootDir, '**/package.json'), { nodir: true });
+  // Find all package.json files under the source directory, excluding node_modules
+  const packageJsonFiles = sync(join(sourceRootDir, '**/package.json'), {
+    nodir: true,
+    ignore: ['**/node_modules/**'],
+  });
 
   // Check each directory containing a package.json file
   for (const packageJsonFile of packageJsonFiles) {
@@ -244,8 +250,11 @@ export function determineCdsProjectsUnderSourceDir(sourceRootDir: string): strin
     }
   }
 
-  // Also check directories that have .cds files but no package.json
-  const cdsFiles = sync(join(sourceRootDir, '**/*.cds'), { nodir: true });
+  // Also check directories that have .cds files but no package.json, excluding node_modules
+  const cdsFiles = sync(join(sourceRootDir, '**/*.cds'), {
+    nodir: true,
+    ignore: ['**/node_modules/**'],
+  });
   const cdsDirsSet = new Set(cdsFiles.map(file => dirname(file)));
   const cdsDirs = Array.from(cdsDirsSet);
 
@@ -279,6 +288,11 @@ export function determineCdsProjectsUnderSourceDir(sourceRootDir: string): strin
  * @returns true if the directory or a parent has been processed
  */
 export function isDirectoryProcessed(dir: string, processedDirectories: Set<string>): boolean {
+  // Skip node_modules directories entirely
+  if (dir.includes('node_modules')) {
+    return true; // Consider node_modules as already processed to skip them
+  }
+
   let currentDir = dir;
 
   while (currentDir) {
@@ -432,6 +446,11 @@ export function findProjectRootFromCdsFile(
   cdsFileDir: string,
   sourceRootDir: string,
 ): string | null {
+  // Skip node_modules directories entirely
+  if (cdsFileDir.includes('node_modules')) {
+    return null;
+  }
+
   let currentDir = cdsFileDir;
 
   // Limit the upward search to the sourceRootDir
@@ -442,7 +461,11 @@ export function findProjectRootFromCdsFile(
       // be the real project root containing both db and srv directories
       const parentDir = dirname(currentDir);
 
-      if (parentDir !== currentDir && parentDir.startsWith(sourceRootDir)) {
+      if (
+        parentDir !== currentDir &&
+        parentDir.startsWith(sourceRootDir) &&
+        !parentDir.includes('node_modules')
+      ) {
         const hasDbDir =
           existsSync(join(parentDir, 'db')) && statSync(join(parentDir, 'db')).isDirectory();
         const hasSrvDir =
@@ -522,6 +545,11 @@ export function getAllCdsFiles(sourceRootDir: string): { filePath: string; proje
  */
 export function isLikelyCdsProject(dir: string): boolean {
   try {
+    // Skip node_modules directories entirely
+    if (dir.includes('node_modules')) {
+      return false;
+    }
+
     // Check if package.json exists and has CAP dependencies
     const packageJsonPath = join(dir, 'package.json');
     const packageJson = readPackageJsonWithCache(packageJsonPath);
@@ -538,12 +566,13 @@ export function isLikelyCdsProject(dir: string): boolean {
       }
     }
 
-    // Check for CDS files in standard locations
+    // Check for CDS files in standard locations (checking both direct and nested files)
     const standardLocations = [join(dir, 'db'), join(dir, 'srv'), join(dir, 'app')];
 
     for (const location of standardLocations) {
       if (existsSync(location) && statSync(location).isDirectory()) {
-        const cdsFiles = sync(join(location, '*.cds'));
+        // Check for any .cds files at any level under these directories
+        const cdsFiles = sync(join(location, '**/*.cds'), { nodir: true });
         if (cdsFiles.length > 0) {
           return true;
         }

extractors/cds/tools/src/indexFiles.ts

@@ -0,0 +1,138 @@
+import { join } from 'path';
+
+import { CdsProject } from './cds/parser/types';
+import { getCdsFilePathsToProcess } from './filesystem';
+
+export interface IndexFilesValidationResult {
+  success: boolean;
+  warnings: string[];
+  errorMessage?: string;
+  discoveredCount: number;
+  responseFileCount: number;
+}
+
+/**
+ * Validates discovered CDS files against the response file for INDEX_FILES mode.
+ * This function performs backwards compatibility validation to ensure that the
+ * project-aware discovery is consistent with the legacy response file approach.
+ *
+ * @param cdsFilePathsToProcess Array of CDS file paths discovered through project-aware discovery
+ * @param sourceRoot The source root directory
+ * @param responseFile Path to the response file
+ * @param platformInfo Platform information object with isWindows property
+ * @returns Validation result with warnings and statistics
+ */
+export function validateIndexFilesMode(
+  cdsFilePathsToProcess: string[],
+  sourceRoot: string,
+  responseFile: string,
+  platformInfo: { isWindows: boolean },
+): IndexFilesValidationResult {
+  const warnings: string[] = [];
+
+  // Validate response file and get the full paths of CDS files from response file
+  const responseFileResult = getCdsFilePathsToProcess(responseFile, platformInfo);
+  if (!responseFileResult.success) {
+    return {
+      success: false,
+      warnings: [],
+      errorMessage: responseFileResult.errorMessage,
+      discoveredCount: cdsFilePathsToProcess.length,
+      responseFileCount: 0,
+    };
+  }
+
+  const responseFilePaths = responseFileResult.cdsFilePaths;
+
+  // Convert discovered files to absolute paths for comparison
+  const discoveredAbsolutePaths = cdsFilePathsToProcess.map(relativePath =>
+    relativePath.startsWith(sourceRoot) ? relativePath : join(sourceRoot, relativePath),
+  );
+
+  // Validate that discovered files are consistent with response file
+  const responseFileBasenames = responseFilePaths.map(path => path.replace(/^.*[/\\]/, ''));
+  const discoveredBasenames = discoveredAbsolutePaths.map(path => path.replace(/^.*[/\\]/, ''));
+
+  const unexpectedFiles = discoveredBasenames.filter(
+    basename => !responseFileBasenames.includes(basename),
+  );
+  const missingFiles = responseFileBasenames.filter(
+    basename => !discoveredBasenames.includes(basename),
+  );
+
+  if (unexpectedFiles.length > 0) {
+    warnings.push(`Discovered CDS files not in response file: ${unexpectedFiles.join(', ')}`);
+  }
+
+  if (missingFiles.length > 0) {
+    warnings.push(`Response file contains CDS files not discovered: ${missingFiles.join(', ')}`);
+  }
+
+  return {
+    success: true,
+    warnings,
+    discoveredCount: cdsFilePathsToProcess.length,
+    responseFileCount: responseFilePaths.length,
+  };
+}
+
+/**
+ * Handles the INDEX_FILES run mode processing.
+ * This function performs project-aware CDS file discovery and validates the results
+ * against the response file for backwards compatibility.
+ *
+ * @param projectMap Map of CDS projects discovered through project-aware parsing
+ * @param sourceRoot The source root directory
+ * @param responseFile Path to the response file
+ * @param platformInfo Platform information object with isWindows property
+ * @returns Object containing the CDS file paths to process and validation result
+ */
+export function handleIndexFilesMode(
+  projectMap: Map<string, CdsProject>,
+  sourceRoot: string,
+  responseFile: string,
+  platformInfo: { isWindows: boolean },
+): {
+  cdsFilePathsToProcess: string[];
+  validationResult: IndexFilesValidationResult;
+} {
+  console.log('Extracting CDS files from discovered projects...');
+
+  // Extract all CDS files from the discovered projects
+  const cdsFilePathsToProcess: string[] = [];
+  for (const [, project] of projectMap.entries()) {
+    cdsFilePathsToProcess.push(...project.cdsFiles);
+  }
+
+  console.log(
+    'Validating discovered CDS files against response file for backwards compatibility...',
+  );
+
+  // Validate discovered files against response file
+  const validationResult = validateIndexFilesMode(
+    cdsFilePathsToProcess,
+    sourceRoot,
+    responseFile,
+    platformInfo,
+  );
+
+  if (!validationResult.success) {
+    return {
+      cdsFilePathsToProcess: [],
+      validationResult,
+    };
+  }
+
+  // Log warnings if any
+  validationResult.warnings.forEach(warning => {
+    console.warn(`Warning: ${warning}`);
+  });
+
+  console.log(`Discovered ${validationResult.discoveredCount} CDS files from project analysis`);
+  console.log(`Response file specified ${validationResult.responseFileCount} CDS files`);
+
+  return {
+    cdsFilePathsToProcess,
+    validationResult,
+  };
+}