Skip to content

Commit 27743ba

Browse files
data-douserdata-douser
committed
Simplify CDS extractor logic and refactor
Refactors cds extractor `src/cds/compiler` and `src/cds/parser` packages for improved maintainability. Simplifies the main logic of the CDS extractor such that we always build a graph that maps CDS projects to their imports / dependencies, which is part of the longer process of deprecating the "index-files" run mode of the CDS extractor (in favor of autobuild, eventually). Attempts to fix CDS file and project parsing for test projects such as: `javascript/frameworks/cap/test/queries/loginjection/log-injection-without-protocol-none`
1 parent bc4a2cd commit 27743ba

File tree

19 files changed

+681
-718
lines changed

19 files changed

+681
-718
lines changed

‎extractors/cds/tools/cds-extractor.ts‎

Lines changed: 15 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -8,9 +8,9 @@ import { CdsProjectMapWithDebugSignals } from './src/cds/parser/types';
88
import { runJavaScriptExtractor } from './src/codeql';
99
import { addCompilationDiagnostic } from './src/diagnostics';
1010
import { configureLgtmIndexFilters, setupAndValidateEnvironment } from './src/environment';
11-
import { handleIndexFilesMode } from './src/indexFiles';
1211
import { installDependencies } from './src/packageManager';
13-
import { RunMode, validateArguments } from './src/utils';
12+
import { RunMode } from './src/runMode';
13+
import { validateArguments } from './src/utils';
1414

1515
// Validate arguments to this script.
1616
// The first argument we pass is the expected run mode, which will be extracted from process.argv[2]
@@ -23,7 +23,7 @@ if (!validationResult.isValid) {
2323
}
2424

2525
// Get the validated and sanitized arguments
26-
const { runMode, sourceRoot, responseFile } = validationResult.args!;
26+
const { runMode, sourceRoot } = validationResult.args!;
2727

2828
// Check for autobuild mode
2929
if (runMode === (RunMode.AUTOBUILD as string)) {
@@ -79,36 +79,26 @@ if (typedProjectMap.__debugParserSuccess) {
7979
process.exit(1);
8080
}
8181

82-
// Install dependencies using the new caching approach
83-
console.log('Installing required CDS compiler versions using cached approach...');
82+
// Install dependencies of discovered CAP/CDS projects
83+
console.log('Ensuring depencencies are installed in cache for required CDS compiler versions...');
8484
const projectCacheDirMap = installDependencies(projectMap, sourceRoot, codeqlExePath);
8585

86-
// Handle run mode specific logic
87-
let cdsFilePathsToProcess: string[] = [];
86+
const cdsFilePathsToProcess: string[] = [];
8887

89-
if (runMode === (RunMode.INDEX_FILES as string)) {
90-
// Use the dedicated function for INDEX_FILES mode
91-
const indexFilesResult = handleIndexFilesMode(projectMap, sourceRoot, responseFile, platformInfo);
88+
console.log('Extracting CDS files from discovered projects...');
9289

93-
if (!indexFilesResult.validationResult.success) {
94-
console.warn(indexFilesResult.validationResult.errorMessage);
95-
// Exit with an error if unable to read the response file
96-
process.exit(1);
97-
}
98-
99-
cdsFilePathsToProcess = indexFilesResult.cdsFilePathsToProcess;
100-
} else {
101-
console.log('Extracting CDS files from discovered projects...');
102-
103-
// Extract all CDS files from the discovered projects for other run modes
104-
for (const [, project] of projectMap.entries()) {
105-
cdsFilePathsToProcess.push(...project.cdsFiles);
106-
}
90+
// Use the project map to collect all `.cds` files from each project.
91+
// We want to "extract" all `.cds` files from all projects so that we have a copy
92+
// of each `.cds` source file in the CodeQL database.
93+
for (const [, project] of projectMap.entries()) {
94+
cdsFilePathsToProcess.push(...project.cdsFiles);
10795
}
10896

10997
console.log('Processing CDS files to JSON ...');
11098

111-
// Compile each `.cds` file to create a `.cds.json` file.
99+
// Evaluate each `.cds` source file to determine if it should be compiled to JSON.
100+
// TODO : use project.cdsFilesToCompile instead of cdsFiles, to avoid compiling files
101+
// that are already imported by other files in the same project.
112102
for (const rawCdsFilePath of cdsFilePathsToProcess) {
113103
try {
114104
// Find which project this CDS file belongs to, to use the correct cache directory

‎extractors/cds/tools/src/cds/compiler/command.ts‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
import { execFileSync } from 'child_process';
2+
import { join } from 'path';
3+
4+
import { fileExists } from '../../filesystem';
5+
6+
/**
7+
* Determine the `cds` command to use based on the environment and cache directory.
8+
* @param cacheDir Optional path to a directory containing installed dependencies
9+
* @returns A string representing the CLI command to run to invoke the
10+
* CDS compiler.
11+
*/
12+
export function determineCdsCommand(cacheDir?: string): string {
13+
// If we have a cache directory, use the cds binary from there
14+
if (cacheDir) {
15+
const localCdsBin = join(cacheDir, 'node_modules', '.bin', 'cds');
16+
17+
// Check if the local cds binary exists in the cache directory
18+
if (fileExists(localCdsBin)) {
19+
// We need to use node to execute the local bin directly to ensure correct resolution
20+
return `node "${localCdsBin}"`;
21+
}
22+
23+
// If there's a cache directory but no local binary, use npx with NODE_PATH
24+
return `npx --no-install cds`;
25+
}
26+
27+
// Default behavior when no cache directory is provided
28+
let cdsCommand = 'cds';
29+
try {
30+
execFileSync('cds', ['--version'], { stdio: 'ignore' });
31+
} catch {
32+
// If 'cds' command is not available, use npx to run it
33+
cdsCommand = 'npx -y --package @sap/cds-dk cds';
34+
}
35+
return cdsCommand;
36+
}

‎extractors/cds/tools/src/cds/compiler/functions.ts‎ renamed to ‎extractors/cds/tools/src/cds/compiler/compile.ts‎

Lines changed: 2 additions & 78 deletions
Original file line numberDiff line numberDiff line change
@@ -1,86 +1,10 @@
1-
import { execFileSync, spawnSync, SpawnSyncOptions } from 'child_process';
1+
import { spawnSync, SpawnSyncOptions } from 'child_process';
22
import { resolve, join, delimiter, relative } from 'path';
33

44
import { CdsCompilationResult } from './types';
5+
import { getCdsVersion } from './version';
56
import { fileExists, dirExists, recursivelyRenameJsonFiles } from '../../filesystem';
67

7-
/**
8-
* Determine the `cds` command to use based on the environment and cache directory.
9-
* @param cacheDir Optional path to a directory containing installed dependencies
10-
* @returns A string representing the CLI command to run to invoke the
11-
* CDS compiler.
12-
*/
13-
export function determineCdsCommand(cacheDir?: string): string {
14-
// If we have a cache directory, use the cds binary from there
15-
if (cacheDir) {
16-
const localCdsBin = join(cacheDir, 'node_modules', '.bin', 'cds');
17-
18-
// Check if the local cds binary exists in the cache directory
19-
if (fileExists(localCdsBin)) {
20-
// We need to use node to execute the local bin directly to ensure correct resolution
21-
return `node "${localCdsBin}"`;
22-
}
23-
24-
// If there's a cache directory but no local binary, use npx with NODE_PATH
25-
return `npx --no-install cds`;
26-
}
27-
28-
// Default behavior when no cache directory is provided
29-
let cdsCommand = 'cds';
30-
try {
31-
execFileSync('cds', ['--version'], { stdio: 'ignore' });
32-
} catch {
33-
// If 'cds' command is not available, use npx to run it
34-
cdsCommand = 'npx -y --package @sap/cds-dk cds';
35-
}
36-
return cdsCommand;
37-
}
38-
39-
/**
40-
* Get the CDS compiler version from a specific command or cache directory
41-
* @param cdsCommand The CDS command to use
42-
* @param cacheDir Optional path to a directory containing installed dependencies
43-
* @returns The CDS compiler version string, or undefined if it couldn't be determined
44-
*/
45-
export function getCdsVersion(cdsCommand: string, cacheDir?: string): string | undefined {
46-
try {
47-
// Set up environment vars if using a cache directory
48-
const spawnOptions: SpawnSyncOptions = {
49-
shell: true,
50-
stdio: 'pipe',
51-
env: { ...process.env },
52-
};
53-
54-
// If a cache directory is provided, set NODE_PATH to use that cache
55-
if (cacheDir) {
56-
const nodePath = join(cacheDir, 'node_modules');
57-
58-
// Set up environment to use the cached dependencies
59-
spawnOptions.env = {
60-
...process.env,
61-
NODE_PATH: `${nodePath}${delimiter}${process.env.NODE_PATH ?? ''}`,
62-
PATH: `${join(nodePath, '.bin')}${delimiter}${process.env.PATH}`,
63-
npm_config_prefix: cacheDir,
64-
};
65-
}
66-
67-
// Execute the CDS command with the --version flag
68-
const result = spawnSync(cdsCommand, ['--version'], spawnOptions);
69-
if (result.status === 0 && result.stdout) {
70-
const versionOutput = result.stdout.toString().trim();
71-
// Extract version number, which is typically in formats like "@sap/cds: 6.1.3" or similar
72-
const match = versionOutput.match(/@sap\/cds[^0-9]*([0-9]+\.[0-9]+\.[0-9]+)/);
73-
if (match?.[1]) {
74-
return match[1]; // Return just the version number
75-
}
76-
return versionOutput; // Return full output if we couldn't parse it
77-
}
78-
return undefined;
79-
} catch {
80-
return undefined;
81-
}
82-
}
83-
848
/**
859
* Compile a CDS file to JSON
8610
* @param cdsFilePath Path to the CDS file

‎extractors/cds/tools/src/cds/compiler/index.ts‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,5 @@
1-
export * from './functions';
1+
export { determineCdsCommand } from './command';
2+
export { compileCdsToJson } from './compile';
3+
export { findProjectForCdsFile } from './project';
24
export * from './types';
3-
export * from './projectMapping';
5+
export { getCdsVersion } from './version';

‎extractors/cds/tools/src/cds/compiler/projectMapping.ts‎ renamed to ‎extractors/cds/tools/src/cds/compiler/project.ts‎

File renamed without changes.

‎extractors/cds/tools/src/cds/compiler/version.ts‎

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
import { spawnSync, SpawnSyncOptions } from 'child_process';
2+
import { join, delimiter } from 'path';
3+
4+
/**
5+
* Get the CDS compiler version from a specific command or cache directory
6+
* @param cdsCommand The CDS command to use
7+
* @param cacheDir Optional path to a directory containing installed dependencies
8+
* @returns The CDS compiler version string, or undefined if it couldn't be determined
9+
*/
10+
export function getCdsVersion(cdsCommand: string, cacheDir?: string): string | undefined {
11+
try {
12+
// Set up environment vars if using a cache directory
13+
const spawnOptions: SpawnSyncOptions = {
14+
shell: true,
15+
stdio: 'pipe',
16+
env: { ...process.env },
17+
};
18+
19+
// If a cache directory is provided, set NODE_PATH to use that cache
20+
if (cacheDir) {
21+
const nodePath = join(cacheDir, 'node_modules');
22+
23+
// Set up environment to use the cached dependencies
24+
spawnOptions.env = {
25+
...process.env,
26+
NODE_PATH: `${nodePath}${delimiter}${process.env.NODE_PATH ?? ''}`,
27+
PATH: `${join(nodePath, '.bin')}${delimiter}${process.env.PATH}`,
28+
npm_config_prefix: cacheDir,
29+
};
30+
}
31+
32+
// Execute the CDS command with the --version flag
33+
const result = spawnSync(cdsCommand, ['--version'], spawnOptions);
34+
if (result.status === 0 && result.stdout) {
35+
const versionOutput = result.stdout.toString().trim();
36+
// Extract version number, which is typically in formats like "@sap/cds: 6.1.3" or similar
37+
const match = versionOutput.match(/@sap\/cds[^0-9]*([0-9]+\.[0-9]+\.[0-9]+)/);
38+
if (match?.[1]) {
39+
return match[1]; // Return just the version number
40+
}
41+
return versionOutput; // Return full output if we couldn't parse it
42+
}
43+
return undefined;
44+
} catch {
45+
return undefined;
46+
}
47+
}

‎extractors/cds/tools/src/cds/parser/debug.ts‎

Lines changed: 6 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -94,12 +94,11 @@ export function writeParserDebugInfo(
9494
}
9595
});
9696

97-
// Check if we have meaningful content to write
98-
const debugContent = outputLines.join('\n');
99-
if (!debugContent.trim()) {
100-
console.warn('No debug information to write. Empty project map or no CDS projects found.');
101-
return false;
102-
}
97+
// Add the stringified JSON representation of the project map at the end
98+
// of the debug content.
99+
outputLines.push('\nProject Map JSON:');
100+
outputLines.push('===================');
101+
outputLines.push(JSON.stringify(Array.from(projectMap.entries()), null, 2));
103102

104103
const debugFilePath = join(scriptDir, PARSER_DEBUG_SUBDIR, PARSER_DEBUG_FILE);
105104

@@ -111,7 +110,7 @@ export function writeParserDebugInfo(
111110

112111
// Intentionally add a newline at the end of the file to make it easier to
113112
// cat/read when debugging.
114-
writeFileSync(debugFilePath, `${debugContent}\n`, 'utf-8');
113+
writeFileSync(debugFilePath, `${outputLines.join('\n')}\n`, 'utf-8');
115114

116115
console.log(`INFO: CDS extractor parser debug information written to: ${debugFilePath}`);
117116
return true;

0 commit comments

Comments
 (0)