Skip to content

Commit d12155b

Browse files
elierichaeliericha
committed
Implement integration of e3-testsuite
1 parent 5dc988a commit d12155b

File tree

3 files changed

+362
-0
lines changed

3 files changed

+362
-0
lines changed

‎integration/vscode/ada/package.json‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -710,6 +710,21 @@
710710
"description": "Enable experimental features still in development."
711711
}
712712
}
713+
},
714+
{
715+
"title": "e3-testsuite",
716+
"properties": {
717+
"e3-testsuite.testsuitePath": {
718+
"type": "string",
719+
"description": "Path to testsuite.py",
720+
"default": "testsuite.py"
721+
},
722+
"e3-testsuite.python": {
723+
"type": "string",
724+
"description": "Path to python interpreter, useful when you want to use a specific venv",
725+
"default": "python"
726+
}
727+
}
713728
}
714729
],
715730
"jsonValidation": [

‎integration/vscode/ada/src/e3Testsuite.ts‎

Lines changed: 344 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,344 @@
1+
import { spawn } from 'child_process';
2+
import { existsSync, readFileSync } from 'fs';
3+
import path from 'path';
4+
import { tmpNameSync } from 'tmp';
5+
import * as vscode from 'vscode';
6+
import * as yaml from 'yaml';
7+
import { setTerminalEnvironment } from './helpers';
8+
9+
interface Testsuite {
10+
uri: vscode.Uri;
11+
python: string;
12+
}
13+
14+
interface TestInfo {
15+
test_name: string;
16+
test_env: { [key: string]: string };
17+
test_dir: string;
18+
test_matcher?: string;
19+
}
20+
21+
const TestStatuses = [
22+
'PASS',
23+
'FAIL',
24+
'XFAIL',
25+
'XPASS',
26+
'VERIFY',
27+
'SKIP',
28+
'NOT_APPLICABLE',
29+
'ERROR',
30+
] as const;
31+
32+
type TestStatus = (typeof TestStatuses)[number];
33+
34+
type IndexEntry = {
35+
test_name: string;
36+
status: TestStatus;
37+
msg?: string;
38+
filename: string;
39+
};
40+
41+
type ReportIndex = {
42+
entries: IndexEntry[];
43+
};
44+
45+
type TestResult = {
46+
log?: string;
47+
};
48+
49+
const showLoadTestListErrorCmdId = 'e3-testsuite.showLoadTestListError';
50+
let lastLoadError: string = '';
51+
52+
// This method is called when your extension is activated
53+
// Your extension is activated the very first time the command is executed
54+
export function activateE3TestsuiteIntegration(context: vscode.ExtensionContext) {
55+
context.subscriptions.push(
56+
vscode.commands.registerCommand(showLoadTestListErrorCmdId, async () => {
57+
const doc = await vscode.workspace.openTextDocument({
58+
content: lastLoadError,
59+
});
60+
await vscode.window.showTextDocument(doc);
61+
}),
62+
);
63+
64+
const controller = vscode.tests.createTestController('e3-testsuite', 'e3-testsuite');
65+
context.subscriptions.push(controller);
66+
67+
const testData: Map<vscode.TestItem, TestInfo> = new Map();
68+
69+
controller.refreshHandler = async function () {
70+
controller.items.replace([]);
71+
testData.clear();
72+
73+
const ts: Testsuite = getTestsuite();
74+
75+
if (!existsSync(ts.uri.fsPath)) {
76+
return;
77+
}
78+
79+
const rootItem = this.createTestItem(
80+
getRootItemId(),
81+
vscode.workspace.asRelativePath(ts.uri),
82+
ts.uri,
83+
);
84+
controller.items.add(rootItem);
85+
86+
const jsonFname = tmpNameSync({ postfix: '.json' });
87+
const cmd = [ts.python, ts.uri.fsPath, `--list-json=${jsonFname}`];
88+
89+
console.log(`Loading tests from: ${ts.uri.fsPath}`);
90+
const testList: TestInfo[] = await new Promise((resolve, reject) => {
91+
const output: Buffer[] = [];
92+
const fullOutput: Buffer[] = [];
93+
const p = spawn(cmd[0], [...cmd].splice(1), {
94+
cwd: vscode.workspace.workspaceFolders![0].uri.fsPath,
95+
});
96+
p.stdout.on('data', (chunk) => {
97+
// eslint-disable-next-line @typescript-eslint/no-unsafe-argument
98+
output.push(chunk);
99+
// eslint-disable-next-line @typescript-eslint/no-unsafe-argument
100+
fullOutput.push(chunk);
101+
});
102+
// eslint-disable-next-line @typescript-eslint/no-unsafe-argument
103+
p.stderr.on('data', (chunk) => fullOutput.push(chunk));
104+
p.on('close', (code) => {
105+
if (!existsSync(jsonFname)) {
106+
reject(new Error(`Expected JSON file not found: ${jsonFname}`));
107+
}
108+
109+
if (code !== 0) {
110+
lastLoadError = `Error getting test list from testsuite.py, ran: ${cmd.join(
111+
' ',
112+
)}\n${Buffer.concat(fullOutput).toString()}`;
113+
const errorItem = this.createTestItem('error', 'Error');
114+
errorItem.error = new vscode.MarkdownString(
115+
`[Failed to load test list](command:${showLoadTestListErrorCmdId})`,
116+
);
117+
errorItem.error.isTrusted = true;
118+
rootItem.children.add(errorItem);
119+
reject(new Error(lastLoadError));
120+
}
121+
122+
resolve(JSON.parse(readFileSync(jsonFname, { encoding: 'utf-8' })) as TestInfo[]);
123+
});
124+
p.on('error', reject);
125+
});
126+
127+
for (const test of testList) {
128+
const testYaml = path.join(test.test_dir, 'test.yaml');
129+
const isYamlTest = existsSync(testYaml);
130+
131+
const item = this.createTestItem(
132+
test.test_name,
133+
test.test_name,
134+
vscode.Uri.file(isYamlTest ? testYaml : test.test_dir),
135+
);
136+
item.range = isYamlTest ? new vscode.Range(0, 0, 0, 0) : undefined;
137+
testData.set(item, test);
138+
rootItem.children.add(item);
139+
}
140+
};
141+
142+
const profile = controller.createRunProfile(
143+
'e3-testsuite',
144+
vscode.TestRunProfileKind.Run,
145+
async function (request, token) {
146+
const ts = getTestsuite();
147+
const cmd = [ts.python, ts.uri.fsPath, '--failure-exit-code=0'];
148+
149+
const remainingSet: Set<vscode.TestItem> = new Set();
150+
function appendLeafs(i: vscode.TestItem) {
151+
if (i.children.size > 0) {
152+
i.children.forEach(appendLeafs);
153+
} else {
154+
remainingSet.add(i);
155+
}
156+
}
157+
158+
function onlyRootSelected(rq: vscode.TestRunRequest) {
159+
return rq.include?.length === 1 && rq.include[0].id === getRootItemId();
160+
}
161+
162+
if (request.include && !onlyRootSelected(request)) {
163+
request.include.forEach(appendLeafs);
164+
remainingSet.forEach((item) => {
165+
const data = testData.get(item)!;
166+
cmd.push(data.test_matcher ?? data.test_dir);
167+
});
168+
} else {
169+
/**
170+
* Do not append tests to the command line. Just run everything.
171+
*/
172+
controller.items.forEach(appendLeafs);
173+
}
174+
175+
const remainingArray = [...remainingSet];
176+
177+
const run = controller.createTestRun(request, 'e3-testsuite');
178+
run.appendOutput(`Running: ${cmd.map((c) => '"' + c + '"').join(' ')}\n\r`);
179+
180+
const cwd = vscode.workspace.workspaceFolders![0].uri.fsPath;
181+
await new Promise<void>((resolve, reject) => {
182+
const p = spawn(cmd[0], cmd.splice(1), {
183+
cwd: cwd,
184+
env: getEnv(),
185+
});
186+
token.onCancellationRequested(() => {
187+
p.kill();
188+
run.appendOutput('\r\n*** Test run was cancelled');
189+
remainingSet.forEach((item) =>
190+
run.errored(item, new vscode.TestMessage('Test run was cancelled.')),
191+
);
192+
resolve();
193+
});
194+
remainingSet.forEach((t) => run.started(t));
195+
function handleChunk(chunk: string | Buffer) {
196+
const decoded: string =
197+
typeof chunk === 'string' ? chunk : chunk.toLocaleString();
198+
run.appendOutput(decoded.replace(/\n/g, '\n\r'));
199+
200+
const statusLineReStr = `^INFO\\s+(${TestStatuses.join(
201+
'|',
202+
)})\\s+(([^ \\t\\n\\r:])+)(:\\s*(.*))?`;
203+
const statusLineRe = RegExp(statusLineReStr);
204+
const match = decoded.match(statusLineRe);
205+
if (match) {
206+
const status = match[1] as TestStatus;
207+
const testName = match[2];
208+
const message = match[5];
209+
if (testName) {
210+
const item = remainingArray.find((v) => v.id === testName);
211+
212+
if (item) {
213+
reportResult(
214+
status,
215+
item,
216+
message ? [new vscode.TestMessage(message)] : undefined,
217+
);
218+
remainingSet.delete(item);
219+
}
220+
}
221+
}
222+
}
223+
p.stdout.on('data', handleChunk);
224+
p.stderr.on('data', handleChunk);
225+
p.on('error', reject);
226+
p.on('close', (code) => {
227+
if (code === 0) {
228+
resolve();
229+
} else {
230+
reject(Error(''));
231+
}
232+
});
233+
});
234+
235+
const indexPath = path.join(cwd, 'out', 'new', '_index.json');
236+
if (existsSync(indexPath)) {
237+
const index = loadReportIndex(indexPath);
238+
for (const entry of index.entries) {
239+
const item = remainingArray.find((i) => i.id === entry.test_name);
240+
if (item) {
241+
const testResultPath = path.join(path.dirname(indexPath), entry.filename);
242+
243+
if (existsSync(testResultPath)) {
244+
const result = loadTestResult(testResultPath);
245+
reportResult(
246+
entry.status,
247+
item,
248+
result.log ? [new vscode.TestMessage(result.log)] : undefined,
249+
);
250+
}
251+
}
252+
}
253+
}
254+
255+
run.end();
256+
257+
function reportResult(
258+
status: string,
259+
item: vscode.TestItem,
260+
messages: vscode.TestMessage[] = [],
261+
) {
262+
switch (status) {
263+
case 'PASS':
264+
case 'XFAIL':
265+
run.passed(item);
266+
break;
267+
268+
case 'FAIL':
269+
case 'XPASS':
270+
case 'VERIFY':
271+
run.failed(item, messages);
272+
break;
273+
case 'SKIP':
274+
run.skipped(item);
275+
break;
276+
case 'NOT_APPLICABLE':
277+
case 'ERROR':
278+
run.errored(item, messages);
279+
break;
280+
}
281+
}
282+
},
283+
);
284+
285+
context.subscriptions.push(profile);
286+
287+
vscode.window.withProgress(
288+
{
289+
location: vscode.ProgressLocation.Notification,
290+
title: 'Loading e3-testsuite tests',
291+
},
292+
async (_, token) => {
293+
await controller.refreshHandler!(token);
294+
},
295+
);
296+
}
297+
298+
function getRootItemId(): string {
299+
return getTestsuite().uri.toString();
300+
}
301+
302+
function getTestsuite() {
303+
const config = vscode.workspace.getConfiguration('e3-testsuite');
304+
const tsPath = config.get<string>('testsuitePath') ?? 'testsuite.py';
305+
const python = config.get<string>('python') ?? 'python';
306+
307+
const tsAbsUri: vscode.Uri = path.isAbsolute(tsPath)
308+
? vscode.Uri.file(tsPath)
309+
: vscode.Uri.joinPath(vscode.workspace.workspaceFolders![0].uri, tsPath);
310+
311+
const ts: Testsuite = { uri: tsAbsUri, python: python };
312+
return ts;
313+
}
314+
315+
function getEnv(): NodeJS.ProcessEnv | undefined {
316+
const env = { ...process.env };
317+
setTerminalEnvironment(env);
318+
return env;
319+
}
320+
321+
function loadReportIndex(indexPath: string): ReportIndex {
322+
const indexText = readFileSync(indexPath, { encoding: 'utf-8' });
323+
const index = JSON.parse(indexText) as ReportIndex;
324+
return index;
325+
}
326+
327+
const testStatusTag: yaml.ScalarTag = {
328+
tag: '!e3.testsuite.result.TestStatus',
329+
resolve(value: string) {
330+
const intVal = Number.parseInt(value);
331+
const index = intVal - 1;
332+
return TestStatuses[index];
333+
},
334+
};
335+
const testResultTag: yaml.CollectionTag = {
336+
tag: '!e3.testsuite.result.TestResult',
337+
collection: 'map',
338+
};
339+
function loadTestResult(testResultPath: string) {
340+
const result = yaml.parse(readFileSync(testResultPath, { encoding: 'utf-8' }), {
341+
customTags: [testResultTag, testStatusTag],
342+
}) as TestResult;
343+
return result;
344+
}

‎integration/vscode/ada/src/extension.ts‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@ import {
3535
* This import gives access to the package.json content.
3636
*/
3737
import * as meta from '../package.json';
38+
import { activateE3TestsuiteIntegration } from './e3Testsuite';
3839

3940
export const EXTENSION_NAME: string = meta.displayName;
4041

@@ -182,6 +183,8 @@ async function activateExtension(context: vscode.ExtensionContext) {
182183
* This can display a dialog to the User so don't wait on the result.
183184
*/
184185
void vscode.commands.executeCommand('ada.addMissingDirsToWorkspace', true);
186+
187+
activateE3TestsuiteIntegration(context);
185188
}
186189

187190
function setUpLogging(context: vscode.ExtensionContext) {

0 commit comments

Comments
 (0)