Skip to content

Commit 9a4b2db

Browse files
plemarquandplemarquand
authored
Swift Testing Attachments Support (#1187)
* Attachments Support Support swift-testing attachments. Currently prints the attachments out as a report at the end of a test run. In order to leverage them we set the --experimental-attachment-path defined by a new swift.attachmentsPath setting. This is a directory path which can be global or scoped to the workspace. It can be relative or absolute. If relative, the absolute path is computed relative to the project workspace root. Defaults to .build/attachments. If the attachmentsPath directory doesn't exist, it is created. Each test run that produces attachments creates a subfolder named with the date/time of the test run, and places the runs attachments within it.
1 parent dd19e22 commit 9a4b2db

File tree

10 files changed

+312
-35
lines changed

10 files changed

+312
-35
lines changed

‎.gitignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ default.profraw
77
*.vsix
88
.vscode-test
99
.build
10+
.index-build
1011
.DS_Store
1112
assets/documentation-webview
1213
assets/test/**/Package.resolved

‎assets/test/defaultPackage/Tests/PackageTests/PackageTests.swift‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -42,9 +42,13 @@ final class DebugReleaseTestSuite: XCTestCase {
4242
}
4343
}
4444

45-
#if swift(>=6.0)
45+
#if swift(>=6.1)
46+
@_spi(Experimental) import Testing
47+
#elseif swift(>=6.0)
4648
import Testing
49+
#endif
4750

51+
#if swift(>=6.0)
4852
@Test func topLevelTestPassing() {
4953
print("A print statement in a test.")
5054
#if !TEST_ARGUMENT_SET_VIA_TEST_BUILD_ARGUMENTS_SETTING
@@ -98,7 +102,12 @@ struct MixedSwiftTestingSuite {
98102
}
99103
#expect(2 == 3)
100104
}
105+
#endif
101106

107+
#if swift(>=6.1)
108+
@Test func testAttachment() throws {
109+
Attachment("Hello, world!", named: "hello.txt").attach()
110+
}
102111
#endif
103112

104113
final class DuplicateSuffixTests: XCTestCase {

‎package.json‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -434,6 +434,12 @@
434434
}
435435
}
436436
}
437+
},
438+
"swift.attachmentsPath": {
439+
"type": "string",
440+
"default": ".build/attachments",
441+
"markdownDescription": "The path to a directory that will be used to store attachments produced during a test run.\n\nA relative path resolves relative to the root directory of the workspace running the test(s)",
442+
"scope": "machine-overridable"
437443
}
438444
}
439445
},

‎src/TestExplorer/TestParsers/SwiftTestingOutputParser.ts‎

Lines changed: 27 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -52,7 +52,8 @@ export type EventRecordPayload =
5252
| TestCaseEnded
5353
| IssueRecorded
5454
| TestSkipped
55-
| RunEnded;
55+
| RunEnded
56+
| ValueAttached;
5657

5758
export interface EventRecord extends VersionedRecord {
5859
kind: "event";
@@ -95,6 +96,15 @@ interface RunEnded {
9596
messages: EventMessage[];
9697
}
9798

99+
interface ValueAttached {
100+
kind: "_valueAttached";
101+
_attachment: {
102+
path?: string;
103+
};
104+
testID: string;
105+
messages: EventMessage[];
106+
}
107+
98108
interface Instant {
99109
absolute: number;
100110
since1970: number;
@@ -148,6 +158,7 @@ export enum TestSymbol {
148158
difference = "difference",
149159
warning = "warning",
150160
details = "details",
161+
attachment = "attachment",
151162
none = "none",
152163
}
153164

@@ -169,7 +180,8 @@ export class SwiftTestingOutputParser {
169180

170181
constructor(
171182
public testRunStarted: () => void,
172-
public addParameterizedTestCase: (testClass: TestClass, parentIndex: number) => void
183+
public addParameterizedTestCase: (testClass: TestClass, parentIndex: number) => void,
184+
public onAttachment: (testIndex: number, path: string) => void
173185
) {}
174186

175187
/**
@@ -457,6 +469,12 @@ export class SwiftTestingOutputParser {
457469
this.completionMap.set(testIndex, true);
458470
runState.completed(testIndex, { timestamp: item.payload.instant.absolute });
459471
return;
472+
} else if (item.payload.kind === "_valueAttached" && item.payload._attachment.path) {
473+
const testID = this.idFromOptionalTestCase(item.payload.testID);
474+
const testIndex = this.getTestCaseIndex(runState, testID);
475+
476+
this.onAttachment(testIndex, item.payload._attachment.path);
477+
return;
460478
}
461479
}
462480
}
@@ -495,7 +513,7 @@ export class SymbolRenderer {
495513
* @param message An event message, typically found on an `EventRecordPayload`.
496514
* @returns A string colorized with ANSI escape codes.
497515
*/
498-
static eventMessageSymbol(symbol: TestSymbol): string {
516+
public static eventMessageSymbol(symbol: TestSymbol): string {
499517
return this.colorize(symbol, this.symbol(symbol));
500518
}
501519

@@ -521,6 +539,8 @@ export class SymbolRenderer {
521539
return "\u{25B2}"; // Unicode: BLACK UP-POINTING TRIANGLE
522540
case TestSymbol.details:
523541
return "\u{2192}"; // Unicode: RIGHTWARDS ARROW
542+
case TestSymbol.attachment:
543+
return "\u{2399}"; // Unicode: PRINT SCREEN SYMBOL
524544
case TestSymbol.none:
525545
return "";
526546
}
@@ -540,6 +560,8 @@ export class SymbolRenderer {
540560
return "\u{26A0}\u{FE0E}"; // Unicode: WARNING SIGN + VARIATION SELECTOR-15 (disable emoji)
541561
case TestSymbol.details:
542562
return "\u{21B3}"; // Unicode: DOWNWARDS ARROW WITH TIP RIGHTWARDS
563+
case TestSymbol.attachment:
564+
return "\u{2399}"; // Unicode: PRINT SCREEN SYMBOL
543565
case TestSymbol.none:
544566
return " ";
545567
}
@@ -562,6 +584,8 @@ export class SymbolRenderer {
562584
return `${SymbolRenderer.ansiEscapeCodePrefix}91m${symbol}${SymbolRenderer.resetANSIEscapeCode}`;
563585
case TestSymbol.warning:
564586
return `${SymbolRenderer.ansiEscapeCodePrefix}93m${symbol}${SymbolRenderer.resetANSIEscapeCode}`;
587+
case TestSymbol.attachment:
588+
return `${SymbolRenderer.ansiEscapeCodePrefix}94m${symbol}${SymbolRenderer.resetANSIEscapeCode}`;
565589
case TestSymbol.none:
566590
default:
567591
return symbol;

‎src/TestExplorer/TestRunner.ts‎

Lines changed: 90 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,11 @@ import {
2727
ParallelXCTestOutputParser,
2828
XCTestOutputParser,
2929
} from "./TestParsers/XCTestOutputParser";
30-
import { SwiftTestingOutputParser } from "./TestParsers/SwiftTestingOutputParser";
30+
import {
31+
SwiftTestingOutputParser,
32+
SymbolRenderer,
33+
TestSymbol,
34+
} from "./TestParsers/SwiftTestingOutputParser";
3135
import { LoggingDebugAdapterTracker } from "../debugger/logTracker";
3236
import { TaskOperation } from "../tasks/TaskQueue";
3337
import { TestXUnitParser } from "./TestXUnitParser";
@@ -36,7 +40,12 @@ import { TestRunArguments } from "./TestRunArguments";
3640
import { TemporaryFolder } from "../utilities/tempFolder";
3741
import { TestClass, runnableTag, upsertTestItem } from "./TestDiscovery";
3842
import { TestCoverage } from "../coverage/LcovResults";
39-
import { BuildConfigurationFactory, TestingConfigurationFactory } from "../debugger/buildConfig";
43+
import {
44+
BuildConfigurationFactory,
45+
SwiftTestingBuildAguments,
46+
SwiftTestingConfigurationSetup,
47+
TestingConfigurationFactory,
48+
} from "../debugger/buildConfig";
4049
import { TestKind, isDebugging, isRelease } from "./TestKind";
4150
import { reduceTestItemChildren } from "./TestUtils";
4251
import { CompositeCancellationToken } from "../utilities/cancellation";
@@ -67,6 +76,7 @@ export class TestRunProxy {
6776
private queuedOutput: string[] = [];
6877
private _testItems: vscode.TestItem[];
6978
private iteration: number | undefined;
79+
private attachments: { [key: string]: string[] } = {};
7080
public coverage: TestCoverage;
7181
public token: CompositeCancellationToken;
7282

@@ -177,6 +187,12 @@ export class TestRunProxy {
177187
this.addedTestItems.push({ testClass, parentIndex });
178188
};
179189

190+
public addAttachment = (testIndex: number, attachment: string) => {
191+
const attachments = this.attachments[testIndex] ?? [];
192+
attachments.push(attachment);
193+
this.attachments[testIndex] = attachments;
194+
};
195+
180196
public getTestIndex(id: string, filename?: string): number {
181197
return this.testItemFinder.getIndex(id, filename);
182198
}
@@ -231,6 +247,8 @@ export class TestRunProxy {
231247
if (!this.runStarted) {
232248
this.testRunStarted();
233249
}
250+
251+
this.reportAttachments();
234252
this.testRun?.end();
235253
this.testRunCompleteEmitter.fire();
236254
this.token.dispose();
@@ -259,6 +277,25 @@ export class TestRunProxy {
259277
}
260278
}
261279

280+
private reportAttachments() {
281+
const attachmentKeys = Object.keys(this.attachments);
282+
if (attachmentKeys.length > 0) {
283+
let attachment = "";
284+
const totalAttachments = attachmentKeys.reduce((acc, key) => {
285+
const attachments = this.attachments[key];
286+
attachment = attachments.length ? attachments[0] : attachment;
287+
return acc + attachments.length;
288+
}, 0);
289+
290+
if (attachment) {
291+
attachment = path.dirname(attachment);
292+
this.appendOutput(
293+
`${SymbolRenderer.eventMessageSymbol(TestSymbol.attachment)} ${SymbolRenderer.ansiEscapeCodePrefix}90mRecorded ${totalAttachments} attachment${totalAttachments === 1 ? "" : "s"} to ${attachment}${SymbolRenderer.resetANSIEscapeCode}`
294+
);
295+
}
296+
}
297+
}
298+
262299
private performAppendOutput(
263300
testRun: vscode.TestRun,
264301
output: string,
@@ -321,7 +358,8 @@ export class TestRunner {
321358
: new XCTestOutputParser();
322359
this.swiftTestOutputParser = new SwiftTestingOutputParser(
323360
this.testRun.testRunStarted,
324-
this.testRun.addParameterizedTestCase
361+
this.testRun.addParameterizedTestCase,
362+
this.testRun.addAttachment
325363
);
326364
}
327365

@@ -334,7 +372,8 @@ export class TestRunner {
334372
// The SwiftTestingOutputParser holds state and needs to be reset between iterations.
335373
this.swiftTestOutputParser = new SwiftTestingOutputParser(
336374
this.testRun.testRunStarted,
337-
this.testRun.addParameterizedTestCase
375+
this.testRun.addParameterizedTestCase,
376+
this.testRun.addAttachment
338377
);
339378
this.testRun.setIteration(iteration);
340379
}
@@ -519,18 +558,28 @@ export class TestRunner {
519558
// Run swift-testing first, then XCTest.
520559
// swift-testing being parallel by default should help these run faster.
521560
if (this.testArgs.hasSwiftTestingTests) {
522-
const fifoPipePath = this.generateFifoPipePath();
561+
const testRunTime = Date.now();
562+
const fifoPipePath = this.generateFifoPipePath(testRunTime);
523563

524-
await TemporaryFolder.withNamedTemporaryFile(fifoPipePath, async () => {
564+
await TemporaryFolder.withNamedTemporaryFiles([fifoPipePath], async () => {
525565
// macOS/Linux require us to create the named pipe before we use it.
526566
// Windows just lets us communicate by specifying a pipe path without any ceremony.
527567
if (process.platform !== "win32") {
528568
await execFile("mkfifo", [fifoPipePath], undefined, this.folderContext);
529569
}
530-
531-
const testBuildConfig = await TestingConfigurationFactory.swiftTestingConfig(
570+
// Create the swift-testing configuration JSON file, peparing any
571+
// directories the configuration may require.
572+
const attachmentFolder = await SwiftTestingConfigurationSetup.setupAttachmentFolder(
532573
this.folderContext,
574+
testRunTime
575+
);
576+
const swiftTestingArgs = await SwiftTestingBuildAguments.build(
533577
fifoPipePath,
578+
attachmentFolder
579+
);
580+
const testBuildConfig = await TestingConfigurationFactory.swiftTestingConfig(
581+
this.folderContext,
582+
swiftTestingArgs,
534583
this.testKind,
535584
this.testArgs.swiftTestArgs,
536585
true
@@ -540,17 +589,25 @@ export class TestRunner {
540589
return this.testRun.runState;
541590
}
542591

592+
const outputStream = this.testOutputWritable(TestLibrary.swiftTesting, runState);
593+
543594
// Watch the pipe for JSONL output and parse the events into test explorer updates.
544595
// The await simply waits for the watching to be configured.
545596
await this.swiftTestOutputParser.watch(fifoPipePath, runState);
546597

547598
await this.launchTests(
548599
runState,
549600
this.testKind === TestKind.parallel ? TestKind.standard : this.testKind,
550-
this.testOutputWritable(TestLibrary.swiftTesting, runState),
601+
outputStream,
551602
testBuildConfig,
552603
TestLibrary.swiftTesting
553604
);
605+
606+
await SwiftTestingConfigurationSetup.cleanupAttachmentFolder(
607+
this.folderContext,
608+
testRunTime,
609+
this.workspaceContext.outputChannel
610+
);
554611
});
555612
}
556613

@@ -774,21 +831,32 @@ export class TestRunner {
774831
throw new Error(`Build failed with exit code ${buildExitCode}`);
775832
}
776833

834+
const testRunTime = Date.now();
777835
const subscriptions: vscode.Disposable[] = [];
778836
const buildConfigs: Array<vscode.DebugConfiguration | undefined> = [];
779-
const fifoPipePath = this.generateFifoPipePath();
837+
const fifoPipePath = this.generateFifoPipePath(testRunTime);
780838

781-
await TemporaryFolder.withNamedTemporaryFile(fifoPipePath, async () => {
839+
await TemporaryFolder.withNamedTemporaryFiles([fifoPipePath], async () => {
782840
if (this.testArgs.hasSwiftTestingTests) {
783841
// macOS/Linux require us to create the named pipe before we use it.
784842
// Windows just lets us communicate by specifying a pipe path without any ceremony.
785843
if (process.platform !== "win32") {
786844
await execFile("mkfifo", [fifoPipePath], undefined, this.folderContext);
787845
}
846+
// Create the swift-testing configuration JSON file, peparing any
847+
// directories the configuration may require.
848+
const attachmentFolder = await SwiftTestingConfigurationSetup.setupAttachmentFolder(
849+
this.folderContext,
850+
testRunTime
851+
);
852+
const swiftTestingArgs = await SwiftTestingBuildAguments.build(
853+
fifoPipePath,
854+
attachmentFolder
855+
);
788856

789857
const swiftTestBuildConfig = await TestingConfigurationFactory.swiftTestingConfig(
790858
this.folderContext,
791-
fifoPipePath,
859+
swiftTestingArgs,
792860
this.testKind,
793861
this.testArgs.swiftTestArgs,
794862
true
@@ -930,9 +998,6 @@ export class TestRunner {
930998
}
931999
},
9321000
reason => {
933-
this.workspaceContext.outputChannel.logDiagnostic(
934-
`Failed to debug test: ${reason}`
935-
);
9361001
subscriptions.forEach(sub => sub.dispose());
9371002
reject(reason);
9381003
}
@@ -942,6 +1007,13 @@ export class TestRunner {
9421007

9431008
// Run each debugging session sequentially
9441009
await debugRuns.reduce((p, fn) => p.then(() => fn()), Promise.resolve());
1010+
1011+
// Clean up any leftover resources
1012+
await SwiftTestingConfigurationSetup.cleanupAttachmentFolder(
1013+
this.folderContext,
1014+
testRunTime,
1015+
this.workspaceContext.outputChannel
1016+
);
9451017
});
9461018
}
9471019

@@ -996,10 +1068,10 @@ export class TestRunner {
9961068
}
9971069
}
9981070

999-
private generateFifoPipePath(): string {
1071+
private generateFifoPipePath(testRunDateNow: number): string {
10001072
return process.platform === "win32"
1001-
? `\\\\.\\pipe\\vscodemkfifo-${Date.now()}`
1002-
: path.join(os.tmpdir(), `vscodemkfifo-${Date.now()}`);
1073+
? `\\\\.\\pipe\\vscodemkfifo-${testRunDateNow}`
1074+
: path.join(os.tmpdir(), `vscodemkfifo-${testRunDateNow}`);
10031075
}
10041076
}
10051077

‎src/configuration.ts‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -65,6 +65,8 @@ export interface FolderConfiguration {
6565
readonly autoGenerateLaunchConfigurations: boolean;
6666
/** disable automatic running of swift package resolve */
6767
readonly disableAutoResolve: boolean;
68+
/** location to save swift-testing attachments */
69+
readonly attachmentsPath: string;
6870
/** look up saved permissions for the supplied plugin */
6971
pluginPermissions(pluginId: string): PluginPermissionConfiguration;
7072
}
@@ -162,6 +164,11 @@ const configuration = {
162164
.getConfiguration("swift", workspaceFolder)
163165
.get<boolean>("searchSubfoldersForPackages", false);
164166
},
167+
get attachmentsPath(): string {
168+
return vscode.workspace
169+
.getConfiguration("swift", workspaceFolder)
170+
.get<string>("attachmentsPath", "./.build/attachments");
171+
},
165172
pluginPermissions(pluginId: string): PluginPermissionConfiguration {
166173
return (
167174
vscode.workspace.getConfiguration("swift", workspaceFolder).get<{

0 commit comments

Comments
 (0)