Add support for collecting stdout and stderr in an exit test. (#773)

This PR adds `stdout` and `stderr` capture to exit tests.

Gathering these streams can be very expensive, so they are _opt-in_.
There is a new argument to `#expect(exitsWith:)`, `observedValues`, to
which you can pass one or more key paths on `ExitTest.Result` to
indicate which bits you want:

```swift
 let result = try await #require(exitsWith: .failure, observing: [\.standardOutputContent]) {
   print("Goodbye, world!")
   fatalError()
 }
 #expect(result.standardOutputContent.contains(UInt8(ascii: "G")))
```

Some changes to pipes were needed to satisfy the compiler's concerns
about `Pipe` being a move-only type that contains other move-only
types—now it's just a function that returns two `FileHandle` instances.
This is simpler anyway.

It is possible some callers will want `stdout` and `stderr` to be set,
but to something other than a pipe (such as a file on disk or a PTY.)
While these are interesting cases, they are beyond the scope of this PR.

### Checklist:

- [x] Code and documentation should follow the style of the [Style
Guide](https://github.com/apple/swift-testing/blob/main/Documentation/StyleGuide.md).
- [x] If public symbols are renamed or modified, DocC references should
be updated.

Author: grynspan

Sources/Testing/ExitTests/ExitCondition.swift

@@ -15,8 +15,8 @@ private import _TestingInternals
 ///
 /// Values of this type are used to describe the conditions under which an exit
 /// test is expected to pass or fail by passing them to
-/// ``expect(exitsWith:_:sourceLocation:performing:)`` or
-/// ``require(exitsWith:_:sourceLocation:performing:)``.
+/// ``expect(exitsWith:observing:_:sourceLocation:performing:)`` or
+/// ``require(exitsWith:observing:_:sourceLocation:performing:)``.
 @_spi(Experimental)
 #if SWT_NO_PROCESS_SPAWNING
 @available(*, unavailable, message: "Exit tests are not available on this platform.")

Sources/Testing/ExitTests/ExitTest.swift

@@ -34,6 +34,41 @@ public struct ExitTest: Sendable, ~Copyable {
   /// The body closure of the exit test.
   fileprivate var body: @Sendable () async throws -> Void = {}
 
+  /// Storage for ``observedValues``.
+  ///
+  /// Key paths are not sendable because the properties they refer to may or may
+  /// not be, so this property needs to be `nonisolated(unsafe)`. It is safe to
+  /// use it in this fashion because `ExitTestArtifacts` is sendable.
+  fileprivate nonisolated(unsafe) var _observedValues = [PartialKeyPath<ExitTestArtifacts>]()
+
+  /// Key paths representing results from within this exit test that should be
+  /// observed and returned to the caller.
+  ///
+  /// The testing library sets this property to match what was passed by the
+  /// developer to the `#expect(exitsWith:)` or `#require(exitsWith:)` macro.
+  /// If you are implementing an exit test handler, you can check the value of
+  /// this property to determine what information you need to preserve from your
+  /// child process.
+  ///
+  /// The value of this property always includes ``Result/exitCondition`` even
+  /// if the test author does not specify it.
+  ///
+  /// Within a child process running an exit test, the value of this property is
+  /// otherwise unspecified.
+  @_spi(ForToolsIntegrationOnly)
+  public var observedValues: [PartialKeyPath<ExitTestArtifacts>] {
+    get {
+      var result = _observedValues
+      if !result.contains(\.exitCondition) { // O(n), but n <= 3 (no Set needed)
+        result.append(\.exitCondition)
+      }
+      return result
+    }
+    set {
+      _observedValues = newValue
+    }
+  }
+
   /// The source location of the exit test.
   ///
   /// The source location is unique to each exit test and is consistent between
@@ -184,6 +219,9 @@ extension ExitTest {
 ///
 /// - Parameters:
 ///   - expectedExitCondition: The expected exit condition.
+///   - observedValues: An array of key paths representing results from within
+///     the exit test that should be observed and returned by this macro. The
+///     ``ExitTestArtifacts/exitCondition`` property is always returned.
 ///   - expression: The expression, corresponding to `condition`, that is being
 ///     evaluated (if available at compile time.)
 ///   - comments: An array of comments describing the expectation. This array
@@ -199,19 +237,21 @@ extension ExitTest {
 /// convention.
 func callExitTest(
   exitsWith expectedExitCondition: ExitCondition,
+  observing observedValues: [PartialKeyPath<ExitTestArtifacts>],
   expression: __Expression,
   comments: @autoclosure () -> [Comment],
   isRequired: Bool,
   isolation: isolated (any Actor)? = #isolation,
   sourceLocation: SourceLocation
-) async -> Result<ExitTestArtifacts, any Error> {
+) async -> Result<ExitTestArtifacts?, any Error> {
   guard let configuration = Configuration.current ?? Configuration.all.first else {
     preconditionFailure("A test must be running on the current task to use #expect(exitsWith:).")
   }
 
   var result: ExitTestArtifacts
   do {
-    let exitTest = ExitTest(expectedExitCondition: expectedExitCondition, sourceLocation: sourceLocation)
+    var exitTest = ExitTest(expectedExitCondition: expectedExitCondition, sourceLocation: sourceLocation)
+    exitTest.observedValues = observedValues
     result = try await configuration.exitTestHandler(exitTest)
 
 #if os(Windows)
@@ -276,11 +316,15 @@ extension ExitTest {
   ///   the exit test.
   ///
   /// This handler is invoked when an exit test (i.e. a call to either
-  /// ``expect(exitsWith:_:sourceLocation:performing:)`` or
-  /// ``require(exitsWith:_:sourceLocation:performing:)``) is started. The
-  /// handler is responsible for initializing a new child environment (typically
-  /// a child process) and running the exit test identified by `sourceLocation`
-  /// there. The exit test's body can be found using ``ExitTest/find(at:)``.
+  /// ``expect(exitsWith:observing:_:sourceLocation:performing:)`` or
+  /// ``require(exitsWith:observing:_:sourceLocation:performing:)``) is started.
+  /// The handler is responsible for initializing a new child environment
+  /// (typically a child process) and running the exit test identified by
+  /// `sourceLocation` there.
+  ///
+  /// In the child environment, you can find the exit test again by calling
+  /// ``ExitTest/find(at:)`` and can run it by calling
+  /// ``ExitTest/callAsFunction()``.
   ///
   /// The parent environment should suspend until the results of the exit test
   /// are available or the child environment is otherwise terminated. The parent
@@ -465,20 +509,43 @@ extension ExitTest {
         childEnvironment["SWT_EXPERIMENTAL_EXIT_TEST_SOURCE_LOCATION"] = String(decoding: json, as: UTF8.self)
       }
 
-      return try await withThrowingTaskGroup(of: ExitTestArtifacts?.self) { taskGroup in
+      typealias ResultUpdater = @Sendable (inout ExitTestArtifacts) -> Void
+      return try await withThrowingTaskGroup(of: ResultUpdater?.self) { taskGroup in
+        // Set up stdout and stderr streams. By POSIX convention, stdin/stdout
+        // are line-buffered by default and stderr is unbuffered by default.
+        // SEE: https://en.cppreference.com/w/cpp/io/c/std_streams
+        var stdoutReadEnd: FileHandle?
+        var stdoutWriteEnd: FileHandle?
+        if exitTest._observedValues.contains(\.standardOutputContent) {
+          try FileHandle.makePipe(readEnd: &stdoutReadEnd, writeEnd: &stdoutWriteEnd)
+          stdoutWriteEnd?.withUnsafeCFILEHandle { stdout in
+            _ = setvbuf(stdout, nil, _IOLBF, Int(BUFSIZ))
+          }
+        }
+        var stderrReadEnd: FileHandle?
+        var stderrWriteEnd: FileHandle?
+        if exitTest._observedValues.contains(\.standardErrorContent) {
+          try FileHandle.makePipe(readEnd: &stderrReadEnd, writeEnd: &stderrWriteEnd)
+          stderrWriteEnd?.withUnsafeCFILEHandle { stderr in
+            _ = setvbuf(stderr, nil, _IONBF, Int(BUFSIZ))
+          }
+        }
+
         // Create a "back channel" pipe to handle events from the child process.
-        let backChannel = try FileHandle.Pipe()
+        var backChannelReadEnd: FileHandle!
+        var backChannelWriteEnd: FileHandle!
+        try FileHandle.makePipe(readEnd: &backChannelReadEnd, writeEnd: &backChannelWriteEnd)
 
         // Let the child process know how to find the back channel by setting a
         // known environment variable to the corresponding file descriptor
         // (HANDLE on Windows.)
         var backChannelEnvironmentVariable: String?
 #if SWT_TARGET_OS_APPLE || os(Linux) || os(FreeBSD)
-        backChannelEnvironmentVariable = backChannel.writeEnd.withUnsafePOSIXFileDescriptor { fd in
+        backChannelEnvironmentVariable = backChannelWriteEnd.withUnsafePOSIXFileDescriptor { fd in
           fd.map(String.init(describing:))
         }
 #elseif os(Windows)
-        backChannelEnvironmentVariable = backChannel.writeEnd.withUnsafeWindowsHANDLE { handle in
+        backChannelEnvironmentVariable = backChannelWriteEnd.withUnsafeWindowsHANDLE { handle in
           handle.flatMap { String(describing: UInt(bitPattern: $0)) }
         }
 #else
@@ -489,32 +556,55 @@ extension ExitTest {
         }
 
         // Spawn the child process.
-        let processID = try withUnsafePointer(to: backChannel.writeEnd) { writeEnd in
+        let processID = try withUnsafePointer(to: backChannelWriteEnd) { backChannelWriteEnd in
           try spawnExecutable(
             atPath: childProcessExecutablePath,
             arguments: childArguments,
             environment: childEnvironment,
-            additionalFileHandles: .init(start: writeEnd, count: 1)
+            standardOutput: stdoutWriteEnd,
+            standardError: stderrWriteEnd,
+            additionalFileHandles: [backChannelWriteEnd]
           )
         }
 
         // Await termination of the child process.
         taskGroup.addTask {
           let exitCondition = try await wait(for: processID)
-          return ExitTestArtifacts(exitCondition: exitCondition)
+          return { $0.exitCondition = exitCondition }
+        }
+
+        // Read back the stdout and stderr streams.
+        if let stdoutReadEnd {
+          stdoutWriteEnd?.close()
+          taskGroup.addTask {
+            let standardOutputContent = try stdoutReadEnd.readToEnd()
+            return { $0.standardOutputContent = standardOutputContent }
+          }
+        }
+        if let stderrReadEnd {
+          stderrWriteEnd?.close()
+          taskGroup.addTask {
+            let standardErrorContent = try stderrReadEnd.readToEnd()
+            return { $0.standardErrorContent = standardErrorContent }
+          }
         }
 
         // Read back all data written to the back channel by the child process
         // and process it as a (minimal) event stream.
-        let readEnd = backChannel.closeWriteEnd()
+        backChannelWriteEnd.close()
         taskGroup.addTask {
-          Self._processRecords(fromBackChannel: readEnd)
+          Self._processRecords(fromBackChannel: backChannelReadEnd)
           return nil
         }
 
-        // This is a roundabout way of saying "and return the exit condition
-        // yielded by wait(for:)".
-        return try await taskGroup.compactMap { $0 }.first { _ in true }!
+        // Collate the various bits of the result. The exit condition .failure
+        // here is just a placeholder and will be replaced by the result of one
+        // of the tasks above.
+        var result = ExitTestArtifacts(exitCondition: .failure)
+        for try await update in taskGroup {
+          update?(&result)
+        }
+        return result
       }
     }
   }

Sources/Testing/ExitTests/ExitTestArtifacts.swift

@@ -11,9 +11,9 @@
 /// A type representing the result of an exit test after it has exited and
 /// returned control to the calling test function.
 ///
-/// Both ``expect(exitsWith:_:sourceLocation:performing:)`` and
-/// ``require(exitsWith:_:sourceLocation:performing:)`` return instances of
-/// this type.
+/// Both ``expect(exitsWith:observing:_:sourceLocation:performing:)`` and
+/// ``require(exitsWith:observing:_:sourceLocation:performing:)`` return
+/// instances of this type.
 ///
 /// - Warning: The name of this type is still unstable and subject to change.
 @_spi(Experimental)
@@ -25,11 +25,64 @@ public struct ExitTestArtifacts: Sendable {
   ///
   /// When the exit test passes, the value of this property is equal to the
   /// value of the `expectedExitCondition` argument passed to
-  /// ``expect(exitsWith:_:sourceLocation:performing:)`` or to
-  /// ``require(exitsWith:_:sourceLocation:performing:)``. You can compare two
-  /// instances of ``ExitCondition`` with ``/Swift/Optional/==(_:_:)``.
+  /// ``expect(exitsWith:observing:_:sourceLocation:performing:)`` or to
+  /// ``require(exitsWith:observing:_:sourceLocation:performing:)``. You can
+  /// compare two instances of ``ExitCondition`` with
+  /// ``/Swift/Optional/==(_:_:)``.
   public var exitCondition: ExitCondition
 
+  /// All bytes written to the standard output stream of the exit test before
+  /// it exited.
+  ///
+  /// The value of this property may contain any arbitrary sequence of bytes,
+  /// including sequences that are not valid UTF-8 and cannot be decoded by
+  /// [`String.init(cString:)`](https://developer.apple.com/documentation/swift/string/init(cstring:)-6kr8s).
+  /// Consider using [`String.init(validatingCString:)`](https://developer.apple.com/documentation/swift/string/init(validatingcstring:)-992vo)
+  /// instead.
+  ///
+  /// When checking the value of this property, keep in mind that the standard
+  /// output stream is globally accessible, and any code running in an exit
+  /// test may write to it including including the operating system and any
+  /// third-party dependencies you have declared in your package. Rather than
+  /// comparing the value of this property with [`==`](https://developer.apple.com/documentation/swift/array/==(_:_:)),
+  /// use [`contains(_:)`](https://developer.apple.com/documentation/swift/collection/contains(_:))
+  /// to check if expected output is present.
+  ///
+  /// To enable gathering output from the standard output stream during an
+  /// exit test, pass `\.standardOutputContent` in the `observedValues`
+  /// argument of ``expect(exitsWith:observing:_:sourceLocation:performing:)``
+  /// or ``require(exitsWith:observing:_:sourceLocation:performing:)``.
+  ///
+  /// If you did not request standard output content when running an exit test,
+  /// the value of this property is the empty array.
+  public var standardOutputContent = [UInt8]()
+
+  /// All bytes written to the standard error stream of the exit test before
+  /// it exited.
+  ///
+  /// The value of this property may contain any arbitrary sequence of bytes,
+  /// including sequences that are not valid UTF-8 and cannot be decoded by
+  /// [`String.init(cString:)`](https://developer.apple.com/documentation/swift/string/init(cstring:)-6kr8s).
+  /// Consider using [`String.init(validatingCString:)`](https://developer.apple.com/documentation/swift/string/init(validatingcstring:)-992vo)
+  /// instead.
+  ///
+  /// When checking the value of this property, keep in mind that the standard
+  /// output stream is globally accessible, and any code running in an exit
+  /// test may write to it including including the operating system and any
+  /// third-party dependencies you have declared in your package. Rather than
+  /// comparing the value of this property with [`==`](https://developer.apple.com/documentation/swift/array/==(_:_:)),
+  /// use [`contains(_:)`](https://developer.apple.com/documentation/swift/collection/contains(_:))
+  /// to check if expected output is present.
+  ///
+  /// To enable gathering output from the standard error stream during an exit
+  /// test, pass `\.standardErrorContent` in the `observedValues` argument of
+  /// ``expect(exitsWith:observing:_:sourceLocation:performing:)`` or
+  /// ``require(exitsWith:observing:_:sourceLocation:performing:)``.
+  ///
+  /// If you did not request standard error content when running an exit test,
+  /// the value of this property is the empty array.
+  public var standardErrorContent = [UInt8]()
+
   @_spi(ForToolsIntegrationOnly)
   public init(exitCondition: ExitCondition) {
     self.exitCondition = exitCondition