Introduce a severity level for issues, and a 'warning' severity

Author: stmontgomery

Sources/Testing/ABI/EntryPoints/EntryPoint.swift

@@ -40,7 +40,7 @@ func entryPoint(passing args: __CommandLineArguments_v0?, eventHandler: Event.Ha
 
     // Set up the event handler.
     configuration.eventHandler = { [oldEventHandler = configuration.eventHandler] event, context in
-      if case let .issueRecorded(issue) = event.kind, !issue.isKnown {
+      if case let .issueRecorded(issue) = event.kind, !issue.isKnown, issue.severity >= .error {
         exitCode.withLock { exitCode in
           exitCode = EXIT_FAILURE
         }

Sources/Testing/Events/Recorder/Event.ConsoleOutputRecorder.swift

@@ -354,7 +354,7 @@ extension Event.ConsoleOutputRecorder {
   ///
   /// The caller is responsible for presenting this message to the user.
   static func warning(_ message: String, options: Event.ConsoleOutputRecorder.Options) -> String {
-    let symbol = Event.Symbol.warning.stringValue(options: options)
+    let symbol = Event.Symbol.warning().stringValue(options: options)
     return "\(symbol) \(message)\n"
   }
 }

Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift

@@ -56,8 +56,11 @@ extension Event {
         /// The instant at which the test started.
         var startInstant: Test.Clock.Instant
 
-        /// The number of issues recorded for the test.
-        var issueCount = 0
+        /// The number of issues with error severity recorded for the test.
+        var errorIssueCount = 0
+
+        /// The number of issues with warning severity recorded for the test.
+        var warningIssueCount = 0
 
         /// The number of known issues recorded for the test.
         var knownIssueCount = 0
@@ -114,27 +117,36 @@ extension Event.HumanReadableOutputRecorder {
   ///   - graph: The graph to walk while counting issues.
   ///
   /// - Returns: A tuple containing the number of issues recorded in `graph`.
-  private func _issueCounts(in graph: Graph<String, Event.HumanReadableOutputRecorder._Context.TestData?>?) -> (issueCount: Int, knownIssueCount: Int, totalIssueCount: Int, description: String) {
+  private func _issueCounts(in graph: Graph<String, Event.HumanReadableOutputRecorder._Context.TestData?>?) -> (errorIssueCount: Int, warningIssueCount: Int, knownIssueCount: Int, totalIssueCount: Int, description: String) {
     guard let graph else {
-      return (0, 0, 0, "")
+      return (0, 0, 0, 0, "")
     }
-    let issueCount = graph.compactMap(\.value?.issueCount).reduce(into: 0, +=)
+    let errorIssueCount = graph.compactMap(\.value?.errorIssueCount).reduce(into: 0, +=)
+    let warningIssueCount = graph.compactMap(\.value?.warningIssueCount).reduce(into: 0, +=)
     let knownIssueCount = graph.compactMap(\.value?.knownIssueCount).reduce(into: 0, +=)
-    let totalIssueCount = issueCount + knownIssueCount
+    let totalIssueCount = errorIssueCount + warningIssueCount + knownIssueCount
 
     // Construct a string describing the issue counts.
-    let description = switch (issueCount > 0, knownIssueCount > 0) {
-    case (true, true):
+    let description = switch (errorIssueCount > 0, warningIssueCount > 0, knownIssueCount > 0) {
+    case (true, true, true):
+      " with \(totalIssueCount.counting("issue")) (including \(warningIssueCount.counting("warning")) and \(knownIssueCount.counting("known issue")))"
+    case (true, false, true):
       " with \(totalIssueCount.counting("issue")) (including \(knownIssueCount.counting("known issue")))"
-    case (false, true):
+    case (false, true, true):
+      " with \(warningIssueCount.counting("warning")) and \(knownIssueCount.counting("known issue"))"
+    case (false, false, true):
       " with \(knownIssueCount.counting("known issue"))"
-    case (true, false):
+    case (true, true, false):
+      " with \(totalIssueCount.counting("issue")) (including \(warningIssueCount.counting("warning")))"
+    case (true, false, false):
       " with \(totalIssueCount.counting("issue"))"
-    case(false, false):
+    case(false, true, false):
+      " with \(warningIssueCount.counting("warning"))"
+    case(false, false, false):
       ""
     }
 
-    return (issueCount, knownIssueCount, totalIssueCount,  description)
+    return (errorIssueCount, warningIssueCount, knownIssueCount, totalIssueCount,  description)
   }
 }
 
@@ -267,7 +279,12 @@ extension Event.HumanReadableOutputRecorder {
         if issue.isKnown {
           testData.knownIssueCount += 1
         } else {
-          testData.issueCount += 1
+          switch issue.severity {
+          case .warning:
+            testData.warningIssueCount += 1
+          case .error:
+            testData.errorIssueCount += 1
+          }
         }
         context.testData[id] = testData
 
@@ -355,15 +372,22 @@ extension Event.HumanReadableOutputRecorder {
       let testData = testDataGraph?.value ?? .init(startInstant: instant)
       let issues = _issueCounts(in: testDataGraph)
       let duration = testData.startInstant.descriptionOfDuration(to: instant)
-      return if issues.issueCount > 0 {
+      return if issues.errorIssueCount > 0 {
         CollectionOfOne(
           Message(
             symbol: .fail,
             stringValue: "\(_capitalizedTitle(for: test)) \(testName) failed after \(duration)\(issues.description)."
           )
         ) + _formattedComments(for: test)
+      } else if issues.warningIssueCount > 0 {
+        [
+          Message(
+           symbol: .warning(warningIssueCount: issues.warningIssueCount),
+           stringValue: "\(_capitalizedTitle(for: test)) \(testName) passed after \(duration)\(issues.description)."
+         )
+       ]
       } else {
-         [
+        [
           Message(
             symbol: .pass(knownIssueCount: issues.knownIssueCount),
             stringValue: "\(_capitalizedTitle(for: test)) \(testName) passed after \(duration)\(issues.description)."
@@ -400,13 +424,19 @@ extension Event.HumanReadableOutputRecorder {
         ""
       }
       let symbol: Event.Symbol
-      let known: String
+      let introducer: String
       if issue.isKnown {
         symbol = .pass(knownIssueCount: 1)
-        known = " known"
+        introducer = "a known"
       } else {
-        symbol = .fail
-        known = "n"
+        switch issue.severity {
+        case .warning:
+          symbol = .warning(warningIssueCount: 1)
+          introducer = "a warning"
+        case .error:
+          symbol = .fail
+          introducer = "an"
+        }
       }
 
       var additionalMessages = [Message]()
@@ -435,13 +465,13 @@ extension Event.HumanReadableOutputRecorder {
       let primaryMessage: Message = if parameterCount == 0 {
         Message(
           symbol: symbol,
-          stringValue: "\(_capitalizedTitle(for: test)) \(testName) recorded a\(known) issue\(atSourceLocation): \(issue.kind)",
+          stringValue: "\(_capitalizedTitle(for: test)) \(testName) recorded \(introducer) issue\(atSourceLocation): \(issue.kind)",
           conciseStringValue: String(describing: issue.kind)
         )
       } else {
         Message(
           symbol: symbol,
-          stringValue: "\(_capitalizedTitle(for: test)) \(testName) recorded a\(known) issue with \(parameterCount.counting("argument")) \(labeledArguments)\(atSourceLocation): \(issue.kind)",
+          stringValue: "\(_capitalizedTitle(for: test)) \(testName) recorded \(introducer) issue with \(parameterCount.counting("argument")) \(labeledArguments)\(atSourceLocation): \(issue.kind)",
           conciseStringValue: String(describing: issue.kind)
         )
       }
@@ -498,13 +528,20 @@ extension Event.HumanReadableOutputRecorder {
       let runStartInstant = context.runStartInstant ?? instant
       let duration = runStartInstant.descriptionOfDuration(to: instant)
 
-      return if issues.issueCount > 0 {
+      return if issues.errorIssueCount > 0 {
         [
           Message(
             symbol: .fail,
             stringValue: "Test run with \(testCount.counting("test")) failed after \(duration)\(issues.description)."
           )
         ]
+      } else if issues.warningIssueCount > 0 {
+        [
+          Message(
+            symbol: .warning(warningIssueCount: issues.warningIssueCount),
+            stringValue: "Test run with \(testCount.counting("test")) passed after \(duration)\(issues.description)."
+          )
+        ]
       } else {
         [
           Message(

Sources/Testing/Events/Recorder/Event.Symbol.swift

@@ -22,8 +22,8 @@ extension Event {
     /// The symbol to use when a test passes.
     ///
     /// - Parameters:
-    ///   - knownIssueCount: The number of known issues encountered by the end
-    ///     of the test.
+    ///   - knownIssueCount: The number of known issues recorded for the test.
+    ///     The default value is `0`.
     case pass(knownIssueCount: Int = 0)
 
     /// The symbol to use when a test fails.
@@ -34,7 +34,11 @@ extension Event {
 
     /// A warning or caution symbol to use when the developer should be aware of
     /// some condition.
-    case warning
+    ///
+    /// - Parameters:
+    ///   - warningIssueCount: The number of issues with warning severity
+    ///     recorded for the test. The default value is `0`.
+    case warning(warningIssueCount: Int = 0)
 
     /// The symbol to use when presenting details about an event to the user.
     case details

Sources/Testing/Issues/Issue.swift

@@ -79,6 +79,35 @@ public struct Issue: Sendable {
   /// The kind of issue this value represents.
   public var kind: Kind
 
+  /// An enumeration representing the level of severity of a recorded issue.
+  ///
+  /// The supported levels, in decreasing order of severity, are:
+  ///
+  /// - ``error``
+  /// - ``warning``
+  @_spi(Experimental)
+  public enum Severity: Sendable {
+    /// The severity level representing an issue which may be a concern but is
+    /// not an error.
+    ///
+    /// An issue with warning severity does not cause the test it's associated
+    /// with to be marked as a failure, but is noted in the results.
+    case warning
+
+    /// The severity level representing an issue which represents an error in a
+    /// test.
+    ///
+    /// An issue with error severity causes the test it's associated with to be
+    /// marked as a failure.
+    case error
+  }
+
+  /// The severity of this issue.
+  ///
+  /// The default value of this property is ``Severity-swift.enum/error``.
+  @_spi(Experimental)
+  public var severity: Severity = .error
+
   /// Any comments provided by the developer and associated with this issue.
   ///
   /// If no comment was supplied when the issue occurred, the value of this
@@ -97,12 +126,15 @@ public struct Issue: Sendable {
   ///
   /// - Parameters:
   ///   - kind: The kind of issue this value represents.
+  ///   - severity: The severity of this issue. The default value is
+  ///     ``Severity-swift.enum/error``.
   ///   - comments: An array of comments describing the issue. This array may be
   ///     empty.
   ///   - sourceContext: A ``SourceContext`` indicating where and how this issue
   ///     occurred.
   init(
     kind: Kind,
+    severity: Severity = .error,
     comments: [Comment],
     sourceContext: SourceContext
   ) {
@@ -154,6 +186,8 @@ public struct Issue: Sendable {
   }
 }
 
+extension Issue.Severity: Comparable {}
+
 // MARK: - CustomStringConvertible, CustomDebugStringConvertible
 
 extension Issue: CustomStringConvertible, CustomDebugStringConvertible {
@@ -164,7 +198,7 @@ extension Issue: CustomStringConvertible, CustomDebugStringConvertible {
     let joinedComments = comments.lazy
       .map(\.rawValue)
       .joined(separator: "\n")
-    return "\(kind): \(joinedComments)"
+    return "\(severity): \(kind): \(joinedComments)"
   }
 
   public var debugDescription: String {
@@ -174,7 +208,7 @@ extension Issue: CustomStringConvertible, CustomDebugStringConvertible {
     let joinedComments: String = comments.lazy
       .map(\.rawValue)
       .joined(separator: "\n")
-    return "\(kind)\(sourceLocation.map { " at \($0)" } ?? ""): \(joinedComments)"
+    return "\(severity): \(kind)\(sourceLocation.map { " at \($0)" } ?? ""): \(joinedComments)"
   }
 }
 
@@ -234,6 +268,17 @@ extension Issue.Kind: CustomStringConvertible {
   }
 }
 
+extension Issue.Severity: CustomStringConvertible {
+  public var description: String {
+    switch self {
+    case .warning:
+      "warning"
+    case .error:
+      "error"
+    }
+  }
+}
+
 #if !SWT_NO_SNAPSHOT_TYPES
 // MARK: - Snapshotting
 
@@ -244,6 +289,9 @@ extension Issue {
     /// The kind of issue this value represents.
     public var kind: Kind.Snapshot
 
+    @_spi(Experimental)
+    public var severity: Severity
+
     /// Any comments provided by the developer and associated with this issue.
     ///
     /// If no comment was supplied when the issue occurred, the value of this
@@ -268,6 +316,7 @@ extension Issue {
         self.kind = Issue.Kind.Snapshot(snapshotting: issue.kind)
         self.comments = issue.comments
       }
+      self.severity = issue.severity
       self.sourceContext = issue.sourceContext
       self.isKnown = issue.isKnown
     }
@@ -295,6 +344,8 @@ extension Issue {
   }
 }
 
+extension Issue.Severity: Codable {}
+
 extension Issue.Kind {
   /// Serializable kinds of issues which may be recorded.
   @_spi(ForToolsIntegrationOnly)
@@ -484,7 +535,7 @@ extension Issue.Snapshot: CustomStringConvertible, CustomDebugStringConvertible
     let joinedComments = comments.lazy
       .map(\.rawValue)
       .joined(separator: "\n")
-    return "\(kind): \(joinedComments)"
+    return "\(severity): \(kind): \(joinedComments)"
   }
 
   public var debugDescription: String {
@@ -494,7 +545,7 @@ extension Issue.Snapshot: CustomStringConvertible, CustomDebugStringConvertible
     let joinedComments: String = comments.lazy
       .map(\.rawValue)
       .joined(separator: "\n")
-    return "\(kind)\(sourceLocation.map { " at \($0)" } ?? ""): \(joinedComments)"
+    return "\(severity): \(kind)\(sourceLocation.map { " at \($0)" } ?? ""): \(joinedComments)"
   }
 }
 

Tests/TestingTests/IssueTests.swift

@@ -1124,12 +1124,12 @@ final class IssueTests: XCTestCase {
     do {
       let sourceLocation = SourceLocation.init(fileID: "FakeModule/FakeFile.swift", filePath: "", line: 9999, column: 1)
       let issue = Issue(kind: .system, comments: ["Some issue"], sourceContext: SourceContext(sourceLocation: sourceLocation))
-      XCTAssertEqual(issue.description, "A system failure occurred: Some issue")
-      XCTAssertEqual(issue.debugDescription, "A system failure occurred at FakeFile.swift:9999:1: Some issue")
+      XCTAssertEqual(issue.description, "error: A system failure occurred: Some issue")
+      XCTAssertEqual(issue.debugDescription, "error: A system failure occurred at FakeFile.swift:9999:1: Some issue")
     }
     do {
       let issue = Issue(kind: .system, comments: ["Some issue"], sourceContext: SourceContext(sourceLocation: nil))
-      XCTAssertEqual(issue.debugDescription, "A system failure occurred: Some issue")
+      XCTAssertEqual(issue.debugDescription, "error: A system failure occurred: Some issue")
     }
   }