Add failure summary to ConsoleOutputRecorder

tienquocbui · stmontgomery · commit d8cfaed1adf5 · 2026-04-08T11:35:25.000-05:00
diff --git a/Sources/Testing/Events/Recorder/Event.ConsoleOutputRecorder.swift b/Sources/Testing/Events/Recorder/Event.ConsoleOutputRecorder.swift
@@ -348,6 +348,17 @@ extension Event.ConsoleOutputRecorder {
     }
 
     write(lines.joined())
+
+    // Print failure summary when run ends, unless an environment variable is
+    // set to explicitly disable it. The summary is printed after the main
+    // output so it appears at the very end of the console output.
+    if case .runEnded = event.kind, Environment.flag(named: "SWT_FAILURE_SUMMARY_ENABLED") != false {
+      if let summary = _humanReadableOutputRecorder.generateFailureSummary(options: options) {
+        // Add blank line before summary for visual separation
+        write("\n\(summary)")
+      }
+    }
+
     return !messages.isEmpty
   }
 
diff --git a/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift b/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift
@@ -13,6 +13,228 @@ private import Synchronization
 #endif
 
 extension Event {
+  /// A type that generates a failure summary from test run data.
+  ///
+  /// This type encapsulates the logic for collecting failed tests from a test
+  /// data graph and formatting them into a human-readable failure summary.
+  private struct TestRunSummary: Sendable {
+    /// Information about a single failed test case (for parameterized tests).
+    struct FailedTestCase: Sendable {
+      /// The test case arguments for this parameterized test case.
+      var arguments: String
+
+      /// All issues recorded for this test case.
+      var issues: [HumanReadableOutputRecorder.Context.TestData.IssueInfo]
+    }
+
+    /// Information about a single failed test.
+    struct FailedTest: Sendable {
+      /// The name components from ``Test/ID`` (excludes the module name).
+      var nameComponents: [String]
+
+      /// The human-readable name for this test, consistent with
+      /// ``Test/humanReadableName(withVerbosity:)``.
+      var humanReadableName: String
+
+      /// Whether this is a suite rather than an individual test.
+      var isSuite: Bool
+
+      /// For non-parameterized tests: issues recorded directly on the test.
+      var issues: [HumanReadableOutputRecorder.Context.TestData.IssueInfo]
+
+      /// For parameterized tests: test cases with their issues.
+      var testCases: [FailedTestCase]
+    }
+
+    /// The list of failed tests collected from the test run.
+    private let failedTests: [FailedTest]
+
+    /// Initialize a test run summary by collecting failures from a test data
+    /// graph.
+    ///
+    /// - Parameters:
+    ///   - testData: The root test data graph to traverse.
+    fileprivate init(from testData: Graph<HumanReadableOutputRecorder.Context.TestDataKey, HumanReadableOutputRecorder.Context.TestData?>) {
+      var testMap: [String: FailedTest] = [:]
+
+      testData.forEach { keyPath, value in
+        guard let testData = value, !testData.issues.isEmpty else { return }
+        guard let testID = testData.testID else { return }
+
+        // Determine if this node represents a parameterized test case by
+        // checking if the key path contains a parameterized test case ID.
+        // Non-parameterized test cases have nil argument IDs and should be
+        // treated as regular tests.
+        let isTestCase = keyPath.contains {
+          if case let .testCaseID(id) = $0 {
+            return id.argumentIDs != nil
+          }
+          return false
+        }
+
+        let nameComponents = testID.nameComponents
+        let pathKey = nameComponents.joined(separator: "/")
+        let humanReadableName = testData.humanReadableName ?? nameComponents.last ?? "Unknown"
+
+        if isTestCase {
+          // This represents a parameterized test case — group it under
+          // its parent test.
+          if var parentTest = testMap[pathKey] {
+            if let arguments = testData.testCaseArguments, !arguments.isEmpty {
+              parentTest.testCases.append(FailedTestCase(
+                arguments: arguments,
+                issues: testData.issues
+              ))
+              testMap[pathKey] = parentTest
+            }
+          } else {
+            let parentTest = FailedTest(
+              nameComponents: nameComponents,
+              humanReadableName: humanReadableName,
+              isSuite: testData.isSuite,
+              issues: [],
+              testCases: (testData.testCaseArguments?.isEmpty ?? true) ? [] : [FailedTestCase(
+                arguments: testData.testCaseArguments ?? "",
+                issues: testData.issues
+              )]
+            )
+            testMap[pathKey] = parentTest
+          }
+        } else {
+          // This represents a test function, not a parameterized test case.
+          let failedTest = FailedTest(
+            nameComponents: nameComponents,
+            humanReadableName: humanReadableName,
+            isSuite: testData.isSuite,
+            issues: testData.issues,
+            testCases: []
+          )
+          testMap[pathKey] = failedTest
+        }
+      }
+
+      self.failedTests = Array(testMap.values).filter { !$0.issues.isEmpty || !$0.testCases.isEmpty }
+    }
+
+    /// Generate a formatted failure summary string.
+    ///
+    /// - Parameters:
+    ///   - options: Options for formatting (e.g., for ANSI colors and symbols).
+    ///
+    /// - Returns: A formatted string containing the failure summary, or `nil`
+    ///   if there were no failures.
+    public func formatted(with options: Event.ConsoleOutputRecorder.Options) -> String? {
+      // If no failures, return nil
+      guard !failedTests.isEmpty else {
+        return nil
+      }
+
+      // Begin with the summary header.
+      var summary = header()
+
+      // Get the failure symbol
+      let failSymbol = Event.Symbol.fail.stringValue(options: options)
+
+      // Format each failed test
+      for failedTest in failedTests {
+        summary += formatFailedTest(failedTest, withSymbol: failSymbol)
+      }
+
+      return summary
+    }
+
+    /// Generate the summary header with failure counts.
+    ///
+    /// - Returns: A string containing the header line.
+    private func header() -> String {
+      let failedTestsPhrase = failedTests.count.counting("test")
+      var totalIssuesCount = 0
+      for test in failedTests {
+        totalIssuesCount += test.issues.count
+        for testCase in test.testCases {
+          totalIssuesCount += testCase.issues.count
+        }
+      }
+      let issuesPhrase = totalIssuesCount.counting("issue")
+      return "Test run had \(failedTestsPhrase) which recorded \(issuesPhrase) total:\n"
+    }
+
+    /// Format a single failed test entry.
+    ///
+    /// - Parameters:
+    ///   - failedTest: The failed test to format.
+    ///   - symbol: The failure symbol string to use.
+    ///
+    /// - Returns: A formatted string representing the failed test and its
+    ///   issues.
+    private func formatFailedTest(_ failedTest: FailedTest, withSymbol symbol: String) -> String {
+      var result = ""
+
+      // Build fully qualified name
+      let fullyQualifiedName = fullyQualifiedName(for: failedTest)
+
+      // Use "Suite" or "Test" based on whether this is a suite
+      let label = failedTest.isSuite ? "Suite" : "Test"
+      result += "\(symbol) \(label) \(fullyQualifiedName)\n"
+
+      // For parameterized tests: show test cases grouped under the parent test
+      if !failedTest.testCases.isEmpty {
+        for testCase in failedTest.testCases {
+          // Show test case with argument count phrase and arguments
+          let argumentCount = testCase.arguments.split(separator: ",").count
+          let argumentPhrase = argumentCount.counting("argument")
+          result += "  Test case with \(argumentPhrase): (\(testCase.arguments))\n"
+          // List each issue for this test case with additional indentation
+          for issue in testCase.issues {
+            result += formatIssue(issue, indentLevel: 2)
+          }
+        }
+      } else {
+        // For non-parameterized tests: show issues directly
+        for issue in failedTest.issues {
+          result += formatIssue(issue)
+        }
+      }
+
+      return result
+    }
+
+    /// Build the fully qualified name for a failed test.
+    ///
+    /// - Parameters:
+    ///   - failedTest: The failed test.
+    ///
+    /// - Returns: The fully qualified name, using the test's
+    ///   ``Test/humanReadableName(withVerbosity:)`` for the final component.
+    private func fullyQualifiedName(for failedTest: FailedTest) -> String {
+      // Use the name components from Test.ID (which already exclude the module
+      // name) and replace the last component with the human-readable name to
+      // be consistent with how tests are described elsewhere in the output.
+      var components = failedTest.nameComponents
+      if !components.isEmpty {
+        components[components.count - 1] = failedTest.humanReadableName
+      }
+      return components.joined(separator: "/")
+    }
+
+    /// Format a single issue entry.
+    ///
+    /// - Parameters:
+    ///   - issue: The issue to format.
+    ///   - indentLevel: The number of indentation levels (each level is 2 spaces).
+    ///     Defaults to 1.
+    ///
+    /// - Returns: A formatted string representing the issue with indentation.
+    private func formatIssue(_ issue: HumanReadableOutputRecorder.Context.TestData.IssueInfo, indentLevel: Int = 1) -> String {
+      let indent = String(repeating: "  ", count: indentLevel)
+      var result = "\(indent)- \(issue.description)\n"
+      if let location = issue.sourceLocation {
+        result += "\(indent)    at \(location)\n"
+      }
+      return result
+    }
+  }
+
   /// A type which handles ``Event`` instances and outputs representations of
   /// them as human-readable messages.
   ///
@@ -77,6 +299,21 @@ extension Event {
 
       /// A type describing data tracked on a per-test basis.
       struct TestData {
+        /// A lightweight struct containing information about a single issue.
+        struct IssueInfo: Sendable {
+          /// The source location where the issue occurred.
+          var sourceLocation: SourceLocation?
+
+          /// A detailed description of what failed (using expanded description).
+          var description: String
+
+          /// Whether this issue is a known issue.
+          var isKnown: Bool
+
+          /// The severity of this issue.
+          var severity: Issue.Severity
+        }
+
         /// The instant at which the test started.
         var startInstant: Test.Clock.Instant
 
@@ -89,6 +326,25 @@ extension Event {
 
         /// Information about the cancellation of this test or test case.
         var cancellationInfo: SkipInfo?
+
+        /// Array of all issues recorded for this test (for failure summary).
+        /// Each issue is stored individually with its own source location.
+        var issues: [IssueInfo] = []
+
+        /// The ID of the test, used to construct its fully qualified name in
+        /// the failure summary.
+        var testID: Test.ID?
+
+        /// The human-readable name for this test, consistent with
+        /// ``Test/humanReadableName(withVerbosity:)``.
+        var humanReadableName: String?
+
+        /// The test case arguments, formatted for display (for parameterized
+        /// tests).
+        var testCaseArguments: String?
+
+        /// Whether this is a suite rather than an individual test.
+        var isSuite: Bool = false
       }
 
       /// Data tracked on a per-test basis.
@@ -330,6 +586,46 @@ extension Event.HumanReadableOutputRecorder {
           let issueCount = testData.issueCount[issue.severity] ?? 0
           testData.issueCount[issue.severity] = issueCount + 1
         }
+
+        // Store individual issue information for failure summary, but only for
+        // issues whose severity is error or greater.
+        if issue.severity >= .error {
+          // Extract detailed failure message
+          let description: String
+          if case let .expectationFailed(expectation) = issue.kind {
+            // Use expandedDebugDescription only when verbose, otherwise use expandedDescription
+            description = if verbosity > 0 {
+              expectation.evaluatedExpression.expandedDebugDescription()
+            } else {
+              expectation.evaluatedExpression.expandedDescription()
+            }
+          } else if let comment = issue.comments.first {
+            description = comment.rawValue
+          } else {
+            description = "Test failed"
+          }
+
+          let issueInfo = Context.TestData.IssueInfo(
+            sourceLocation: issue.sourceLocation,
+            description: description,
+            isKnown: issue.isKnown,
+            severity: issue.severity
+          )
+          testData.issues.append(issueInfo)
+
+          // Capture test metadata once per test (not per issue).
+          if testData.testID == nil {
+            testData.testID = test?.id
+          }
+          if testData.humanReadableName == nil {
+            testData.humanReadableName = test?.humanReadableName(withVerbosity: verbosity)
+          }
+          if testData.testCaseArguments == nil {
+            testData.testCaseArguments = testCase?.labeledArguments()
+          }
+          testData.isSuite = test?.isSuite ?? false
+        }
+
         context.testData[keyPath] = testData
       
       case .testCaseStarted:
@@ -648,6 +944,22 @@ extension Event.HumanReadableOutputRecorder {
 
     return []
   }
+
+  /// Generate a failure summary string with all failed tests and their issues.
+  ///
+  /// This method creates a ``TestRunSummary`` from the test data graph and
+  /// formats it for display.
+  ///
+  /// - Parameters:
+  ///   - options: Options for formatting (e.g., for ANSI colors and symbols).
+  ///
+  /// - Returns: A formatted string containing the failure summary, or `nil`
+  ///   if there were no failures.
+  func generateFailureSummary(options: Event.ConsoleOutputRecorder.Options) -> String? {
+    let testData = _context.value.withLock { $0.testData }
+    let summary = Event.TestRunSummary(from: testData)
+    return summary.formatted(with: options)
+  }
 }
 
 extension Test.ID {
