From d8cfaed1adf546cef965a0fdc9b8fdf1acc886a3 Mon Sep 17 00:00:00 2001 From: tienquocbui Date: Mon, 9 Feb 2026 03:14:31 +0100 Subject: [PATCH 1/3] Add failure summary to ConsoleOutputRecorder --- .../Event.ConsoleOutputRecorder.swift | 11 + .../Event.HumanReadableOutputRecorder.swift | 312 ++++++++++++++++++ 2 files changed, 323 insertions(+) diff --git a/Sources/Testing/Events/Recorder/Event.ConsoleOutputRecorder.swift b/Sources/Testing/Events/Recorder/Event.ConsoleOutputRecorder.swift index b9585fc6b..be01fb094 100644 --- a/Sources/Testing/Events/Recorder/Event.ConsoleOutputRecorder.swift +++ b/Sources/Testing/Events/Recorder/Event.ConsoleOutputRecorder.swift @@ -348,6 +348,17 @@ extension Event.ConsoleOutputRecorder { } write(lines.joined()) + + // Print failure summary when run ends, unless an environment variable is + // set to explicitly disable it. The summary is printed after the main + // output so it appears at the very end of the console output. + if case .runEnded = event.kind, Environment.flag(named: "SWT_FAILURE_SUMMARY_ENABLED") != false { + if let summary = _humanReadableOutputRecorder.generateFailureSummary(options: options) { + // Add blank line before summary for visual separation + write("\n\(summary)") + } + } + return !messages.isEmpty } diff --git a/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift b/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift index 8d4277d75..2b34484d6 100644 --- a/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift +++ b/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift @@ -13,6 +13,228 @@ private import Synchronization #endif extension Event { + /// A type that generates a failure summary from test run data. + /// + /// This type encapsulates the logic for collecting failed tests from a test + /// data graph and formatting them into a human-readable failure summary. + private struct TestRunSummary: Sendable { + /// Information about a single failed test case (for parameterized tests). + struct FailedTestCase: Sendable { + /// The test case arguments for this parameterized test case. + var arguments: String + + /// All issues recorded for this test case. + var issues: [HumanReadableOutputRecorder.Context.TestData.IssueInfo] + } + + /// Information about a single failed test. + struct FailedTest: Sendable { + /// The name components from ``Test/ID`` (excludes the module name). + var nameComponents: [String] + + /// The human-readable name for this test, consistent with + /// ``Test/humanReadableName(withVerbosity:)``. + var humanReadableName: String + + /// Whether this is a suite rather than an individual test. + var isSuite: Bool + + /// For non-parameterized tests: issues recorded directly on the test. + var issues: [HumanReadableOutputRecorder.Context.TestData.IssueInfo] + + /// For parameterized tests: test cases with their issues. + var testCases: [FailedTestCase] + } + + /// The list of failed tests collected from the test run. + private let failedTests: [FailedTest] + + /// Initialize a test run summary by collecting failures from a test data + /// graph. + /// + /// - Parameters: + /// - testData: The root test data graph to traverse. + fileprivate init(from testData: Graph) { + var testMap: [String: FailedTest] = [:] + + testData.forEach { keyPath, value in + guard let testData = value, !testData.issues.isEmpty else { return } + guard let testID = testData.testID else { return } + + // Determine if this node represents a parameterized test case by + // checking if the key path contains a parameterized test case ID. + // Non-parameterized test cases have nil argument IDs and should be + // treated as regular tests. + let isTestCase = keyPath.contains { + if case let .testCaseID(id) = $0 { + return id.argumentIDs != nil + } + return false + } + + let nameComponents = testID.nameComponents + let pathKey = nameComponents.joined(separator: "/") + let humanReadableName = testData.humanReadableName ?? nameComponents.last ?? "Unknown" + + if isTestCase { + // This represents a parameterized test case — group it under + // its parent test. + if var parentTest = testMap[pathKey] { + if let arguments = testData.testCaseArguments, !arguments.isEmpty { + parentTest.testCases.append(FailedTestCase( + arguments: arguments, + issues: testData.issues + )) + testMap[pathKey] = parentTest + } + } else { + let parentTest = FailedTest( + nameComponents: nameComponents, + humanReadableName: humanReadableName, + isSuite: testData.isSuite, + issues: [], + testCases: (testData.testCaseArguments?.isEmpty ?? true) ? [] : [FailedTestCase( + arguments: testData.testCaseArguments ?? "", + issues: testData.issues + )] + ) + testMap[pathKey] = parentTest + } + } else { + // This represents a test function, not a parameterized test case. + let failedTest = FailedTest( + nameComponents: nameComponents, + humanReadableName: humanReadableName, + isSuite: testData.isSuite, + issues: testData.issues, + testCases: [] + ) + testMap[pathKey] = failedTest + } + } + + self.failedTests = Array(testMap.values).filter { !$0.issues.isEmpty || !$0.testCases.isEmpty } + } + + /// Generate a formatted failure summary string. + /// + /// - Parameters: + /// - options: Options for formatting (e.g., for ANSI colors and symbols). + /// + /// - Returns: A formatted string containing the failure summary, or `nil` + /// if there were no failures. + public func formatted(with options: Event.ConsoleOutputRecorder.Options) -> String? { + // If no failures, return nil + guard !failedTests.isEmpty else { + return nil + } + + // Begin with the summary header. + var summary = header() + + // Get the failure symbol + let failSymbol = Event.Symbol.fail.stringValue(options: options) + + // Format each failed test + for failedTest in failedTests { + summary += formatFailedTest(failedTest, withSymbol: failSymbol) + } + + return summary + } + + /// Generate the summary header with failure counts. + /// + /// - Returns: A string containing the header line. + private func header() -> String { + let failedTestsPhrase = failedTests.count.counting("test") + var totalIssuesCount = 0 + for test in failedTests { + totalIssuesCount += test.issues.count + for testCase in test.testCases { + totalIssuesCount += testCase.issues.count + } + } + let issuesPhrase = totalIssuesCount.counting("issue") + return "Test run had \(failedTestsPhrase) which recorded \(issuesPhrase) total:\n" + } + + /// Format a single failed test entry. + /// + /// - Parameters: + /// - failedTest: The failed test to format. + /// - symbol: The failure symbol string to use. + /// + /// - Returns: A formatted string representing the failed test and its + /// issues. + private func formatFailedTest(_ failedTest: FailedTest, withSymbol symbol: String) -> String { + var result = "" + + // Build fully qualified name + let fullyQualifiedName = fullyQualifiedName(for: failedTest) + + // Use "Suite" or "Test" based on whether this is a suite + let label = failedTest.isSuite ? "Suite" : "Test" + result += "\(symbol) \(label) \(fullyQualifiedName)\n" + + // For parameterized tests: show test cases grouped under the parent test + if !failedTest.testCases.isEmpty { + for testCase in failedTest.testCases { + // Show test case with argument count phrase and arguments + let argumentCount = testCase.arguments.split(separator: ",").count + let argumentPhrase = argumentCount.counting("argument") + result += " Test case with \(argumentPhrase): (\(testCase.arguments))\n" + // List each issue for this test case with additional indentation + for issue in testCase.issues { + result += formatIssue(issue, indentLevel: 2) + } + } + } else { + // For non-parameterized tests: show issues directly + for issue in failedTest.issues { + result += formatIssue(issue) + } + } + + return result + } + + /// Build the fully qualified name for a failed test. + /// + /// - Parameters: + /// - failedTest: The failed test. + /// + /// - Returns: The fully qualified name, using the test's + /// ``Test/humanReadableName(withVerbosity:)`` for the final component. + private func fullyQualifiedName(for failedTest: FailedTest) -> String { + // Use the name components from Test.ID (which already exclude the module + // name) and replace the last component with the human-readable name to + // be consistent with how tests are described elsewhere in the output. + var components = failedTest.nameComponents + if !components.isEmpty { + components[components.count - 1] = failedTest.humanReadableName + } + return components.joined(separator: "/") + } + + /// Format a single issue entry. + /// + /// - Parameters: + /// - issue: The issue to format. + /// - indentLevel: The number of indentation levels (each level is 2 spaces). + /// Defaults to 1. + /// + /// - Returns: A formatted string representing the issue with indentation. + private func formatIssue(_ issue: HumanReadableOutputRecorder.Context.TestData.IssueInfo, indentLevel: Int = 1) -> String { + let indent = String(repeating: " ", count: indentLevel) + var result = "\(indent)- \(issue.description)\n" + if let location = issue.sourceLocation { + result += "\(indent) at \(location)\n" + } + return result + } + } + /// A type which handles ``Event`` instances and outputs representations of /// them as human-readable messages. /// @@ -77,6 +299,21 @@ extension Event { /// A type describing data tracked on a per-test basis. struct TestData { + /// A lightweight struct containing information about a single issue. + struct IssueInfo: Sendable { + /// The source location where the issue occurred. + var sourceLocation: SourceLocation? + + /// A detailed description of what failed (using expanded description). + var description: String + + /// Whether this issue is a known issue. + var isKnown: Bool + + /// The severity of this issue. + var severity: Issue.Severity + } + /// The instant at which the test started. var startInstant: Test.Clock.Instant @@ -89,6 +326,25 @@ extension Event { /// Information about the cancellation of this test or test case. var cancellationInfo: SkipInfo? + + /// Array of all issues recorded for this test (for failure summary). + /// Each issue is stored individually with its own source location. + var issues: [IssueInfo] = [] + + /// The ID of the test, used to construct its fully qualified name in + /// the failure summary. + var testID: Test.ID? + + /// The human-readable name for this test, consistent with + /// ``Test/humanReadableName(withVerbosity:)``. + var humanReadableName: String? + + /// The test case arguments, formatted for display (for parameterized + /// tests). + var testCaseArguments: String? + + /// Whether this is a suite rather than an individual test. + var isSuite: Bool = false } /// Data tracked on a per-test basis. @@ -330,6 +586,46 @@ extension Event.HumanReadableOutputRecorder { let issueCount = testData.issueCount[issue.severity] ?? 0 testData.issueCount[issue.severity] = issueCount + 1 } + + // Store individual issue information for failure summary, but only for + // issues whose severity is error or greater. + if issue.severity >= .error { + // Extract detailed failure message + let description: String + if case let .expectationFailed(expectation) = issue.kind { + // Use expandedDebugDescription only when verbose, otherwise use expandedDescription + description = if verbosity > 0 { + expectation.evaluatedExpression.expandedDebugDescription() + } else { + expectation.evaluatedExpression.expandedDescription() + } + } else if let comment = issue.comments.first { + description = comment.rawValue + } else { + description = "Test failed" + } + + let issueInfo = Context.TestData.IssueInfo( + sourceLocation: issue.sourceLocation, + description: description, + isKnown: issue.isKnown, + severity: issue.severity + ) + testData.issues.append(issueInfo) + + // Capture test metadata once per test (not per issue). + if testData.testID == nil { + testData.testID = test?.id + } + if testData.humanReadableName == nil { + testData.humanReadableName = test?.humanReadableName(withVerbosity: verbosity) + } + if testData.testCaseArguments == nil { + testData.testCaseArguments = testCase?.labeledArguments() + } + testData.isSuite = test?.isSuite ?? false + } + context.testData[keyPath] = testData case .testCaseStarted: @@ -648,6 +944,22 @@ extension Event.HumanReadableOutputRecorder { return [] } + + /// Generate a failure summary string with all failed tests and their issues. + /// + /// This method creates a ``TestRunSummary`` from the test data graph and + /// formats it for display. + /// + /// - Parameters: + /// - options: Options for formatting (e.g., for ANSI colors and symbols). + /// + /// - Returns: A formatted string containing the failure summary, or `nil` + /// if there were no failures. + func generateFailureSummary(options: Event.ConsoleOutputRecorder.Options) -> String? { + let testData = _context.value.withLock { $0.testData } + let summary = Event.TestRunSummary(from: testData) + return summary.formatted(with: options) + } } extension Test.ID { From ae0a5d5fad01e7866cdf15cd88c83a4b2487e765 Mon Sep 17 00:00:00 2001 From: Stuart Montgomery Date: Wed, 8 Apr 2026 11:31:25 -0500 Subject: [PATCH 2/3] Suppress failure summary if verbosity == .min --- .../Events/Recorder/Event.ConsoleOutputRecorder.swift | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/Sources/Testing/Events/Recorder/Event.ConsoleOutputRecorder.swift b/Sources/Testing/Events/Recorder/Event.ConsoleOutputRecorder.swift index be01fb094..22376470b 100644 --- a/Sources/Testing/Events/Recorder/Event.ConsoleOutputRecorder.swift +++ b/Sources/Testing/Events/Recorder/Event.ConsoleOutputRecorder.swift @@ -350,9 +350,12 @@ extension Event.ConsoleOutputRecorder { write(lines.joined()) // Print failure summary when run ends, unless an environment variable is - // set to explicitly disable it. The summary is printed after the main - // output so it appears at the very end of the console output. - if case .runEnded = event.kind, Environment.flag(named: "SWT_FAILURE_SUMMARY_ENABLED") != false { + // set to explicitly disable it or verbosity is set to the lowest value. The + // summary is printed after the main output so it appears at the very end of + // the console output. + if case .runEnded = event.kind, + Environment.flag(named: "SWT_FAILURE_SUMMARY_ENABLED") != false, + context.configuration?.verbosity != .min { if let summary = _humanReadableOutputRecorder.generateFailureSummary(options: options) { // Add blank line before summary for visual separation write("\n\(summary)") From 84d0048f7a073ec7f1dc9cc8f1f65835ce385e81 Mon Sep 17 00:00:00 2001 From: Stuart Montgomery Date: Wed, 8 Apr 2026 11:43:37 -0500 Subject: [PATCH 3/3] Fixups after rebasing --- .../Event.HumanReadableOutputRecorder.swift | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift b/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift index 2b34484d6..6d2d85091 100644 --- a/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift +++ b/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift @@ -591,18 +591,12 @@ extension Event.HumanReadableOutputRecorder { // issues whose severity is error or greater. if issue.severity >= .error { // Extract detailed failure message - let description: String - if case let .expectationFailed(expectation) = issue.kind { - // Use expandedDebugDescription only when verbose, otherwise use expandedDescription - description = if verbosity > 0 { - expectation.evaluatedExpression.expandedDebugDescription() - } else { - expectation.evaluatedExpression.expandedDescription() - } + let description = if case let .expectationFailed(expectation) = issue.kind { + expectation.evaluatedExpression.expandedDescription(verbose: verbosity > 0) } else if let comment = issue.comments.first { - description = comment.rawValue + comment.rawValue } else { - description = "Test failed" + "Test failed" } let issueInfo = Context.TestData.IssueInfo(