From 4799f196b48b6d6ad66da6842ce2e321c5e74889 Mon Sep 17 00:00:00 2001 From: Will Downey Date: Wed, 25 Feb 2026 13:04:07 -0500 Subject: [PATCH 1/6] Diagnostics for issues recorded after their associated test has finished Adds a warning diagnostic message for issue that are recorded after their associated test has finished. This commonly occurs when asynchronous work completes after the test its associated test has ended. While responsibility falls on the user in this scenario, clear messaging can point them in the correct direction. Resolves swiftlang#1283 --- .../Event.HumanReadableOutputRecorder.swift | 15 +++++++++++++++ Tests/TestingTests/EventRecorderTests.swift | 19 +++++++++++++++++++ 2 files changed, 34 insertions(+) diff --git a/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift b/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift index c36a764ad..b17834744 100644 --- a/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift +++ b/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift @@ -80,6 +80,9 @@ extension Event { /// The instant at which the test started. var startInstant: Test.Clock.Instant + /// Whether the test has ended. + var hasEnded = false + /// The number of issues recorded for the test, grouped by their /// level of severity. var issueCount: [Issue.Severity: Int] = [:] @@ -335,6 +338,9 @@ extension Event.HumanReadableOutputRecorder { case .testCaseStarted: context.testData[keyPath] = .init(startInstant: instant) + case .testEnded, .testCaseEnded: + context.testData[keyPath]?.hasEnded = true + case let .testCancelled(skipInfo), let .testCaseCancelled(skipInfo): context.testData[keyPath]?.cancellationInfo = skipInfo @@ -477,6 +483,7 @@ extension Event.HumanReadableOutputRecorder { break case let .issueRecorded(issue): + let wasRecordedAfterTestEnded = context.testData[keyPath]?.hasEnded == true let parameterCount = if let parameters = test?.parameters { parameters.count } else { @@ -508,6 +515,14 @@ extension Event.HumanReadableOutputRecorder { additionalMessages.append(Message(symbol: .difference, stringValue: differenceDescription)) } additionalMessages += _formattedComments(issue.comments) + if wasRecordedAfterTestEnded { + additionalMessages.append( + Message( + symbol: .warning, + stringValue: "This issue was recorded after its associated test ended. Ensure asynchronous work has completed before your test ends." + ) + ) + } if let knownIssueComment = issue.knownIssueContext?.comment { additionalMessages.append(_formattedComment(knownIssueComment)) } diff --git a/Tests/TestingTests/EventRecorderTests.swift b/Tests/TestingTests/EventRecorderTests.swift index 7c21e022d..8692ac260 100644 --- a/Tests/TestingTests/EventRecorderTests.swift +++ b/Tests/TestingTests/EventRecorderTests.swift @@ -521,6 +521,25 @@ struct EventRecorderTests { ) } + @Test("HumanReadableOutputRecorder emits guidance for issues recorded after test end") + func humanReadableRecorderExplainsLateIssueRecording() { + let test = Test(name: "example") {} + let context = Event.Context(test: test, testCase: nil, iteration: nil, configuration: nil) + let recorder = Event.HumanReadableOutputRecorder() + + recorder.record(Event(.testStarted, testID: test.id, testCaseID: nil), in: context) + recorder.record(Event(.testEnded, testID: test.id, testCaseID: nil), in: context) + + let issue = Issue(kind: .unconditional, comments: ["Late"], sourceContext: .init()) + let messages = recorder.record(Event(.issueRecorded(issue), testID: test.id, testCaseID: nil), in: context) + + #expect( + messages.contains { message in + message.stringValue.contains("recorded after this test ended") + } + ) + } + @Test("JUnitXMLRecorder counts issues without associated tests") func junitRecorderCountsIssuesWithoutTests() async throws { let issue = Issue(kind: .unconditional) From 25024fea9eb9ac3e37b93aba29a8225c645e9fc2 Mon Sep 17 00:00:00 2001 From: Will Downey Date: Sat, 28 Feb 2026 20:36:17 -0500 Subject: [PATCH 2/6] Record an additional late issue for issues recorded outside test case execution Removed previous changed to HumanReadableEventRecorder. Instead added a computed property on Test.Case indicating whether the test has finished. The computed property accesses modifies a property of a backing class on the Test.Case object. When a testCaseEnded event is posted for a given test case, the hasFinished property is set to true. When issueRecorded events are posted, they are checked to see if the corresponding test case has finished. If the test case has finished, a second late issue is emitted. --- Sources/Testing/Events/Event.swift | 21 ++++++ .../Event.HumanReadableOutputRecorder.swift | 15 ----- Sources/Testing/Issues/Issue.swift | 4 ++ .../Testing/Parameterization/Test.Case.swift | 15 +++++ Sources/Testing/Running/Runner.swift | 1 + Tests/TestingTests/EventRecorderTests.swift | 66 +++++++++++++++---- 6 files changed, 94 insertions(+), 28 deletions(-) diff --git a/Sources/Testing/Events/Event.swift b/Sources/Testing/Events/Event.swift index 2404a60fd..6373c5bf2 100644 --- a/Sources/Testing/Events/Event.swift +++ b/Sources/Testing/Events/Event.swift @@ -264,6 +264,27 @@ public struct Event: Sendable { let event = Event(kind, testID: test?.id, testCaseID: testCase?.id, instant: instant) let context = Event.Context(test: test, testCase: testCase, iteration: iteration, configuration: nil) event._post(in: context, configuration: configuration) + + if case let .issueRecorded(issue) = kind, let testCase, testCase.hasFinished { + // An issue was recorded after the associated test case ended. Record a + // second issue, which emits another issue recorded event, and guard + // against recursing. + guard !issue.isLate else { return } + + var lateRecordedIssue = Issue( + kind: .apiMisused, + severity: .error, + comments: [ + """ + An issue was recorded after its associated test ended. Ensure \ + asynchronous work has completed before your test ends. + """ + ], + sourceContext: issue.sourceContext + ) + lateRecordedIssue.isLate = true + lateRecordedIssue.record() + } } } diff --git a/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift b/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift index b17834744..c36a764ad 100644 --- a/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift +++ b/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift @@ -80,9 +80,6 @@ extension Event { /// The instant at which the test started. var startInstant: Test.Clock.Instant - /// Whether the test has ended. - var hasEnded = false - /// The number of issues recorded for the test, grouped by their /// level of severity. var issueCount: [Issue.Severity: Int] = [:] @@ -338,9 +335,6 @@ extension Event.HumanReadableOutputRecorder { case .testCaseStarted: context.testData[keyPath] = .init(startInstant: instant) - case .testEnded, .testCaseEnded: - context.testData[keyPath]?.hasEnded = true - case let .testCancelled(skipInfo), let .testCaseCancelled(skipInfo): context.testData[keyPath]?.cancellationInfo = skipInfo @@ -483,7 +477,6 @@ extension Event.HumanReadableOutputRecorder { break case let .issueRecorded(issue): - let wasRecordedAfterTestEnded = context.testData[keyPath]?.hasEnded == true let parameterCount = if let parameters = test?.parameters { parameters.count } else { @@ -515,14 +508,6 @@ extension Event.HumanReadableOutputRecorder { additionalMessages.append(Message(symbol: .difference, stringValue: differenceDescription)) } additionalMessages += _formattedComments(issue.comments) - if wasRecordedAfterTestEnded { - additionalMessages.append( - Message( - symbol: .warning, - stringValue: "This issue was recorded after its associated test ended. Ensure asynchronous work has completed before your test ends." - ) - ) - } if let knownIssueComment = issue.knownIssueContext?.comment { additionalMessages.append(_formattedComment(knownIssueComment)) } diff --git a/Sources/Testing/Issues/Issue.swift b/Sources/Testing/Issues/Issue.swift index 299c5d9a2..96cc050e5 100644 --- a/Sources/Testing/Issues/Issue.swift +++ b/Sources/Testing/Issues/Issue.swift @@ -77,6 +77,10 @@ public struct Issue: Sendable { /// The kind of issue this value represents. public var kind: Kind + + /// Whether or not the issue is recorded after its associated test has + /// finished. + var isLate: Bool = false /// An enumeration representing the level of severity of a recorded issue. /// diff --git a/Sources/Testing/Parameterization/Test.Case.swift b/Sources/Testing/Parameterization/Test.Case.swift index ab9183cf8..35837c3a4 100644 --- a/Sources/Testing/Parameterization/Test.Case.swift +++ b/Sources/Testing/Parameterization/Test.Case.swift @@ -233,6 +233,21 @@ extension Test { /// Do not invoke this closure directly. Always use a ``Runner`` to invoke a /// test or test case. var body: @Sendable () async throws -> Void + + /// Whether or not the test case has finished. + var hasFinished: Bool { + get { _executionState.hasFinished.withLock { $0 }} + nonmutating set { _executionState.hasFinished.withLock { $0 = newValue } } + } + + /// The execution state of this test case. + private var _executionState: _ExecutionState = .init() + + /// A backing class that stores execution state for this test case. + private final class _ExecutionState: Sendable { + /// Whether or not the test case has finished executing. + let hasFinished: Mutex = .init(false) + } } /// A type representing a single parameter to a parameterized test function. diff --git a/Sources/Testing/Running/Runner.swift b/Sources/Testing/Running/Runner.swift index 9838ad5b9..f44e6ed98 100644 --- a/Sources/Testing/Running/Runner.swift +++ b/Sources/Testing/Running/Runner.swift @@ -409,6 +409,7 @@ extension Runner { Event.post(.testCaseStarted, for: (step.test, testCase), iteration: context.iteration, configuration: configuration) defer { + testCase.hasFinished = true Event.post(.testCaseEnded, for: (step.test, testCase), iteration: context.iteration, configuration: configuration) } diff --git a/Tests/TestingTests/EventRecorderTests.swift b/Tests/TestingTests/EventRecorderTests.swift index 8692ac260..f5266d7bd 100644 --- a/Tests/TestingTests/EventRecorderTests.swift +++ b/Tests/TestingTests/EventRecorderTests.swift @@ -521,23 +521,63 @@ struct EventRecorderTests { ) } - @Test("HumanReadableOutputRecorder emits guidance for issues recorded after test end") - func humanReadableRecorderExplainsLateIssueRecording() { - let test = Test(name: "example") {} - let context = Event.Context(test: test, testCase: nil, iteration: nil, configuration: nil) - let recorder = Event.HumanReadableOutputRecorder() + @Test("An additional late issue is recorded for issues recorded after test case end") + func lateIssueRecordedAfterTestCaseEnd() async throws { + var configuration = Configuration() + let recordedIssues = Mutex<[Issue]>([]) + configuration.eventHandler = { event, _ in + guard case let .issueRecorded(issue) = event.kind else { return } + recordedIssues.withLock { + $0.append(issue) + } + } - recorder.record(Event(.testStarted, testID: test.id, testCaseID: nil), in: context) - recorder.record(Event(.testEnded, testID: test.id, testCaseID: nil), in: context) + let recordIssueLateTask = Mutex?>(nil) + await Test { + recordIssueLateTask.withLock { + $0 = Task { + try? await Task.sleep(for: .seconds(1)) + Issue.record("Late") + } + } + }.run(configuration: configuration) - let issue = Issue(kind: .unconditional, comments: ["Late"], sourceContext: .init()) - let messages = recorder.record(Event(.issueRecorded(issue), testID: test.id, testCaseID: nil), in: context) + let lateTask = recordIssueLateTask.withLock { $0 } + await lateTask?.value - #expect( - messages.contains { message in - message.stringValue.contains("recorded after this test ended") - } + let issues = recordedIssues.rawValue + let originalIssue = issues[0], lateIssue = issues[1] + #expect(issues.count == 2) + guard case .unconditional = originalIssue.kind else { + Issue.record( + "Unexpected issue kind \(originalIssue.kind)" + ) + return + } + + guard case .apiMisused = lateIssue.kind else { + Issue.record( + "Unexpected issue kind \(originalIssue.kind)" + ) + return + } + + let lateIssueFirstComment = try #require( + lateIssue.comments.first, + "Late issue should have a comment" + ) + + #expect(lateIssueFirstComment.rawValue == + """ + An issue was recorded after its associated test ended. Ensure \ + asynchronous work has completed before your test ends. + """ ) + + let originalIssueSnapshot = Issue.Snapshot(snapshotting: originalIssue) + let lateIssueSnapshot = Issue.Snapshot(snapshotting: lateIssue) + #expect(originalIssueSnapshot.sourceContext == lateIssueSnapshot.sourceContext) + #expect(originalIssueSnapshot.sourceLocation == lateIssueSnapshot.sourceLocation) } @Test("JUnitXMLRecorder counts issues without associated tests") From f32ff3321719a94b4d6f37f919b980607f6c7477 Mon Sep 17 00:00:00 2001 From: Will Downey Date: Sat, 28 Feb 2026 21:01:26 -0500 Subject: [PATCH 3/6] Directly emit another event for late recorded issues Issues that are recorded late directly call the _post function. This removes the need for guarding against recursing and also prevents us from re-computing expensive state (source location, configuration). --- Sources/Testing/Events/Event.swift | 10 ++++------ Tests/TestingTests/EventRecorderTests.swift | 2 +- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/Sources/Testing/Events/Event.swift b/Sources/Testing/Events/Event.swift index 6373c5bf2..d5d02312e 100644 --- a/Sources/Testing/Events/Event.swift +++ b/Sources/Testing/Events/Event.swift @@ -267,13 +267,9 @@ public struct Event: Sendable { if case let .issueRecorded(issue) = kind, let testCase, testCase.hasFinished { // An issue was recorded after the associated test case ended. Record a - // second issue, which emits another issue recorded event, and guard - // against recursing. - guard !issue.isLate else { return } - + // second issue, which emits another issue recorded event. var lateRecordedIssue = Issue( kind: .apiMisused, - severity: .error, comments: [ """ An issue was recorded after its associated test ended. Ensure \ @@ -283,7 +279,9 @@ public struct Event: Sendable { sourceContext: issue.sourceContext ) lateRecordedIssue.isLate = true - lateRecordedIssue.record() + + let lateEvent = Event(.issueRecorded(lateRecordedIssue), testID: test?.id, testCaseID: testCase.id, instant: instant) + lateEvent._post(in: context, configuration: configuration) } } } diff --git a/Tests/TestingTests/EventRecorderTests.swift b/Tests/TestingTests/EventRecorderTests.swift index f5266d7bd..67641fcc1 100644 --- a/Tests/TestingTests/EventRecorderTests.swift +++ b/Tests/TestingTests/EventRecorderTests.swift @@ -536,7 +536,7 @@ struct EventRecorderTests { await Test { recordIssueLateTask.withLock { $0 = Task { - try? await Task.sleep(for: .seconds(1)) + try? await Task.sleep(for: .milliseconds(1)) Issue.record("Late") } } From 89689db846ae9748abc97c856a80f565af4951eb Mon Sep 17 00:00:00 2001 From: Will Downey Date: Sat, 28 Feb 2026 21:08:30 -0500 Subject: [PATCH 4/6] Remove isLate flag on Issue Now that we are directly posting the late issue recorded event, the isLate flag is redundant. --- Sources/Testing/Events/Event.swift | 3 +-- Sources/Testing/Issues/Issue.swift | 4 ---- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/Sources/Testing/Events/Event.swift b/Sources/Testing/Events/Event.swift index d5d02312e..32bcf05dd 100644 --- a/Sources/Testing/Events/Event.swift +++ b/Sources/Testing/Events/Event.swift @@ -268,7 +268,7 @@ public struct Event: Sendable { if case let .issueRecorded(issue) = kind, let testCase, testCase.hasFinished { // An issue was recorded after the associated test case ended. Record a // second issue, which emits another issue recorded event. - var lateRecordedIssue = Issue( + let lateRecordedIssue = Issue( kind: .apiMisused, comments: [ """ @@ -278,7 +278,6 @@ public struct Event: Sendable { ], sourceContext: issue.sourceContext ) - lateRecordedIssue.isLate = true let lateEvent = Event(.issueRecorded(lateRecordedIssue), testID: test?.id, testCaseID: testCase.id, instant: instant) lateEvent._post(in: context, configuration: configuration) diff --git a/Sources/Testing/Issues/Issue.swift b/Sources/Testing/Issues/Issue.swift index 96cc050e5..299c5d9a2 100644 --- a/Sources/Testing/Issues/Issue.swift +++ b/Sources/Testing/Issues/Issue.swift @@ -77,10 +77,6 @@ public struct Issue: Sendable { /// The kind of issue this value represents. public var kind: Kind - - /// Whether or not the issue is recorded after its associated test has - /// finished. - var isLate: Bool = false /// An enumeration representing the level of severity of a recorded issue. /// From 9d11f68d3bb78ce07181a7f26ff6539872ce4335 Mon Sep 17 00:00:00 2001 From: Will Downey Date: Sun, 15 Mar 2026 09:37:20 -0400 Subject: [PATCH 5/6] Track finished items in event handler Utilizes the existing event handler for tracking if issues are recorded to keep a set of finished items (test or test case). When a `testEnded` or `testCaseEnded` event is received, it will add that ID of that item to the set. When `issueRecorded` events are handled we then check if the associated test is in the finished set and emit an additional issue if it is. --- Sources/Testing/Events/Event.swift | 18 ------ Sources/Testing/Issues/Issue.swift | 4 ++ .../Testing/Parameterization/Test.Case.swift | 15 ----- Sources/Testing/Running/Runner.swift | 55 +++++++++++++++++-- Tests/TestingTests/EventRecorderTests.swift | 2 +- 5 files changed, 56 insertions(+), 38 deletions(-) diff --git a/Sources/Testing/Events/Event.swift b/Sources/Testing/Events/Event.swift index 32bcf05dd..2404a60fd 100644 --- a/Sources/Testing/Events/Event.swift +++ b/Sources/Testing/Events/Event.swift @@ -264,24 +264,6 @@ public struct Event: Sendable { let event = Event(kind, testID: test?.id, testCaseID: testCase?.id, instant: instant) let context = Event.Context(test: test, testCase: testCase, iteration: iteration, configuration: nil) event._post(in: context, configuration: configuration) - - if case let .issueRecorded(issue) = kind, let testCase, testCase.hasFinished { - // An issue was recorded after the associated test case ended. Record a - // second issue, which emits another issue recorded event. - let lateRecordedIssue = Issue( - kind: .apiMisused, - comments: [ - """ - An issue was recorded after its associated test ended. Ensure \ - asynchronous work has completed before your test ends. - """ - ], - sourceContext: issue.sourceContext - ) - - let lateEvent = Event(.issueRecorded(lateRecordedIssue), testID: test?.id, testCaseID: testCase.id, instant: instant) - lateEvent._post(in: context, configuration: configuration) - } } } diff --git a/Sources/Testing/Issues/Issue.swift b/Sources/Testing/Issues/Issue.swift index 299c5d9a2..89c1964f0 100644 --- a/Sources/Testing/Issues/Issue.swift +++ b/Sources/Testing/Issues/Issue.swift @@ -77,6 +77,10 @@ public struct Issue: Sendable { /// The kind of issue this value represents. public var kind: Kind + + /// Whether or not the issue is recorded after its associated test has + /// finished. + var isLate: Bool = false /// An enumeration representing the level of severity of a recorded issue. /// diff --git a/Sources/Testing/Parameterization/Test.Case.swift b/Sources/Testing/Parameterization/Test.Case.swift index 35837c3a4..ab9183cf8 100644 --- a/Sources/Testing/Parameterization/Test.Case.swift +++ b/Sources/Testing/Parameterization/Test.Case.swift @@ -233,21 +233,6 @@ extension Test { /// Do not invoke this closure directly. Always use a ``Runner`` to invoke a /// test or test case. var body: @Sendable () async throws -> Void - - /// Whether or not the test case has finished. - var hasFinished: Bool { - get { _executionState.hasFinished.withLock { $0 }} - nonmutating set { _executionState.hasFinished.withLock { $0 = newValue } } - } - - /// The execution state of this test case. - private var _executionState: _ExecutionState = .init() - - /// A backing class that stores execution state for this test case. - private final class _ExecutionState: Sendable { - /// Whether or not the test case has finished executing. - let hasFinished: Mutex = .init(false) - } } /// A type representing a single parameter to a parameterized test function. diff --git a/Sources/Testing/Running/Runner.swift b/Sources/Testing/Running/Runner.swift index f44e6ed98..10f376d24 100644 --- a/Sources/Testing/Running/Runner.swift +++ b/Sources/Testing/Running/Runner.swift @@ -409,7 +409,6 @@ extension Runner { Event.post(.testCaseStarted, for: (step.test, testCase), iteration: context.iteration, configuration: configuration) defer { - testCase.hasFinished = true Event.post(.testCaseEnded, for: (step.test, testCase), iteration: context.iteration, configuration: configuration) } @@ -439,6 +438,13 @@ extension Runner { public func run() async { await Self._run(self) } + + /// An enumeration describing a test or test case that has already finished + /// running. + private enum _FinishedItem: Sendable, Equatable, Hashable { + case test(Test.ID) + case testCase(Test.Case.ID) + } /// Run the tests in a runner's plan with a given configuration. /// @@ -456,11 +462,52 @@ extension Runner { // Track whether or not any issues were recorded across the entire run. let issueRecorded = Mutex(false) + + // Track whether or not an issue is recorded after its test ends. + let finishedItems = Allocated(Mutex>([])) + runner.configuration.eventHandler = { [eventHandler = runner.configuration.eventHandler] event, context in - if case let .issueRecorded(issue) = event.kind, !issue.isKnown { - issueRecorded.withLock { issueRecorded in - issueRecorded = true + switch event.kind { + case .testEnded: + finishedItems.value.withLock { finishedItems in + _ = finishedItems.insert(.test(event.testID!)) + } + case .testCaseEnded: + finishedItems.value.withLock { finishedItems in + _ = finishedItems.insert(.testCase(event.testCaseID!)) + } + case .issueRecorded(let issue): + if !issue.isKnown { + issueRecorded.withLock { issueRecorded in + issueRecorded = true + } + } + + guard !issue.isLate else { break } + + var lateRecordedIssue = Issue( + kind: .apiMisused, + comments: [ + """ + An issue was recorded after its associated test ended. Ensure \ + asynchronous work has completed before your test ends. + """ + ], + sourceContext: issue.sourceContext + ) + lateRecordedIssue.isLate = true + + let finishedItems = finishedItems.value.rawValue + if let testCaseID = event.testCaseID, + finishedItems.contains(.testCase(testCaseID)) { + Event.post(.issueRecorded(lateRecordedIssue)) + + } else if let testID = event.testID, + finishedItems.contains(.test(testID)) { + Event.post(.issueRecorded(lateRecordedIssue)) } + default: + break } eventHandler(event, context) } diff --git a/Tests/TestingTests/EventRecorderTests.swift b/Tests/TestingTests/EventRecorderTests.swift index 67641fcc1..27981a8ec 100644 --- a/Tests/TestingTests/EventRecorderTests.swift +++ b/Tests/TestingTests/EventRecorderTests.swift @@ -546,7 +546,7 @@ struct EventRecorderTests { await lateTask?.value let issues = recordedIssues.rawValue - let originalIssue = issues[0], lateIssue = issues[1] + let originalIssue = issues[1], lateIssue = issues[0] #expect(issues.count == 2) guard case .unconditional = originalIssue.kind else { Issue.record( From e1814ba3f443e23afa24d3066574c88e828cfd9f Mon Sep 17 00:00:00 2001 From: Will Downey Date: Wed, 8 Apr 2026 18:00:55 -0400 Subject: [PATCH 6/6] Track both testID and testCaseID for testCase FinishedItem Storing the Test ID along with the Test Case ID for finished items, allows us to differentiate test cases that belong to different tests. Additionally clean up some of the other logic for recording the late issue. --- Sources/Testing/Running/Runner.swift | 40 +++++++++++++-------- Tests/TestingTests/EventRecorderTests.swift | 5 +-- 2 files changed, 28 insertions(+), 17 deletions(-) diff --git a/Sources/Testing/Running/Runner.swift b/Sources/Testing/Running/Runner.swift index a377da2ce..8a4e6ce2b 100644 --- a/Sources/Testing/Running/Runner.swift +++ b/Sources/Testing/Running/Runner.swift @@ -443,7 +443,7 @@ extension Runner { /// running. private enum _FinishedItem: Sendable, Equatable, Hashable { case test(Test.ID) - case testCase(Test.Case.ID) + case testCase(Test.ID, Test.Case.ID) } /// Run the tests in a runner's plan with a given configuration. @@ -463,19 +463,39 @@ extension Runner { // Track whether or not an issue is recorded after its test ends. let finishedItems = Allocated(Mutex>([])) - runner.configuration.eventHandler = { [eventHandler = runner.configuration.eventHandler] event, context in + runner.configuration.eventHandler = { [oldEventHandler = runner.configuration.eventHandler] event, context in + defer { + oldEventHandler(event, context) + } + + guard let testID = event.testID, let testCaseID = event.testCaseID else { + return + } + switch event.kind { case .testEnded: finishedItems.value.withLock { finishedItems in - _ = finishedItems.insert(.test(event.testID!)) + _ = finishedItems.insert(.test(testID)) } case .testCaseEnded: finishedItems.value.withLock { finishedItems in - _ = finishedItems.insert(.testCase(event.testCaseID!)) + _ = finishedItems.insert(.testCase(testID, testCaseID)) } case .issueRecorded(let issue): guard !issue.isLate else { break } + let shouldRecordIssue = finishedItems.value.withLock { finishedItems in + let testCaseFinished = finishedItems.contains( + .testCase(testID, testCaseID) + ) + let testFinished = finishedItems.contains(.test(testID)) + return testCaseFinished || testFinished + } + + guard shouldRecordIssue else { + break + } + var lateRecordedIssue = Issue( kind: .apiMisused, comments: [ @@ -487,20 +507,10 @@ extension Runner { sourceContext: issue.sourceContext ) lateRecordedIssue.isLate = true - - let finishedItems = finishedItems.value.rawValue - if let testCaseID = event.testCaseID, - finishedItems.contains(.testCase(testCaseID)) { - Event.post(.issueRecorded(lateRecordedIssue)) - - } else if let testID = event.testID, - finishedItems.contains(.test(testID)) { - Event.post(.issueRecorded(lateRecordedIssue)) - } + Event.post(.issueRecorded(lateRecordedIssue)) default: break } - eventHandler(event, context) } // Track whether or not any issues were recorded across the entire run. diff --git a/Tests/TestingTests/EventRecorderTests.swift b/Tests/TestingTests/EventRecorderTests.swift index 27981a8ec..91bb0c84b 100644 --- a/Tests/TestingTests/EventRecorderTests.swift +++ b/Tests/TestingTests/EventRecorderTests.swift @@ -536,7 +536,7 @@ struct EventRecorderTests { await Test { recordIssueLateTask.withLock { $0 = Task { - try? await Task.sleep(for: .milliseconds(1)) + try? await Task.sleep(for: .milliseconds(10)) Issue.record("Late") } } @@ -546,8 +546,9 @@ struct EventRecorderTests { await lateTask?.value let issues = recordedIssues.rawValue - let originalIssue = issues[1], lateIssue = issues[0] #expect(issues.count == 2) + let lateIssue = try #require(issues.first) + let originalIssue = try #require(issues.last) guard case .unconditional = originalIssue.kind else { Issue.record( "Unexpected issue kind \(originalIssue.kind)"