diff --git a/Sources/Testing/ABI/Encoded/ABI.EncodedEvent.swift b/Sources/Testing/ABI/Encoded/ABI.EncodedEvent.swift index 8c496d8e6..357930c2c 100644 --- a/Sources/Testing/ABI/Encoded/ABI.EncodedEvent.swift +++ b/Sources/Testing/ABI/Encoded/ABI.EncodedEvent.swift @@ -117,10 +117,18 @@ extension ABI { case .testStarted: kind = .testStarted case .testCaseStarted: + // For non-parameterized tests, we elide `testCaseStarted` calls because it would be + // redundant. However, for multiple iterations of a test case within a non-parameterized + // function, we need to emit another `testStarted` event. if eventContext.test?.isParameterized == false { - return nil + if let iteration = eventContext.iteration, iteration > 1 { + kind = .testStarted + } else { + return nil + } + } else { + kind = .testCaseStarted } - kind = .testCaseStarted case let .issueRecorded(recordedIssue): kind = .issueRecorded issue = EncodedIssue(encoding: recordedIssue, in: eventContext) @@ -129,9 +137,14 @@ extension ABI { self.attachment = EncodedAttachment(encoding: attachment) case .testCaseEnded: if eventContext.test?.isParameterized == false { - return nil + if let iteration = eventContext.iteration, iteration > 1 { + kind = .testEnded + } else { + return nil + } + } else { + kind = .testCaseEnded } - kind = .testCaseEnded case .testCaseCancelled: kind = .testCaseCancelled case .testEnded: @@ -164,6 +177,7 @@ extension ABI { let .testCancelled(skipInfo): _comments = Array(skipInfo.comment).map(\.rawValue) _sourceLocation = skipInfo.sourceLocation.map { EncodedSourceLocation(encoding: $0) } + _iteration = eventContext.iteration default: break } diff --git a/Sources/Testing/ABI/EntryPoints/EntryPoint.swift b/Sources/Testing/ABI/EntryPoints/EntryPoint.swift index 64cef7759..a484f01b4 100644 --- a/Sources/Testing/ABI/EntryPoints/EntryPoint.swift +++ b/Sources/Testing/ABI/EntryPoints/EntryPoint.swift @@ -333,6 +333,9 @@ public struct __CommandLineArguments_v0: Sendable { /// The value of the `--repeat-until` argument. public var repeatUntil: String? + /// Whether or not to use the per-test-case repetition mode. + var usePerTestCaseRepetition: Bool = false + /// The value of the `--attachments-path` argument. public var attachmentsPath: String? } @@ -537,6 +540,9 @@ func parseCommandLineArguments(from args: [String]) throws -> __CommandLineArgum if let repeatUntil = args.argumentValue(forLabel: "--repeat-until") { result.repeatUntil = repeatUntil } + if args.contains("--experimental-per-test-case-repetition") { + result.usePerTestCaseRepetition = true + } return result } @@ -665,6 +671,9 @@ public func configurationForEntryPoint(from args: __CommandLineArguments_v0) thr } configuration.repetitionPolicy = repetitionPolicy + // Opt in to per-test-case repetition + configuration.shouldUseLegacyPlanLevelRepetition = !args.usePerTestCaseRepetition + #if !SWT_NO_EXIT_TESTS // Enable exit test handling via __swiftPMEntryPoint(). configuration.exitTestHandler = ExitTest.handlerForEntryPoint() diff --git a/Sources/Testing/Events/Event.swift b/Sources/Testing/Events/Event.swift index db2066c9f..0d7354636 100644 --- a/Sources/Testing/Events/Event.swift +++ b/Sources/Testing/Events/Event.swift @@ -238,7 +238,6 @@ public struct Event: Sendable { static func post( _ kind: Kind, for testAndTestCase: (Test?, Test.Case?) = currentTestAndTestCase(), - iteration: Int? = nil, instant: Test.Clock.Instant = .now, configuration: Configuration? = nil ) { @@ -262,7 +261,12 @@ public struct Event: Sendable { } } let event = Event(kind, testID: test?.id, testCaseID: testCase?.id, instant: instant) - let context = Event.Context(test: test, testCase: testCase, iteration: iteration, configuration: nil) + let context = Event.Context( + test: test, + testCase: testCase, + iteration: Test.currentIteration, + configuration: nil + ) event._post(in: context, configuration: configuration) } } @@ -327,8 +331,6 @@ extension Event { iteration: Int?, configuration: Configuration? ) { - // Ensure that if `iteration` is specified, the test is also specified. - precondition(iteration == nil || (iteration != nil && test != nil)) self.test = test self.testCase = testCase self.iteration = iteration diff --git a/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift b/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift index 8d4277d75..4fe30be10 100644 --- a/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift +++ b/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift @@ -365,6 +365,10 @@ extension Event.HumanReadableOutputRecorder { String(describing: TimeValue(rawValue: end.suspending.rawValue - start.suspending.rawValue)) } + func testStartedMessage(for test: Test) -> String { + "\(_capitalizedTitle(for: test)) \(testName) started" + } + // Finally, produce any messages for the event. switch event.kind { case .testDiscovered: @@ -425,7 +429,7 @@ extension Event.HumanReadableOutputRecorder { return [ Message( symbol: .default, - stringValue: "\(_capitalizedTitle(for: test)) \(testName) started." + stringValue: "\(testStartedMessage(for: test))." ) ] @@ -565,15 +569,26 @@ extension Event.HumanReadableOutputRecorder { return result case .testCaseStarted: - guard let testCase, testCase.isParameterized, let arguments = testCase.arguments else { + guard let test, let testCase else { break } + let iteration = eventContext.iteration ?? 1 + + var message: String + if testCase.isParameterized, let arguments = testCase.arguments { + message = "Test case passing \(arguments.count.counting("argument")) \(testCase.labeledArguments(includingQualifiedTypeNames: verbosity > 0)) to \(testName) started" + } else if iteration > 1 { + message = testStartedMessage(for: test) + } else { break } + if iteration > 1 { + message += " (repetition \(iteration))" + } + + message += "." + return [ - Message( - symbol: .default, - stringValue: "Test case passing \(arguments.count.counting("argument")) \(testCase.labeledArguments(includingQualifiedTypeNames: verbosity > 0)) to \(testName) started." - ) + Message(symbol: .default, stringValue: message) ] case .testCaseEnded: diff --git a/Sources/Testing/Running/Configuration.swift b/Sources/Testing/Running/Configuration.swift index 4c028aff2..7f8a70e09 100644 --- a/Sources/Testing/Running/Configuration.swift +++ b/Sources/Testing/Running/Configuration.swift @@ -132,6 +132,11 @@ public struct Configuration: Sendable { } } + /// Whether to perform test repetition at the plan level or on a per-test- + /// case basis. + @_spi(Experimental) + public var shouldUseLegacyPlanLevelRepetition: Bool = true + /// Whether or not, and how, to iterate the test plan repeatedly. /// /// By default, the value of this property allows for a single iteration. diff --git a/Sources/Testing/Running/Runner.RuntimeState.swift b/Sources/Testing/Running/Runner.RuntimeState.swift index a9f1819d4..a3f0ebb2b 100644 --- a/Sources/Testing/Running/Runner.RuntimeState.swift +++ b/Sources/Testing/Running/Runner.RuntimeState.swift @@ -29,6 +29,9 @@ extension Runner { /// The test case that is running on the current task, if any. var testCase: Test.Case? + /// The current iteration of the test repetition policy, if any. + var iteration: Int? + /// The runtime state related to the runner running on the current task, /// if any. @TaskLocal @@ -198,7 +201,7 @@ extension Configuration { } } -// MARK: - Current test and test case +// MARK: - Current test, test case, and iteration extension Test { /// The test that is running on the current task, if any. @@ -233,6 +236,28 @@ extension Test { try await test.withCancellationHandling(body) } } + + /// The current iteration of the currently-running test case, if any. + static var currentIteration: Int? { + Runner.RuntimeState.current?.iteration + } + + /// Call a function while the value of ``Test/currentIteration`` is set. + /// + /// - Parameters: + /// - iteration: The new value to set for ``Test/currentIteration``. + /// - body: A function to call. + /// + /// - Returns: Whatever is returned by `body`. + /// + /// - Throws: Whatever is thrown by `body`. + static func withCurrentIteration(_ iteration: Int?, perform body: () async throws -> R) async rethrows -> R { + var runtimeState = Runner.RuntimeState.current ?? .init() + runtimeState.iteration = iteration + return try await Runner.RuntimeState.$current.withValue(runtimeState) { + try await body() + } + } } extension Test.Case { diff --git a/Sources/Testing/Running/Runner.swift b/Sources/Testing/Running/Runner.swift index 36d42c5c1..9cd19a3a3 100644 --- a/Sources/Testing/Running/Runner.swift +++ b/Sources/Testing/Running/Runner.swift @@ -82,9 +82,6 @@ extension Runner { private struct _Context: Sendable { /// A serializer used to reduce parallelism among test cases. var testCaseSerializer: Serializer? - - /// Which iteration of the test plan is being executed. - var iteration: Int } /// Apply the custom scope for any test scope providers of the traits @@ -230,10 +227,10 @@ extension Runner { // Determine what kind of event to send for this step based on its action. switch step.action { case .run: - Event.post(.testStarted, for: (step.test, nil), iteration: context.iteration, configuration: configuration) + Event.post(.testStarted, for: (step.test, nil), configuration: configuration) shouldSendTestEnded = true case let .skip(skipInfo): - Event.post(.testSkipped(skipInfo), for: (step.test, nil), iteration: context.iteration, configuration: configuration) + Event.post(.testSkipped(skipInfo), for: (step.test, nil), configuration: configuration) shouldSendTestEnded = false case let .recordIssue(issue): // Scope posting the issue recorded event such that issue handling @@ -257,7 +254,7 @@ extension Runner { defer { if let step = stepGraph.value { if shouldSendTestEnded { - Event.post(.testEnded, for: (step.test, nil), iteration: context.iteration, configuration: configuration) + Event.post(.testEnded, for: (step.test, nil), configuration: configuration) } Event.post(.planStepEnded(step), for: (step.test, nil), configuration: configuration) } @@ -388,9 +385,9 @@ extension Runner { if let testCaseSerializer = context.testCaseSerializer { // Note that if .serialized is applied to an inner scope, we still use // this serializer (if set) so that we don't overcommit. - await testCaseSerializer.run { await _runTestCase(testCase, within: step, context: context) } + await testCaseSerializer.run { await _runTestCase(testCase, within: step) } } else { - await _runTestCase(testCase, within: step, context: context) + await _runTestCase(testCase, within: step) } } } @@ -400,16 +397,36 @@ extension Runner { /// - Parameters: /// - testCase: The test case to run. /// - step: The runner plan step associated with this test case. - /// - context: Context for the test run. /// /// This function sets ``Test/Case/current``, then invokes the test case's /// body closure. - private static func _runTestCase(_ testCase: Test.Case, within step: Plan.Step, context: _Context) async { + private static func _runTestCase( + _ testCase: Test.Case, + within step: Plan.Step + ) async { + if _configuration.shouldUseLegacyPlanLevelRepetition { + await _runSingleTestCaseIteration(testCase, within: step) + } else { + await _applyRepetitionPolicy { + await _runSingleTestCaseIteration(testCase, within: step) + } + } + } + + /// Run a single iteration of a test case. + /// + /// - Parameters: + /// - testCase: The test case to run. + /// - step: The runner plan step associated with this test case. + /// + /// This function sets ``Test/Case/current``, then invokes the test case's + /// body closure. + private static func _runSingleTestCaseIteration(_ testCase: Test.Case, within step: Plan.Step) async { let configuration = _configuration - Event.post(.testCaseStarted, for: (step.test, testCase), iteration: context.iteration, configuration: configuration) + Event.post(.testCaseStarted, for: (step.test, testCase), configuration: configuration) defer { - Event.post(.testCaseEnded, for: (step.test, testCase), iteration: context.iteration, configuration: configuration) + Event.post(.testCaseEnded, for: (step.test, testCase), configuration: configuration) } await Test.Case.withCurrent(testCase) { @@ -434,6 +451,47 @@ extension Runner { } } + /// Applies the repetition policy specified in the current configuration by running the provided test case + /// repeatedly until the continuation condition is satisfied. + /// + /// - Parameters: + /// - body: The actual body of the function which must ultimately call into the test function. + /// + /// - Note: This function updates ``Configuration/current`` before invoking the test body. + private static func _applyRepetitionPolicy( + perform body: () async -> Void + ) async { + for iteration in 1..._configuration.repetitionPolicy.maximumIterationCount { + let issueRecorded = Atomic(false) + var config = _configuration + config.eventHandler = { [eventHandler = config.eventHandler] event, context in + if case let .issueRecorded(issue) = event.kind, !issue.isKnown { + issueRecorded.store(true, ordering: .sequentiallyConsistent) + } + eventHandler(event, context) + } + + await Test.withCurrentIteration(iteration) { + await Configuration.withCurrent(config) { + await body() + } + } + + // Determine if the test plan should iterate again. + let shouldContinue = switch config.repetitionPolicy.continuationCondition { + case nil: + true + case .untilIssueRecorded: + !issueRecorded.load(ordering: .sequentiallyConsistent) + case .whileIssueRecorded: + issueRecorded.load(ordering: .sequentiallyConsistent) + } + guard shouldContinue else { + break + } + } + } + /// Run the tests in this runner's plan. public func run() async { await Self._run(self) @@ -454,21 +512,12 @@ extension Runner { #endif _ = Event.installFallbackEventHandler() - // Track whether or not any issues were recorded across the entire run. - let issueRecorded = Atomic(false) - runner.configuration.eventHandler = { [eventHandler = runner.configuration.eventHandler] event, context in - if case let .issueRecorded(issue) = event.kind, !issue.isKnown { - issueRecorded.store(true, ordering: .sequentiallyConsistent) - } - eventHandler(event, context) - } - // Context to pass into the test run. We intentionally don't pass the Runner // itself (implicitly as `self` nor as an argument) because we don't want to // accidentally depend on e.g. the `configuration` property rather than the // current configuration. let context: _Context = { - var context = _Context(iteration: 0) + var context = _Context() let maximumParallelizationWidth = runner.configuration.maximumParallelizationWidth if maximumParallelizationWidth > 1 && maximumParallelizationWidth < .max { @@ -492,45 +541,30 @@ extension Runner { Event.post(.runEnded, for: (nil, nil), configuration: runner.configuration) } - let repetitionPolicy = runner.configuration.repetitionPolicy - let iterationCount = repetitionPolicy.maximumIterationCount - for iterationIndex in 0 ..< iterationCount { - Event.post(.iterationStarted(iterationIndex), for: (nil, nil), configuration: runner.configuration) - defer { - Event.post(.iterationEnded(iterationIndex), for: (nil, nil), configuration: runner.configuration) - } + if runner.configuration.shouldUseLegacyPlanLevelRepetition { + await _applyRepetitionPolicy { [runner] in + let iteration = Test.currentIteration ?? 1 - await withTaskGroup { [runner] taskGroup in - var taskAction: String? - if iterationCount > 1 { - taskAction = "running iteration #\(iterationIndex + 1)" - } - _ = taskGroup.addTaskUnlessCancelled(name: decorateTaskName("test run", withAction: taskAction)) { - var iterationContext = context - // `iteration` is one-indexed, so offset that here. - iterationContext.iteration = iterationIndex + 1 - try? await _runStep(atRootOf: runner.plan.stepGraph, context: iterationContext) + // Legacy clients expect these values to be zero-indexed. + let iterationIndex = iteration - 1 + Event.post(.iterationStarted(iterationIndex), configuration: runner.configuration) + defer { + Event.post(.iterationEnded(iterationIndex), configuration: runner.configuration) } - await taskGroup.waitForAll() - } - - // Determine if the test plan should iterate again. (The iteration count - // is handled by the outer for-loop.) - let shouldContinue = switch repetitionPolicy.continuationCondition { - case nil: - true - case .untilIssueRecorded: - !issueRecorded.load(ordering: .sequentiallyConsistent) - case .whileIssueRecorded: - issueRecorded.load(ordering: .sequentiallyConsistent) - } - guard shouldContinue else { - break + await runner._runAllTests(context: context) } + } else { + await runner._runAllTests(context: context) + } + } + } - // Reset the run-wide "issue was recorded" flag for this iteration. - issueRecorded.store(false, ordering: .sequentiallyConsistent) + private func _runAllTests(context: _Context) async { + await withTaskGroup { taskGroup in + _ = taskGroup.addTaskUnlessCancelled(name: decorateTaskName("test run", withAction: nil)) { + try? await Self._runStep(atRootOf: plan.stepGraph, context: context) } + await taskGroup.waitForAll() } } } diff --git a/Tests/TestingTests/EventIterationTests.swift b/Tests/TestingTests/EventIterationTests.swift index 40f9a8c27..43131c3c3 100644 --- a/Tests/TestingTests/EventIterationTests.swift +++ b/Tests/TestingTests/EventIterationTests.swift @@ -44,7 +44,7 @@ struct EventIterationTests { // Verify all expected iterations were recorded let iteration = recordedIteration.load(ordering: .sequentiallyConsistent) - #expect(iteration == expectedIterations, "Final observed iteration did not match expected number of iterations", sourceLocation: location) + #expect(iteration == expectedIterations, sourceLocation: location) } } @@ -60,17 +60,6 @@ struct EventIterationTests { } } - @Test - func `testStarted and testEnded events include iteration in context`() async { - await verifyIterations( - for: [.testStarted, .testEnded], - repetitionPolicy: .once, - expectedIterations: 1 - ) { _ in - // Do nothing, just pass - } - } - @Test func `testCaseStarted and testCaseEnded events include iteration in context`() async { await verifyIterations( @@ -96,9 +85,7 @@ struct EventIterationTests { repetitionPolicy: policy, expectedIterations: expectedIterations ) { iteration in - if iteration < 3 { - Issue.record("Failure") - } + #expect(iteration >= 3) } } } diff --git a/Tests/TestingTests/PlanIterationTests.swift b/Tests/TestingTests/LegacyPlanIterationTests.swift similarity index 98% rename from Tests/TestingTests/PlanIterationTests.swift rename to Tests/TestingTests/LegacyPlanIterationTests.swift index ceeb36576..ec713d6f2 100644 --- a/Tests/TestingTests/PlanIterationTests.swift +++ b/Tests/TestingTests/LegacyPlanIterationTests.swift @@ -14,8 +14,8 @@ import Synchronization #endif -@Suite("Configuration.RepetitionPolicy Tests") -struct PlanIterationTests { +@Suite +struct LegacyPlanIterationTests { @Test("One iteration (default behavior)") func oneIteration() async { await confirmation("N iterations started") { started in diff --git a/Tests/TestingTests/SwiftPMTests.swift b/Tests/TestingTests/SwiftPMTests.swift index 75e07a5d1..0bca3d8ee 100644 --- a/Tests/TestingTests/SwiftPMTests.swift +++ b/Tests/TestingTests/SwiftPMTests.swift @@ -511,6 +511,15 @@ struct SwiftPMTests { #expect(configuration.repetitionPolicy.continuationCondition == .whileIssueRecorded) } + @Test("--experimental-per-test-case-repetition") + func experimentalPerTestCaseRepetition() throws { + let defaultConfig = try configurationForEntryPoint(withArguments: ["PATH"]) + #expect(defaultConfig.shouldUseLegacyPlanLevelRepetition) + + let configuration = try configurationForEntryPoint(withArguments: ["PATH", "--experimental-per-test-case-repetition"]) + #expect(!configuration.shouldUseLegacyPlanLevelRepetition) + } + @Test("list subcommand") func list() async throws { do { diff --git a/Tests/TestingTests/TestCaseIterationTests.swift b/Tests/TestingTests/TestCaseIterationTests.swift new file mode 100644 index 000000000..ea25837a1 --- /dev/null +++ b/Tests/TestingTests/TestCaseIterationTests.swift @@ -0,0 +1,122 @@ +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2026 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for Swift project authors +// + +@testable @_spi(Experimental) @_spi(ForToolsIntegrationOnly) import Testing + +#if !SWT_TARGET_OS_APPLE && canImport(Synchronization) +import Synchronization +#endif + +@Suite +struct TestCaseIterationTests { + @Test("One iteration (default behavior)") + func oneIteration() async { + await confirmation("N iterations started") { started in + await confirmation("N iterations ended") { ended in + var configuration = Configuration() + configuration.shouldUseLegacyPlanLevelRepetition = false + configuration.eventHandler = { event, _ in + if case .testCaseStarted = event.kind { + started() + } else if case .testCaseEnded = event.kind { + ended() + } + } + configuration.repetitionPolicy = .once + + await Test { + }.run(configuration: configuration) + } + } + } + + @Test("Unconditional iteration") + func unconditionalIteration() async { + let iterationCount = 10 + await confirmation("N iterations started", expectedCount: iterationCount) { started in + await confirmation("N iterations ended", expectedCount: iterationCount) { ended in + var configuration = Configuration() + configuration.shouldUseLegacyPlanLevelRepetition = false + configuration.eventHandler = { event, _ in + if case .testCaseStarted = event.kind { + started() + } else if case .testCaseEnded = event.kind { + ended() + } + } + configuration.repetitionPolicy = .repeating(maximumIterationCount: iterationCount) + + await Test { + if Bool.random() { + #expect(Bool(false)) + } + }.run(configuration: configuration) + } + } + } + + @Test("Iteration until issue recorded") + func iterationUntilIssueRecorded() async { + let iterations = Atomic(0) + let iterationCount = 10 + let iterationWithIssue = 5 + await confirmation("N iterations started", expectedCount: iterationWithIssue) { started in + await confirmation("N iterations ended", expectedCount: iterationWithIssue) { ended in + var configuration = Configuration() + configuration.shouldUseLegacyPlanLevelRepetition = false + configuration.eventHandler = { event, context in + guard let iteration = context.iteration else { return } + if case .testCaseStarted = event.kind { + iterations.store(iteration, ordering: .sequentiallyConsistent) + started() + } else if case .testCaseEnded = event.kind { + ended() + } + } + configuration.repetitionPolicy = .repeating(.untilIssueRecorded, maximumIterationCount: iterationCount) + + await Test { + let iterations = iterations.load(ordering: .sequentiallyConsistent) + #expect(iterations < iterationWithIssue) + }.run(configuration: configuration) + } + } + } + + @Test("Iteration while issue recorded") + func iterationWhileIssueRecorded() async { + let iterations = Atomic(0) + let iterationCount = 10 + let iterationWithoutIssue = 5 + await confirmation("N iterations started", expectedCount: iterationWithoutIssue) { started in + await confirmation("N iterations ended", expectedCount: iterationWithoutIssue) { ended in + var configuration = Configuration() + configuration.shouldUseLegacyPlanLevelRepetition = false + configuration.eventHandler = { event, context in + guard let iteration = context.iteration else { return } + if case .testCaseStarted = event.kind { + iterations.store(iteration, ordering: .sequentiallyConsistent) + started() + } else if case .testCaseEnded = event.kind { + ended() + } + } + configuration.repetitionPolicy = .repeating(.whileIssueRecorded, maximumIterationCount: iterationCount) + + await Test { + let iterations = iterations.load(ordering: .sequentiallyConsistent) + if iterations < iterationWithoutIssue { + #expect(Bool(false)) + } + }.run(configuration: configuration) + } + } + } +}