Sink iteration down to a per-test-case iteration

Author: harlanhaskins

Sources/Testing/Running/Runner.swift

@@ -82,9 +82,6 @@ extension Runner {
   private struct _Context: Sendable {
     /// A serializer used to reduce parallelism among test cases.
     var testCaseSerializer: Serializer?
-
-    /// Which iteration of the test plan is being executed.
-    var iteration: Int
   }
 
   /// Apply the custom scope for any test scope providers of the traits
@@ -405,31 +402,79 @@ extension Runner {
   /// This function sets ``Test/Case/current``, then invokes the test case's
   /// body closure.
   private static func _runTestCase(_ testCase: Test.Case, within step: Plan.Step, context: _Context) async {
-    let configuration = _configuration
+    await _applyRepetitionPolicy(test: step.test, testCase: testCase) { iteration in
+      let configuration = _configuration
 
-    Event.post(.testCaseStarted, for: (step.test, testCase), iteration: context.iteration, configuration: configuration)
-    defer {
-      Event.post(.testCaseEnded, for: (step.test, testCase), iteration: context.iteration, configuration: configuration)
-    }
+      Event.post(.testCaseStarted, for: (step.test, testCase), iteration: iteration, configuration: configuration)
+      defer {
+        Event.post(.testCaseEnded, for: (step.test, testCase), iteration: iteration, configuration: configuration)
+      }
+
+      await Test.Case.withCurrent(testCase) {
+        let sourceLocation = step.test.sourceLocation
+        await Issue.withErrorRecording(at: sourceLocation, configuration: configuration) {
+          // Exit early if the task has already been cancelled.
+          try Task.checkCancellation()
 
-    await Test.Case.withCurrent(testCase) {
-      let sourceLocation = step.test.sourceLocation
-      await Issue.withErrorRecording(at: sourceLocation, configuration: configuration) {
-        // Exit early if the task has already been cancelled.
-        try Task.checkCancellation()
+          try await withTimeLimit(for: step.test, configuration: configuration) {
+            try await _applyScopingTraits(for: step.test, testCase: testCase) {
+              try await testCase.body()
+            }
+          } timeoutHandler: { timeLimit in
+            let issue = Issue(
+              kind: .timeLimitExceeded(timeLimitComponents: timeLimit),
+              comments: [],
+              sourceContext: .init(backtrace: .current(), sourceLocation: sourceLocation)
+            )
+            issue.record(configuration: configuration)
+          }
+        }
+      }
+    }
+  }
 
-        try await withTimeLimit(for: step.test, configuration: configuration) {
-          try await _applyScopingTraits(for: step.test, testCase: testCase) {
-            try await testCase.body()
+  /// Applies the repetition policy specified in the current configuration by running the provided test case
+  /// repeatedly until the continuation condition is satisfied.
+  ///
+  /// - Parameters:
+  ///   - test: The test being executed.
+  ///   - testCase: The test case being iterated.
+  ///   - body: The actual body of the function which must ultimately call into the test function.
+  ///
+  /// - Note: This function updates ``Configuration/current`` before invoking the test body.
+  private static func _applyRepetitionPolicy(
+    test: Test,
+    testCase: Test.Case,
+    perform body: (Int) async -> Void
+  ) async {
+    var config = _configuration
+
+    for iteration in 1...config.repetitionPolicy.maximumIterationCount {
+      let issueRecorded = Mutex(false)
+      config.eventHandler = { [eventHandler = config.eventHandler] event, context in
+        if case let .issueRecorded(issue) = event.kind, !issue.isKnown {
+          issueRecorded.withLock { issueRecorded in
+            issueRecorded = true
           }
-        } timeoutHandler: { timeLimit in
-          let issue = Issue(
-            kind: .timeLimitExceeded(timeLimitComponents: timeLimit),
-            comments: [],
-            sourceContext: .init(backtrace: .current(), sourceLocation: sourceLocation)
-          )
-          issue.record(configuration: configuration)
         }
+        eventHandler(event, context)
+      }
+
+      await Configuration.withCurrent(config) {
+        await body(iteration)
+      }
+
+      // Determine if the test plan should iterate again.
+      let shouldContinue = switch config.repetitionPolicy.continuationCondition {
+      case nil:
+        true
+      case .untilIssueRecorded:
+        !issueRecorded.rawValue
+      case .whileIssueRecorded:
+        issueRecorded.rawValue
+      }
+      guard shouldContinue else {
+        break
       }
     }
   }
@@ -453,23 +498,12 @@ extension Runner {
     runner.configureAttachmentHandling()
 #endif
 
-    // Track whether or not any issues were recorded across the entire run.
-    let issueRecorded = Mutex(false)
-    runner.configuration.eventHandler = { [eventHandler = runner.configuration.eventHandler] event, context in
-      if case let .issueRecorded(issue) = event.kind, !issue.isKnown {
-        issueRecorded.withLock { issueRecorded in
-          issueRecorded = true
-        }
-      }
-      eventHandler(event, context)
-    }
-
     // Context to pass into the test run. We intentionally don't pass the Runner
     // itself (implicitly as `self` nor as an argument) because we don't want to
     // accidentally depend on e.g. the `configuration` property rather than the
     // current configuration.
     let context: _Context = {
-      var context = _Context(iteration: 0)
+      var context = _Context()
 
       let maximumParallelizationWidth = runner.configuration.maximumParallelizationWidth
       if maximumParallelizationWidth > 1 && maximumParallelizationWidth < .max {
@@ -493,46 +527,11 @@ extension Runner {
         Event.post(.runEnded, for: (nil, nil), configuration: runner.configuration)
       }
 
-      let repetitionPolicy = runner.configuration.repetitionPolicy
-      let iterationCount = repetitionPolicy.maximumIterationCount
-      for iterationIndex in 0 ..< iterationCount {
-        Event.post(.iterationStarted(iterationIndex), for: (nil, nil), configuration: runner.configuration)
-        defer {
-          Event.post(.iterationEnded(iterationIndex), for: (nil, nil), configuration: runner.configuration)
-        }
-
-        await withTaskGroup { [runner] taskGroup in
-          var taskAction: String?
-          if iterationCount > 1 {
-            taskAction = "running iteration #\(iterationIndex + 1)"
-          }
-          _ = taskGroup.addTaskUnlessCancelled(name: decorateTaskName("test run", withAction: taskAction)) {
-            var iterationContext = context
-            // `iteration` is one-indexed, so offset that here.
-            iterationContext.iteration = iterationIndex + 1
-            try? await _runStep(atRootOf: runner.plan.stepGraph, context: iterationContext)
-          }
-          await taskGroup.waitForAll()
-        }
-
-        // Determine if the test plan should iterate again. (The iteration count
-        // is handled by the outer for-loop.)
-        let shouldContinue = switch repetitionPolicy.continuationCondition {
-        case nil:
-          true
-        case .untilIssueRecorded:
-          !issueRecorded.rawValue
-        case .whileIssueRecorded:
-          issueRecorded.rawValue
-        }
-        guard shouldContinue else {
-          break
-        }
-
-        // Reset the run-wide "issue was recorded" flag for this iteration.
-        issueRecorded.withLock { issueRecorded in
-          issueRecorded = false
+      await withTaskGroup { [runner] taskGroup in
+        _ = taskGroup.addTaskUnlessCancelled(name: decorateTaskName("test run", withAction: nil)) {
+          try? await _runStep(atRootOf: runner.plan.stepGraph, context: context)
         }
+        await taskGroup.waitForAll()
       }
     }
   }

Tests/TestingTests/EventIterationTests.swift

@@ -44,7 +44,7 @@ struct EventIterationTests {
 
       // Verify all expected iterations were recorded
       let iteration = recordedIteration.rawValue
-      #expect(iteration == expectedIterations, "Final observed iteration did not match expected number of iterations", sourceLocation: location)
+      #expect(iteration == expectedIterations, sourceLocation: location)
     }
   }
 
@@ -96,9 +96,7 @@ struct EventIterationTests {
       repetitionPolicy: policy,
       expectedIterations: expectedIterations
     ) { iteration in
-      if iteration < 3 {
-        Issue.record("Failure")
-      }
+      #expect(iteration >= 3)
     }
   }
 }

Tests/TestingTests/PlanIterationTests.swift

@@ -15,16 +15,16 @@ import Synchronization
 #endif
 
 @Suite("Configuration.RepetitionPolicy Tests")
-struct PlanIterationTests {
+struct TestCaseIterationTests {
   @Test("One iteration (default behavior)")
   func oneIteration() async {
-    await confirmation("N iterations started") { started in
-      await confirmation("N iterations ended") { ended in
+    await confirmation("1 iteration started") { started in
+      await confirmation("1 iteration ended") { ended in
         var configuration = Configuration()
         configuration.eventHandler = { event, _ in
-          if case .iterationStarted = event.kind {
+          if case .testStarted = event.kind {
             started()
-          } else if case .iterationEnded = event.kind {
+          } else if case .testEnded = event.kind {
             ended()
           }
         }
@@ -43,9 +43,9 @@ struct PlanIterationTests {
       await confirmation("N iterations ended", expectedCount: iterationCount) { ended in
         var configuration = Configuration()
         configuration.eventHandler = { event, _ in
-          if case .iterationStarted = event.kind {
+          if case .testStarted = event.kind {
             started()
-          } else if case .iterationEnded = event.kind {
+          } else if case .testEnded = event.kind {
             ended()
           }
         }
@@ -62,62 +62,92 @@ struct PlanIterationTests {
 
   @Test("Iteration until issue recorded")
   func iterationUntilIssueRecorded() async {
-    let iterationIndex = Mutex(0)
+    let iterations = Mutex(0)
     let iterationCount = 10
     let iterationWithIssue = 5
     await confirmation("N iterations started", expectedCount: iterationWithIssue + 1) { started in
       await confirmation("N iterations ended", expectedCount: iterationWithIssue + 1) { ended in
         var configuration = Configuration()
-        configuration.eventHandler = { event, _ in
-          if case let .iterationStarted(index) = event.kind {
-            iterationIndex.withLock { iterationIndex in
-              iterationIndex = index
-            }
+        configuration.eventHandler = { event, context in
+          guard let iteration = context.iteration else { return }
+          if case .testStarted = event.kind {
+            iterations.withLock { $0 = iteration }
             started()
-          } else if case .iterationEnded = event.kind {
+          } else if case .testEnded = event.kind {
             ended()
           }
         }
         configuration.repetitionPolicy = .repeating(.untilIssueRecorded, maximumIterationCount: iterationCount)
 
         await Test {
-          if iterationIndex.rawValue == iterationWithIssue {
-            #expect(Bool(false))
-          }
+          #expect(iterations.rawValue < iterationWithIssue)
         }.run(configuration: configuration)
       }
     }
   }
 
   @Test("Iteration while issue recorded")
   func iterationWhileIssueRecorded() async {
-    let iterationIndex = Mutex(0)
+    let iterations = Mutex(0)
     let iterationCount = 10
     let iterationWithoutIssue = 5
-    await confirmation("N iterations started", expectedCount: iterationWithoutIssue + 1) { started in
-      await confirmation("N iterations ended", expectedCount: iterationWithoutIssue + 1) { ended in
+    await confirmation("N iterations started", expectedCount: iterationWithoutIssue) { started in
+      await confirmation("N iterations ended", expectedCount: iterationWithoutIssue) { ended in
         var configuration = Configuration()
-        configuration.eventHandler = { event, _ in
-          if case let .iterationStarted(index) = event.kind {
-            iterationIndex.withLock { iterationIndex in
-              iterationIndex = index
-            }
+        configuration.eventHandler = { event, context in
+          guard let iteration = context.iteration else { return }
+          if case .testStarted = event.kind {
+            iterations.withLock { $0 = iteration }
             started()
-          } else if case .iterationEnded = event.kind {
+          } else if case .testEnded = event.kind {
             ended()
           }
         }
         configuration.repetitionPolicy = .repeating(.whileIssueRecorded, maximumIterationCount: iterationCount)
 
         await Test {
-          if iterationIndex.rawValue < iterationWithoutIssue {
-            #expect(Bool(false))
-          }
+          #expect(iterations.rawValue >= iterationWithoutIssue)
         }.run(configuration: configuration)
       }
     }
   }
 
+  @Test
+  func iterationOnlyRepeatsFailingTest() async {
+    let iterationForFailingTest = Mutex(0)
+    let iterationForSucceedingTest = Mutex(0)
+
+    let iterationCount = 10
+    let iterationWithoutIssue = 5
+
+    var configuration = Configuration()
+    configuration.eventHandler = { event, context in
+      guard let test = context.test, let iteration = context.iteration else { return }
+      if test.name.contains("Failing") {
+        iterationForFailingTest.withLock { $0 = iteration }
+      }
+      if test.name.contains("Succeeding") {
+        iterationForSucceedingTest.withLock { $0 = iteration }
+      }
+    }
+    configuration.repetitionPolicy = .repeating(.whileIssueRecorded, maximumIterationCount: iterationCount)
+
+    let runner = await Runner(testing: [
+      Test(name: "Failing") {
+        #expect(iterationForFailingTest.rawValue >= iterationWithoutIssue)
+      },
+      Test(name: "Succeeding") {
+        #expect(Bool(true))
+      },
+
+    ], configuration: configuration)
+
+    await runner.run()
+
+    #expect(iterationForFailingTest.rawValue == iterationWithoutIssue)
+    #expect(iterationForSucceedingTest.rawValue == 1)
+  }
+
 #if !SWT_NO_EXIT_TESTS
   @Test("Iteration count must be positive")
   func positiveIterationCount() async {