Sink iteration down to a per-test-case iteration

Author: harlanhaskins

Sources/Testing/ABI/Encoded/ABI.EncodedEvent.swift

@@ -118,9 +118,14 @@ extension ABI {
         kind = .testStarted
       case .testCaseStarted:
         if eventContext.test?.isParameterized == false {
-          return nil
+          if let iteration = eventContext.iteration, iteration > 1 {
+            kind = .testStarted
+          } else {
+            return nil
+          }
+        } else {
+          kind = .testCaseStarted
         }
-        kind = .testCaseStarted
       case let .issueRecorded(recordedIssue):
         kind = .issueRecorded
         issue = EncodedIssue(encoding: recordedIssue, in: eventContext)
@@ -129,9 +134,14 @@ extension ABI {
         self.attachment = EncodedAttachment(encoding: attachment)
       case .testCaseEnded:
         if eventContext.test?.isParameterized == false {
-          return nil
+          if let iteration = eventContext.iteration, iteration > 1 {
+            kind = .testEnded
+          } else {
+            return nil
+          }
+        } else {
+          kind = .testCaseEnded
         }
-        kind = .testCaseEnded
       case .testCaseCancelled:
         kind = .testCaseCancelled
       case .testEnded:
@@ -164,6 +174,7 @@ extension ABI {
           let .testCancelled(skipInfo):
           _comments = Array(skipInfo.comment).map(\.rawValue)
           _sourceLocation = skipInfo.sourceLocation.map { EncodedSourceLocation(encoding: $0) }
+          _iteration = eventContext.iteration
         default:
           break
         }

Sources/Testing/Running/Runner.swift

@@ -82,9 +82,6 @@ extension Runner {
   private struct _Context: Sendable {
     /// A serializer used to reduce parallelism among test cases.
     var testCaseSerializer: Serializer<Void>?
-
-    /// Which iteration of the test plan is being executed.
-    var iteration: Int
   }
 
   /// Apply the custom scope for any test scope providers of the traits
@@ -230,10 +227,10 @@ extension Runner {
       // Determine what kind of event to send for this step based on its action.
       switch step.action {
       case .run:
-        Event.post(.testStarted, for: (step.test, nil), iteration: context.iteration, configuration: configuration)
+        Event.post(.testStarted, for: (step.test, nil), iteration: 1, configuration: configuration)
         shouldSendTestEnded = true
       case let .skip(skipInfo):
-        Event.post(.testSkipped(skipInfo), for: (step.test, nil), iteration: context.iteration, configuration: configuration)
+        Event.post(.testSkipped(skipInfo), for: (step.test, nil), iteration: 1, configuration: configuration)
         shouldSendTestEnded = false
       case let .recordIssue(issue):
         // Scope posting the issue recorded event such that issue handling
@@ -257,7 +254,7 @@ extension Runner {
     defer {
       if let step = stepGraph.value {
         if shouldSendTestEnded {
-          Event.post(.testEnded, for: (step.test, nil), iteration: context.iteration, configuration: configuration)
+          Event.post(.testEnded, for: (step.test, nil), iteration: 1, configuration: configuration)
         }
         Event.post(.planStepEnded(step), for: (step.test, nil), configuration: configuration)
       }
@@ -405,35 +402,81 @@ extension Runner {
   /// This function sets ``Test/Case/current``, then invokes the test case's
   /// body closure.
   private static func _runTestCase(_ testCase: Test.Case, within step: Plan.Step, context: _Context) async {
-    let configuration = _configuration
+    await _applyRepetitionPolicy(test: step.test, testCase: testCase) { iteration in
+      let configuration = _configuration
 
-    Event.post(.testCaseStarted, for: (step.test, testCase), iteration: context.iteration, configuration: configuration)
-    defer {
-      Event.post(.testCaseEnded, for: (step.test, testCase), iteration: context.iteration, configuration: configuration)
-    }
+      Event.post(.testCaseStarted, for: (step.test, testCase), iteration: iteration, configuration: configuration)
+      defer {
+        Event.post(.testCaseEnded, for: (step.test, testCase), iteration: iteration, configuration: configuration)
+      }
 
-    await Test.Case.withCurrent(testCase) {
-      let sourceLocation = step.test.sourceLocation
-      await Issue.withErrorRecording(at: sourceLocation, configuration: configuration) {
-        // Exit early if the task has already been cancelled.
-        try Task.checkCancellation()
+      await Test.Case.withCurrent(testCase) {
+        let sourceLocation = step.test.sourceLocation
+        await Issue.withErrorRecording(at: sourceLocation, configuration: configuration) {
+          // Exit early if the task has already been cancelled.
+          try Task.checkCancellation()
 
-        try await withTimeLimit(for: step.test, configuration: configuration) {
-          try await _applyScopingTraits(for: step.test, testCase: testCase) {
-            try await testCase.body()
+          try await withTimeLimit(for: step.test, configuration: configuration) {
+            try await _applyScopingTraits(for: step.test, testCase: testCase) {
+              try await testCase.body()
+            }
+          } timeoutHandler: { timeLimit in
+            let issue = Issue(
+              kind: .timeLimitExceeded(timeLimit: timeLimit),
+              comments: [],
+              sourceContext: .init(backtrace: .current(), sourceLocation: sourceLocation)
+            )
+            issue.record(configuration: configuration)
           }
-        } timeoutHandler: { timeLimit in
-          let issue = Issue(
-            kind: .timeLimitExceeded(timeLimit: timeLimit),
-            comments: [],
-            sourceContext: .init(backtrace: .current(), sourceLocation: sourceLocation)
-          )
-          issue.record(configuration: configuration)
         }
       }
     }
   }
 
+  /// Applies the repetition policy specified in the current configuration by running the provided test case
+  /// repeatedly until the continuation condition is satisfied.
+  ///
+  /// - Parameters:
+  ///   - test: The test being executed.
+  ///   - testCase: The test case being iterated.
+  ///   - body: The actual body of the function which must ultimately call into the test function.
+  ///
+  /// - Note: This function updates ``Configuration/current`` before invoking the test body.
+  private static func _applyRepetitionPolicy(
+    test: Test,
+    testCase: Test.Case,
+    perform body: (Int) async -> Void
+  ) async {
+    var config = _configuration
+
+    for iteration in 1...config.repetitionPolicy.maximumIterationCount {
+      let issueRecorded = Atomic(false)
+      config.eventHandler = { [eventHandler = config.eventHandler] event, context in
+        if case let .issueRecorded(issue) = event.kind, !issue.isKnown {
+          issueRecorded.store(true, ordering: .sequentiallyConsistent)
+        }
+        eventHandler(event, context)
+      }
+
+      await Configuration.withCurrent(config) {
+        await body(iteration)
+      }
+
+      // Determine if the test plan should iterate again.
+      let shouldContinue = switch config.repetitionPolicy.continuationCondition {
+      case nil:
+        true
+      case .untilIssueRecorded:
+        !issueRecorded.load(ordering: .sequentiallyConsistent)
+      case .whileIssueRecorded:
+        issueRecorded.load(ordering: .sequentiallyConsistent)
+      }
+      guard shouldContinue else {
+        break
+      }
+    }
+  }
+
   /// Run the tests in this runner's plan.
   public func run() async {
     await Self._run(self)
@@ -454,21 +497,12 @@ extension Runner {
 #endif
     _ = Event.installFallbackEventHandler()
 
-    // Track whether or not any issues were recorded across the entire run.
-    let issueRecorded = Atomic(false)
-    runner.configuration.eventHandler = { [eventHandler = runner.configuration.eventHandler] event, context in
-      if case let .issueRecorded(issue) = event.kind, !issue.isKnown {
-        issueRecorded.store(true, ordering: .sequentiallyConsistent)
-      }
-      eventHandler(event, context)
-    }
-
     // Context to pass into the test run. We intentionally don't pass the Runner
     // itself (implicitly as `self` nor as an argument) because we don't want to
     // accidentally depend on e.g. the `configuration` property rather than the
     // current configuration.
     let context: _Context = {
-      var context = _Context(iteration: 0)
+      var context = _Context()
 
       let maximumParallelizationWidth = runner.configuration.maximumParallelizationWidth
       if maximumParallelizationWidth > 1 && maximumParallelizationWidth < .max {
@@ -492,44 +526,11 @@ extension Runner {
         Event.post(.runEnded, for: (nil, nil), configuration: runner.configuration)
       }
 
-      let repetitionPolicy = runner.configuration.repetitionPolicy
-      let iterationCount = repetitionPolicy.maximumIterationCount
-      for iterationIndex in 0 ..< iterationCount {
-        Event.post(.iterationStarted(iterationIndex), for: (nil, nil), configuration: runner.configuration)
-        defer {
-          Event.post(.iterationEnded(iterationIndex), for: (nil, nil), configuration: runner.configuration)
+      await withTaskGroup { [runner] taskGroup in
+        _ = taskGroup.addTaskUnlessCancelled(name: decorateTaskName("test run", withAction: nil)) {
+          try? await _runStep(atRootOf: runner.plan.stepGraph, context: context)
         }
-
-        await withTaskGroup { [runner] taskGroup in
-          var taskAction: String?
-          if iterationCount > 1 {
-            taskAction = "running iteration #\(iterationIndex + 1)"
-          }
-          _ = taskGroup.addTaskUnlessCancelled(name: decorateTaskName("test run", withAction: taskAction)) {
-            var iterationContext = context
-            // `iteration` is one-indexed, so offset that here.
-            iterationContext.iteration = iterationIndex + 1
-            try? await _runStep(atRootOf: runner.plan.stepGraph, context: iterationContext)
-          }
-          await taskGroup.waitForAll()
-        }
-
-        // Determine if the test plan should iterate again. (The iteration count
-        // is handled by the outer for-loop.)
-        let shouldContinue = switch repetitionPolicy.continuationCondition {
-        case nil:
-          true
-        case .untilIssueRecorded:
-          !issueRecorded.load(ordering: .sequentiallyConsistent)
-        case .whileIssueRecorded:
-          issueRecorded.load(ordering: .sequentiallyConsistent)
-        }
-        guard shouldContinue else {
-          break
-        }
-
-        // Reset the run-wide "issue was recorded" flag for this iteration.
-        issueRecorded.store(false, ordering: .sequentiallyConsistent)
+        await taskGroup.waitForAll()
       }
     }
   }

Tests/TestingTests/EventIterationTests.swift

@@ -44,7 +44,7 @@ struct EventIterationTests {
 
       // Verify all expected iterations were recorded
       let iteration = recordedIteration.load(ordering: .sequentiallyConsistent)
-      #expect(iteration == expectedIterations, "Final observed iteration did not match expected number of iterations", sourceLocation: location)
+      #expect(iteration == expectedIterations, sourceLocation: location)
     }
   }
 
@@ -60,17 +60,6 @@ struct EventIterationTests {
     }
   }
 
-  @Test
-  func `testStarted and testEnded events include iteration in context`() async {
-    await verifyIterations(
-      for: [.testStarted, .testEnded],
-      repetitionPolicy: .once,
-      expectedIterations: 1
-    ) { _ in
-      // Do nothing, just pass
-    }
-  }
-
   @Test
   func `testCaseStarted and testCaseEnded events include iteration in context`() async {
     await verifyIterations(
@@ -96,9 +85,7 @@ struct EventIterationTests {
       repetitionPolicy: policy,
       expectedIterations: expectedIterations
     ) { iteration in
-      if iteration < 3 {
-        Issue.record("Failure")
-      }
+      #expect(iteration >= 3)
     }
   }
 }