Sink iteration down to a per-test-case iteration

Author: harlanhaskins

Sources/Testing/ABI/Encoded/ABI.EncodedEvent.swift

@@ -117,10 +117,18 @@ extension ABI {
       case .testStarted:
         kind = .testStarted
       case .testCaseStarted:
+        // For non-parameterized tests, we elide `testCaseStarted` calls because it would be
+        // redundant. However, for multiple iterations of a test case within a non-parameterized
+        // function, we need to emit another `testStarted` event.
         if eventContext.test?.isParameterized == false {
-          return nil
+          if let iteration = eventContext.iteration, iteration > 1 {
+            kind = .testStarted
+          } else {
+            return nil
+          }
+        } else {
+          kind = .testCaseStarted
         }
-        kind = .testCaseStarted
       case let .issueRecorded(recordedIssue):
         kind = .issueRecorded
         issue = EncodedIssue(encoding: recordedIssue, in: eventContext)
@@ -129,9 +137,14 @@ extension ABI {
         self.attachment = EncodedAttachment(encoding: attachment)
       case .testCaseEnded:
         if eventContext.test?.isParameterized == false {
-          return nil
+          if let iteration = eventContext.iteration, iteration > 1 {
+            kind = .testEnded
+          } else {
+            return nil
+          }
+        } else {
+          kind = .testCaseEnded
         }
-        kind = .testCaseEnded
       case .testCaseCancelled:
         kind = .testCaseCancelled
       case .testEnded:
@@ -164,6 +177,7 @@ extension ABI {
           let .testCancelled(skipInfo):
           _comments = Array(skipInfo.comment).map(\.rawValue)
           _sourceLocation = skipInfo.sourceLocation.map { EncodedSourceLocation(encoding: $0) }
+          _iteration = eventContext.iteration
         default:
           break
         }

Sources/Testing/ABI/EntryPoints/EntryPoint.swift

@@ -333,6 +333,9 @@ public struct __CommandLineArguments_v0: Sendable {
   /// The value of the `--repeat-until` argument.
   public var repeatUntil: String?
 
+  /// Whether or not to use the per-test-case repetition mode.
+  var usePerTestCaseRepetition: Bool = false
+
   /// The value of the `--attachments-path` argument.
   public var attachmentsPath: String?
 }
@@ -537,6 +540,9 @@ func parseCommandLineArguments(from args: [String]) throws -> __CommandLineArgum
   if let repeatUntil = args.argumentValue(forLabel: "--repeat-until") {
     result.repeatUntil = repeatUntil
   }
+  if args.contains("--experimental-per-test-case-repetition") {
+    result.usePerTestCaseRepetition = true
+  }
 
   return result
 }
@@ -665,6 +671,9 @@ public func configurationForEntryPoint(from args: __CommandLineArguments_v0) thr
   }
   configuration.repetitionPolicy = repetitionPolicy
 
+  // Opt in to per-test-case repetition
+  configuration.shouldUseLegacyPlanLevelRepetition = !args.usePerTestCaseRepetition
+
 #if !SWT_NO_EXIT_TESTS
   // Enable exit test handling via __swiftPMEntryPoint().
   configuration.exitTestHandler = ExitTest.handlerForEntryPoint()

Sources/Testing/Events/Event.swift

@@ -238,7 +238,6 @@ public struct Event: Sendable {
   static func post(
     _ kind: Kind,
     for testAndTestCase: (Test?, Test.Case?) = currentTestAndTestCase(),
-    iteration: Int? = nil,
     instant: Test.Clock.Instant = .now,
     configuration: Configuration? = nil
   ) {
@@ -262,7 +261,12 @@ public struct Event: Sendable {
       }
     }
     let event = Event(kind, testID: test?.id, testCaseID: testCase?.id, instant: instant)
-    let context = Event.Context(test: test, testCase: testCase, iteration: iteration, configuration: nil)
+    let context = Event.Context(
+      test: test,
+      testCase: testCase,
+      iteration: Test.currentIteration,
+      configuration: nil
+    )
     event._post(in: context, configuration: configuration)
   }
 }
@@ -327,8 +331,6 @@ extension Event {
       iteration: Int?,
       configuration: Configuration?
     ) {
-      // Ensure that if `iteration` is specified, the test is also specified.
-      precondition(iteration == nil || (iteration != nil && test != nil))
       self.test = test
       self.testCase = testCase
       self.iteration = iteration

Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift

@@ -365,6 +365,10 @@ extension Event.HumanReadableOutputRecorder {
       String(describing: TimeValue(rawValue: end.suspending.rawValue - start.suspending.rawValue))
     }
 
+    func testStartedMessage(for test: Test) -> String {
+      "\(_capitalizedTitle(for: test)) \(testName) started"
+    }
+
     // Finally, produce any messages for the event.
     switch event.kind {
     case .testDiscovered:
@@ -425,7 +429,7 @@ extension Event.HumanReadableOutputRecorder {
       return [
         Message(
           symbol: .default,
-          stringValue: "\(_capitalizedTitle(for: test)) \(testName) started."
+          stringValue: "\(testStartedMessage(for: test))."
         )
       ]
 
@@ -565,15 +569,26 @@ extension Event.HumanReadableOutputRecorder {
       return result
 
     case .testCaseStarted:
-      guard let testCase, testCase.isParameterized, let arguments = testCase.arguments else {
+      guard let test, let testCase else { break }
+      let iteration = eventContext.iteration ?? 1
+
+      var message: String
+      if testCase.isParameterized, let arguments = testCase.arguments {
+        message = "Test case passing \(arguments.count.counting("argument")) \(testCase.labeledArguments(includingQualifiedTypeNames: verbosity > 0)) to \(testName) started"
+      } else if iteration > 1 {
+        message = testStartedMessage(for: test)
+      } else {
         break
       }
 
+      if iteration > 1 {
+        message += " (repetition \(iteration))"
+      }
+
+      message += "."
+
       return [
-        Message(
-          symbol: .default,
-          stringValue: "Test case passing \(arguments.count.counting("argument")) \(testCase.labeledArguments(includingQualifiedTypeNames: verbosity > 0)) to \(testName) started."
-        )
+        Message(symbol: .default, stringValue: message)
       ]
 
     case .testCaseEnded:

Sources/Testing/Running/Configuration.swift

@@ -132,6 +132,11 @@ public struct Configuration: Sendable {
     }
   }
 
+  /// Whether to perform test repetition at the plan level or on a per-test-
+  /// case basis.
+  @_spi(Experimental)
+  public var shouldUseLegacyPlanLevelRepetition: Bool = true
+
   /// Whether or not, and how, to iterate the test plan repeatedly.
   ///
   /// By default, the value of this property allows for a single iteration.

Sources/Testing/Running/Runner.RuntimeState.swift

@@ -29,6 +29,9 @@ extension Runner {
     /// The test case that is running on the current task, if any.
     var testCase: Test.Case?
 
+    /// The current iteration of the test repetition policy, if any.
+    var iteration: Int?
+
     /// The runtime state related to the runner running on the current task,
     /// if any.
     @TaskLocal
@@ -198,7 +201,7 @@ extension Configuration {
   }
 }
 
-// MARK: - Current test and test case
+// MARK: - Current test, test case, and iteration
 
 extension Test {
   /// The test that is running on the current task, if any.
@@ -233,6 +236,28 @@ extension Test {
       try await test.withCancellationHandling(body)
     }
   }
+
+  /// The current iteration of the currently-running test case, if any.
+  static var currentIteration: Int? {
+    Runner.RuntimeState.current?.iteration
+  }
+
+  /// Call a function while the value of ``Test/currentIteration`` is set.
+  ///
+  /// - Parameters:
+  ///   - iteration: The new value to set for ``Test/currentIteration``.
+  ///   - body: A function to call.
+  ///
+  /// - Returns: Whatever is returned by `body`.
+  ///
+  /// - Throws: Whatever is thrown by `body`.
+  static func withCurrentIteration<R>(_ iteration: Int?, perform body: () async throws -> R) async rethrows -> R {
+    var runtimeState = Runner.RuntimeState.current ?? .init()
+    runtimeState.iteration = iteration
+    return try await Runner.RuntimeState.$current.withValue(runtimeState) {
+      try await body()
+    }
+  }
 }
 
 extension Test.Case {