[WIP] Third-party testing library discovery

Author: grynspan

Sources/Testing/ABI/ABI.Record+Streaming.swift

@@ -24,7 +24,13 @@ extension ABI.Version {
 
     let humanReadableOutputRecorder = Event.HumanReadableOutputRecorder()
     return { [eventHandler = eventHandlerCopy] event, context in
-      if case .testDiscovered = event.kind, let test = context.test {
+      if case let .libraryDiscovered(library) = event.kind {
+        if let libraryRecord = ABI.Record<Self>(encoding: library) {
+          try? JSON.withEncoding(of: libraryRecord) { libraryJSON in
+            eventHandler(libraryJSON)
+          }
+        }
+      } else if case .testDiscovered = event.kind, let test = context.test {
         try? JSON.withEncoding(of: ABI.Record<Self>(encoding: test)) { testJSON in
           eventHandler(testJSON)
         }
@@ -47,23 +53,24 @@ extension ABI.Xcode16 {
     forwardingTo eventHandler: @escaping @Sendable (_ recordJSON: borrowing RawSpan) -> Void
   ) -> Event.Handler {
     return { event, context in
-      if case .testDiscovered = event.kind {
+      switch event.kind {
+      case .libraryDiscovered, .testDiscovered:
         // Discard events of this kind rather than forwarding them to avoid a
         // crash in Xcode 16 (which does not expect any events to occur before
         // .runStarted.)
         return
-      }
-
-      struct EventAndContextSnapshot: Codable {
-        var event: Event.Snapshot
-        var eventContext: Event.Context.Snapshot
-      }
-      let snapshot = EventAndContextSnapshot(
-        event: Event.Snapshot(snapshotting: event),
-        eventContext: Event.Context.Snapshot(snapshotting: context)
-      )
-      try? JSON.withEncoding(of: snapshot) { eventAndContextJSON in
-        eventHandler(eventAndContextJSON)
+      default:
+        struct EventAndContextSnapshot: Codable {
+          var event: Event.Snapshot
+          var eventContext: Event.Context.Snapshot
+        }
+        let snapshot = EventAndContextSnapshot(
+          event: Event.Snapshot(snapshotting: event),
+          eventContext: Event.Context.Snapshot(snapshotting: context)
+        )
+        try? JSON.withEncoding(of: snapshot) { eventAndContextJSON in
+          eventHandler(eventAndContextJSON)
+        }
       }
     }
   }

Sources/Testing/ABI/ABI.Record.swift

@@ -18,6 +18,11 @@ extension ABI {
   struct Record<V>: Sendable where V: ABI.Version {
     /// An enumeration describing the various kinds of record.
     enum Kind: Sendable {
+      /// A testing library.
+      ///
+      /// - Warning: Testing libraries are not yet part of the JSON schema.
+      case library(EncodedLibrary<V>)
+
       /// A test record.
       case test(EncodedTest<V>)
 
@@ -28,6 +33,13 @@ extension ABI {
     /// The kind of record.
     var kind: Kind
 
+    init?(encoding library: borrowing Library) {
+      guard V.includesExperimentalFields else {
+        return nil
+      }
+      kind = .library(EncodedLibrary(encoding: library))
+    }
+
     init(encoding test: borrowing Test) {
       kind = .test(EncodedTest(encoding: test))
     }
@@ -58,6 +70,9 @@ extension ABI.Record: Codable {
     var container = encoder.container(keyedBy: CodingKeys.self)
     try container.encode(V.versionNumber, forKey: .version)
     switch kind {
+    case let .library(library):
+      try container.encode("_library", forKey: .kind)
+      try container.encode(library, forKey: .payload)
     case let .test(test):
       try container.encode("test", forKey: .kind)
       try container.encode(test, forKey: .payload)
@@ -92,6 +107,9 @@ extension ABI.Record: Codable {
     try validateVersionNumber(versionNumber)
 
     switch try container.decode(String.self, forKey: .kind) {
+    case "_library":
+      let library = try container.decode(ABI.EncodedLibrary<V>.self, forKey: .payload)
+      kind = .library(library)
     case "test":
       let test = try container.decode(ABI.EncodedTest<V>.self, forKey: .payload)
       kind = .test(test)

Sources/Testing/ABI/ABI.swift

@@ -67,6 +67,12 @@ extension ABI {
     forVersionNumber versionNumber: VersionNumber,
     givenSwiftCompilerVersion swiftCompilerVersion: @autoclosure () -> VersionNumber = swiftCompilerVersion
   ) -> (any Version.Type)? {
+    // Special-case the experimental ABI version number (which is intentionally
+    // higher than any Swift release's version number).
+    if versionNumber == ExperimentalVersion.versionNumber {
+      return ExperimentalVersion.self
+    }
+
     if versionNumber > ABI.HighestVersion.versionNumber {
       // If the caller requested an ABI version higher than the current Swift
       // compiler version and it's not an ABI version we've explicitly defined,

Sources/Testing/ABI/Encoded/ABI.EncodedLibrary.swift

@@ -0,0 +1,39 @@
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2024 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for Swift project authors
+//
+
+extension ABI {
+  /// A type implementing the JSON encoding of ``Library`` for the ABI entry
+  /// point and event stream output.
+  ///
+  /// The properties and members of this type are documented in ABI/JSON.md.
+  ///
+  /// This type is not part of the public interface of the testing library. It
+  /// assists in converting values to JSON; clients that consume this JSON are
+  /// expected to write their own decoders.
+  ///
+  /// - Warning: Testing libraries are not yet part of the JSON schema.
+  struct EncodedLibrary<V>: Sendable where V: ABI.Version {
+    /// The human-readable name of the library.
+    var name: String
+
+    /// The canonical form of the "hint" to run the testing library's tests at
+    /// runtime.
+    var canonicalHint: String
+
+    init(encoding library: borrowing Library) {
+      name = library.name
+      canonicalHint = library.canonicalHint
+    }
+  }
+}
+
+// MARK: - Codable
+
+extension ABI.EncodedLibrary: Codable {}

Sources/Testing/ABI/EntryPoints/EntryPoint.swift

@@ -34,13 +34,25 @@ func entryPoint(passing args: __CommandLineArguments_v0?, eventHandler: Event.Ha
 
   do {
 #if !SWT_NO_EXIT_TESTS
-      // If an exit test was specified, run it. `exitTest` returns `Never`.
-      if let exitTest = ExitTest.findInEnvironmentForEntryPoint() {
-        await exitTest()
-      }
+    // If an exit test was specified, run it. `exitTest` returns `Never`.
+    if let exitTest = ExitTest.findInEnvironmentForEntryPoint() {
+      await exitTest()
+    }
 #endif
 
     let args = try args ?? parseCommandLineArguments(from: CommandLine.arguments)
+
+#if !SWT_NO_RUNTIME_LIBRARY_DISCOVERY
+    // If the user requested a different testing library, run it instead of
+    // Swift Testing. (If they requested Swift Testing, we're already here so
+    // there's no real need to recurse).
+    if args.experimentalListLibraries != true,
+       let library = args.testingLibrary.flatMap(Library.init(withHint:)),
+       library.canonicalHint != "swift-testing" {
+      return await library.callEntryPoint(passing: args)
+    }
+#endif
+
     // Configure the test runner.
     var configuration = try configurationForEntryPoint(from: args)
 
@@ -95,9 +107,10 @@ func entryPoint(passing args: __CommandLineArguments_v0?, eventHandler: Event.Ha
 
     // The set of matching tests (or, in the case of `swift test list`, the set
     // of all tests.)
-    let tests: [Test]
+    var tests = [Test]()
+    var libraries = [Library]()
 
-    if args.listTests ?? false {
+    if args.listTests == true {
       tests = await Array(Test.all)
 
       if args.verbosity > .min {
@@ -116,6 +129,29 @@ func entryPoint(passing args: __CommandLineArguments_v0?, eventHandler: Event.Ha
       for test in tests {
         Event.post(.testDiscovered, for: (test, nil), configuration: configuration)
       }
+    } else if args.experimentalListLibraries == true {
+#if !SWT_NO_RUNTIME_LIBRARY_DISCOVERY
+      libraries = Array(Library.all)
+#else
+      libraries = [Library.swiftTesting]
+#endif
+
+      if args.verbosity > .min {
+        for library in libraries {
+          // Print the test ID to stdout (classical CLI behavior.)
+          let libraryDescription = "\(library.name) (swift test --experimental-testing-library \(library.canonicalHint))"
+#if SWT_TARGET_OS_APPLE && !SWT_NO_FILE_IO
+          try? FileHandle.stdout.write("\(libraryDescription)\n")
+#else
+          print(libraryDescription)
+#endif
+        }
+      }
+
+      // Post an event for every discovered library (as with tests above).
+      for library in libraries {
+        Event.post(.libraryDiscovered(library), for: (nil, nil), configuration: configuration)
+      }
     } else {
       // Run the tests.
       let runner = await Runner(configuration: configuration)
@@ -126,7 +162,7 @@ func entryPoint(passing args: __CommandLineArguments_v0?, eventHandler: Event.Ha
     // If there were no matching tests, exit with a dedicated exit code so that
     // the caller (assumed to be Swift Package Manager) can implement special
     // handling.
-    if tests.isEmpty {
+    if tests.isEmpty && libraries.isEmpty {
       exitCode.withLock { exitCode in
         if exitCode == EXIT_SUCCESS {
           exitCode = EXIT_NO_TESTS_FOUND
@@ -211,6 +247,9 @@ public struct __CommandLineArguments_v0: Sendable {
   /// The value of the `--list-tests` argument.
   public var listTests: Bool?
 
+  /// The value of the `--experimental-list-libraries` argument.
+  public var experimentalListLibraries: Bool?
+
   /// The value of the `--parallel` or `--no-parallel` argument.
   public var parallel: Bool?
 
@@ -335,13 +374,17 @@ public struct __CommandLineArguments_v0: Sendable {
 
   /// The value of the `--attachments-path` argument.
   public var attachmentsPath: String?
+
+  /// The value of the `--testing-library` argument.
+  public var testingLibrary: String?
 }
 
 extension __CommandLineArguments_v0: Codable {
   // Explicitly list the coding keys so that storage properties like _verbosity
   // do not end up with leading underscores when encoded.
   enum CodingKeys: String, CodingKey {
     case listTests
+    case experimentalListLibraries
     case parallel
     case experimentalMaximumParallelizationWidth
     case symbolicateBacktraces
@@ -357,6 +400,7 @@ extension __CommandLineArguments_v0: Codable {
     case repetitions
     case repeatUntil
     case attachmentsPath
+    case testingLibrary
   }
 }
 
@@ -468,6 +512,11 @@ func parseCommandLineArguments(from args: [String]) throws -> __CommandLineArgum
   }
 #endif
 
+  // Testing library
+  if let testingLibrary = Environment.variable(named: "SWT_EXPERIMENTAL_LIBRARY") ?? args.argumentValue(forLabel: "--testing-library") {
+    result.testingLibrary = testingLibrary
+  }
+
   // XML output
   if let xunitOutputPath = args.argumentValue(forLabel: "--xunit-output") {
     result.xunitOutput = xunitOutputPath
@@ -486,6 +535,9 @@ func parseCommandLineArguments(from args: [String]) throws -> __CommandLineArgum
     // makes invocation from e.g. Wasmtime a bit more intuitive/idiomatic.
     result.listTests = true
   }
+  if Environment.flag(named: "SWT_EXPERIMENTAL_LIST_LIBRARIES") == true || args.contains("--experimental-list-libraries") {
+    result.experimentalListLibraries = true
+  }
 
   // Parallelization (on by default)
   if args.contains("--no-parallel") {