diff --git a/Sources/Testing/ABI/ABI.Record+Streaming.swift b/Sources/Testing/ABI/ABI.Record+Streaming.swift index 1aa1362ec..4f703dea8 100644 --- a/Sources/Testing/ABI/ABI.Record+Streaming.swift +++ b/Sources/Testing/ABI/ABI.Record+Streaming.swift @@ -24,7 +24,13 @@ extension ABI.Version { let humanReadableOutputRecorder = Event.HumanReadableOutputRecorder() return { [eventHandler = eventHandlerCopy] event, context in - if case .testDiscovered = event.kind, let test = context.test { + if case let .libraryDiscovered(library) = event.kind { + if let libraryRecord = ABI.Record(encoding: library) { + try? JSON.withEncoding(of: libraryRecord) { libraryJSON in + eventHandler(libraryJSON) + } + } + } else if case .testDiscovered = event.kind, let test = context.test { try? JSON.withEncoding(of: ABI.Record(encoding: test)) { testJSON in eventHandler(testJSON) } @@ -47,24 +53,25 @@ extension ABI.Xcode16 { forwardingTo eventHandler: @escaping @Sendable (_ recordJSON: UnsafeRawBufferPointer) -> Void ) -> Event.Handler { return { event, context in - if case .testDiscovered = event.kind { + switch event.kind { + case .libraryDiscovered, .testDiscovered: // Discard events of this kind rather than forwarding them to avoid a // crash in Xcode 16 (which does not expect any events to occur before // .runStarted.) return - } - - struct EventAndContextSnapshot: Codable { - var event: Event.Snapshot - var eventContext: Event.Context.Snapshot - } - let snapshot = EventAndContextSnapshot( - event: Event.Snapshot(snapshotting: event), - eventContext: Event.Context.Snapshot(snapshotting: context) - ) - try? JSON.withEncoding(of: snapshot) { eventAndContextJSON in - eventAndContextJSON.withUnsafeBytes { eventAndContextJSON in - eventHandler(eventAndContextJSON) + default: + struct EventAndContextSnapshot: Codable { + var event: Event.Snapshot + var eventContext: Event.Context.Snapshot + } + let snapshot = EventAndContextSnapshot( + event: Event.Snapshot(snapshotting: event), + eventContext: Event.Context.Snapshot(snapshotting: context) + ) + try? JSON.withEncoding(of: snapshot) { eventAndContextJSON in + eventAndContextJSON.withUnsafeBytes { eventAndContextJSON in + eventHandler(eventAndContextJSON) + } } } } diff --git a/Sources/Testing/ABI/ABI.Record.swift b/Sources/Testing/ABI/ABI.Record.swift index 2c101ec40..db667bdd0 100644 --- a/Sources/Testing/ABI/ABI.Record.swift +++ b/Sources/Testing/ABI/ABI.Record.swift @@ -18,6 +18,11 @@ extension ABI { struct Record: Sendable where V: ABI.Version { /// An enumeration describing the various kinds of record. enum Kind: Sendable { + /// A testing library. + /// + /// - Warning: Testing libraries are not yet part of the JSON schema. + case library(EncodedLibrary) + /// A test record. case test(EncodedTest) @@ -28,6 +33,13 @@ extension ABI { /// The kind of record. var kind: Kind + init?(encoding library: borrowing Library) { + guard V.includesExperimentalFields else { + return nil + } + kind = .library(EncodedLibrary(encoding: library)) + } + init(encoding test: borrowing Test) { kind = .test(EncodedTest(encoding: test)) } @@ -58,6 +70,9 @@ extension ABI.Record: Codable { var container = encoder.container(keyedBy: CodingKeys.self) try container.encode(V.versionNumber, forKey: .version) switch kind { + case let .library(library): + try container.encode("_library", forKey: .kind) + try container.encode(library, forKey: .payload) case let .test(test): try container.encode("test", forKey: .kind) try container.encode(test, forKey: .payload) @@ -92,6 +107,9 @@ extension ABI.Record: Codable { try validateVersionNumber(versionNumber) switch try container.decode(String.self, forKey: .kind) { + case "_library": + let library = try container.decode(ABI.EncodedLibrary.self, forKey: .payload) + kind = .library(library) case "test": let test = try container.decode(ABI.EncodedTest.self, forKey: .payload) kind = .test(test) diff --git a/Sources/Testing/ABI/Encoded/ABI.EncodedLibrary.swift b/Sources/Testing/ABI/Encoded/ABI.EncodedLibrary.swift new file mode 100644 index 000000000..62823c722 --- /dev/null +++ b/Sources/Testing/ABI/Encoded/ABI.EncodedLibrary.swift @@ -0,0 +1,38 @@ +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2024 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for Swift project authors +// + +extension ABI { + /// A type implementing the JSON encoding of ``Library`` for the ABI entry + /// point and event stream output. + /// + /// The properties and members of this type are documented in ABI/JSON.md. + /// + /// This type is not part of the public interface of the testing library. It + /// assists in converting values to JSON; clients that consume this JSON are + /// expected to write their own decoders. + /// + /// - Warning: Testing libraries are not yet part of the JSON schema. + struct EncodedLibrary: Sendable where V: ABI.Version { + /// The human-readable name of this library. + var displayName: String + + /// The name of this testing library. + var name: String + + init(encoding library: borrowing Library) { + displayName = library.displayName + name = library.name + } + } +} + +// MARK: - Codable + +extension ABI.EncodedLibrary: Codable {} diff --git a/Sources/Testing/ABI/EntryPoints/EntryPoint.swift b/Sources/Testing/ABI/EntryPoints/EntryPoint.swift index c2f82d4fa..33c53d6b6 100644 --- a/Sources/Testing/ABI/EntryPoints/EntryPoint.swift +++ b/Sources/Testing/ABI/EntryPoints/EntryPoint.swift @@ -34,13 +34,25 @@ func entryPoint(passing args: __CommandLineArguments_v0?, eventHandler: Event.Ha do { #if !SWT_NO_EXIT_TESTS - // If an exit test was specified, run it. `exitTest` returns `Never`. - if let exitTest = ExitTest.findInEnvironmentForEntryPoint() { - await exitTest() - } + // If an exit test was specified, run it. `exitTest` returns `Never`. + if let exitTest = ExitTest.findInEnvironmentForEntryPoint() { + await exitTest() + } #endif let args = try args ?? parseCommandLineArguments(from: CommandLine.arguments) + +#if !SWT_NO_RUNTIME_LIBRARY_DISCOVERY + // If the user requested a different testing library, run it instead of + // Swift Testing. (If they requested Swift Testing, we're already here so + // there's no real need to recurse). + if args.experimentalListLibraries != true, + let library = args.testingLibrary.flatMap(Library.init(named:)), + library.name != "swift-testing" { + return await library.callEntryPoint(passing: args) + } +#endif + // Configure the test runner. var configuration = try configurationForEntryPoint(from: args) @@ -95,9 +107,10 @@ func entryPoint(passing args: __CommandLineArguments_v0?, eventHandler: Event.Ha // The set of matching tests (or, in the case of `swift test list`, the set // of all tests.) - var tests: [Test] + var tests = [Test]() + var libraries = [Library]() - if args.listTests ?? false { + if args.listTests == true { tests = await Array(Test.all) if args.verbosity > .min { @@ -120,6 +133,29 @@ func entryPoint(passing args: __CommandLineArguments_v0?, eventHandler: Event.Ha for test in tests { Event.post(.testDiscovered, for: (test, nil), configuration: configuration) } + } else if args.experimentalListLibraries == true { +#if !SWT_NO_RUNTIME_LIBRARY_DISCOVERY + libraries = Array(Library.all) +#else + libraries = [Library.swiftTesting] +#endif + + if args.verbosity > .min { + for library in libraries { + // Print the test ID to stdout (classical CLI behavior.) + let libraryDescription = "\(library.displayName) (swift test --experimental-testing-library \(library.name))" +#if SWT_TARGET_OS_APPLE && !SWT_NO_FILE_IO + try? FileHandle.stdout.write("\(libraryDescription)\n") +#else + print(libraryDescription) +#endif + } + } + + // Post an event for every discovered library (as with tests above). + for library in libraries { + Event.post(.libraryDiscovered(library), for: (nil, nil), configuration: configuration) + } } else { // Run the tests. let runner = await Runner(configuration: configuration) @@ -130,7 +166,7 @@ func entryPoint(passing args: __CommandLineArguments_v0?, eventHandler: Event.Ha // If there were no matching tests, exit with a dedicated exit code so that // the caller (assumed to be Swift Package Manager) can implement special // handling. - if tests.isEmpty { + if tests.isEmpty && libraries.isEmpty { exitCode.withLock { exitCode in if exitCode == EXIT_SUCCESS { exitCode = EXIT_NO_TESTS_FOUND @@ -215,6 +251,9 @@ public struct __CommandLineArguments_v0: Sendable { /// The value of the `--list-tests` argument. public var listTests: Bool? + /// The value of the `--experimental-list-libraries` argument. + public var experimentalListLibraries: Bool? + /// The value of the `--parallel` or `--no-parallel` argument. public var parallel: Bool? @@ -339,6 +378,9 @@ public struct __CommandLineArguments_v0: Sendable { /// The value of the `--attachments-path` argument. public var attachmentsPath: String? + + /// The value of the `--testing-library` argument. + public var testingLibrary: String? } extension __CommandLineArguments_v0: Codable { @@ -346,6 +388,7 @@ extension __CommandLineArguments_v0: Codable { // do not end up with leading underscores when encoded. enum CodingKeys: String, CodingKey { case listTests + case experimentalListLibraries case parallel case experimentalMaximumParallelizationWidth case symbolicateBacktraces @@ -361,6 +404,7 @@ extension __CommandLineArguments_v0: Codable { case repetitions case repeatUntil case attachmentsPath + case testingLibrary } } @@ -474,6 +518,11 @@ func parseCommandLineArguments(from args: [String]) throws -> __CommandLineArgum } #endif + // Testing library + if let testingLibrary = Environment.variable(named: "SWT_EXPERIMENTAL_LIBRARY") ?? args.argumentValue(forLabel: "--testing-library") { + result.testingLibrary = testingLibrary + } + // XML output if let xunitOutputPath = args.argumentValue(forLabel: "--xunit-output") { result.xunitOutput = xunitOutputPath @@ -492,6 +541,9 @@ func parseCommandLineArguments(from args: [String]) throws -> __CommandLineArgum // makes invocation from e.g. Wasmtime a bit more intuitive/idiomatic. result.listTests = true } + if Environment.flag(named: "SWT_EXPERIMENTAL_LIST_LIBRARIES") == true || args.contains("--experimental-list-libraries") { + result.experimentalListLibraries = true + } // Parallelization (on by default) if args.contains("--no-parallel") { diff --git a/Sources/Testing/ABI/EntryPoints/Library.swift b/Sources/Testing/ABI/EntryPoints/Library.swift new file mode 100644 index 000000000..455fd0bee --- /dev/null +++ b/Sources/Testing/ABI/EntryPoints/Library.swift @@ -0,0 +1,371 @@ +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2023 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for Swift project authors +// + +@_spi(Experimental) @_spi(ForToolsIntegrationOnly) private import _TestDiscovery +internal import _TestingInternals + +/// A type representing a testing library such as Swift Testing or XCTest. +@_spi(Experimental) +@frozen public struct Library: Sendable { + /// The underlying instance of ``Library/Record``. + /// + /// - Important: The in-memory layout of ``Library`` must _exactly_ match the + /// layout of ``Library/Record``. As such, ``Library`` must not contain any + /// other stored properties. + nonisolated(unsafe) var rawValue: Record + + /// The human-readable name of this library. + /// + /// For example, the value of this property for an instance of this type that + /// represents the Swift Testing library is `"Swift Testing"`. + public var displayName: String { + String(validatingCString: rawValue.displayName) ?? "" + } + + /// The name of this testing library. + /// + /// For example, the value of this property for an instance of this type that + /// represents the Swift Testing library is `"swift-testing"`. + @_spi(ForToolsIntegrationOnly) + public var name: String { + String(validatingCString: rawValue.name) ?? "" + } + +#if !SWT_NO_RUNTIME_LIBRARY_DISCOVERY + /// Call the entry point function of this library. + /// + /// - Parameters: + /// - args: The arguments to pass to the testing library as its + /// configuration JSON. + /// - recordHandler: A callback to invoke once per record. + /// + /// - Returns: A process exit code such as `EXIT_SUCCESS`. + /// + /// - Warning: The signature of this function is subject to change as + /// `__CommandLineArguments_v0` is not a stable interface. + @_spi(ForToolsIntegrationOnly) + public func callEntryPoint( + passing args: __CommandLineArguments_v0? = nil, + recordHandler: (@Sendable (_ recordJSON: UnsafeRawBufferPointer) -> Void)? = nil + ) async -> CInt { + do { + return try await _callEntryPoint(passing: args, recordHandler: recordHandler) + } catch { + // TODO: more advanced error recovery? + return EXIT_FAILURE + } + } + + /// The implementation of ``callEntryPoint(passing:recordHandler:)``. + /// + /// - Parameters: + /// - args: The arguments to pass to the testing library as its + /// configuration JSON. + /// - recordHandler: A callback to invoke once per record. + /// + /// - Returns: A process exit code such as `EXIT_SUCCESS`. + private func _callEntryPoint( + passing args: __CommandLineArguments_v0?, + recordHandler: (@Sendable (_ recordJSON: UnsafeRawBufferPointer) -> Void)? + ) async throws -> CInt { + var recordHandler = recordHandler + + let args = try args ?? parseCommandLineArguments(from: CommandLine.arguments) + + // Event stream output + // Automatically write record JSON as JSON lines to the event stream if + // specified by the user. + if let eventStreamOutputPath = args.eventStreamOutputPath { + let file = try FileHandle(forWritingAtPath: eventStreamOutputPath) + recordHandler = { [oldRecordHandler = recordHandler] recordJSON in + JSON.asJSONLine(recordJSON) { recordJSON in + _ = try? file.withLock { + try file.write(recordJSON) + try file.write("\n") + } + } + oldRecordHandler?(recordJSON) + } + } + + let configurationJSON = try JSON.withEncoding(of: args) { configurationJSON in + configurationJSON.withMemoryRebound(to: UInt8.self) { Array($0) } + } + + let resultJSON: [UInt8] = await withCheckedContinuation { continuation in + struct Context { + var continuation: CheckedContinuation<[UInt8], Never> + var recordHandler: (@Sendable (UnsafeRawBufferPointer) -> Void)? + } + let context = Unmanaged.passRetained( + Context( + continuation: continuation, + recordHandler: recordHandler + ) as AnyObject + ).toOpaque() + configurationJSON.withUnsafeBytes { configurationJSON in + rawValue.entryPoint( + configurationJSON.baseAddress!, + configurationJSON.count, + 0, + context, + /* recordJSONHandler: */ { recordJSON, recordJSONByteCount, _, context in + guard let context = Unmanaged.fromOpaque(context!).takeUnretainedValue() as? Context else { + return + } + let recordJSON = UnsafeRawBufferPointer(start: recordJSON, count: recordJSONByteCount) + context.recordHandler?(recordJSON) + }, + /* completionHandler: */ { resultJSON, resultJSONByteCount, _, context in + guard let context = Unmanaged.fromOpaque(context!).takeRetainedValue() as? Context else { + return + } + // TODO: interpret more complex results than a process exit code + let resultJSON = [UInt8](UnsafeRawBufferPointer(start: resultJSON, count: resultJSONByteCount)) + context.continuation.resume(returning: resultJSON) + } + ) + } + } + + do { + return try resultJSON.withUnsafeBytes { resultJSON in + try JSON.decode(CInt.self, from: resultJSON) + } + } catch { + // TODO: more advanced error recovery? + return EXIT_FAILURE + } + } +#endif +} + +#if !SWT_NO_RUNTIME_LIBRARY_DISCOVERY +// MARK: - C structure + +extension Library { + @usableFromInline typealias EntryPoint = @convention(c) ( + _ configurationJSON: UnsafeRawPointer, + _ configurationJSONByteCount: Int, + _ reserved: UInt, + _ context: UnsafeRawPointer?, + _ recordJSONHandler: EntryPointRecordJSONHandler, + _ completionHandler: EntryPointCompletionHandler + ) -> Void + + @usableFromInline typealias EntryPointRecordJSONHandler = @convention(c) ( + _ recordJSON: UnsafeRawPointer, + _ recordJSONByteCount: Int, + _ reserved: UInt, + _ context: UnsafeRawPointer? + ) -> Void + + @usableFromInline typealias EntryPointCompletionHandler = @convention(c) ( + _ resultJSON: UnsafeRawPointer, + _ resultJSONByteCount: Int, + _ reserved: UInt, + _ context: UnsafeRawPointer? + ) -> Void + + /// A type that provides the C-compatible in-memory layout of the ``Library`` + /// Swift type. + @usableFromInline typealias Record = ( + displayName: UnsafePointer, + name: UnsafePointer, + entryPoint: EntryPoint, + reserved: (UInt, UInt, UInt, UInt, UInt) + ) +} + +// MARK: - Discovery + +/// A helper protocol that prevents the conformance of ``Library`` to +/// ``DiscoverableAsTestContent`` from being emitted into the testing library's +/// Swift module or interface files. +private protocol _DiscoverableAsTestContent: DiscoverableAsTestContent {} + +extension Library: _DiscoverableAsTestContent { + fileprivate static var testContentKind: TestContentKind { + "main" + } + + fileprivate typealias TestContentAccessorHint = ( + name: UnsafePointer, + reserved: UInt + ) +} + +@_spi(Experimental) +extension Library { + /// Perform a one-time check that the in-memory layout of ``Library`` matches + /// that of ``Library/Record``. + private static let _validateMemoryLayout: Void = { + assert(MemoryLayout.size == MemoryLayout.size, "Library.size (\(MemoryLayout.size)) != Library.Record.size (\(MemoryLayout.size)). Please file a bug report at https://github.com/swiftlang/swift-testing/issues/new") + assert(MemoryLayout.stride == MemoryLayout.stride, "Library.stride (\(MemoryLayout.stride)) != Library.Record.stride (\(MemoryLayout.stride)). Please file a bug report at https://github.com/swiftlang/swift-testing/issues/new") + assert(MemoryLayout.alignment == MemoryLayout.alignment, "Library.alignment (\(MemoryLayout.alignment)) != Library.Record.alignment (\(MemoryLayout.alignment)). Please file a bug report at https://github.com/swiftlang/swift-testing/issues/new") + }() + + /// Create an instance of this type representing the testing library with the + /// given name. + /// + /// - Parameters: + /// - name: The name to match against such as `"swift-testing"`. Whether or + /// not the library's display name is accepted is library-specific. + /// + /// If no matching testing library is found, this initializer returns `nil`. + @_spi(ForToolsIntegrationOnly) + public init?(named name: String) { + Self._validateMemoryLayout + let result = name.withCString { name in + let hint = TestContentAccessorHint( + name: name, + reserved: 0 + ) + return Library.allTestContentRecords().lazy + .compactMap { $0.load(withHint: hint) } + .first + } + if let result { + self = result + } else { + return nil + } + } + + /// All testing libraries known to the system including Swift Testing. + @_spi(ForToolsIntegrationOnly) + public static var all: some Sequence { + Self._validateMemoryLayout + return Library.allTestContentRecords().lazy.compactMap { $0.load() } + } +} +#endif + +// MARK: - Referring to Swift Testing directly + +extension Library { + /// The ABI entry point function for the testing library, thunked so that it + /// is compatible with the ``Library`` ABI. + private static let _libraryRecordEntryPoint: Library.EntryPoint = { configurationJSON, configurationJSONByteCount, _, context, recordJSONHandler, completionHandler in +#if !SWT_NO_RUNTIME_LIBRARY_DISCOVERY + // Capture appropriate state from the arguments to forward into the + // canonical entry point function. + let contextBitPattern = UInt(bitPattern: context) + let configurationJSON = UnsafeRawBufferPointer(start: configurationJSON, count: configurationJSONByteCount) + var args: __CommandLineArguments_v0 + let eventHandler: Event.Handler + do { + args = try JSON.decode(__CommandLineArguments_v0.self, from: configurationJSON) + eventHandler = try eventHandlerForStreamingEvents(withVersionNumber: args.eventStreamVersionNumber, encodeAsJSONLines: false) { recordJSON in + let context = UnsafeRawPointer(bitPattern: contextBitPattern)! + recordJSONHandler(recordJSON.baseAddress!, recordJSON.count, 0, context) + } + } catch { + // TODO: interpret more complex results than a process exit code + var resultJSON = "\(EXIT_FAILURE)" + return resultJSON.withUTF8 { resultJSON in + completionHandler(resultJSON.baseAddress!, resultJSON.count, 0, context) + } + } + + // Avoid infinite recursion and double JSON output. (Other libraries don't + // need to clear these fields.) + args.testingLibrary = nil + args.eventStreamOutputPath = nil + + // Create an async context and run tests within it. + let run = { @Sendable [args] in + let context = UnsafeRawPointer(bitPattern: contextBitPattern)! + let exitCode = await Testing.entryPoint(passing: args, eventHandler: eventHandler) + var resultJSON = "\(exitCode)" + resultJSON.withUTF8 { resultJSON in + completionHandler(resultJSON.baseAddress!, resultJSON.count, 0, context) + } + } + +#if !SWT_NO_UNSTRUCTURED_TASKS + Task.detached { await run() } +#else + Task.runInline { await run() } +#endif +#else + // There is no way to call this function without pointer shenanigans because + // we are not exposing `callEntryPoint()` nor are we emitting a record into + // the test content section. + swt_unreachable() +#endif + } + + /// An instance of this type representing Swift Testing itself. + public static let swiftTesting: Self = { + Self( + rawValue: ( + displayName: StaticString("Swift Testing").constUTF8CString, + name: StaticString("swift-testing").constUTF8CString, + entryPoint: _libraryRecordEntryPoint, + reserved: (0, 0, 0, 0, 0) + ) + ) + }() +} + +#if !SWT_NO_RUNTIME_LIBRARY_DISCOVERY +// MARK: - Our very own library record + +private func _libraryRecordAccessor(_ outValue: UnsafeMutableRawPointer, _ type: UnsafeRawPointer, _ hint: UnsafeRawPointer?, _ reserved: UInt) -> CBool { +#if !hasFeature(Embedded) + // Make sure that the caller supplied the right Swift type. If a testing + // library is implemented in a language other than Swift, they can either: + // ignore this argument; or ask the Swift runtime for the type metadata + // pointer and compare it against the value `type.pointee` (`*type` in C). + guard type.load(as: Any.Type.self) == Library.self else { + return false + } +#endif + + // Check if the name of the testing library the caller wants is equivalent to + // "Swift Testing", ignoring case and punctuation. The `hint` argument is a + // structure whose first field is the name of the library as a UTF-8 C string. + // If the caller passed `nil` for `hint`, it wants records for all libraries. + // + // Other libraries may opt to compare their names differently or accept + // multiple different strings/patterns. + let hint = hint.map { $0.load(as: Library.TestContentAccessorHint.self) } + if let hint { + guard let name = String(validatingCString: hint.name), + String(name.filter(\.isLetter)).lowercased() == "swifttesting" else { + return false + } + } + + // Initialize the provided memory to the (ABI-stable) library structure. + _ = outValue.initializeMemory(as: Library.self, to: .swiftTesting) + + return true +} + +#if objectFormat(MachO) +@section("__DATA_CONST,__swift5_tests") +#elseif objectFormat(ELF) || objectFormat(Wasm) +@section("swift5_tests") +#elseif objectFormat(COFF) +@section(".sw5test$B") +#else +@__testing(warning: "Platform-specific implementation missing: test content section name unavailable") +#endif +@used +private let _libraryRecord: __TestContentRecord = ( + kind: 0x6D61696E, /* 'main' */ + reserved1: 0, + accessor: _libraryRecordAccessor, + context: 0, + reserved2: 0 +) +#endif diff --git a/Sources/Testing/Events/Event.swift b/Sources/Testing/Events/Event.swift index 2404a60fd..cb8e4aba0 100644 --- a/Sources/Testing/Events/Event.swift +++ b/Sources/Testing/Events/Event.swift @@ -13,6 +13,21 @@ public struct Event: Sendable { /// An enumeration describing the various kinds of event that can be observed. public enum Kind: Sendable { + /// A library was discovered during test run planning. + /// + /// - Parameters: + /// - library: A description of the discovered library. + /// + /// This event is recorded once, before events of kind ``testDiscovered``, + /// and indicates which testing library's tests will run. + /// + /// This event is also posted once per library when + /// `swift test --experimental-list-libraries` is called. In that case, + /// events are posted for all discovered libraries regardless of whether or + /// not they would run. + @_spi(Experimental) + indirect case libraryDiscovered(_ library: Library) + /// A test was discovered during test run planning. /// /// This event is recorded once per discovered test when ``Runner/run()`` is @@ -420,6 +435,18 @@ extension Event { extension Event.Kind { /// A serializable enumeration describing the various kinds of event that can be observed. public enum Snapshot: Sendable, Codable { + /// A library was discovered during test run planning. + /// + /// This event is recorded once, before events of kind ``testDiscovered``, + /// and indicates which testing library's tests will run. + /// + /// This event is also posted once per library when + /// `swift test --experimental-list-libraries` is called. In that case, + /// events are posted for all discovered libraries regardless of whether or + /// not they would run. + @_spi(Experimental) + case libraryDiscovered + /// A test was discovered during test run planning. /// /// This event is recorded once per discovered test when ``Runner/run()`` is @@ -565,6 +592,8 @@ extension Event.Kind { /// - kind: The original event kind to snapshot. public init(snapshotting kind: Event.Kind) { switch kind { + case .libraryDiscovered: + self = .libraryDiscovered case .testDiscovered: self = .testDiscovered case .runStarted: diff --git a/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift b/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift index 0d8ed6286..04f08da90 100644 --- a/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift +++ b/Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift @@ -362,7 +362,7 @@ extension Event.HumanReadableOutputRecorder { // Finally, produce any messages for the event. switch event.kind { - case .testDiscovered: + case .libraryDiscovered, .testDiscovered: // Suppress events of this kind from output as they are not generally // interesting in human-readable output. break diff --git a/Sources/Testing/Running/Runner.swift b/Sources/Testing/Running/Runner.swift index 9838ad5b9..b544d246f 100644 --- a/Sources/Testing/Running/Runner.swift +++ b/Sources/Testing/Running/Runner.swift @@ -482,11 +482,14 @@ extension Runner { await Configuration.withCurrent(runner.configuration) { // Post an event for every test in the test plan being run. These events // are turned into JSON objects if JSON output is enabled. - let tests = runner.plan.stepGraph.compactMap { $0.value?.test } - for test in tests { - Event.post(.testDiscovered, for: (test, nil), configuration: runner.configuration) + do { + Event.post(.libraryDiscovered(.swiftTesting), for: (nil, nil), configuration: runner.configuration) + let tests = runner.plan.steps.map(\.test) + for test in tests { + Event.post(.testDiscovered, for: (test, nil), configuration: runner.configuration) + } + schedule(tests) } - schedule(tests) Event.post(.runStarted, for: (nil, nil), configuration: runner.configuration) defer { diff --git a/Sources/Testing/Support/Additions/StringAdditions.swift b/Sources/Testing/Support/Additions/StringAdditions.swift new file mode 100644 index 000000000..ef7c59fe8 --- /dev/null +++ b/Sources/Testing/Support/Additions/StringAdditions.swift @@ -0,0 +1,20 @@ +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2025 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for Swift project authors +// + +extension StaticString { + /// This string as a compile-time constant C string. + /// + /// - Precondition: This instance of `StaticString` must have been constructed + /// from a string literal, not a Unicode scalar value. + var constUTF8CString: UnsafePointer { + precondition(hasPointerRepresentation, "Cannot construct a compile-time constant C string from a StaticString without pointer representation.") + return UnsafeRawPointer(utf8Start).bindMemory(to: CChar.self, capacity: utf8CodeUnitCount) + } +} diff --git a/Tests/TestingTests/LibraryTests.swift b/Tests/TestingTests/LibraryTests.swift new file mode 100644 index 000000000..f72e9eb73 --- /dev/null +++ b/Tests/TestingTests/LibraryTests.swift @@ -0,0 +1,192 @@ +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2023 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for Swift project authors +// + +#if !SWT_NO_RUNTIME_LIBRARY_DISCOVERY +@testable @_spi(Experimental) @_spi(ForToolsIntegrationOnly) import Testing +private import _TestingInternals + +private import Foundation // for JSONSerialization + +struct `Library tests` { + @Test func `Find all libraries`() throws { + let libraries = Array(Library.all) + #expect(libraries.count > 0) + #expect(libraries.map(\.displayName).contains("Swift Testing")) + } + + @Test func `Find Swift Testing library`() throws { + let library = try #require(Library(named: "SwIfTtEsTiNg")) + #expect(library.displayName == "Swift Testing") + #expect(library.name == "swift-testing") + } + + @Test func `Run mock library`() async throws { + try await confirmation("(Mock) issue recorded") { issueRecorded in + let library = try #require(Library(named: "mock")) + + var args = __CommandLineArguments_v0() + args.eventStreamVersionNumber = ABI.v0.versionNumber + let exitCode = await library.callEntryPoint(passing: args) { recordJSON in + do { + let recordJSON = Data(recordJSON) + let jsonObject = try JSONSerialization.jsonObject(with: recordJSON) + let record = try #require(jsonObject as? [String: Any]) + if let kind = record["kind"] as? String, let payload = record["payload"] as? [String: Any] { + if kind == "event", let eventKind = payload["kind"] as? String { + if eventKind == "issueRecorded" { + issueRecorded() + } + } + } + } catch { + Issue.record(error) + } + } + #expect(exitCode == EXIT_SUCCESS) + } + } +} + +// MARK: - Fixtures + +extension Library { + private static let _mockRecordEntryPoint: Library.EntryPoint = { configurationJSON, configurationJSONByteCount, _, context, recordJSONHandler, completionHandler in + let tests: [[String: Any]] = [ + [ + "kind": "function", + "name": "mock_test_1", + "sourceLocation": [ + "filePath": "/tmp/mock_file.pascal", + "_filePath": "/tmp/mock_file.pascal", + "line": 1, + "column": 1, + ], + "id": "mock_test_1_id", + "isParameterized": false + ] + ] + + let events: [[String: Any]] = [ + [ + "kind": "runStarted", + ], + [ + "kind": "testStarted", + "testID": "mock_test_1_id" + ], + [ + "kind": "issueRecorded", + "testID": "mock_test_1_id", + "issue": [ + "isKnown": false, + "sourceLocation": [ + "filePath": "/tmp/mock_file.pascal", + "_filePath": "/tmp/mock_file.pascal", + "line": 20, + "column": 1, + ], + ] + ], + [ + "kind": "testEnded", + "testID": "mock_test_1_id" + ], + [ + "kind": "runEnded", + ], + ] + + for var test in tests { + test = [ + "version": 0, + "kind": "test", + "payload": test + ] + let json = try! JSONSerialization.data(withJSONObject: test) + json.withUnsafeBytes { json in + recordJSONHandler(json.baseAddress!, json.count, 0, context) + } + } + let now1970 = Date().timeIntervalSince1970 + for var (i, event) in events.enumerated() { + event["instant"] = [ + "absolute": i, + "since1970": now1970 + Double(i), + ] + event = [ + "version": 0, + "kind": "event", + "payload": event + ] + let json = try! JSONSerialization.data(withJSONObject: event) + json.withUnsafeBytes { json in + recordJSONHandler(json.baseAddress!, json.count, 0, context) + } + } + + var resultJSON = "0" + resultJSON.withUTF8 { resultJSON in + completionHandler(resultJSON.baseAddress!, resultJSON.count, 0, context) + } + } + + static let mock = Self( + rawValue: ( + displayName: StaticString("Mock Testing Library").constUTF8CString, + name: StaticString("mock").constUTF8CString, + entryPoint: _mockRecordEntryPoint, + reserved: (0, 0, 0, 0, 0) + ) + ) +} + +#if objectFormat(MachO) +@section("__DATA_CONST,__swift5_tests") +#elseif objectFormat(ELF) || objectFormat(Wasm) +@section("swift5_tests") +#elseif objectFormat(COFF) +@section(".sw5test$B") +#else +@__testing(warning: "Platform-specific implementation missing: test content section name unavailable") +#endif +@used +private let _mockLibraryRecord: __TestContentRecord = ( + kind: 0x6D61696E, /* 'main' */ + reserved1: 0, + accessor: _mockLibraryRecordAccessor, + context: 0, + reserved2: 0 +) + +private func _mockLibraryRecordAccessor(_ outValue: UnsafeMutableRawPointer, _ type: UnsafeRawPointer, _ hint: UnsafeRawPointer?, _ reserved: UInt) -> CBool { +#if !hasFeature(Embedded) + // Make sure that the caller supplied the right Swift type. If a testing + // library is implemented in a language other than Swift, they can either: + // ignore this argument; or ask the Swift runtime for the type metadata + // pointer and compare it against the value `type.pointee` (`*type` in C). + guard type.load(as: Any.Type.self) == Library.self else { + return false + } +#endif + + // Check if the name of the testing library the caller wants is equivalent to + // "Swift Testing", ignoring case and punctuation. (If the caller did not + // specify a library name, the caller wants records for all libraries.) + let hint = hint.map { $0.load(as: UnsafePointer.self) } + if let hint, 0 != strcasecmp(hint, "mock") { + return false + } + + // Initialize the provided memory to the (ABI-stable) library structure. + _ = outValue.initializeMemory(as: Library.self, to: .mock) + + return true +} +#endif diff --git a/Tests/TestingTests/SwiftPMTests.swift b/Tests/TestingTests/SwiftPMTests.swift index 029fa1200..84078b235 100644 --- a/Tests/TestingTests/SwiftPMTests.swift +++ b/Tests/TestingTests/SwiftPMTests.swift @@ -369,6 +369,7 @@ struct SwiftPMTests { ("--experimental-event-stream-output", "--experimental-event-stream-version", ABI.v6_3.versionNumber), ("--experimental-event-stream-output", "--experimental-event-stream-version", ABI.v6_4.versionNumber), ("--event-stream-output-path", "--event-stream-version", ABI.v6_4.versionNumber), + ("--experimental-event-stream-output", "--experimental-event-stream-version", ABI.ExperimentalVersion.versionNumber), ]) func eventStreamOutput(outputArgumentName: String, versionArgumentName: String, version: VersionNumber) async throws { let version = try #require(ABI.version(forVersionNumber: version)) @@ -398,6 +399,9 @@ struct SwiftPMTests { ) {_ in} let eventContext = Event.Context(test: test, testCase: nil, iteration: nil, configuration: nil) + if V.includesExperimentalFields { + configuration.handleEvent(Event(.libraryDiscovered(.swiftTesting), testID: test.id, testCaseID: nil), in: eventContext) + } configuration.handleEvent(Event(.testDiscovered, testID: test.id, testCaseID: nil), in: eventContext) configuration.handleEvent(Event(.runStarted, testID: nil, testCaseID: nil), in: eventContext) do {