Skip to content

Commit a952758

Browse files
committed
[ST-0019] Augment JSON ABI
Enhance Swift Testing's JSON event ABI by exposing test metadata that is currently unavailable to external tools. By including test tags, bug associations, and time limits in the JSON output, this allows third-party tools to provide richer insights and more sophisticated test management capabilities. Motivation: Swift Testing's JSON event stream provides data for external tooling, enabling developers to build test analysis and reporting tools. However, the current implementation lacks access to some test metadata that developers may want to use to organize and manage their test suites. Currently missing from the JSON output are: - **Test tags**: Used for categorization - **Bug associations**: Tracks bugs associated with tests - **Time limits**: Useful for performance monitoring and timeout management This missing metadata limits the capabilities of external tools. For example: - IDE extensions cannot provide tag-based test filtering - CI/CD systems cannot generate reports grouped by test categories - Performance monitoring tools cannot track tests with specific time constraints - Bug tracking integrations cannot correlate test failures with known issues
1 parent 5e86c57 commit a952758

5 files changed

Lines changed: 98 additions & 11 deletions

File tree

Documentation/ABI/JSON.md

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -160,10 +160,21 @@ additional `"testCases"` field describing the individual test cases.
160160
["displayName": <string>,] ; the user-supplied custom display name
161161
"sourceLocation": <source-location>, ; where the test is defined
162162
"id": <test-id>,
163-
"isParameterized": <bool> ; is this a parameterized test function or not?
163+
"isParameterized": <bool>, ; is this a parameterized test function or not?
164+
["tags": <array:tag>,] ; the tags associated with this test function
165+
["bugs": <array:bug>,] ; the bugs associated with this test function
166+
["timeLimit": <number>] ; the time limit associated with this test function
164167
}
165168
166169
<test-id> ::= <string> ; an opaque string representing the test case
170+
171+
<tag> ::= <string> ; a string representation of a tag
172+
173+
<bug> ::= {
174+
["url": <string>,] ; the bug URL
175+
["id": <string>,] ; the bug id
176+
["title": <string>] ; the human readable bug title
177+
}
167178
```
168179

169180
<!--

Sources/Testing/ABI/ABI.swift

Lines changed: 16 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -43,11 +43,11 @@ extension ABI {
4343
}
4444

4545
/// The current supported ABI version (ignoring any experimental versions.)
46-
typealias CurrentVersion = v6_3
46+
typealias CurrentVersion = v6_4
4747

4848
/// The highest defined and supported ABI version (including any experimental
4949
/// versions.)
50-
typealias HighestVersion = v6_3
50+
typealias HighestVersion = v6_4
5151

5252
#if !hasFeature(Embedded)
5353
/// Get the type representing a given ABI version.
@@ -85,6 +85,8 @@ extension ABI {
8585
}
8686

8787
return switch versionNumber {
88+
case ABI.v6_4.versionNumber...:
89+
ABI.v6_4.self
8890
case ABI.v6_3.versionNumber...:
8991
ABI.v6_3.self
9092
case ABI.v0.versionNumber...:
@@ -167,6 +169,18 @@ extension ABI {
167169
}
168170
}
169171

172+
/// A namespace and type for ABI version 6.4 symbols.
173+
///
174+
/// @Metadata {
175+
/// @Available(Swift, introduced: 6.4).
176+
/// }
177+
@_spi(Experimental)
178+
public enum v6_4: Sendable, Version {
179+
static var versionNumber: VersionNumber {
180+
VersionNumber(6, 4)
181+
}
182+
}
183+
170184
/// A namespace and type representing the ABI version whose symbols are
171185
/// considered experimental.
172186
enum ExperimentalVersion: Sendable, Version {

Sources/Testing/ABI/Encoded/ABI.EncodedTest.swift

Lines changed: 39 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
//
22
// This source file is part of the Swift.org open source project
33
//
4-
// Copyright (c) 2024 Apple Inc. and the Swift project authors
4+
// Copyright (c) 2024-2026 Apple Inc. and the Swift project authors
55
// Licensed under Apache License v2.0 with Runtime Library Exception
66
//
77
// See https://swift.org/LICENSE.txt for license information
@@ -73,10 +73,30 @@ extension ABI {
7373
/// is `nil`.
7474
var isParameterized: Bool?
7575

76+
77+
/// An equivalent of ``tags`` that preserved ABIv6.3 support.
78+
var _tags: [String]?
79+
7680
/// The tags associated with the test.
7781
///
78-
/// - Warning: Tags are not yet part of the JSON schema.
79-
var _tags: [String]?
82+
/// @Metadata {
83+
/// @Available(Swift, introduced: 6.4)
84+
/// }
85+
var tags: [String]?
86+
87+
/// The bugs associated with the test.
88+
///
89+
/// @Metadata {
90+
/// @Available(Swift, introduced: 6.4)
91+
/// }
92+
var bugs: [Bug]?
93+
94+
/// The time limits associated with the test.
95+
///
96+
/// @Metadata {
97+
/// @Available(Swift, introduced: 6.4)
98+
/// }
99+
var timeLimit: Double?
80100

81101
init(encoding test: borrowing Test) {
82102
if test.isSuite {
@@ -95,11 +115,25 @@ extension ABI {
95115
if isParameterized == true {
96116
_testCases = test.uncheckedTestCases?.map(EncodedTestCase.init(encoding:))
97117
}
98-
99118
let tags = test.tags
100119
if !tags.isEmpty {
101-
_tags = tags.map(String.init(describing:))
120+
self._tags = tags.map(String.init(describing:))
121+
}
122+
}
123+
124+
if V.versionNumber >= ABI.v6_4.versionNumber {
125+
self.tags = test.tags.sorted().map { tag in
126+
switch tag.kind {
127+
case .staticMember(let value): value
128+
}
129+
}
130+
let bugs = test.associatedBugs
131+
if !bugs.isEmpty {
132+
self.bugs = bugs
102133
}
134+
self.timeLimit = test.timeLimit
135+
.map(TimeValue.init)
136+
.map(Double.init)
103137
}
104138
}
105139
}

Tests/TestingTests/EventHandlingInteropTests.swift

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -63,7 +63,7 @@ struct EventHandlingInteropTests {
6363
try Self.handlerContents.withLock {
6464
let contents = try #require(
6565
$0, "Fallback should have been called with non nil contents")
66-
#expect(contents.version == "6.3")
66+
#expect(contents.version == "\(ABI.CurrentVersion.versionNumber)")
6767
#expect(contents.record?.contains("A system failure occurred") ?? false)
6868
}
6969
}

Tests/TestingTests/SwiftPMTests.swift

Lines changed: 30 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -367,6 +367,8 @@ struct SwiftPMTests {
367367
("--event-stream-output-path", "--event-stream-version", ABI.v0.versionNumber),
368368
("--experimental-event-stream-output", "--experimental-event-stream-version", ABI.v0.versionNumber),
369369
("--experimental-event-stream-output", "--experimental-event-stream-version", ABI.v6_3.versionNumber),
370+
("--experimental-event-stream-output", "--experimental-event-stream-version", ABI.v6_4.versionNumber),
371+
("--event-stream-output-path", "--event-stream-version", ABI.v6_4.versionNumber),
370372
])
371373
func eventStreamOutput(outputArgumentName: String, versionArgumentName: String, version: VersionNumber) async throws {
372374
let version = try #require(ABI.version(forVersionNumber: version))
@@ -381,9 +383,19 @@ struct SwiftPMTests {
381383
defer {
382384
_ = remove(temporaryFilePath)
383385
}
386+
let testTimeLimit = 3
387+
let expectedArgs = ["argument1", "argument2"]
384388
do {
385389
let configuration = try configurationForEntryPoint(withArguments: ["PATH", outputArgumentName, temporaryFilePath, versionArgumentName, "\(version.versionNumber)"])
386-
let test = Test(.tags(.blue)) {}
390+
let test = Test(
391+
.tags(.blue),
392+
.bug("https://my.defect.com/1234"),
393+
.bug("other defect"),
394+
.timeLimit(Swift.Duration.seconds(testTimeLimit + 100)),
395+
.timeLimit(Swift.Duration.seconds(testTimeLimit)),
396+
.timeLimit(Swift.Duration.seconds(testTimeLimit + 10)),
397+
arguments: expectedArgs as [String]
398+
) {_ in}
387399
let eventContext = Event.Context(test: test, testCase: nil, configuration: nil)
388400

389401
configuration.handleEvent(Event(.testDiscovered, testID: test.id, testCaseID: nil), in: eventContext)
@@ -407,10 +419,26 @@ struct SwiftPMTests {
407419
#expect(testRecords.count == 1)
408420
for testRecord in testRecords {
409421
if version.includesExperimentalFields {
422+
let actualTestCases = testRecord._testCases
423+
let testCases = try #require(actualTestCases)
424+
#expect(testCases.count == expectedArgs.count)
410425
#expect(testRecord._tags != nil)
411426
} else {
412-
#expect(testRecord._tags == nil)
427+
#expect(testRecord._testCases == nil)
413428
}
429+
430+
if version.versionNumber >= ABI.v6_4.versionNumber {
431+
let testTags = try #require(testRecord.tags)
432+
#expect(testTags.count >= 1)
433+
for tag in testTags {
434+
#expect(!tag.starts(with: "."))
435+
}
436+
let bugs = try #require(testRecord.bugs)
437+
#expect(bugs.count == 2)
438+
let timeLimit = try #require(testRecord.timeLimit)
439+
#expect(timeLimit == Double(testTimeLimit))
440+
}
441+
414442
}
415443
let eventRecords = decodedRecords.compactMap { record in
416444
if case let .event(event) = record.kind {

0 commit comments

Comments
 (0)