Skip to content

Commit 22ffa1f

Browse files
grynspangrynspan
authored
Add support for .expectationFailed in interop and exit tests. (#1608)
This PR enhances the JSON event stream to allow us to recognize and recover issues of kind `.expectationFailed`. This then improves the fidelity of these issues when passed through XCTest interop and exit test process boundaries. For example, given the following exit test: ```swift @test func foo() async { await #expect(processExitsWith: .success) { struct S: Equatable { var x: Int var y: String } let lhs = S(x: 1, y: "abc") let rhs = S(x: 2, y: "def") #expect(lhs == rhs) } } ``` We'll now get a breakdown of the failed expectation (note I recorded this example with `--verbose` so type info would be visible too): ``` <X> Test foo() recorded an issue at [...]: Expectation failed: lhs == rhs `-> lhs == rhs: Swift.Bool → false `-> lhs: ModuleName.S → S(x: 1, y: "abc") `-> x: Swift.Int → 1 `-> y: Swift.String → "abc" `-> rhs: ModuleName.S → S(x: 2, y: "def") `-> x: Swift.Int → 2 `-> y: Swift.String → "def" ``` In the process of implementing this change, I was able to reconcile the issue-decoding logic used in interop and exit tests so they share an implementation. This PR also resolves an issue running interop tests inside Xcode 26.4 where the tests could spuriously fail. ### Checklist: - [x] Code and documentation should follow the style of the [Style Guide](https://github.com/apple/swift-testing/blob/main/Documentation/StyleGuide.md). - [x] If public symbols are renamed or modified, DocC references should be updated.
1 parent 93837c0 commit 22ffa1f

8 files changed

Lines changed: 167 additions & 103 deletions

File tree

‎Sources/Testing/ABI/Encoded/ABI.EncodedExpression.swift‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -51,3 +51,25 @@ extension ABI {
5151
// MARK: - Codable
5252

5353
extension ABI.EncodedExpression: Codable {}
54+
55+
// MARK: - Conversion to/from library types
56+
57+
extension __Expression {
58+
/// Initialize an instance of this type from the given value.
59+
///
60+
/// - Parameters:
61+
/// - expression: The encoded expression to initialize this instance from.
62+
init?<V>(decoding expression: ABI.EncodedExpression<V>) {
63+
self.init(expression.sourceCode)
64+
if let runtimeValue = expression.runtimeValue,
65+
let runtimeTypeName = expression.runtimeTypeName {
66+
self.runtimeValue = __Expression.Value(
67+
description: runtimeValue,
68+
typeInfo: TypeInfo(fullyQualifiedName: runtimeTypeName, mangledName: nil)
69+
)
70+
}
71+
if let children = expression.children {
72+
self.subexpressions = children.compactMap(__Expression.init(decoding:))
73+
}
74+
}
75+
}

‎Sources/Testing/ABI/Encoded/ABI.EncodedIssue.swift‎

Lines changed: 69 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -99,3 +99,72 @@ extension ABI {
9999

100100
extension ABI.EncodedIssue: Codable {}
101101
extension ABI.EncodedIssue.Severity: Codable {}
102+
103+
// MARK: - Conversion to/from library types
104+
105+
extension Issue {
106+
/// Initialize an instance of this type from the given value.
107+
///
108+
/// - Parameters:
109+
/// - event: The encoded event to initialize this instance from.
110+
///
111+
/// If `event` does not represent a recorded issue, the initializer returns
112+
/// `nil`.
113+
init?<V>(decoding event: ABI.EncodedEvent<V>) {
114+
guard let issue = event.issue else {
115+
return nil
116+
}
117+
self.init(decoding: issue)
118+
if let comments = event._comments {
119+
self.comments += comments.map(Comment.init(rawValue:))
120+
}
121+
}
122+
123+
/// Initialize an instance of this type from the given value.
124+
///
125+
/// - Parameters:
126+
/// - issue: The encoded issue to initialize this instance from.
127+
///
128+
/// - Note: For higher fidelity, initialize the issue with an encoded event
129+
/// representing a recorded issue rather than just the encoded issue.
130+
init?<V>(decoding issue: ABI.EncodedIssue<V>) {
131+
let issueKind: Issue.Kind
132+
if let error = issue._error {
133+
issueKind = .errorCaught(error)
134+
} else if let expectation = issue._expectation,
135+
let expression = __Expression(decoding: expectation._expression),
136+
let sourceLocation = issue.sourceLocation.flatMap(SourceLocation.init) {
137+
let expectation = Expectation(
138+
evaluatedExpression: expression,
139+
isPassing: false,
140+
isRequired: false,
141+
sourceLocation: sourceLocation
142+
)
143+
issueKind = .expectationFailed(expectation)
144+
} else {
145+
// TODO: improve fidelity of issue kind reporting (especially those without associated values)
146+
issueKind = .unconditional
147+
}
148+
let severity: Issue.Severity = switch issue.severity {
149+
case .warning:
150+
.warning
151+
case .error, nil:
152+
// Prior to 6.3, all Issues are errors
153+
.error
154+
}
155+
let sourceContext = SourceContext(
156+
backtrace: issue._backtrace.map { Backtrace(addresses: $0.symbolicatedAddresses.map(\.address)) },
157+
sourceLocation: issue.sourceLocation.flatMap(SourceLocation.init)
158+
)
159+
self.init(
160+
kind: issueKind,
161+
severity: severity,
162+
comments: [],
163+
sourceContext: sourceContext
164+
)
165+
if issue.isKnown {
166+
// FIXME: The known issue comment is not currently encoded.
167+
self.knownIssueContext = Issue.KnownIssueContext()
168+
}
169+
}
170+
}

‎Sources/Testing/Events/Event+FallbackEventHandler.swift‎

Lines changed: 4 additions & 66 deletions
Original file line numberDiff line numberDiff line change
@@ -10,63 +10,6 @@
1010

1111
private import _TestingInternals
1212

13-
extension Issue {
14-
/// Attempt to create an `Issue` from a foreign `EncodedIssue`.
15-
///
16-
/// Typically, another testing library transforms its own test issue into the
17-
/// `EncodedEvent` format and then passes it through Swift Testing's installed
18-
/// fallback event handler to be converted into an `Issue`.
19-
///
20-
/// The fidelity of this conversion is limited by the fields present in
21-
/// `EncodedIssue`, in addition to how well the foreign test issue is
22-
/// represented by the schema.
23-
///
24-
/// - Parameter event: The `EncodedIssue` wrapped in an `EncodedEvent`.
25-
/// - Returns: `nil` if this is not an issueRecorded kind of event, or if the
26-
/// event doesn't include an `EncodedIssue`.
27-
init?<V>(event: ABI.EncodedEvent<V>) where V: ABI.Version {
28-
switch event.kind {
29-
case .issueRecorded:
30-
guard let issue = event.issue else { return nil }
31-
let issueKind: Issue.Kind =
32-
if let error = issue._error {
33-
.errorCaught(error)
34-
} else {
35-
// The encoded Issue doesn't include enough information to determine
36-
// the exact kind of issue, so a expectation and unconditional failure
37-
// have the same representation.
38-
.unconditional
39-
}
40-
41-
let severity: Issue.Severity =
42-
switch issue.severity {
43-
case .warning: .warning
44-
case nil, .error: .error
45-
}
46-
47-
let comments = {
48-
let returnedComments = event.messages.map { $0.text }.map(Comment.init(rawValue:))
49-
return if returnedComments.isEmpty {
50-
[Comment("Unknown issue")]
51-
} else {
52-
returnedComments
53-
}
54-
}()
55-
56-
let sourceContext = SourceContext(
57-
backtrace: nil, // Requires backtrace information from the EncodedIssue
58-
sourceLocation: event._sourceLocation.flatMap(SourceLocation.init)
59-
)
60-
61-
self.init(
62-
kind: issueKind, severity: severity, comments: comments, sourceContext: sourceContext)
63-
default:
64-
// The fallback handler does not support this event type
65-
return nil
66-
}
67-
}
68-
}
69-
7013
extension Event {
7114
/// Attempt to handle an event encoded as JSON as if it had been generated in
7215
/// the current testing context.
@@ -84,24 +27,19 @@ extension Event {
8427
static func handle<V>(_ recordJSON: UnsafeRawBufferPointer, encodedWith version: V.Type) throws
8528
where V: ABI.Version {
8629
let record = try JSON.decode(ABI.Record<V>.self, from: recordJSON)
87-
guard
88-
case .event(let event) = record.kind,
89-
let issue = Issue(event: event)
90-
else {
30+
guard case .event(let event) = record.kind,
31+
let issue = Issue(decoding: event) else {
9132
return
9233
}
9334

9435
// For the time being, assume that foreign test events originate from XCTest
9536
let warnForXCTestUsageIssue = {
96-
let sourceContext = SourceContext(
97-
backtrace: issue.sourceContext.backtrace,
98-
sourceLocation: event._sourceLocation.flatMap(SourceLocation.init)
99-
)
10037
return Issue(
10138
kind: .apiMisused, severity: .warning,
10239
comments: [
10340
"XCTest API was used in a Swift Testing test. Adopt Swift Testing primitives, such as #expect, instead."
104-
], sourceContext: sourceContext)
41+
], sourceContext: issue.sourceContext
42+
)
10543
}()
10644

10745
issue.record()

‎Sources/Testing/ExitTests/ExitTest.swift‎

Lines changed: 16 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -1056,39 +1056,25 @@ extension ExitTest {
10561056
return
10571057
}
10581058

1059-
lazy var comments: [Comment] = event._comments?.map(Comment.init(rawValue:)) ?? []
1060-
lazy var sourceContext = SourceContext(
1061-
backtrace: nil, // A backtrace from the child process will have the wrong address space.
1062-
sourceLocation: event._sourceLocation.flatMap(SourceLocation.init)
1063-
)
1064-
lazy var skipInfo = SkipInfo(comment: comments.first, sourceContext: sourceContext)
1065-
if let issue = event.issue {
1066-
// Translate the issue back into a "real" issue and record it
1067-
// in the parent process. This translation is, of course, lossy
1068-
// due to the process boundary, but we make a best effort.
1069-
let issueKind: Issue.Kind = if let error = issue._error {
1070-
.errorCaught(error)
1071-
} else {
1072-
// TODO: improve fidelity of issue kind reporting (especially those without associated values)
1073-
.unconditional
1074-
}
1075-
let severity: Issue.Severity = switch issue.severity {
1076-
case .warning:
1077-
.warning
1078-
case .error, nil:
1079-
// Prior to 6.3, all Issues are errors
1080-
.error
1081-
}
1082-
var issueCopy = Issue(kind: issueKind, severity: severity, comments: comments, sourceContext: sourceContext)
1083-
if issue.isKnown {
1084-
// The known issue comment, if there was one, is already included in
1085-
// the `comments` array above.
1086-
issueCopy.knownIssueContext = Issue.KnownIssueContext()
1087-
}
1088-
issueCopy.record()
1059+
// Translate the event back into a "real" event (such as "issue recorded")
1060+
// and post it in the parent process. This translation is, of course, lossy
1061+
// due to the process boundary, but we make a best effort.
1062+
if var issue = Issue(decoding: event) {
1063+
// A backtrace from the child process will have the wrong address space,
1064+
// so remove the backtrace if present before recording it.
1065+
issue.sourceContext.backtrace = nil
1066+
issue.record()
10891067
} else if let attachment = event.attachment {
10901068
Attachment.record(attachment, sourceLocation: event._sourceLocation.flatMap(SourceLocation.init)!)
10911069
} else if case .testCancelled = event.kind {
1070+
let comment = event._comments?.lazy
1071+
.map(Comment.init(rawValue:))
1072+
.first
1073+
let sourceContext = SourceContext(
1074+
backtrace: nil, // A backtrace from the child process will have the wrong address space.
1075+
sourceLocation: event._sourceLocation.flatMap(SourceLocation.init)
1076+
)
1077+
let skipInfo = SkipInfo(comment: comment, sourceContext: sourceContext)
10921078
_ = try? Test.cancel(with: skipInfo)
10931079
}
10941080
}

‎Sources/Testing/Parameterization/TypeInfo.swift‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -69,11 +69,14 @@ public struct TypeInfo: Sendable {
6969
/// - Parameters:
7070
/// - fullyQualifiedName: The fully-qualified name of the type, with its
7171
/// components separated by a period character (`"."`).
72-
/// - unqualified: The unqualified name of the type.
72+
/// - unqualified: The unqualified name of the type. If `nil`, this string
73+
/// is derived from `fullyQualifiedName`.
7374
/// - mangled: The mangled name of the type, if available.
74-
init(fullyQualifiedName: String, unqualifiedName: String, mangledName: String?) {
75+
init(fullyQualifiedName: String, unqualifiedName: String? = nil, mangledName: String?) {
76+
let fullyQualifiedNameComponents = Self.fullyQualifiedNameComponents(ofTypeWithName: fullyQualifiedName)
77+
let unqualifiedName = unqualifiedName ?? fullyQualifiedNameComponents.last ?? fullyQualifiedName
7578
self.init(
76-
fullyQualifiedNameComponents: Self.fullyQualifiedNameComponents(ofTypeWithName: fullyQualifiedName),
79+
fullyQualifiedNameComponents: fullyQualifiedNameComponents,
7780
unqualifiedName: unqualifiedName,
7881
mangledName: mangledName
7982
)

‎Sources/Testing/SourceAttribution/Expression.swift‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -126,6 +126,23 @@ public struct __Expression: Sendable {
126126
isCollection = mirror.displayStyle?.isCollection ?? false
127127
}
128128

129+
/// Initialize an instance of this type with a previously-generated
130+
/// description of some subject.
131+
///
132+
/// - Parameters:
133+
/// - description: A description of the subject.
134+
/// - typeInfo: The type of the subject.
135+
///
136+
/// This initializer is only used when decoding an instance of
137+
/// ``ABI/EncodedExpression``. Callers should prefer other initializers
138+
/// where possible.
139+
init(description: String, typeInfo: TypeInfo) {
140+
self.description = description
141+
self.debugDescription = description
142+
self.typeInfo = typeInfo
143+
self.isCollection = false
144+
}
145+
129146
/// Initialize an instance of this type with the specified description.
130147
///
131148
/// - Parameters:

‎Tests/TestingTests/EventHandlingInteropTests.swift‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -18,9 +18,9 @@ import Foundation
1818
import Synchronization
1919
#endif
2020

21-
#if !SWIFT_PACKAGE && SWT_TARGET_OS_APPLE
21+
#if SWT_TARGET_OS_APPLE
2222
// Xcode already installs a handler, so the preconditions for this suite may not be met
23-
let interopHandlerMayBeInstalled = true
23+
let interopHandlerMayBeInstalled = Environment.variable(named: "XCTestSessionIdentifier") != nil
2424
#else
2525
let interopHandlerMayBeInstalled = false
2626
#endif
@@ -224,7 +224,7 @@ struct EventHandlingInteropTests {
224224
#expect(
225225
issues.rawValue.map { $0.description }.sorted() == [
226226
"An API was misused (warning): XCTest API was used in a Swift Testing test. Adopt Swift Testing primitives, such as #expect, instead.",
227-
"Issue recorded (error): Unknown issue",
227+
"Issue recorded (error)",
228228
]
229229
)
230230
}

‎Tests/TestingTests/ExitTestTests.swift‎

Lines changed: 30 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -214,7 +214,7 @@ private import _TestingInternals
214214

215215
await Test {
216216
await #expect(processExitsWith: .success) {
217-
#expect(Bool(false), "Something went wrong!")
217+
Issue.record("Something went wrong!")
218218
exit(0)
219219
}
220220
await #expect(processExitsWith: .failure) {
@@ -225,6 +225,35 @@ private import _TestingInternals
225225
}
226226
}
227227

228+
@Test("Exit test issues contain expression trees") func expressionsInIssues() async {
229+
await confirmation("Expectation failed") { expectationFailed in
230+
var configuration = Configuration()
231+
configuration.eventHandler = { event, _ in
232+
guard case let .issueRecorded(issue) = event.kind else {
233+
return
234+
}
235+
if case let .expectationFailed(expectation) = issue.kind,
236+
expectation.evaluatedExpression.sourceCode == "lhs == rhs",
237+
expectation.evaluatedExpression.subexpressions.count > 1 {
238+
expectationFailed()
239+
}
240+
}
241+
configuration.exitTestHandler = ExitTest.handlerForEntryPoint()
242+
243+
await Test {
244+
await #expect(processExitsWith: .success) {
245+
struct S: Equatable {
246+
var x: Int
247+
var y: String
248+
}
249+
let lhs = S(x: 1, y: "abc")
250+
let rhs = S(x: 2, y: "def")
251+
#expect(lhs == rhs)
252+
}
253+
}.run(configuration: configuration)
254+
}
255+
}
256+
228257
private static let attachmentPayload = [UInt8](0...255)
229258

230259
@Test("Exit test forwards attachments") func forwardsAttachments() async {

0 commit comments

Comments
 (0)