Skip to content

Commit bb8abff

Browse files
authored
Show subexpressions separately in expectation failures. (#1572)
This PR refactors how we capture and display subexpressions when an expectation fails. For example, given the following test: ```swift @test func foo() { let lhs = "abc123" let rhs = "d" #expect(lhs.contains(rhs)) } ``` We currently output: ``` < > Test foo() started. <X> Test foo() recorded an issue at FileName.swift:X:Y: Expectation failed: (lhs → "abc123").contains(rhs → "d") <X> Test foo() failed after 0.003 seconds with 1 issue. ``` This can be hard to read for more complex expressions as the real values are nested inside a single line. With this change, we instead output: ``` < > Test foo() started. <X> Test foo() recorded an issue at FileName.swift:X:Y: Expectation failed: lhs.contains(rhs) --> lhs.contains(rhs) → false --> lhs → "abc123" --> rhs → "d" <X> Test foo() failed after 0.003 seconds with 1 issue. ``` This change also adds the subexpression breakdown to our JSON schema as an experimental field of the "issue" event record: ```json { "kind" : "event", "payload" : { /* ... */ "issue" : { /* ... */ "_expectation" : { "_expression" : { "children" : [ { "runtimeTypeName" : "Swift.String", "runtimeValue" : "\"abc123\"", "sourceCode" : "lhs" }, { "runtimeTypeName" : "Swift.String", "runtimeValue" : "\"d\"", "sourceCode" : "rhs" } ], "runtimeTypeName" : "Swift.Bool", "runtimeValue" : "false", "sourceCode" : "lhs.contains(rhs)" } }, /* ... */ }, }, "version" : "99.0.0" } ``` ### Checklist: - [x] Code and documentation should follow the style of the [Style Guide](https://github.com/apple/swift-testing/blob/main/Documentation/StyleGuide.md). - [x] If public symbols are renamed or modified, DocC references should be updated.
1 parent 91f685b commit bb8abff

14 files changed

Lines changed: 342 additions & 368 deletions
Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
//
2+
// This source file is part of the Swift.org open source project
3+
//
4+
// Copyright (c) 2025 Apple Inc. and the Swift project authors
5+
// Licensed under Apache License v2.0 with Runtime Library Exception
6+
//
7+
// See https://swift.org/LICENSE.txt for license information
8+
// See https://swift.org/CONTRIBUTORS.txt for Swift project authors
9+
//
10+
11+
extension ABI {
12+
/// A type implementing the JSON encoding of ``Expectation`` for the ABI entry
13+
/// point and event stream output.
14+
///
15+
/// This type is not part of the public interface of the testing library. It
16+
/// assists in converting values to JSON; clients that consume this JSON are
17+
/// expected to write their own decoders.
18+
///
19+
/// - Warning: Expectations are not yet part of the JSON schema.
20+
struct EncodedExpectation<V>: Sendable where V: ABI.Version {
21+
/// The expression evaluated by this expectation.
22+
///
23+
/// - Warning: Expressions are not yet part of the JSON schema.
24+
var _expression: EncodedExpression<V>
25+
26+
init(encoding expectation: borrowing Expectation, in eventContext: borrowing Event.Context) {
27+
_expression = EncodedExpression<V>(encoding: expectation.evaluatedExpression, in: eventContext)
28+
}
29+
}
30+
}
31+
32+
// MARK: - Codable
33+
34+
extension ABI.EncodedExpectation: Codable {}
Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
1+
//
2+
// This source file is part of the Swift.org open source project
3+
//
4+
// Copyright (c) 2025 Apple Inc. and the Swift project authors
5+
// Licensed under Apache License v2.0 with Runtime Library Exception
6+
//
7+
// See https://swift.org/LICENSE.txt for license information
8+
// See https://swift.org/CONTRIBUTORS.txt for Swift project authors
9+
//
10+
11+
extension ABI {
12+
/// A type implementing the JSON encoding of ``Expression`` for the ABI entry
13+
/// point and event stream output.
14+
///
15+
/// This type is not part of the public interface of the testing library. It
16+
/// assists in converting values to JSON; clients that consume this JSON are
17+
/// expected to write their own decoders.
18+
///
19+
/// - Warning: Expressions are not yet part of the JSON schema.
20+
struct EncodedExpression<V>: Sendable where V: ABI.Version {
21+
/// The source code of the original captured expression.
22+
var sourceCode: String
23+
24+
/// A string representation of the runtime value of this expression.
25+
///
26+
/// If the runtime value of this expression has not been evaluated, the
27+
/// value of this property is `nil`.
28+
var runtimeValue: String?
29+
30+
/// The fully-qualified name of the type of value represented by
31+
/// `runtimeValue`, or `nil` if that value has not been captured.
32+
var runtimeTypeName: String?
33+
34+
/// Any child expressions within this expression.
35+
var children: [EncodedExpression]?
36+
37+
init(encoding expression: borrowing __Expression, in eventContext: borrowing Event.Context) {
38+
sourceCode = expression.sourceCode
39+
runtimeValue = expression.runtimeValue.map(String.init(describingForTest:))
40+
runtimeTypeName = expression.runtimeValue.map(\.typeInfo.fullyQualifiedName)
41+
if !expression.subexpressions.isEmpty {
42+
children = expression.subexpressions.map { [eventContext = copy eventContext] subexpression in
43+
Self(encoding: subexpression, in: eventContext)
44+
}
45+
}
46+
}
47+
}
48+
}
49+
50+
// MARK: - Codable
51+
52+
extension ABI.EncodedExpression: Codable {}

Sources/Testing/ABI/Encoded/ABI.EncodedIssue.swift

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -60,6 +60,11 @@ extension ABI {
6060
/// - Warning: Errors are not yet part of the JSON schema.
6161
var _error: EncodedError<V>?
6262

63+
/// The expectation associated with this issue, if applicable.
64+
///
65+
/// - Warning: Expectations are not yet part of the JSON schema.
66+
var _expectation: EncodedExpectation<V>?
67+
6368
init(encoding issue: borrowing Issue, in eventContext: borrowing Event.Context) {
6469
// >= v0
6570
isKnown = issue.isKnown
@@ -82,6 +87,9 @@ extension ABI {
8287
if let error = issue.error {
8388
_error = EncodedError(encoding: error, in: eventContext)
8489
}
90+
if case let .expectationFailed(expectation) = issue.kind {
91+
_expectation = EncodedExpectation(encoding: expectation, in: eventContext)
92+
}
8593
}
8694
}
8795
}

Sources/Testing/ABI/Encoded/ABI.EncodedMessage.swift

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -67,9 +67,20 @@ extension ABI {
6767
/// The human-readable, unformatted text associated with this message.
6868
var text: String
6969

70+
/// How much to indent this message when presenting it.
71+
///
72+
/// - Warning: This property is not yet part of the JSON schema.
73+
var _indentation: Int?
74+
7075
init(encoding message: borrowing Event.HumanReadableOutputRecorder.Message) {
7176
symbol = Symbol(encoding: message.symbol ?? .default)
7277
text = message.conciseStringValue ?? message.stringValue
78+
79+
if V.includesExperimentalFields {
80+
if message.indentation > 0 {
81+
_indentation = message.indentation
82+
}
83+
}
7384
}
7485
}
7586
}

Sources/Testing/CMakeLists.txt

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,8 @@ add_library(Testing
1818
ABI/Encoded/ABI.EncodedBacktrace.swift
1919
ABI/Encoded/ABI.EncodedError.swift
2020
ABI/Encoded/ABI.EncodedEvent.swift
21+
ABI/Encoded/ABI.EncodedExpectation.swift
22+
ABI/Encoded/ABI.EncodedExpression.swift
2123
ABI/Encoded/ABI.EncodedInstant.swift
2224
ABI/Encoded/ABI.EncodedIssue.swift
2325
ABI/Encoded/ABI.EncodedMessage.swift

Sources/Testing/Events/Recorder/Event.ConsoleOutputRecorder.swift

Lines changed: 25 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -129,6 +129,22 @@ private let _ansiEscapeCodePrefix = "\u{001B}["
129129
private let _resetANSIEscapeCode = "\(_ansiEscapeCodePrefix)0m"
130130

131131
extension Event.Symbol {
132+
/// Get the string value to use for a message with no associated symbol.
133+
///
134+
/// - Parameters:
135+
/// - options: Options to use when writing the symbol.
136+
///
137+
/// - Returns: A string representation of "no symbol" appropriate for writing
138+
/// to a stream.
139+
fileprivate static func placeholderStringValue(options: Event.ConsoleOutputRecorder.Options) -> String {
140+
#if os(macOS) || (os(iOS) && targetEnvironment(macCatalyst))
141+
if options.useSFSymbols {
142+
return " "
143+
}
144+
#endif
145+
return " "
146+
}
147+
132148
/// Get the string value for this symbol with the given write options.
133149
///
134150
/// - Parameters:
@@ -174,7 +190,7 @@ extension Event.Symbol {
174190
return symbolCharacter
175191
}
176192
}
177-
return "\(symbolCharacter)"
193+
return symbolCharacter
178194
}
179195
}
180196

@@ -305,18 +321,12 @@ extension Event.ConsoleOutputRecorder {
305321
/// - Returns: Whether any output was produced and written to this instance's
306322
/// destination.
307323
@discardableResult public func record(_ event: borrowing Event, in context: borrowing Event.Context) -> Bool {
308-
let messages = _humanReadableOutputRecorder.record(event, in: context)
309-
310-
// Padding to use in place of a symbol for messages that don't have one.
311-
var padding = " "
312-
#if os(macOS) || (os(iOS) && targetEnvironment(macCatalyst))
313-
if options.useSFSymbols {
314-
padding = " "
315-
}
316-
#endif
324+
let symbolPlaceholder = Event.Symbol.placeholderStringValue(options: options)
317325

326+
let messages = _humanReadableOutputRecorder.record(event, in: context)
318327
let lines = messages.lazy.map { [test = context.test] message in
319-
let symbol = message.symbol?.stringValue(options: options) ?? padding
328+
let symbol = message.symbol?.stringValue(options: options) ?? symbolPlaceholder
329+
let indentation = String(repeating: " ", count: message.indentation)
320330

321331
if case .details = message.symbol {
322332
// Special-case the detail symbol to apply grey to the entire line of
@@ -325,17 +335,17 @@ extension Event.ConsoleOutputRecorder {
325335
// to the indentation provided by the symbol.
326336
var lines = message.stringValue.split(omittingEmptySubsequences: false, whereSeparator: \.isNewline)
327337
lines = CollectionOfOne(lines[0]) + lines.dropFirst().map { line in
328-
"\(padding) \(line)"
338+
"\(indentation)\(symbolPlaceholder) \(line)"
329339
}
330340
let stringValue = lines.joined(separator: "\n")
331341
if options.useANSIEscapeCodes, options.ansiColorBitDepth > 1 {
332-
return "\(_ansiEscapeCodePrefix)90m\(symbol) \(stringValue)\(_resetANSIEscapeCode)\n"
342+
return "\(_ansiEscapeCodePrefix)90m\(symbol) \(indentation)\(stringValue)\(_resetANSIEscapeCode)\n"
333343
} else {
334-
return "\(symbol) \(stringValue)\n"
344+
return "\(symbol) \(indentation)\(stringValue)\n"
335345
}
336346
} else {
337347
let colorDots = test.map { self.colorDots(for: $0.tags) } ?? ""
338-
return "\(symbol) \(colorDots)\(message.stringValue)\n"
348+
return "\(symbol) \(indentation)\(colorDots)\(message.stringValue)\n"
339349
}
340350
}
341351

Sources/Testing/Events/Recorder/Event.HumanReadableOutputRecorder.swift

Lines changed: 18 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,15 @@ extension Event {
2929
/// The symbol associated with this message, if any.
3030
var symbol: Symbol?
3131

32+
/// How much to indent this message when presenting it.
33+
///
34+
/// The way in which this additional indentation is rendered is
35+
/// implementation-defined. Typically, the greater the value of this
36+
/// property, the more whitespace characters are inserted.
37+
///
38+
/// Rendering of indentation is optional.
39+
var indentation = 0
40+
3241
/// The human-readable message.
3342
var stringValue: String
3443

@@ -503,20 +512,18 @@ extension Event.HumanReadableOutputRecorder {
503512
additionalMessages.append(_formattedComment(knownIssueComment))
504513
}
505514

506-
if verbosity > 0, case let .expectationFailed(expectation) = issue.kind {
515+
if verbosity >= 0, case let .expectationFailed(expectation) = issue.kind {
507516
let expression = expectation.evaluatedExpression
508-
func addMessage(about expression: __Expression) {
509-
let description = expression.expandedDebugDescription()
510-
additionalMessages.append(Message(symbol: .details, stringValue: description))
511-
}
512-
let subexpressions = expression.subexpressions
513-
if subexpressions.isEmpty {
514-
addMessage(about: expression)
515-
} else {
516-
for subexpression in subexpressions {
517-
addMessage(about: subexpression)
517+
func addMessage(about expression: __Expression, depth: Int) {
518+
let description = expression.expandedDescription(verbose: verbosity > 0)
519+
if description != expression.sourceCode {
520+
additionalMessages.append(Message(symbol: .details, indentation: depth, stringValue: description))
521+
}
522+
for subexpression in expression.subexpressions {
523+
addMessage(about: subexpression, depth: depth + 1)
518524
}
519525
}
526+
addMessage(about: expression, depth: 0)
520527
}
521528

522529
let atSourceLocation = issue.sourceLocation.map { " at \($0)" } ?? ""

0 commit comments

Comments
 (0)