Include a note after Swift Testing tests finish if any XCTests failed (#9646)

This adds a note in the console output after Swift Testing tests finish
if one or more XCTests failed, so the user knows to look earlier in the
log output for those details.

Fixes rdar://168311253

### Motivation:

Swift Package Manager runs XCTests followed by Swift Testing tests, and
they are separate subprocess invocations. Each test framework prints its
own output to the console and each has a summary at the end, which
includes the total number of failures. Since Swift Testing finishes
second (assuming both frameworks are enabled), its summary always
appears at the bottom and it can potentially be misleading to a user if
there were one or more XCTest failures but zero Swift Testing failures
because they may incorrectly believe zero tests failed overall.

Long-term, we are planning to unify the console output between these
testing frameworks so that they present a consolidated summary. However,
that remains an ambition that is still being planned and will likely
take time to arrive.

Another motivator for this is that soon, I anticipate we'll land an
enhancement in Swift Testing which will expand its failure summary to
span more lines and make it easier for users to locate which Swift
Testing tests failed. (For details, see
swiftlang/swift-testing#1420.) This will be
helpful, but also has the potential to exacerbate the pre-existing
confusion situation if (say) a mixture of Swift Testing and XCTests
failed.

### Modifications:

Modify the `swift test` command such that if Swift Testing is enabled,
it will keep track of whether any XCTests failed before running Swift
Testing, and if any did, emit a note at the very end (after Swift
Testing finishes) indicating that to the user.

### Result:

When relevant, the new note described above will be included in the
console output.

Author: stmontgomery

Sources/Commands/SwiftTestCommand.swift

@@ -279,6 +279,13 @@ public struct SwiftTestCommand: AsyncSwiftCommand {
     @OptionGroup()
     var options: TestCommandOptions
 
+    /// The text of a note emitted after Swift Testing tests finish running if
+    /// at least one XCTest has failed, to inform the user.
+    ///
+    /// - Note: This is exposed as a property so it can be referenced by an
+    ///     accompanying test as well as the implementation.
+    public static let xctestFailedNote = "Note: One or more XCTests failed, see logging above for details."
+
     private func run(_ swiftCommandState: SwiftCommandState, buildParameters: BuildParameters, testProducts: [BuiltTestProduct]) async throws {
         // Remove test output from prior runs and validate priors.
         if self.options.enableExperimentalTestOutput && buildParameters.triple.supportsTestSummary {
@@ -359,6 +366,9 @@ public struct SwiftTestCommand: AsyncSwiftCommand {
 
         // Run Swift Testing (parallel or not, it has a single entry point.)
         if options.testLibraryOptions.isEnabled(.swiftTesting, swiftCommandState: swiftCommandState) {
+            // Determine whether any XCTest runs performed above failed, before Swift Testing runs.
+            let anyXCTestFailed = results.reduce() == .failure
+
             lazy var testEntryPointPath = testProducts.lazy.compactMap(\.testEntryPointPath).first
             if options.testLibraryOptions.isExplicitlyEnabled(.swiftTesting, swiftCommandState: swiftCommandState) || testEntryPointPath == nil {
                 results.append(
@@ -377,6 +387,15 @@ public struct SwiftTestCommand: AsyncSwiftCommand {
                     debug: "Skipping automatic Swift Testing invocation because a test entry point path is present: \(testEntryPointPath)"
                 )
             }
+
+            // After running Swift Testing tests, if we determined that any XCTests failed earlier,
+            // emit a message informing the user so they aren't misled and know to look elsewhere for
+            // those details.
+            if anyXCTestFailed {
+                // In theory this could, or should, use `observabilityScope.print(_:verbose:)`,
+                // but that causes tests which check for this output to fail in CI.
+                print(Self.xctestFailedNote)
+            }
         }
 
         switch results.reduce() {

Tests/CommandsTests/TestCommandTests.swift

@@ -734,6 +734,110 @@ struct TestCommandTests {
         }
     }
 
+    /// An argument to the test function `noteXCTestFailures()`.
+    struct XCTestFailureNoteTestArgument: CustomStringConvertible {
+        /// The relative path to a test fixture in this project.
+        var fixturePath: String
+
+        /// The setting representing whether XCTest should be enabled or disabled
+        /// for the test command, if any. When the value of this property is `nil`,
+        /// no flag will be passed. If non-`nil`, either `--enable-xctest` or
+        /// `--disable-xctest` will be passed representing the value. The default
+        /// value is `nil`, meaning no flag will be passed but the command defaults
+        /// to having XCTest enabled.
+        var isXCTestEnabled: Bool? = nil
+
+        /// The setting representing whether Swift Testing should be enabled or disabled
+        /// for the test command, if any. When the value of this property is `nil`,
+        /// no flag will be passed. If non-`nil`, either `--enable-swift-testing` or
+        /// `--disable-swift-testing` will be passed representing the value. The default
+        /// value is `nil`, meaning no flag will be passed but the command defaults
+        /// to having Swift Testing enabled.
+        var isSwiftTestingEnabled: Bool? = nil
+
+        /// Whether the test command output is expected to include the note.
+        var expectNote: Bool
+
+        var description: String {
+            var description = "fixture: '\((fixturePath as NSString).lastPathComponent)'"
+            if let isXCTestEnabled {
+                description.append(", XCTest enabled: \(isXCTestEnabled)")
+            }
+            if let isSwiftTestingEnabled {
+                description.append(", Swift Testing enabled: \(isSwiftTestingEnabled)")
+            }
+            description.append(", expectNote: \(expectNote)")
+            return description
+        }
+    }
+
+    /// Test whether a note is emitted to stdout indicating that XCTests failed
+    /// after Swift Testing tests finish running.
+    @Test(
+        .tags(
+            .Feature.TargetType.Test,
+            .Feature.CommandLineArguments.TestEnableXCTest,
+            .Feature.CommandLineArguments.TestDisableXCTest,
+            .Feature.CommandLineArguments.TestEnableSwiftTesting,
+            .Feature.CommandLineArguments.TestDisableSwiftTesting,
+        ),
+        arguments: [
+            .init(
+                fixturePath: "Miscellaneous/TestDiscovery/Simple",
+                expectNote: false,
+            ),
+            .init(
+                fixturePath: "Miscellaneous/TestSingleFailureXCTest",
+                expectNote: true,
+            ),
+            .init(
+                fixturePath: "Miscellaneous/TestSingleFailureSwiftTesting",
+                expectNote: false,
+            ),
+            .init(
+                fixturePath: "Miscellaneous/TestSingleFailureXCTest",
+                isXCTestEnabled: false,
+                expectNote: false,
+            ),
+            .init(
+                fixturePath: "Miscellaneous/TestSingleFailureXCTest",
+                isSwiftTestingEnabled: false,
+                expectNote: false,
+            ),
+            .init(
+                fixturePath: "Miscellaneous/TestSingleFailureXCTest",
+                isXCTestEnabled: false,
+                isSwiftTestingEnabled: false,
+                expectNote: false,
+            ),
+        ] as [XCTestFailureNoteTestArgument]
+    )
+    func noteXCTestFailures(noteArgument arg: XCTestFailureNoteTestArgument) async throws {
+        try await fixture(name: arg.fixturePath) { fixturePath in
+            var args: [String] = []
+
+            switch arg.isXCTestEnabled {
+            case .none: break
+            case .some(true): args.append("--enable-xctest")
+            case .some(false): args.append("--disable-xctest")
+            }
+
+            switch arg.isSwiftTestingEnabled {
+            case .none: break
+            case .some(true): args.append("--enable-swift-testing")
+            case .some(false): args.append("--disable-swift-testing")
+            }
+
+            let (stdout, stderr) = try await execute(
+                args,
+                packagePath: fixturePath,
+                buildSystem: .native,
+                throwIfCommandFails: false,
+            )
+            #expect(stdout.contains(SwiftTestCommand.xctestFailedNote) == arg.expectNote, "stdout: \(stdout), stderr: \(stderr)")
+        }
+    }
+
     @Test(
          .tags(
             .Feature.TargetType.Executable,