Correctly block @Suite and @Test on XCTest.XCTest and XCTestSuite. (#1456)

This PR ensures that we diagnose if `@Suite` or `@Test` is applied to a
class (or method on such class) that subclasses
[`XCTest.XCTest`](https://developer.apple.com/documentation/xctest/xctest)
or
[`XCTestSuite`](https://developer.apple.com/documentation/xctest/xctestsuite),
not just
[`XCTestCase`](https://developer.apple.com/documentation/xctest/xctestcase).

This change also suppresses the emitted call to the helper function
`__invokeXCTest[Case]Method()` if the suite type is _known_ not to
subclass `XCTest.XCTest` (e.g. because it's a `struct`.) This slightly
improves the performance of non-class suite types (although probably not
measurably so).

Finally, this change improves the emitted diagnostic when `@Test` is
used on a method in an `XCTest.XCTest` subclass such that it explicitly
states the reason why the attribute can't be used:

> 🛑 Attribute 'Test' cannot be applied to a function within class 'C'
because it is a subclass of 'XCTest', 'XCTestCase', or 'XCTestSuite'

### Checklist:

- [x] Code and documentation should follow the style of the [Style
Guide](https://github.com/apple/swift-testing/blob/main/Documentation/StyleGuide.md).
- [x] If public symbols are renamed or modified, DocC references should
be updated.

Author: grynspan

Sources/Testing/ExitTests/ExitTest.swift

@@ -812,7 +812,7 @@ extension ExitTest {
         ?? parentArguments.dropFirst().last
       // If the running executable appears to be the XCTest runner executable in
       // Xcode, figure out the path to the running XCTest bundle. If we can find
-      // it, then we can re-run the host XCTestCase instance.
+      // it, then we can spawn a child process of it.
       var isHostedByXCTest = false
       if let executablePath = try? childProcessExecutablePath.get() {
         executablePath.withCString { childProcessExecutablePath in
@@ -825,12 +825,9 @@ extension ExitTest {
       }
 
       if isHostedByXCTest, let xctestTargetPath {
-        // HACK: if the current test is being run from within Xcode, we don't
-        // always know we're being hosted by an XCTestCase instance. In cases
-        // where we don't, but the XCTest environment variable specifying the
-        // test bundle is set, assume we _are_ being hosted and specify a
-        // blank test identifier ("/") to force the xctest command-line tool
-        // to run.
+        // HACK: specify a blank test identifier ("/") to force the xctest
+        // command-line tool to run. Xcode will then (eventually) invoke the
+        // testing library which will then start the exit test.
         result += ["-XCTest", "/", xctestTargetPath]
       }
 

Sources/Testing/Test+Macro.swift

@@ -568,54 +568,51 @@ public var __defaultSynchronousIsolationContext: (any Actor)? {
   Configuration.current?.defaultSynchronousIsolationContext ?? #isolation
 }
 
-/// Run a test function as an `XCTestCase`-compatible method.
+/// Run a test function as an XCTest-compatible method.
 ///
 /// This overload is used for types that are not classes. It always returns
 /// `false`.
 ///
 /// - Warning: This function is used to implement the `@Test` macro. Do not call
 ///   it directly.
-@inlinable public func __invokeXCTestCaseMethod<T>(
+@inlinable public func __invokeXCTestMethod<T>(
   _ selector: __XCTestCompatibleSelector?,
   onInstanceOf type: T.Type,
   sourceLocation: SourceLocation
 ) async throws -> Bool where T: ~Copyable {
   false
 }
 
-// TODO: implement a hook in XCTest that __invokeXCTestCaseMethod() can call to
-// run an XCTestCase nested in the current @Test function.
-
-/// The `XCTestCase` Objective-C class.
-let xcTestCaseClass: AnyClass? = {
+/// The `XCTest.XCTest` Objective-C class.
+let xcTestClass: AnyClass? = {
 #if _runtime(_ObjC)
-  objc_getClass("XCTestCase") as? AnyClass
+  objc_getClass("XCTest") as? AnyClass
 #else
-  _typeByName("6XCTest0A4CaseC") as? AnyClass // _mangledTypeName(XCTest.XCTestCase.self)
+  _typeByName("6XCTestAAC") as? AnyClass // _mangledTypeName(XCTest.XCTest.self)
 #endif
 }()
 
-/// Run a test function as an `XCTestCase`-compatible method.
+/// Run a test function as an XCTest-compatible method.
 ///
 /// This overload is used for types that are classes. If the type is not a
-/// subclass of `XCTestCase`, or if XCTest is not loaded in the current process,
-/// this function returns immediately.
+/// subclass of `XCTest.XCTest`, or if XCTest is not loaded in the current
+/// process, this function returns immediately.
 ///
 /// - Warning: This function is used to implement the `@Test` macro. Do not call
 ///   it directly.
-public func __invokeXCTestCaseMethod<T>(
+public func __invokeXCTestMethod<T>(
   _ selector: __XCTestCompatibleSelector?,
-  onInstanceOf xcTestCaseSubclass: T.Type,
+  onInstanceOf xcTestSubclass: T.Type,
   sourceLocation: SourceLocation
 ) async throws -> Bool where T: AnyObject {
   // All classes will end up on this code path, so only record an issue if it is
-  // really an XCTestCase subclass.
-  guard let xcTestCaseClass, isClass(xcTestCaseSubclass, subclassOf: xcTestCaseClass) else {
+  // really an XCTest.XCTest subclass.
+  guard let xcTestClass, isClass(xcTestSubclass, subclassOf: xcTestClass) else {
     return false
   }
   let issue = Issue(
     kind: .apiMisused,
-    comments: ["The @Test attribute cannot be applied to methods on a subclass of XCTestCase."],
+    comments: ["The 'Test' attribute cannot be applied to a method on a subclass of 'XCTest', 'XCTestCase', or 'XCTestSuite'."],
     sourceContext: .init(backtrace: nil, sourceLocation: sourceLocation)
   )
   issue.record()

Sources/TestingMacros/SuiteDeclarationMacro.swift

@@ -74,14 +74,14 @@ public struct SuiteDeclarationMacro: MemberMacro, PeerMacro, Sendable {
     diagnostics += diagnoseIssuesWithLexicalContext(context.lexicalContext, containing: declaration, attribute: suiteAttribute)
     diagnostics += diagnoseIssuesWithLexicalContext(declaration, containing: declaration, attribute: suiteAttribute)
 
-    // Suites inheriting from XCTestCase are not supported. This check is
+    // Suites inheriting from XCTest.XCTest are not supported. This check is
     // duplicated in TestDeclarationMacro but is not part of
     // diagnoseIssuesWithLexicalContext() because it doesn't need to recurse
     // across the entire lexical context list, just the innermost type
     // declaration.
-    if let declaration = declaration.asProtocol((any DeclGroupSyntax).self),
-       declaration.inherits(fromTypeNamed: "XCTestCase", inModuleNamed: "XCTest") {
-      diagnostics.append(.xcTestCaseNotSupported(declaration, whenUsing: suiteAttribute))
+    let inheritsFromXCTestClass = declarationInheritsFromXCTestClass(declaration)
+    if inheritsFromXCTestClass == true {
+      diagnostics.append(.xcTestSubclassNotSupported(declaration, whenUsing: suiteAttribute))
     }
 
     // @Suite cannot be applied to a type extension (although a type extension

Sources/TestingMacros/Support/DiagnosticMessage+Diagnosing.swift

@@ -281,3 +281,39 @@ func makeGenericGuardDecl(
   private static let \(genericGuardName): Void = ()
   """
 }
+
+// MARK: -
+
+/// Check whether or not the given declaration inherits from `XCTest.XCTest` or
+/// its known subclasses `XCTestCase` and `XCTestSuite`.
+///
+/// - Parameters:
+/// 	- decl: The declaration to examine.
+///
+/// - Returns: Whether or not `decl` inherits from `XCTest.XCTest`. If the
+///   result could not be determined from the available syntax, returns `nil`.
+func declarationInheritsFromXCTestClass(_ decl: some DeclSyntaxProtocol) -> Bool? {
+  switch decl.kind {
+  case .structDecl, .enumDecl:
+    // Value types can never inherit from XCTest.XCTest because it's a class, so
+    // we can confidently return `false` here.
+    return false
+
+  default:
+    if let decl = decl.asProtocol((any DeclGroupSyntax).self) {
+      let xctestClassNames = ["XCTest", "XCTestCase", "XCTestSuite"]
+      let inheritsFromXCTestClass = xctestClassNames.contains { className in
+        decl.inherits(fromTypeNamed: className, inModuleNamed: "XCTest")
+      }
+      if inheritsFromXCTestClass {
+        // We can plainly see the inheritance, so return `true`. Note we don't
+        // return `false` along this branch because we can't be sure it doesn't
+        // inherit via an intermediate class, typealias, etc.
+        return true
+      }
+    }
+  }
+
+  // We couldn't tell either way.
+  return nil
+}

Sources/TestingMacros/Support/DiagnosticMessage.swift

@@ -375,6 +375,8 @@ struct DiagnosticMessage: SwiftDiagnostics.DiagnosticMessage {
     }
     if escapableNonConformance != nil {
       message += " because its conformance to 'Escapable' has been suppressed"
+    } else if let decl = node.as(DeclSyntax.self), declarationInheritsFromXCTestClass(decl) == true {
+      message += " because it is a subclass of 'XCTest', 'XCTestCase', or 'XCTestSuite'"
     }
 
     return Self(syntax: syntax, message: message, severity: .error)
@@ -529,18 +531,18 @@ struct DiagnosticMessage: SwiftDiagnostics.DiagnosticMessage {
   }
 
   /// Create a diagnostic message stating that `@Test` or `@Suite` is
-  /// incompatible with `XCTestCase` and its subclasses.
+  /// incompatible with `XCTest.XCTest` and its subclasses.
   ///
   /// - Parameters:
   ///   - decl: The expression or declaration referring to the unsupported
   ///     XCTest symbol.
   ///   - attribute: The `@Test` or `@Suite` attribute.
   ///
   /// - Returns: A diagnostic message.
-  static func xcTestCaseNotSupported(_ decl: some SyntaxProtocol, whenUsing attribute: AttributeSyntax) -> Self {
+  static func xcTestSubclassNotSupported(_ decl: some SyntaxProtocol, whenUsing attribute: AttributeSyntax) -> Self {
     Self(
       syntax: Syntax(decl),
-      message: "Attribute \(_macroName(attribute)) cannot be applied to a subclass of 'XCTestCase'",
+      message: "Attribute \(_macroName(attribute)) cannot be applied to a subclass of 'XCTest', 'XCTestCase', or 'XCTestSuite'",
       severity: .error
     )
   }

Sources/TestingMacros/TestDeclarationMacro.swift

@@ -27,14 +27,15 @@ public struct TestDeclarationMacro: PeerMacro, Sendable {
     providingPeersOf declaration: some DeclSyntaxProtocol,
     in context: some MacroExpansionContext
   ) throws -> [DeclSyntax] {
-    guard _diagnoseIssues(with: declaration, testAttribute: node, in: context) else {
+    var inheritsFromXCTestClass: Bool?
+    guard _diagnoseIssues(with: declaration, testAttribute: node, inheritsFromXCTestClass: &inheritsFromXCTestClass, in: context) else {
       return []
     }
 
     let functionDecl = declaration.cast(FunctionDeclSyntax.self)
     let typeName = context.typeOfLexicalContext
 
-    return _createTestDecls(for: functionDecl, on: typeName, testAttribute: node, in: context)
+    return _createTestDecls(for: functionDecl, on: typeName, testAttribute: node, inheritsFromXCTestClass: inheritsFromXCTestClass, in: context)
   }
 
   public static var formatMode: FormatMode {
@@ -46,20 +47,26 @@ public struct TestDeclarationMacro: PeerMacro, Sendable {
   /// - Parameters:
   ///   - declaration: The function declaration to diagnose.
   ///   - testAttribute: The `@Test` attribute applied to `declaration`.
+  ///   - inheritsFromXCTestClass: On return, whether or not the type containing
+  ///     `declaration` (if any) is known to inherit from `XCTest.XCTest`.
   ///   - context: The macro context in which the expression is being parsed.
   ///
   /// - Returns: Whether or not macro expansion should continue (i.e. stopping
   ///   if a fatal error was diagnosed.)
   private static func _diagnoseIssues(
     with declaration: some DeclSyntaxProtocol,
     testAttribute: AttributeSyntax,
+    inheritsFromXCTestClass: inout Bool?,
     in context: some MacroExpansionContext
   ) -> Bool {
     var diagnostics = [DiagnosticMessage]()
     defer {
       context.diagnose(diagnostics)
     }
 
+    // Default to "we don't know".
+    inheritsFromXCTestClass = nil
+
     // The @Test attribute is only supported on function declarations.
     guard let function = declaration.as(FunctionDeclSyntax.self), !function.isOperator else {
       diagnostics.append(.attributeNotSupported(testAttribute, on: declaration))
@@ -70,14 +77,16 @@ public struct TestDeclarationMacro: PeerMacro, Sendable {
     let lexicalContext = context.lexicalContext
     diagnostics += diagnoseIssuesWithLexicalContext(lexicalContext, containing: declaration, attribute: testAttribute)
 
-    // Suites inheriting from XCTestCase are not supported. We are a bit
+    // Suites inheriting from XCTest.XCTest are not supported. We are a bit
     // conservative here in this check and only check the immediate context.
     // Presumably, if there's an intermediate lexical context that is *not* a
     // type declaration, then it must be a function or closure (disallowed
     // elsewhere) and thus the test function is not a member of any type.
-    if let containingTypeDecl = lexicalContext.first?.asProtocol((any DeclGroupSyntax).self),
-       containingTypeDecl.inherits(fromTypeNamed: "XCTestCase", inModuleNamed: "XCTest") {
-      diagnostics.append(.containingNodeUnsupported(containingTypeDecl, whenUsing: testAttribute, on: declaration))
+    if let containingTypeDecl = lexicalContext.first?.asProtocol((any DeclGroupSyntax).self) {
+      inheritsFromXCTestClass = declarationInheritsFromXCTestClass(containingTypeDecl)
+      if inheritsFromXCTestClass == true {
+        diagnostics.append(.containingNodeUnsupported(containingTypeDecl, whenUsing: testAttribute, on: declaration))
+      }
     }
 
     // Only one @Test attribute is supported.
@@ -291,7 +300,7 @@ public struct TestDeclarationMacro: PeerMacro, Sendable {
           let sourceLocationExpr = createSourceLocationExpr(of: functionDecl.name, context: context)
 
           thunkBody = """
-          if try await Testing.__invokeXCTestCaseMethod(\(selectorExpr), onInstanceOf: \(typeName).self, sourceLocation: \(sourceLocationExpr)) {
+          if try await Testing.__invokeXCTestMethod(\(selectorExpr), onInstanceOf: \(typeName).self, sourceLocation: \(sourceLocationExpr)) {
             return
           }
           \(thunkBody)
@@ -368,6 +377,8 @@ public struct TestDeclarationMacro: PeerMacro, Sendable {
   ///   - typeName: The name of the type of which `functionDecl` is a member, if
   ///     any.
   ///   - testAttribute: The `@Test` attribute applied to `declaration`.
+  ///   - inheritsFromXCTestClass: Whether or not the type containing
+  ///     `functionDecl` (if any) is known to inherit from `XCTest.XCTest`.
   ///   - context: The macro context in which the expression is being parsed.
   ///
   /// - Returns: An array of declarations providing runtime information about
@@ -376,6 +387,7 @@ public struct TestDeclarationMacro: PeerMacro, Sendable {
     for functionDecl: FunctionDeclSyntax,
     on typeName: TypeSyntax?,
     testAttribute: AttributeSyntax,
+    inheritsFromXCTestClass: Bool?,
     in context: some MacroExpansionContext
   ) -> [DeclSyntax] {
     var result = [DeclSyntax]()
@@ -401,7 +413,8 @@ public struct TestDeclarationMacro: PeerMacro, Sendable {
 
     // Generate a selector expression compatible with XCTest.
     var selectorExpr: ExprSyntax?
-    if let selector = functionDecl.xcTestCompatibleSelector {
+    if inheritsFromXCTestClass != false, // definitely does or maybe does
+       let selector = functionDecl.xcTestCompatibleSelector {
       let selectorLiteral = String(selector.tokens(viewMode: .fixedUp).lazy.flatMap(\.textWithoutBackticks))
       selectorExpr = "Testing.__xcTestCompatibleSelector(\(literal: selectorLiteral))"
     }

Tests/TestingMacrosTests/TestDeclarationMacroTests.swift

@@ -80,15 +80,31 @@ struct TestDeclarationMacroTests {
       "@_unavailableFromAsync @Suite actor A {}":
         "Attribute 'Suite' cannot be applied to this actor because it has been marked '@_unavailableFromAsync'",
 
-      // XCTestCase
+      // XCTest/XCTestCase/XCTestSuite
+      "@Suite final class C: XCTest {}":
+        "Attribute 'Suite' cannot be applied to a subclass of 'XCTest', 'XCTestCase', or 'XCTestSuite'",
       "@Suite final class C: XCTestCase {}":
-        "Attribute 'Suite' cannot be applied to a subclass of 'XCTestCase'",
+        "Attribute 'Suite' cannot be applied to a subclass of 'XCTest', 'XCTestCase', or 'XCTestSuite'",
+      "@Suite final class C: XCTestSuite {}":
+        "Attribute 'Suite' cannot be applied to a subclass of 'XCTest', 'XCTestCase', or 'XCTestSuite'",
+      "@Suite final class C: XCTest.XCTest {}":
+        "Attribute 'Suite' cannot be applied to a subclass of 'XCTest', 'XCTestCase', or 'XCTestSuite'",
       "@Suite final class C: XCTest.XCTestCase {}":
-        "Attribute 'Suite' cannot be applied to a subclass of 'XCTestCase'",
+        "Attribute 'Suite' cannot be applied to a subclass of 'XCTest', 'XCTestCase', or 'XCTestSuite'",
+      "@Suite final class C: XCTest.XCTestSuite {}":
+        "Attribute 'Suite' cannot be applied to a subclass of 'XCTest', 'XCTestCase', or 'XCTestSuite'",
+      "final class C: XCTest { @Test func f() {} }":
+        "Attribute 'Test' cannot be applied to a function within class 'C' because it is a subclass of 'XCTest', 'XCTestCase', or 'XCTestSuite'",
       "final class C: XCTestCase { @Test func f() {} }":
-        "Attribute 'Test' cannot be applied to a function within class 'C'",
+        "Attribute 'Test' cannot be applied to a function within class 'C' because it is a subclass of 'XCTest', 'XCTestCase', or 'XCTestSuite'",
+      "final class C: XCTestSuite { @Test func f() {} }":
+        "Attribute 'Test' cannot be applied to a function within class 'C' because it is a subclass of 'XCTest', 'XCTestCase', or 'XCTestSuite'",
+      "final class C: XCTest.XCTest { @Test func f() {} }":
+        "Attribute 'Test' cannot be applied to a function within class 'C' because it is a subclass of 'XCTest', 'XCTestCase', or 'XCTestSuite'",
       "final class C: XCTest.XCTestCase { @Test func f() {} }":
-        "Attribute 'Test' cannot be applied to a function within class 'C'",
+        "Attribute 'Test' cannot be applied to a function within class 'C' because it is a subclass of 'XCTest', 'XCTestCase', or 'XCTestSuite'",
+      "final class C: XCTest.XCTestSuite { @Test func f() {} }":
+        "Attribute 'Test' cannot be applied to a function within class 'C' because it is a subclass of 'XCTest', 'XCTestCase', or 'XCTestSuite'",
 
       // Unsupported inheritance
       "@Suite protocol P {}":
@@ -440,10 +456,10 @@ struct TestDeclarationMacroTests {
       ("@Test @available(*, noasync) func f() {}", nil, "__requiringTry"),
       ("@Test @_unavailableFromAsync func f() {}", nil, "__requiringTry"),
       ("@Test(arguments: []) func f(f: () -> String) {}", "(() -> String).self", nil),
-      ("struct S {\n\t@Test func testF() {} }", nil, "__invokeXCTestCaseMethod"),
-      ("struct S {\n\t@Test func testF() throws {} }", nil, "__invokeXCTestCaseMethod"),
-      ("struct S {\n\t@Test func testF() async {} }", nil, "__invokeXCTestCaseMethod"),
-      ("struct S {\n\t@Test func testF() async throws {} }", nil, "__invokeXCTestCaseMethod"),
+      ("class S {\n\t@Test func testF() {} }", nil, "__invokeXCTestMethod"),
+      ("class S {\n\t@Test func testF() throws {} }", nil, "__invokeXCTestMethod"),
+      ("class S {\n\t@Test func testF() async {} }", nil, "__invokeXCTestMethod"),
+      ("class S {\n\t@Test func testF() async throws {} }", nil, "__invokeXCTestMethod"),
       (
         """
         struct S {

Tests/TestingTests/NonCopyableSuiteTests.swift

@@ -16,7 +16,6 @@ struct NonCopyableTests: ~Copyable {
   @Test borrowing func borrowMe() {}
   @Test consuming func consumeMe() {}
   @Test mutating func mutateMe() {}
-  @Test borrowing func testNotAnXCTestCaseMethod() {}
 
   @Test borrowing func typeComparison() {
     let lhs = TypeInfo(describing: Self.self)
@@ -31,3 +30,7 @@ struct NonCopyableTests: ~Copyable {
     #expect(TypeInfo(describing: Self.self).mangledName != nil)
   }
 }
+
+extension NonCopyableTests {
+  @Test borrowing func testNotAnXCTestCaseMethod() {}
+}

Tests/TestingTests/ObjCInteropTests.swift

@@ -99,7 +99,7 @@ struct ObjCAndXCTestInteropTests {
         if case let .issueRecorded(issue) = event.kind,
            case .apiMisused = issue.kind,
            let comment = issue.comments.first,
-           comment == "The @Test attribute cannot be applied to methods on a subclass of XCTestCase." {
+           comment == "The 'Test' attribute cannot be applied to a method on a subclass of 'XCTest', 'XCTestCase', or 'XCTestSuite'." {
           issueRecorded()
         }
       }

Tests/TestingTests/TypeNameConflictTests.swift

@@ -37,12 +37,12 @@ fileprivate func __forwardNoAsync<R>(_ value: @autoclosure () throws -> R) throw
   Issue.record("Called wrong __forwardNoAsync()")
 }
 
-fileprivate func __invokeXCTestCaseMethod<T>(
+fileprivate func __invokeXCTestMethod<T>(
   _ selector: __XCTestCompatibleSelector?,
-  onInstanceOf xcTestCaseSubclass: T.Type,
+  onInstanceOf xcTestSubclass: T.Type,
   sourceLocation: SourceLocation
 ) {
-  Issue.record("Called wrong __invokeXCTestCaseMethod()")
+  Issue.record("Called wrong __invokeXCTestMethod()")
 }
 
 fileprivate func __xcTestCompatibleSelector(_ selector: String) -> __XCTestCompatibleSelector? {