[SER] Diagnose HitObject in unsupported declaration contexts (#7376)

- Generalize long vector diagnostics code to HitObjects.
- Diagnose unsupported use of HitObject in globals, entry params/returns
and various other shader-kind-specific contexts.
- Create HitObject variants from the invalid-longvec-decls*.hlsl tests
to make sure all cases are covered.

Specification:
https://github.com/microsoft/hlsl-specs/blob/main/proposals/0027-shader-execution-reordering.md

Closes #7234 [SER] Diagnose and validate illegal use of HitObject in
unsupported contexts (discussed offline)

Author: simoll

tools/clang/include/clang/AST/DeclCXX.h

@@ -465,10 +465,6 @@ class CXXRecordDecl : public RecordDecl {
     /// \brief Whether we are currently parsing base specifiers.
     bool IsParsingBaseSpecifiers : 1;
 
-    /// \brief Whether this class contains at least one member or base
-    ///  class containing an HLSL vector longer than 4 elements.
-    bool HasHLSLLongVector : 1;
-
     /// \brief The number of base class specifiers in Bases.
     unsigned NumBases;
 
@@ -1022,13 +1018,6 @@ class CXXRecordDecl : public RecordDecl {
     return data().NeedOverloadResolutionForDestructor;
   }
 
-  // HLSL Change add HLSL Long vector bit.
-  /// \brief Determine whether this class contains an HLSL long vector
-  /// of over 4 elements.
-  bool hasHLSLLongVector() { return data().HasHLSLLongVector; }
-  /// \brief Set that this class contains an HLSL long vector of over 4 elements
-  bool setHasHLSLLongVector() { return data().HasHLSLLongVector = true; }
-
   /// \brief Determine whether this class describes a lambda function object.
   bool isLambda() const {
     // An update record can't turn a non-lambda into a lambda.

tools/clang/include/clang/Basic/DiagnosticSemaKinds.td

@@ -7558,8 +7558,6 @@ def err_hlsl_missing_type_specifier : Error< // Patterened after err_missing_typ
   "HLSL requires a type specifier for all declarations">;
 def err_hlsl_multiple_concrete_bases : Error<
   "multiple concrete base types specified">;
-def err_hlsl_objectintemplateargument : Error<
-  "%0 is an object and cannot be used as a type parameter">;
 def err_hlsl_packoffset_requires_cbuffer : Error<
   "packoffset is only allowed in a constant buffer">;
 def warn_hlsl_packoffset_mix : Warning<
@@ -7886,6 +7884,15 @@ def err_hlsl_unsupported_long_vector
     "entry function parameters|entry function return type|"
     "patch constant function parameters|patch constant function return type|"
     "payload parameters|attributes}0 are not supported">;
+// First %select options must match err_hlsl_unsupported_long_vector (same index used)
+def err_hlsl_unsupported_object_context
+    : Error<"object %0 is not allowed in "
+    "%select{ConstantBuffers or TextureBuffers|"
+    "tessellation patches|geometry streams|node records|"
+    "cbuffers or tbuffers|user-defined struct parameter|"
+    "entry function parameters|entry function return type|"
+    "patch constant function parameters|patch constant function return type|"
+    "payload parameters|attributes|builtin template parameters|structured buffers|global variables|groupshared variables}1">;
 def err_hlsl_logical_binop_scalar : Error<
    "operands for short-circuiting logical binary operator must be scalar, for non-scalar types use '%select{and|or}0'">;
 def err_hlsl_ternary_scalar : Error<
@@ -7970,8 +7977,6 @@ def err_hlsl_too_many_node_inputs : Error<
    "Node shader '%0' may not have more than one input record">;
 def err_hlsl_node_record_type : Error<
    "%0 is not valid as a node record type - struct/class required">;
-def err_hlsl_node_record_object : Error<
-   "object %0 may not appear in a node record">;
 def err_hlsl_array_disallowed : Error<
    "%select{entry parameter|declaration}1 of type %0 may not be an array">;
 def err_hlsl_inputpatch_size: Error<

tools/clang/include/clang/Sema/SemaHLSL.h

@@ -59,6 +59,38 @@ bool DiagnoseNodeStructArgument(clang::Sema *self,
                                 clang::QualType ArgTy, bool &Empty,
                                 const clang::FieldDecl *FD = nullptr);
 
+// Keep this in sync with err_hlsl_unsupported_object in DiagnosticSemaKinds.td
+enum class TypeDiagContext {
+  // Indices that the type context is valid and no diagnostics should be emitted
+  // for this type category.
+  Valid = -1,
+  // Supported indices for both `err_hlsl_unsupported_object_context` and
+  // `err_hlsl_unsupported_long_vector`
+  ConstantBuffersOrTextureBuffers = 0,
+  TessellationPatches = 1,
+  GeometryStreams = 2,
+  NodeRecords = 3,
+  CBuffersOrTBuffers = 4,
+  UserDefinedStructParameter = 5,
+  EntryFunctionParameters = 6,
+  EntryFunctionReturnType = 7,
+  PatchConstantFunctionParameters = 8,
+  PatchConstantFunctionReturnType = 9,
+  PayloadParameters = 10,
+  Attributes = 11,
+  TypeParameter = 12,
+  LongVecDiagMaxSelectIndex = TypeParameter,
+  // Below only supported for `err_hlsl_diag_unsupported_object_context`
+  StructuredBuffers = 13,
+  GlobalVariables = 14,
+  GroupShared = 15,
+  DiagMaxSelectIndex = 15,
+};
+bool DiagnoseTypeElements(clang::Sema &S, clang::SourceLocation Loc,
+                          clang::QualType Ty, TypeDiagContext ObjDiagContext,
+                          TypeDiagContext LongVecDiagContext,
+                          const clang::FieldDecl *FD = nullptr);
+
 void DiagnoseControlFlowConditionForHLSL(clang::Sema *self,
                                          clang::Expr *condExpr,
                                          llvm::StringRef StmtName);

tools/clang/lib/AST/DeclCXX.cpp

@@ -72,8 +72,8 @@ CXXRecordDecl::DefinitionData::DefinitionData(CXXRecordDecl *D)
       ImplicitCopyAssignmentHasConstParam(true),
       HasDeclaredCopyConstructorWithConstParam(false),
       HasDeclaredCopyAssignmentWithConstParam(false), IsLambda(false),
-      IsParsingBaseSpecifiers(false), HasHLSLLongVector(false), NumBases(0),
-      NumVBases(0), Bases(), VBases(), Definition(D), FirstFriend() {}
+      IsParsingBaseSpecifiers(false), NumBases(0), NumVBases(0), Bases(),
+      VBases(), Definition(D), FirstFriend() {}
 // HLSL Change End - Add HasLongVector and clang-format
 
 CXXBaseSpecifier *CXXRecordDecl::DefinitionData::getBasesSlowCase() const {
@@ -203,11 +203,6 @@ CXXRecordDecl::setBases(CXXBaseSpecifier const * const *Bases,
     if (!BaseClassDecl->isStandardLayout())
       data().IsStandardLayout = false;
 
-    // HLSL Change Begin - Propagate presence of long vector to child classes.
-    if (BaseClassDecl->hasHLSLLongVector())
-      data().HasHLSLLongVector = true;
-    // HLSL Change End
-
     // Record if this base is the first non-literal field or base.
     if (!hasNonLiteralTypeFieldsOrBases() && !BaseType->isLiteralType(C))
       data().HasNonLiteralTypeFieldsOrBases = true;
@@ -389,11 +384,6 @@ void CXXRecordDecl::addedClassSubobject(CXXRecordDecl *Subobj) {
     data().NeedOverloadResolutionForMoveConstructor = true;
     data().NeedOverloadResolutionForDestructor = true;
   }
-
-  // HLSL Change Begin - Propagate presence of long vector to child classes.
-  if (Subobj->hasHLSLLongVector())
-    data().HasHLSLLongVector = true;
-  // HLSL Change End
 }
 
 /// Callback function for CXXRecordDecl::forallBases that acknowledges

tools/clang/lib/AST/HlslTypes.cpp

@@ -120,6 +120,13 @@ bool IsHLSLCopyableAnnotatableRecord(clang::QualType QT) {
       if (!IsHLSLNumericOrAggregateOfNumericType(Member->getType()))
         return false;
     }
+    if (auto *CXXRD = dyn_cast<CXXRecordDecl>(RD)) {
+      // Walk up the inheritance chain and check base class fields
+      for (const auto &Base : CXXRD->bases()) {
+        if (!IsHLSLCopyableAnnotatableRecord(Base.getType()))
+          return false;
+      }
+    }
     return true;
   }
   return false;

tools/clang/lib/Sema/SemaDXR.cpp

@@ -827,19 +827,16 @@ void DiagnoseBuiltinCallWithPayload(Sema &S, const VarDecl *Payload,
   }
 
   // Verify that the payload type is legal
-  if (!hlsl::IsHLSLCopyableAnnotatableRecord(Payload->getType())) {
+  if (!hlsl::IsHLSLCopyableAnnotatableRecord(Payload->getType()))
     S.Diag(Payload->getLocation(), diag::err_payload_attrs_must_be_udt)
         << /*payload|attributes|callable*/ 0 << /*parameter %2|type*/ 0
         << Payload;
-    return;
-  }
 
-  if (ContainsLongVector(Payload->getType())) {
-    const unsigned PayloadParametersIdx = 10;
-    S.Diag(Payload->getLocation(), diag::err_hlsl_unsupported_long_vector)
-        << PayloadParametersIdx;
+  // This will produce more details, but also catch disallowed long vectors
+  const TypeDiagContext DiagContext = TypeDiagContext::PayloadParameters;
+  if (DiagnoseTypeElements(S, Payload->getLocation(), Payload->getType(),
+                           DiagContext, DiagContext))
     return;
-  }
 
   CollectNonAccessableFields(PayloadType, CallerStage, {}, {},
                              NonWriteableFields, NonReadableFields);