[SER] Generalize long vector diagnostics to HitObject

- Generalize long vector diagnostics code to HitObjects.
- Diagnose unsupported use of HitObject in globals, entry params/returns
  and various other shader-kind-specific contexts.
- Create HitObject variants from the invalid-longvec-decls*.hlsl tests
  to make sure all cases are covered.

Specification: https://github.com/microsoft/hlsl-specs/blob/main/proposals/0027-shader-execution-reordering.md

Bug: #7234 [SER] Diagnose and validate illegal use of HitObject in unsupported contexts

Author: simoll

tools/clang/include/clang/AST/DeclCXX.h

@@ -469,6 +469,10 @@ class CXXRecordDecl : public RecordDecl {
     ///  class containing an HLSL vector longer than 4 elements.
     bool HasHLSLLongVector : 1;
 
+    /// \brief Whether this class contains at least one member or base
+    ///  class containing an HLSL HitObject.
+    bool HasHLSLHitObject : 1;
+
     /// \brief The number of base class specifiers in Bases.
     unsigned NumBases;
 
@@ -1029,6 +1033,11 @@ class CXXRecordDecl : public RecordDecl {
   /// \brief Set that this class contains an HLSL long vector of over 4 elements
   bool setHasHLSLLongVector() { return data().HasHLSLLongVector = true; }
 
+  // \brief Determine whether this class contains an HLSL HitObject.
+  bool hasHLSLHitObject() { return data().HasHLSLHitObject; }
+  /// \brief Set that this class contains an HLSL HitObject.
+  bool setHasHLSLHitObject() { return data().HasHLSLHitObject = true; }
+
   /// \brief Determine whether this class describes a lambda function object.
   bool isLambda() const {
     // An update record can't turn a non-lambda into a lambda.

tools/clang/include/clang/Basic/DiagnosticSemaKinds.td

@@ -7878,14 +7878,14 @@ def err_hlsl_load_from_mesh_out_arrays: Error<
    "output arrays of a mesh shader can not be read from">;
 def err_hlsl_out_indices_array_incorrect_access: Error<
    "a vector in out indices array must be accessed as a whole">;
-def err_hlsl_unsupported_long_vector
-    : Error<"vectors of over 4 elements in "
+def err_hlsl_unsupported_type
+    : Error<"%select{vectors of over 4 elements|HitObjects}0 in "
     "%select{ConstantBuffers or TextureBuffers|"
     "tessellation patches|geometry streams|node records|"
     "cbuffers or tbuffers|user-defined struct parameter|"
     "entry function parameters|entry function return type|"
     "patch constant function parameters|patch constant function return type|"
-    "payload parameters|attributes}0 are not supported">;
+    "payload parameters|attributes}1 are not supported">;
 def err_hlsl_logical_binop_scalar : Error<
    "operands for short-circuiting logical binary operator must be scalar, for non-scalar types use '%select{and|or}0'">;
 def err_hlsl_ternary_scalar : Error<

tools/clang/include/clang/Sema/SemaHLSL.h

@@ -129,6 +129,14 @@ unsigned CaculateInitListArraySizeForHLSL(clang::Sema *sema,
                                           const clang::QualType EltTy);
 
 bool ContainsLongVector(clang::QualType);
+bool ContainsHitObject(clang::QualType);
+
+/// \brief Determine if the given type contains a long vector or a hit object.
+/// \param QT The type to check.
+/// \param DiagTypeIdx The index of the type in the diagnostic message.
+/// \returns True if the type contains a long vector or a hit object, false
+/// otherwise.
+bool ContainsLongVecOrHitObject(clang::QualType QT, unsigned &DiagTypeIdx);
 
 bool IsConversionToLessOrEqualElements(clang::Sema *self,
                                        const clang::ExprResult &sourceExpr,

tools/clang/lib/AST/DeclCXX.cpp

@@ -72,8 +72,9 @@ CXXRecordDecl::DefinitionData::DefinitionData(CXXRecordDecl *D)
       ImplicitCopyAssignmentHasConstParam(true),
       HasDeclaredCopyConstructorWithConstParam(false),
       HasDeclaredCopyAssignmentWithConstParam(false), IsLambda(false),
-      IsParsingBaseSpecifiers(false), HasHLSLLongVector(false), NumBases(0),
-      NumVBases(0), Bases(), VBases(), Definition(D), FirstFriend() {}
+      IsParsingBaseSpecifiers(false), HasHLSLLongVector(false),
+      HasHLSLHitObject(false), NumBases(0), NumVBases(0), Bases(), VBases(),
+      Definition(D), FirstFriend() {}
 // HLSL Change End - Add HasLongVector and clang-format
 
 CXXBaseSpecifier *CXXRecordDecl::DefinitionData::getBasesSlowCase() const {
@@ -206,6 +207,8 @@ CXXRecordDecl::setBases(CXXBaseSpecifier const * const *Bases,
     // HLSL Change Begin - Propagate presence of long vector to child classes.
     if (BaseClassDecl->hasHLSLLongVector())
       data().HasHLSLLongVector = true;
+    if (BaseClassDecl->hasHLSLHitObject())
+      data().HasHLSLHitObject = true;
     // HLSL Change End
 
     // Record if this base is the first non-literal field or base.
@@ -393,6 +396,8 @@ void CXXRecordDecl::addedClassSubobject(CXXRecordDecl *Subobj) {
   // HLSL Change Begin - Propagate presence of long vector to child classes.
   if (Subobj->hasHLSLLongVector())
     data().HasHLSLLongVector = true;
+  if (Subobj->hasHLSLHitObject())
+    data().HasHLSLHitObject = true;
   // HLSL Change End
 }
 

tools/clang/lib/AST/HlslTypes.cpp

@@ -25,6 +25,7 @@
 #include "clang/AST/DeclTemplate.h"
 #include "clang/AST/Type.h"
 #include "clang/Sema/AttributeList.h" // conceptually ParsedAttributes
+#include "clang/Sema/SemaHLSL.h"
 #include "llvm/ADT/StringSwitch.h"
 
 using namespace clang;
@@ -124,6 +125,8 @@ bool IsHLSLNumericUserDefinedType(clang::QualType type) {
 // which can't be annotated. But includes UDTs of trivially copyable data and
 // the builtin trivially copyable raytracing structs.
 bool IsHLSLCopyableAnnotatableRecord(clang::QualType QT) {
+  if (ContainsHitObject(QT))
+    return false;
   return IsHLSLNumericUserDefinedType(QT) ||
          IsHLSLBuiltinRayAttributeStruct(QT);
 }

tools/clang/lib/Sema/SemaDXR.cpp

@@ -835,9 +835,11 @@ void DiagnoseBuiltinCallWithPayload(Sema &S, const VarDecl *Payload,
   }
 
   if (ContainsLongVector(Payload->getType())) {
+    // No need to check for HitObject, checked with payload diagnosis.
+    const unsigned LongVectorIdx = 0;
     const unsigned PayloadParametersIdx = 10;
-    S.Diag(Payload->getLocation(), diag::err_hlsl_unsupported_long_vector)
-        << PayloadParametersIdx;
+    S.Diag(Payload->getLocation(), diag::err_hlsl_unsupported_type)
+        << LongVectorIdx << PayloadParametersIdx;
     return;
   }
 

tools/clang/lib/Sema/SemaDecl.cpp

@@ -13247,10 +13247,16 @@ void Sema::ActOnFields(Scope *S, SourceLocation RecLoc, Decl *EnclosingDecl,
   SmallVector<FieldDecl*, 32> RecFields;
 
   bool ARCErrReported = false;
+  bool HasHitObject = false;
   for (ArrayRef<Decl *>::iterator i = Fields.begin(), end = Fields.end();
        i != end; ++i) {
     FieldDecl *FD = cast<FieldDecl>(*i);
 
+    // HLSL Change Begin - set HitObject bit for fields
+    if (!HasHitObject && hlsl::ContainsHitObject(FD->getType()))
+      HasHitObject = true;
+    // HLSL Change End
+
     // Get the type for the field.
     const Type *FDTy = FD->getType().getTypePtr();
 
@@ -13439,7 +13445,12 @@ void Sema::ActOnFields(Scope *S, SourceLocation RecLoc, Decl *EnclosingDecl,
                I = CXXRecord->conversion_begin(),
                E = CXXRecord->conversion_end(); I != E; ++I)
           I.setAccess((*I)->getAccess());
-        
+
+        // HLSL Change Begin - set HitObject bit for fields
+        if (HasHitObject)
+          CXXRecord->setHasHLSLHitObject();
+        // HLSL Change End
+
         if (!CXXRecord->isDependentType()) {
           if (CXXRecord->hasUserDeclaredDestructor()) {
             // Adjust user-defined destructor exception spec.

tools/clang/lib/Sema/SemaHLSL.cpp

@@ -5533,10 +5533,11 @@ class HLSLExternalSource : public ExternalSemaSource {
         m_sema->RequireCompleteType(argSrcLoc, argType,
                                     diag::err_typecheck_decl_incomplete_type);
 
-        if (ContainsLongVector(argType)) {
+        unsigned DiagTypeIdx = 0;
+        if (hlsl::ContainsLongVecOrHitObject(argType, DiagTypeIdx)) {
           const unsigned ConstantBuffersOrTextureBuffersIdx = 0;
-          m_sema->Diag(argSrcLoc, diag::err_hlsl_unsupported_long_vector)
-              << ConstantBuffersOrTextureBuffersIdx;
+          m_sema->Diag(argSrcLoc, diag::err_hlsl_unsupported_type)
+              << DiagTypeIdx << ConstantBuffersOrTextureBuffersIdx;
           return true;
         }
       }
@@ -5641,11 +5642,11 @@ class HLSLExternalSource : public ExternalSemaSource {
       CXXRecordDecl *Decl = arg.getAsType()->getAsCXXRecordDecl();
       if (Decl && !Decl->isCompleteDefinition())
         return true;
-      if (ContainsLongVector(arg.getAsType())) {
+      unsigned DiagTypeIdx = 0;
+      if (ContainsLongVecOrHitObject(arg.getAsType(), DiagTypeIdx)) {
         const unsigned TessellationPatchesIDx = 1;
-        m_sema->Diag(argLoc.getLocation(),
-                     diag::err_hlsl_unsupported_long_vector)
-            << TessellationPatchesIDx;
+        m_sema->Diag(argLoc.getLocation(), diag::err_hlsl_unsupported_type)
+            << DiagTypeIdx << TessellationPatchesIDx;
         return true;
       }
     } else if (Template->getTemplatedDecl()->hasAttr<HLSLStreamOutputAttr>()) {
@@ -5660,11 +5661,11 @@ class HLSLExternalSource : public ExternalSemaSource {
       CXXRecordDecl *Decl = arg.getAsType()->getAsCXXRecordDecl();
       if (Decl && !Decl->isCompleteDefinition())
         return true;
-      if (ContainsLongVector(arg.getAsType())) {
+      unsigned DiagTypeIdx = 0;
+      if (ContainsLongVecOrHitObject(arg.getAsType(), DiagTypeIdx)) {
         const unsigned GeometryStreamsIdx = 2;
-        m_sema->Diag(argLoc.getLocation(),
-                     diag::err_hlsl_unsupported_long_vector)
-            << GeometryStreamsIdx;
+        m_sema->Diag(argLoc.getLocation(), diag::err_hlsl_unsupported_type)
+            << DiagTypeIdx << GeometryStreamsIdx;
         return true;
       }
     }
@@ -10779,8 +10780,11 @@ bool DiagnoseIntersectionAttributes(Sema &S, SourceLocation Loc, QualType Ty) {
   }
 
   if (ContainsLongVector(Ty)) {
+    // No need to check for HitObject, checked with attributes diagnosis.
+    const unsigned LongVectorIdx = 0;
     const unsigned AttributesIdx = 11;
-    S.Diag(Loc, diag::err_hlsl_unsupported_long_vector) << AttributesIdx;
+    S.Diag(Loc, diag::err_hlsl_unsupported_type)
+        << LongVectorIdx << AttributesIdx;
     return false;
   }
   return true;
@@ -12119,9 +12123,10 @@ bool hlsl::DiagnoseNodeStructArgument(Sema *self, TemplateArgumentLoc ArgLoc,
   switch (shapeKind) {
   case AR_TOBJ_VECTOR:
     if (GetHLSLVecSize(ArgTy) > DXIL::kDefaultMaxVectorLength) {
+      const unsigned LongVectorIdx = 0;
       const unsigned NodeRecordsIdx = 3;
-      self->Diag(ArgLoc.getLocation(), diag::err_hlsl_unsupported_long_vector)
-          << NodeRecordsIdx;
+      self->Diag(ArgLoc.getLocation(), diag::err_hlsl_unsupported_type)
+          << LongVectorIdx << NodeRecordsIdx;
       Empty = false;
       return false;
     }
@@ -12591,6 +12596,24 @@ bool hlsl::ShouldSkipNRVO(clang::Sema &sema, clang::QualType returnType,
   return false;
 }
 
+bool hlsl::ContainsHitObject(QualType QT) {
+  if (QT.isNull() || QT->isDependentType())
+    return false;
+
+  while (const ArrayType *Arr = QT->getAsArrayTypeUnsafe())
+    QT = Arr->getElementType();
+
+  if (IsHLSLHitObjectType(QT))
+    return true;
+
+  if (CXXRecordDecl *Decl = QT->getAsCXXRecordDecl()) {
+    if (!Decl->isCompleteDefinition())
+      return false;
+    return Decl->hasHLSLHitObject();
+  }
+  return false;
+}
+
 bool hlsl::ContainsLongVector(QualType QT) {
   if (QT.isNull() || QT->isDependentType())
     return false;
@@ -12606,6 +12629,19 @@ bool hlsl::ContainsLongVector(QualType QT) {
   return false;
 }
 
+bool hlsl::ContainsLongVecOrHitObject(clang::QualType QT,
+                                      unsigned &DiagTypeIdx) {
+  if (ContainsHitObject(QT)) {
+    DiagTypeIdx = 1;
+    return true;
+  }
+  if (ContainsLongVector(QT)) {
+    DiagTypeIdx = 0;
+    return true;
+  }
+  return false;
+}
+
 bool hlsl::IsConversionToLessOrEqualElements(
     clang::Sema *self, const clang::ExprResult &sourceExpr,
     const clang::QualType &targetType, bool explicitConversion) {
@@ -15277,18 +15313,27 @@ bool Sema::DiagnoseHLSLDecl(Declarator &D, DeclContext *DC, Expr *BitWidth,
   }
 
   // Disallow long vecs from $Global cbuffers.
-  if (isGlobal && !isStatic && !isGroupShared && !IS_BASIC_OBJECT(basicKind)) {
+  const unsigned CbuffersOrTbuffersIdx = 4;
+  if (isGlobal && !isStatic) {
     // Suppress actual emitting of errors for incompletable types here
     // They are redundant to those produced in ActOnUninitializedDecl.
     struct SilentDiagnoser : public TypeDiagnoser {
       SilentDiagnoser() : TypeDiagnoser(true) {}
       virtual void diagnose(Sema &S, SourceLocation Loc, QualType T) {}
     } SD;
     RequireCompleteType(D.getLocStart(), qt, SD);
-    if (ContainsLongVector(qt)) {
-      unsigned CbuffersOrTbuffersIdx = 4;
-      Diag(D.getLocStart(), diag::err_hlsl_unsupported_long_vector)
-          << CbuffersOrTbuffersIdx;
+    unsigned DiagTypeIdx = 0;
+    bool HasError = false;
+    if (!isGroupShared && !IS_BASIC_OBJECT(basicKind) && ContainsLongVector(qt))
+      HasError = true;
+    else if (ContainsHitObject(qt)) {
+      const unsigned HitObjectTypeIdx = 1;
+      DiagTypeIdx = HitObjectTypeIdx;
+      HasError = true;
+    }
+    if (HasError) {
+      Diag(D.getLocStart(), diag::err_hlsl_unsupported_type)
+          << DiagTypeIdx << CbuffersOrTbuffersIdx;
       result = false;
     }
   }
@@ -16195,10 +16240,11 @@ static bool isRelatedDeclMarkedNointerpolation(Expr *E) {
 
 // Verify that user-defined intrinsic struct args contain no long vectors
 static bool CheckUDTIntrinsicArg(Sema *S, Expr *Arg) {
-  if (ContainsLongVector(Arg->getType())) {
+  unsigned DiagTypeIdx = 0;
+  if (ContainsLongVecOrHitObject(Arg->getType(), DiagTypeIdx)) {
     const unsigned UserDefinedStructParameterIdx = 5;
-    S->Diag(Arg->getExprLoc(), diag::err_hlsl_unsupported_long_vector)
-        << UserDefinedStructParameterIdx;
+    S->Diag(Arg->getExprLoc(), diag::err_hlsl_unsupported_type)
+        << DiagTypeIdx << UserDefinedStructParameterIdx;
     return true;
   }
   return false;
@@ -16938,17 +16984,19 @@ void DiagnoseEntry(Sema &S, FunctionDecl *FD) {
   // Would be nice to check for resources here as they crash the compiler now.
   // See issue #7186.
   for (const auto *param : FD->params()) {
-    if (ContainsLongVector(param->getType())) {
+    unsigned DiagTypeIdx = 0;
+    if (ContainsLongVecOrHitObject(param->getType(), DiagTypeIdx)) {
       const unsigned EntryFunctionParametersIdx = 6;
-      S.Diag(param->getLocation(), diag::err_hlsl_unsupported_long_vector)
-          << EntryFunctionParametersIdx;
+      S.Diag(param->getLocation(), diag::err_hlsl_unsupported_type)
+          << DiagTypeIdx << EntryFunctionParametersIdx;
     }
   }
 
-  if (ContainsLongVector(FD->getReturnType())) {
+  unsigned DiagTypeIdx = 0;
+  if (ContainsLongVecOrHitObject(FD->getReturnType(), DiagTypeIdx)) {
     const unsigned EntryFunctionReturnIdx = 7;
-    S.Diag(FD->getLocation(), diag::err_hlsl_unsupported_long_vector)
-        << EntryFunctionReturnIdx;
+    S.Diag(FD->getLocation(), diag::err_hlsl_unsupported_type)
+        << DiagTypeIdx << EntryFunctionReturnIdx;
   }
 
   DXIL::ShaderKind Stage =

tools/clang/lib/Sema/SemaHLSLDiagnoseTU.cpp

@@ -709,19 +709,20 @@ void hlsl::DiagnoseTranslationUnit(clang::Sema *self) {
               << hullPatchCount.value();
         }
       }
-      for (const auto *param : pPatchFnDecl->params())
-        if (ContainsLongVector(param->getType())) {
-          const unsigned PatchConstantFunctionParametersIdx = 8;
-          self->Diag(param->getLocation(),
-                     diag::err_hlsl_unsupported_long_vector)
-              << PatchConstantFunctionParametersIdx;
-        }
+      for (const auto *param : pPatchFnDecl->params()) {
+        unsigned TypeDiagIdx = 0;
+        const unsigned PatchConstantFunctionParametersIdx = 8;
+        if (ContainsLongVecOrHitObject(param->getType(), TypeDiagIdx))
+          self->Diag(param->getLocation(), diag::err_hlsl_unsupported_type)
+              << TypeDiagIdx << PatchConstantFunctionParametersIdx;
+      }
 
-      if (ContainsLongVector(pPatchFnDecl->getReturnType())) {
+      unsigned TypeDiagIdx = 0;
+      if (ContainsLongVecOrHitObject(pPatchFnDecl->getReturnType(),
+                                     TypeDiagIdx)) {
         const unsigned PatchConstantFunctionReturnIdx = 9;
-        self->Diag(pPatchFnDecl->getLocation(),
-                   diag::err_hlsl_unsupported_long_vector)
-            << PatchConstantFunctionReturnIdx;
+        self->Diag(pPatchFnDecl->getLocation(), diag::err_hlsl_unsupported_type)
+            << TypeDiagIdx << PatchConstantFunctionReturnIdx;
       }
     }
     DXIL::ShaderKind EntrySK = shaderModel->GetKind();

tools/clang/test/SemaHLSL/hlsl/types/invalid-hitobject-decls-hs.hlsl

@@ -0,0 +1,24 @@
+// RUN: %dxc -DTYPE=float -T hs_6_9 -verify %s
+
+struct HsConstantData {
+  float Edges[3] : SV_TessFactor;
+  dx::HitObject hit;
+};
+
+struct LongVec {
+  float4 f;
+  dx::HitObject hit;
+};
+
+HsConstantData PatchConstantFunction( // expected-error{{HitObjects in patch constant function return type are not supported}}
+				      dx::HitObject hit : V, // expected-error{{HitObjects in patch constant function parameters are not supported}}
+				      LongVec lv : L) { // expected-error{{HitObjects in patch constant function parameters are not supported}}
+  return (HsConstantData)0;
+}
+
+[domain("tri")]
+[outputtopology("triangle_cw")]
+[outputcontrolpoints(32)]
+[patchconstantfunc("PatchConstantFunction")]
+void main() {
+}