Fix detection of builtin UDT DXR struct types (#7452)

Built-in DXR struct types RayDesc and
BuiltInTriangleIntersectionAttributes were not treated identically to
other UDT types.

This caused differences in intrinsic codegen when one of these types is
returned.

This change corrects this difference so these builtin structs are
handled in the same way as other UDTs.

Fixes #7450.

Author: tex3d

tools/clang/include/clang/AST/HlslTypes.h

@@ -494,7 +494,6 @@ DXIL::NodeIOKind GetNodeIOType(clang::QualType type);
 
 bool IsHLSLStructuredBufferType(clang::QualType type);
 bool IsHLSLNumericOrAggregateOfNumericType(clang::QualType type);
-bool IsHLSLNumericUserDefinedType(clang::QualType type);
 bool IsHLSLCopyableAnnotatableRecord(clang::QualType QT);
 bool IsHLSLBuiltinRayAttributeStruct(clang::QualType QT);
 bool IsHLSLAggregateType(clang::QualType type);

tools/clang/lib/AST/HlslTypes.cpp

@@ -103,31 +103,26 @@ bool IsHLSLNumericOrAggregateOfNumericType(clang::QualType type) {
          BuiltinTy->getKind() != BuiltinType::Kind::Char_S;
 }
 
-bool IsHLSLNumericUserDefinedType(clang::QualType type) {
-  const clang::Type *Ty = type.getCanonicalType().getTypePtr();
+// In some cases we need record types that are annotatable and trivially
+// copyable from outside the shader. This excludes resource types which may be
+// trivially copyable inside the shader, and builtin matrix and vector types
+// which can't be annotated. But includes UDTs of trivially copyable data and
+// the builtin trivially copyable raytracing structs.
+bool IsHLSLCopyableAnnotatableRecord(clang::QualType QT) {
+  const clang::Type *Ty = QT.getCanonicalType().getTypePtr();
   if (const RecordType *RT = dyn_cast<RecordType>(Ty)) {
     const RecordDecl *RD = RT->getDecl();
-    if (!IsUserDefinedRecordType(type))
+    if (!IsUserDefinedRecordType(QT))
       return false;
-    for (auto member : RD->fields()) {
-      if (!IsHLSLNumericOrAggregateOfNumericType(member->getType()))
+    for (auto Member : RD->fields()) {
+      if (!IsHLSLNumericOrAggregateOfNumericType(Member->getType()))
         return false;
     }
     return true;
   }
   return false;
 }
 
-// In some cases we need record types that are annotatable and trivially
-// copyable from outside the shader. This excludes resource types which may be
-// trivially copyable inside the shader, and builtin matrix and vector types
-// which can't be annotated. But includes UDTs of trivially copyable data and
-// the builtin trivially copyable raytracing structs.
-bool IsHLSLCopyableAnnotatableRecord(clang::QualType QT) {
-  return IsHLSLNumericUserDefinedType(QT) ||
-         IsHLSLBuiltinRayAttributeStruct(QT);
-}
-
 bool IsHLSLBuiltinRayAttributeStruct(clang::QualType QT) {
   QT = QT.getCanonicalType();
   const clang::Type *Ty = QT.getTypePtr();
@@ -609,7 +604,8 @@ bool IsUserDefinedRecordType(clang::QualType QT) {
   const clang::Type *Ty = QT.getCanonicalType().getTypePtr();
   if (const RecordType *RT = dyn_cast<RecordType>(Ty)) {
     const RecordDecl *RD = RT->getDecl();
-    if (RD->isImplicit())
+    // Built-in ray tracing struct types are considered user defined types.
+    if (RD->isImplicit() && !IsHLSLBuiltinRayAttributeStruct(QT))
       return false;
     if (auto TD = dyn_cast<ClassTemplateSpecializationDecl>(RD))
       if (TD->getSpecializedTemplate()->isImplicit())

tools/clang/test/CodeGenDXIL/hlsl/objects/HitObject/hitobject_attributes_builtin.hlsl

@@ -0,0 +1,42 @@
+// RUN: %dxc /Tlib_6_9 %s | FileCheck %s
+// RUN: %dxc /Tlib_6_9 -fcgl %s | FileCheck %s -check-prefix=FCGL
+
+// Make sure that we can use the BuiltInTriangleIntersectionAttributes struct
+// as a template argument to GetAttributes.
+
+// For -fcgl, just check the form of the HL call.
+// FCGL: %{{[^ ]+}} = call %struct.BuiltInTriangleIntersectionAttributes* @"dx.hl.op..%struct.BuiltInTriangleIntersectionAttributes* (i32, %dx.types.HitObject*)"(i32 364, %dx.types.HitObject* %{{[^ ]+}})
+
+// CHECK: %[[ATTR:[^ ]+]] = alloca %struct.BuiltInTriangleIntersectionAttributes
+// CHECK: call void @dx.op.hitObject_Attributes.struct.BuiltInTriangleIntersectionAttributes(i32 289, %dx.types.HitObject %{{[^ ]+}}, %struct.BuiltInTriangleIntersectionAttributes* nonnull %[[ATTR]])
+
+RaytracingAccelerationStructure Scene : register(t0, space0);
+RWTexture2D<float4> RenderTarget : register(u0);
+
+struct [raypayload] RayPayload
+{
+    float4 color : write(caller, closesthit, miss) : read(caller);
+};
+
+typedef BuiltInTriangleIntersectionAttributes MyAttribs;
+
+[shader("raygeneration")]
+void MyRaygenShader()
+{
+    RayDesc ray;
+    ray.Origin = float3(0,0,0);
+    ray.Direction = float3(0, 0, 1);
+    ray.TMin = 0.001;
+    ray.TMax = 10000.0;
+
+    RayPayload payload = { float4(0, 0, 0, 0) };
+    float4 color = float4(1,1,1,1);
+
+    dx::HitObject hit = dx::HitObject::TraceRay(Scene, RAY_FLAG_NONE, ~0, 0, 1, 0, ray, payload);
+
+    MyAttribs attr = hit.GetAttributes<MyAttribs>();
+    payload.color += float4(attr,0,1);
+
+    // Write the raytraced color to the output texture.
+    RenderTarget[DispatchRaysIndex().xy] = payload.color;
+}