[HLSL] Add __builtin_hlsl_is_scalarized_layout_compatible (llvm#102227)

HLSL tends to rely pretty aggressively on scalarization occuring in the
complier, which allows for some relaxed language behaviors when types
are fully sclarized to equivalent scalar representations.

This change adds a new queryable trait builtin for scalarized layout
compatability.

Resolves llvm#100614

---------

Co-authored-by: Aaron Ballman <[email protected]>

clang/include/clang/Basic/TokenKinds.def

@@ -660,6 +660,9 @@ KEYWORD(out                         , KEYHLSL)
 #define HLSL_INTANGIBLE_TYPE(Name, Id, SingletonId) KEYWORD(Name, KEYHLSL)
 #include "clang/Basic/HLSLIntangibleTypes.def"
 
+// HLSL Type traits.
+TYPE_TRAIT_2(__builtin_hlsl_is_scalarized_layout_compatible, IsScalarizedLayoutCompatible, KEYHLSL)
+
 // OpenMP Type Traits
 UNARY_EXPR_OR_TYPE_TRAIT(__builtin_omp_required_simd_align, OpenMPRequiredSimdAlign, KEYALL)
 

clang/include/clang/Sema/SemaHLSL.h

@@ -61,6 +61,9 @@ class SemaHLSL : public SemaBase {
   void handleParamModifierAttr(Decl *D, const ParsedAttr &AL);
 
   bool CheckBuiltinFunctionCall(unsigned BuiltinID, CallExpr *TheCall);
+
+  // HLSL Type trait implementations
+  bool IsScalarizedLayoutCompatible(QualType T1, QualType T2) const;
 };
 
 } // namespace clang

clang/lib/Sema/SemaExprCXX.cpp

@@ -39,6 +39,7 @@
 #include "clang/Sema/Scope.h"
 #include "clang/Sema/ScopeInfo.h"
 #include "clang/Sema/SemaCUDA.h"
+#include "clang/Sema/SemaHLSL.h"
 #include "clang/Sema/SemaInternal.h"
 #include "clang/Sema/SemaLambda.h"
 #include "clang/Sema/SemaObjC.h"
@@ -6248,6 +6249,23 @@ static bool EvaluateBinaryTypeTrait(Sema &Self, TypeTrait BTT, const TypeSourceI
                TSTToBeDeduced->getTemplateName().getAsTemplateDecl(), RhsT,
                Info) == TemplateDeductionResult::Success;
   }
+  case BTT_IsScalarizedLayoutCompatible: {
+    if (!LhsT->isVoidType() && !LhsT->isIncompleteArrayType() &&
+        Self.RequireCompleteType(Lhs->getTypeLoc().getBeginLoc(), LhsT,
+                                 diag::err_incomplete_type))
+      return true;
+    if (!RhsT->isVoidType() && !RhsT->isIncompleteArrayType() &&
+        Self.RequireCompleteType(Rhs->getTypeLoc().getBeginLoc(), RhsT,
+                                 diag::err_incomplete_type))
+      return true;
+
+    DiagnoseVLAInCXXTypeTrait(
+        Self, Lhs, tok::kw___builtin_hlsl_is_scalarized_layout_compatible);
+    DiagnoseVLAInCXXTypeTrait(
+        Self, Rhs, tok::kw___builtin_hlsl_is_scalarized_layout_compatible);
+
+    return Self.HLSL().IsScalarizedLayoutCompatible(LhsT, RhsT);
+  }
   default:
     llvm_unreachable("not a BTT");
   }

clang/lib/Sema/SemaHLSL.cpp

@@ -1524,3 +1524,85 @@ bool SemaHLSL::CheckBuiltinFunctionCall(unsigned BuiltinID, CallExpr *TheCall) {
   }
   return false;
 }
+
+static void BuildFlattenedTypeList(QualType BaseTy,
+                                   llvm::SmallVectorImpl<QualType> &List) {
+  llvm::SmallVector<QualType, 16> WorkList;
+  WorkList.push_back(BaseTy);
+  while (!WorkList.empty()) {
+    QualType T = WorkList.pop_back_val();
+    T = T.getCanonicalType().getUnqualifiedType();
+    assert(!isa<MatrixType>(T) && "Matrix types not yet supported in HLSL");
+    if (const auto *AT = dyn_cast<ConstantArrayType>(T)) {
+      llvm::SmallVector<QualType, 16> ElementFields;
+      // Generally I've avoided recursion in this algorithm, but arrays of
+      // structs could be time-consuming to flatten and churn through on the
+      // work list. Hopefully nesting arrays of structs containing arrays
+      // of structs too many levels deep is unlikely.
+      BuildFlattenedTypeList(AT->getElementType(), ElementFields);
+      // Repeat the element's field list n times.
+      for (uint64_t Ct = 0; Ct < AT->getZExtSize(); ++Ct)
+        List.insert(List.end(), ElementFields.begin(), ElementFields.end());
+      continue;
+    }
+    // Vectors can only have element types that are builtin types, so this can
+    // add directly to the list instead of to the WorkList.
+    if (const auto *VT = dyn_cast<VectorType>(T)) {
+      List.insert(List.end(), VT->getNumElements(), VT->getElementType());
+      continue;
+    }
+    if (const auto *RT = dyn_cast<RecordType>(T)) {
+      const RecordDecl *RD = RT->getDecl();
+      if (RD->isUnion()) {
+        List.push_back(T);
+        continue;
+      }
+      const CXXRecordDecl *CXXD = dyn_cast<CXXRecordDecl>(RD);
+
+      llvm::SmallVector<QualType, 16> FieldTypes;
+      if (CXXD && CXXD->isStandardLayout())
+        RD = CXXD->getStandardLayoutBaseWithFields();
+
+      for (const auto *FD : RD->fields())
+        FieldTypes.push_back(FD->getType());
+      // Reverse the newly added sub-range.
+      std::reverse(FieldTypes.begin(), FieldTypes.end());
+      WorkList.insert(WorkList.end(), FieldTypes.begin(), FieldTypes.end());
+
+      // If this wasn't a standard layout type we may also have some base
+      // classes to deal with.
+      if (CXXD && !CXXD->isStandardLayout()) {
+        FieldTypes.clear();
+        for (const auto &Base : CXXD->bases())
+          FieldTypes.push_back(Base.getType());
+        std::reverse(FieldTypes.begin(), FieldTypes.end());
+        WorkList.insert(WorkList.end(), FieldTypes.begin(), FieldTypes.end());
+      }
+      continue;
+    }
+    List.push_back(T);
+  }
+}
+
+bool SemaHLSL::IsScalarizedLayoutCompatible(QualType T1, QualType T2) const {
+  if (T1.isNull() || T2.isNull())
+    return false;
+
+  T1 = T1.getCanonicalType().getUnqualifiedType();
+  T2 = T2.getCanonicalType().getUnqualifiedType();
+
+  // If both types are the same canonical type, they're obviously compatible.
+  if (SemaRef.getASTContext().hasSameType(T1, T2))
+    return true;
+
+  llvm::SmallVector<QualType, 16> T1Types;
+  BuildFlattenedTypeList(T1, T1Types);
+  llvm::SmallVector<QualType, 16> T2Types;
+  BuildFlattenedTypeList(T2, T2Types);
+
+  // Check the flattened type list
+  return llvm::equal(T1Types, T2Types,
+                     [this](QualType LHS, QualType RHS) -> bool {
+                       return SemaRef.IsLayoutCompatible(LHS, RHS);
+                     });
+}

clang/test/SemaHLSL/Types/Traits/ScalarizedLayoutCompatible.hlsl

@@ -0,0 +1,132 @@
+// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.6-library -finclude-default-header -verify %s
+// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.6-library -finclude-default-header -fnative-half-type -verify %s
+// expected-no-diagnostics
+
+// Case 1: How many ways can I come up with to represent three float values?
+struct ThreeFloats1 {
+  float X, Y, Z;
+};
+
+struct ThreeFloats2 {
+  float X[3];
+};
+
+struct ThreeFloats3 {
+  float3 V;
+};
+
+struct ThreeFloats4 {
+  float2 V;
+  float F;
+};
+
+_Static_assert(__builtin_hlsl_is_scalarized_layout_compatible(float3, float[3]), "");
+_Static_assert(__builtin_hlsl_is_scalarized_layout_compatible(float3, ThreeFloats1), "");
+_Static_assert(__builtin_hlsl_is_scalarized_layout_compatible(float3, ThreeFloats2), "");
+_Static_assert(__builtin_hlsl_is_scalarized_layout_compatible(float3, ThreeFloats3), "");
+_Static_assert(__builtin_hlsl_is_scalarized_layout_compatible(float3, ThreeFloats4), "");
+
+// Case 2: structs and base classes and arrays, oh my!
+struct Dog {
+  int Leg[4];
+  bool Tail;
+  float Fur;
+};
+
+struct Shiba {
+  int4 StubbyLegs;
+  bool CurlyTail;
+  struct Coating {
+    float Fur;
+  } F;
+};
+
+struct FourLegged {
+  int FR, FL, BR, BL;
+};
+
+struct Doggo : FourLegged {
+  bool WaggyBit;
+  float Fuzz;
+};
+
+_Static_assert(__builtin_hlsl_is_scalarized_layout_compatible(Dog, Shiba), "");
+_Static_assert(__builtin_hlsl_is_scalarized_layout_compatible(Dog, Doggo), "");
+
+// Case 3: Arrays of structs inside structs
+
+struct Cat {
+  struct Leg {
+    int L;
+  } Legs[4];
+  struct Other {
+    bool Tail;
+    float Furs;
+  } Bits;
+};
+
+_Static_assert(__builtin_hlsl_is_scalarized_layout_compatible(Dog, Cat), "");
+
+// case 4: Arrays of structs inside arrays of structs.
+struct Pets {
+  Dog Puppers[6];
+  Cat Kitties[4];
+};
+
+struct Animals {
+  Dog Puppers[2];
+  Cat Kitties[8];
+};
+
+_Static_assert(__builtin_hlsl_is_scalarized_layout_compatible(Pets, Animals), "");
+
+// Case 5: Turtles all the way down...
+
+typedef int Turtle;
+
+enum Ninja : Turtle {
+  Leonardo,
+  Donatello,
+  Michelangelo,
+  Raphael,
+};
+
+enum NotNinja : Turtle {
+  Fred,
+  Mikey,
+};
+
+enum Mammals : uint {
+  Dog,
+  Cat,
+};
+
+_Static_assert(__builtin_hlsl_is_scalarized_layout_compatible(Ninja, NotNinja), "");
+_Static_assert(!__builtin_hlsl_is_scalarized_layout_compatible(Ninja, Mammals), "");
+
+// Case 6: Some basic types.
+_Static_assert(__builtin_hlsl_is_scalarized_layout_compatible(int, int32_t), "");
+_Static_assert(__builtin_hlsl_is_scalarized_layout_compatible(uint, uint32_t), "");
+_Static_assert(!__builtin_hlsl_is_scalarized_layout_compatible(int, uint), "");
+_Static_assert(!__builtin_hlsl_is_scalarized_layout_compatible(int, float), "");
+
+// Even though half and float may be the same size we don't want them to be
+// layout compatible since they are different types.
+_Static_assert(!__builtin_hlsl_is_scalarized_layout_compatible(half, float), "");
+
+// Case 6: Empty classes... because they're fun.
+
+struct NotEmpty { int X; };
+struct Empty {};
+struct AlsoEmpty {};
+
+struct DerivedEmpty : Empty {};
+
+struct DerivedNotEmpty : Empty { int X; };
+struct DerivedEmptyNotEmptyBase : NotEmpty {};
+
+_Static_assert(__builtin_hlsl_is_scalarized_layout_compatible(Empty, AlsoEmpty), "");
+_Static_assert(__builtin_hlsl_is_scalarized_layout_compatible(Empty, DerivedEmpty), "");
+
+_Static_assert(__builtin_hlsl_is_scalarized_layout_compatible(NotEmpty, DerivedNotEmpty), "");
+_Static_assert(__builtin_hlsl_is_scalarized_layout_compatible(NotEmpty, DerivedEmptyNotEmptyBase), "");

clang/test/SemaHLSL/Types/Traits/ScalarizedLayoutCompatibleErrors.hlsl

@@ -0,0 +1,64 @@
+// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.6-library  -finclude-default-header -verify %s
+
+// Some things that don't work!
+
+// Case 1: Both types must be complete!
+struct Defined {
+  int X;
+};
+
+
+struct Undefined; // expected-note {{forward declaration of 'Undefined'}}
+
+_Static_assert(__builtin_hlsl_is_scalarized_layout_compatible(Undefined, Defined), ""); // expected-error{{incomplete type 'Undefined' where a complete type is required}}
+
+// Case 2: No variable length arrays!
+
+void fn(int X) {
+  // expected-error@#vla {{variable length arrays are not supported for the current target}}
+  // expected-error@#vla {{variable length arrays are not supported in '__builtin_hlsl_is_scalarized_layout_compatible'}}
+  // expected-error@#vla {{static assertion failed due to requirement '__builtin_hlsl_is_scalarized_layout_compatible(int[4], int[X])'}}
+  // expected-warning@#vla {{variable length arrays in C++ are a Clang extension}}
+  _Static_assert(__builtin_hlsl_is_scalarized_layout_compatible(int[4], int[X]), ""); // #vla
+}
+
+// Case 3: Make this always fail for unions.
+// HLSL doesn't really support unions, and the places where scalarized layouts
+// are valid is probably going to be really confusing for unions, so we should
+// just make sure unions are never scalarized compatible with anything other
+// than themselves.
+
+union Wah {
+  int OhNo;
+  float NotAgain;
+};
+
+struct OneInt {
+  int I;
+};
+
+struct OneFloat {
+  float F;
+};
+
+struct HasUnion {
+  int I;
+  Wah W;
+};
+
+struct HasUnionSame {
+  int I;
+  Wah W;
+};
+
+struct HasUnionDifferent {
+  Wah W;
+  int I;
+};
+
+_Static_assert(__builtin_hlsl_is_scalarized_layout_compatible(Wah, Wah), "Identical types are always compatible");
+_Static_assert(!__builtin_hlsl_is_scalarized_layout_compatible(Wah, OneInt), "Unions are not compatible with anything else");
+_Static_assert(!__builtin_hlsl_is_scalarized_layout_compatible(Wah, OneFloat), "Unions are not compatible with anything else");
+
+_Static_assert(__builtin_hlsl_is_scalarized_layout_compatible(HasUnion, HasUnionSame), "");
+_Static_assert(!__builtin_hlsl_is_scalarized_layout_compatible(HasUnion, HasUnionDifferent), "");