[clang][bytecode] Fix D3DCOLORtoUBYTE4 hlsl test (llvm#151819)

HLSL is using CK_FloatingToIntegral casts to cast to vectors, which we
don't support (neither does the current interpreter). Also fix a crash
when trying to promote the HLSL bool, which can't be promoted it seems.

Author: tbaederr

clang/lib/AST/ByteCode/Compiler.cpp

@@ -331,6 +331,8 @@ bool Compiler<Emitter>::VisitCastExpr(const CastExpr *CE) {
   }
 
   case CK_FloatingToIntegral: {
+    if (!CE->getType()->isIntegralOrEnumerationType())
+      return false;
     if (!this->visit(SubExpr))
       return false;
     PrimType ToT = classifyPrim(CE);
@@ -1369,10 +1371,15 @@ bool Compiler<Emitter>::VisitVectorBinOp(const BinaryOperator *E) {
   // BitAdd/BitOr/BitXor/Shl/Shr doesn't support bool type, we need perform the
   // integer promotion.
   bool NeedIntPromot = ElemT == PT_Bool && (E->isBitwiseOp() || E->isShiftOp());
-  QualType PromotTy =
-      Ctx.getASTContext().getPromotedIntegerType(Ctx.getASTContext().BoolTy);
-  PrimType PromotT = classifyPrim(PromotTy);
-  PrimType OpT = NeedIntPromot ? PromotT : ElemT;
+  QualType PromotTy;
+  PrimType PromotT = PT_Bool;
+  PrimType OpT = ElemT;
+  if (NeedIntPromot) {
+    PromotTy =
+        Ctx.getASTContext().getPromotedIntegerType(Ctx.getASTContext().BoolTy);
+    PromotT = classifyPrim(PromotTy);
+    OpT = PromotT;
+  }
 
   auto getElem = [=](unsigned Offset, PrimType ElemT, unsigned Index) {
     if (!this->emitGetLocal(PT_Ptr, Offset, E))

clang/test/AST/ByteCode/hlsl.hlsl

@@ -29,3 +29,11 @@ export void fn() {
   // smaller vector, then truncated to a float as a constant expression.
   _Static_assert(((float2)float4(6, 5, 4, 3)).x == 6, "Woo!");
 }
+
+int4 test_D3DCOLORtoUBYTE4(float4 p1) {
+  return D3DCOLORtoUBYTE4(p1);
+}
+
+int4 test_constant_inputs() {
+  return D3DCOLORtoUBYTE4(float4(0, 11.11, -50.5, 100));
+}