'cmeq' and 'fcmeq' Vector64<T>.Zero/Vector128<T>.Zero ARM64 containment optimizations

src/coreclr/jit/codegenlinear.cpp

@@ -1657,7 +1657,7 @@ void CodeGen::genConsumeRegs(GenTree* tree)
 #ifdef FEATURE_SIMD
             // (In)Equality operation that produces bool result, when compared
             // against Vector zero, marks its Vector Zero operand as contained.
-            assert(tree->OperIsLeaf() || tree->IsSIMDZero());
+            assert(tree->OperIsLeaf() || tree->IsSIMDZero() || tree->IsVectorZero());
 #else
             assert(tree->OperIsLeaf());
 #endif

src/coreclr/jit/emitarm64.cpp

@@ -12990,6 +12990,11 @@ void emitter::emitDispIns(
                 emitDispVectorReg(id->idReg1(), id->idInsOpt(), true);
                 emitDispVectorReg(id->idReg2(), id->idInsOpt(), false);
             }
+            if (ins == INS_cmeq)
+            {
+                printf(", ");
+                emitDispFloatZero();
+            }
             break;
 
         case IF_DV_2N: // DV_2N   .........iiiiiii ......nnnnnddddd      Vd Vn imm   (shift - scalar)

src/coreclr/jit/gentree.cpp

@@ -17807,6 +17807,20 @@ bool GenTree::isContainableHWIntrinsic() const
             return true;
         }
 
+        default:
+        {
+            return false;
+        }
+    }
+#elif TARGET_ARM64
+    switch (AsHWIntrinsic()->GetHWIntrinsicId())
+    {
+        case NI_Vector64_get_Zero:
+        case NI_Vector128_get_Zero:
+        {
+            return true;
+        }
+
         default:
         {
             return false;

src/coreclr/jit/gentree.h

@@ -1730,6 +1730,8 @@ struct GenTree
     inline bool IsIntegralConst(ssize_t constVal) const;
     inline bool IsIntegralConstVector(ssize_t constVal) const;
     inline bool IsSIMDZero() const;
+    inline bool IsFloatPositiveZero() const;
+    inline bool IsVectorZero() const;
 
     inline bool IsBoxedValue();
 
@@ -2132,7 +2134,7 @@ struct GenTree
 
     inline bool IsCnsFltOrDbl() const;
 
-    inline bool IsCnsNonZeroFltOrDbl();
+    inline bool IsCnsNonZeroFltOrDbl() const;
 
     bool IsIconHandle() const
     {
@@ -7689,6 +7691,77 @@ inline bool GenTree::IsSIMDZero() const
     return false;
 }
 
+//-------------------------------------------------------------------
+// IsFloatPositiveZero: returns true if this is exactly a const float value of postive zero (+0.0)
+//
+// Returns:
+//     True if this represents a const floating-point value of exactly positive zero (+0.0).
+//     Will return false if the value is negative zero (-0.0).
+//
+inline bool GenTree::IsFloatPositiveZero() const
+{
+    return !(IsCnsNonZeroFltOrDbl());
+}
+
+//-------------------------------------------------------------------
+// IsVectorZero: returns true if this is an integral or floating-point (SIMD or HW intrinsic) vector
+// with all its elements equal to zero.
+//
+// Returns:
+//     True if this represents an integral or floating-point const (SIMD or HW intrinsic) vector with all its elements equal to zero.
+//
+// TODO: We already have IsSIMDZero() and IsIntegralConstVector(0),
+//       however, IsSIMDZero() does not cover hardware intrinsics, and IsIntegralConstVector(0) does not cover floating
+//       point. In order to not risk adverse behaviour by modifying those, this function 'IsVectorZero' was introduced.
+//       At some point, it makes sense to normalize this logic to be a single function call rather than have several
+//       separate ones; preferably this one.
+inline bool GenTree::IsVectorZero() const
+{
+#ifdef FEATURE_SIMD
+    if (gtOper == GT_SIMD)
+    {
+        const GenTreeSIMD* node = AsSIMD();
+
+        if (node->GetSIMDIntrinsicId() == SIMDIntrinsicInit)
+        {
+            return (node->Op(1)->IsIntegralConst(0) || node->Op(1)->IsFloatPositiveZero());
+        }
+    }
+#endif
+
+#ifdef FEATURE_HW_INTRINSICS
+    if (gtOper == GT_HWINTRINSIC)
+    {
+        const GenTreeHWIntrinsic* node = AsHWIntrinsic();
+        const var_types simdBaseType = node->GetSimdBaseType();
+
+        if (varTypeIsIntegral(simdBaseType) || varTypeIsFloating(simdBaseType))
+        {
+            const NamedIntrinsic intrinsicId = node->GetHWIntrinsicId();
+
+            if (node->GetOperandCount() == 0)
+            {
+#if defined(TARGET_XARCH)
+                return (intrinsicId == NI_Vector128_get_Zero) || (intrinsicId == NI_Vector256_get_Zero);
+#elif defined(TARGET_ARM64)
+                return (intrinsicId == NI_Vector64_get_Zero) || (intrinsicId == NI_Vector128_get_Zero);
+#endif // !TARGET_XARCH && !TARGET_ARM64
+            }
+            else if ((node->GetOperandCount() == 1) && (node->Op(1)->IsIntegralConst(0) || node->Op(1)->IsFloatPositiveZero()))
+            {
+#if defined(TARGET_XARCH)
+                return (intrinsicId == NI_Vector128_Create) || (intrinsicId == NI_Vector256_Create);
+#elif defined(TARGET_ARM64)
+                return (intrinsicId == NI_Vector64_Create) || (intrinsicId == NI_Vector128_Create);
+#endif // !TARGET_XARCH && !TARGET_ARM64
+            }
+        }
+    }
+#endif // FEATURE_HW_INTRINSICS
+
+    return false;
+}
+
 inline bool GenTree::IsBoxedValue()
 {
     assert(gtOper != GT_BOX || AsBox()->BoxOp() != nullptr);
@@ -8353,7 +8426,7 @@ inline bool GenTree::IsCnsFltOrDbl() const
     return OperGet() == GT_CNS_DBL;
 }
 
-inline bool GenTree::IsCnsNonZeroFltOrDbl()
+inline bool GenTree::IsCnsNonZeroFltOrDbl() const
 {
     if (OperGet() == GT_CNS_DBL)
     {

src/coreclr/jit/hwintrinsiccodegenarm64.cpp

@@ -496,6 +496,27 @@ void CodeGen::genHWIntrinsic(GenTreeHWIntrinsic* node)
                 GetEmitter()->emitIns_R_R_R(ins, emitSize, targetReg, op1Reg, op2Reg, opt);
                 break;
 
+            case NI_AdvSimd_CompareEqual:
+                if (intrin.op1->isContained())
+                {
+                    assert(HWIntrinsicInfo::SupportsContainment(intrin.id));
+                    assert(intrin.op1->IsVectorZero());
+
+                    GetEmitter()->emitIns_R_R(ins, emitSize, targetReg, op1Reg, opt);
+                }
+                else if (intrin.op2->isContained())
+                {
+                    assert(HWIntrinsicInfo::SupportsContainment(intrin.id));
+                    assert(intrin.op2->IsVectorZero());
+
+                    GetEmitter()->emitIns_R_R(ins, emitSize, targetReg, op1Reg, opt);
+                }
+                else
+                {
+                    GetEmitter()->emitIns_R_R_R(ins, emitSize, targetReg, op1Reg, op2Reg, opt);
+                }
+                break;
+
             case NI_AdvSimd_AbsoluteCompareLessThan:
             case NI_AdvSimd_AbsoluteCompareLessThanOrEqual:
             case NI_AdvSimd_CompareLessThan:

src/coreclr/jit/hwintrinsiclistarm64.h

@@ -214,7 +214,7 @@ HARDWARE_INTRINSIC(AdvSimd,       BitwiseClear,
 HARDWARE_INTRINSIC(AdvSimd,       BitwiseSelect,                                                    -1,      3,     {INS_bsl,            INS_bsl,            INS_bsl,            INS_bsl,            INS_bsl,            INS_bsl,            INS_bsl,            INS_bsl,            INS_bsl,            INS_bsl},         HW_Category_SIMD,                  HW_Flag_SpecialCodeGen)
 HARDWARE_INTRINSIC(AdvSimd,       Ceiling,                                                          -1,      1,     {INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_frintp,         INS_invalid},     HW_Category_SIMD,                  HW_Flag_NoFlag)
 HARDWARE_INTRINSIC(AdvSimd,       CeilingScalar,                                                     8,      1,     {INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_frintp,         INS_frintp},      HW_Category_SIMD,                  HW_Flag_SIMDScalar)
-HARDWARE_INTRINSIC(AdvSimd,       CompareEqual,                                                     -1,      2,     {INS_cmeq,           INS_cmeq,           INS_cmeq,           INS_cmeq,           INS_cmeq,           INS_cmeq,           INS_invalid,        INS_invalid,        INS_fcmeq,          INS_invalid},     HW_Category_SIMD,                  HW_Flag_Commutative)
+HARDWARE_INTRINSIC(AdvSimd,       CompareEqual,                                                     -1,      2,     {INS_cmeq,           INS_cmeq,           INS_cmeq,           INS_cmeq,           INS_cmeq,           INS_cmeq,           INS_invalid,        INS_invalid,        INS_fcmeq,          INS_invalid},     HW_Category_SIMD,                  HW_Flag_Commutative|HW_Flag_SpecialCodeGen|HW_Flag_SupportsContainment)
 HARDWARE_INTRINSIC(AdvSimd,       CompareGreaterThan,                                               -1,      2,     {INS_cmgt,           INS_cmhi,           INS_cmgt,           INS_cmhi,           INS_cmgt,           INS_cmhi,           INS_invalid,        INS_invalid,        INS_fcmgt,          INS_invalid},     HW_Category_SIMD,                  HW_Flag_NoFlag)
 HARDWARE_INTRINSIC(AdvSimd,       CompareGreaterThanOrEqual,                                        -1,      2,     {INS_cmge,           INS_cmhs,           INS_cmge,           INS_cmhs,           INS_cmge,           INS_cmhs,           INS_invalid,        INS_invalid,        INS_fcmge,          INS_invalid},     HW_Category_SIMD,                  HW_Flag_NoFlag)
 HARDWARE_INTRINSIC(AdvSimd,       CompareLessThan,                                                  -1,      2,     {INS_cmgt,           INS_cmhi,           INS_cmgt,           INS_cmhi,           INS_cmgt,           INS_cmhi,           INS_invalid,        INS_invalid,        INS_fcmgt,          INS_invalid},     HW_Category_SIMD,                  HW_Flag_SpecialCodeGen)

src/coreclr/jit/instrsarm64.h

@@ -312,8 +312,8 @@ INST4(neg,         "neg",          0,      IF_EN4G,   0x4B0003E0,  0x4B0003E0,
 INST4(cmeq,        "cmeq",         0,      IF_EN4H,   0x7EE08C00,  0x2E208C00,  0x5E209800,  0x0E209800)
                                    //  cmeq    Vd,Vn,Vm             DV_3E  01111110111mmmmm 100011nnnnnddddd   7EE0 8C00   Vd,Vn,Vm   (scalar)
                                    //  cmeq    Vd,Vn,Vm             DV_3A  0Q101110XX1mmmmm 100011nnnnnddddd   2E20 8C00   Vd,Vn,Vm   (vector)
-                                   //  cmeq    Vd,Vn                DV_2L  01011110XX100000 100110nnnnnddddd   5E20 9800   Vd,Vn      (scalar)
-                                   //  cmeq    Vd,Vn                DV_2M  0Q001110XX100000 100110nnnnnddddd   0E20 9800   Vd,Vn      (vector)
+                                   //  cmeq    Vd,Vn                DV_2L  01011110XX100000 100110nnnnnddddd   5E20 9800   Vd,Vn,#0   (scalar - with zero)
+                                   //  cmeq    Vd,Vn                DV_2M  0Q001110XX100000 100110nnnnnddddd   0E20 9800   Vd,Vn,#0   (vector - with zero)
 
 INST4(cmge,        "cmge",         0,      IF_EN4H,   0x5EE03C00,  0x0E203C00,  0x7E208800,  0x2E208800)
                                    //  cmge    Vd,Vn,Vm             DV_3E  01011110111mmmmm 001111nnnnnddddd   5EE0 3C00   Vd,Vn,Vm   (scalar)

src/coreclr/jit/lowerarmarch.cpp

@@ -1853,6 +1853,19 @@ void Lowering::ContainCheckHWIntrinsic(GenTreeHWIntrinsic* node)
                 MakeSrcContained(node, intrin.op4);
                 break;
 
+            case NI_AdvSimd_CompareEqual:
+            {
+                if (intrin.op1->IsVectorZero())
+                {
+                    MakeSrcContained(node, intrin.op1);
+                }
+                else if (intrin.op2->IsVectorZero())
+                {
+                    MakeSrcContained(node, intrin.op2);
+                }
+                break;
+            }
+
             case NI_Vector64_CreateScalarUnsafe:
             case NI_Vector128_CreateScalarUnsafe:
             case NI_AdvSimd_DuplicateToVector64:

src/coreclr/jit/lsraarm64.cpp

@@ -1011,40 +1011,49 @@ int LinearScan::BuildHWIntrinsic(GenTreeHWIntrinsic* intrinsicTree)
 
     if (intrin.op1 != nullptr)
     {
-        bool simdRegToSimdRegMove = false;
-
-        if ((intrin.id == NI_Vector64_CreateScalarUnsafe) || (intrin.id == NI_Vector128_CreateScalarUnsafe))
-        {
-            simdRegToSimdRegMove = varTypeIsFloating(intrin.op1);
-        }
-        else if (intrin.id == NI_AdvSimd_Arm64_DuplicateToVector64)
+        // Do not give a register to the second operand if it is contained.
+        if (!hasImmediateOperand && intrin.op1->isContained())
         {
-            simdRegToSimdRegMove = (intrin.op1->TypeGet() == TYP_DOUBLE);
+            assert(HWIntrinsicInfo::SupportsContainment(intrin.id));
+            assert(intrin.op1->IsVectorZero());
         }
-        else if ((intrin.id == NI_Vector64_ToScalar) || (intrin.id == NI_Vector128_ToScalar))
+        else
         {
-            simdRegToSimdRegMove = varTypeIsFloating(intrinsicTree);
-        }
+            bool simdRegToSimdRegMove = false;
 
-        // If we have an RMW intrinsic or an intrinsic with simple move semantic between two SIMD registers,
-        // we want to preference op1Reg to the target if op1 is not contained.
-        if (isRMW || simdRegToSimdRegMove)
-        {
-            tgtPrefOp1 = !intrin.op1->isContained();
-        }
+            if ((intrin.id == NI_Vector64_CreateScalarUnsafe) || (intrin.id == NI_Vector128_CreateScalarUnsafe))
+            {
+                simdRegToSimdRegMove = varTypeIsFloating(intrin.op1);
+            }
+            else if (intrin.id == NI_AdvSimd_Arm64_DuplicateToVector64)
+            {
+                simdRegToSimdRegMove = (intrin.op1->TypeGet() == TYP_DOUBLE);
+            }
+            else if ((intrin.id == NI_Vector64_ToScalar) || (intrin.id == NI_Vector128_ToScalar))
+            {
+                simdRegToSimdRegMove = varTypeIsFloating(intrinsicTree);
+            }
 
-        if (intrinsicTree->OperIsMemoryLoadOrStore())
-        {
-            srcCount += BuildAddrUses(intrin.op1);
-        }
-        else if (tgtPrefOp1)
-        {
-            tgtPrefUse = BuildUse(intrin.op1);
-            srcCount++;
-        }
-        else
-        {
-            srcCount += BuildOperandUses(intrin.op1);
+            // If we have an RMW intrinsic or an intrinsic with simple move semantic between two SIMD registers,
+            // we want to preference op1Reg to the target if op1 is not contained.
+            if (isRMW || simdRegToSimdRegMove)
+            {
+                tgtPrefOp1 = !intrin.op1->isContained();
+            }
+
+            if (intrinsicTree->OperIsMemoryLoadOrStore())
+            {
+                srcCount += BuildAddrUses(intrin.op1);
+            }
+            else if (tgtPrefOp1)
+            {
+                tgtPrefUse = BuildUse(intrin.op1);
+                srcCount++;
+            }
+            else
+            {
+                srcCount += BuildOperandUses(intrin.op1);
+            }
         }
     }
 
@@ -1091,9 +1100,15 @@ int LinearScan::BuildHWIntrinsic(GenTreeHWIntrinsic* intrinsicTree)
             }
         }
     }
-    else
+    else if (intrin.op2 != nullptr)
     {
-        if (intrin.op2 != nullptr)
+        // Do not give a register to the second operand if it is contained.
+        if (!hasImmediateOperand && intrin.op2->isContained())
+        {
+            assert(HWIntrinsicInfo::SupportsContainment(intrin.id));
+            assert(intrin.op2->IsVectorZero());
+        }
+        else
         {
             // RMW intrinsic operands doesn't have to be delayFree when they can be assigned the same register as op1Reg
             // (i.e. a register that corresponds to read-modify-write operand) and one of them is the last use.