Fix GatherPrefetch and Prefetch APIs (#104494)

kunalspathak · web-flow · commit 9882188fbe16 · 2024-07-08T20:33:47.000-07:00
diff --git a/src/coreclr/jit/hwintrinsicarm64.cpp b/src/coreclr/jit/hwintrinsicarm64.cpp
@@ -2784,7 +2784,7 @@ GenTree* Compiler::impSpecialIntrinsic(NamedIntrinsic        intrinsic,
                 op4     = getArgForHWIntrinsic(argType, argClass);
 
                 assert(HWIntrinsicInfo::isImmOp(intrinsic, op4));
-                op3 = addRangeCheckIfNeeded(intrinsic, op4, mustExpand, immLowerBound, immUpperBound);
+                op4 = addRangeCheckIfNeeded(intrinsic, op4, mustExpand, immLowerBound, immUpperBound);
 
                 argType                    = JITtype2varType(strip(info.compCompHnd->getArgType(sig, arg3, &argClass)));
                 op3                        = getArgForHWIntrinsic(argType, argClass);
diff --git a/src/coreclr/jit/hwintrinsiccodegenarm64.cpp b/src/coreclr/jit/hwintrinsiccodegenarm64.cpp
@@ -308,18 +308,18 @@ void CodeGen::genHWIntrinsic(GenTreeHWIntrinsic* node)
         emitSize = emitActualTypeSize(intrin.baseType);
         opt      = INS_OPTS_NONE;
     }
+    else if (HWIntrinsicInfo::IsScalable(intrin.id))
+    {
+        emitSize = EA_SCALABLE;
+        opt      = emitter::optGetSveInsOpt(emitTypeSize(intrin.baseType));
+    }
     else if (intrin.category == HW_Category_Special)
     {
         assert(intrin.id == NI_ArmBase_Yield);
 
         emitSize = EA_UNKNOWN;
         opt      = INS_OPTS_NONE;
     }
-    else if (HWIntrinsicInfo::IsScalable(intrin.id))
-    {
-        emitSize = EA_SCALABLE;
-        opt      = emitter::optGetSveInsOpt(emitTypeSize(intrin.baseType));
-    }
     else
     {
         emitSize = emitActualTypeSize(Compiler::getSIMDTypeForSize(node->GetSimdSize()));
diff --git a/src/coreclr/jit/hwintrinsiclistarm64sve.h b/src/coreclr/jit/hwintrinsiclistarm64sve.h
@@ -99,10 +99,10 @@ HARDWARE_INTRINSIC(Sve,           FusedMultiplyAddNegated,
 HARDWARE_INTRINSIC(Sve,           FusedMultiplySubtract,                                            -1,     -1,      false, {INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_fmls,       INS_sve_fmls},    HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_LowMaskedOperation|HW_Flag_FmaIntrinsic|HW_Flag_SpecialCodeGen)
 HARDWARE_INTRINSIC(Sve,           FusedMultiplySubtractBySelectedScalar,                            -1,      4,      true,  {INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_fmls,       INS_sve_fmls},    HW_Category_SIMDByIndexedElement,  HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics|HW_Flag_FmaIntrinsic|HW_Flag_LowVectorOperation)
 HARDWARE_INTRINSIC(Sve,           FusedMultiplySubtractNegated,                                     -1,     -1,      false, {INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_fnmls,      INS_sve_fnmls},   HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_LowMaskedOperation|HW_Flag_FmaIntrinsic|HW_Flag_SpecialCodeGen)
-HARDWARE_INTRINSIC(Sve,           GatherPrefetch16Bit,                                              -1,     -1,      false, {INS_invalid,        INS_invalid,        INS_sve_prfh,       INS_sve_prfh,       INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid},     HW_Category_Special,               HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_HasImmediateOperand|HW_Flag_HasEnumOperand)
-HARDWARE_INTRINSIC(Sve,           GatherPrefetch32Bit,                                              -1,     -1,      false, {INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_prfw,       INS_sve_prfw,       INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid},     HW_Category_Special,               HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_HasImmediateOperand|HW_Flag_HasEnumOperand)
-HARDWARE_INTRINSIC(Sve,           GatherPrefetch64Bit,                                              -1,     -1,      false, {INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_prfd,       INS_sve_prfd,       INS_invalid,        INS_invalid},     HW_Category_Special,               HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_HasImmediateOperand|HW_Flag_HasEnumOperand)
-HARDWARE_INTRINSIC(Sve,           GatherPrefetch8Bit,                                               -1,     -1,      false, {INS_sve_prfb,       INS_sve_prfb,       INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid},     HW_Category_Special,               HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_HasImmediateOperand|HW_Flag_HasEnumOperand)
+HARDWARE_INTRINSIC(Sve,           GatherPrefetch16Bit,                                              -1,     -1,      false, {INS_invalid,        INS_invalid,        INS_sve_prfh,       INS_sve_prfh,       INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid},     HW_Category_Special,               HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_HasImmediateOperand|HW_Flag_HasEnumOperand|HW_Flag_SpecialSideEffect_Other)
+HARDWARE_INTRINSIC(Sve,           GatherPrefetch32Bit,                                              -1,     -1,      false, {INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_prfw,       INS_sve_prfw,       INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid},     HW_Category_Special,               HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_HasImmediateOperand|HW_Flag_HasEnumOperand|HW_Flag_SpecialSideEffect_Other)
+HARDWARE_INTRINSIC(Sve,           GatherPrefetch64Bit,                                              -1,     -1,      false, {INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_prfd,       INS_sve_prfd,       INS_invalid,        INS_invalid},     HW_Category_Special,               HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_HasImmediateOperand|HW_Flag_HasEnumOperand|HW_Flag_SpecialSideEffect_Other)
+HARDWARE_INTRINSIC(Sve,           GatherPrefetch8Bit,                                               -1,     -1,      false, {INS_sve_prfb,       INS_sve_prfb,       INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid},     HW_Category_Special,               HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_HasImmediateOperand|HW_Flag_HasEnumOperand|HW_Flag_SpecialSideEffect_Other)
 HARDWARE_INTRINSIC(Sve,           GatherVector,                                                     -1,     -1,      false, {INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_ld1w,       INS_sve_ld1w,       INS_sve_ld1d,       INS_sve_ld1d,       INS_sve_ld1w,       INS_sve_ld1d},    HW_Category_MemoryLoad,            HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation)
 HARDWARE_INTRINSIC(Sve,           GatherVectorByteZeroExtend,                                       -1,     -1,      false, {INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_ld1b,       INS_sve_ld1b,       INS_sve_ld1b,       INS_sve_ld1b,       INS_invalid,        INS_invalid},     HW_Category_MemoryLoad,            HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation)
 HARDWARE_INTRINSIC(Sve,           GatherVectorInt16SignExtend,                                      -1,     -1,      false, {INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_ld1sh,      INS_sve_ld1sh,      INS_sve_ld1sh,      INS_sve_ld1sh,      INS_invalid,        INS_invalid},     HW_Category_MemoryLoad,            HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation)
@@ -192,10 +192,10 @@ HARDWARE_INTRINSIC(Sve,           Not,
 HARDWARE_INTRINSIC(Sve,           Or,                                                               -1,     -1,      false, {INS_sve_orr,        INS_sve_orr,        INS_sve_orr,        INS_sve_orr,        INS_sve_orr,        INS_sve_orr,        INS_sve_orr,        INS_sve_orr,        INS_invalid,        INS_invalid},     HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_OptionalEmbeddedMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_LowMaskedOperation)
 HARDWARE_INTRINSIC(Sve,           OrAcross,                                                         -1,     -1,      false, {INS_sve_orv,        INS_sve_orv,        INS_sve_orv,        INS_sve_orv,        INS_sve_orv,        INS_sve_orv,        INS_sve_orv,        INS_sve_orv,        INS_invalid,        INS_invalid},     HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation)
 HARDWARE_INTRINSIC(Sve,           PopCount,                                                         -1,     -1,      false, {INS_sve_cnt,        INS_sve_cnt,        INS_sve_cnt,        INS_sve_cnt,        INS_sve_cnt,        INS_sve_cnt,        INS_sve_cnt,        INS_sve_cnt,        INS_sve_cnt,        INS_sve_cnt},     HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation)
-HARDWARE_INTRINSIC(Sve,           PrefetchBytes,                                                    -1,      3,      false, {INS_invalid,        INS_sve_prfb,       INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid},     HW_Category_Special,               HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_BaseTypeFromFirstArg|HW_Flag_HasImmediateOperand|HW_Flag_HasEnumOperand)
-HARDWARE_INTRINSIC(Sve,           PrefetchInt16,                                                    -1,      3,      false, {INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_prfh,       INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid},     HW_Category_Special,               HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_BaseTypeFromFirstArg|HW_Flag_HasImmediateOperand|HW_Flag_HasEnumOperand)
-HARDWARE_INTRINSIC(Sve,           PrefetchInt32,                                                    -1,      3,      false, {INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_prfw,       INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid},     HW_Category_Special,               HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_BaseTypeFromFirstArg|HW_Flag_HasImmediateOperand|HW_Flag_HasEnumOperand)
-HARDWARE_INTRINSIC(Sve,           PrefetchInt64,                                                    -1,      3,      false, {INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_prfd,       INS_invalid,        INS_invalid},     HW_Category_Special,               HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_BaseTypeFromFirstArg|HW_Flag_HasImmediateOperand|HW_Flag_HasEnumOperand)
+HARDWARE_INTRINSIC(Sve,           PrefetchBytes,                                                    -1,      3,      false, {INS_invalid,        INS_sve_prfb,       INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid},     HW_Category_Special,               HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_BaseTypeFromFirstArg|HW_Flag_HasImmediateOperand|HW_Flag_HasEnumOperand|HW_Flag_SpecialSideEffect_Other)
+HARDWARE_INTRINSIC(Sve,           PrefetchInt16,                                                    -1,      3,      false, {INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_prfh,       INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid},     HW_Category_Special,               HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_BaseTypeFromFirstArg|HW_Flag_HasImmediateOperand|HW_Flag_HasEnumOperand|HW_Flag_SpecialSideEffect_Other)
+HARDWARE_INTRINSIC(Sve,           PrefetchInt32,                                                    -1,      3,      false, {INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_prfw,       INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid},     HW_Category_Special,               HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_BaseTypeFromFirstArg|HW_Flag_HasImmediateOperand|HW_Flag_HasEnumOperand|HW_Flag_SpecialSideEffect_Other)
+HARDWARE_INTRINSIC(Sve,           PrefetchInt64,                                                    -1,      3,      false, {INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_prfd,       INS_invalid,        INS_invalid},     HW_Category_Special,               HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_BaseTypeFromFirstArg|HW_Flag_HasImmediateOperand|HW_Flag_HasEnumOperand|HW_Flag_SpecialSideEffect_Other)
 HARDWARE_INTRINSIC(Sve,           ReciprocalEstimate,                                               -1,      1,      true,  {INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_frecpe,     INS_sve_frecpe},  HW_Category_SIMD,                  HW_Flag_Scalable)
 HARDWARE_INTRINSIC(Sve,           ReciprocalExponent,                                               -1,      -1,     false, {INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_frecpx,     INS_sve_frecpx},  HW_Category_SIMD,                  HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation)
 HARDWARE_INTRINSIC(Sve,           ReciprocalSqrtEstimate,                                           -1,      1,      true,  {INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_invalid,        INS_sve_frsqrte,    INS_sve_frsqrte}, HW_Category_SIMD,                  HW_Flag_Scalable)
diff --git a/src/coreclr/jit/lsraarm64.cpp b/src/coreclr/jit/lsraarm64.cpp
@@ -1980,18 +1980,10 @@ int LinearScan::BuildHWIntrinsic(GenTreeHWIntrinsic* intrinsicTree, int* pDstCou
                 case NI_Sve_LoadVectorNonTemporal:
                 case NI_Sve_LoadVector128AndReplicateToVector:
                 case NI_Sve_StoreAndZip:
-                case NI_Sve_PrefetchBytes:
-                case NI_Sve_PrefetchInt16:
-                case NI_Sve_PrefetchInt32:
-                case NI_Sve_PrefetchInt64:
                     assert(intrinsicTree->OperIsMemoryLoadOrStore());
                     srcCount += BuildAddrUses(intrin.op2);
                     break;
 
-                case NI_Sve_GatherPrefetch8Bit:
-                case NI_Sve_GatherPrefetch16Bit:
-                case NI_Sve_GatherPrefetch32Bit:
-                case NI_Sve_GatherPrefetch64Bit:
                 case NI_Sve_GatherVector:
                 case NI_Sve_GatherVectorByteZeroExtend:
                 case NI_Sve_GatherVectorInt16SignExtend:
@@ -2004,6 +1996,16 @@ int LinearScan::BuildHWIntrinsic(GenTreeHWIntrinsic* intrinsicTree, int* pDstCou
                 case NI_Sve_GatherVectorUInt32WithByteOffsetsZeroExtend:
                 case NI_Sve_GatherVectorUInt32ZeroExtend:
                     assert(intrinsicTree->OperIsMemoryLoadOrStore());
+                    FALLTHROUGH;
+
+                case NI_Sve_PrefetchBytes:
+                case NI_Sve_PrefetchInt16:
+                case NI_Sve_PrefetchInt32:
+                case NI_Sve_PrefetchInt64:
+                case NI_Sve_GatherPrefetch8Bit:
+                case NI_Sve_GatherPrefetch16Bit:
+                case NI_Sve_GatherPrefetch32Bit:
+                case NI_Sve_GatherPrefetch64Bit:
                     if (!varTypeIsSIMD(intrin.op2->gtType))
                     {
                         srcCount += BuildAddrUses(intrin.op2);

Original file line number	Diff line number	Diff line change
`@@ -308,18 +308,18 @@ void CodeGen::genHWIntrinsic(GenTreeHWIntrinsic* node)`
`308`	`308`	`emitSize = emitActualTypeSize(intrin.baseType);`
`309`	`309`	`opt = INS_OPTS_NONE;`
`310`	`310`	`}`
	`311`	`+ else if (HWIntrinsicInfo::IsScalable(intrin.id))`
	`312`	`+ {`
	`313`	`+ emitSize = EA_SCALABLE;`
	`314`	`+ opt = emitter::optGetSveInsOpt(emitTypeSize(intrin.baseType));`
	`315`	`+ }`
`311`	`316`	`else if (intrin.category == HW_Category_Special)`
`312`	`317`	`{`
`313`	`318`	`assert(intrin.id == NI_ArmBase_Yield);`
`314`	`319`
`315`	`320`	`emitSize = EA_UNKNOWN;`
`316`	`321`	`opt = INS_OPTS_NONE;`
`317`	`322`	`}`
`318`		`- else if (HWIntrinsicInfo::IsScalable(intrin.id))`
`319`		`- {`
`320`		`- emitSize = EA_SCALABLE;`
`321`		`- opt = emitter::optGetSveInsOpt(emitTypeSize(intrin.baseType));`
`322`		`- }`
`323`	`323`	`else`
`324`	`324`	`{`
`325`	`325`	`emitSize = emitActualTypeSize(Compiler::getSIMDTypeForSize(node->GetSimdSize()));`