Skip to content

arm64: Add Sve2 Saturating functions #117758

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Jul 18, 2025
Merged

arm64: Add Sve2 Saturating functions #117758

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions src/coreclr/jit/codegenarm64test.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5722,13 +5722,13 @@ void CodeGen::genArm64EmitterUnitTestsSve()
theEmitter->emitIns_R_R_R(INS_sve_sqdmlslbt, EA_SCALABLE, REG_V6, REG_V7, REG_V8,
INS_OPTS_SCALABLE_D); // SQDMLSLBT <Zda>.<T>, <Zn>.<Tb>, <Zm>.<Tb>
theEmitter->emitIns_R_R_R(INS_sve_sqdmlalb, EA_SCALABLE, REG_V0, REG_V1, REG_V2,
INS_OPTS_SCALABLE_H); // SQDMLALB <Zda>.<T>, <Zn>.<Tb>, <Zm>.<Tb>
INS_OPTS_SCALABLE_S); // SQDMLALB <Zda>.<T>, <Zn>.<Tb>, <Zm>.<Tb>
theEmitter->emitIns_R_R_R(INS_sve_sqdmlalt, EA_SCALABLE, REG_V3, REG_V4, REG_V5,
INS_OPTS_SCALABLE_S); // SQDMLALT <Zda>.<T>, <Zn>.<Tb>, <Zm>.<Tb>
theEmitter->emitIns_R_R_R(INS_sve_sqdmlslb, EA_SCALABLE, REG_V6, REG_V7, REG_V8,
INS_OPTS_SCALABLE_D); // SQDMLSLB <Zda>.<T>, <Zn>.<Tb>, <Zm>.<Tb>
theEmitter->emitIns_R_R_R(INS_sve_sqdmlslt, EA_SCALABLE, REG_V9, REG_V10, REG_V11,
INS_OPTS_SCALABLE_H); // SQDMLSLT <Zda>.<T>, <Zn>.<Tb>, <Zm>.<Tb>
INS_OPTS_SCALABLE_D); // SQDMLSLT <Zda>.<T>, <Zn>.<Tb>, <Zm>.<Tb>
theEmitter->emitIns_R_R_R(INS_sve_sabalb, EA_SCALABLE, REG_V0, REG_V1, REG_V2,
INS_OPTS_SCALABLE_H); // SABALB <Zda>.<T>, <Zn>.<Tb>, <Zm>.<Tb>
theEmitter->emitIns_R_R_R(INS_sve_sabalt, EA_SCALABLE, REG_V3, REG_V4, REG_V5,
Expand Down
4 changes: 2 additions & 2 deletions src/coreclr/jit/emitarm64sve.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5534,15 +5534,15 @@ void emitter::emitInsSve_R_R_R_I(instruction ins,
assert(isVectorRegister(reg2)); // nnnnn
assert(isLowVectorRegister(reg3)); // mmmm

if (opt == INS_OPTS_SCALABLE_H)
if (opt == INS_OPTS_SCALABLE_S)
{
assert((REG_V0 <= reg3) && (reg3 <= REG_V7)); // mmm
assert(isValidUimm<3>(imm)); // ii i
fmt = IF_SVE_FJ_3A;
}
else
{
assert(opt == INS_OPTS_SCALABLE_S);
assert(opt == INS_OPTS_SCALABLE_D);
assert(isValidUimm<2>(imm)); // ii
fmt = IF_SVE_FJ_3B;
}
Expand Down
8 changes: 8 additions & 0 deletions src/coreclr/jit/hwintrinsicarm64.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -306,6 +306,10 @@ void Compiler::getHWIntrinsicImmTypes(NamedIntrinsic intrinsic,
intrinsic == NI_Sve2_MultiplyBySelectedScalarWideningOdd ||
intrinsic == NI_Sve2_MultiplyBySelectedScalarWideningOddAndAdd ||
intrinsic == NI_Sve2_MultiplyBySelectedScalarWideningOddAndSubtract ||
intrinsic == NI_Sve2_MultiplyDoublingWideningBySelectedScalarAndAddSaturateEven ||
intrinsic == NI_Sve2_MultiplyDoublingWideningBySelectedScalarAndAddSaturateOdd ||
intrinsic == NI_Sve2_MultiplyDoublingWideningBySelectedScalarAndSubtractSaturateEven ||
intrinsic == NI_Sve2_MultiplyDoublingWideningBySelectedScalarAndSubtractSaturateOdd ||
intrinsic == NI_Sve2_MultiplySubtractBySelectedScalar)
{
indexedElementBaseType = simdBaseType;
Expand Down Expand Up @@ -386,6 +390,10 @@ void HWIntrinsicInfo::lookupImmBounds(
case NI_Sve2_MultiplyBySelectedScalarWideningOdd:
case NI_Sve2_MultiplyBySelectedScalarWideningOddAndAdd:
case NI_Sve2_MultiplyBySelectedScalarWideningOddAndSubtract:
case NI_Sve2_MultiplyDoublingWideningBySelectedScalarAndAddSaturateEven:
case NI_Sve2_MultiplyDoublingWideningBySelectedScalarAndAddSaturateOdd:
case NI_Sve2_MultiplyDoublingWideningBySelectedScalarAndSubtractSaturateEven:
case NI_Sve2_MultiplyDoublingWideningBySelectedScalarAndSubtractSaturateOdd:
// Index is on the half-width vector, hence double the maximum index.
immUpperBound = Compiler::getSIMDVectorLength(simdSize, baseType) * 2 - 1;
break;
Expand Down
15 changes: 14 additions & 1 deletion src/coreclr/jit/hwintrinsiclistarm64sve.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -312,7 +312,9 @@ HARDWARE_INTRINSIC(Sve, ZipLow,
// ISA Function name SIMD size NumArg Instructions Category Flags
// {TYP_BYTE, TYP_UBYTE, TYP_SHORT, TYP_USHORT, TYP_INT, TYP_UINT, TYP_LONG, TYP_ULONG, TYP_FLOAT, TYP_DOUBLE}
// ***************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************************
#define FIRST_NI_Sve2 NI_Sve2_AbsoluteDifferenceAdd
// SVE2 Intrinsics
#define FIRST_NI_Sve2 NI_Sve2_AbsSaturate
HARDWARE_INTRINSIC(Sve2, AbsSaturate, -1, -1, {INS_sve_sqabs, INS_invalid, INS_sve_sqabs, INS_invalid, INS_sve_sqabs, INS_invalid, INS_sve_sqabs, INS_invalid, INS_invalid, INS_invalid}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation)
HARDWARE_INTRINSIC(Sve2, AbsoluteDifferenceAdd, -1, 3, {INS_sve_saba, INS_sve_uaba, INS_sve_saba, INS_sve_uaba, INS_sve_saba, INS_sve_uaba, INS_sve_saba, INS_sve_uaba, INS_invalid, INS_invalid}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasRMWSemantics)
HARDWARE_INTRINSIC(Sve2, AbsoluteDifferenceWideningEven, -1, 2, {INS_invalid, INS_invalid, INS_sve_sabdlb, INS_sve_uabdlb, INS_sve_sabdlb, INS_sve_uabdlb, INS_sve_sabdlb, INS_sve_uabdlb, INS_invalid, INS_invalid}, HW_Category_SIMD, HW_Flag_Scalable)
HARDWARE_INTRINSIC(Sve2, AbsoluteDifferenceWideningLowerAndAddEven, -1, 3, {INS_invalid, INS_invalid, INS_sve_sabalb, INS_sve_uabalb, INS_sve_sabalb, INS_sve_uabalb, INS_sve_sabalb, INS_sve_uabalb, INS_invalid, INS_invalid}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_HasRMWSemantics)
Expand Down Expand Up @@ -349,13 +351,24 @@ HARDWARE_INTRINSIC(Sve2, MultiplyBySelectedScalarWideningEvenAndSubtrac
HARDWARE_INTRINSIC(Sve2, MultiplyBySelectedScalarWideningOdd, -1, 3, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_smullt, INS_sve_umullt, INS_sve_smullt, INS_sve_umullt, INS_invalid, INS_invalid}, HW_Category_SIMDByIndexedElement, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_LowVectorOperation)
HARDWARE_INTRINSIC(Sve2, MultiplyBySelectedScalarWideningOddAndAdd, -1, 4, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_smlalt, INS_sve_umlalt, INS_sve_smlalt, INS_sve_umlalt, INS_invalid, INS_invalid}, HW_Category_SIMDByIndexedElement, HW_Flag_Scalable|HW_Flag_HasRMWSemantics|HW_Flag_HasImmediateOperand|HW_Flag_LowVectorOperation)
HARDWARE_INTRINSIC(Sve2, MultiplyBySelectedScalarWideningOddAndSubtract, -1, 4, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_smlslt, INS_sve_umlslt, INS_sve_smlslt, INS_sve_umlslt, INS_invalid, INS_invalid}, HW_Category_SIMDByIndexedElement, HW_Flag_Scalable|HW_Flag_HasRMWSemantics|HW_Flag_HasImmediateOperand|HW_Flag_LowVectorOperation)
HARDWARE_INTRINSIC(Sve2, MultiplyDoublingWideningAndAddSaturateEven, -1, 3, {INS_invalid, INS_invalid, INS_sve_sqdmlalb, INS_invalid, INS_sve_sqdmlalb, INS_invalid, INS_sve_sqdmlalb, INS_invalid, INS_invalid, INS_invalid}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasRMWSemantics)
HARDWARE_INTRINSIC(Sve2, MultiplyDoublingWideningAndAddSaturateEvenOdd, -1, 3, {INS_invalid, INS_invalid, INS_sve_sqdmlalbt, INS_invalid, INS_sve_sqdmlalbt, INS_invalid, INS_sve_sqdmlalbt, INS_invalid, INS_invalid, INS_invalid}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasRMWSemantics)
HARDWARE_INTRINSIC(Sve2, MultiplyDoublingWideningAndAddSaturateOdd, -1, 3, {INS_invalid, INS_invalid, INS_sve_sqdmlalt, INS_invalid, INS_sve_sqdmlalt, INS_invalid, INS_sve_sqdmlalt, INS_invalid, INS_invalid, INS_invalid}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasRMWSemantics)
HARDWARE_INTRINSIC(Sve2, MultiplyDoublingWideningAndSubtractSaturateEven, -1, 3, {INS_invalid, INS_invalid, INS_sve_sqdmlslb, INS_invalid, INS_sve_sqdmlslb, INS_invalid, INS_sve_sqdmlslb, INS_invalid, INS_invalid, INS_invalid}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasRMWSemantics)
HARDWARE_INTRINSIC(Sve2, MultiplyDoublingWideningAndSubtractSaturateEvenOdd, -1, 3, {INS_invalid, INS_invalid, INS_sve_sqdmlslbt, INS_invalid, INS_sve_sqdmlslbt, INS_invalid, INS_sve_sqdmlslbt, INS_invalid, INS_invalid, INS_invalid}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasRMWSemantics)
HARDWARE_INTRINSIC(Sve2, MultiplyDoublingWideningAndSubtractSaturateOdd, -1, 3, {INS_invalid, INS_invalid, INS_sve_sqdmlslt, INS_invalid, INS_sve_sqdmlslt, INS_invalid, INS_sve_sqdmlslt, INS_invalid, INS_invalid, INS_invalid}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasRMWSemantics)
HARDWARE_INTRINSIC(Sve2, MultiplyDoublingWideningBySelectedScalarAndAddSaturateEven, -1, 4, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_sqdmlalb, INS_invalid, INS_sve_sqdmlalb, INS_invalid, INS_invalid, INS_invalid}, HW_Category_SIMDByIndexedElement, HW_Flag_Scalable|HW_Flag_HasRMWSemantics|HW_Flag_HasImmediateOperand|HW_Flag_LowVectorOperation)
HARDWARE_INTRINSIC(Sve2, MultiplyDoublingWideningBySelectedScalarAndAddSaturateOdd, -1, 4, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_sqdmlalt, INS_invalid, INS_sve_sqdmlalt, INS_invalid, INS_invalid, INS_invalid}, HW_Category_SIMDByIndexedElement, HW_Flag_Scalable|HW_Flag_HasRMWSemantics|HW_Flag_HasImmediateOperand|HW_Flag_LowVectorOperation)
HARDWARE_INTRINSIC(Sve2, MultiplyDoublingWideningBySelectedScalarAndSubtractSaturateEven, -1, 4, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_sqdmlslb, INS_invalid, INS_sve_sqdmlslb, INS_invalid, INS_invalid, INS_invalid}, HW_Category_SIMDByIndexedElement, HW_Flag_Scalable|HW_Flag_HasRMWSemantics|HW_Flag_HasImmediateOperand|HW_Flag_LowVectorOperation)
HARDWARE_INTRINSIC(Sve2, MultiplyDoublingWideningBySelectedScalarAndSubtractSaturateOdd, -1, 4, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_sqdmlslt, INS_invalid, INS_sve_sqdmlslt, INS_invalid, INS_invalid, INS_invalid}, HW_Category_SIMDByIndexedElement, HW_Flag_Scalable|HW_Flag_HasRMWSemantics|HW_Flag_HasImmediateOperand|HW_Flag_LowVectorOperation)
HARDWARE_INTRINSIC(Sve2, MultiplySubtractBySelectedScalar, -1, 4, {INS_invalid, INS_invalid, INS_sve_mls, INS_sve_mls, INS_sve_mls, INS_sve_mls, INS_sve_mls, INS_sve_mls, INS_invalid, INS_invalid}, HW_Category_SIMDByIndexedElement, HW_Flag_Scalable|HW_Flag_HasImmediateOperand|HW_Flag_HasRMWSemantics|HW_Flag_LowVectorOperation)
HARDWARE_INTRINSIC(Sve2, MultiplyWideningEven, -1, 2, {INS_invalid, INS_invalid, INS_sve_smullb, INS_sve_umullb, INS_sve_smullb, INS_sve_umullb, INS_sve_smullb, INS_sve_umullb, INS_invalid, INS_invalid}, HW_Category_SIMD, HW_Flag_Scalable)
HARDWARE_INTRINSIC(Sve2, MultiplyWideningEvenAndAdd, -1, 3, {INS_invalid, INS_invalid, INS_sve_smlalb, INS_sve_umlalb, INS_sve_smlalb, INS_sve_umlalb, INS_sve_smlalb, INS_sve_umlalb, INS_invalid, INS_invalid}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasRMWSemantics)
HARDWARE_INTRINSIC(Sve2, MultiplyWideningEvenAndSubtract, -1, 3, {INS_invalid, INS_invalid, INS_sve_smlslb, INS_sve_umlslb, INS_sve_smlslb, INS_sve_umlslb, INS_sve_smlslb, INS_sve_umlslb, INS_invalid, INS_invalid}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasRMWSemantics)
HARDWARE_INTRINSIC(Sve2, MultiplyWideningOdd, -1, 2, {INS_invalid, INS_invalid, INS_sve_smullt, INS_sve_umullt, INS_sve_smullt, INS_sve_umullt, INS_sve_smullt, INS_sve_umullt, INS_invalid, INS_invalid}, HW_Category_SIMD, HW_Flag_Scalable)
HARDWARE_INTRINSIC(Sve2, MultiplyWideningOddAndAdd, -1, 3, {INS_invalid, INS_invalid, INS_sve_smlalt, INS_sve_umlalt, INS_sve_smlalt, INS_sve_umlalt, INS_sve_smlalt, INS_sve_umlalt, INS_invalid, INS_invalid}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasRMWSemantics)
HARDWARE_INTRINSIC(Sve2, MultiplyWideningOddAndSubtract, -1, 3, {INS_invalid, INS_invalid, INS_sve_smlslt, INS_sve_umlslt, INS_sve_smlslt, INS_sve_umlslt, INS_sve_smlslt, INS_sve_umlslt, INS_invalid, INS_invalid}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_HasRMWSemantics)
HARDWARE_INTRINSIC(Sve2, NegateSaturate, -1, -1, {INS_sve_sqneg, INS_invalid, INS_sve_sqneg, INS_invalid, INS_sve_sqneg, INS_invalid, INS_sve_sqneg, INS_invalid, INS_invalid, INS_invalid}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation)
HARDWARE_INTRINSIC(Sve2, PolynomialMultiply, -1, 2, {INS_sve_pmul, INS_sve_pmul, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_SIMD, HW_Flag_Scalable)
HARDWARE_INTRINSIC(Sve2, PolynomialMultiplyWideningEven, -1, 2, {INS_invalid, INS_invalid, INS_invalid, INS_sve_pmullb, INS_invalid, INS_invalid, INS_invalid, INS_sve_pmullb, INS_invalid, INS_invalid}, HW_Category_SIMD, HW_Flag_Scalable)
HARDWARE_INTRINSIC(Sve2, PolynomialMultiplyWideningOdd, -1, 2, {INS_invalid, INS_invalid, INS_invalid, INS_sve_pmullt, INS_invalid, INS_invalid, INS_invalid, INS_sve_pmullt, INS_invalid, INS_invalid}, HW_Category_SIMD, HW_Flag_Scalable)
Expand Down
4 changes: 4 additions & 0 deletions src/coreclr/jit/lsraarm64.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2170,6 +2170,10 @@ SingleTypeRegSet LinearScan::getOperandCandidates(GenTreeHWIntrinsic* intrinsicT
case NI_Sve2_MultiplySubtractBySelectedScalar:
case NI_Sve2_MultiplyBySelectedScalarWideningEvenAndSubtract:
case NI_Sve2_MultiplyBySelectedScalarWideningOddAndSubtract:
case NI_Sve2_MultiplyDoublingWideningBySelectedScalarAndAddSaturateEven:
case NI_Sve2_MultiplyDoublingWideningBySelectedScalarAndAddSaturateOdd:
case NI_Sve2_MultiplyDoublingWideningBySelectedScalarAndSubtractSaturateEven:
case NI_Sve2_MultiplyDoublingWideningBySelectedScalarAndSubtractSaturateOdd:
isLowVectorOpNum = (opNum == 3);
break;
case NI_Sve_MultiplyBySelectedScalar:
Expand Down
Loading
Loading