From 2e693e6d4f36dbf9063bbe7ab5ebfa9df1cd83b9 Mon Sep 17 00:00:00 2001 From: Swapnil Gaikwad Date: Wed, 12 Jun 2024 11:27:41 +0100 Subject: [PATCH] Add support for Sve.ReverseBits() and Sve.Splice() --- src/coreclr/jit/hwintrinsiccodegenarm64.cpp | 28 +- src/coreclr/jit/hwintrinsiclistarm64sve.h | 2 + src/coreclr/jit/lsraarm64.cpp | 34 +- .../Arm/Sve.PlatformNotSupported.cs | 114 +++++ .../src/System/Runtime/Intrinsics/Arm/Sve.cs | 114 +++++ .../ref/System.Runtime.Intrinsics.cs | 20 + .../GenerateHWIntrinsicTests_Arm.cs | 21 + .../HardwareIntrinsics/Arm/Shared/Helpers.cs | 296 ++++++++++++ .../_SveTernOpMaskedOpTestTemplate.template | 446 ++++++++++++++++++ 9 files changed, 1066 insertions(+), 9 deletions(-) create mode 100644 src/tests/JIT/HardwareIntrinsics/Arm/Shared/_SveTernOpMaskedOpTestTemplate.template diff --git a/src/coreclr/jit/hwintrinsiccodegenarm64.cpp b/src/coreclr/jit/hwintrinsiccodegenarm64.cpp index 38a480ae77ef5a..8845400a1d4226 100644 --- a/src/coreclr/jit/hwintrinsiccodegenarm64.cpp +++ b/src/coreclr/jit/hwintrinsiccodegenarm64.cpp @@ -840,15 +840,31 @@ void CodeGen::genHWIntrinsic(GenTreeHWIntrinsic* node) case 3: if (isRMW) { - if (targetReg != op1Reg) + if (HWIntrinsicInfo::IsExplicitMaskedOperation(intrin.id)) { - assert(targetReg != op2Reg); - assert(targetReg != op3Reg); + if (targetReg != op2Reg) + { + assert(targetReg != op1Reg); + assert(targetReg != op3Reg); - GetEmitter()->emitIns_Mov(INS_mov, emitTypeSize(node), targetReg, op1Reg, - /* canSkip */ true); + GetEmitter()->emitIns_Mov(INS_mov, emitTypeSize(node), targetReg, op2Reg, + /* canSkip */ true); + } + + GetEmitter()->emitIns_R_R_R(ins, emitSize, targetReg, op1Reg, op3Reg, opt); + } + else + { + if (targetReg != op1Reg) + { + assert(targetReg != op2Reg); + assert(targetReg != op3Reg); + + GetEmitter()->emitIns_Mov(INS_mov, emitTypeSize(node), targetReg, op1Reg, + /* canSkip */ true); + } + GetEmitter()->emitIns_R_R_R(ins, emitSize, targetReg, op2Reg, op3Reg, opt); } - GetEmitter()->emitIns_R_R_R(ins, emitSize, targetReg, op2Reg, op3Reg, opt); } else { diff --git a/src/coreclr/jit/hwintrinsiclistarm64sve.h b/src/coreclr/jit/hwintrinsiclistarm64sve.h index 600c1c7d9eaaaa..c4b6ff01677ef9 100644 --- a/src/coreclr/jit/hwintrinsiclistarm64sve.h +++ b/src/coreclr/jit/hwintrinsiclistarm64sve.h @@ -165,6 +165,7 @@ HARDWARE_INTRINSIC(Sve, PrefetchBytes, HARDWARE_INTRINSIC(Sve, PrefetchInt16, -1, 3, false, {INS_invalid, INS_invalid, INS_invalid, INS_sve_prfh, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_BaseTypeFromFirstArg|HW_Flag_HasImmediateOperand|HW_Flag_HasEnumOperand) HARDWARE_INTRINSIC(Sve, PrefetchInt32, -1, 3, false, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_prfw, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_BaseTypeFromFirstArg|HW_Flag_HasImmediateOperand|HW_Flag_HasEnumOperand) HARDWARE_INTRINSIC(Sve, PrefetchInt64, -1, 3, false, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_prfd, INS_invalid, INS_invalid}, HW_Category_MemoryLoad, HW_Flag_Scalable|HW_Flag_SpecialCodeGen|HW_Flag_ExplicitMaskedOperation|HW_Flag_LowMaskedOperation|HW_Flag_BaseTypeFromFirstArg|HW_Flag_HasImmediateOperand|HW_Flag_HasEnumOperand) +HARDWARE_INTRINSIC(Sve, ReverseBits, -1, -1, false, {INS_sve_rbit, INS_sve_rbit, INS_sve_rbit, INS_sve_rbit, INS_sve_rbit, INS_sve_rbit, INS_sve_rbit, INS_sve_rbit, INS_invalid, INS_invalid}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) HARDWARE_INTRINSIC(Sve, ReverseElement, -1, 1, true, {INS_sve_rev, INS_sve_rev, INS_sve_rev, INS_sve_rev, INS_sve_rev, INS_sve_rev, INS_sve_rev, INS_sve_rev, INS_sve_rev, INS_sve_rev}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_SpecialCodeGen) HARDWARE_INTRINSIC(Sve, ReverseElement16, -1, -1, false, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_revh, INS_sve_revh, INS_sve_revh, INS_sve_revh, INS_invalid, INS_invalid}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) HARDWARE_INTRINSIC(Sve, ReverseElement32, -1, -1, false, {INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_sve_revw, INS_sve_revw, INS_invalid, INS_invalid}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) @@ -184,6 +185,7 @@ HARDWARE_INTRINSIC(Sve, SignExtend32, HARDWARE_INTRINSIC(Sve, SignExtend8, -1, -1, false, {INS_invalid, INS_invalid, INS_sve_sxtb, INS_invalid, INS_sve_sxtb, INS_invalid, INS_sve_sxtb, INS_invalid, INS_invalid, INS_invalid}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_EmbeddedMaskedOperation|HW_Flag_LowMaskedOperation) HARDWARE_INTRINSIC(Sve, SignExtendWideningLower, -1, 1, true, {INS_sve_sunpklo, INS_invalid, INS_sve_sunpklo, INS_invalid, INS_sve_sunpklo, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg) HARDWARE_INTRINSIC(Sve, SignExtendWideningUpper, -1, 1, true, {INS_sve_sunpkhi, INS_invalid, INS_sve_sunpkhi, INS_invalid, INS_sve_sunpkhi, INS_invalid, INS_invalid, INS_invalid, INS_invalid, INS_invalid}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg) +HARDWARE_INTRINSIC(Sve, Splice, -1, 3, true, {INS_sve_splice, INS_sve_splice, INS_sve_splice, INS_sve_splice, INS_sve_splice, INS_sve_splice, INS_sve_splice, INS_sve_splice, INS_sve_splice, INS_sve_splice}, HW_Category_SIMD, HW_Flag_Scalable|HW_Flag_ExplicitMaskedOperation|HW_Flag_HasRMWSemantics|HW_Flag_LowMaskedOperation) HARDWARE_INTRINSIC(Sve, StoreAndZip, -1, 3, true, {INS_sve_st1b, INS_sve_st1b, INS_sve_st1h, INS_sve_st1h, INS_sve_st1w, INS_sve_st1w, INS_sve_st1d, INS_sve_st1d, INS_sve_st1w, INS_sve_st1d}, HW_Category_MemoryStore, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_ExplicitMaskedOperation|HW_Flag_SpecialImport|HW_Flag_SpecialCodeGen|HW_Flag_LowMaskedOperation) HARDWARE_INTRINSIC(Sve, StoreNarrowing, -1, 3, true, {INS_sve_st1b, INS_sve_st1b, INS_sve_st1h, INS_sve_st1h, INS_sve_st1w, INS_sve_st1w, INS_sve_st1d, INS_sve_st1d, INS_invalid, INS_invalid}, HW_Category_MemoryStore, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_ExplicitMaskedOperation|HW_Flag_SpecialImport|HW_Flag_SpecialCodeGen|HW_Flag_LowMaskedOperation) HARDWARE_INTRINSIC(Sve, StoreNonTemporal, -1, 3, true, {INS_sve_stnt1b, INS_sve_stnt1b, INS_sve_stnt1h, INS_sve_stnt1h, INS_sve_stnt1w, INS_sve_stnt1w, INS_sve_stnt1d, INS_sve_stnt1d, INS_sve_stnt1w, INS_sve_stnt1d}, HW_Category_MemoryStore, HW_Flag_Scalable|HW_Flag_BaseTypeFromFirstArg|HW_Flag_ExplicitMaskedOperation|HW_Flag_SpecialCodeGen|HW_Flag_LowMaskedOperation) diff --git a/src/coreclr/jit/lsraarm64.cpp b/src/coreclr/jit/lsraarm64.cpp index 0256b105a774bf..72c2a417d932cf 100644 --- a/src/coreclr/jit/lsraarm64.cpp +++ b/src/coreclr/jit/lsraarm64.cpp @@ -1514,6 +1514,7 @@ int LinearScan::BuildHWIntrinsic(GenTreeHWIntrinsic* intrinsicTree, int* pDstCou const bool isRMW = intrinsicTree->isRMWHWIntrinsic(compiler); bool tgtPrefOp1 = false; + bool op2IsTarget = (isRMW && HWIntrinsicInfo::IsExplicitMaskedOperation(intrin.id)); bool delayFreeMultiple = false; if (intrin.op1 != nullptr) { @@ -1568,7 +1569,8 @@ int LinearScan::BuildHWIntrinsic(GenTreeHWIntrinsic* intrinsicTree, int* pDstCou // If we have an RMW intrinsic or an intrinsic with simple move semantic between two SIMD registers, // we want to preference op1Reg to the target if op1 is not contained. - if (isRMW || simdRegToSimdRegMove) + + if ((isRMW || simdRegToSimdRegMove) && !HWIntrinsicInfo::IsExplicitMaskedOperation(intrin.id)) { tgtPrefOp1 = !intrin.op1->isContained(); } @@ -1617,6 +1619,12 @@ int LinearScan::BuildHWIntrinsic(GenTreeHWIntrinsic* intrinsicTree, int* pDstCou predMask = RBM_LOWMASK.GetPredicateRegSet(); } + if (HWIntrinsicInfo::IsExplicitMaskedOperation(intrin.id) && isRMW) + { + srcCount += BuildDelayFreeUses(intrin.op1, intrin.op2, predMask); + } + else + { srcCount += BuildOperandUses(intrin.op1, predMask); } } @@ -1978,6 +1986,18 @@ int LinearScan::BuildHWIntrinsic(GenTreeHWIntrinsic* intrinsicTree, int* pDstCou (argNum == lowVectorOperandNum) ? lowVectorCandidates : RBM_NONE); } } + else if (op2IsTarget) + { + if (!intrin.op2->isContained()) + { + tgtPrefUse = BuildUse(intrin.op2); + srcCount ++; + } + else + { + srcCount += BuildOperandUses(intrin.op2); + } + } else { switch (intrin.id) @@ -2021,12 +2041,20 @@ int LinearScan::BuildHWIntrinsic(GenTreeHWIntrinsic* intrinsicTree, int* pDstCou { SingleTypeRegSet candidates = lowVectorOperandNum == 3 ? lowVectorCandidates : RBM_NONE; - srcCount += isRMW ? BuildDelayFreeUses(intrin.op3, intrin.op1, candidates) - : BuildOperandUses(intrin.op3, candidates); + if (op2IsTarget) + { + srcCount += BuildDelayFreeUses(intrin.op3, intrin.op2, candidates); + } + else + { + srcCount += isRMW ? BuildDelayFreeUses(intrin.op3, intrin.op1, candidates) + : BuildOperandUses(intrin.op3, candidates); + } if (intrin.op4 != nullptr) { assert(lowVectorOperandNum != 4); + assert(!op2IsTarget); srcCount += isRMW ? BuildDelayFreeUses(intrin.op4, intrin.op1) : BuildOperandUses(intrin.op4); } } diff --git a/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Arm/Sve.PlatformNotSupported.cs b/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Arm/Sve.PlatformNotSupported.cs index 493afebaab801e..2b5baf0dff844c 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Arm/Sve.PlatformNotSupported.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Arm/Sve.PlatformNotSupported.cs @@ -4207,6 +4207,57 @@ internal Arm64() { } public static unsafe void PrefetchInt64(Vector mask, void* address, [ConstantExpected] SvePrefetchType prefetchType) { throw new PlatformNotSupportedException(); } + /// Reverse bits + + /// + /// svuint8_t svrbit[_u8]_m(svuint8_t inactive, svbool_t pg, svuint8_t op) + /// RBIT Ztied.B, Pg/M, Zop.B + /// + public static unsafe Vector ReverseBits(Vector value) { throw new PlatformNotSupportedException(); } + + /// + /// svint16_t svrbit[_s16]_m(svint16_t inactive, svbool_t pg, svint16_t op) + /// RBIT Ztied.H, Pg/M, Zop.H + /// + public static unsafe Vector ReverseBits(Vector value) { throw new PlatformNotSupportedException(); } + + /// + /// svint32_t svrbit[_s32]_m(svint32_t inactive, svbool_t pg, svint32_t op) + /// RBIT Ztied.S, Pg/M, Zop.S + /// + public static unsafe Vector ReverseBits(Vector value) { throw new PlatformNotSupportedException(); } + + /// + /// svint64_t svrbit[_s64]_m(svint64_t inactive, svbool_t pg, svint64_t op) + /// RBIT Ztied.D, Pg/M, Zop.D + /// + public static unsafe Vector ReverseBits(Vector value) { throw new PlatformNotSupportedException(); } + + /// + /// svint8_t svrbit[_s8]_m(svint8_t inactive, svbool_t pg, svint8_t op) + /// RBIT Ztied.B, Pg/M, Zop.B + /// + public static unsafe Vector ReverseBits(Vector value) { throw new PlatformNotSupportedException(); } + + /// + /// svuint16_t svrbit[_u16]_m(svuint16_t inactive, svbool_t pg, svuint16_t op) + /// RBIT Ztied.H, Pg/M, Zop.H + /// + public static unsafe Vector ReverseBits(Vector value) { throw new PlatformNotSupportedException(); } + + /// + /// svuint32_t svrbit[_u32]_m(svuint32_t inactive, svbool_t pg, svuint32_t op) + /// RBIT Ztied.S, Pg/M, Zop.S + /// + public static unsafe Vector ReverseBits(Vector value) { throw new PlatformNotSupportedException(); } + + /// + /// svuint64_t svrbit[_u64]_m(svuint64_t inactive, svbool_t pg, svuint64_t op) + /// RBIT Ztied.D, Pg/M, Zop.D + /// + public static unsafe Vector ReverseBits(Vector value) { throw new PlatformNotSupportedException(); } + + /// Reverse all elements /// @@ -5002,6 +5053,69 @@ internal Arm64() { } public static unsafe Vector SignExtend8(Vector value) { throw new PlatformNotSupportedException(); } + /// Splice two vectors under predicate control + + /// + /// svuint8_t svsplice[_u8](svbool_t pg, svuint8_t op1, svuint8_t op2) + /// SPLICE Ztied1.B, Pg, Ztied1.B, Zop2.B + /// + public static unsafe Vector Splice(Vector mask, Vector left, Vector right) { throw new PlatformNotSupportedException(); } + + /// + /// svfloat64_t svsplice[_f64](svbool_t pg, svfloat64_t op1, svfloat64_t op2) + /// SPLICE Ztied1.D, Pg, Ztied1.D, Zop2.D + /// + public static unsafe Vector Splice(Vector mask, Vector left, Vector right) { throw new PlatformNotSupportedException(); } + + /// + /// svint16_t svsplice[_s16](svbool_t pg, svint16_t op1, svint16_t op2) + /// SPLICE Ztied1.H, Pg, Ztied1.H, Zop2.H + /// + public static unsafe Vector Splice(Vector mask, Vector left, Vector right) { throw new PlatformNotSupportedException(); } + + /// + /// svint32_t svsplice[_s32](svbool_t pg, svint32_t op1, svint32_t op2) + /// SPLICE Ztied1.S, Pg, Ztied1.S, Zop2.S + /// + public static unsafe Vector Splice(Vector mask, Vector left, Vector right) { throw new PlatformNotSupportedException(); } + + /// + /// svint64_t svsplice[_s64](svbool_t pg, svint64_t op1, svint64_t op2) + /// SPLICE Ztied1.D, Pg, Ztied1.D, Zop2.D + /// + public static unsafe Vector Splice(Vector mask, Vector left, Vector right) { throw new PlatformNotSupportedException(); } + + /// + /// svint8_t svsplice[_s8](svbool_t pg, svint8_t op1, svint8_t op2) + /// SPLICE Ztied1.B, Pg, Ztied1.B, Zop2.B + /// + public static unsafe Vector Splice(Vector mask, Vector left, Vector right) { throw new PlatformNotSupportedException(); } + + /// + /// svfloat32_t svsplice[_f32](svbool_t pg, svfloat32_t op1, svfloat32_t op2) + /// SPLICE Ztied1.S, Pg, Ztied1.S, Zop2.S + /// + public static unsafe Vector Splice(Vector mask, Vector left, Vector right) { throw new PlatformNotSupportedException(); } + + /// + /// svuint16_t svsplice[_u16](svbool_t pg, svuint16_t op1, svuint16_t op2) + /// SPLICE Ztied1.H, Pg, Ztied1.H, Zop2.H + /// + public static unsafe Vector Splice(Vector mask, Vector left, Vector right) { throw new PlatformNotSupportedException(); } + + /// + /// svuint32_t svsplice[_u32](svbool_t pg, svuint32_t op1, svuint32_t op2) + /// SPLICE Ztied1.S, Pg, Ztied1.S, Zop2.S + /// + public static unsafe Vector Splice(Vector mask, Vector left, Vector right) { throw new PlatformNotSupportedException(); } + + /// + /// svuint64_t svsplice[_u64](svbool_t pg, svuint64_t op1, svuint64_t op2) + /// SPLICE Ztied1.D, Pg, Ztied1.D, Zop2.D + /// + public static unsafe Vector Splice(Vector mask, Vector left, Vector right) { throw new PlatformNotSupportedException(); } + + /// Non-truncating store /// diff --git a/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Arm/Sve.cs b/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Arm/Sve.cs index fba023cbb54c60..8d0d1d5eb2b119 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Arm/Sve.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Runtime/Intrinsics/Arm/Sve.cs @@ -4261,6 +4261,57 @@ internal Arm64() { } /// public static unsafe void PrefetchInt64(Vector mask, void* address, [ConstantExpected] SvePrefetchType prefetchType) => PrefetchInt64(mask, address, prefetchType); + /// Reverse bits + + /// + /// svuint8_t svrbit[_u8]_x(svbool_t pg, svuint8_t op) + /// RBIT Ztied.B, Pg/M, Ztied.B + /// + public static unsafe Vector ReverseBits(Vector value) => ReverseBits(value); + + /// + /// svint16_t svrbit[_s16]_m(svint16_t inactive, svbool_t pg, svint16_t op) + /// RBIT Ztied.H, Pg/M, Zop.H + /// + public static unsafe Vector ReverseBits(Vector value) => ReverseBits(value); + + /// + /// svint32_t svrbit[_s32]_m(svint32_t inactive, svbool_t pg, svint32_t op) + /// RBIT Ztied.S, Pg/M, Zop.S + /// + public static unsafe Vector ReverseBits(Vector value) => ReverseBits(value); + + /// + /// svint64_t svrbit[_s64]_m(svint64_t inactive, svbool_t pg, svint64_t op) + /// RBIT Ztied.D, Pg/M, Zop.D + /// + public static unsafe Vector ReverseBits(Vector value) => ReverseBits(value); + + /// + /// svint8_t svrbit[_s8]_m(svint8_t inactive, svbool_t pg, svint8_t op) + /// RBIT Ztied.B, Pg/M, Zop.B + /// + public static unsafe Vector ReverseBits(Vector value) => ReverseBits(value); + + /// + /// svuint16_t svrbit[_u16]_m(svuint16_t inactive, svbool_t pg, svuint16_t op) + /// RBIT Ztied.H, Pg/M, Zop.H + /// + public static unsafe Vector ReverseBits(Vector value) => ReverseBits(value); + + /// + /// svuint32_t svrbit[_u32]_m(svuint32_t inactive, svbool_t pg, svuint32_t op) + /// RBIT Ztied.S, Pg/M, Zop.S + /// + public static unsafe Vector ReverseBits(Vector value) => ReverseBits(value); + + /// + /// svuint64_t svrbit[_u64]_m(svuint64_t inactive, svbool_t pg, svuint64_t op) + /// RBIT Ztied.D, Pg/M, Zop.D + /// + public static unsafe Vector ReverseBits(Vector value) => ReverseBits(value); + + /// Reverse all elements /// @@ -5085,6 +5136,69 @@ internal Arm64() { } public static unsafe Vector SignExtendWideningUpper(Vector value) => SignExtendWideningUpper(value); + /// Splice two vectors under predicate control + + /// + /// svuint8_t svsplice[_u8](svbool_t pg, svuint8_t op1, svuint8_t op2) + /// SPLICE Ztied1.B, Pg, Ztied1.B, Zop2.B + /// + public static unsafe Vector Splice(Vector mask, Vector left, Vector right) => Splice(mask, left, right); + + /// + /// svfloat64_t svsplice[_f64](svbool_t pg, svfloat64_t op1, svfloat64_t op2) + /// SPLICE Ztied1.D, Pg, Ztied1.D, Zop2.D + /// + public static unsafe Vector Splice(Vector mask, Vector left, Vector right) => Splice(mask, left, right); + + /// + /// svint16_t svsplice[_s16](svbool_t pg, svint16_t op1, svint16_t op2) + /// SPLICE Ztied1.H, Pg, Ztied1.H, Zop2.H + /// + public static unsafe Vector Splice(Vector mask, Vector left, Vector right) => Splice(mask, left, right); + + /// + /// svint32_t svsplice[_s32](svbool_t pg, svint32_t op1, svint32_t op2) + /// SPLICE Ztied1.S, Pg, Ztied1.S, Zop2.S + /// + public static unsafe Vector Splice(Vector mask, Vector left, Vector right) => Splice(mask, left, right); + + /// + /// svint64_t svsplice[_s64](svbool_t pg, svint64_t op1, svint64_t op2) + /// SPLICE Ztied1.D, Pg, Ztied1.D, Zop2.D + /// + public static unsafe Vector Splice(Vector mask, Vector left, Vector right) => Splice(mask, left, right); + + /// + /// svint8_t svsplice[_s8](svbool_t pg, svint8_t op1, svint8_t op2) + /// SPLICE Ztied1.B, Pg, Ztied1.B, Zop2.B + /// + public static unsafe Vector Splice(Vector mask, Vector left, Vector right) => Splice(mask, left, right); + + /// + /// svfloat32_t svsplice[_f32](svbool_t pg, svfloat32_t op1, svfloat32_t op2) + /// SPLICE Ztied1.S, Pg, Ztied1.S, Zop2.S + /// + public static unsafe Vector Splice(Vector mask, Vector left, Vector right) => Splice(mask, left, right); + + /// + /// svuint16_t svsplice[_u16](svbool_t pg, svuint16_t op1, svuint16_t op2) + /// SPLICE Ztied1.H, Pg, Ztied1.H, Zop2.H + /// + public static unsafe Vector Splice(Vector mask, Vector left, Vector right) => Splice(mask, left, right); + + /// + /// svuint32_t svsplice[_u32](svbool_t pg, svuint32_t op1, svuint32_t op2) + /// SPLICE Ztied1.S, Pg, Ztied1.S, Zop2.S + /// + public static unsafe Vector Splice(Vector mask, Vector left, Vector right) => Splice(mask, left, right); + + /// + /// svuint64_t svsplice[_u64](svbool_t pg, svuint64_t op1, svuint64_t op2) + /// SPLICE Ztied1.D, Pg, Ztied1.D, Zop2.D + /// + public static unsafe Vector Splice(Vector mask, Vector left, Vector right) => Splice(mask, left, right); + + /// Non-truncating store /// diff --git a/src/libraries/System.Runtime.Intrinsics/ref/System.Runtime.Intrinsics.cs b/src/libraries/System.Runtime.Intrinsics/ref/System.Runtime.Intrinsics.cs index 3e85d1af8d5839..630e3f1fe77193 100644 --- a/src/libraries/System.Runtime.Intrinsics/ref/System.Runtime.Intrinsics.cs +++ b/src/libraries/System.Runtime.Intrinsics/ref/System.Runtime.Intrinsics.cs @@ -4793,6 +4793,15 @@ internal Arm64() { } public static unsafe void PrefetchInt32(System.Numerics.Vector mask, void* address, [ConstantExpected] SvePrefetchType prefetchType) { throw null; } public static unsafe void PrefetchInt64(System.Numerics.Vector mask, void* address, [ConstantExpected] SvePrefetchType prefetchType) { throw null; } + public static System.Numerics.Vector ReverseBits(System.Numerics.Vector value) { throw null; } + public static System.Numerics.Vector ReverseBits(System.Numerics.Vector value) { throw null; } + public static System.Numerics.Vector ReverseBits(System.Numerics.Vector value) { throw null; } + public static System.Numerics.Vector ReverseBits(System.Numerics.Vector value) { throw null; } + public static System.Numerics.Vector ReverseBits(System.Numerics.Vector value) { throw null; } + public static System.Numerics.Vector ReverseBits(System.Numerics.Vector value) { throw null; } + public static System.Numerics.Vector ReverseBits(System.Numerics.Vector value) { throw null; } + public static System.Numerics.Vector ReverseBits(System.Numerics.Vector value) { throw null; } + public static System.Numerics.Vector ReverseElement(System.Numerics.Vector value) { throw null; } public static System.Numerics.Vector ReverseElement(System.Numerics.Vector value) { throw null; } public static System.Numerics.Vector ReverseElement(System.Numerics.Vector value) { throw null; } @@ -4922,6 +4931,17 @@ internal Arm64() { } public static System.Numerics.Vector SignExtendWideningUpper(System.Numerics.Vector value) { throw null; } public static System.Numerics.Vector SignExtendWideningUpper(System.Numerics.Vector value) { throw null; } + public static System.Numerics.Vector Splice(System.Numerics.Vector mask, System.Numerics.Vector left, System.Numerics.Vector right) { throw null; } + public static System.Numerics.Vector Splice(System.Numerics.Vector mask, System.Numerics.Vector left, System.Numerics.Vector right) { throw null; } + public static System.Numerics.Vector Splice(System.Numerics.Vector mask, System.Numerics.Vector left, System.Numerics.Vector right) { throw null; } + public static System.Numerics.Vector Splice(System.Numerics.Vector mask, System.Numerics.Vector left, System.Numerics.Vector right) { throw null; } + public static System.Numerics.Vector Splice(System.Numerics.Vector mask, System.Numerics.Vector left, System.Numerics.Vector right) { throw null; } + public static System.Numerics.Vector Splice(System.Numerics.Vector mask, System.Numerics.Vector left, System.Numerics.Vector right) { throw null; } + public static System.Numerics.Vector Splice(System.Numerics.Vector mask, System.Numerics.Vector left, System.Numerics.Vector right) { throw null; } + public static System.Numerics.Vector Splice(System.Numerics.Vector mask, System.Numerics.Vector left, System.Numerics.Vector right) { throw null; } + public static System.Numerics.Vector Splice(System.Numerics.Vector mask, System.Numerics.Vector left, System.Numerics.Vector right) { throw null; } + public static System.Numerics.Vector Splice(System.Numerics.Vector mask, System.Numerics.Vector left, System.Numerics.Vector right) { throw null; } + public static unsafe void StoreAndZip(System.Numerics.Vector mask, byte* address, System.Numerics.Vector data) { throw null; } public static unsafe void StoreAndZip(System.Numerics.Vector mask, byte* address, (System.Numerics.Vector Value1, System.Numerics.Vector Value2) data) { throw null; } public static unsafe void StoreAndZip(System.Numerics.Vector mask, byte* address, (System.Numerics.Vector Value1, System.Numerics.Vector Value2, System.Numerics.Vector Value3) data) { throw null; } diff --git a/src/tests/Common/GenerateHWIntrinsicTests/GenerateHWIntrinsicTests_Arm.cs b/src/tests/Common/GenerateHWIntrinsicTests/GenerateHWIntrinsicTests_Arm.cs index 6a1b0391cdaff0..80203aa05dbd37 100644 --- a/src/tests/Common/GenerateHWIntrinsicTests/GenerateHWIntrinsicTests_Arm.cs +++ b/src/tests/Common/GenerateHWIntrinsicTests/GenerateHWIntrinsicTests_Arm.cs @@ -186,6 +186,7 @@ ("_SveTernOpTestTemplate.template", "SveVecTernOpTest.template", new Dictionary { ["TemplateName"] = "Simple", ["TemplateValidationLogic"] = SimpleVecOpTest_ValidationLogic, ["TemplateValidationLogicForCndSel"] = SimpleTernVecOpTest_ValidationLogicForCndSel }), ("_SveTernOpFirstArgTestTemplate.template", "SveVecTernOpFirstArgTest.template", new Dictionary { ["TemplateName"] = "Simple", ["TemplateValidationLogic"] = SimpleVecOpTest_ValidationLogic, ["TemplateValidationLogicForCndSel"] = SimpleTernVecOpTest_ValidationLogicForCndSel }), ("_SveImmTernOpTestTemplate.template", "SveVecImmTernOpTest.template", new Dictionary { ["TemplateName"] = "Simple", ["TemplateValidationLogic"] = SimpleVecOpTest_ValidationLogic, ["TemplateValidationLogicForCndSel"] = SimpleTernVecOpTest_ValidationLogicForCndSel }), + ("_SveTernOpMaskedOpTestTemplate.template", "SveVecTernOpMaskedTest.template", new Dictionary { ["TemplateName"] = "Simple", ["TemplateValidationLogic"] = SimpleVecOpTest_ValidationLogic, ["TemplateValidationLogicForCndSel"] = SimpleTernVecOpTest_ValidationLogicForCndSel }), ("_SveImmTernOpFirstArgTestTemplate.template", "SveVecImmTernOpFirstArgTest.template", new Dictionary { ["TemplateName"] = "Simple", ["TemplateValidationLogic"] = SimpleVecOpTest_ValidationLogic, ["TemplateValidationLogicForCndSel"] = SimpleTernVecOpTest_ValidationLogicForCndSel }), ("_SveImm2UnaryOpTestTemplate.template", "SveVecImm2UnOpTest.template", new Dictionary { ["TemplateName"] = "Imm", ["TemplateValidationLogic"] = SimpleVecOpTest_ValidationLogic }), ("_SveMinimalUnaryOpTestTemplate.template", "SveVecReduceUnOpTest.template", new Dictionary { ["TemplateName"] = "Simple", ["TemplateValidationLogic"] = VecReduceOpTest_ValidationLogic }), @@ -3901,6 +3902,26 @@ ("SveVecPairBinOpTest.template", new Dictionary { ["TestName"] = "SveTransposeOdd_ushort", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "TransposeOdd", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt16", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "UInt16", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "UInt16", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetUInt16()", ["NextValueOp2"] = "TestLibrary.Generator.GetUInt16()", ["ValidateEntry"] = "result[i] != left[index * 2 + 1] || result[i + 1] != right[index * 2 + 1]"}), ("SveVecPairBinOpTest.template", new Dictionary { ["TestName"] = "SveTransposeOdd_uint", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "TransposeOdd", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt32", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "UInt32", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "UInt32", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetUInt32()", ["NextValueOp2"] = "TestLibrary.Generator.GetUInt32()", ["ValidateEntry"] = "result[i] != left[index * 2 + 1] || result[i + 1] != right[index * 2 + 1]"}), ("SveVecPairBinOpTest.template", new Dictionary { ["TestName"] = "SveTransposeOdd_ulong", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "TransposeOdd", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt64", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "UInt64", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "UInt64", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetUInt64()", ["NextValueOp2"] = "TestLibrary.Generator.GetUInt64()", ["ValidateEntry"] = "result[i] != left[index * 2 + 1] || result[i + 1] != right[index * 2 + 1]"}), + + ("SveSimpleVecOpTest.template", new Dictionary { ["TestName"] = "SveReverseBits_byte", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "ReverseBits", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Byte", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Byte", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetByte()", ["ValidateIterResult"] = "result[i] != Helpers.ReverseElementBits(firstOp[i])", ["GetIterResult"] = "Helpers.ReverseElementBits(leftOp[i])"}), + ("SveSimpleVecOpTest.template", new Dictionary { ["TestName"] = "SveReverseBits_short", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "ReverseBits", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Int16", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Int16", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetInt16()", ["ValidateIterResult"] = "result[i] != Helpers.ReverseElementBits(firstOp[i])", ["GetIterResult"] = "Helpers.ReverseElementBits(leftOp[i])"}), + ("SveSimpleVecOpTest.template", new Dictionary { ["TestName"] = "SveReverseBits_int", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "ReverseBits", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Int32", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Int32", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetInt32()", ["ValidateIterResult"] = "result[i] != Helpers.ReverseElementBits(firstOp[i])", ["GetIterResult"] = "Helpers.ReverseElementBits(leftOp[i])"}), + ("SveSimpleVecOpTest.template", new Dictionary { ["TestName"] = "SveReverseBits_long", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "ReverseBits", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Int64", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Int64", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetInt64()", ["ValidateIterResult"] = "result[i] != Helpers.ReverseElementBits(firstOp[i])", ["GetIterResult"] = "Helpers.ReverseElementBits(leftOp[i])"}), + ("SveSimpleVecOpTest.template", new Dictionary { ["TestName"] = "SveReverseBits_sbyte", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "ReverseBits", ["RetVectorType"] = "Vector", ["RetBaseType"] = "SByte", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "SByte", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetSByte()", ["ValidateIterResult"] = "result[i] != Helpers.ReverseElementBits(firstOp[i])", ["GetIterResult"] = "Helpers.ReverseElementBits(leftOp[i])"}), + ("SveSimpleVecOpTest.template", new Dictionary { ["TestName"] = "SveReverseBits_ushort", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "ReverseBits", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt16", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "UInt16", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetUInt16()", ["ValidateIterResult"] = "result[i] != Helpers.ReverseElementBits(firstOp[i])", ["GetIterResult"] = "Helpers.ReverseElementBits(leftOp[i])"}), + ("SveSimpleVecOpTest.template", new Dictionary { ["TestName"] = "SveReverseBits_uint", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "ReverseBits", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt32", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "UInt32", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetUInt32()", ["ValidateIterResult"] = "result[i] != Helpers.ReverseElementBits(firstOp[i])", ["GetIterResult"] = "Helpers.ReverseElementBits(leftOp[i])"}), + ("SveSimpleVecOpTest.template", new Dictionary { ["TestName"] = "SveReverseBits_ulong", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "ReverseBits", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt64", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "UInt64", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetUInt64()", ["ValidateIterResult"] = "result[i] != Helpers.ReverseElementBits(firstOp[i])", ["GetIterResult"] = "Helpers.ReverseElementBits(leftOp[i])"}), + + ("SveVecTernOpMaskedTest.template", new Dictionary { ["TestName"] = "SveSplice_float", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "Splice", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Single", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Single", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "Single", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetSingle()", ["NextValueOp2"] = "TestLibrary.Generator.GetSingle()", ["ValidateIterResult"] = "result[i] != Helpers.Splice(first, second, maskArray, i)", ["GetIterResult"] = "Helpers.Splice(left, right, mask, i)", ["ConvertFunc"] = "BitConverter.SingleToInt32Bits"}), + ("SveVecTernOpMaskedTest.template", new Dictionary { ["TestName"] = "SveSplice_double", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "Splice", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Double", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Double", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "Double", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetDouble()", ["NextValueOp2"] = "TestLibrary.Generator.GetDouble()", ["ValidateIterResult"] = "result[i] != Helpers.Splice(first, second, maskArray, i)", ["GetIterResult"] = "Helpers.Splice(left, right, mask, i)", ["ConvertFunc"] = "BitConverter.DoubleToInt64Bits"}), + ("SveVecTernOpMaskedTest.template", new Dictionary { ["TestName"] = "SveSplice_sbyte", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "Splice", ["RetVectorType"] = "Vector", ["RetBaseType"] = "SByte", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "SByte", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "SByte", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetSByte()", ["NextValueOp2"] = "TestLibrary.Generator.GetSByte()", ["ValidateIterResult"] = "result[i] != Helpers.Splice(first, second, maskArray, i)", ["GetIterResult"] = "Helpers.Splice(left, right, mask, i)", ["ConvertFunc"] = ""}), + ("SveVecTernOpMaskedTest.template", new Dictionary { ["TestName"] = "SveSplice_short", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "Splice", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Int16", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Int16", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "Int16", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetInt16()", ["NextValueOp2"] = "TestLibrary.Generator.GetInt16()", ["ValidateIterResult"] = "result[i] != Helpers.Splice(first, second, maskArray, i)", ["GetIterResult"] = "Helpers.Splice(left, right, mask, i)", ["ConvertFunc"] = ""}), + ("SveVecTernOpMaskedTest.template", new Dictionary { ["TestName"] = "SveSplice_int", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "Splice", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Int32", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Int32", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "Int32", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetInt32()", ["NextValueOp2"] = "TestLibrary.Generator.GetInt32()", ["ValidateIterResult"] = "result[i] != Helpers.Splice(first, second, maskArray, i)", ["GetIterResult"] = "Helpers.Splice(left, right, mask, i)", ["ConvertFunc"] = ""}), + ("SveVecTernOpMaskedTest.template", new Dictionary { ["TestName"] = "SveSplice_long", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "Splice", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Int64", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Int64", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "Int64", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetInt64()", ["NextValueOp2"] = "TestLibrary.Generator.GetInt64()", ["ValidateIterResult"] = "result[i] != Helpers.Splice(first, second, maskArray, i)", ["GetIterResult"] = "Helpers.Splice(left, right, mask, i)", ["ConvertFunc"] = ""}), + ("SveVecTernOpMaskedTest.template", new Dictionary { ["TestName"] = "SveSplice_byte", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "Splice", ["RetVectorType"] = "Vector", ["RetBaseType"] = "Byte", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "Byte", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "Byte", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetByte()", ["NextValueOp2"] = "TestLibrary.Generator.GetByte()", ["ValidateIterResult"] = "result[i] != Helpers.Splice(first, second, maskArray, i)", ["GetIterResult"] = "Helpers.Splice(left, right, mask, i)", ["ConvertFunc"] = ""}), + ("SveVecTernOpMaskedTest.template", new Dictionary { ["TestName"] = "SveSplice_ushort", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "Splice", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt16", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "UInt16", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "UInt16", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetUInt16()", ["NextValueOp2"] = "TestLibrary.Generator.GetUInt16()", ["ValidateIterResult"] = "result[i] != Helpers.Splice(first, second, maskArray, i)", ["GetIterResult"] = "Helpers.Splice(left, right, mask, i)", ["ConvertFunc"] = ""}), + ("SveVecTernOpMaskedTest.template", new Dictionary { ["TestName"] = "SveSplice_uint", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "Splice", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt32", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "UInt32", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "UInt32", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetUInt32()", ["NextValueOp2"] = "TestLibrary.Generator.GetUInt32()", ["ValidateIterResult"] = "result[i] != Helpers.Splice(first, second, maskArray, i)", ["GetIterResult"] = "Helpers.Splice(left, right, mask, i)", ["ConvertFunc"] = ""}), + ("SveVecTernOpMaskedTest.template", new Dictionary { ["TestName"] = "SveSplice_ulong", ["Isa"] = "Sve", ["LoadIsa"] = "Sve", ["Method"] = "Splice", ["RetVectorType"] = "Vector", ["RetBaseType"] = "UInt64", ["Op1VectorType"] = "Vector", ["Op1BaseType"] = "UInt64", ["Op2VectorType"] = "Vector", ["Op2BaseType"] = "UInt64", ["LargestVectorSize"] = "64", ["NextValueOp1"] = "TestLibrary.Generator.GetUInt64()", ["NextValueOp2"] = "TestLibrary.Generator.GetUInt64()", ["ValidateIterResult"] = "result[i] != Helpers.Splice(first, second, maskArray, i)", ["GetIterResult"] = "Helpers.Splice(left, right, mask, i)", ["ConvertFunc"] = ""}), }; diff --git a/src/tests/JIT/HardwareIntrinsics/Arm/Shared/Helpers.cs b/src/tests/JIT/HardwareIntrinsics/Arm/Shared/Helpers.cs index f47938b15f927b..c0a2c0eda41278 100644 --- a/src/tests/JIT/HardwareIntrinsics/Arm/Shared/Helpers.cs +++ b/src/tests/JIT/HardwareIntrinsics/Arm/Shared/Helpers.cs @@ -323,6 +323,24 @@ public static byte ReverseElementBits(byte op1) return (byte)result; } + public static short ReverseElementBits(short op1) + { + short val = (short)op1; + short result = 0; + const int bitsize = sizeof(short) * 8; + const short cst_one = 1; + + for (int i = 0; i < bitsize; i++) + { + if ((val & (cst_one << i)) != 0) + { + result |= (short)(cst_one << (bitsize - 1 - i)); + } + } + + return (short)result; + } + public static int ReverseElementBits(int op1) { uint val = (uint)op1; @@ -377,6 +395,24 @@ public static sbyte ReverseElementBits(sbyte op1) return (sbyte)result; } + public static ushort ReverseElementBits(ushort op1) + { + ushort val = (ushort)op1; + ushort result = 0; + const int bitsize = sizeof(ushort) * 8; + const ushort cst_one = 1; + + for (int i = 0; i < bitsize; i++) + { + if ((val & (cst_one << i)) != 0) + { + result |= (ushort)(cst_one << (bitsize - 1 - i)); + } + } + + return (ushort)result; + } + public static uint ReverseElementBits(uint op1) { uint val = (uint)op1; @@ -6841,5 +6877,265 @@ public static ulong[] Compact(ulong[] op1, ulong[] op2) return result; } + public static Byte Splice(Byte[] first, Byte[] second, Byte[] maskArray, Int32 index) + { + int start = -1; + int end = -1; + + for(var i = 0; i < maskArray.Length; i++) + { + if (maskArray[i] != 0) + { + if (start == -1) + { + start = i; + } + end = i; + } + } + + if (start == -1) + { + return second[index]; + } + + var rangeSize = end - start + 1; + return (index < rangeSize) ? first[start + index] : second[index - rangeSize]; + } + + public static double Splice(double[] first, double[] second, double[] maskArray, Int32 index) + { + int start = -1; + int end = -1; + + for(var i = 0; i < maskArray.Length; i++) + { + if (Double.IsNaN(maskArray[i]) || maskArray[i] > 0.0d) + { + if (start == -1) + { + start = i; + } + end = i; + } + } + + if (start == -1) + { + return second[index]; + } + + var rangeSize = end - start + 1; + return (index < rangeSize) ? first[start + index] : second[index - rangeSize]; + } + + public static float Splice(float[] first, float[] second, float[] maskArray, Int32 index) + { + int start = -1; + int end = -1; + + for(var i = 0; i < maskArray.Length; i++) + { + if (maskArray[i] != 0.0f) + { + if (start == -1) + { + start = i; + } + end = i; + } + } + + if (start == -1) + { + return second[index]; + } + + var rangeSize = end - start + 1; + return (index < rangeSize) ? first[start + index] : second[index - rangeSize]; + } + + public static Int16 Splice(Int16[] first, Int16[] second, Int16[] maskArray, Int32 index) + { + int start = -1; + int end = -1; + + for(var i = 0; i < maskArray.Length; i++) + { + if (maskArray[i] != 0) + { + if (start == -1) + { + start = i; + } + end = i; + } + } + + if (start == -1) + { + return second[index]; + } + + var rangeSize = end - start + 1; + return (index < rangeSize) ? first[start + index] : second[index - rangeSize]; + } + + public static Int32 Splice(Int32[] first, Int32[] second, Int32[] maskArray, Int32 index) + { + int start = -1; + int end = -1; + + for(var i = 0; i < maskArray.Length; i++) + { + if (maskArray[i] != 0) + { + if (start == -1) + { + start = i; + } + end = i; + } + } + + if (start == -1) + { + return second[index]; + } + + var rangeSize = end - start + 1; + return (index < rangeSize) ? first[start + index] : second[index - rangeSize]; + } + + public static Int64 Splice(Int64[] first, Int64[] second, Int64[] maskArray, Int32 index) + { + int start = -1; + int end = -1; + + for(var i = 0; i < maskArray.Length; i++) + { + if (maskArray[i] != 0) + { + if (start == -1) + { + start = i; + } + end = i; + } + } + + if (start == -1) + { + return second[index]; + } + + var rangeSize = end - start + 1; + return (index < rangeSize) ? first[start + index] : second[index - rangeSize]; + } + + public static SByte Splice(SByte[] first, SByte[] second, SByte[] maskArray, Int32 index) + { + int start = -1; + int end = -1; + + for(var i = 0; i < maskArray.Length; i++) + { + if (maskArray[i] != 0) + { + if (start == -1) + { + start = i; + } + end = i; + } + } + + if (start == -1) + { + return second[index]; + } + + var rangeSize = end - start + 1; + return (index < rangeSize) ? first[start + index] : second[index - rangeSize]; + } + + public static UInt16 Splice(UInt16[] first, UInt16[] second, UInt16[] maskArray, Int32 index) + { + int start = -1; + int end = -1; + + for(var i = 0; i < maskArray.Length; i++) + { + if (maskArray[i] != 0) + { + if (start == -1) + { + start = i; + } + end = i; + } + } + + if (start == -1) + { + return second[index]; + } + + var rangeSize = end - start + 1; + return (index < rangeSize) ? first[start + index] : second[index - rangeSize]; + } + + public static UInt32 Splice(UInt32[] first, UInt32[] second, UInt32[] maskArray, Int32 index) + { + int start = -1; + int end = -1; + + for(var i = 0; i < maskArray.Length; i++) + { + if (maskArray[i] != 0) + { + if (start == -1) + { + start = i; + } + end = i; + } + } + + if (start == -1) + { + return second[index]; + } + + var rangeSize = end - start + 1; + return (index < rangeSize) ? first[start + index] : second[index - rangeSize]; + } + + public static ulong Splice(ulong[] first, ulong[] second, ulong[] maskArray, int index) + { + int start = -1; + int end = -1; + + for(var i = 0; i < maskArray.Length; i++) + { + if (maskArray[i] != 0) + { + if (start == -1) + { + start = i; + } + end = i; + } + } + + if (start == -1) + { + return second[index]; + } + + var rangeSize = end - start + 1; + return (index < rangeSize) ? first[start + index] : second[index - rangeSize]; + } + } } diff --git a/src/tests/JIT/HardwareIntrinsics/Arm/Shared/_SveTernOpMaskedOpTestTemplate.template b/src/tests/JIT/HardwareIntrinsics/Arm/Shared/_SveTernOpMaskedOpTestTemplate.template new file mode 100644 index 00000000000000..84935d765458b7 --- /dev/null +++ b/src/tests/JIT/HardwareIntrinsics/Arm/Shared/_SveTernOpMaskedOpTestTemplate.template @@ -0,0 +1,446 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +/****************************************************************************** + * This file is auto-generated from a template file by the GenerateTests.csx * + * script in tests\src\JIT\HardwareIntrinsics\Arm\Shared. In order to make * + * changes, please update the corresponding template and run according to the * + * directions listed in the file. * + ******************************************************************************/ + +using System; +using System.Numerics; +using System.Reflection; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; +using System.Runtime.Intrinsics; +using System.Runtime.Intrinsics.Arm; +using Xunit; + +namespace JIT.HardwareIntrinsics.Arm._{Isa} +{ + public static partial class Program + { + [Fact] + public static void {TestName}() + { + var test = new {Method}Test__{TestName}(); + + if (test.IsSupported) + { + // Validates basic functionality works, using Unsafe.Read + test.RunBasicScenario_UnsafeRead(); + + if ({Isa}.IsSupported) + { + // Validates basic functionality works, using Load + test.RunBasicScenario_Load(); + } + + // Validates calling via reflection works, using Unsafe.Read + test.RunReflectionScenario_UnsafeRead(); + + // Validates passing a local works, using Unsafe.Read + test.RunLclVarScenario_UnsafeRead(); + + // Validates passing an instance member of a class works + test.RunClassFldScenario(); + + // Validates passing the field of a local struct works + test.RunStructLclFldScenario(); + + // Validates passing an instance member of a struct works + test.RunStructFldScenario(); + + // Validates executing the test inside conditional, with op1 as falseValue + test.ConditionalSelect_Op1(); + + // Validates executing the test inside conditional, with op3 as falseValue + test.ConditionalSelect_FalseOp(); + + // Validates executing the test inside conditional, with op3 as zero + test.ConditionalSelect_ZeroOp(); + } + else + { + // Validates we throw on unsupported hardware + test.RunUnsupportedScenario(); + } + + if (!test.Succeeded) + { + throw new Exception("One or more scenarios did not complete as expected."); + } + } + } + + public sealed unsafe class {Method}Test__{TestName} + { + private struct DataTable + { + private byte[] inArray1; + private byte[] inArray2; + private byte[] outArray; + private byte[] maskArray; + + private GCHandle inHandle1; + private GCHandle inHandle2; + private GCHandle outHandle; + private GCHandle maskHandle; + + private ulong alignment; + + public DataTable({Op1BaseType}[] inArray1, {Op2BaseType}[] inArray2, {RetBaseType}[] outArray, {Op1BaseType}[] maskArray, int alignment) + { + int sizeOfinArray1 = inArray1.Length * Unsafe.SizeOf<{Op1BaseType}>(); + int sizeOfinArray2 = inArray2.Length * Unsafe.SizeOf<{Op2BaseType}>(); + int sizeOfoutArray = outArray.Length * Unsafe.SizeOf<{RetBaseType}>(); + int sizeOfmaskArray = maskArray.Length * Unsafe.SizeOf<{Op1BaseType}>(); + + if ((alignment != 64 && alignment != 16 && alignment != 8) || (alignment * 2) < sizeOfinArray1 || (alignment * 2) < sizeOfinArray2 || (alignment * 2) < sizeOfoutArray || (alignment * 2) < sizeOfmaskArray) + { + throw new ArgumentException("Invalid value of alignment"); + } + + this.inArray1 = new byte[alignment * 2]; + this.inArray2 = new byte[alignment * 2]; + this.outArray = new byte[alignment * 2]; + this.maskArray = new byte[alignment * 2]; + + this.inHandle1 = GCHandle.Alloc(this.inArray1, GCHandleType.Pinned); + this.inHandle2 = GCHandle.Alloc(this.inArray2, GCHandleType.Pinned); + this.outHandle = GCHandle.Alloc(this.outArray, GCHandleType.Pinned); + this.maskHandle = GCHandle.Alloc(this.maskArray, GCHandleType.Pinned); + + this.alignment = (ulong)alignment; + + Unsafe.CopyBlockUnaligned(ref Unsafe.AsRef(inArray1Ptr), ref Unsafe.As<{Op1BaseType}, byte>(ref inArray1[0]), (uint)sizeOfinArray1); + Unsafe.CopyBlockUnaligned(ref Unsafe.AsRef(inArray2Ptr), ref Unsafe.As<{Op2BaseType}, byte>(ref inArray2[0]), (uint)sizeOfinArray2); + Unsafe.CopyBlockUnaligned(ref Unsafe.AsRef(maskArrayPtr), ref Unsafe.As<{RetBaseType}, byte>(ref maskArray[0]), (uint)sizeOfmaskArray); + } + + public void* inArray1Ptr => Align((byte*)(inHandle1.AddrOfPinnedObject().ToPointer()), alignment); + public void* inArray2Ptr => Align((byte*)(inHandle2.AddrOfPinnedObject().ToPointer()), alignment); + public void* outArrayPtr => Align((byte*)(outHandle.AddrOfPinnedObject().ToPointer()), alignment); + public void* maskArrayPtr => Align((byte*)(maskHandle.AddrOfPinnedObject().ToPointer()), alignment); + + public void Dispose() + { + inHandle1.Free(); + inHandle2.Free(); + outHandle.Free(); + maskHandle.Free(); + } + + public static unsafe void* Align(byte* buffer, ulong expectedAlignment) + { + return (void*)(((ulong)buffer + expectedAlignment - 1) & ~(expectedAlignment - 1)); + } + } + + + private struct TestStruct + { + public Vector<{Op1BaseType}> _fld1; + public Vector<{Op2BaseType}> _fld2; + public Vector<{Op1BaseType}> _storeMask; + private GCHandle _outHandle; + + public static TestStruct Create() + { + var testStruct = new TestStruct(); + + for (var i = 0; i < Op1ElementCount; i++) { _data1[i] = {NextValueOp1}; } + Unsafe.CopyBlockUnaligned(ref Unsafe.As, byte>(ref testStruct._fld1), ref Unsafe.As<{Op1BaseType}, byte>(ref _data1[0]), (uint)Unsafe.SizeOf>()); + for (var i = 0; i < Op2ElementCount; i++) { _data2[i] = {NextValueOp2}; } + Unsafe.CopyBlockUnaligned(ref Unsafe.As, byte>(ref testStruct._fld2), ref Unsafe.As<{Op2BaseType}, byte>(ref _data2[0]), (uint)Unsafe.SizeOf>()); + for (var i = 0; i < MaskElementCount; i++) { _maskData[i] = ({Op1BaseType})(Helpers.getMask{Op1BaseType}()); } + Unsafe.CopyBlockUnaligned(ref Unsafe.As, byte>(ref testStruct._storeMask), ref Unsafe.As<{Op1BaseType}, byte>(ref _maskData[0]), (uint)Unsafe.SizeOf>()); + + ulong alignment = (ulong)LargestVectorSize; + byte[] _outArray = new byte[alignment * 2]; + testStruct._outHandle = GCHandle.Alloc(_outArray, GCHandleType.Pinned); + + return testStruct; + } + + public void* _outArrayPtr => DataTable.Align((byte*)(_outHandle.AddrOfPinnedObject().ToPointer()), (ulong)LargestVectorSize); + + public void Dispose() + { + _outHandle.Free(); + } + + public void RunStructFldScenario({Method}Test__{TestName} testClass) + { + var result = {Isa}.{Method}(_storeMask, _fld1, _fld2); + + testClass.ValidateResult(_fld1, _fld2, result, _storeMask); + } + } + + public void ConditionalSelect_Op1() + { + TestLibrary.TestFramework.BeginScenario("ConditionalSelect_Op1_mask"); + ConditionalSelectScenario(_mask, _fld1, _fld2, _fld2); + + TestLibrary.TestFramework.BeginScenario("ConditionalSelect_Op1_zero"); + ConditionalSelectScenario(Vector<{Op1BaseType}>.Zero, _fld1, _fld2, _fld2); + + TestLibrary.TestFramework.BeginScenario("ConditionalSelect_Op1_all"); + ConditionalSelectScenario(Vector<{Op1BaseType}>.AllBitsSet, _fld1, _fld2, _fld2); + } + + public void ConditionalSelect_FalseOp() + { + TestLibrary.TestFramework.BeginScenario(nameof(ConditionalSelect_FalseOp)); + ConditionalSelectScenario(_mask, _fld1, _fld2, _falseFld); + + TestLibrary.TestFramework.BeginScenario("ConditionalSelect_FalseOp_zero"); + ConditionalSelectScenario(Vector<{Op1BaseType}>.Zero, _fld1, _fld2, _falseFld); + + TestLibrary.TestFramework.BeginScenario("ConditionalSelect_FalseOp_all"); + ConditionalSelectScenario(Vector<{Op1BaseType}>.AllBitsSet, _fld1, _fld2, _falseFld); + } + + public void ConditionalSelect_ZeroOp() + { + TestLibrary.TestFramework.BeginScenario(nameof(ConditionalSelect_ZeroOp)); + ConditionalSelectScenario(_mask, _fld1, _fld2, Vector<{Op1BaseType}>.Zero); + + TestLibrary.TestFramework.BeginScenario("ConditionalSelect_ZeroOp_zero"); + ConditionalSelectScenario(Vector<{Op1BaseType}>.Zero, _fld1, _fld2, Vector<{Op1BaseType}>.Zero); + + TestLibrary.TestFramework.BeginScenario("ConditionalSelect_ZeroOp_all"); + ConditionalSelectScenario(Vector<{Op1BaseType}>.AllBitsSet, _fld1, _fld2, Vector<{Op1BaseType}>.Zero); + } + + [method: MethodImpl(MethodImplOptions.AggressiveInlining)] + private void ConditionalSelectScenario(Vector<{Op1BaseType}> mask, Vector<{Op1BaseType}> op1, Vector<{Op2BaseType}> op2, Vector<{Op1BaseType}> falseOp) + { + var result = Sve.ConditionalSelect(mask, {Isa}.{Method}(mask, op1, op2), falseOp); + + Unsafe.Write(_dataTable.outArrayPtr, result); + ValidateConditionalSelectResult(mask, op1, op2, falseOp, _dataTable.outArrayPtr); + } + + private static readonly int LargestVectorSize = 64; + + private static readonly int Op1ElementCount = Unsafe.SizeOf>() / sizeof({Op1BaseType}); + private static readonly int Op2ElementCount = Unsafe.SizeOf>() / sizeof({Op2BaseType}); + private static readonly int RetElementCount = Unsafe.SizeOf>() / sizeof({RetBaseType}); + private static readonly int MaskElementCount = Unsafe.SizeOf>() / sizeof({Op1BaseType}); + + private static {Op1BaseType}[] _data1 = new {Op1BaseType}[Op1ElementCount]; + private static {Op2BaseType}[] _data2 = new {Op2BaseType}[Op1ElementCount]; + private static {Op1BaseType}[] _maskData = new {Op1BaseType}[MaskElementCount]; + + private Vector<{Op1BaseType}> _fld1; + private Vector<{Op2BaseType}> _fld2; + private Vector<{Op1BaseType}> _mask; + private Vector<{Op1BaseType}> _falseFld; + + private DataTable _dataTable; + + public {Method}Test__{TestName}() + { + Succeeded = true; + + for (var i = 0; i < Op1ElementCount; i++) { _data1[i] = {NextValueOp1}; } + for (var i = 0; i < Op2ElementCount; i++) { _data2[i] = {NextValueOp2}; } + for (var i = 0; i < MaskElementCount; i++) { _maskData[i] = ({Op1BaseType})(Helpers.getMask{Op1BaseType}()); } + Unsafe.CopyBlockUnaligned(ref Unsafe.As, byte>(ref _fld1), ref Unsafe.As<{Op1BaseType}, byte>(ref _data1[0]), (uint)Unsafe.SizeOf>()); + Unsafe.CopyBlockUnaligned(ref Unsafe.As, byte>(ref _fld2), ref Unsafe.As<{Op2BaseType}, byte>(ref _data2[0]), (uint)Unsafe.SizeOf>()); + Unsafe.CopyBlockUnaligned(ref Unsafe.As, byte>(ref _mask), ref Unsafe.As<{Op1BaseType}, byte>(ref _maskData[0]), (uint)Unsafe.SizeOf>()); + Unsafe.CopyBlockUnaligned(ref Unsafe.As, byte>(ref _falseFld), ref Unsafe.As<{Op1BaseType}, byte>(ref _data1[0]), (uint)Unsafe.SizeOf>()); + + _dataTable = new DataTable(_data1, _data2, new {RetBaseType}[RetElementCount], _maskData, LargestVectorSize); + } + + public bool IsSupported => {Isa}.IsSupported; + + public bool Succeeded { get; set; } + + public void RunBasicScenario_UnsafeRead() + { + TestLibrary.TestFramework.BeginScenario(nameof(RunBasicScenario_UnsafeRead)); + + var result = {Isa}.{Method}(Unsafe.Read>(_dataTable.maskArrayPtr), Unsafe.Read>(_dataTable.inArray1Ptr), Unsafe.Read>(_dataTable.inArray2Ptr)); + + Unsafe.Write(_dataTable.outArrayPtr, (Vector<{RetBaseType}>)result); + ValidateResult(_dataTable.inArray1Ptr, _dataTable.inArray2Ptr, _dataTable.outArrayPtr, _dataTable.maskArrayPtr); + } + + public void RunBasicScenario_Load() + { + TestLibrary.TestFramework.BeginScenario(nameof(RunBasicScenario_Load)); + Vector<{Op1BaseType}> loadMask = {Isa}.CreateTrueMask{Op1BaseType}(SveMaskPattern.All); + + var result = {Isa}.{Method}({LoadIsa}.LoadVector(loadMask, ({Op1BaseType}*)(_dataTable.maskArrayPtr)), {LoadIsa}.LoadVector(loadMask, ({Op1BaseType}*)(_dataTable.inArray1Ptr)), {LoadIsa}.LoadVector(loadMask, ({Op2BaseType}*)(_dataTable.inArray2Ptr))); + + Unsafe.Write(_dataTable.outArrayPtr, (Vector<{RetBaseType}>)result); + ValidateResult(_dataTable.inArray1Ptr, _dataTable.inArray2Ptr, _dataTable.outArrayPtr, _dataTable.maskArrayPtr); + } + + public void RunReflectionScenario_UnsafeRead() + { + TestLibrary.TestFramework.BeginScenario(nameof(RunReflectionScenario_UnsafeRead)); + + var first = Unsafe.Read>(_dataTable.inArray1Ptr); + var second = Unsafe.Read>(_dataTable.inArray2Ptr); + var mask = Unsafe.Read>(_dataTable.maskArrayPtr); + var result = typeof({Isa}).GetMethod(nameof({Isa}.{Method}), new Type[] { typeof(Vector<{Op1BaseType}>), typeof(Vector<{Op1BaseType}>), typeof(Vector<{Op2BaseType}>) }) + .Invoke(null, new object[] { mask, + first, second + }); + Unsafe.Write(_dataTable.outArrayPtr, (Vector<{RetBaseType}>)result); + ValidateResult(_dataTable.inArray1Ptr, _dataTable.inArray2Ptr, _dataTable.outArrayPtr, _dataTable.maskArrayPtr); + } + + public void RunLclVarScenario_UnsafeRead() + { + TestLibrary.TestFramework.BeginScenario(nameof(RunLclVarScenario_UnsafeRead)); + + Vector<{Op1BaseType}> op1 = Unsafe.Read>(_dataTable.inArray1Ptr); + Vector<{Op2BaseType}> op2 = Unsafe.Read>(_dataTable.inArray2Ptr); + Vector<{RetBaseType}> storeMask = Unsafe.Read>(_dataTable.maskArrayPtr); + + var result = {Isa}.{Method}(storeMask, op1, op2); + + ValidateResult(op1, op2, result, storeMask); + } + + public void RunClassFldScenario() + { + TestLibrary.TestFramework.BeginScenario(nameof(RunClassFldScenario)); + + Vector<{Op1BaseType}> op1 = Unsafe.Read>(_dataTable.inArray1Ptr); + Vector<{Op2BaseType}> op2 = Unsafe.Read>(_dataTable.inArray2Ptr); + Vector<{RetBaseType}> storeMask = Unsafe.Read>(_dataTable.maskArrayPtr); + + var result = {Isa}.{Method}(storeMask, op1, op2); + + ValidateResult(op1, op2, result, storeMask); + } + + public void RunStructLclFldScenario() + { + TestLibrary.TestFramework.BeginScenario(nameof(RunStructLclFldScenario)); + + var test = TestStruct.Create(); + var result = {Isa}.{Method}(test._storeMask, test._fld1, test._fld2); + + ValidateResult(test._fld1, test._fld2, result, test._storeMask); + } + + public void RunStructFldScenario() + { + TestLibrary.TestFramework.BeginScenario(nameof(RunStructFldScenario)); + + var test = TestStruct.Create(); + test.RunStructFldScenario(this); + } + + public void RunUnsupportedScenario() + { + TestLibrary.TestFramework.BeginScenario(nameof(RunUnsupportedScenario)); + + bool succeeded = false; + + try + { + RunBasicScenario_UnsafeRead(); + } + catch (PlatformNotSupportedException) + { + succeeded = true; + } + + if (!succeeded) + { + Succeeded = false; + } + } + + private void ValidateConditionalSelectResult(Vector<{Op1BaseType}> maskOp, Vector<{Op1BaseType}> leftOp, Vector<{Op2BaseType}> rightOp, Vector<{Op1BaseType}> falseOp, void* output, [CallerMemberName] string method = "") + { + {Op1BaseType}[] mask = new {Op1BaseType}[Op1ElementCount]; + {Op1BaseType}[] left = new {Op1BaseType}[Op1ElementCount]; + {Op2BaseType}[] right = new {Op2BaseType}[Op2ElementCount]; + {Op1BaseType}[] falseVal = new {Op1BaseType}[Op1ElementCount]; + {RetBaseType}[] result = new {RetBaseType}[RetElementCount]; + + Unsafe.WriteUnaligned(ref Unsafe.As<{Op1BaseType}, byte>(ref mask[0]), maskOp); + Unsafe.WriteUnaligned(ref Unsafe.As<{Op1BaseType}, byte>(ref left[0]), leftOp); + Unsafe.WriteUnaligned(ref Unsafe.As<{Op2BaseType}, byte>(ref right[0]), rightOp); + Unsafe.WriteUnaligned(ref Unsafe.As<{Op1BaseType}, byte>(ref falseVal[0]), falseOp); + Unsafe.CopyBlockUnaligned(ref Unsafe.As<{Op1BaseType}, byte>(ref result[0]), ref Unsafe.AsRef(output), (uint)Unsafe.SizeOf>()); + + bool succeeded = true; + + {TemplateValidationLogicForCndSel} + + if (!succeeded) + { + TestLibrary.TestFramework.LogInformation($"{nameof(Sve)}.{nameof({Isa}.{Method})}<{Op1BaseType}>(Vector<{Op1BaseType}>, Vector<{Op1BaseType}>, Vector<{Op2BaseType}>): {method} failed:"); + TestLibrary.TestFramework.LogInformation($" mask: ({string.Join(", ", mask)})"); + TestLibrary.TestFramework.LogInformation($" left: ({string.Join(", ", left)})"); + TestLibrary.TestFramework.LogInformation($" right: ({string.Join(", ", right)})"); + TestLibrary.TestFramework.LogInformation($" falseOp: ({string.Join(", ", falseVal)})"); + TestLibrary.TestFramework.LogInformation($" result: ({string.Join(", ", result)})"); + TestLibrary.TestFramework.LogInformation(string.Empty); + + Succeeded = false; + } + } + + private void ValidateResult(Vector<{Op1BaseType}> op1, Vector<{Op2BaseType}> op2, Vector<{RetBaseType}> result, Vector<{Op1BaseType}> storeMask, [CallerMemberName] string method = "") + { + {Op1BaseType}[] inArray1 = new {Op1BaseType}[Op1ElementCount]; + {Op2BaseType}[] inArray2 = new {Op2BaseType}[Op2ElementCount]; + {RetBaseType}[] outArray = new {RetBaseType}[RetElementCount]; + {Op1BaseType}[] maskArray = new {Op1BaseType}[MaskElementCount]; + + Unsafe.WriteUnaligned(ref Unsafe.As<{Op1BaseType}, byte>(ref inArray1[0]), op1); + Unsafe.WriteUnaligned(ref Unsafe.As<{Op2BaseType}, byte>(ref inArray2[0]), op2); + Unsafe.WriteUnaligned(ref Unsafe.As<{Op1BaseType}, byte>(ref maskArray[0]), storeMask); + Unsafe.WriteUnaligned(ref Unsafe.As<{RetBaseType}, byte>(ref outArray[0]), result); + + ValidateResult(inArray1, inArray2, outArray, maskArray, method); + } + + private void ValidateResult(void* op1, void* op2, void* result, void* mask, [CallerMemberName] string method = "") + { + {Op1BaseType}[] inArray1 = new {Op1BaseType}[Op1ElementCount]; + {Op2BaseType}[] inArray2 = new {Op2BaseType}[Op2ElementCount]; + {RetBaseType}[] outArray = new {RetBaseType}[RetElementCount]; + {Op1BaseType}[] maskArray = new {Op1BaseType}[MaskElementCount]; + + Unsafe.CopyBlockUnaligned(ref Unsafe.As<{Op1BaseType}, byte>(ref inArray1[0]), ref Unsafe.AsRef(op1), (uint)Unsafe.SizeOf>()); + Unsafe.CopyBlockUnaligned(ref Unsafe.As<{Op2BaseType}, byte>(ref inArray2[0]), ref Unsafe.AsRef(op2), (uint)Unsafe.SizeOf>()); + Unsafe.CopyBlockUnaligned(ref Unsafe.As<{RetBaseType}, byte>(ref outArray[0]), ref Unsafe.AsRef(result), (uint)Unsafe.SizeOf>()); + Unsafe.CopyBlockUnaligned(ref Unsafe.As<{Op1BaseType}, byte>(ref maskArray[0]), ref Unsafe.AsRef(mask), (uint)Unsafe.SizeOf>()); + + ValidateResult(inArray1, inArray2, outArray, maskArray, method); + } + + private void ValidateResult({Op1BaseType}[] first, {Op2BaseType}[] second, {RetBaseType}[] result, {Op1BaseType}[] maskArray, [CallerMemberName] string method = "") + { + bool succeeded = true; + + {TemplateValidationLogic} + + if (!succeeded) + { + TestLibrary.TestFramework.LogInformation($"{nameof({Isa})}.{nameof({Isa}.{Method})}<{RetBaseType}>(Vector<{Op1BaseType}>, Vector<{Op1BaseType}>, Vector<{Op2BaseType}>): {method} failed:"); + TestLibrary.TestFramework.LogInformation($" first: ({string.Join(", ", first)})"); + TestLibrary.TestFramework.LogInformation($" second: ({string.Join(", ", second)})"); + TestLibrary.TestFramework.LogInformation($" mask: ({string.Join(", ", maskArray)})"); + TestLibrary.TestFramework.LogInformation($" result: ({string.Join(", ", result)})"); + TestLibrary.TestFramework.LogInformation(string.Empty); + + Succeeded = false; + } + } + } +}