Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Arm64/Sve: Rewrite how ConditionalSelect wraps the embedded mask operations #104248

Merged
merged 30 commits into from
Jul 3, 2024
Merged

Arm64/Sve: Rewrite how ConditionalSelect wraps the embedded mask operations #104248

Show file tree
Hide file tree
Changes from 16 commits
Commits
Show all changes
30 commits
Select commit Hold shift + click to select a range
d832b42
All current embedded scenarios work
kunalspathak Jun 29, 2024
90df0aa
Add test coverage for _SveUnaryOpTestTemplate
kunalspathak Jun 29, 2024
310e93d
wip: review comments
kunalspathak Jul 2, 2024
6a77306
Merge remote-tracking branch 'origin/main' into cndsel
kunalspathak Jul 2, 2024
2ecd169
fix the assert for GatherPrefetch
kunalspathak Jul 3, 2024
b4a6328
review commetns
kunalspathak Jul 3, 2024
c58ada2
jit format
kunalspathak Jul 3, 2024
c5ee94e
Add coverage for _SveBinaryOpTestTemplate.template
kunalspathak Jul 3, 2024
02f5cf5
review comments
kunalspathak Jul 3, 2024
dd8f935
Add test coverage in _SveTernOpTestTemplate.template
kunalspathak Jul 3, 2024
89040a4
Add for _SveUnaryOpDifferentRetTypeTestTemplate
kunalspathak Jul 3, 2024
7a0a854
Add _SveBinaryMaskOpTestTemplate.template
kunalspathak Jul 3, 2024
e5e45c0
minor feedback
kunalspathak Jul 3, 2024
15ca5a0
coverge for _SveBinaryOpDifferentTypesTestTemplate
kunalspathak Jul 3, 2024
6d5f6ef
Add for _SveBinaryOpDifferentTypesTestTemplate.template
kunalspathak Jul 3, 2024
2626609
Add for _SveImmBinaryOpTestTemplate.template
kunalspathak Jul 3, 2024
a1bfc5e
Add for _SveImmTernOpFirstArgTestTemplate.template
kunalspathak Jul 3, 2024
ccb3a6f
Add for _SveImmTernOpTestTemplate.template
kunalspathak Jul 3, 2024
4344d96
Add for _SveImmUnaryOpTestTemplate.template
kunalspathak Jul 3, 2024
d38199d
Add for _SveTernOpFirstArgTestTemplate.template
kunalspathak Jul 3, 2024
3655ecc
Add for _SveTernOpMaskedOpTestTemplate.template
kunalspathak Jul 3, 2024
9c6ea01
Add for SveLoadNonFaultingUnOpTest.template
kunalspathak Jul 3, 2024
e7660f0
Add for SveGatherVectorVectorBases.template
kunalspathak Jul 3, 2024
f5b34ff
Add for SveGatherVectorIndices.template
kunalspathak Jul 3, 2024
d49611d
Add for SveGatherVectorByteOffsets.template
kunalspathak Jul 3, 2024
492ab8e
fix the typos
kunalspathak Jul 3, 2024
8584e9e
minor test feedback
kunalspathak Jul 3, 2024
c5bd12e
jit format
kunalspathak Jul 3, 2024
c5a5709
revert fix for GatherPrefetch*
kunalspathak Jul 3, 2024
4a365f6
missed a change
kunalspathak Jul 3, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 4 additions & 4 deletions src/coreclr/jit/lowerarmarch.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1349,6 +1349,10 @@ GenTree* Lowering::LowerHWIntrinsic(GenTreeHWIntrinsic* node)
DISPTREERANGE(BlockRange(), condSelNode);
JITDUMP("\n");
}
else
{
assert(!"Embedded mask operation is not used anywhere.");
}
}

ContainCheckHWIntrinsic(node);
Expand Down Expand Up @@ -3507,7 +3511,6 @@ GenTree* Lowering::LowerHWIntrinsicCndSel(GenTreeHWIntrinsic* cndSelNode)
// Handle cases where there is a nested ConditionalSelect for
// `trueValue`
GenTreeHWIntrinsic* nestedCndSel = op2->AsHWIntrinsic();
const HWIntrinsic intrinEmbMask(nestedCndSel);
GenTree* nestedOp1 = nestedCndSel->Op(1);
assert(varTypeIsMask(nestedOp1));

Expand All @@ -3516,8 +3519,6 @@ GenTree* Lowering::LowerHWIntrinsicCndSel(GenTreeHWIntrinsic* cndSelNode)
GenTree* nestedOp2 = nestedCndSel->Op(2);
GenTree* nestedOp3 = nestedCndSel->Op(3);

assert(nestedOp2->isEmbeddedMaskingCompatibleHWIntrinsic());

JITDUMP("lowering ConditionalSelect HWIntrinisic (before):\n");
DISPTREERANGE(BlockRange(), cndSelNode);
JITDUMP("\n");
Expand All @@ -3538,7 +3539,6 @@ GenTree* Lowering::LowerHWIntrinsicCndSel(GenTreeHWIntrinsic* cndSelNode)
}

nestedOp1->SetUnusedValue();
nestedCndSel->SetUnusedValue();
BlockRange().Remove(nestedCndSel);

JITDUMP("lowering ConditionalSelect HWIntrinisic (after):\n");
Expand Down
114 changes: 97 additions & 17 deletions src/tests/JIT/HardwareIntrinsics/Arm/Shared/SveGatherVectorByteOffsets.template
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -363,35 +363,62 @@ namespace JIT.HardwareIntrinsics.Arm

public void ConditionalSelect_FalseOp()
{
TestLibrary.TestFramework.BeginScenario(nameof(ConditionalSelect_FalseOp));
ConditionalSelectScenario(_mask, _fld1, _fld2, _fld3, _falseFld);
TestLibrary.TestFramework.BeginScenario("ConditionalSelect_FalseOp_mask - operation in TrueValue");
ConditionalSelectScenario_TrueValue(_mask, _fld1, _fld2, _fld3, _falseFld);

TestLibrary.TestFramework.BeginScenario("ConditionalSelect_FalseOp_zero");
ConditionalSelectScenario({Op1VectorType}<{Op1BaseType}>.Zero, _fld1, _fld2, _fld3, _falseFld);
TestLibrary.TestFramework.BeginScenario("ConditionalSelect_FalseOp_zero - operation in TrueValue");
ConditionalSelectScenario_TrueValue({Op1VectorType}<{Op1BaseType}>.Zero, _fld1, _fld2, _fld3, _falseFld);

TestLibrary.TestFramework.BeginScenario("ConditionalSelect_FalseOp_all");
ConditionalSelectScenario({Op1VectorType}<{Op1BaseType}>.AllBitsSet, _fld1, _fld2, _fld3, _falseFld);
TestLibrary.TestFramework.BeginScenario("ConditionalSelect_FalseOp_all - operation in TrueValue");
ConditionalSelectScenario_TrueValue({Op1VectorType}<{Op1BaseType}>.AllBitsSet, _fld1, _fld2, _fld3, _falseFld);

TestLibrary.TestFramework.BeginScenario("ConditionalSelect_FalseOp_mask - operation in FalseValue");
ConditionalSelectScenario_FalseValue(_mask, _fld1, _fld2, _fld3, _falseFld);

TestLibrary.TestFramework.BeginScenario("ConditionalSelect_FalseOp_zero - operation in FalseValue");
ConditionalSelectScenario_FalseValue({Op1VectorType}<{Op1BaseType}>.Zero, _fld1, _fld2, _fld3, _falseFld);

TestLibrary.TestFramework.BeginScenario("ConditionalSelect_FalseOp_all - operation in FalseValue");
ConditionalSelectScenario_FalseValue({Op1VectorType}<{Op1BaseType}>.AllBitsSet, _fld1, _fld2, _fld3, _falseFld);
}

public void ConditionalSelect_ZeroOp()
{
TestLibrary.TestFramework.BeginScenario(nameof(ConditionalSelect_ZeroOp));
ConditionalSelectScenario(_mask, _fld1, _fld2, _fld3, {Op1VectorType}<{RetBaseType}>.Zero);
TestLibrary.TestFramework.BeginScenario("ConditionalSelect_ZeroOp_mask - operation in TrueValue");
ConditionalSelectScenario_TrueValue(_mask, _fld1, _fld2, _fld3, {Op1VectorType}<{RetBaseType}>.Zero);

TestLibrary.TestFramework.BeginScenario("ConditionalSelect_ZeroOp_zero - operation in TrueValue");
ConditionalSelectScenario_TrueValue({Op1VectorType}<{Op1BaseType}>.Zero, _fld1, _fld2, _fld3, {Op1VectorType}<{Op1BaseType}>.Zero);

TestLibrary.TestFramework.BeginScenario("ConditionalSelect_ZeroOp_zero");
ConditionalSelectScenario({Op1VectorType}<{Op1BaseType}>.Zero, _fld1, _fld2, _fld3, {Op1VectorType}<{Op1BaseType}>.Zero);
TestLibrary.TestFramework.BeginScenario("ConditionalSelect_ZeroOp_all - operation in TrueValue");
ConditionalSelectScenario_TrueValue({Op1VectorType}<{Op1BaseType}>.AllBitsSet, _fld1, _fld2, _fld3, {Op1VectorType}<{Op1BaseType}>.Zero);

TestLibrary.TestFramework.BeginScenario("ConditionalSelect_ZeroOp_all");
ConditionalSelectScenario({Op1VectorType}<{Op1BaseType}>.AllBitsSet, _fld1, _fld2, _fld3, {Op1VectorType}<{Op1BaseType}>.Zero);
TestLibrary.TestFramework.BeginScenario("ConditionalSelect_ZeroOp_mask - operation in FalseValue");
ConditionalSelectScenario_FalseValue(_mask, _fld1, _fld2, _fld3, {Op1VectorType}<{RetBaseType}>.Zero);

TestLibrary.TestFramework.BeginScenario("ConditionalSelect_ZeroOp_zero - operation in FalseValue");
ConditionalSelectScenario_FalseValue({Op1VectorType}<{Op1BaseType}>.Zero, _fld1, _fld2, _fld3, {Op1VectorType}<{Op1BaseType}>.Zero);

TestLibrary.TestFramework.BeginScenario("ConditionalSelect_ZeroOp_all - operation in FalseValue");
ConditionalSelectScenario_FalseValue({Op1VectorType}<{Op1BaseType}>.AllBitsSet, _fld1, _fld2, _fld3, {Op1VectorType}<{Op1BaseType}>.Zero);
}

[method: MethodImpl(MethodImplOptions.AggressiveInlining)]
private void ConditionalSelectScenario({RetVectorType}<{RetBaseType}> mask, {Op1VectorType}<{Op1BaseType}> op1, {Op2BaseType}* op2, {Op3VectorType}<{Op3BaseType}> op3, {Op1VectorType}<{Op1BaseType}> falseOp)
private void ConditionalSelectScenario_TrueValue({RetVectorType}<{RetBaseType}> mask, {Op1VectorType}<{Op1BaseType}> op1, {Op2BaseType}* op2, {Op3VectorType}<{Op3BaseType}> op3, {Op1VectorType}<{Op1BaseType}> falseOp)
{
var result = Sve.ConditionalSelect(mask, {Isa}.{Method}(op1, op2, op3), falseOp);

Unsafe.Write(_dataTable.outArrayPtr, result);
ValidateConditionalSelectResult(mask, op1, op2, op3, falseOp, _dataTable.outArrayPtr);
ValidateConditionalSelectResult_TrueValue(mask, op1, op2, op3, falseOp, _dataTable.outArrayPtr);
}

[method: MethodImpl(MethodImplOptions.AggressiveInlining)]
private void ConditionalSelectScenario_FalseValue({RetVectorType}<{RetBaseType}> mask, {Op1VectorType}<{Op1BaseType}> op1, {Op2BaseType}* op2, {Op3VectorType}<{Op3BaseType}> op3, {Op1VectorType}<{Op1BaseType}> trueOp)
{
var result = Sve.ConditionalSelect(mask, trueOp, {Isa}.{Method}(op1, op2, op3));

Unsafe.Write(_dataTable.outArrayPtr, result);
ValidateConditionalSelectResult_TrueValue(mask, op1, op2, op3, trueOp, _dataTable.outArrayPtr);
}

public void RunUnsupportedScenario()
Expand Down Expand Up @@ -524,7 +551,7 @@ namespace JIT.HardwareIntrinsics.Arm
}
}

private void ValidateConditionalSelectResult({RetVectorType}<{RetBaseType}> maskOp, {Op1VectorType}<{Op1BaseType}> op1, {Op2BaseType}* op2, {Op3VectorType}<{Op3BaseType}> op3, {RetVectorType}<{RetBaseType}> falseOp, void* result, [CallerMemberName] string method = "")
private void ValidateConditionalSelectResult_TrueValue({RetVectorType}<{RetBaseType}> maskOp, {Op1VectorType}<{Op1BaseType}> op1, {Op2BaseType}* op2, {Op3VectorType}<{Op3BaseType}> op3, {RetVectorType}<{RetBaseType}> falseOp, void* result, [CallerMemberName] string method = "")
{
{RetBaseType}[] maskArray = new {RetBaseType}[RetElementCount];
{Op1BaseType}[] op1Array = new {Op1BaseType}[Op1ElementCount];
Expand All @@ -540,10 +567,10 @@ namespace JIT.HardwareIntrinsics.Arm
Unsafe.WriteUnaligned(ref Unsafe.As<{RetBaseType}, byte>(ref falseValArray[0]), falseOp);
Unsafe.CopyBlockUnaligned(ref Unsafe.As<{RetBaseType}, byte>(ref resultArray[0]), ref Unsafe.AsRef<byte>(result), (uint)Unsafe.SizeOf<{RetVectorType}<{RetBaseType}>>());

ValidateConditionalSelectResult(maskArray, op1Array, op2Array, op3Array, falseValArray, resultArray, method);
ValidateConditionalSelectResult_TrueValue(maskArray, op1Array, op2Array, op3Array, falseValArray, resultArray, method);
}

private void ValidateConditionalSelectResult({RetBaseType}[] maskOp, {Op1BaseType}[] firstOp, byte[] secondOp, {Op3BaseType}[] thirdOp, {RetBaseType}[] falseOp, {RetBaseType}[] result, [CallerMemberName] string method = "")
private void ValidateConditionalSelectResult_TrueValue({RetBaseType}[] maskOp, {Op1BaseType}[] firstOp, byte[] secondOp, {Op3BaseType}[] thirdOp, {RetBaseType}[] falseOp, {RetBaseType}[] result, [CallerMemberName] string method = "")
{
bool succeeded = true;

Expand Down Expand Up @@ -573,5 +600,58 @@ namespace JIT.HardwareIntrinsics.Arm
Succeeded = false;
}
}

private void ValidateConditionalSelectResult_FalseValue({RetVectorType}<{RetBaseType}> maskOp, {Op1VectorType}<{Op1BaseType}> op1, {Op2BaseType}* op2, {Op3VectorType}<{Op3BaseType}> op3, {RetVectorType}<{RetBaseType}> trueOp, void* result, [CallerMemberName] string method = "")
{
{RetBaseType}[] maskArray = new {RetBaseType}[RetElementCount];
{Op1BaseType}[] op1Array = new {Op1BaseType}[Op1ElementCount];
byte[] op2Array = new byte[Unsafe.SizeOf<{Op2BaseType}>() * Op2ElementCount];
{Op3BaseType}[] op3Array = new {Op3BaseType}[Op3ElementCount];
{RetBaseType}[] trueValArray = new {RetBaseType}[RetElementCount];
{RetBaseType}[] resultArray = new {RetBaseType}[RetElementCount];

Unsafe.WriteUnaligned(ref Unsafe.As<{RetBaseType}, byte>(ref maskArray[0]), maskOp);
Unsafe.WriteUnaligned(ref Unsafe.As<{Op1BaseType}, byte>(ref op1Array[0]), op1);
Unsafe.CopyBlockUnaligned(ref Unsafe.As<byte, byte>(ref op2Array[0]), ref Unsafe.AsRef<byte>(op2), (uint)(Unsafe.SizeOf<{Op2BaseType}>() * Op2ElementCount));
Unsafe.WriteUnaligned(ref Unsafe.As<{Op3BaseType}, byte>(ref op3Array[0]), op3);
Unsafe.WriteUnaligned(ref Unsafe.As<{RetBaseType}, byte>(ref trueValArray[0]), trueOp);
Unsafe.CopyBlockUnaligned(ref Unsafe.As<{RetBaseType}, byte>(ref resultArray[0]), ref Unsafe.AsRef<byte>(result), (uint)Unsafe.SizeOf<{RetVectorType}<{RetBaseType}>>());

ValidateConditionalSelectResult_FalseValue(maskArray, op1Array, op2Array, op3Array, trueValArray, resultArray, method);
}

private void ValidateConditionalSelectResult_FalseValue({RetBaseType}[] maskOp, {Op1BaseType}[] firstOp, byte[] secondOp, {Op3BaseType}[] thirdOp, {RetBaseType}[] trueOp, {RetBaseType}[] result, [CallerMemberName] string method = "")
{
bool succeeded = true;

for (var i = 0; i < RetElementCount; i++)
{
{RetBaseType} element = Helpers.Load{RetBaseType}FromByteArray(secondOp, thirdOp[i]);
{RetBaseType} gatherResult = ({RetBaseType})(firstOp[i] == 0 ? 0 : element);
{RetBaseType} iterResult = (maskOp[i] != 0) ? trueOp[i] : gatherResult;
if (maskOp[i] != 0)
{
if (iterResult != result[i])
{
succeeded = false;
break;
}
}
}

if (!succeeded)
{
TestLibrary.TestFramework.LogInformation($"{nameof({Isa})}.{nameof({Isa}.{Method})}<{RetBaseType}>({Op1BaseType}, {Op2BaseType}, {Op3BaseType}): {method} failed:");
TestLibrary.TestFramework.LogInformation($" maskOp: ({string.Join(", ", maskOp)})");
TestLibrary.TestFramework.LogInformation($" firstOp: ({string.Join(", ", firstOp)})");
TestLibrary.TestFramework.LogInformation($" secondOp: ({string.Join(", ", secondOp)})");
TestLibrary.TestFramework.LogInformation($" thirdOp: ({string.Join(", ", thirdOp)})");
TestLibrary.TestFramework.LogInformation($" trueOp: ({string.Join(", ", trueOp)})");
TestLibrary.TestFramework.LogInformation($" result: ({string.Join(", ", result)})");
TestLibrary.TestFramework.LogInformation(string.Empty);

Succeeded = false;
}
}
}
}
Loading
Loading