Skip to content

Commit 62baf21

Browse files
sdesmalen-armsdesmalen-arm
authored
[AArch64] Check for streaming mode in HasSME* features. (llvm#96302)
This also fixes up some asserts in copyPhysReg, loadRegFromStackSlot and storeRegToStackSlot.
1 parent 2ae0905 commit 62baf21

File tree

111 files changed

+186
-172
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

111 files changed

+186
-172
lines changed

‎llvm/lib/Target/AArch64/AArch64ISelLowering.cpp‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -423,7 +423,7 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
423423
addQRType(MVT::v8bf16);
424424
}
425425

426-
if (Subtarget->hasSVEorSME()) {
426+
if (Subtarget->isSVEorStreamingSVEAvailable()) {
427427
// Add legal sve predicate types
428428
addRegisterClass(MVT::nxv1i1, &AArch64::PPRRegClass);
429429
addRegisterClass(MVT::nxv2i1, &AArch64::PPRRegClass);
@@ -1408,7 +1408,7 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
14081408

14091409
// FIXME: Move lowering for more nodes here if those are common between
14101410
// SVE and SME.
1411-
if (Subtarget->hasSVEorSME()) {
1411+
if (Subtarget->isSVEorStreamingSVEAvailable()) {
14121412
for (auto VT :
14131413
{MVT::nxv16i1, MVT::nxv8i1, MVT::nxv4i1, MVT::nxv2i1, MVT::nxv1i1}) {
14141414
setOperationAction(ISD::SPLAT_VECTOR, VT, Custom);

‎llvm/lib/Target/AArch64/AArch64InstrInfo.cpp‎

Lines changed: 22 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -4497,7 +4497,8 @@ void AArch64InstrInfo::copyPhysReg(MachineBasicBlock &MBB,
44974497
// Copy a Predicate register by ORRing with itself.
44984498
if (AArch64::PPRRegClass.contains(DestReg) &&
44994499
AArch64::PPRRegClass.contains(SrcReg)) {
4500-
assert(Subtarget.hasSVEorSME() && "Unexpected SVE register.");
4500+
assert(Subtarget.isSVEorStreamingSVEAvailable() &&
4501+
"Unexpected SVE register.");
45014502
BuildMI(MBB, I, DL, get(AArch64::ORR_PPzPP), DestReg)
45024503
.addReg(SrcReg) // Pg
45034504
.addReg(SrcReg)
@@ -4510,8 +4511,6 @@ void AArch64InstrInfo::copyPhysReg(MachineBasicBlock &MBB,
45104511
bool DestIsPNR = AArch64::PNRRegClass.contains(DestReg);
45114512
bool SrcIsPNR = AArch64::PNRRegClass.contains(SrcReg);
45124513
if (DestIsPNR || SrcIsPNR) {
4513-
assert((Subtarget.hasSVE2p1() || Subtarget.hasSME2()) &&
4514-
"Unexpected predicate-as-counter register.");
45154514
auto ToPPR = [](MCRegister R) -> MCRegister {
45164515
return (R - AArch64::PN0) + AArch64::P0;
45174516
};
@@ -4532,7 +4531,8 @@ void AArch64InstrInfo::copyPhysReg(MachineBasicBlock &MBB,
45324531
// Copy a Z register by ORRing with itself.
45334532
if (AArch64::ZPRRegClass.contains(DestReg) &&
45344533
AArch64::ZPRRegClass.contains(SrcReg)) {
4535-
assert(Subtarget.hasSVEorSME() && "Unexpected SVE register.");
4534+
assert(Subtarget.isSVEorStreamingSVEAvailable() &&
4535+
"Unexpected SVE register.");
45364536
BuildMI(MBB, I, DL, get(AArch64::ORR_ZZZ), DestReg)
45374537
.addReg(SrcReg)
45384538
.addReg(SrcReg, getKillRegState(KillSrc));
@@ -4544,7 +4544,8 @@ void AArch64InstrInfo::copyPhysReg(MachineBasicBlock &MBB,
45444544
AArch64::ZPR2StridedOrContiguousRegClass.contains(DestReg)) &&
45454545
(AArch64::ZPR2RegClass.contains(SrcReg) ||
45464546
AArch64::ZPR2StridedOrContiguousRegClass.contains(SrcReg))) {
4547-
assert(Subtarget.hasSVEorSME() && "Unexpected SVE register.");
4547+
assert(Subtarget.isSVEorStreamingSVEAvailable() &&
4548+
"Unexpected SVE register.");
45484549
static const unsigned Indices[] = {AArch64::zsub0, AArch64::zsub1};
45494550
copyPhysRegTuple(MBB, I, DL, DestReg, SrcReg, KillSrc, AArch64::ORR_ZZZ,
45504551
Indices);
@@ -4554,7 +4555,8 @@ void AArch64InstrInfo::copyPhysReg(MachineBasicBlock &MBB,
45544555
// Copy a Z register triple by copying the individual sub-registers.
45554556
if (AArch64::ZPR3RegClass.contains(DestReg) &&
45564557
AArch64::ZPR3RegClass.contains(SrcReg)) {
4557-
assert(Subtarget.hasSVEorSME() && "Unexpected SVE register.");
4558+
assert(Subtarget.isSVEorStreamingSVEAvailable() &&
4559+
"Unexpected SVE register.");
45584560
static const unsigned Indices[] = {AArch64::zsub0, AArch64::zsub1,
45594561
AArch64::zsub2};
45604562
copyPhysRegTuple(MBB, I, DL, DestReg, SrcReg, KillSrc, AArch64::ORR_ZZZ,
@@ -4567,7 +4569,8 @@ void AArch64InstrInfo::copyPhysReg(MachineBasicBlock &MBB,
45674569
AArch64::ZPR4StridedOrContiguousRegClass.contains(DestReg)) &&
45684570
(AArch64::ZPR4RegClass.contains(SrcReg) ||
45694571
AArch64::ZPR4StridedOrContiguousRegClass.contains(SrcReg))) {
4570-
assert(Subtarget.hasSVEorSME() && "Unexpected SVE register.");
4572+
assert(Subtarget.isSVEorStreamingSVEAvailable() &&
4573+
"Unexpected SVE register.");
45714574
static const unsigned Indices[] = {AArch64::zsub0, AArch64::zsub1,
45724575
AArch64::zsub2, AArch64::zsub3};
45734576
copyPhysRegTuple(MBB, I, DL, DestReg, SrcReg, KillSrc, AArch64::ORR_ZZZ,
@@ -4830,14 +4833,12 @@ void AArch64InstrInfo::storeRegToStackSlot(MachineBasicBlock &MBB,
48304833
Opc = AArch64::STRBui;
48314834
break;
48324835
case 2: {
4833-
bool IsPNR = AArch64::PNRRegClass.hasSubClassEq(RC);
48344836
if (AArch64::FPR16RegClass.hasSubClassEq(RC))
48354837
Opc = AArch64::STRHui;
4836-
else if (IsPNR || AArch64::PPRRegClass.hasSubClassEq(RC)) {
4837-
assert(Subtarget.hasSVEorSME() &&
4838+
else if (AArch64::PNRRegClass.hasSubClassEq(RC) ||
4839+
AArch64::PPRRegClass.hasSubClassEq(RC)) {
4840+
assert(Subtarget.isSVEorStreamingSVEAvailable() &&
48384841
"Unexpected register store without SVE store instructions");
4839-
assert((!IsPNR || Subtarget.hasSVE2p1() || Subtarget.hasSME2()) &&
4840-
"Unexpected register store without SVE2p1 or SME2");
48414842
Opc = AArch64::STR_PXI;
48424843
StackID = TargetStackID::ScalableVector;
48434844
}
@@ -4886,7 +4887,7 @@ void AArch64InstrInfo::storeRegToStackSlot(MachineBasicBlock &MBB,
48864887
AArch64::sube64, AArch64::subo64, FI, MMO);
48874888
return;
48884889
} else if (AArch64::ZPRRegClass.hasSubClassEq(RC)) {
4889-
assert(Subtarget.hasSVEorSME() &&
4890+
assert(Subtarget.isSVEorStreamingSVEAvailable() &&
48904891
"Unexpected register store without SVE store instructions");
48914892
Opc = AArch64::STR_ZXI;
48924893
StackID = TargetStackID::ScalableVector;
@@ -4910,7 +4911,7 @@ void AArch64InstrInfo::storeRegToStackSlot(MachineBasicBlock &MBB,
49104911
Offset = false;
49114912
} else if (AArch64::ZPR2RegClass.hasSubClassEq(RC) ||
49124913
AArch64::ZPR2StridedOrContiguousRegClass.hasSubClassEq(RC)) {
4913-
assert(Subtarget.hasSVEorSME() &&
4914+
assert(Subtarget.isSVEorStreamingSVEAvailable() &&
49144915
"Unexpected register store without SVE store instructions");
49154916
Opc = AArch64::STR_ZZXI;
49164917
StackID = TargetStackID::ScalableVector;
@@ -4922,7 +4923,7 @@ void AArch64InstrInfo::storeRegToStackSlot(MachineBasicBlock &MBB,
49224923
Opc = AArch64::ST1Threev2d;
49234924
Offset = false;
49244925
} else if (AArch64::ZPR3RegClass.hasSubClassEq(RC)) {
4925-
assert(Subtarget.hasSVEorSME() &&
4926+
assert(Subtarget.isSVEorStreamingSVEAvailable() &&
49264927
"Unexpected register store without SVE store instructions");
49274928
Opc = AArch64::STR_ZZZXI;
49284929
StackID = TargetStackID::ScalableVector;
@@ -4935,7 +4936,7 @@ void AArch64InstrInfo::storeRegToStackSlot(MachineBasicBlock &MBB,
49354936
Offset = false;
49364937
} else if (AArch64::ZPR4RegClass.hasSubClassEq(RC) ||
49374938
AArch64::ZPR4StridedOrContiguousRegClass.hasSubClassEq(RC)) {
4938-
assert(Subtarget.hasSVEorSME() &&
4939+
assert(Subtarget.isSVEorStreamingSVEAvailable() &&
49394940
"Unexpected register store without SVE store instructions");
49404941
Opc = AArch64::STR_ZZZZXI;
49414942
StackID = TargetStackID::ScalableVector;
@@ -5008,10 +5009,8 @@ void AArch64InstrInfo::loadRegFromStackSlot(MachineBasicBlock &MBB,
50085009
if (AArch64::FPR16RegClass.hasSubClassEq(RC))
50095010
Opc = AArch64::LDRHui;
50105011
else if (IsPNR || AArch64::PPRRegClass.hasSubClassEq(RC)) {
5011-
assert(Subtarget.hasSVEorSME() &&
5012+
assert(Subtarget.isSVEorStreamingSVEAvailable() &&
50125013
"Unexpected register load without SVE load instructions");
5013-
assert((!IsPNR || Subtarget.hasSVE2p1() || Subtarget.hasSME2()) &&
5014-
"Unexpected register load without SVE2p1 or SME2");
50155014
if (IsPNR)
50165015
PNRReg = DestReg;
50175016
Opc = AArch64::LDR_PXI;
@@ -5062,7 +5061,7 @@ void AArch64InstrInfo::loadRegFromStackSlot(MachineBasicBlock &MBB,
50625061
AArch64::subo64, FI, MMO);
50635062
return;
50645063
} else if (AArch64::ZPRRegClass.hasSubClassEq(RC)) {
5065-
assert(Subtarget.hasSVEorSME() &&
5064+
assert(Subtarget.isSVEorStreamingSVEAvailable() &&
50665065
"Unexpected register load without SVE load instructions");
50675066
Opc = AArch64::LDR_ZXI;
50685067
StackID = TargetStackID::ScalableVector;
@@ -5086,7 +5085,7 @@ void AArch64InstrInfo::loadRegFromStackSlot(MachineBasicBlock &MBB,
50865085
Offset = false;
50875086
} else if (AArch64::ZPR2RegClass.hasSubClassEq(RC) ||
50885087
AArch64::ZPR2StridedOrContiguousRegClass.hasSubClassEq(RC)) {
5089-
assert(Subtarget.hasSVEorSME() &&
5088+
assert(Subtarget.isSVEorStreamingSVEAvailable() &&
50905089
"Unexpected register load without SVE load instructions");
50915090
Opc = AArch64::LDR_ZZXI;
50925091
StackID = TargetStackID::ScalableVector;
@@ -5098,7 +5097,7 @@ void AArch64InstrInfo::loadRegFromStackSlot(MachineBasicBlock &MBB,
50985097
Opc = AArch64::LD1Threev2d;
50995098
Offset = false;
51005099
} else if (AArch64::ZPR3RegClass.hasSubClassEq(RC)) {
5101-
assert(Subtarget.hasSVEorSME() &&
5100+
assert(Subtarget.isSVEorStreamingSVEAvailable() &&
51025101
"Unexpected register load without SVE load instructions");
51035102
Opc = AArch64::LDR_ZZZXI;
51045103
StackID = TargetStackID::ScalableVector;
@@ -5111,7 +5110,7 @@ void AArch64InstrInfo::loadRegFromStackSlot(MachineBasicBlock &MBB,
51115110
Offset = false;
51125111
} else if (AArch64::ZPR4RegClass.hasSubClassEq(RC) ||
51135112
AArch64::ZPR4StridedOrContiguousRegClass.hasSubClassEq(RC)) {
5114-
assert(Subtarget.hasSVEorSME() &&
5113+
assert(Subtarget.isSVEorStreamingSVEAvailable() &&
51155114
"Unexpected register load without SVE load instructions");
51165115
Opc = AArch64::LDR_ZZZZXI;
51175116
StackID = TargetStackID::ScalableVector;

‎llvm/lib/Target/AArch64/AArch64InstrInfo.td‎

Lines changed: 30 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -141,35 +141,41 @@ def HasSPE : Predicate<"Subtarget->hasSPE()">,
141141
def HasFuseAES : Predicate<"Subtarget->hasFuseAES()">,
142142
AssemblerPredicateWithAll<(all_of FeatureFuseAES),
143143
"fuse-aes">;
144-
def HasSVE : Predicate<"Subtarget->hasSVE()">,
144+
def HasSVE : Predicate<"Subtarget->isSVEAvailable()">,
145145
AssemblerPredicateWithAll<(all_of FeatureSVE), "sve">;
146-
def HasSVE2 : Predicate<"Subtarget->hasSVE2()">,
146+
def HasSVE2 : Predicate<"Subtarget->isSVEAvailable() && Subtarget->hasSVE2()">,
147147
AssemblerPredicateWithAll<(all_of FeatureSVE2), "sve2">;
148-
def HasSVE2p1 : Predicate<"Subtarget->hasSVE2p1()">,
148+
def HasSVE2p1 : Predicate<"Subtarget->isSVEAvailable() && Subtarget->hasSVE2p1()">,
149149
AssemblerPredicateWithAll<(all_of FeatureSVE2p1), "sve2p1">;
150-
def HasSVE2AES : Predicate<"Subtarget->hasSVE2AES()">,
150+
def HasSVE2AES : Predicate<"Subtarget->isSVEAvailable() && Subtarget->hasSVE2AES()">,
151151
AssemblerPredicateWithAll<(all_of FeatureSVE2AES), "sve2-aes">;
152-
def HasSVE2SM4 : Predicate<"Subtarget->hasSVE2SM4()">,
152+
def HasSVE2SM4 : Predicate<"Subtarget->isSVEAvailable() && Subtarget->hasSVE2SM4()">,
153153
AssemblerPredicateWithAll<(all_of FeatureSVE2SM4), "sve2-sm4">;
154-
def HasSVE2SHA3 : Predicate<"Subtarget->hasSVE2SHA3()">,
154+
def HasSVE2SHA3 : Predicate<"Subtarget->isSVEAvailable() && Subtarget->hasSVE2SHA3()">,
155155
AssemblerPredicateWithAll<(all_of FeatureSVE2SHA3), "sve2-sha3">;
156-
def HasSVE2BitPerm : Predicate<"Subtarget->hasSVE2BitPerm()">,
156+
def HasSVE2BitPerm : Predicate<"Subtarget->isSVEAvailable() && Subtarget->hasSVE2BitPerm()">,
157157
AssemblerPredicateWithAll<(all_of FeatureSVE2BitPerm), "sve2-bitperm">;
158158
def HasB16B16 : Predicate<"Subtarget->hasB16B16()">,
159159
AssemblerPredicateWithAll<(all_of FeatureB16B16), "b16b16">;
160-
def HasSME : Predicate<"Subtarget->hasSME()">,
160+
def HasSMEandIsNonStreamingSafe
161+
: Predicate<"Subtarget->hasSME()">,
161162
AssemblerPredicateWithAll<(all_of FeatureSME), "sme">;
162-
def HasSMEF64F64 : Predicate<"Subtarget->hasSMEF64F64()">,
163+
def HasSME : Predicate<"Subtarget->isStreaming() && Subtarget->hasSME()">,
164+
AssemblerPredicateWithAll<(all_of FeatureSME), "sme">;
165+
def HasSMEF64F64 : Predicate<"Subtarget->isStreaming() && Subtarget->hasSMEF64F64()">,
163166
AssemblerPredicateWithAll<(all_of FeatureSMEF64F64), "sme-f64f64">;
164-
def HasSMEF16F16 : Predicate<"Subtarget->hasSMEF16F16()">,
167+
def HasSMEF16F16 : Predicate<"Subtarget->isStreaming() && Subtarget->hasSMEF16F16()">,
165168
AssemblerPredicateWithAll<(all_of FeatureSMEF16F16), "sme-f16f16">;
166-
def HasSMEFA64 : Predicate<"Subtarget->hasSMEFA64()">,
169+
def HasSMEFA64 : Predicate<"Subtarget->isStreaming() && Subtarget->hasSMEFA64()">,
167170
AssemblerPredicateWithAll<(all_of FeatureSMEFA64), "sme-fa64">;
168-
def HasSMEI16I64 : Predicate<"Subtarget->hasSMEI16I64()">,
171+
def HasSMEI16I64 : Predicate<"Subtarget->isStreaming() && Subtarget->hasSMEI16I64()">,
169172
AssemblerPredicateWithAll<(all_of FeatureSMEI16I64), "sme-i16i64">;
170-
def HasSME2 : Predicate<"Subtarget->hasSME2()">,
173+
def HasSME2andIsNonStreamingSafe
174+
: Predicate<"Subtarget->hasSME2()">,
175+
AssemblerPredicateWithAll<(all_of FeatureSME2), "sme2">;
176+
def HasSME2 : Predicate<"Subtarget->isStreaming() && Subtarget->hasSME2()">,
171177
AssemblerPredicateWithAll<(all_of FeatureSME2), "sme2">;
172-
def HasSME2p1 : Predicate<"Subtarget->hasSME2p1()">,
178+
def HasSME2p1 : Predicate<"Subtarget->isStreaming() && Subtarget->hasSME2p1()">,
173179
AssemblerPredicateWithAll<(all_of FeatureSME2p1), "sme2p1">;
174180
def HasFP8 : Predicate<"Subtarget->hasFP8()">,
175181
AssemblerPredicateWithAll<(all_of FeatureFP8), "fp8">;
@@ -198,39 +204,39 @@ def HasSSVE_FP8DOT4 : Predicate<"Subtarget->hasSSVE_FP8DOT4() || "
198204
"ssve-fp8dot4 or (sve2 and fp8dot4)">;
199205
def HasLUT : Predicate<"Subtarget->hasLUT()">,
200206
AssemblerPredicateWithAll<(all_of FeatureLUT), "lut">;
201-
def HasSME_LUTv2 : Predicate<"Subtarget->hasSME_LUTv2()">,
207+
def HasSME_LUTv2 : Predicate<"Subtarget->isStreaming() && Subtarget->hasSME_LUTv2()">,
202208
AssemblerPredicateWithAll<(all_of FeatureSME_LUTv2), "sme-lutv2">;
203-
def HasSMEF8F16 : Predicate<"Subtarget->hasSMEF8F16()">,
209+
def HasSMEF8F16 : Predicate<"Subtarget->isStreaming() && Subtarget->hasSMEF8F16()">,
204210
AssemblerPredicateWithAll<(all_of FeatureSMEF8F16), "sme-f8f16">;
205-
def HasSMEF8F32 : Predicate<"Subtarget->hasSMEF8F32()">,
211+
def HasSMEF8F32 : Predicate<"Subtarget->isStreaming() && Subtarget->hasSMEF8F32()">,
206212
AssemblerPredicateWithAll<(all_of FeatureSMEF8F32), "sme-f8f32">;
207213

208214
// A subset of SVE(2) instructions are legal in Streaming SVE execution mode,
209215
// they should be enabled if either has been specified.
210216
def HasSVEorSME
211-
: Predicate<"Subtarget->hasSVEorSME()">,
217+
: Predicate<"Subtarget->hasSVE() || (Subtarget->isStreaming() && Subtarget->hasSME())">,
212218
AssemblerPredicateWithAll<(any_of FeatureSVE, FeatureSME),
213219
"sve or sme">;
214220
def HasSVE2orSME
215-
: Predicate<"Subtarget->hasSVE2() || Subtarget->hasSME()">,
221+
: Predicate<"Subtarget->hasSVE2() || (Subtarget->isStreaming() && Subtarget->hasSME())">,
216222
AssemblerPredicateWithAll<(any_of FeatureSVE2, FeatureSME),
217223
"sve2 or sme">;
218224
def HasSVE2orSME2
219-
: Predicate<"Subtarget->hasSVE2() || Subtarget->hasSME2()">,
225+
: Predicate<"Subtarget->hasSVE2() || (Subtarget->isStreaming() && Subtarget->hasSME2())">,
220226
AssemblerPredicateWithAll<(any_of FeatureSVE2, FeatureSME2),
221227
"sve2 or sme2">;
222228
def HasSVE2p1_or_HasSME
223-
: Predicate<"Subtarget->hasSVE2p1() || Subtarget->hasSME()">,
229+
: Predicate<"Subtarget->hasSVE2p1() || (Subtarget->isStreaming() && Subtarget->hasSME())">,
224230
AssemblerPredicateWithAll<(any_of FeatureSME, FeatureSVE2p1), "sme or sve2p1">;
225231
def HasSVE2p1_or_HasSME2
226-
: Predicate<"Subtarget->hasSVE2p1() || Subtarget->hasSME2()">,
232+
: Predicate<"Subtarget->hasSVE2p1() || (Subtarget->isStreaming() && Subtarget->hasSME2())">,
227233
AssemblerPredicateWithAll<(any_of FeatureSME2, FeatureSVE2p1), "sme2 or sve2p1">;
228234
def HasSVE2p1_or_HasSME2p1
229-
: Predicate<"Subtarget->hasSVE2p1() || Subtarget->hasSME2p1()">,
235+
: Predicate<"Subtarget->hasSVE2p1() || (Subtarget->isStreaming() && Subtarget->hasSME2p1())">,
230236
AssemblerPredicateWithAll<(any_of FeatureSME2p1, FeatureSVE2p1), "sme2p1 or sve2p1">;
231237

232238
def HasSMEF16F16orSMEF8F16
233-
: Predicate<"Subtarget->hasSMEF16F16() || Subtarget->hasSMEF8F16()">,
239+
: Predicate<"Subtarget->isStreaming() && (Subtarget->hasSMEF16F16() || Subtarget->hasSMEF8F16())">,
234240
AssemblerPredicateWithAll<(any_of FeatureSMEF16F16, FeatureSMEF8F16),
235241
"sme-f16f16 or sme-f8f16">;
236242

0 commit comments

Comments
 (0)