@@ -5483,199 +5483,21 @@ AArch64InstrInfo::findRegisterToSaveLRTo(const outliner::Candidate &C) const {
5483
5483
return 0u ;
5484
5484
}
5485
5485
5486
- static bool
5487
- outliningCandidatesSigningScopeConsensus (const outliner::Candidate &a,
5488
- const outliner::Candidate &b) {
5489
- const Function &Fa = a.getMF ()->getFunction ();
5490
- const Function &Fb = b.getMF ()->getFunction ();
5491
-
5492
- // If none of the functions have the "sign-return-address" attribute their
5493
- // signing behaviour is equal
5494
- if (!Fa.hasFnAttribute (" sign-return-address" ) &&
5495
- !Fb.hasFnAttribute (" sign-return-address" )) {
5496
- return true ;
5497
- }
5498
-
5499
- // If both functions have the "sign-return-address" attribute their signing
5500
- // behaviour is equal, if the values of the attributes are equal
5501
- if (Fa.hasFnAttribute (" sign-return-address" ) &&
5502
- Fb.hasFnAttribute (" sign-return-address" )) {
5503
- StringRef ScopeA =
5504
- Fa.getFnAttribute (" sign-return-address" ).getValueAsString ();
5505
- StringRef ScopeB =
5506
- Fb.getFnAttribute (" sign-return-address" ).getValueAsString ();
5507
- return ScopeA.equals (ScopeB);
5508
- }
5509
-
5510
- // If function B doesn't have the "sign-return-address" attribute but A does,
5511
- // the functions' signing behaviour is equal if A's value for
5512
- // "sign-return-address" is "none" and vice versa.
5513
- if (Fa.hasFnAttribute (" sign-return-address" )) {
5514
- StringRef ScopeA =
5515
- Fa.getFnAttribute (" sign-return-address" ).getValueAsString ();
5516
- return ScopeA.equals (" none" );
5517
- }
5518
-
5519
- if (Fb.hasFnAttribute (" sign-return-address" )) {
5520
- StringRef ScopeB =
5521
- Fb.getFnAttribute (" sign-return-address" ).getValueAsString ();
5522
- return ScopeB.equals (" none" );
5523
- }
5524
-
5525
- llvm_unreachable (" Unkown combination of sign-return-address attributes" );
5526
- }
5527
-
5528
- static bool
5529
- outliningCandidatesSigningKeyConsensus (const outliner::Candidate &a,
5530
- const outliner::Candidate &b) {
5531
- const Function &Fa = a.getMF ()->getFunction ();
5532
- const Function &Fb = b.getMF ()->getFunction ();
5533
-
5534
- // If none of the functions have the "sign-return-address-key" attribute
5535
- // their keys are equal
5536
- if (!Fa.hasFnAttribute (" sign-return-address-key" ) &&
5537
- !Fb.hasFnAttribute (" sign-return-address-key" )) {
5538
- return true ;
5539
- }
5540
-
5541
- // If both functions have the "sign-return-address-key" attribute their
5542
- // keys are equal if the values of "sign-return-address-key" are equal
5543
- if (Fa.hasFnAttribute (" sign-return-address-key" ) &&
5544
- Fb.hasFnAttribute (" sign-return-address-key" )) {
5545
- StringRef KeyA =
5546
- Fa.getFnAttribute (" sign-return-address-key" ).getValueAsString ();
5547
- StringRef KeyB =
5548
- Fb.getFnAttribute (" sign-return-address-key" ).getValueAsString ();
5549
- return KeyA.equals (KeyB);
5550
- }
5551
-
5552
- // If B doesn't have the "sign-return-address-key" attribute, both keys are
5553
- // equal, if function a has the default key (a_key)
5554
- if (Fa.hasFnAttribute (" sign-return-address-key" )) {
5555
- StringRef KeyA =
5556
- Fa.getFnAttribute (" sign-return-address-key" ).getValueAsString ();
5557
- return KeyA.equals_lower (" a_key" );
5558
- }
5559
-
5560
- if (Fb.hasFnAttribute (" sign-return-address-key" )) {
5561
- StringRef KeyB =
5562
- Fb.getFnAttribute (" sign-return-address-key" ).getValueAsString ();
5563
- return KeyB.equals_lower (" a_key" );
5564
- }
5565
-
5566
- llvm_unreachable (" Unkown combination of sign-return-address-key attributes" );
5567
- }
5568
-
5569
- static bool outliningCandidatesV8_3OpsConsensus (const outliner::Candidate &a,
5570
- const outliner::Candidate &b) {
5571
- const AArch64Subtarget &SubtargetA =
5572
- a.getMF ()->getSubtarget <AArch64Subtarget>();
5573
- const AArch64Subtarget &SubtargetB =
5574
- b.getMF ()->getSubtarget <AArch64Subtarget>();
5575
- return SubtargetA.hasV8_3aOps () == SubtargetB.hasV8_3aOps ();
5576
- }
5577
-
5578
- outliner::OutlinedFunction AArch64InstrInfo::getOutliningCandidateInfo (
5486
+ outliner::OutlinedFunction
5487
+ AArch64InstrInfo::getOutliningCandidateInfo (
5579
5488
std::vector<outliner::Candidate> &RepeatedSequenceLocs) const {
5580
5489
outliner::Candidate &FirstCand = RepeatedSequenceLocs[0 ];
5581
5490
unsigned SequenceSize =
5582
5491
std::accumulate (FirstCand.front (), std::next (FirstCand.back ()), 0 ,
5583
5492
[this ](unsigned Sum, const MachineInstr &MI) {
5584
5493
return Sum + getInstSizeInBytes (MI);
5585
5494
});
5586
- unsigned NumBytesToCreateFrame = 0 ;
5587
-
5588
- // We only allow outlining for functions having exactly matching return
5589
- // address signing attributes, i.e., all share the same value for the
5590
- // attribute "sign-return-address" and all share the same type of key they
5591
- // are signed with.
5592
- // Additionally we require all functions to simultaniously either support
5593
- // v8.3a features or not. Otherwise an outlined function could get signed
5594
- // using dedicated v8.3 instructions and a call from a function that doesn't
5595
- // support v8.3 instructions would therefore be invalid.
5596
- if (std::adjacent_find (
5597
- RepeatedSequenceLocs.begin (), RepeatedSequenceLocs.end (),
5598
- [](const outliner::Candidate &a, const outliner::Candidate &b) {
5599
- // Return true if a and b are non-equal w.r.t. return address
5600
- // signing or support of v8.3a features
5601
- if (outliningCandidatesSigningScopeConsensus (a, b) &&
5602
- outliningCandidatesSigningKeyConsensus (a, b) &&
5603
- outliningCandidatesV8_3OpsConsensus (a, b)) {
5604
- return false ;
5605
- }
5606
- return true ;
5607
- }) != RepeatedSequenceLocs.end ()) {
5608
- return outliner::OutlinedFunction ();
5609
- }
5610
-
5611
- // Since at this point all candidates agree on their return address signing
5612
- // picking just one is fine. If the candidate functions potentially sign their
5613
- // return addresses, the outlined function should do the same. Note that in
5614
- // the case of "sign-return-address"="non-leaf" this is an assumption: It is
5615
- // not certainly true that the outlined function will have to sign its return
5616
- // address but this decision is made later, when the decision to outline
5617
- // has already been made.
5618
- // The same holds for the number of additional instructions we need: On
5619
- // v8.3a RET can be replaced by RETAA/RETAB and no AUT instruction is
5620
- // necessary. However, at this point we don't know if the outlined function
5621
- // will have a RET instruction so we assume the worst.
5622
- const Function &FCF = FirstCand.getMF ()->getFunction ();
5623
- const TargetRegisterInfo &TRI = getRegisterInfo ();
5624
- if (FCF.hasFnAttribute (" sign-return-address" )) {
5625
- // One PAC and one AUT instructions
5626
- NumBytesToCreateFrame += 8 ;
5627
-
5628
- // We have to check if sp modifying instructions would get outlined.
5629
- // If so we only allow outlining if sp is unchanged overall, so matching
5630
- // sub and add instructions are okay to outline, all other sp modifications
5631
- // are not
5632
- auto hasIllegalSPModification = [&TRI](outliner::Candidate &C) {
5633
- int SPValue = 0 ;
5634
- MachineBasicBlock::iterator MBBI = C.front ();
5635
- for (;;) {
5636
- if (MBBI->modifiesRegister (AArch64::SP, &TRI)) {
5637
- switch (MBBI->getOpcode ()) {
5638
- case AArch64::ADDXri:
5639
- case AArch64::ADDWri:
5640
- assert (MBBI->getNumOperands () == 4 && " Wrong number of operands" );
5641
- assert (MBBI->getOperand (2 ).isImm () &&
5642
- " Expected operand to be immediate" );
5643
- SPValue += MBBI->getOperand (2 ).getImm ();
5644
- break ;
5645
- case AArch64::SUBXri:
5646
- case AArch64::SUBWri:
5647
- assert (MBBI->getNumOperands () == 4 && " Wrong number of operands" );
5648
- assert (MBBI->getOperand (2 ).isImm () &&
5649
- " Expected operand to be immediate" );
5650
- SPValue -= MBBI->getOperand (2 ).getImm ();
5651
- break ;
5652
- default :
5653
- return true ;
5654
- }
5655
- }
5656
- if (MBBI == C.back ())
5657
- break ;
5658
- ++MBBI;
5659
- }
5660
- if (SPValue)
5661
- return true ;
5662
- return false ;
5663
- };
5664
- // Remove candidates with illegal stack modifying instructions
5665
- RepeatedSequenceLocs.erase (std::remove_if (RepeatedSequenceLocs.begin (),
5666
- RepeatedSequenceLocs.end (),
5667
- hasIllegalSPModification),
5668
- RepeatedSequenceLocs.end ());
5669
-
5670
- // If the sequence doesn't have enough candidates left, then we're done.
5671
- if (RepeatedSequenceLocs.size () < 2 )
5672
- return outliner::OutlinedFunction ();
5673
- }
5674
5495
5675
5496
// Properties about candidate MBBs that hold for all of them.
5676
5497
unsigned FlagsSetInAll = 0xF ;
5677
5498
5678
5499
// Compute liveness information for each candidate, and set FlagsSetInAll.
5500
+ const TargetRegisterInfo &TRI = getRegisterInfo ();
5679
5501
std::for_each (RepeatedSequenceLocs.begin (), RepeatedSequenceLocs.end (),
5680
5502
[&FlagsSetInAll](outliner::Candidate &C) {
5681
5503
FlagsSetInAll &= C.Flags ;
@@ -5731,7 +5553,7 @@ outliner::OutlinedFunction AArch64InstrInfo::getOutliningCandidateInfo(
5731
5553
};
5732
5554
5733
5555
unsigned FrameID = MachineOutlinerDefault;
5734
- NumBytesToCreateFrame + = 4 ;
5556
+ unsigned NumBytesToCreateFrame = 4 ;
5735
5557
5736
5558
bool HasBTI = any_of (RepeatedSequenceLocs, [](outliner::Candidate &C) {
5737
5559
return C.getMF ()->getFunction ().hasFnAttribute (" branch-target-enforcement" );
@@ -6000,19 +5822,6 @@ AArch64InstrInfo::getOutliningType(MachineBasicBlock::iterator &MIT,
6000
5822
MachineFunction *MF = MBB->getParent ();
6001
5823
AArch64FunctionInfo *FuncInfo = MF->getInfo <AArch64FunctionInfo>();
6002
5824
6003
- // Don't outline anything used for return address signing. The outlined
6004
- // function will get signed later if needed
6005
- switch (MI.getOpcode ()) {
6006
- case AArch64::PACIASP:
6007
- case AArch64::PACIBSP:
6008
- case AArch64::AUTIASP:
6009
- case AArch64::AUTIBSP:
6010
- case AArch64::RETAA:
6011
- case AArch64::RETAB:
6012
- case AArch64::EMITBKEY:
6013
- return outliner::InstrType::Illegal;
6014
- }
6015
-
6016
5825
// Don't outline LOHs.
6017
5826
if (FuncInfo->getLOHRelated ().count (&MI))
6018
5827
return outliner::InstrType::Illegal;
@@ -6165,59 +5974,6 @@ void AArch64InstrInfo::fixupPostOutline(MachineBasicBlock &MBB) const {
6165
5974
}
6166
5975
}
6167
5976
6168
- static void signOutlinedFunction (MachineFunction &MF, MachineBasicBlock &MBB,
6169
- bool ShouldSignReturnAddr,
6170
- bool ShouldSignReturnAddrWithAKey) {
6171
- if (ShouldSignReturnAddr) {
6172
- MachineBasicBlock::iterator MBBPAC = MBB.begin ();
6173
- MachineBasicBlock::iterator MBBAUT = MBB.getFirstTerminator ();
6174
- const AArch64Subtarget &Subtarget = MF.getSubtarget <AArch64Subtarget>();
6175
- const TargetInstrInfo *TII = Subtarget.getInstrInfo ();
6176
- DebugLoc DL;
6177
-
6178
- if (MBBAUT != MBB.end ())
6179
- DL = MBBAUT->getDebugLoc ();
6180
-
6181
- // At the very beginning of the basic block we insert the following
6182
- // depending on the key type
6183
- //
6184
- // a_key: b_key:
6185
- // PACIASP EMITBKEY
6186
- // CFI_INSTRUCTION PACIBSP
6187
- // CFI_INSTRUCTION
6188
- if (ShouldSignReturnAddrWithAKey) {
6189
- BuildMI (MBB, MBBPAC, DebugLoc (), TII->get (AArch64::PACIASP))
6190
- .setMIFlag (MachineInstr::FrameSetup);
6191
- } else {
6192
- BuildMI (MBB, MBBPAC, DebugLoc (), TII->get (AArch64::EMITBKEY))
6193
- .setMIFlag (MachineInstr::FrameSetup);
6194
- BuildMI (MBB, MBBPAC, DebugLoc (), TII->get (AArch64::PACIBSP))
6195
- .setMIFlag (MachineInstr::FrameSetup);
6196
- }
6197
- unsigned CFIIndex =
6198
- MF.addFrameInst (MCCFIInstruction::createNegateRAState (nullptr ));
6199
- BuildMI (MBB, MBBPAC, DebugLoc (), TII->get (AArch64::CFI_INSTRUCTION))
6200
- .addCFIIndex (CFIIndex)
6201
- .setMIFlags (MachineInstr::FrameSetup);
6202
-
6203
- // If v8.3a features are available we can replace a RET instruction by
6204
- // RETAA or RETAB and omit the AUT instructions
6205
- if (Subtarget.hasV8_3aOps () && MBBAUT != MBB.end () &&
6206
- MBBAUT->getOpcode () == AArch64::RET) {
6207
- BuildMI (MBB, MBBAUT, DL,
6208
- TII->get (ShouldSignReturnAddrWithAKey ? AArch64::RETAA
6209
- : AArch64::RETAB))
6210
- .copyImplicitOps (*MBBAUT);
6211
- MBB.erase (MBBAUT);
6212
- } else {
6213
- BuildMI (MBB, MBBAUT, DL,
6214
- TII->get (ShouldSignReturnAddrWithAKey ? AArch64::AUTIASP
6215
- : AArch64::AUTIBSP))
6216
- .setMIFlag (MachineInstr::FrameDestroy);
6217
- }
6218
- }
6219
- }
6220
-
6221
5977
void AArch64InstrInfo::buildOutlinedFrame (
6222
5978
MachineBasicBlock &MBB, MachineFunction &MF,
6223
5979
const outliner::OutlinedFunction &OF) const {
@@ -6233,28 +5989,23 @@ void AArch64InstrInfo::buildOutlinedFrame(
6233
5989
TailOpcode = AArch64::TCRETURNriALL;
6234
5990
}
6235
5991
MachineInstr *TC = BuildMI (MF, DebugLoc (), get (TailOpcode))
6236
- .add (Call->getOperand (0 ))
6237
- .addImm (0 );
5992
+ .add (Call->getOperand (0 ))
5993
+ .addImm (0 );
6238
5994
MBB.insert (MBB.end (), TC);
6239
5995
Call->eraseFromParent ();
6240
5996
}
6241
5997
6242
- bool IsLeafFunction = true ;
6243
-
6244
5998
// Is there a call in the outlined range?
6245
- auto IsNonTailCall = [](const MachineInstr &MI) {
5999
+ auto IsNonTailCall = [](MachineInstr &MI) {
6246
6000
return MI.isCall () && !MI.isReturn ();
6247
6001
};
6248
-
6249
6002
if (std::any_of (MBB.instr_begin (), MBB.instr_end (), IsNonTailCall)) {
6250
6003
// Fix up the instructions in the range, since we're going to modify the
6251
6004
// stack.
6252
6005
assert (OF.FrameConstructionID != MachineOutlinerDefault &&
6253
6006
" Can only fix up stack references once" );
6254
6007
fixupPostOutline (MBB);
6255
6008
6256
- IsLeafFunction = false ;
6257
-
6258
6009
// LR has to be a live in so that we can save it.
6259
6010
MBB.addLiveIn (AArch64::LR);
6260
6011
@@ -6301,47 +6052,16 @@ void AArch64InstrInfo::buildOutlinedFrame(
6301
6052
Et = MBB.insert (Et, LDRXpost);
6302
6053
}
6303
6054
6304
- // If a bunch of candidates reach this point they must agree on their return
6305
- // address signing. It is therefore enough to just consider the signing
6306
- // behaviour of one of them
6307
- const Function &CF = OF.Candidates .front ().getMF ()->getFunction ();
6308
- bool ShouldSignReturnAddr = false ;
6309
- if (CF.hasFnAttribute (" sign-return-address" )) {
6310
- StringRef Scope =
6311
- CF.getFnAttribute (" sign-return-address" ).getValueAsString ();
6312
- if (Scope.equals (" all" ))
6313
- ShouldSignReturnAddr = true ;
6314
- else if (Scope.equals (" non-leaf" ) && !IsLeafFunction)
6315
- ShouldSignReturnAddr = true ;
6316
- }
6317
-
6318
- // a_key is the default
6319
- bool ShouldSignReturnAddrWithAKey = true ;
6320
- if (CF.hasFnAttribute (" sign-return-address-key" )) {
6321
- const StringRef Key =
6322
- CF.getFnAttribute (" sign-return-address-key" ).getValueAsString ();
6323
- // Key can either be a_key or b_key
6324
- assert ((Key.equals_lower (" a_key" ) || Key.equals_lower (" b_key" )) &&
6325
- " Return address signing key must be either a_key or b_key" );
6326
- ShouldSignReturnAddrWithAKey = Key.equals_lower (" a_key" );
6327
- }
6328
-
6329
6055
// If this is a tail call outlined function, then there's already a return.
6330
6056
if (OF.FrameConstructionID == MachineOutlinerTailCall ||
6331
- OF.FrameConstructionID == MachineOutlinerThunk) {
6332
- signOutlinedFunction (MF, MBB, ShouldSignReturnAddr,
6333
- ShouldSignReturnAddrWithAKey);
6057
+ OF.FrameConstructionID == MachineOutlinerThunk)
6334
6058
return ;
6335
- }
6336
6059
6337
6060
// It's not a tail call, so we have to insert the return ourselves.
6338
6061
MachineInstr *ret = BuildMI (MF, DebugLoc (), get (AArch64::RET))
6339
6062
.addReg (AArch64::LR, RegState::Undef);
6340
6063
MBB.insert (MBB.end (), ret);
6341
6064
6342
- signOutlinedFunction (MF, MBB, ShouldSignReturnAddr,
6343
- ShouldSignReturnAddrWithAKey);
6344
-
6345
6065
// Did we have to modify the stack by saving the link register?
6346
6066
if (OF.FrameConstructionID != MachineOutlinerDefault)
6347
6067
return ;
0 commit comments