Skip to content

Commit f8b7eff

Browse files
fhahnfhahn
committed
[VPlan] Move getCanonicalIV to VPRegionBlock (NFC).
The canonical IV is tied to region blocks; move getCanonicalIV there and update all users.
1 parent ae1cd7c commit f8b7eff

File tree

7 files changed

+72
-55
lines changed

7 files changed

+72
-55
lines changed

‎llvm/lib/Transforms/Vectorize/LoopVectorize.cpp‎

Lines changed: 12 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -8242,12 +8242,12 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(
82428242
// count is >= increment and a multiple of the increment.
82438243
bool HasNUW = !IVUpdateMayOverflow || Style == TailFoldingStyle::None;
82448244
if (!HasNUW) {
8245-
auto *IVInc = Plan->getVectorLoopRegion()
8246-
->getExitingBasicBlock()
8247-
->getTerminator()
8248-
->getOperand(0);
8249-
assert(match(IVInc, m_VPInstruction<Instruction::Add>(
8250-
m_Specific(Plan->getCanonicalIV()), m_VPValue())) &&
8245+
VPRegionBlock *LoopRegion = Plan->getVectorLoopRegion();
8246+
auto *IVInc =
8247+
LoopRegion->getExitingBasicBlock()->getTerminator()->getOperand(0);
8248+
assert(match(IVInc,
8249+
m_VPInstruction<Instruction::Add>(
8250+
m_Specific(LoopRegion->getCanonicalIV()), m_VPValue())) &&
82518251
"Did not find the canonical IV increment");
82528252
cast<VPRecipeWithIRFlags>(IVInc)->dropPoisonGeneratingFlags();
82538253
}
@@ -8377,8 +8377,8 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(
83778377
for (VPValue *Old : Old2New.keys())
83788378
Old->getDefiningRecipe()->eraseFromParent();
83798379

8380-
assert(isa<VPRegionBlock>(Plan->getVectorLoopRegion()) &&
8381-
!Plan->getVectorLoopRegion()->getEntryBasicBlock()->empty() &&
8380+
assert(isa<VPRegionBlock>(LoopRegion) &&
8381+
!LoopRegion->getEntryBasicBlock()->empty() &&
83828382
"entry block must be set to a VPRegionBlock having a non-empty entry "
83838383
"VPBasicBlock");
83848384

@@ -9320,8 +9320,9 @@ static void preparePlanForMainVectorLoop(VPlan &MainPlan, VPlan &EpiPlan) {
93209320
if (ResumePhiIter == MainScalarPH->phis().end()) {
93219321
VPBuilder ScalarPHBuilder(MainScalarPH, MainScalarPH->begin());
93229322
ResumePhi = ScalarPHBuilder.createScalarPhi(
9323-
{VectorTC, MainPlan.getCanonicalIV()->getStartValue()}, {},
9324-
"vec.epilog.resume.val");
9323+
{VectorTC,
9324+
MainPlan.getVectorLoopRegion()->getCanonicalIV()->getStartValue()},
9325+
{}, "vec.epilog.resume.val");
93259326
} else {
93269327
ResumePhi = cast<VPPhi>(&*ResumePhiIter);
93279328
if (MainScalarPH->begin() == MainScalarPH->end())
@@ -9348,7 +9349,7 @@ static SmallVector<Instruction *> preparePlanForEpilogueVectorLoop(
93489349
VPBasicBlock *Header = VectorLoop->getEntryBasicBlock();
93499350
Header->setName("vec.epilog.vector.body");
93509351

9351-
VPCanonicalIVPHIRecipe *IV = Plan.getCanonicalIV();
9352+
VPCanonicalIVPHIRecipe *IV = VectorLoop->getCanonicalIV();
93529353
// When vectorizing the epilogue loop, the canonical induction needs to be
93539354
// adjusted by the value after the main vector loop. Find the resume value
93549355
// created during execution of the main VPlan. It must be the first phi in the

‎llvm/lib/Transforms/Vectorize/VPlan.h‎

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -4066,6 +4066,16 @@ class LLVM_ABI_FOR_TEST VPRegionBlock : public VPBlockBase {
40664066
/// Remove the current region from its VPlan, connecting its predecessor to
40674067
/// its entry, and its exiting block to its successor.
40684068
void dissolveToCFGLoop();
4069+
4070+
/// Returns the canonical induction recipe of the region.
4071+
VPCanonicalIVPHIRecipe *getCanonicalIV() {
4072+
VPBasicBlock *EntryVPBB = getEntryBasicBlock();
4073+
if (EntryVPBB->empty()) {
4074+
// VPlan native path.
4075+
EntryVPBB = cast<VPBasicBlock>(EntryVPBB->getSingleSuccessor());
4076+
}
4077+
return cast<VPCanonicalIVPHIRecipe>(&*EntryVPBB->begin());
4078+
}
40694079
};
40704080

40714081
/// VPlan models a candidate for vectorization, encoding various decisions take
@@ -4377,16 +4387,6 @@ class VPlan {
43774387
LLVM_DUMP_METHOD void dump() const;
43784388
#endif
43794389

4380-
/// Returns the canonical induction recipe of the vector loop.
4381-
VPCanonicalIVPHIRecipe *getCanonicalIV() {
4382-
VPBasicBlock *EntryVPBB = getVectorLoopRegion()->getEntryBasicBlock();
4383-
if (EntryVPBB->empty()) {
4384-
// VPlan native path.
4385-
EntryVPBB = cast<VPBasicBlock>(EntryVPBB->getSingleSuccessor());
4386-
}
4387-
return cast<VPCanonicalIVPHIRecipe>(&*EntryVPBB->begin());
4388-
}
4389-
43904390
VPValue *getSCEVExpansion(const SCEV *S) const {
43914391
return SCEVToExpansion.lookup(S);
43924392
}

‎llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp‎

Lines changed: 9 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -661,9 +661,11 @@ void VPlanTransforms::attachCheckBlock(VPlan &Plan, Value *Cond,
661661
}
662662

663663
VPIRMetadata VPBranchWeights;
664-
auto *Term = VPBuilder(CheckBlockVPBB)
665-
.createNaryOp(VPInstruction::BranchOnCond, {CondVPV},
666-
Plan.getCanonicalIV()->getDebugLoc());
664+
auto *Term =
665+
VPBuilder(CheckBlockVPBB)
666+
.createNaryOp(
667+
VPInstruction::BranchOnCond, {CondVPV},
668+
Plan.getVectorLoopRegion()->getCanonicalIV()->getDebugLoc());
667669
if (AddBranchWeights) {
668670
MDBuilder MDB(Plan.getContext());
669671
MDNode *BranchWeights =
@@ -925,8 +927,8 @@ bool VPlanTransforms::handleMaxMinNumReductions(VPlan &Plan) {
925927
if (auto *DerivedIV = dyn_cast<VPDerivedIVRecipe>(VecV)) {
926928
if (DerivedIV->getNumUsers() == 1 &&
927929
DerivedIV->getOperand(1) == &Plan.getVectorTripCount()) {
928-
auto *NewSel = Builder.createSelect(AnyNaN, Plan.getCanonicalIV(),
929-
&Plan.getVectorTripCount());
930+
auto *NewSel = Builder.createSelect(
931+
AnyNaN, LoopRegion->getCanonicalIV(), &Plan.getVectorTripCount());
930932
DerivedIV->moveAfter(&*Builder.getInsertPoint());
931933
DerivedIV->setOperand(1, NewSel);
932934
continue;
@@ -939,7 +941,8 @@ bool VPlanTransforms::handleMaxMinNumReductions(VPlan &Plan) {
939941
"FMaxNum/FMinNum reduction.\n");
940942
return false;
941943
}
942-
auto *NewSel = Builder.createSelect(AnyNaN, Plan.getCanonicalIV(), VecV);
944+
auto *NewSel =
945+
Builder.createSelect(AnyNaN, LoopRegion->getCanonicalIV(), VecV);
943946
ResumeR->setOperand(0, NewSel);
944947
}
945948

‎llvm/lib/Transforms/Vectorize/VPlanPredicator.cpp‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -168,7 +168,8 @@ void VPPredicator::createHeaderMask(VPBasicBlock *HeaderVPBB, bool FoldTail) {
168168
// non-phi instructions.
169169

170170
auto &Plan = *HeaderVPBB->getPlan();
171-
auto *IV = new VPWidenCanonicalIVRecipe(Plan.getCanonicalIV());
171+
auto *IV = new VPWidenCanonicalIVRecipe(
172+
Plan.getVectorLoopRegion()->getCanonicalIV());
172173
Builder.setInsertPoint(HeaderVPBB, HeaderVPBB->getFirstNonPhi());
173174
Builder.insert(IV);
174175

‎llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp‎

Lines changed: 31 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -501,7 +501,8 @@ static void removeRedundantInductionCasts(VPlan &Plan) {
501501
/// Try to replace VPWidenCanonicalIVRecipes with a widened canonical IV
502502
/// recipe, if it exists.
503503
static void removeRedundantCanonicalIVs(VPlan &Plan) {
504-
VPCanonicalIVPHIRecipe *CanonicalIV = Plan.getCanonicalIV();
504+
VPRegionBlock *LoopRegion = Plan.getVectorLoopRegion();
505+
VPCanonicalIVPHIRecipe *CanonicalIV = LoopRegion->getCanonicalIV();
505506
VPWidenCanonicalIVRecipe *WidenNewIV = nullptr;
506507
for (VPUser *U : CanonicalIV->users()) {
507508
WidenNewIV = dyn_cast<VPWidenCanonicalIVRecipe>(U);
@@ -512,7 +513,7 @@ static void removeRedundantCanonicalIVs(VPlan &Plan) {
512513
if (!WidenNewIV)
513514
return;
514515

515-
VPBasicBlock *HeaderVPBB = Plan.getVectorLoopRegion()->getEntryBasicBlock();
516+
VPBasicBlock *HeaderVPBB = LoopRegion->getEntryBasicBlock();
516517
for (VPRecipeBase &Phi : HeaderVPBB->phis()) {
517518
auto *WidenOriginalIV = dyn_cast<VPWidenIntOrFpInductionRecipe>(&Phi);
518519

@@ -582,8 +583,9 @@ createScalarIVSteps(VPlan &Plan, InductionDescriptor::InductionKind Kind,
582583
FPMathOperator *FPBinOp, Instruction *TruncI,
583584
VPValue *StartV, VPValue *Step, DebugLoc DL,
584585
VPBuilder &Builder) {
585-
VPBasicBlock *HeaderVPBB = Plan.getVectorLoopRegion()->getEntryBasicBlock();
586-
VPCanonicalIVPHIRecipe *CanonicalIV = Plan.getCanonicalIV();
586+
VPRegionBlock *LoopRegion = Plan.getVectorLoopRegion();
587+
VPBasicBlock *HeaderVPBB = LoopRegion->getEntryBasicBlock();
588+
VPCanonicalIVPHIRecipe *CanonicalIV = LoopRegion->getCanonicalIV();
587589
VPSingleDefRecipe *BaseIV = Builder.createDerivedIV(
588590
Kind, FPBinOp, StartV, CanonicalIV, Step, "offset.idx");
589591

@@ -800,8 +802,9 @@ static VPValue *optimizeEarlyExitInductionUser(VPlan &Plan,
800802
return nullptr;
801803

802804
// Calculate the final index.
803-
VPValue *EndValue = Plan.getCanonicalIV();
804-
auto CanonicalIVType = Plan.getCanonicalIV()->getScalarType();
805+
VPRegionBlock *LoopRegion = Plan.getVectorLoopRegion();
806+
VPValue *EndValue = LoopRegion->getCanonicalIV();
807+
auto CanonicalIVType = LoopRegion->getCanonicalIV()->getScalarType();
805808
VPBuilder B(cast<VPBasicBlock>(PredVPBB));
806809

807810
DebugLoc DL = cast<VPInstruction>(Op)->getDebugLoc();
@@ -1532,7 +1535,7 @@ static bool isConditionTrueViaVFAndUF(VPValue *Cond, VPlan &Plan,
15321535
return isConditionTrueViaVFAndUF(C, Plan, BestVF, BestUF, SE);
15331536
});
15341537

1535-
auto *CanIV = Plan.getCanonicalIV();
1538+
auto *CanIV = Plan.getVectorLoopRegion()->getCanonicalIV();
15361539
if (!match(Cond, m_SpecificICmp(CmpInst::ICMP_EQ,
15371540
m_Specific(CanIV->getBackedgeValue()),
15381541
m_Specific(&Plan.getVectorTripCount()))))
@@ -2316,7 +2319,7 @@ static VPActiveLaneMaskPHIRecipe *addVPLaneMaskPhiAndUpdateExitBranch(
23162319
VPlan &Plan, bool DataAndControlFlowWithoutRuntimeCheck) {
23172320
VPRegionBlock *TopRegion = Plan.getVectorLoopRegion();
23182321
VPBasicBlock *EB = TopRegion->getExitingBasicBlock();
2319-
auto *CanonicalIVPHI = Plan.getCanonicalIV();
2322+
auto *CanonicalIVPHI = TopRegion->getCanonicalIV();
23202323
VPValue *StartV = CanonicalIVPHI->getStartValue();
23212324

23222325
auto *CanonicalIVIncrement =
@@ -2355,7 +2358,7 @@ static VPActiveLaneMaskPHIRecipe *addVPLaneMaskPhiAndUpdateExitBranch(
23552358

23562359
// Create the active lane mask instruction in the VPlan preheader.
23572360
VPValue *ALMMultiplier = Plan.getOrAddLiveIn(
2358-
ConstantInt::get(Plan.getCanonicalIV()->getScalarType(), 1));
2361+
ConstantInt::get(TopRegion->getCanonicalIV()->getScalarType(), 1));
23592362
auto *EntryALM = Builder.createNaryOp(VPInstruction::ActiveLaneMask,
23602363
{EntryIncrement, TC, ALMMultiplier}, DL,
23612364
"active.lane.mask.entry");
@@ -2391,21 +2394,23 @@ static VPActiveLaneMaskPHIRecipe *addVPLaneMaskPhiAndUpdateExitBranch(
23912394
/// TODO: Introduce explicit recipe for header-mask instead of searching
23922395
/// for the header-mask pattern manually.
23932396
static VPSingleDefRecipe *findHeaderMask(VPlan &Plan) {
2397+
VPRegionBlock *LoopRegion = Plan.getVectorLoopRegion();
23942398
SmallVector<VPValue *> WideCanonicalIVs;
2395-
auto *FoundWidenCanonicalIVUser = find_if(Plan.getCanonicalIV()->users(),
2396-
IsaPred<VPWidenCanonicalIVRecipe>);
2397-
assert(count_if(Plan.getCanonicalIV()->users(),
2399+
auto *FoundWidenCanonicalIVUser = find_if(
2400+
LoopRegion->getCanonicalIV()->users(), IsaPred<VPWidenCanonicalIVRecipe>);
2401+
assert(count_if(LoopRegion->getCanonicalIV()->users(),
23982402
IsaPred<VPWidenCanonicalIVRecipe>) <= 1 &&
23992403
"Must have at most one VPWideCanonicalIVRecipe");
2400-
if (FoundWidenCanonicalIVUser != Plan.getCanonicalIV()->users().end()) {
2404+
if (FoundWidenCanonicalIVUser !=
2405+
LoopRegion->getCanonicalIV()->users().end()) {
24012406
auto *WideCanonicalIV =
24022407
cast<VPWidenCanonicalIVRecipe>(*FoundWidenCanonicalIVUser);
24032408
WideCanonicalIVs.push_back(WideCanonicalIV);
24042409
}
24052410

24062411
// Also include VPWidenIntOrFpInductionRecipes that represent a widened
24072412
// version of the canonical induction.
2408-
VPBasicBlock *HeaderVPBB = Plan.getVectorLoopRegion()->getEntryBasicBlock();
2413+
VPBasicBlock *HeaderVPBB = LoopRegion->getEntryBasicBlock();
24092414
for (VPRecipeBase &Phi : HeaderVPBB->phis()) {
24102415
auto *WidenOriginalIV = dyn_cast<VPWidenIntOrFpInductionRecipe>(&Phi);
24112416
if (WidenOriginalIV && WidenOriginalIV->isCanonical())
@@ -2438,8 +2443,9 @@ void VPlanTransforms::addActiveLaneMask(
24382443
"DataAndControlFlowWithoutRuntimeCheck implies "
24392444
"UseActiveLaneMaskForControlFlow");
24402445

2441-
auto *FoundWidenCanonicalIVUser = find_if(Plan.getCanonicalIV()->users(),
2442-
IsaPred<VPWidenCanonicalIVRecipe>);
2446+
VPRegionBlock *LoopRegion = Plan.getVectorLoopRegion();
2447+
auto *FoundWidenCanonicalIVUser = find_if(
2448+
LoopRegion->getCanonicalIV()->users(), IsaPred<VPWidenCanonicalIVRecipe>);
24432449
assert(FoundWidenCanonicalIVUser &&
24442450
"Must have widened canonical IV when tail folding!");
24452451
VPSingleDefRecipe *HeaderMask = findHeaderMask(Plan);
@@ -2452,7 +2458,7 @@ void VPlanTransforms::addActiveLaneMask(
24522458
} else {
24532459
VPBuilder B = VPBuilder::getToInsertAfter(WideCanonicalIV);
24542460
VPValue *ALMMultiplier = Plan.getOrAddLiveIn(
2455-
ConstantInt::get(Plan.getCanonicalIV()->getScalarType(), 1));
2461+
ConstantInt::get(LoopRegion->getCanonicalIV()->getScalarType(), 1));
24562462
LaneMask =
24572463
B.createNaryOp(VPInstruction::ActiveLaneMask,
24582464
{WideCanonicalIV, Plan.getTripCount(), ALMMultiplier},
@@ -2562,9 +2568,10 @@ static void transformRecipestoEVLRecipes(VPlan &Plan, VPValue &EVL) {
25622568
});
25632569

25642570
assert(all_of(Plan.getVFxUF().users(),
2565-
[&Plan](VPUser *U) {
2566-
return match(U, m_c_Add(m_Specific(Plan.getCanonicalIV()),
2567-
m_Specific(&Plan.getVFxUF()))) ||
2571+
[&LoopRegion, &Plan](VPUser *U) {
2572+
return match(U,
2573+
m_c_Add(m_Specific(LoopRegion->getCanonicalIV()),
2574+
m_Specific(&Plan.getVFxUF()))) ||
25682575
isa<VPWidenPointerInductionRecipe>(U);
25692576
}) &&
25702577
"Only users of VFxUF should be VPWidenPointerInductionRecipe and the "
@@ -2719,9 +2726,10 @@ void VPlanTransforms::addExplicitVectorLength(
27192726
VPlan &Plan, const std::optional<unsigned> &MaxSafeElements) {
27202727
if (Plan.hasScalarVFOnly())
27212728
return;
2722-
VPBasicBlock *Header = Plan.getVectorLoopRegion()->getEntryBasicBlock();
2729+
VPRegionBlock *LoopRegion = Plan.getVectorLoopRegion();
2730+
VPBasicBlock *Header = LoopRegion->getEntryBasicBlock();
27232731

2724-
auto *CanonicalIVPHI = Plan.getCanonicalIV();
2732+
auto *CanonicalIVPHI = LoopRegion->getCanonicalIV();
27252733
auto *CanIVTy = CanonicalIVPHI->getScalarType();
27262734
VPValue *StartV = CanonicalIVPHI->getStartValue();
27272735

@@ -4165,7 +4173,7 @@ void VPlanTransforms::narrowInterleaveGroups(VPlan &Plan, ElementCount VF,
41654173

41664174
// Adjust induction to reflect that the transformed plan only processes one
41674175
// original iteration.
4168-
auto *CanIV = Plan.getCanonicalIV();
4176+
auto *CanIV = VectorLoop->getCanonicalIV();
41694177
auto *Inc = cast<VPInstruction>(CanIV->getBackedgeValue());
41704178
VPBuilder PHBuilder(Plan.getVectorPreheader());
41714179

‎llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -69,7 +69,8 @@ class UnrollState {
6969
VPBasicBlock::iterator InsertPtForPhi);
7070

7171
VPValue *getConstantVPV(unsigned Part) {
72-
Type *CanIVIntTy = Plan.getCanonicalIV()->getScalarType();
72+
Type *CanIVIntTy =
73+
Plan.getVectorLoopRegion()->getCanonicalIV()->getScalarType();
7374
return Plan.getOrAddLiveIn(ConstantInt::get(CanIVIntTy, Part));
7475
}
7576

‎llvm/lib/Transforms/Vectorize/VPlanUtils.cpp‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -67,8 +67,10 @@ bool vputils::isHeaderMask(const VPValue *V, VPlan &Plan) {
6767

6868
if (match(V, m_ActiveLaneMask(m_VPValue(A), m_VPValue(B), m_One())))
6969
return B == Plan.getTripCount() &&
70-
(match(A, m_ScalarIVSteps(m_Specific(Plan.getCanonicalIV()), m_One(),
71-
m_Specific(&Plan.getVF()))) ||
70+
(match(A,
71+
m_ScalarIVSteps(
72+
m_Specific(Plan.getVectorLoopRegion()->getCanonicalIV()),
73+
m_One(), m_Specific(&Plan.getVF()))) ||
7274
IsWideCanonicalIV(A));
7375

7476
return match(V, m_ICmp(m_VPValue(A), m_VPValue(B))) && IsWideCanonicalIV(A) &&
@@ -102,7 +104,8 @@ bool vputils::isUniformAcrossVFsAndUFs(VPValue *V) {
102104
return all_of(R->operands(), isUniformAcrossVFsAndUFs);
103105
}
104106

105-
auto *CanonicalIV = R->getParent()->getPlan()->getCanonicalIV();
107+
auto *CanonicalIV =
108+
R->getParent()->getPlan()->getVectorLoopRegion()->getCanonicalIV();
106109
// Canonical IV chain is uniform.
107110
if (V == CanonicalIV || V == CanonicalIV->getBackedgeValue())
108111
return true;

0 commit comments

Comments
 (0)