Skip to content

Commit

Permalink
[NFC][LoopVectorize] Cache result of requiresScalarEpilogue
Browse files Browse the repository at this point in the history 
Caching the decision returned by requiresScalarEpilogue means that
we can avoid printing out the same debug many times, and also
avoids repeating the same calculation. This function will get more
complex when we start to reason about more early exit loops, such
as in PR llvm#88385. The only problem with this is we sometimes have to
invalidate the previous result due to changes in the scalar epilogue
status or interleave groups.
  • Loading branch information
@david-arm
david-arm committed Nov 13, 2024
1 parent 77afd9e commit 0bb1b0a
Show file tree
Hide file tree
Showing 2 changed files with 61 additions and 29 deletions.
78 changes: 57 additions & 21 deletions llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1347,27 +1347,46 @@ class LoopVectorizationCostModel {
return InterleaveInfo.getInterleaveGroup(Instr);
}

/// Calculate in advance whether a scalar epilogue is required when
/// vectorizing and not vectorizing. If \p Invalidate is true then
/// invalidate a previous decision.
void collectScalarEpilogueRequirements(bool Invalidate) {
auto NeedsScalarEpilogue = [&](bool IsVectorizing) -> bool {
if (!isScalarEpilogueAllowed()) {
LLVM_DEBUG(dbgs() << "LV: Loop does not require scalar epilogue");
return false;
}
// If we might exit from anywhere but the latch, must run the exiting
// iteration in scalar form.
if (TheLoop->getExitingBlock() != TheLoop->getLoopLatch()) {
LLVM_DEBUG(dbgs() << "LV: Loop requires scalar epilogue: not exiting "
"from latch block\n");
return true;
}
if (IsVectorizing && InterleaveInfo.requiresScalarEpilogue()) {
LLVM_DEBUG(dbgs() << "LV: Loop requires scalar epilogue: "
"interleaved group requires scalar epilogue");
return true;
}
LLVM_DEBUG(dbgs() << "LV: Loop does not require scalar epilogue");
return false;
};

assert((Invalidate || !RequiresScalarEpilogue) &&
"Already determined scalar epilogue requirements!");
std::pair<bool, bool> Result;
Result.first = NeedsScalarEpilogue(true);
LLVM_DEBUG(dbgs() << ", when vectorizing\n");
Result.second = NeedsScalarEpilogue(false);
LLVM_DEBUG(dbgs() << ", when not vectorizing\n");
RequiresScalarEpilogue = Result;
}

/// Returns true if we're required to use a scalar epilogue for at least
/// the final iteration of the original loop.
bool requiresScalarEpilogue(bool IsVectorizing) const {
if (!isScalarEpilogueAllowed()) {
LLVM_DEBUG(dbgs() << "LV: Loop does not require scalar epilogue\n");
return false;
}
// If we might exit from anywhere but the latch, must run the exiting
// iteration in scalar form.
if (TheLoop->getExitingBlock() != TheLoop->getLoopLatch()) {
LLVM_DEBUG(dbgs() << "LV: Loop requires scalar epilogue: not exiting "
"from latch block\n");
return true;
}
if (IsVectorizing && InterleaveInfo.requiresScalarEpilogue()) {
LLVM_DEBUG(dbgs() << "LV: Loop requires scalar epilogue: "
"interleaved group requires scalar epilogue\n");
return true;
}
LLVM_DEBUG(dbgs() << "LV: Loop does not require scalar epilogue\n");
return false;
auto &CachedResult = *RequiresScalarEpilogue;
return IsVectorizing ? CachedResult.first : CachedResult.second;
}

/// Returns true if we're required to use a scalar epilogue for at least
Expand All @@ -1391,6 +1410,15 @@ class LoopVectorizationCostModel {
return ScalarEpilogueStatus == CM_ScalarEpilogueAllowed;
}

/// Update the ScalarEpilogueStatus to a new value, potentially triggering a
/// recalculation of the scalar epilogue requirements.
void setScalarEpilogueStatus(ScalarEpilogueLowering Status) {
bool Changed = ScalarEpilogueStatus != Status;
ScalarEpilogueStatus = Status;
if (Changed)
collectScalarEpilogueRequirements(/*Invalidate=*/true);
}

/// Returns the TailFoldingStyle that is best for the current loop.
TailFoldingStyle getTailFoldingStyle(bool IVUpdateMayOverflow = true) const {
if (!ChosenTailFoldingStyle)
Expand Down Expand Up @@ -1771,6 +1799,9 @@ class LoopVectorizationCostModel {

/// All element types found in the loop.
SmallPtrSet<Type *, 16> ElementTypesInLoop;

/// Keeps track of whether we require a scalar epilogue.
std::optional<std::pair<bool, bool>> RequiresScalarEpilogue;
};
} // end namespace llvm

Expand Down Expand Up @@ -4058,7 +4089,7 @@ LoopVectorizationCostModel::computeMaxVF(ElementCount UserVF, unsigned UserIC) {
if (ScalarEpilogueStatus == CM_ScalarEpilogueNotNeededUsePredicate) {
LLVM_DEBUG(dbgs() << "LV: Cannot fold tail by masking: vectorize with a "
"scalar epilogue instead.\n");
ScalarEpilogueStatus = CM_ScalarEpilogueAllowed;
setScalarEpilogueStatus(CM_ScalarEpilogueAllowed);
return computeFeasibleMaxVF(MaxTC, UserVF, false);
}
return FixedScalableVFPair::getNone();
Expand All @@ -4074,6 +4105,7 @@ LoopVectorizationCostModel::computeMaxVF(ElementCount UserVF, unsigned UserIC) {
// Note: There is no need to invalidate any cost modeling decisions here, as
// none were taken so far.
InterleaveInfo.invalidateGroupsRequiringScalarEpilogue();
collectScalarEpilogueRequirements(/*Invalidate=*/true);
}

FixedScalableVFPair MaxFactors = computeFeasibleMaxVF(MaxTC, UserVF, true);
Expand Down Expand Up @@ -4145,7 +4177,7 @@ LoopVectorizationCostModel::computeMaxVF(ElementCount UserVF, unsigned UserIC) {
if (ScalarEpilogueStatus == CM_ScalarEpilogueNotNeededUsePredicate) {
LLVM_DEBUG(dbgs() << "LV: Cannot fold tail by masking: vectorize with a "
"scalar epilogue instead.\n");
ScalarEpilogueStatus = CM_ScalarEpilogueAllowed;
setScalarEpilogueStatus(CM_ScalarEpilogueAllowed);
return MaxFactors;
}

Expand Down Expand Up @@ -7058,6 +7090,7 @@ LoopVectorizationPlanner::planInVPlanNativePath(ElementCount UserVF) {
if (!OrigLoop->isInnermost()) {
// If the user doesn't provide a vectorization factor, determine a
// reasonable one.
CM.collectScalarEpilogueRequirements(/*Invalidate=*/false);
if (UserVF.isZero()) {
VF = determineVPlanVF(TTI, CM);
LLVM_DEBUG(dbgs() << "LV: VPlan computed VF " << VF << ".\n");
Expand Down Expand Up @@ -7102,6 +7135,7 @@ LoopVectorizationPlanner::planInVPlanNativePath(ElementCount UserVF) {

void LoopVectorizationPlanner::plan(ElementCount UserVF, unsigned UserIC) {
assert(OrigLoop->isInnermost() && "Inner loop expected.");
CM.collectScalarEpilogueRequirements(/*Invalidate=*/false);
CM.collectValuesToIgnore();
CM.collectElementTypesForWidening();

Expand All @@ -7116,11 +7150,13 @@ void LoopVectorizationPlanner::plan(ElementCount UserVF, unsigned UserIC) {
dbgs()
<< "LV: Invalidate all interleaved groups due to fold-tail by masking "
"which requires masked-interleaved support.\n");
if (CM.InterleaveInfo.invalidateGroups())
if (CM.InterleaveInfo.invalidateGroups()) {
// Invalidating interleave groups also requires invalidating all decisions
// based on them, which includes widening decisions and uniform and scalar
// values.
CM.invalidateCostModelingDecisions();
CM.collectScalarEpilogueRequirements(/*Invalidate=*/true);
}
}

if (CM.foldTailByMasking())
Expand Down
12 changes: 4 additions & 8 deletions llvm/test/Transforms/LoopVectorize/RISCV/riscv-vector-reverse.ll
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,8 @@ define void @vector_reverse_i64(ptr nocapture noundef writeonly %A, ptr nocaptur
; CHECK-NEXT: LV: Found an induction variable.
; CHECK-NEXT: LV: Did not find one integer induction var.
; CHECK-NEXT: LV: We can vectorize this loop (with a runtime bound check)!
; CHECK-NEXT: LV: Loop does not require scalar epilogue
; CHECK-NEXT: LV: Loop does not require scalar epilogue, when vectorizing
; CHECK-NEXT: LV: Loop does not require scalar epilogue, when not vectorizing
; CHECK-NEXT: LV: Found trip count: 0
; CHECK-NEXT: LV: Found maximum trip count: 4294967295
; CHECK-NEXT: LV: Scalable vectorization is available
Expand Down Expand Up @@ -45,7 +46,6 @@ define void @vector_reverse_i64(ptr nocapture noundef writeonly %A, ptr nocaptur
; CHECK-NEXT: LV: Found an estimated cost of 1 for VF vscale x 4 For instruction: %indvars.iv.next = add nsw i64 %indvars.iv, -1
; CHECK-NEXT: LV: Found an estimated cost of 0 for VF vscale x 4 For instruction: br i1 %cmp, label %for.body, label %for.cond.cleanup.loopexit, !llvm.loop !0
; CHECK-NEXT: LV: Using user VF vscale x 4.
; CHECK-NEXT: LV: Loop does not require scalar epilogue
; CHECK-NEXT: LV: Scalarizing: %i.0 = add nsw i32 %i.0.in8, -1
; CHECK-NEXT: LV: Scalarizing: %idxprom = zext i32 %i.0 to i64
; CHECK-NEXT: LV: Scalarizing: %arrayidx = getelementptr inbounds i32, ptr %B, i64 %idxprom
Expand Down Expand Up @@ -134,7 +134,6 @@ define void @vector_reverse_i64(ptr nocapture noundef writeonly %A, ptr nocaptur
; CHECK-NEXT: LV(REG): RegisterClass: RISCV::GPRRC, 1 registers
; CHECK-NEXT: LV: The target has 31 registers of RISCV::GPRRC register class
; CHECK-NEXT: LV: The target has 32 registers of RISCV::VRRC register class
; CHECK-NEXT: LV: Loop does not require scalar epilogue
; CHECK-NEXT: LV: Loop cost is 32
; CHECK-NEXT: LV: IC is 1
; CHECK-NEXT: LV: VF is vscale x 4
Expand Down Expand Up @@ -194,7 +193,6 @@ define void @vector_reverse_i64(ptr nocapture noundef writeonly %A, ptr nocaptur
; CHECK: IR %indvars.iv.next = add nsw i64 %indvars.iv, -1
; CHECK-NEXT: No successors
; CHECK-NEXT: }
; CHECK: LV: Loop does not require scalar epilogue
;
entry:
%cmp7 = icmp sgt i32 %n, 0
Expand Down Expand Up @@ -231,7 +229,8 @@ define void @vector_reverse_f32(ptr nocapture noundef writeonly %A, ptr nocaptur
; CHECK-NEXT: LV: Found FP op with unsafe algebra.
; CHECK-NEXT: LV: Did not find one integer induction var.
; CHECK-NEXT: LV: We can vectorize this loop (with a runtime bound check)!
; CHECK-NEXT: LV: Loop does not require scalar epilogue
; CHECK-NEXT: LV: Loop does not require scalar epilogue, when vectorizing
; CHECK-NEXT: LV: Loop does not require scalar epilogue, when not vectorizing
; CHECK-NEXT: LV: Found trip count: 0
; CHECK-NEXT: LV: Found maximum trip count: 4294967295
; CHECK-NEXT: LV: Scalable vectorization is available
Expand Down Expand Up @@ -259,7 +258,6 @@ define void @vector_reverse_f32(ptr nocapture noundef writeonly %A, ptr nocaptur
; CHECK-NEXT: LV: Found an estimated cost of 1 for VF vscale x 4 For instruction: %indvars.iv.next = add nsw i64 %indvars.iv, -1
; CHECK-NEXT: LV: Found an estimated cost of 0 for VF vscale x 4 For instruction: br i1 %cmp, label %for.body, label %for.cond.cleanup.loopexit, !llvm.loop !0
; CHECK-NEXT: LV: Using user VF vscale x 4.
; CHECK-NEXT: LV: Loop does not require scalar epilogue
; CHECK-NEXT: LV: Scalarizing: %i.0 = add nsw i32 %i.0.in8, -1
; CHECK-NEXT: LV: Scalarizing: %idxprom = zext i32 %i.0 to i64
; CHECK-NEXT: LV: Scalarizing: %arrayidx = getelementptr inbounds float, ptr %B, i64 %idxprom
Expand Down Expand Up @@ -348,7 +346,6 @@ define void @vector_reverse_f32(ptr nocapture noundef writeonly %A, ptr nocaptur
; CHECK-NEXT: LV(REG): RegisterClass: RISCV::GPRRC, 1 registers
; CHECK-NEXT: LV: The target has 31 registers of RISCV::GPRRC register class
; CHECK-NEXT: LV: The target has 32 registers of RISCV::VRRC register class
; CHECK-NEXT: LV: Loop does not require scalar epilogue
; CHECK-NEXT: LV: Loop cost is 34
; CHECK-NEXT: LV: IC is 1
; CHECK-NEXT: LV: VF is vscale x 4
Expand Down Expand Up @@ -408,7 +405,6 @@ define void @vector_reverse_f32(ptr nocapture noundef writeonly %A, ptr nocaptur
; CHECK: IR %indvars.iv.next = add nsw i64 %indvars.iv, -1
; CHECK-NEXT: No successors
; CHECK-NEXT: }
; CHECK: LV: Loop does not require scalar epilogue
;
entry:
%cmp7 = icmp sgt i32 %n, 0
Expand Down

0 comments on commit 0bb1b0a

Please sign in to comment.