Skip to content

[VPlan] Manage Sentinel value for FindLastIV in VPlan. #142291

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
Jun 13, 2025
Merged

[VPlan] Manage Sentinel value for FindLastIV in VPlan. #142291

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
abc7c36
[VPlan] Manage Sentinel value for FindLastIV in VPlan.
fhahn May 29, 2025
abb0072
Merge branch 'main' into vplan-findlastiv-sentinel-value
fhahn Jun 13, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 12 additions & 8 deletions llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7266,9 +7266,11 @@ static void fixReductionScalarResumeWhenVectorizingEpilog(
using namespace llvm::PatternMatch;
Value *Cmp, *OrigResumeV, *CmpOp;
Copy link
Collaborator

@ayalz ayalz Jun 13, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Value *Sentinel = ‏EpiRedResult->getOperand(2)->getLiveInIRValue();

Perhaps worth a Value *vputils::getSentinel(VPInstruction*)?

Copy link
Contributor Author

@fhahn fhahn Jun 14, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Done, thanks!

bool IsExpectedPattern =
match(MainResumeValue, m_Select(m_OneUse(m_Value(Cmp)),
m_Specific(RdxDesc.getSentinelValue()),
m_Value(OrigResumeV))) &&
match(MainResumeValue,
m_Select(
m_OneUse(m_Value(Cmp)),
m_Specific(EpiRedResult->getOperand(2)->getLiveInIRValue()),
m_Value(OrigResumeV))) &&
(match(Cmp, m_SpecificICmp(ICmpInst::ICMP_EQ, m_Specific(OrigResumeV),
m_Value(CmpOp))) &&
((CmpOp == StartV && isGuaranteedNotToBeUndefOrPoison(CmpOp))));
Expand Down Expand Up @@ -9235,9 +9237,11 @@ void LoopVectorizationPlanner::adjustRecipesForReductions(
if (RecurrenceDescriptor::isFindLastIVRecurrenceKind(
RdxDesc.getRecurrenceKind())) {
VPValue *Start = PhiR->getStartValue();
Copy link
Collaborator

@ayalz ayalz Jun 13, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

VPValue *Sentinel = ...

Copy link
Contributor Author

@fhahn fhahn Jun 14, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Done thanks

FinalReductionResult =
Builder.createNaryOp(VPInstruction::ComputeFindLastIVResult,
{PhiR, Start, NewExitingVPV}, ExitDL);
FinalReductionResult = Builder.createNaryOp(
VPInstruction::ComputeFindLastIVResult,
{PhiR, Start, Plan->getOrAddLiveIn(RdxDesc.getSentinelValue()),
NewExitingVPV},
ExitDL);
} else if (RecurrenceDescriptor::isAnyOfRecurrenceKind(
RdxDesc.getRecurrenceKind())) {
VPValue *Start = PhiR->getStartValue();
Expand Down Expand Up @@ -9825,8 +9829,8 @@ preparePlanForEpilogueVectorLoop(VPlan &Plan, Loop *L,
BasicBlock *ResumeBB = cast<Instruction>(ResumeV)->getParent();
IRBuilder<> Builder(ResumeBB, ResumeBB->getFirstNonPHIIt());
Value *Cmp = Builder.CreateICmpEQ(ResumeV, ToFrozen[StartV]);
Copy link
Collaborator

@ayalz ayalz Jun 13, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Value *Sentinel = ...

Copy link
Contributor Author

@fhahn fhahn Jun 14, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Done thanks

ResumeV =
Builder.CreateSelect(Cmp, RdxDesc.getSentinelValue(), ResumeV);
ResumeV = Builder.CreateSelect(
Cmp, RdxResult->getOperand(2)->getLiveInIRValue(), ResumeV);
} else {
VPValue *StartVal = Plan.getOrAddLiveIn(ResumeV);
auto *PhiR = dyn_cast<VPReductionPHIRecipe>(&R);
Expand Down
19 changes: 19 additions & 0 deletions llvm/lib/Transforms/Vectorize/VPlanPatternMatch.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -318,6 +318,25 @@ m_VPInstruction(const Op0_t &Op0, const Op1_t &Op1, const Op2_t &Op2) {
{Op0, Op1, Op2});
}

template <typename Op0_t, typename Op1_t, typename Op2_t, typename Op3_t,
unsigned Opcode, bool Commutative, typename... RecipeTys>
using Recipe4Op_match = Recipe_match<std::tuple<Op0_t, Op1_t, Op2_t, Op3_t>,
Opcode, Commutative, RecipeTys...>;

template <typename Op0_t, typename Op1_t, typename Op2_t, typename Op3_t,
unsigned Opcode>
using VPInstruction4Op_match =
Recipe4Op_match<Op0_t, Op1_t, Op2_t, Op3_t, Opcode, /*Commutative*/ false,
VPInstruction>;

template <unsigned Opcode, typename Op0_t, typename Op1_t, typename Op2_t,
typename Op3_t>
inline VPInstruction4Op_match<Op0_t, Op1_t, Op2_t, Op3_t, Opcode>
m_VPInstruction(const Op0_t &Op0, const Op1_t &Op1, const Op2_t &Op2,
const Op3_t &Op3) {
return VPInstruction4Op_match<Op0_t, Op1_t, Op2_t, Op3_t, Opcode>(
{Op0, Op1, Op2, Op3});
}
template <typename Op0_t>
inline UnaryVPInstruction_match<Op0_t, Instruction::Freeze>
m_Freeze(const Op0_t &Op0) {
Expand Down
8 changes: 4 additions & 4 deletions llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -645,16 +645,16 @@ Value *VPInstruction::generate(VPTransformState &State) {

// The recipe's operands are the reduction phi, followed by one operand for
// each part of the reduction.
Copy link
Collaborator

@ayalz ayalz Jun 13, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Comment needs updating? Operands better be explaining under /// somewhere?

Copy link
Contributor Author

@fhahn fhahn Jun 14, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Done thanks

unsigned UF = getNumOperands() - 2;
Value *ReducedPartRdx = State.get(getOperand(2));
unsigned UF = getNumOperands() - 3;
Value *ReducedPartRdx = State.get(getOperand(3));
for (unsigned Part = 1; Part < UF; ++Part) {
ReducedPartRdx = createMinMaxOp(Builder, RecurKind::SMax, ReducedPartRdx,
State.get(getOperand(2 + Part)));
State.get(getOperand(3 + Part)));
}

Copy link
Collaborator

@ayalz ayalz Jun 13, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Value *Sentinel = ...

Copy link
Contributor Author

@fhahn fhahn Jun 14, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Done, also for Start

return createFindLastIVReduction(Builder, ReducedPartRdx,
State.get(getOperand(1), true),
RdxDesc.getSentinelValue());
getOperand(2)->getLiveInIRValue());
}
case VPInstruction::ComputeReductionResult: {
// FIXME: The cross-recipe dependency on VPReductionPHIRecipe is temporary
Expand Down
2 changes: 1 addition & 1 deletion llvm/lib/Transforms/Vectorize/VPlanUnroll.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -347,7 +347,7 @@ void UnrollState::unrollBlock(VPBlockBase *VPB) {
match(&R, m_VPInstruction<VPInstruction::ComputeReductionResult>(
m_VPValue(), m_VPValue(Op1))) ||
match(&R, m_VPInstruction<VPInstruction::ComputeFindLastIVResult>(
m_VPValue(), m_VPValue(), m_VPValue(Op1)))) {
m_VPValue(), m_VPValue(), m_VPValue(), m_VPValue(Op1)))) {
addUniformForAllParts(cast<VPInstruction>(&R));
for (unsigned Part = 1; Part != UF; ++Part)
R.addOperand(getValueForPart(Op1, Part));
Expand Down
2 changes: 1 addition & 1 deletion llvm/test/Transforms/LoopVectorize/vplan-printing-reductions.ll
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -240,7 +240,7 @@ define i64 @find_last_iv(ptr %a, i64 %n, i64 %start) {
; CHECK-NEXT: Successor(s): middle.block
; CHECK-EMPTY:
; CHECK-NEXT: middle.block:
; CHECK-NEXT: EMIT vp<[[RDX_RES:%.+]]> = compute-find-last-iv-result ir<%rdx>, ir<%start>, ir<%cond>
; CHECK-NEXT: EMIT vp<[[RDX_RES:%.+]]> = compute-find-last-iv-result ir<%rdx>, ir<%start>, ir<-9223372036854775808>, ir<%cond>
; CHECK-NEXT: EMIT vp<[[EXT:%.+]]> = extract-last-element vp<[[RDX_RES]]>
; CHECK-NEXT: EMIT vp<%cmp.n> = icmp eq ir<%n>, vp<{{.+}}>
; CHECK-NEXT: EMIT branch-on-cond vp<%cmp.n>
Expand Down
Loading