Skip to content

[LV] Bundle sub reductions into VPExpressionRecipe #147255

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 10 commits into from
Sep 1, 2025
Merged

[LV] Bundle sub reductions into VPExpressionRecipe #147255

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
d64d52c
[LV] Bundle sub reductions into VPExpressionRecipe
SamTebbs33 Jun 30, 2025
3171e7b
Move IsNegated parameter and cost sub
SamTebbs33 Jul 8, 2025
cacb89e
Improve getMulAccReductionCost comment
SamTebbs33 Aug 13, 2025
53fca5c
Accept reductions with sub opcode
SamTebbs33 Aug 20, 2025
55a9c3e
Rebase and remove negated expression type
SamTebbs33 Aug 20, 2025
27e462a
Rename printing test blocks
SamTebbs33 Aug 21, 2025
67a0604
Address review
SamTebbs33 Aug 27, 2025
33236a3
Rebase
SamTebbs33 Aug 29, 2025
b988436
Address approval review
SamTebbs33 Sep 1, 2025
0c55abc
Remove NFCs
SamTebbs33 Sep 1, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 5 additions & 5 deletions llvm/include/llvm/Analysis/TargetTransformInfo.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1647,12 +1647,12 @@ class TargetTransformInfo {
TTI::TargetCostKind CostKind = TTI::TCK_RecipThroughput) const;

/// Calculate the cost of an extended reduction pattern, similar to
/// getArithmeticReductionCost of an Add reduction with multiply and optional
/// extensions. This is the cost of as:
/// ResTy vecreduce.add(mul (A, B)).
/// ResTy vecreduce.add(mul(ext(Ty A), ext(Ty B)).
/// getArithmeticReductionCost of an Add/Sub reduction with multiply and
/// optional extensions. This is the cost of as:
/// * ResTy vecreduce.add/sub(mul (A, B)) or,
/// * ResTy vecreduce.add/sub(mul(ext(Ty A), ext(Ty B)).
LLVM_ABI InstructionCost getMulAccReductionCost(
bool IsUnsigned, Type *ResTy, VectorType *Ty,
bool IsUnsigned, unsigned RedOpcode, Type *ResTy, VectorType *Ty,
TTI::TargetCostKind CostKind = TTI::TCK_RecipThroughput) const;

/// Calculate the cost of an extended reduction pattern, similar to
Expand Down
4 changes: 2 additions & 2 deletions llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -971,8 +971,8 @@ class TargetTransformInfoImplBase {
}

virtual InstructionCost
getMulAccReductionCost(bool IsUnsigned, Type *ResTy, VectorType *Ty,
TTI::TargetCostKind CostKind) const {
getMulAccReductionCost(bool IsUnsigned, unsigned RedOpcode, Type *ResTy,
VectorType *Ty, TTI::TargetCostKind CostKind) const {
return 1;
}

Expand Down
7 changes: 5 additions & 2 deletions llvm/include/llvm/CodeGen/BasicTTIImpl.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3260,14 +3260,17 @@ class BasicTTIImplBase : public TargetTransformInfoImplCRTPBase<T> {
}

InstructionCost
getMulAccReductionCost(bool IsUnsigned, Type *ResTy, VectorType *Ty,
getMulAccReductionCost(bool IsUnsigned, unsigned RedOpcode, Type *ResTy,
VectorType *Ty,
TTI::TargetCostKind CostKind) const override {
// Without any native support, this is equivalent to the cost of
// vecreduce.add(mul(ext(Ty A), ext(Ty B))) or
// vecreduce.add(mul(A, B)).
assert((RedOpcode == Instruction::Add || RedOpcode == Instruction::Sub) &&
"The reduction opcode is expected to be Add or Sub.");
VectorType *ExtTy = VectorType::get(ResTy, Ty);
InstructionCost RedCost = thisT()->getArithmeticReductionCost(
Instruction::Add, ExtTy, std::nullopt, CostKind);
RedOpcode, ExtTy, std::nullopt, CostKind);
InstructionCost ExtCost = thisT()->getCastInstrCost(
IsUnsigned ? Instruction::ZExt : Instruction::SExt, ExtTy, Ty,
TTI::CastContextHint::None, CostKind);
Expand Down
5 changes: 3 additions & 2 deletions llvm/lib/Analysis/TargetTransformInfo.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1283,9 +1283,10 @@ InstructionCost TargetTransformInfo::getExtendedReductionCost(
}

InstructionCost TargetTransformInfo::getMulAccReductionCost(
bool IsUnsigned, Type *ResTy, VectorType *Ty,
bool IsUnsigned, unsigned RedOpcode, Type *ResTy, VectorType *Ty,
TTI::TargetCostKind CostKind) const {
return TTIImpl->getMulAccReductionCost(IsUnsigned, ResTy, Ty, CostKind);
return TTIImpl->getMulAccReductionCost(IsUnsigned, RedOpcode, ResTy, Ty,
CostKind);
}

InstructionCost
Expand Down
10 changes: 6 additions & 4 deletions llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5486,13 +5486,14 @@ InstructionCost AArch64TTIImpl::getExtendedReductionCost(
}

InstructionCost
AArch64TTIImpl::getMulAccReductionCost(bool IsUnsigned, Type *ResTy,
VectorType *VecTy,
AArch64TTIImpl::getMulAccReductionCost(bool IsUnsigned, unsigned RedOpcode,
Type *ResTy, VectorType *VecTy,
TTI::TargetCostKind CostKind) const {
EVT VecVT = TLI->getValueType(DL, VecTy);
EVT ResVT = TLI->getValueType(DL, ResTy);

if (ST->hasDotProd() && VecVT.isSimple() && ResVT.isSimple()) {
if (ST->hasDotProd() && VecVT.isSimple() && ResVT.isSimple() &&
RedOpcode == Instruction::Add) {
std::pair<InstructionCost, MVT> LT = getTypeLegalizationCost(VecTy);

// The legal cases with dotprod are
Expand All @@ -5503,7 +5504,8 @@ AArch64TTIImpl::getMulAccReductionCost(bool IsUnsigned, Type *ResTy,
return LT.first + 2;
}

return BaseT::getMulAccReductionCost(IsUnsigned, ResTy, VecTy, CostKind);
return BaseT::getMulAccReductionCost(IsUnsigned, RedOpcode, ResTy, VecTy,
CostKind);
}

InstructionCost
Expand Down
2 changes: 1 addition & 1 deletion llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -460,7 +460,7 @@ class AArch64TTIImpl final : public BasicTTIImplBase<AArch64TTIImpl> {
TTI::TargetCostKind CostKind) const override;

InstructionCost getMulAccReductionCost(
bool IsUnsigned, Type *ResTy, VectorType *Ty,
bool IsUnsigned, unsigned RedOpcode, Type *ResTy, VectorType *Ty,
TTI::TargetCostKind CostKind = TTI::TCK_RecipThroughput) const override;

InstructionCost
Expand Down
9 changes: 6 additions & 3 deletions llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1916,9 +1916,11 @@ InstructionCost ARMTTIImpl::getExtendedReductionCost(
}

InstructionCost
ARMTTIImpl::getMulAccReductionCost(bool IsUnsigned, Type *ResTy,
VectorType *ValTy,
ARMTTIImpl::getMulAccReductionCost(bool IsUnsigned, unsigned RedOpcode,
Type *ResTy, VectorType *ValTy,
TTI::TargetCostKind CostKind) const {
if (RedOpcode != Instruction::Add)
return InstructionCost::getInvalid(CostKind);
EVT ValVT = TLI->getValueType(DL, ValTy);
EVT ResVT = TLI->getValueType(DL, ResTy);

Expand All @@ -1939,7 +1941,8 @@ ARMTTIImpl::getMulAccReductionCost(bool IsUnsigned, Type *ResTy,
return ST->getMVEVectorCostFactor(CostKind) * LT.first;
}

return BaseT::getMulAccReductionCost(IsUnsigned, ResTy, ValTy, CostKind);
return BaseT::getMulAccReductionCost(IsUnsigned, RedOpcode, ResTy, ValTy,
CostKind);
}

InstructionCost
Expand Down
3 changes: 2 additions & 1 deletion llvm/lib/Target/ARM/ARMTargetTransformInfo.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -299,7 +299,8 @@ class ARMTTIImpl final : public BasicTTIImplBase<ARMTTIImpl> {
VectorType *ValTy, std::optional<FastMathFlags> FMF,
TTI::TargetCostKind CostKind) const override;
InstructionCost
getMulAccReductionCost(bool IsUnsigned, Type *ResTy, VectorType *ValTy,
getMulAccReductionCost(bool IsUnsigned, unsigned RedOpcode, Type *ResTy,
VectorType *ValTy,
TTI::TargetCostKind CostKind) const override;

InstructionCost
Expand Down
9 changes: 6 additions & 3 deletions llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5414,7 +5414,8 @@ LoopVectorizationCostModel::getReductionPatternCost(Instruction *I,
TTI::CastContextHint::None, CostKind, RedOp);

InstructionCost RedCost = TTI.getMulAccReductionCost(
IsUnsigned, RdxDesc.getRecurrenceType(), ExtType, CostKind);
IsUnsigned, RdxDesc.getOpcode(), RdxDesc.getRecurrenceType(), ExtType,
CostKind);

if (RedCost.isValid() &&
RedCost < ExtCost * 2 + MulCost + Ext2Cost + BaseCost)
Expand Down Expand Up @@ -5459,7 +5460,8 @@ LoopVectorizationCostModel::getReductionPatternCost(Instruction *I,
TTI.getArithmeticInstrCost(Instruction::Mul, VectorTy, CostKind);

InstructionCost RedCost = TTI.getMulAccReductionCost(
IsUnsigned, RdxDesc.getRecurrenceType(), ExtType, CostKind);
IsUnsigned, RdxDesc.getOpcode(), RdxDesc.getRecurrenceType(), ExtType,
CostKind);
InstructionCost ExtraExtCost = 0;
if (Op0Ty != LargestOpTy || Op1Ty != LargestOpTy) {
Instruction *ExtraExtOp = (Op0Ty != LargestOpTy) ? Op0 : Op1;
Expand All @@ -5478,7 +5480,8 @@ LoopVectorizationCostModel::getReductionPatternCost(Instruction *I,
TTI.getArithmeticInstrCost(Instruction::Mul, VectorTy, CostKind);

InstructionCost RedCost = TTI.getMulAccReductionCost(
true, RdxDesc.getRecurrenceType(), VectorTy, CostKind);
true, RdxDesc.getOpcode(), RdxDesc.getRecurrenceType(), VectorTy,
CostKind);

if (RedCost.isValid() && RedCost < MulCost + BaseCost)
return I == RetI ? RedCost : 0;
Expand Down
9 changes: 5 additions & 4 deletions llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2803,24 +2803,25 @@ InstructionCost VPExpressionRecipe::computeCost(ElementCount VF,
toVectorTy(Ctx.Types.inferScalarType(getOperand(0)), VF));
assert(RedTy->isIntegerTy() &&
"VPExpressionRecipe only supports integer types currently.");
unsigned Opcode = RecurrenceDescriptor::getOpcode(
cast<VPReductionRecipe>(ExpressionRecipes.back())->getRecurrenceKind());
switch (ExpressionType) {
case ExpressionTypes::ExtendedReduction: {
unsigned Opcode = RecurrenceDescriptor::getOpcode(
cast<VPReductionRecipe>(ExpressionRecipes[1])->getRecurrenceKind());
return Ctx.TTI.getExtendedReductionCost(
Opcode,
cast<VPWidenCastRecipe>(ExpressionRecipes.front())->getOpcode() ==
Instruction::ZExt,
RedTy, SrcVecTy, std::nullopt, Ctx.CostKind);
}
case ExpressionTypes::MulAccReduction:
return Ctx.TTI.getMulAccReductionCost(false, RedTy, SrcVecTy, Ctx.CostKind);
return Ctx.TTI.getMulAccReductionCost(false, Opcode, RedTy, SrcVecTy,
Ctx.CostKind);

case ExpressionTypes::ExtMulAccReduction:
return Ctx.TTI.getMulAccReductionCost(
cast<VPWidenCastRecipe>(ExpressionRecipes.front())->getOpcode() ==
Instruction::ZExt,
RedTy, SrcVecTy, Ctx.CostKind);
Opcode, RedTy, SrcVecTy, Ctx.CostKind);
}
llvm_unreachable("Unknown VPExpressionRecipe::ExpressionTypes enum");
}
Expand Down
6 changes: 3 additions & 3 deletions llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3151,7 +3151,7 @@ static VPExpressionRecipe *
tryToMatchAndCreateMulAccumulateReduction(VPReductionRecipe *Red,
VPCostContext &Ctx, VFRange &Range) {
unsigned Opcode = RecurrenceDescriptor::getOpcode(Red->getRecurrenceKind());
if (Opcode != Instruction::Add)
if (Opcode != Instruction::Add && Opcode != Instruction::Sub)
return nullptr;

Type *RedTy = Ctx.Types.inferScalarType(Red);
Expand All @@ -3166,8 +3166,8 @@ tryToMatchAndCreateMulAccumulateReduction(VPReductionRecipe *Red,
Type *SrcTy =
Ext0 ? Ctx.Types.inferScalarType(Ext0->getOperand(0)) : RedTy;
auto *SrcVecTy = cast<VectorType>(toVectorTy(SrcTy, VF));
InstructionCost MulAccCost =
Ctx.TTI.getMulAccReductionCost(isZExt, RedTy, SrcVecTy, CostKind);
InstructionCost MulAccCost = Ctx.TTI.getMulAccReductionCost(
isZExt, Opcode, RedTy, SrcVecTy, CostKind);
InstructionCost MulCost = Mul->computeCost(VF, Ctx);
InstructionCost RedCost = Red->computeCost(VF, Ctx);
InstructionCost ExtCost = 0;
Expand Down
4 changes: 2 additions & 2 deletions llvm/lib/Transforms/Vectorize/VectorCombine.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1468,8 +1468,8 @@ static void analyzeCostOfVecReduction(const IntrinsicInst &II,
TTI::CastContextHint::None, CostKind, RedOp);

CostBeforeReduction = ExtCost * 2 + MulCost + Ext2Cost;
CostAfterReduction =
TTI.getMulAccReductionCost(IsUnsigned, II.getType(), ExtType, CostKind);
CostAfterReduction = TTI.getMulAccReductionCost(
IsUnsigned, ReductionOpc, II.getType(), ExtType, CostKind);
Comment on lines +1471 to +1472
Copy link
Contributor

@fhahn fhahn Aug 29, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

it would be nice to have a test for this, but not sure if that's possible.

Copy link
Collaborator Author

@SamTebbs33 SamTebbs33 Sep 1, 2025 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I've been trying to make a test but I don't think this code is ever reached. The RedOp && match(RedOp, m_ZExtOrSExt(m_Value())) check above fully (AFAIK) encompasses this check so that code path is always followed instead. If I move this if statement block above that one above then the compiler fails the assertion at Type.cpp:805. This happens on main as well.

return;
}
CostAfterReduction = TTI.getArithmeticReductionCost(ReductionOpc, VecRedTy,
Expand Down
Loading