diff --git a/llvm/include/llvm/Transforms/InstCombine/InstCombiner.h b/llvm/include/llvm/Transforms/InstCombine/InstCombiner.h index 93090431cbb69..ea1f4fc3b85dc 100644 --- a/llvm/include/llvm/Transforms/InstCombine/InstCombiner.h +++ b/llvm/include/llvm/Transforms/InstCombine/InstCombiner.h @@ -76,6 +76,7 @@ class LLVM_LIBRARY_VISIBILITY InstCombiner { SimplifyQuery SQ; OptimizationRemarkEmitter &ORE; BlockFrequencyInfo *BFI; + BranchProbabilityInfo *BPI; ProfileSummaryInfo *PSI; DomConditionCache DC; @@ -96,13 +97,13 @@ class LLVM_LIBRARY_VISIBILITY InstCombiner { bool MinimizeSize, AAResults *AA, AssumptionCache &AC, TargetLibraryInfo &TLI, TargetTransformInfo &TTI, DominatorTree &DT, OptimizationRemarkEmitter &ORE, - BlockFrequencyInfo *BFI, ProfileSummaryInfo *PSI, - const DataLayout &DL, LoopInfo *LI) + BlockFrequencyInfo *BFI, BranchProbabilityInfo *BPI, + ProfileSummaryInfo *PSI, const DataLayout &DL, LoopInfo *LI) : TTI(TTI), Builder(Builder), Worklist(Worklist), MinimizeSize(MinimizeSize), AA(AA), AC(AC), TLI(TLI), DT(DT), DL(DL), SQ(DL, &TLI, &DT, &AC, nullptr, /*UseInstrInfo*/ true, /*CanUseUndef*/ true, &DC), - ORE(ORE), BFI(BFI), PSI(PSI), LI(LI) {} + ORE(ORE), BFI(BFI), BPI(BPI), PSI(PSI), LI(LI) {} virtual ~InstCombiner() = default; diff --git a/llvm/lib/Transforms/InstCombine/InstCombineInternal.h b/llvm/lib/Transforms/InstCombine/InstCombineInternal.h index f4d559036b31f..4479afbd09afd 100644 --- a/llvm/lib/Transforms/InstCombine/InstCombineInternal.h +++ b/llvm/lib/Transforms/InstCombine/InstCombineInternal.h @@ -67,10 +67,10 @@ class LLVM_LIBRARY_VISIBILITY InstCombinerImpl final bool MinimizeSize, AAResults *AA, AssumptionCache &AC, TargetLibraryInfo &TLI, TargetTransformInfo &TTI, DominatorTree &DT, OptimizationRemarkEmitter &ORE, - BlockFrequencyInfo *BFI, ProfileSummaryInfo *PSI, - const DataLayout &DL, LoopInfo *LI) + BlockFrequencyInfo *BFI, BranchProbabilityInfo *BPI, + ProfileSummaryInfo *PSI, const DataLayout &DL, LoopInfo *LI) : InstCombiner(Worklist, Builder, MinimizeSize, AA, AC, TLI, TTI, DT, ORE, - BFI, PSI, DL, LI) {} + BFI, BPI, PSI, DL, LI) {} virtual ~InstCombinerImpl() = default; diff --git a/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp b/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp index ee9b1afbb86ac..3ac5c2559ddf9 100644 --- a/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp +++ b/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp @@ -1347,9 +1347,13 @@ void InstCombinerImpl::freelyInvertAllUsersOf(Value *I, Value *IgnoredUser) { SI->swapProfMetadata(); break; } - case Instruction::Br: - cast(U)->swapSuccessors(); // swaps prof metadata too + case Instruction::Br: { + BranchInst *BI = cast(U); + BI->swapSuccessors(); // swaps prof metadata too + if (BPI) + BPI->swapSuccEdgesProbabilities(BI->getParent()); break; + } case Instruction::Xor: replaceInstUsesWith(cast(*U), I); // Add to worklist for DCE. @@ -3525,6 +3529,8 @@ Instruction *InstCombinerImpl::visitBranchInst(BranchInst &BI) { if (match(Cond, m_Not(m_Value(X))) && !isa(X)) { // Swap Destinations and condition... BI.swapSuccessors(); + if (BPI) + BPI->swapSuccEdgesProbabilities(BI.getParent()); return replaceOperand(BI, 0, X); } @@ -3538,6 +3544,8 @@ Instruction *InstCombinerImpl::visitBranchInst(BranchInst &BI) { Value *NotX = Builder.CreateNot(X, "not." + X->getName()); Value *Or = Builder.CreateLogicalOr(NotX, Y); BI.swapSuccessors(); + if (BPI) + BPI->swapSuccEdgesProbabilities(BI.getParent()); return replaceOperand(BI, 0, Or); } @@ -3554,6 +3562,8 @@ Instruction *InstCombinerImpl::visitBranchInst(BranchInst &BI) { auto *Cmp = cast(Cond); Cmp->setPredicate(CmpInst::getInversePredicate(Pred)); BI.swapSuccessors(); + if (BPI) + BPI->swapSuccEdgesProbabilities(BI.getParent()); Worklist.push(Cmp); return &BI; } @@ -5248,7 +5258,8 @@ static bool combineInstructionsOverFunction( Function &F, InstructionWorklist &Worklist, AliasAnalysis *AA, AssumptionCache &AC, TargetLibraryInfo &TLI, TargetTransformInfo &TTI, DominatorTree &DT, OptimizationRemarkEmitter &ORE, BlockFrequencyInfo *BFI, - ProfileSummaryInfo *PSI, LoopInfo *LI, const InstCombineOptions &Opts) { + BranchProbabilityInfo *BPI, ProfileSummaryInfo *PSI, LoopInfo *LI, + const InstCombineOptions &Opts) { auto &DL = F.getParent()->getDataLayout(); /// Builder - This is an IRBuilder that automatically inserts new @@ -5286,7 +5297,7 @@ static bool combineInstructionsOverFunction( << F.getName() << "\n"); InstCombinerImpl IC(Worklist, Builder, F.hasMinSize(), AA, AC, TLI, TTI, DT, - ORE, BFI, PSI, DL, LI); + ORE, BFI, BPI, PSI, DL, LI); IC.MaxArraySizeForCombine = MaxArraySize; bool MadeChangeInThisIteration = IC.prepareWorklist(F, RPOT); MadeChangeInThisIteration |= IC.run(); @@ -5347,9 +5358,10 @@ PreservedAnalyses InstCombinePass::run(Function &F, MAMProxy.getCachedResult(*F.getParent()); auto *BFI = (PSI && PSI->hasProfileSummary()) ? &AM.getResult(F) : nullptr; + auto *BPI = AM.getCachedResult(F); if (!combineInstructionsOverFunction(F, Worklist, AA, AC, TLI, TTI, DT, ORE, - BFI, PSI, LI, Options)) + BFI, BPI, PSI, LI, Options)) // No changes, all analyses are preserved. return PreservedAnalyses::all(); @@ -5396,9 +5408,14 @@ bool InstructionCombiningPass::runOnFunction(Function &F) { (PSI && PSI->hasProfileSummary()) ? &getAnalysis().getBFI() : nullptr; + BranchProbabilityInfo *BPI = nullptr; + if (auto *WrapperPass = + getAnalysisIfAvailable()) + BPI = &WrapperPass->getBPI(); return combineInstructionsOverFunction(F, Worklist, AA, AC, TLI, TTI, DT, ORE, - BFI, PSI, LI, InstCombineOptions()); + BFI, BPI, PSI, LI, + InstCombineOptions()); } char InstructionCombiningPass::ID = 0; diff --git a/llvm/test/Transforms/InstCombine/update-bpi.ll b/llvm/test/Transforms/InstCombine/update-bpi.ll index 190a686c8f078..fadb2ab16bff6 100644 --- a/llvm/test/Transforms/InstCombine/update-bpi.ll +++ b/llvm/test/Transforms/InstCombine/update-bpi.ll @@ -6,8 +6,8 @@ ; CHECK-NEXT: edge %entry -> %bb2 probability is 0x79e79e7a / 0x80000000 = 95.24% [HOT edge] ; CHECK-NEXT: Printing analysis 'Branch Probability Analysis' for function 'invert_cond': ; CHECK-NEXT: ---- Branch Probabilities ---- -; CHECK-NEXT: edge %entry -> %bb2 probability is 0x06186186 / 0x80000000 = 4.76% -; CHECK-NEXT: edge %entry -> %bb1 probability is 0x79e79e7a / 0x80000000 = 95.24% [HOT edge] +; CHECK-NEXT: edge %entry -> %bb2 probability is 0x79e79e7a / 0x80000000 = 95.24% [HOT edge] +; CHECK-NEXT: edge %entry -> %bb1 probability is 0x06186186 / 0x80000000 = 4.76% define i32 @invert_cond(ptr %p) { ; CHECK-LABEL: define i32 @invert_cond(