From 93cdd1b5cfa3735c599949b77e24dbfbe570441a Mon Sep 17 00:00:00 2001 From: Arthur Eubanks Date: Mon, 12 Feb 2024 15:52:08 -0700 Subject: [PATCH] [PGO] Add ability to mark cold functions as optsize/minsize/optnone (#69030) The performance of cold functions shouldn't matter too much, so if we care about binary sizes, add an option to mark cold functions as optsize/minsize for binary size, or optnone for compile times [1]. Clang patch will be in a future patch. This is intended to replace `shouldOptimizeForSize(Function&, ...)`. We've seen multiple cases where calls to this expensive function, if not careful, can blow up compile times. I will clean up users of that function in a followup patch. Initial version: https://reviews.llvm.org/D149800 [1] https://discourse.llvm.org/t/rfc-new-feature-proposal-de-optimizing-cold-functions-using-pgo-info/56388 --- clang/lib/CodeGen/BackendUtil.cpp | 18 ++-- llvm/include/llvm/Support/PGOOptions.h | 3 + .../Instrumentation/PGOForceFunctionAttrs.h | 29 +++++ llvm/lib/LTO/LTOBackend.cpp | 12 ++- llvm/lib/Passes/PassBuilder.cpp | 1 + llvm/lib/Passes/PassBuilderPipelines.cpp | 10 ++ llvm/lib/Passes/PassRegistry.def | 1 + llvm/lib/Support/PGOOptions.cpp | 7 +- .../Transforms/Instrumentation/CMakeLists.txt | 1 + .../Instrumentation/PGOForceFunctionAttrs.cpp | 61 +++++++++++ .../PGOForceFunctionAttrs/basic.ll | 102 ++++++++++++++++++ llvm/tools/opt/NewPMDriver.cpp | 25 ++++- .../lib/Transforms/Instrumentation/BUILD.gn | 1 + 13 files changed, 253 insertions(+), 18 deletions(-) create mode 100644 llvm/include/llvm/Transforms/Instrumentation/PGOForceFunctionAttrs.h create mode 100644 llvm/lib/Transforms/Instrumentation/PGOForceFunctionAttrs.cpp create mode 100644 llvm/test/Instrumentation/PGOForceFunctionAttrs/basic.ll diff --git a/clang/lib/CodeGen/BackendUtil.cpp b/clang/lib/CodeGen/BackendUtil.cpp index e2511aa1b5c996..a310825240237c 100644 --- a/clang/lib/CodeGen/BackendUtil.cpp +++ b/clang/lib/CodeGen/BackendUtil.cpp @@ -748,7 +748,8 @@ void EmitAssemblyHelper::RunOptimizationPipeline( CodeGenOpts.InstrProfileOutput.empty() ? getDefaultProfileGenName() : CodeGenOpts.InstrProfileOutput, "", "", CodeGenOpts.MemoryProfileUsePath, nullptr, PGOOptions::IRInstr, - PGOOptions::NoCSAction, CodeGenOpts.DebugInfoForProfiling, + PGOOptions::NoCSAction, PGOOptions::ColdFuncOpt::Default, + CodeGenOpts.DebugInfoForProfiling, /*PseudoProbeForProfiling=*/false, CodeGenOpts.AtomicProfileUpdate); else if (CodeGenOpts.hasProfileIRUse()) { // -fprofile-use. @@ -757,28 +758,32 @@ void EmitAssemblyHelper::RunOptimizationPipeline( PGOOpt = PGOOptions( CodeGenOpts.ProfileInstrumentUsePath, "", CodeGenOpts.ProfileRemappingFile, CodeGenOpts.MemoryProfileUsePath, VFS, - PGOOptions::IRUse, CSAction, CodeGenOpts.DebugInfoForProfiling); + PGOOptions::IRUse, CSAction, PGOOptions::ColdFuncOpt::Default, + CodeGenOpts.DebugInfoForProfiling); } else if (!CodeGenOpts.SampleProfileFile.empty()) // -fprofile-sample-use PGOOpt = PGOOptions( CodeGenOpts.SampleProfileFile, "", CodeGenOpts.ProfileRemappingFile, CodeGenOpts.MemoryProfileUsePath, VFS, PGOOptions::SampleUse, - PGOOptions::NoCSAction, CodeGenOpts.DebugInfoForProfiling, - CodeGenOpts.PseudoProbeForProfiling); + PGOOptions::NoCSAction, PGOOptions::ColdFuncOpt::Default, + CodeGenOpts.DebugInfoForProfiling, CodeGenOpts.PseudoProbeForProfiling); else if (!CodeGenOpts.MemoryProfileUsePath.empty()) // -fmemory-profile-use (without any of the above options) PGOOpt = PGOOptions("", "", "", CodeGenOpts.MemoryProfileUsePath, VFS, PGOOptions::NoAction, PGOOptions::NoCSAction, + PGOOptions::ColdFuncOpt::Default, CodeGenOpts.DebugInfoForProfiling); else if (CodeGenOpts.PseudoProbeForProfiling) // -fpseudo-probe-for-profiling PGOOpt = PGOOptions("", "", "", /*MemoryProfile=*/"", nullptr, PGOOptions::NoAction, PGOOptions::NoCSAction, + PGOOptions::ColdFuncOpt::Default, CodeGenOpts.DebugInfoForProfiling, true); else if (CodeGenOpts.DebugInfoForProfiling) // -fdebug-info-for-profiling PGOOpt = PGOOptions("", "", "", /*MemoryProfile=*/"", nullptr, - PGOOptions::NoAction, PGOOptions::NoCSAction, true); + PGOOptions::NoAction, PGOOptions::NoCSAction, + PGOOptions::ColdFuncOpt::Default, true); // Check to see if we want to generate a CS profile. if (CodeGenOpts.hasProfileCSIRInstr()) { @@ -801,7 +806,8 @@ void EmitAssemblyHelper::RunOptimizationPipeline( ? getDefaultProfileGenName() : CodeGenOpts.InstrProfileOutput, "", /*MemoryProfile=*/"", nullptr, PGOOptions::NoAction, - PGOOptions::CSIRInstr, CodeGenOpts.DebugInfoForProfiling); + PGOOptions::CSIRInstr, PGOOptions::ColdFuncOpt::Default, + CodeGenOpts.DebugInfoForProfiling); } if (TM) TM->setPGOOption(PGOOpt); diff --git a/llvm/include/llvm/Support/PGOOptions.h b/llvm/include/llvm/Support/PGOOptions.h index 87eb29a8de48a0..de981abf187058 100644 --- a/llvm/include/llvm/Support/PGOOptions.h +++ b/llvm/include/llvm/Support/PGOOptions.h @@ -27,10 +27,12 @@ class FileSystem; struct PGOOptions { enum PGOAction { NoAction, IRInstr, IRUse, SampleUse }; enum CSPGOAction { NoCSAction, CSIRInstr, CSIRUse }; + enum class ColdFuncOpt { Default, OptSize, MinSize, OptNone }; PGOOptions(std::string ProfileFile, std::string CSProfileGenFile, std::string ProfileRemappingFile, std::string MemoryProfile, IntrusiveRefCntPtr FS, PGOAction Action = NoAction, CSPGOAction CSAction = NoCSAction, + ColdFuncOpt ColdType = ColdFuncOpt::Default, bool DebugInfoForProfiling = false, bool PseudoProbeForProfiling = false, bool AtomicCounterUpdate = false); @@ -44,6 +46,7 @@ struct PGOOptions { std::string MemoryProfile; PGOAction Action; CSPGOAction CSAction; + ColdFuncOpt ColdOptType; bool DebugInfoForProfiling; bool PseudoProbeForProfiling; bool AtomicCounterUpdate; diff --git a/llvm/include/llvm/Transforms/Instrumentation/PGOForceFunctionAttrs.h b/llvm/include/llvm/Transforms/Instrumentation/PGOForceFunctionAttrs.h new file mode 100644 index 00000000000000..785448e0dec4d9 --- /dev/null +++ b/llvm/include/llvm/Transforms/Instrumentation/PGOForceFunctionAttrs.h @@ -0,0 +1,29 @@ +//===- PGOForceFunctionAttrs.h - --------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_TRANSFORMS_INSTRUMENTATION_PGOFORCEFUNCTIONATTRS_H +#define LLVM_TRANSFORMS_INSTRUMENTATION_PGOFORCEFUNCTIONATTRS_H + +#include "llvm/IR/PassManager.h" +#include "llvm/Support/PGOOptions.h" + +namespace llvm { + +struct PGOForceFunctionAttrsPass + : public PassInfoMixin { + PGOForceFunctionAttrsPass(PGOOptions::ColdFuncOpt ColdType) + : ColdType(ColdType) {} + PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM); + +private: + PGOOptions::ColdFuncOpt ColdType; +}; + +} // namespace llvm + +#endif // LLVM_TRANSFORMS_INSTRUMENTATION_PGOFORCEFUNCTIONATTRS_H diff --git a/llvm/lib/LTO/LTOBackend.cpp b/llvm/lib/LTO/LTOBackend.cpp index ccc4276e36dacf..7b3a7590dfa743 100644 --- a/llvm/lib/LTO/LTOBackend.cpp +++ b/llvm/lib/LTO/LTOBackend.cpp @@ -243,19 +243,23 @@ static void runNewPMPasses(const Config &Conf, Module &Mod, TargetMachine *TM, if (!Conf.SampleProfile.empty()) PGOOpt = PGOOptions(Conf.SampleProfile, "", Conf.ProfileRemapping, /*MemoryProfile=*/"", FS, PGOOptions::SampleUse, - PGOOptions::NoCSAction, true); + PGOOptions::NoCSAction, + PGOOptions::ColdFuncOpt::Default, true); else if (Conf.RunCSIRInstr) { PGOOpt = PGOOptions("", Conf.CSIRProfile, Conf.ProfileRemapping, /*MemoryProfile=*/"", FS, PGOOptions::IRUse, - PGOOptions::CSIRInstr, Conf.AddFSDiscriminator); + PGOOptions::CSIRInstr, PGOOptions::ColdFuncOpt::Default, + Conf.AddFSDiscriminator); } else if (!Conf.CSIRProfile.empty()) { PGOOpt = PGOOptions(Conf.CSIRProfile, "", Conf.ProfileRemapping, /*MemoryProfile=*/"", FS, PGOOptions::IRUse, - PGOOptions::CSIRUse, Conf.AddFSDiscriminator); + PGOOptions::CSIRUse, PGOOptions::ColdFuncOpt::Default, + Conf.AddFSDiscriminator); NoPGOWarnMismatch = !Conf.PGOWarnMismatch; } else if (Conf.AddFSDiscriminator) { PGOOpt = PGOOptions("", "", "", /*MemoryProfile=*/"", nullptr, - PGOOptions::NoAction, PGOOptions::NoCSAction, true); + PGOOptions::NoAction, PGOOptions::NoCSAction, + PGOOptions::ColdFuncOpt::Default, true); } TM->setPGOOption(PGOOpt); diff --git a/llvm/lib/Passes/PassBuilder.cpp b/llvm/lib/Passes/PassBuilder.cpp index e3f25028fdabd8..f26d95ab1e479c 100644 --- a/llvm/lib/Passes/PassBuilder.cpp +++ b/llvm/lib/Passes/PassBuilder.cpp @@ -172,6 +172,7 @@ #include "llvm/Transforms/Instrumentation/KCFI.h" #include "llvm/Transforms/Instrumentation/MemProfiler.h" #include "llvm/Transforms/Instrumentation/MemorySanitizer.h" +#include "llvm/Transforms/Instrumentation/PGOForceFunctionAttrs.h" #include "llvm/Transforms/Instrumentation/PGOInstrumentation.h" #include "llvm/Transforms/Instrumentation/PoisonChecking.h" #include "llvm/Transforms/Instrumentation/SanitizerBinaryMetadata.h" diff --git a/llvm/lib/Passes/PassBuilderPipelines.cpp b/llvm/lib/Passes/PassBuilderPipelines.cpp index 4e233d99f942e9..142bd50b3798e0 100644 --- a/llvm/lib/Passes/PassBuilderPipelines.cpp +++ b/llvm/lib/Passes/PassBuilderPipelines.cpp @@ -74,6 +74,7 @@ #include "llvm/Transforms/Instrumentation/InstrOrderFile.h" #include "llvm/Transforms/Instrumentation/InstrProfiling.h" #include "llvm/Transforms/Instrumentation/MemProfiler.h" +#include "llvm/Transforms/Instrumentation/PGOForceFunctionAttrs.h" #include "llvm/Transforms/Instrumentation/PGOInstrumentation.h" #include "llvm/Transforms/Scalar/ADCE.h" #include "llvm/Transforms/Scalar/AlignmentFromAssumptions.h" @@ -213,6 +214,12 @@ static cl::opt cl::desc("Enable DFA jump threading"), cl::init(false), cl::Hidden); +// TODO: turn on and remove flag +static cl::opt EnablePGOForceFunctionAttrs( + "enable-pgo-force-function-attrs", + cl::desc("Enable pass to set function attributes based on PGO profiles"), + cl::init(false)); + static cl::opt EnableHotColdSplit("hot-cold-split", cl::desc("Enable hot-cold splitting pass")); @@ -1146,6 +1153,9 @@ PassBuilder::buildModuleSimplificationPipeline(OptimizationLevel Level, if (EnableSyntheticCounts && !PGOOpt) MPM.addPass(SyntheticCountsPropagation()); + if (EnablePGOForceFunctionAttrs) + MPM.addPass(PGOForceFunctionAttrsPass(PGOOpt->ColdOptType)); + MPM.addPass(AlwaysInlinerPass(/*InsertLifetimeIntrinsics=*/true)); if (EnableModuleInliner) diff --git a/llvm/lib/Passes/PassRegistry.def b/llvm/lib/Passes/PassRegistry.def index afa5a65fcbdef3..093c1f8aaad438 100644 --- a/llvm/lib/Passes/PassRegistry.def +++ b/llvm/lib/Passes/PassRegistry.def @@ -85,6 +85,7 @@ MODULE_PASS("lower-emutls", LowerEmuTLSPass()) MODULE_PASS("lower-global-dtors", LowerGlobalDtorsPass()) MODULE_PASS("lower-ifunc", LowerIFuncPass()) MODULE_PASS("lowertypetests", LowerTypeTestsPass()) +MODULE_PASS("pgo-force-function-attrs", PGOForceFunctionAttrsPass(PGOOpt ? PGOOpt->ColdOptType : PGOOptions::ColdFuncOpt::Default)) MODULE_PASS("memprof-context-disambiguation", MemProfContextDisambiguation()) MODULE_PASS("memprof-module", ModuleMemProfilerPass()) MODULE_PASS("mergefunc", MergeFunctionsPass()) diff --git a/llvm/lib/Support/PGOOptions.cpp b/llvm/lib/Support/PGOOptions.cpp index 7e57b52e4ba2f5..5981dff9e09468 100644 --- a/llvm/lib/Support/PGOOptions.cpp +++ b/llvm/lib/Support/PGOOptions.cpp @@ -15,11 +15,12 @@ PGOOptions::PGOOptions(std::string ProfileFile, std::string CSProfileGenFile, std::string ProfileRemappingFile, std::string MemoryProfile, IntrusiveRefCntPtr FS, PGOAction Action, - CSPGOAction CSAction, bool DebugInfoForProfiling, - bool PseudoProbeForProfiling, bool AtomicCounterUpdate) + CSPGOAction CSAction, ColdFuncOpt ColdType, + bool DebugInfoForProfiling, bool PseudoProbeForProfiling, + bool AtomicCounterUpdate) : ProfileFile(ProfileFile), CSProfileGenFile(CSProfileGenFile), ProfileRemappingFile(ProfileRemappingFile), MemoryProfile(MemoryProfile), - Action(Action), CSAction(CSAction), + Action(Action), CSAction(CSAction), ColdOptType(ColdType), DebugInfoForProfiling(DebugInfoForProfiling || (Action == SampleUse && !PseudoProbeForProfiling)), PseudoProbeForProfiling(PseudoProbeForProfiling), diff --git a/llvm/lib/Transforms/Instrumentation/CMakeLists.txt b/llvm/lib/Transforms/Instrumentation/CMakeLists.txt index 424f1d43360677..ee9aa73ff03403 100644 --- a/llvm/lib/Transforms/Instrumentation/CMakeLists.txt +++ b/llvm/lib/Transforms/Instrumentation/CMakeLists.txt @@ -13,6 +13,7 @@ add_llvm_component_library(LLVMInstrumentation InstrOrderFile.cpp InstrProfiling.cpp KCFI.cpp + PGOForceFunctionAttrs.cpp PGOInstrumentation.cpp PGOMemOPSizeOpt.cpp PoisonChecking.cpp diff --git a/llvm/lib/Transforms/Instrumentation/PGOForceFunctionAttrs.cpp b/llvm/lib/Transforms/Instrumentation/PGOForceFunctionAttrs.cpp new file mode 100644 index 00000000000000..c0ebdd7ed88635 --- /dev/null +++ b/llvm/lib/Transforms/Instrumentation/PGOForceFunctionAttrs.cpp @@ -0,0 +1,61 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "llvm/Transforms/Instrumentation/PGOForceFunctionAttrs.h" +#include "llvm/Analysis/BlockFrequencyInfo.h" +#include "llvm/Analysis/ProfileSummaryInfo.h" +#include "llvm/IR/PassManager.h" +#include "llvm/Support/ErrorHandling.h" + +using namespace llvm; + +static bool shouldRunOnFunction(Function &F, ProfileSummaryInfo &PSI, + FunctionAnalysisManager &FAM) { + if (F.isDeclaration()) + return false; + // Respect existing attributes. + if (F.hasOptNone() || F.hasOptSize() || F.hasMinSize()) + return false; + if (F.hasFnAttribute(Attribute::Cold)) + return true; + if (!PSI.hasProfileSummary()) + return false; + BlockFrequencyInfo &BFI = FAM.getResult(F); + return PSI.isFunctionColdInCallGraph(&F, BFI); +} + +PreservedAnalyses PGOForceFunctionAttrsPass::run(Module &M, + ModuleAnalysisManager &AM) { + if (ColdType == PGOOptions::ColdFuncOpt::Default) + return PreservedAnalyses::all(); + ProfileSummaryInfo &PSI = AM.getResult(M); + FunctionAnalysisManager &FAM = + AM.getResult(M).getManager(); + bool MadeChange = false; + for (Function &F : M) { + if (!shouldRunOnFunction(F, PSI, FAM)) + continue; + MadeChange = true; + switch (ColdType) { + case PGOOptions::ColdFuncOpt::Default: + llvm_unreachable("bailed out for default above"); + break; + case PGOOptions::ColdFuncOpt::OptSize: + F.addFnAttr(Attribute::OptimizeForSize); + break; + case PGOOptions::ColdFuncOpt::MinSize: + F.addFnAttr(Attribute::MinSize); + break; + case PGOOptions::ColdFuncOpt::OptNone: + F.addFnAttr(Attribute::OptimizeNone); + F.addFnAttr(Attribute::NoInline); + break; + } + } + return MadeChange ? PreservedAnalyses::none() : PreservedAnalyses::all(); +} diff --git a/llvm/test/Instrumentation/PGOForceFunctionAttrs/basic.ll b/llvm/test/Instrumentation/PGOForceFunctionAttrs/basic.ll new file mode 100644 index 00000000000000..29ebc0366040e1 --- /dev/null +++ b/llvm/test/Instrumentation/PGOForceFunctionAttrs/basic.ll @@ -0,0 +1,102 @@ +; RUN: opt < %s -passes=pgo-force-function-attrs -pgo-kind=pgo-instr-use-pipeline -S -pgo-cold-func-opt=default | FileCheck %s --check-prefixes=NONE,CHECK +; RUN: opt < %s -passes=pgo-force-function-attrs -pgo-kind=pgo-instr-use-pipeline -S -pgo-cold-func-opt=optsize | FileCheck %s --check-prefixes=OPTSIZE,CHECK +; RUN: opt < %s -passes=pgo-force-function-attrs -pgo-kind=pgo-instr-use-pipeline -S -pgo-cold-func-opt=minsize | FileCheck %s --check-prefixes=MINSIZE,CHECK +; RUN: opt < %s -passes=pgo-force-function-attrs -pgo-kind=pgo-instr-use-pipeline -S -pgo-cold-func-opt=optnone | FileCheck %s --check-prefixes=OPTNONE,CHECK + +; Should be no changes without profile data +; RUN: opt < %s -passes=pgo-force-function-attrs -S -pgo-cold-func-opt=minsize | FileCheck %s --check-prefixes=NONE,CHECK + +; NONE-NOT: Function Attrs: +; OPTSIZE: Function Attrs: optsize{{$}} +; MINSIZE: Function Attrs: minsize{{$}} +; OPTNONE: Function Attrs: noinline optnone{{$}} +; CHECK: define void @cold() + +; CHECK: Function Attrs: optsize{{$}} +; CHECK-NEXT: define void @cold_optsize() + +; CHECK: Function Attrs: minsize{{$}} +; CHECK-NEXT: define void @cold_minsize() + +; CHECK: Function Attrs: noinline optnone{{$}} +; CHECK-NEXT: define void @cold_optnone() + +; NONE: Function Attrs: cold{{$}} +; OPTSIZE: Function Attrs: cold optsize{{$}} +; MINSIZE: Function Attrs: cold minsize{{$}} +; OPTNONE: Function Attrs: cold noinline optnone{{$}} +; CHECK-NEXT: define void @cold_attr() + +; CHECK-NOT: Function Attrs: {{.*}}optsize +; CHECK-NOT: Function Attrs: {{.*}}minsize +; CHECK-NOT: Function Attrs: {{.*}}optnone + +@s = global i32 0 + +define void @cold() !prof !27 { + store i32 1, ptr @s, align 4 + ret void +} + +define void @cold_optsize() optsize !prof !27 { + store i32 1, ptr @s, align 4 + ret void +} + +define void @cold_minsize() minsize !prof !27 { + store i32 1, ptr @s, align 4 + ret void +} + +define void @cold_optnone() noinline optnone !prof !27 { + store i32 1, ptr @s, align 4 + ret void +} + +define void @cold_attr() cold { + store i32 1, ptr @s, align 4 + ret void +} + +define void @hot() !prof !28 { + %l = load i32, ptr @s, align 4 + %add = add nsw i32 %l, 4 + store i32 %add, ptr @s, align 4 + ret void +} + +attributes #0 = { optsize } +attributes #1 = { minsize } +attributes #2 = { noinline optnone } + +!llvm.module.flags = !{!0} + +!0 = !{i32 1, !"ProfileSummary", !1} +!1 = !{!2, !3, !4, !5, !6, !7, !8, !9} +!2 = !{!"ProfileFormat", !"InstrProf"} +!3 = !{!"TotalCount", i64 9040} +!4 = !{!"MaxCount", i64 9000} +!5 = !{!"MaxInternalCount", i64 0} +!6 = !{!"MaxFunctionCount", i64 9000} +!7 = !{!"NumCounts", i64 5} +!8 = !{!"NumFunctions", i64 5} +!9 = !{!"DetailedSummary", !10} +!10 = !{!11, !12, !13, !14, !15, !16, !17, !18, !19, !20, !21, !22, !23, !24, !25, !26} +!11 = !{i32 10000, i64 9000, i32 1} +!12 = !{i32 100000, i64 9000, i32 1} +!13 = !{i32 200000, i64 9000, i32 1} +!14 = !{i32 300000, i64 9000, i32 1} +!15 = !{i32 400000, i64 9000, i32 1} +!16 = !{i32 500000, i64 9000, i32 1} +!17 = !{i32 600000, i64 9000, i32 1} +!18 = !{i32 700000, i64 9000, i32 1} +!19 = !{i32 800000, i64 9000, i32 1} +!20 = !{i32 900000, i64 9000, i32 1} +!21 = !{i32 950000, i64 9000, i32 1} +!22 = !{i32 990000, i64 9000, i32 1} +!23 = !{i32 999000, i64 10, i32 5} +!24 = !{i32 999900, i64 10, i32 5} +!25 = !{i32 999990, i64 10, i32 5} +!26 = !{i32 999999, i64 10, i32 5} +!27 = !{!"function_entry_count", i64 10} +!28 = !{!"function_entry_count", i64 9000} diff --git a/llvm/tools/opt/NewPMDriver.cpp b/llvm/tools/opt/NewPMDriver.cpp index fdfb4df53273fe..374698083763b0 100644 --- a/llvm/tools/opt/NewPMDriver.cpp +++ b/llvm/tools/opt/NewPMDriver.cpp @@ -202,6 +202,19 @@ static cl::opt cl::desc("Path to the profile remapping file."), cl::Hidden); +static cl::opt PGOColdFuncAttr( + "pgo-cold-func-opt", cl::init(PGOOptions::ColdFuncOpt::Default), cl::Hidden, + cl::desc( + "Function attribute to apply to cold functions as determined by PGO"), + cl::values(clEnumValN(PGOOptions::ColdFuncOpt::Default, "default", + "Default (no attribute)"), + clEnumValN(PGOOptions::ColdFuncOpt::OptSize, "optsize", + "Mark cold functions with optsize."), + clEnumValN(PGOOptions::ColdFuncOpt::MinSize, "minsize", + "Mark cold functions with minsize."), + clEnumValN(PGOOptions::ColdFuncOpt::OptNone, "optnone", + "Mark cold functions with optnone."))); + static cl::opt DebugInfoForProfiling( "debug-info-for-profiling", cl::init(false), cl::Hidden, cl::desc("Emit special debug info to enable PGO profile generation.")); @@ -341,22 +354,24 @@ bool llvm::runPassPipeline( switch (PGOKindFlag) { case InstrGen: P = PGOOptions(ProfileFile, "", "", MemoryProfileFile, FS, - PGOOptions::IRInstr); + PGOOptions::IRInstr, PGOOptions::NoCSAction, + PGOColdFuncAttr); break; case InstrUse: P = PGOOptions(ProfileFile, "", ProfileRemappingFile, MemoryProfileFile, FS, - PGOOptions::IRUse); + PGOOptions::IRUse, PGOOptions::NoCSAction, PGOColdFuncAttr); break; case SampleUse: P = PGOOptions(ProfileFile, "", ProfileRemappingFile, MemoryProfileFile, FS, - PGOOptions::SampleUse); + PGOOptions::SampleUse, PGOOptions::NoCSAction, + PGOColdFuncAttr); break; case NoPGO: if (DebugInfoForProfiling || PseudoProbeForProfiling || !MemoryProfileFile.empty()) P = PGOOptions("", "", "", MemoryProfileFile, FS, PGOOptions::NoAction, - PGOOptions::NoCSAction, DebugInfoForProfiling, - PseudoProbeForProfiling); + PGOOptions::NoCSAction, PGOColdFuncAttr, + DebugInfoForProfiling, PseudoProbeForProfiling); else P = std::nullopt; } diff --git a/llvm/utils/gn/secondary/llvm/lib/Transforms/Instrumentation/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/Transforms/Instrumentation/BUILD.gn index 0cce6558e0093c..00e1888da64d26 100644 --- a/llvm/utils/gn/secondary/llvm/lib/Transforms/Instrumentation/BUILD.gn +++ b/llvm/utils/gn/secondary/llvm/lib/Transforms/Instrumentation/BUILD.gn @@ -23,6 +23,7 @@ static_library("Instrumentation") { "InstrProfiling.cpp", "Instrumentation.cpp", "KCFI.cpp", + "PGOForceFunctionAttrs.cpp", "MemProfiler.cpp", "MemorySanitizer.cpp", "PGOInstrumentation.cpp",