Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Unify and split the pass pipeline #49798

Merged
merged 2 commits into from
Jun 7, 2023
Merged

Unify and split the pass pipeline #49798

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
d71c799
Unify and split the pass pipeline
pchintalapudi May 12, 2023
c3462ed
remove IF_O macros
pchintalapudi Jun 7, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
274 changes: 114 additions & 160 deletions src/pipeline.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -208,6 +208,9 @@ namespace {
// .sinkCommonInsts(true)
;
}
#if JL_LLVM_VERSION < 150000
#define LICMOptions()
#endif

// TODO(vchuravy/maleadt):
// Since we are not using the PassBuilder fully and instead rolling our own, we are missing out on
Expand Down Expand Up @@ -244,8 +247,7 @@ namespace {

#define JULIA_PASS(ADD_PASS) if (!options.llvm_only) { ADD_PASS; } else do { } while (0)

//Use for O1 and below
static void buildBasicPipeline(ModulePassManager &MPM, PassBuilder *PB, OptimizationLevel O, OptimizationOptions options) JL_NOTSAFEPOINT {
static void buildEarlySimplificationPipeline(ModulePassManager &MPM, PassBuilder *PB, OptimizationLevel O, const OptimizationOptions &options) JL_NOTSAFEPOINT {
#ifdef JL_DEBUG_BUILD
addVerificationPasses(MPM, options.llvm_only);
#endif
Expand All @@ -254,118 +256,32 @@ static void buildBasicPipeline(ModulePassManager &MPM, PassBuilder *PB, Optimiza
invokePipelineStartCallbacks(MPM, PB, O);
MPM.addPass(Annotation2MetadataPass());
MPM.addPass(ConstantMergePass());
if (!options.dump_native) {
JULIA_PASS(MPM.addPass(CPUFeaturesPass()));
if (O.getSpeedupLevel() > 0) {
MPM.addPass(createModuleToFunctionPassAdaptor(InstSimplifyPass()));
}
}
{
FunctionPassManager FPM;
FPM.addPass(LowerExpectIntrinsicPass());
if (O.getSpeedupLevel() >= 2) {
JULIA_PASS(FPM.addPass(PropagateJuliaAddrspacesPass()));
}
FPM.addPass(SimplifyCFGPass(basicSimplifyCFGOptions()));
if (O.getSpeedupLevel() > 0) {
if (O.getSpeedupLevel() >= 1) {
FPM.addPass(DCEPass());
FPM.addPass(SROAPass());
FPM.addPass(InstCombinePass());
FPM.addPass(EarlyCSEPass());
}
FPM.addPass(MemCpyOptPass());
MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM)));
}
invokeEarlySimplificationCallbacks(MPM, PB, O);
MPM.addPass(AlwaysInlinerPass());
{
CGSCCPassManager CGPM;
invokeCGSCCCallbacks(CGPM, PB, O);
MPM.addPass(createModuleToPostOrderCGSCCPassAdaptor(std::move(CGPM)));
}
invokeOptimizerEarlyCallbacks(MPM, PB, O);
JULIA_PASS(MPM.addPass(LowerSIMDLoopPass()));
{
FunctionPassManager FPM;
{
LoopPassManager LPM;
invokeLateLoopOptimizationCallbacks(LPM, PB, O);
invokeLoopOptimizerEndCallbacks(LPM, PB, O);
FPM.addPass(createFunctionToLoopPassAdaptor(std::move(LPM)));
}
invokeScalarOptimizerCallbacks(FPM, PB, O);
invokeVectorizerCallbacks(FPM, PB, O);
MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM)));
}
if (options.lower_intrinsics) {
//TODO no barrier pass?
{
FunctionPassManager FPM;
JULIA_PASS(FPM.addPass(LowerExcHandlersPass()));
JULIA_PASS(FPM.addPass(GCInvariantVerifierPass(false)));
MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM)));
}
JULIA_PASS(MPM.addPass(RemoveNIPass()));
JULIA_PASS(MPM.addPass(createModuleToFunctionPassAdaptor(LateLowerGCPass())));
JULIA_PASS(MPM.addPass(FinalLowerGCPass()));
JULIA_PASS(MPM.addPass(LowerPTLSPass(options.dump_native)));
} else {
JULIA_PASS(MPM.addPass(RemoveNIPass()));
}
JULIA_PASS(MPM.addPass(LowerSIMDLoopPass())); // TODO why do we do this twice
if (options.dump_native) {
JULIA_PASS(MPM.addPass(MultiVersioningPass(options.external_use)));
JULIA_PASS(MPM.addPass(CPUFeaturesPass()));
if (O.getSpeedupLevel() > 0) {
FunctionPassManager FPM;
FPM.addPass(InstSimplifyPass());
FPM.addPass(SimplifyCFGPass(basicSimplifyCFGOptions()));
MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM)));
}
}
invokeOptimizerLastCallbacks(MPM, PB, O);
{
FunctionPassManager FPM;
FPM.addPass(WarnMissedTransformationsPass());
FPM.addPass(AnnotationRemarksPass());
MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM)));
}
addSanitizerPasses(MPM, O);
JULIA_PASS(MPM.addPass(createModuleToFunctionPassAdaptor(DemoteFloat16Pass())));
}

//Use for O2 and above
static void buildFullPipeline(ModulePassManager &MPM, PassBuilder *PB, OptimizationLevel O, OptimizationOptions options) JL_NOTSAFEPOINT {
#ifdef JL_DEBUG_BUILD
addVerificationPasses(MPM, options.llvm_only);
#endif
// Place after verification in case we want to force it anyways
MPM.addPass(ForceFunctionAttrsPass());
invokePipelineStartCallbacks(MPM, PB, O);
MPM.addPass(Annotation2MetadataPass());
MPM.addPass(ConstantMergePass());
{
FunctionPassManager FPM;
FPM.addPass(LowerExpectIntrinsicPass());
JULIA_PASS(FPM.addPass(PropagateJuliaAddrspacesPass()));
//TODO consider not using even basic simplification
//options here, and adding a run of CVP to take advantage
//of the unsimplified codegen information (e.g. known
//zeros or ones)
FPM.addPass(SimplifyCFGPass(basicSimplifyCFGOptions()));
FPM.addPass(DCEPass());
FPM.addPass(SROAPass());
MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM)));
}
invokeEarlySimplificationCallbacks(MPM, PB, O);
MPM.addPass(AlwaysInlinerPass());
static void buildEarlyOptimizerPipeline(ModulePassManager &MPM, PassBuilder *PB, OptimizationLevel O, const OptimizationOptions &options) JL_NOTSAFEPOINT {
invokeOptimizerEarlyCallbacks(MPM, PB, O);
{
CGSCCPassManager CGPM;
invokeCGSCCCallbacks(CGPM, PB, O);
{
if (O.getSpeedupLevel() >= 2) {
FunctionPassManager FPM;
JULIA_PASS(FPM.addPass(AllocOptPass()));
FPM.addPass(Float2IntPass());
FPM.addPass(LowerConstantIntrinsicsPass());
FPM.addPass(InstCombinePass());
FPM.addPass(SimplifyCFGPass(basicSimplifyCFGOptions()));
CGPM.addPass(createCGSCCToFunctionPassAdaptor(std::move(FPM)));
}
MPM.addPass(createModuleToPostOrderCGSCCPassAdaptor(std::move(CGPM)));
Expand All @@ -374,53 +290,68 @@ static void buildFullPipeline(ModulePassManager &MPM, PassBuilder *PB, Optimizat
JULIA_PASS(MPM.addPass(MultiVersioningPass(options.external_use)));
}
JULIA_PASS(MPM.addPass(CPUFeaturesPass()));
{
if (O.getSpeedupLevel() >= 1) {
FunctionPassManager FPM;
FPM.addPass(SROAPass());
// SROA can duplicate PHI nodes which can block LowerSIMD
FPM.addPass(InstCombinePass());
FPM.addPass(JumpThreadingPass());
FPM.addPass(CorrelatedValuePropagationPass());
FPM.addPass(ReassociatePass());
FPM.addPass(EarlyCSEPass());
JULIA_PASS(FPM.addPass(AllocOptPass()));
if (O.getSpeedupLevel() >= 2) {
FPM.addPass(SROAPass());
// SROA can duplicate PHI nodes which can block LowerSIMD
FPM.addPass(InstCombinePass());
FPM.addPass(JumpThreadingPass());
FPM.addPass(CorrelatedValuePropagationPass());
FPM.addPass(ReassociatePass());
FPM.addPass(EarlyCSEPass());
JULIA_PASS(FPM.addPass(AllocOptPass()));
} else { // if (O.getSpeedupLevel() >= 1) (exactly)
FPM.addPass(InstCombinePass());
FPM.addPass(EarlyCSEPass());
}
invokePeepholeEPCallbacks(FPM, PB, O);
MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM)));
}
JULIA_PASS(MPM.addPass(LowerSIMDLoopPass()));
}

static void buildLoopOptimizerPipeline(FunctionPassManager &FPM, PassBuilder *PB, OptimizationLevel O, const OptimizationOptions &options) JL_NOTSAFEPOINT {
{
FunctionPassManager FPM;
{
LoopPassManager LPM1, LPM2;
LPM1.addPass(LoopRotatePass());
invokeLateLoopOptimizationCallbacks(LPM1, PB, O);
//We don't know if the loop callbacks support MSSA
FPM.addPass(createFunctionToLoopPassAdaptor(std::move(LPM1), /*UseMemorySSA = */false));
#if JL_LLVM_VERSION < 150000
#define LICMOptions()
#endif
LPM2.addPass(LICMPass(LICMOptions()));
JULIA_PASS(LPM2.addPass(JuliaLICMPass()));
LPM2.addPass(SimpleLoopUnswitchPass(/*NonTrivial*/true, true));
LPM2.addPass(LICMPass(LICMOptions()));
JULIA_PASS(LPM2.addPass(JuliaLICMPass()));
//LICM needs MemorySSA now, so we must use it
FPM.addPass(createFunctionToLoopPassAdaptor(std::move(LPM2), /*UseMemorySSA = */true));
LoopPassManager LPM;
if (O.getSpeedupLevel() >= 2) {
LPM.addPass(LoopRotatePass());
}
invokeLateLoopOptimizationCallbacks(LPM, PB, O);
//We don't know if the loop callbacks support MSSA
FPM.addPass(createFunctionToLoopPassAdaptor(std::move(LPM), /*UseMemorySSA = */false));
}
if (O.getSpeedupLevel() >= 2) {
LoopPassManager LPM;
LPM.addPass(LICMPass(LICMOptions()));
LPM.addPass(JuliaLICMPass());
LPM.addPass(SimpleLoopUnswitchPass(/*NonTrivial*/true, true));
LPM.addPass(LICMPass(LICMOptions()));
LPM.addPass(JuliaLICMPass());
//LICM needs MemorySSA now, so we must use it
FPM.addPass(createFunctionToLoopPassAdaptor(std::move(LPM), /*UseMemorySSA = */true));
}
if (O.getSpeedupLevel() >= 2) {
FPM.addPass(IRCEPass());
{
LoopPassManager LPM;
}
{
LoopPassManager LPM;
if (O.getSpeedupLevel() >= 2) {
LPM.addPass(LoopInstSimplifyPass());
LPM.addPass(LoopIdiomRecognizePass());
LPM.addPass(IndVarSimplifyPass());
LPM.addPass(LoopDeletionPass());
// This unroll will only unroll loops when the trip count is known and small,
// so that no loop remains
LPM.addPass(LoopFullUnrollPass());
invokeLoopOptimizerEndCallbacks(LPM, PB, O);
//We don't know if the loop end callbacks support MSSA
FPM.addPass(createFunctionToLoopPassAdaptor(std::move(LPM), /*UseMemorySSA = */false));
}
invokeLoopOptimizerEndCallbacks(LPM, PB, O);
//We don't know if the loop end callbacks support MSSA
FPM.addPass(createFunctionToLoopPassAdaptor(std::move(LPM), /*UseMemorySSA = */false));
}
}

static void buildScalarOptimizerPipeline(FunctionPassManager &FPM, PassBuilder *PB, OptimizationLevel O, const OptimizationOptions &options) JL_NOTSAFEPOINT {
if (O.getSpeedupLevel() >= 2) {
JULIA_PASS(FPM.addPass(AllocOptPass()));
FPM.addPass(SROAPass());
FPM.addPass(InstSimplifyPass());
Expand All @@ -432,9 +363,11 @@ static void buildFullPipeline(ModulePassManager &MPM, PassBuilder *PB, Optimizat
FPM.addPass(IRCEPass());
FPM.addPass(InstCombinePass());
FPM.addPass(JumpThreadingPass());
if (O.getSpeedupLevel() >= 3) {
FPM.addPass(GVNPass());
}
}
if (O.getSpeedupLevel() >= 3) {
FPM.addPass(GVNPass());
}
if (O.getSpeedupLevel() >= 2) {
FPM.addPass(DSEPass());
invokePeepholeEPCallbacks(FPM, PB, O);
FPM.addPass(SimplifyCFGPass(aggressiveSimplifyCFGOptions()));
Expand All @@ -446,24 +379,28 @@ static void buildFullPipeline(ModulePassManager &MPM, PassBuilder *PB, Optimizat
FPM.addPass(createFunctionToLoopPassAdaptor(std::move(LPM)));
}
FPM.addPass(LoopDistributePass());
FPM.addPass(InjectTLIMappings());
invokeScalarOptimizerCallbacks(FPM, PB, O);
//TODO look into loop vectorize options
FPM.addPass(LoopVectorizePass());
FPM.addPass(LoopLoadEliminationPass());
FPM.addPass(InstCombinePass());
FPM.addPass(SimplifyCFGPass(aggressiveSimplifyCFGOptions()));
FPM.addPass(SLPVectorizerPass());
invokeVectorizerCallbacks(FPM, PB, O);
FPM.addPass(VectorCombinePass());
FPM.addPass(ADCEPass());
//TODO add BDCEPass here?
// This unroll will unroll vectorized loops
// as well as loops that we tried but failed to vectorize
FPM.addPass(LoopUnrollPass(LoopUnrollOptions(O.getSpeedupLevel(), /*OnlyWhenForced = */ false, /*ForgetSCEV = */false)));
FPM.addPass(WarnMissedTransformationsPass());
MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM)));
}
invokeScalarOptimizerCallbacks(FPM, PB, O);
}

static void buildVectorPipeline(FunctionPassManager &FPM, PassBuilder *PB, OptimizationLevel O, const OptimizationOptions &options) JL_NOTSAFEPOINT {
//TODO look into loop vectorize options
FPM.addPass(InjectTLIMappings());
FPM.addPass(LoopVectorizePass());
FPM.addPass(LoopLoadEliminationPass());
FPM.addPass(InstCombinePass());
FPM.addPass(SimplifyCFGPass(aggressiveSimplifyCFGOptions()));
FPM.addPass(SLPVectorizerPass());
invokeVectorizerCallbacks(FPM, PB, O);
FPM.addPass(VectorCombinePass());
FPM.addPass(ADCEPass());
//TODO add BDCEPass here?
// This unroll will unroll vectorized loops
// as well as loops that we tried but failed to vectorize
FPM.addPass(LoopUnrollPass(LoopUnrollOptions(O.getSpeedupLevel(), /*OnlyWhenForced = */ false, /*ForgetSCEV = */false)));
}

static void buildIntrinsicLoweringPipeline(ModulePassManager &MPM, PassBuilder *PB, OptimizationLevel O, const OptimizationOptions &options) JL_NOTSAFEPOINT {
if (options.lower_intrinsics) {
//TODO barrier pass?
{
Expand All @@ -477,15 +414,15 @@ static void buildFullPipeline(ModulePassManager &MPM, PassBuilder *PB, Optimizat
JULIA_PASS(MPM.addPass(RemoveNIPass()));
JULIA_PASS(MPM.addPass(createModuleToFunctionPassAdaptor(LateLowerGCPass())));
JULIA_PASS(MPM.addPass(FinalLowerGCPass()));
{
if (O.getSpeedupLevel() >= 2) {
FunctionPassManager FPM;
FPM.addPass(GVNPass());
FPM.addPass(SCCPPass());
FPM.addPass(DCEPass());
MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM)));
}
JULIA_PASS(MPM.addPass(LowerPTLSPass(options.dump_native)));
{
if (O.getSpeedupLevel() >= 1) {
FunctionPassManager FPM;
FPM.addPass(InstCombinePass());
FPM.addPass(SimplifyCFGPass(aggressiveSimplifyCFGOptions()));
Expand All @@ -494,7 +431,10 @@ static void buildFullPipeline(ModulePassManager &MPM, PassBuilder *PB, Optimizat
} else {
JULIA_PASS(MPM.addPass(RemoveNIPass()));
}
{
}

static void buildCleanupPipeline(ModulePassManager &MPM, PassBuilder *PB, OptimizationLevel O, const OptimizationOptions &options) JL_NOTSAFEPOINT {
if (O.getSpeedupLevel() >= 2) {
FunctionPassManager FPM;
JULIA_PASS(FPM.addPass(CombineMulAddPass()));
FPM.addPass(DivRemPairsPass());
Expand All @@ -506,9 +446,30 @@ static void buildFullPipeline(ModulePassManager &MPM, PassBuilder *PB, Optimizat
{
FunctionPassManager FPM;
JULIA_PASS(FPM.addPass(DemoteFloat16Pass()));
FPM.addPass(GVNPass());
if (O.getSpeedupLevel() >= 2) {
FPM.addPass(GVNPass());
}
MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM)));
}
}

static void buildPipeline(ModulePassManager &MPM, PassBuilder *PB, OptimizationLevel O, const OptimizationOptions &options) JL_NOTSAFEPOINT {
buildEarlySimplificationPipeline(MPM, PB, O, options);
MPM.addPass(AlwaysInlinerPass());
buildEarlyOptimizerPipeline(MPM, PB, O, options);
MPM.addPass(LowerSIMDLoopPass());
{
FunctionPassManager FPM;
buildLoopOptimizerPipeline(FPM, PB, O, options);
buildScalarOptimizerPipeline(FPM, PB, O, options);
if (O.getSpeedupLevel() >= 2) {
buildVectorPipeline(FPM, PB, O, options);
}
FPM.addPass(WarnMissedTransformationsPass());
MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM)));
}
buildIntrinsicLoweringPipeline(MPM, PB, O, options);
buildCleanupPipeline(MPM, PB, O, options);
}

#undef JULIA_PASS
Expand Down Expand Up @@ -586,10 +547,7 @@ PIC->addClassToPassName(decltype(CREATE_PASS)::name(), NAME);

ModulePassManager createMPM(PassBuilder &PB, OptimizationLevel O, OptimizationOptions options) JL_NOTSAFEPOINT {
ModulePassManager MPM;
if (O.getSpeedupLevel() < 2)
buildBasicPipeline(MPM, &PB, O, options);
else
buildFullPipeline(MPM, &PB, O, options);
buildPipeline(MPM, &PB, O, options);
return MPM;
}
}
Expand Down Expand Up @@ -726,11 +684,7 @@ void registerCallbacks(PassBuilder &PB) JL_NOTSAFEPOINT {
auto julia_options = parseJuliaPipelineOptions(Name);
if (julia_options) {
ModulePassManager pipeline;
if (julia_options->first.getSpeedupLevel() < 2) {
buildBasicPipeline(pipeline, nullptr, julia_options->first, julia_options->second);
} else {
buildFullPipeline(pipeline, nullptr, julia_options->first, julia_options->second);
}
buildPipeline(pipeline, nullptr, julia_options->first, julia_options->second);
PM.addPass(std::move(pipeline));
return true;
}
Expand Down