Skip to content

Commit

Permalink
Unify and split the pass pipeline (#49798)
Browse files Browse the repository at this point in the history 
Unify and split the pass pipeline
  • Loading branch information
@pchintalapudi
pchintalapudi authored Jun 7, 2023
1 parent 2e76fc4 commit 4200aa9
Showing 1 changed file with 114 additions and 160 deletions.
274 changes: 114 additions & 160 deletions src/pipeline.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -208,6 +208,9 @@ namespace {
// .sinkCommonInsts(true)
;
}
#if JL_LLVM_VERSION < 150000
#define LICMOptions()
#endif

// At any given time exactly one of each pair of overloads is strictly unused
#ifdef _COMPILER_GCC_
Expand Down Expand Up @@ -323,8 +326,7 @@ namespace {

#define JULIA_PASS(ADD_PASS) if (!options.llvm_only) { ADD_PASS; } else do { } while (0)

//Use for O1 and below
static void buildBasicPipeline(ModulePassManager &MPM, PassBuilder *PB, OptimizationLevel O, OptimizationOptions options) JL_NOTSAFEPOINT {
static void buildEarlySimplificationPipeline(ModulePassManager &MPM, PassBuilder *PB, OptimizationLevel O, const OptimizationOptions &options) JL_NOTSAFEPOINT {
#ifdef JL_DEBUG_BUILD
addVerificationPasses(MPM, options.llvm_only);
#endif
Expand All @@ -333,118 +335,32 @@ static void buildBasicPipeline(ModulePassManager &MPM, PassBuilder *PB, Optimiza
invokePipelineStartCallbacks(MPM, PB, O);
MPM.addPass(Annotation2MetadataPass());
MPM.addPass(ConstantMergePass());
if (!options.dump_native) {
JULIA_PASS(MPM.addPass(CPUFeaturesPass()));
if (O.getSpeedupLevel() > 0) {
MPM.addPass(createModuleToFunctionPassAdaptor(InstSimplifyPass()));
}
}
{
FunctionPassManager FPM;
FPM.addPass(LowerExpectIntrinsicPass());
if (O.getSpeedupLevel() >= 2) {
JULIA_PASS(FPM.addPass(PropagateJuliaAddrspacesPass()));
}
FPM.addPass(SimplifyCFGPass(basicSimplifyCFGOptions()));
if (O.getSpeedupLevel() > 0) {
if (O.getSpeedupLevel() >= 1) {
FPM.addPass(DCEPass());
FPM.addPass(SROAPass());
FPM.addPass(InstCombinePass());
FPM.addPass(EarlyCSEPass());
}
FPM.addPass(MemCpyOptPass());
MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM)));
}
invokeEarlySimplificationCallbacks(MPM, PB, O);
MPM.addPass(AlwaysInlinerPass());
{
CGSCCPassManager CGPM;
invokeCGSCCCallbacks(CGPM, PB, O);
MPM.addPass(createModuleToPostOrderCGSCCPassAdaptor(std::move(CGPM)));
}
invokeOptimizerEarlyCallbacks(MPM, PB, O);
JULIA_PASS(MPM.addPass(LowerSIMDLoopPass()));
{
FunctionPassManager FPM;
{
LoopPassManager LPM;
invokeLateLoopOptimizationCallbacks(LPM, PB, O);
invokeLoopOptimizerEndCallbacks(LPM, PB, O);
FPM.addPass(createFunctionToLoopPassAdaptor(std::move(LPM)));
}
invokeScalarOptimizerCallbacks(FPM, PB, O);
invokeVectorizerCallbacks(FPM, PB, O);
MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM)));
}
if (options.lower_intrinsics) {
//TODO no barrier pass?
{
FunctionPassManager FPM;
JULIA_PASS(FPM.addPass(LowerExcHandlersPass()));
JULIA_PASS(FPM.addPass(GCInvariantVerifierPass(false)));
MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM)));
}
JULIA_PASS(MPM.addPass(RemoveNIPass()));
JULIA_PASS(MPM.addPass(createModuleToFunctionPassAdaptor(LateLowerGCPass())));
JULIA_PASS(MPM.addPass(FinalLowerGCPass()));
JULIA_PASS(MPM.addPass(LowerPTLSPass(options.dump_native)));
} else {
JULIA_PASS(MPM.addPass(RemoveNIPass()));
}
JULIA_PASS(MPM.addPass(LowerSIMDLoopPass())); // TODO why do we do this twice
if (options.dump_native) {
JULIA_PASS(MPM.addPass(MultiVersioningPass(options.external_use)));
JULIA_PASS(MPM.addPass(CPUFeaturesPass()));
if (O.getSpeedupLevel() > 0) {
FunctionPassManager FPM;
FPM.addPass(InstSimplifyPass());
FPM.addPass(SimplifyCFGPass(basicSimplifyCFGOptions()));
MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM)));
}
}
invokeOptimizerLastCallbacks(MPM, PB, O);
{
FunctionPassManager FPM;
FPM.addPass(WarnMissedTransformationsPass());
FPM.addPass(AnnotationRemarksPass());
MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM)));
}
addSanitizerPasses(MPM, O);
JULIA_PASS(MPM.addPass(createModuleToFunctionPassAdaptor(DemoteFloat16Pass())));
}

//Use for O2 and above
static void buildFullPipeline(ModulePassManager &MPM, PassBuilder *PB, OptimizationLevel O, OptimizationOptions options) JL_NOTSAFEPOINT {
#ifdef JL_DEBUG_BUILD
addVerificationPasses(MPM, options.llvm_only);
#endif
// Place after verification in case we want to force it anyways
MPM.addPass(ForceFunctionAttrsPass());
invokePipelineStartCallbacks(MPM, PB, O);
MPM.addPass(Annotation2MetadataPass());
MPM.addPass(ConstantMergePass());
{
FunctionPassManager FPM;
FPM.addPass(LowerExpectIntrinsicPass());
JULIA_PASS(FPM.addPass(PropagateJuliaAddrspacesPass()));
//TODO consider not using even basic simplification
//options here, and adding a run of CVP to take advantage
//of the unsimplified codegen information (e.g. known
//zeros or ones)
FPM.addPass(SimplifyCFGPass(basicSimplifyCFGOptions()));
FPM.addPass(DCEPass());
FPM.addPass(SROAPass());
MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM)));
}
invokeEarlySimplificationCallbacks(MPM, PB, O);
MPM.addPass(AlwaysInlinerPass());
static void buildEarlyOptimizerPipeline(ModulePassManager &MPM, PassBuilder *PB, OptimizationLevel O, const OptimizationOptions &options) JL_NOTSAFEPOINT {
invokeOptimizerEarlyCallbacks(MPM, PB, O);
{
CGSCCPassManager CGPM;
invokeCGSCCCallbacks(CGPM, PB, O);
{
if (O.getSpeedupLevel() >= 2) {
FunctionPassManager FPM;
JULIA_PASS(FPM.addPass(AllocOptPass()));
FPM.addPass(Float2IntPass());
FPM.addPass(LowerConstantIntrinsicsPass());
FPM.addPass(InstCombinePass());
FPM.addPass(SimplifyCFGPass(basicSimplifyCFGOptions()));
CGPM.addPass(createCGSCCToFunctionPassAdaptor(std::move(FPM)));
}
MPM.addPass(createModuleToPostOrderCGSCCPassAdaptor(std::move(CGPM)));
Expand All @@ -453,53 +369,68 @@ static void buildFullPipeline(ModulePassManager &MPM, PassBuilder *PB, Optimizat
JULIA_PASS(MPM.addPass(MultiVersioningPass(options.external_use)));
}
JULIA_PASS(MPM.addPass(CPUFeaturesPass()));
{
if (O.getSpeedupLevel() >= 1) {
FunctionPassManager FPM;
FPM.addPass(SROAPass());
// SROA can duplicate PHI nodes which can block LowerSIMD
FPM.addPass(InstCombinePass());
FPM.addPass(JumpThreadingPass());
FPM.addPass(CorrelatedValuePropagationPass());
FPM.addPass(ReassociatePass());
FPM.addPass(EarlyCSEPass());
JULIA_PASS(FPM.addPass(AllocOptPass()));
if (O.getSpeedupLevel() >= 2) {
FPM.addPass(SROAPass());
// SROA can duplicate PHI nodes which can block LowerSIMD
FPM.addPass(InstCombinePass());
FPM.addPass(JumpThreadingPass());
FPM.addPass(CorrelatedValuePropagationPass());
FPM.addPass(ReassociatePass());
FPM.addPass(EarlyCSEPass());
JULIA_PASS(FPM.addPass(AllocOptPass()));
} else { // if (O.getSpeedupLevel() >= 1) (exactly)
FPM.addPass(InstCombinePass());
FPM.addPass(EarlyCSEPass());
}
invokePeepholeEPCallbacks(FPM, PB, O);
MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM)));
}
JULIA_PASS(MPM.addPass(LowerSIMDLoopPass()));
}

static void buildLoopOptimizerPipeline(FunctionPassManager &FPM, PassBuilder *PB, OptimizationLevel O, const OptimizationOptions &options) JL_NOTSAFEPOINT {
{
FunctionPassManager FPM;
{
LoopPassManager LPM1, LPM2;
LPM1.addPass(LoopRotatePass());
invokeLateLoopOptimizationCallbacks(LPM1, PB, O);
//We don't know if the loop callbacks support MSSA
FPM.addPass(createFunctionToLoopPassAdaptor(std::move(LPM1), /*UseMemorySSA = */false));
#if JL_LLVM_VERSION < 150000
#define LICMOptions()
#endif
LPM2.addPass(LICMPass(LICMOptions()));
JULIA_PASS(LPM2.addPass(JuliaLICMPass()));
LPM2.addPass(SimpleLoopUnswitchPass(/*NonTrivial*/true, true));
LPM2.addPass(LICMPass(LICMOptions()));
JULIA_PASS(LPM2.addPass(JuliaLICMPass()));
//LICM needs MemorySSA now, so we must use it
FPM.addPass(createFunctionToLoopPassAdaptor(std::move(LPM2), /*UseMemorySSA = */true));
LoopPassManager LPM;
if (O.getSpeedupLevel() >= 2) {
LPM.addPass(LoopRotatePass());
}
invokeLateLoopOptimizationCallbacks(LPM, PB, O);
//We don't know if the loop callbacks support MSSA
FPM.addPass(createFunctionToLoopPassAdaptor(std::move(LPM), /*UseMemorySSA = */false));
}
if (O.getSpeedupLevel() >= 2) {
LoopPassManager LPM;
LPM.addPass(LICMPass(LICMOptions()));
LPM.addPass(JuliaLICMPass());
LPM.addPass(SimpleLoopUnswitchPass(/*NonTrivial*/true, true));
LPM.addPass(LICMPass(LICMOptions()));
LPM.addPass(JuliaLICMPass());
//LICM needs MemorySSA now, so we must use it
FPM.addPass(createFunctionToLoopPassAdaptor(std::move(LPM), /*UseMemorySSA = */true));
}
if (O.getSpeedupLevel() >= 2) {
FPM.addPass(IRCEPass());
{
LoopPassManager LPM;
}
{
LoopPassManager LPM;
if (O.getSpeedupLevel() >= 2) {
LPM.addPass(LoopInstSimplifyPass());
LPM.addPass(LoopIdiomRecognizePass());
LPM.addPass(IndVarSimplifyPass());
LPM.addPass(LoopDeletionPass());
// This unroll will only unroll loops when the trip count is known and small,
// so that no loop remains
LPM.addPass(LoopFullUnrollPass());
invokeLoopOptimizerEndCallbacks(LPM, PB, O);
//We don't know if the loop end callbacks support MSSA
FPM.addPass(createFunctionToLoopPassAdaptor(std::move(LPM), /*UseMemorySSA = */false));
}
invokeLoopOptimizerEndCallbacks(LPM, PB, O);
//We don't know if the loop end callbacks support MSSA
FPM.addPass(createFunctionToLoopPassAdaptor(std::move(LPM), /*UseMemorySSA = */false));
}
}

static void buildScalarOptimizerPipeline(FunctionPassManager &FPM, PassBuilder *PB, OptimizationLevel O, const OptimizationOptions &options) JL_NOTSAFEPOINT {
if (O.getSpeedupLevel() >= 2) {
JULIA_PASS(FPM.addPass(AllocOptPass()));
FPM.addPass(SROAPass());
FPM.addPass(InstSimplifyPass());
Expand All @@ -511,9 +442,11 @@ static void buildFullPipeline(ModulePassManager &MPM, PassBuilder *PB, Optimizat
FPM.addPass(IRCEPass());
FPM.addPass(InstCombinePass());
FPM.addPass(JumpThreadingPass());
if (O.getSpeedupLevel() >= 3) {
FPM.addPass(GVNPass());
}
}
if (O.getSpeedupLevel() >= 3) {
FPM.addPass(GVNPass());
}
if (O.getSpeedupLevel() >= 2) {
FPM.addPass(DSEPass());
invokePeepholeEPCallbacks(FPM, PB, O);
FPM.addPass(SimplifyCFGPass(aggressiveSimplifyCFGOptions()));
Expand All @@ -525,24 +458,28 @@ static void buildFullPipeline(ModulePassManager &MPM, PassBuilder *PB, Optimizat
FPM.addPass(createFunctionToLoopPassAdaptor(std::move(LPM)));
}
FPM.addPass(LoopDistributePass());
FPM.addPass(InjectTLIMappings());
invokeScalarOptimizerCallbacks(FPM, PB, O);
//TODO look into loop vectorize options
FPM.addPass(LoopVectorizePass());
FPM.addPass(LoopLoadEliminationPass());
FPM.addPass(InstCombinePass());
FPM.addPass(SimplifyCFGPass(aggressiveSimplifyCFGOptions()));
FPM.addPass(SLPVectorizerPass());
invokeVectorizerCallbacks(FPM, PB, O);
FPM.addPass(VectorCombinePass());
FPM.addPass(ADCEPass());
//TODO add BDCEPass here?
// This unroll will unroll vectorized loops
// as well as loops that we tried but failed to vectorize
FPM.addPass(LoopUnrollPass(LoopUnrollOptions(O.getSpeedupLevel(), /*OnlyWhenForced = */ false, /*ForgetSCEV = */false)));
FPM.addPass(WarnMissedTransformationsPass());
MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM)));
}
invokeScalarOptimizerCallbacks(FPM, PB, O);
}

static void buildVectorPipeline(FunctionPassManager &FPM, PassBuilder *PB, OptimizationLevel O, const OptimizationOptions &options) JL_NOTSAFEPOINT {
//TODO look into loop vectorize options
FPM.addPass(InjectTLIMappings());
FPM.addPass(LoopVectorizePass());
FPM.addPass(LoopLoadEliminationPass());
FPM.addPass(InstCombinePass());
FPM.addPass(SimplifyCFGPass(aggressiveSimplifyCFGOptions()));
FPM.addPass(SLPVectorizerPass());
invokeVectorizerCallbacks(FPM, PB, O);
FPM.addPass(VectorCombinePass());
FPM.addPass(ADCEPass());
//TODO add BDCEPass here?
// This unroll will unroll vectorized loops
// as well as loops that we tried but failed to vectorize
FPM.addPass(LoopUnrollPass(LoopUnrollOptions(O.getSpeedupLevel(), /*OnlyWhenForced = */ false, /*ForgetSCEV = */false)));
}

static void buildIntrinsicLoweringPipeline(ModulePassManager &MPM, PassBuilder *PB, OptimizationLevel O, const OptimizationOptions &options) JL_NOTSAFEPOINT {
if (options.lower_intrinsics) {
//TODO barrier pass?
{
Expand All @@ -556,15 +493,15 @@ static void buildFullPipeline(ModulePassManager &MPM, PassBuilder *PB, Optimizat
JULIA_PASS(MPM.addPass(RemoveNIPass()));
JULIA_PASS(MPM.addPass(createModuleToFunctionPassAdaptor(LateLowerGCPass())));
JULIA_PASS(MPM.addPass(FinalLowerGCPass()));
{
if (O.getSpeedupLevel() >= 2) {
FunctionPassManager FPM;
FPM.addPass(GVNPass());
FPM.addPass(SCCPPass());
FPM.addPass(DCEPass());
MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM)));
}
JULIA_PASS(MPM.addPass(LowerPTLSPass(options.dump_native)));
{
if (O.getSpeedupLevel() >= 1) {
FunctionPassManager FPM;
FPM.addPass(InstCombinePass());
FPM.addPass(SimplifyCFGPass(aggressiveSimplifyCFGOptions()));
Expand All @@ -573,7 +510,10 @@ static void buildFullPipeline(ModulePassManager &MPM, PassBuilder *PB, Optimizat
} else {
JULIA_PASS(MPM.addPass(RemoveNIPass()));
}
{
}

static void buildCleanupPipeline(ModulePassManager &MPM, PassBuilder *PB, OptimizationLevel O, const OptimizationOptions &options) JL_NOTSAFEPOINT {
if (O.getSpeedupLevel() >= 2) {
FunctionPassManager FPM;
JULIA_PASS(FPM.addPass(CombineMulAddPass()));
FPM.addPass(DivRemPairsPass());
Expand All @@ -585,9 +525,30 @@ static void buildFullPipeline(ModulePassManager &MPM, PassBuilder *PB, Optimizat
{
FunctionPassManager FPM;
JULIA_PASS(FPM.addPass(DemoteFloat16Pass()));
FPM.addPass(GVNPass());
if (O.getSpeedupLevel() >= 2) {
FPM.addPass(GVNPass());
}
MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM)));
}
}

static void buildPipeline(ModulePassManager &MPM, PassBuilder *PB, OptimizationLevel O, const OptimizationOptions &options) JL_NOTSAFEPOINT {
buildEarlySimplificationPipeline(MPM, PB, O, options);
MPM.addPass(AlwaysInlinerPass());
buildEarlyOptimizerPipeline(MPM, PB, O, options);
MPM.addPass(LowerSIMDLoopPass());
{
FunctionPassManager FPM;
buildLoopOptimizerPipeline(FPM, PB, O, options);
buildScalarOptimizerPipeline(FPM, PB, O, options);
if (O.getSpeedupLevel() >= 2) {
buildVectorPipeline(FPM, PB, O, options);
}
FPM.addPass(WarnMissedTransformationsPass());
MPM.addPass(createModuleToFunctionPassAdaptor(std::move(FPM)));
}
buildIntrinsicLoweringPipeline(MPM, PB, O, options);
buildCleanupPipeline(MPM, PB, O, options);
}

#undef JULIA_PASS
Expand Down Expand Up @@ -665,10 +626,7 @@ PIC->addClassToPassName(decltype(CREATE_PASS)::name(), NAME);

ModulePassManager createMPM(PassBuilder &PB, OptimizationLevel O, OptimizationOptions options) JL_NOTSAFEPOINT {
ModulePassManager MPM;
if (O.getSpeedupLevel() < 2)
buildBasicPipeline(MPM, &PB, O, options);
else
buildFullPipeline(MPM, &PB, O, options);
buildPipeline(MPM, &PB, O, options);
return MPM;
}
}
Expand Down Expand Up @@ -805,11 +763,7 @@ void registerCallbacks(PassBuilder &PB) JL_NOTSAFEPOINT {
auto julia_options = parseJuliaPipelineOptions(Name);
if (julia_options) {
ModulePassManager pipeline;
if (julia_options->first.getSpeedupLevel() < 2) {
buildBasicPipeline(pipeline, nullptr, julia_options->first, julia_options->second);
} else {
buildFullPipeline(pipeline, nullptr, julia_options->first, julia_options->second);
}
buildPipeline(pipeline, nullptr, julia_options->first, julia_options->second);
PM.addPass(std::move(pipeline));
return true;
}
Expand Down

0 comments on commit 4200aa9

Please sign in to comment.