@@ -26,6 +26,7 @@ use crate::{check_inline, util};
2626
2727pub ( crate ) mod cycle;
2828
29+ const HISTORY_DEPTH_LIMIT : usize = 20 ;
2930const TOP_DOWN_DEPTH_LIMIT : usize = 5 ;
3031
3132#[ derive( Clone , Debug ) ]
@@ -117,6 +118,13 @@ trait Inliner<'tcx> {
117118 /// Should inlining happen for a given callee?
118119fn should_inline_for_callee ( & self , def_id : DefId ) -> bool ;
119120
121+ fn check_codegen_attributes_extra (
122+ & self ,
123+ _callee_attrs : & CodegenFnAttrs ,
124+ ) -> Result < ( ) , & ' static str > {
125+ Ok ( ( ) )
126+ }
127+
120128 fn check_caller_mir_body ( & self , body : & Body < ' tcx > ) -> bool ;
121129
122130 /// Returns inlining decision that is based on the examination of callee MIR body.
@@ -128,10 +136,6 @@ trait Inliner<'tcx> {
128136 callee_attrs : & CodegenFnAttrs ,
129137 ) -> Result < ( ) , & ' static str > ;
130138
131- // How many callsites in a body are we allowed to inline? We need to limit this in order
132- // to prevent super-linear growth in MIR size.
133- fn inline_limit_for_block ( & self ) -> Option < usize > ;
134-
135139 /// Called when inlining succeeds.
136140fn on_inline_success (
137141 & mut self ,
@@ -142,9 +146,6 @@ trait Inliner<'tcx> {
142146
143147 /// Called when inlining failed or was not performed.
144148fn on_inline_failure ( & self , callsite : & CallSite < ' tcx > , reason : & ' static str ) ;
145-
146- /// Called when the inline limit for a body is reached.
147- fn on_inline_limit_reached ( & self ) -> bool ;
148149}
149150
150151struct ForceInliner < ' tcx > {
@@ -224,10 +225,6 @@ impl<'tcx> Inliner<'tcx> for ForceInliner<'tcx> {
224225 }
225226 }
226227
227- fn inline_limit_for_block ( & self ) -> Option < usize > {
228- Some ( usize:: MAX )
229- }
230-
231228 fn on_inline_success (
232229 & mut self ,
233230 callsite : & CallSite < ' tcx > ,
@@ -261,10 +258,6 @@ impl<'tcx> Inliner<'tcx> for ForceInliner<'tcx> {
261258 justification : justification. map ( |sym| crate :: errors:: ForceInlineJustification { sym } ) ,
262259 } ) ;
263260 }
264-
265- fn on_inline_limit_reached ( & self ) -> bool {
266- false
267- }
268261}
269262
270263struct NormalInliner < ' tcx > {
@@ -278,13 +271,23 @@ struct NormalInliner<'tcx> {
278271/// The number of `DefId`s is finite, so checking history is enough
279272/// to ensure that we do not loop endlessly while inlining.
280273history : Vec < DefId > ,
274+ /// How many (multi-call) callsites have we inlined for the top-level call?
275+ ///
276+ /// We need to limit this in order to prevent super-linear growth in MIR size.
277+ top_down_counter : usize ,
281278 /// Indicates that the caller body has been modified.
282279changed : bool ,
283280 /// Indicates that the caller is #[inline] and just calls another function,
284281/// and thus we can inline less into it as it'll be inlined itself.
285282caller_is_inline_forwarder : bool ,
286283}
287284
285+ impl < ' tcx > NormalInliner < ' tcx > {
286+ fn past_depth_limit ( & self ) -> bool {
287+ self . history . len ( ) > HISTORY_DEPTH_LIMIT || self . top_down_counter > TOP_DOWN_DEPTH_LIMIT
288+ }
289+ }
290+
288291impl < ' tcx > Inliner < ' tcx > for NormalInliner < ' tcx > {
289292 fn new ( tcx : TyCtxt < ' tcx > , def_id : DefId , body : & Body < ' tcx > ) -> Self {
290293 let typing_env = body. typing_env ( tcx) ;
@@ -295,6 +298,7 @@ impl<'tcx> Inliner<'tcx> for NormalInliner<'tcx> {
295298 typing_env,
296299 def_id,
297300 history : Vec :: new ( ) ,
301+ top_down_counter : 0 ,
298302 changed : false ,
299303 caller_is_inline_forwarder : matches ! (
300304 codegen_fn_attrs. inline,
@@ -327,6 +331,17 @@ impl<'tcx> Inliner<'tcx> for NormalInliner<'tcx> {
327331 true
328332 }
329333
334+ fn check_codegen_attributes_extra (
335+ & self ,
336+ callee_attrs : & CodegenFnAttrs ,
337+ ) -> Result < ( ) , & ' static str > {
338+ if self . past_depth_limit ( ) && matches ! ( callee_attrs. inline, InlineAttr :: None ) {
339+ Err ( "Past depth limit so not inspecting unmarked callee" )
340+ } else {
341+ Ok ( ( ) )
342+ }
343+ }
344+
330345 fn check_caller_mir_body ( & self , body : & Body < ' tcx > ) -> bool {
331346 // Avoid inlining into coroutines, since their `optimized_mir` is used for layout computation,
332347 // which can create a cycle, even when no attempt is made to inline the function in the other
@@ -351,6 +366,10 @@ impl<'tcx> Inliner<'tcx> for NormalInliner<'tcx> {
351366 return Err ( "body has errors" ) ;
352367 }
353368
369+ if self . past_depth_limit ( ) && callee_body. basic_blocks . len ( ) > 1 {
370+ return Err ( "Not inlining multi-block body as we're past a depth limit" ) ;
371+ }
372+
354373 let mut threshold = if self . caller_is_inline_forwarder {
355374 tcx. sess . opts . unstable_opts . inline_mir_forwarder_threshold . unwrap_or ( 30 )
356375 } else if tcx. cross_crate_inlinable ( callsite. callee . def_id ( ) ) {
@@ -431,14 +450,6 @@ impl<'tcx> Inliner<'tcx> for NormalInliner<'tcx> {
431450 }
432451 }
433452
434- fn inline_limit_for_block ( & self ) -> Option < usize > {
435- match self . history . len ( ) {
436- 0 => Some ( usize:: MAX ) ,
437- 1 ..=TOP_DOWN_DEPTH_LIMIT => Some ( 1 ) ,
438- _ => None ,
439- }
440- }
441-
442453 fn on_inline_success (
443454 & mut self ,
444455 callsite : & CallSite < ' tcx > ,
@@ -447,13 +458,26 @@ impl<'tcx> Inliner<'tcx> for NormalInliner<'tcx> {
447458 ) {
448459 self . changed = true ;
449460
461+ let new_calls_count = new_blocks
462+ . clone ( )
463+ . filter ( |& bb| {
464+ matches ! (
465+ caller_body. basic_blocks[ bb] . terminator( ) . kind,
466+ TerminatorKind :: Call { .. } ,
467+ )
468+ } )
469+ . count ( ) ;
470+ if new_calls_count > 1 {
471+ self . top_down_counter += 1 ;
472+ }
473+
450474 self . history . push ( callsite. callee . def_id ( ) ) ;
451475 process_blocks ( self , caller_body, new_blocks) ;
452476 self . history . pop ( ) ;
453- }
454477
455- fn on_inline_limit_reached ( & self ) -> bool {
456- true
478+ if self . history . is_empty ( ) {
479+ self . top_down_counter = 0 ;
480+ }
457481 }
458482
459483 fn on_inline_failure ( & self , _: & CallSite < ' tcx > , _: & ' static str ) { }
@@ -482,8 +506,6 @@ fn process_blocks<'tcx, I: Inliner<'tcx>>(
482506 caller_body : & mut Body < ' tcx > ,
483507 blocks : Range < BasicBlock > ,
484508) {
485- let Some ( inline_limit) = inliner. inline_limit_for_block ( ) else { return } ;
486- let mut inlined_count = 0 ;
487509 for bb in blocks {
488510 let bb_data = & caller_body[ bb] ;
489511 if bb_data. is_cleanup {
@@ -505,13 +527,6 @@ fn process_blocks<'tcx, I: Inliner<'tcx>>(
505527 Ok ( new_blocks) => {
506528 debug ! ( "inlined {}" , callsite. callee) ;
507529 inliner. on_inline_success ( & callsite, caller_body, new_blocks) ;
508-
509- inlined_count += 1 ;
510- if inlined_count == inline_limit {
511- if inliner. on_inline_limit_reached ( ) {
512- return ;
513- }
514- }
515530 }
516531 }
517532 }
@@ -584,6 +599,7 @@ fn try_inlining<'tcx, I: Inliner<'tcx>>(
584599 let callee_attrs = tcx. codegen_fn_attrs ( callsite. callee . def_id ( ) ) ;
585600 check_inline:: is_inline_valid_on_fn ( tcx, callsite. callee . def_id ( ) ) ?;
586601 check_codegen_attributes ( inliner, callsite, callee_attrs) ?;
602+ inliner. check_codegen_attributes_extra ( callee_attrs) ?;
587603
588604 let terminator = caller_body[ callsite. block ] . terminator . as_ref ( ) . unwrap ( ) ;
589605 let TerminatorKind :: Call { args, destination, .. } = & terminator. kind else { bug ! ( ) } ;
@@ -770,6 +786,8 @@ fn check_codegen_attributes<'tcx, I: Inliner<'tcx>>(
770786 return Err ( "has DoNotOptimize attribute" ) ;
771787 }
772788
789+ inliner. check_codegen_attributes_extra ( callee_attrs) ?;
790+
773791 // Reachability pass defines which functions are eligible for inlining. Generally inlining
774792 // other functions is incorrect because they could reference symbols that aren't exported.
775793 let is_generic = callsite. callee . args . non_erasable_generics ( ) . next ( ) . is_some ( ) ;
0 commit comments