@@ -85,13 +85,18 @@ fn inline<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) -> bool {
8585 }
8686
8787 let param_env = tcx. param_env_reveal_all_normalized ( def_id) ;
88+ let codegen_fn_attrs = tcx. codegen_fn_attrs ( def_id) ;
8889
8990 let mut this = Inliner {
9091 tcx,
9192 param_env,
92- codegen_fn_attrs : tcx . codegen_fn_attrs ( def_id ) ,
93+ codegen_fn_attrs,
9394 history : Vec :: new ( ) ,
9495 changed : false ,
96+ caller_is_inline_forwarder : matches ! (
97+ codegen_fn_attrs. inline,
98+ InlineAttr :: Hint | InlineAttr :: Always
99+ ) && body_is_forwarder ( body) ,
95100 } ;
96101 let blocks = START_BLOCK ..body. basic_blocks . next_index ( ) ;
97102 this. process_blocks ( body, blocks) ;
@@ -111,6 +116,9 @@ struct Inliner<'tcx> {
111116 history : Vec < DefId > ,
112117 /// Indicates that the caller body has been modified.
113118 changed : bool ,
119+ /// Indicates that the caller is #[inline] and just calls another function,
120+ /// and thus we can inline less into it as it'll be inlined itself.
121+ caller_is_inline_forwarder : bool ,
114122}
115123
116124impl < ' tcx > Inliner < ' tcx > {
@@ -485,7 +493,9 @@ impl<'tcx> Inliner<'tcx> {
485493 ) -> Result < ( ) , & ' static str > {
486494 let tcx = self . tcx ;
487495
488- let mut threshold = if cross_crate_inlinable {
496+ let mut threshold = if self . caller_is_inline_forwarder {
497+ self . tcx . sess . opts . unstable_opts . inline_mir_forwarder_threshold . unwrap_or ( 30 )
498+ } else if cross_crate_inlinable {
489499 self . tcx . sess . opts . unstable_opts . inline_mir_hint_threshold . unwrap_or ( 100 )
490500 } else {
491501 self . tcx . sess . opts . unstable_opts . inline_mir_threshold . unwrap_or ( 50 )
@@ -504,6 +514,8 @@ impl<'tcx> Inliner<'tcx> {
504514 let mut checker =
505515 CostChecker :: new ( self . tcx , self . param_env , Some ( callsite. callee ) , callee_body) ;
506516
517+ checker. add_function_level_costs ( ) ;
518+
507519 // Traverse the MIR manually so we can account for the effects of inlining on the CFG.
508520 let mut work_list = vec ! [ START_BLOCK ] ;
509521 let mut visited = BitSet :: new_empty ( callee_body. basic_blocks . len ( ) ) ;
@@ -1091,3 +1103,37 @@ fn try_instance_mir<'tcx>(
10911103 }
10921104 Ok ( tcx. instance_mir ( instance) )
10931105}
1106+
1107+ fn body_is_forwarder ( body : & Body < ' _ > ) -> bool {
1108+ let TerminatorKind :: Call { target, .. } = body. basic_blocks [ START_BLOCK ] . terminator ( ) . kind
1109+ else {
1110+ return false ;
1111+ } ;
1112+ if let Some ( target) = target {
1113+ let TerminatorKind :: Return = body. basic_blocks [ target] . terminator ( ) . kind else {
1114+ return false ;
1115+ } ;
1116+ }
1117+
1118+ let max_blocks = if !body. is_polymorphic {
1119+ 2
1120+ } else if target. is_none ( ) {
1121+ 3
1122+ } else {
1123+ 4
1124+ } ;
1125+ if body. basic_blocks . len ( ) > max_blocks {
1126+ return false ;
1127+ }
1128+
1129+ body. basic_blocks . iter_enumerated ( ) . all ( |( bb, bb_data) | {
1130+ bb == START_BLOCK
1131+ || matches ! (
1132+ bb_data. terminator( ) . kind,
1133+ TerminatorKind :: Return
1134+ | TerminatorKind :: Drop { .. }
1135+ | TerminatorKind :: UnwindResume
1136+ | TerminatorKind :: UnwindTerminate ( _)
1137+ )
1138+ } )
1139+ }
0 commit comments