Auto merge of #116505 - saethlin:infer-inline, r=<try>

Automatically enable cross-crate inlining for small functions

This is a work-in-progress. For example I have not thought at all about the cost model and I am sure that the threshold is too high.

But I'm curious to know how this looks in perf. It certainly has some unique effects on codegen.

compiler/rustc_metadata/src/rmeta/decoder.rs

@@ -1273,6 +1273,15 @@ impl<'a, 'tcx> CrateMetadataRef<'a> {
  self.root.tables.optimized_mir.get(self, id).is_some()
  }
 
+ fn cross_crate_inlinable(self, tcx: TyCtxt<'tcx>, id: DefIndex) -> bool {
+ self.root
+ .tables
+ .cross_crate_inlinable
+ .get(self, id)
+ .map(|v| v.decode((self, tcx.sess)))
+ .unwrap_or(false)
+ }
+
  fn get_fn_has_self_parameter(self, id: DefIndex, sess: &'a Session) -> bool {
  self.root
  .tables

compiler/rustc_metadata/src/rmeta/decoder/cstore_impl.rs

@@ -287,6 +287,7 @@ provide! { tcx, def_id, other, cdata,
  item_attrs => { tcx.arena.alloc_from_iter(cdata.get_item_attrs(def_id.index, tcx.sess)) }
  is_mir_available => { cdata.is_item_mir_available(def_id.index) }
  is_ctfe_mir_available => { cdata.is_ctfe_mir_available(def_id.index) }
+ cross_crate_inlinable => { cdata.cross_crate_inlinable(tcx, def_id.index) }
 
  dylib_dependency_formats => { cdata.get_dylib_dependency_formats(tcx) }
  is_private_dep => {

compiler/rustc_metadata/src/rmeta/encoder.rs

@@ -1046,7 +1046,7 @@ fn should_encode_mir(
  || (tcx.sess.opts.output_types.should_codegen()
  && reachable_set.contains(&def_id)
  && (generics.requires_monomorphization(tcx)
- || tcx.codegen_fn_attrs(def_id).requests_inline()));
+ || tcx.cross_crate_inlinable(def_id)));
  // The function has a `const` modifier or is in a `#[const_trait]`.
  let is_const_fn = tcx.is_const_fn_raw(def_id.to_def_id())
  || tcx.is_const_default_method(def_id.to_def_id());
@@ -1612,6 +1612,7 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> {
  debug!("EntryBuilder::encode_mir({:?})", def_id);
  if encode_opt {
  record!(self.tables.optimized_mir[def_id.to_def_id()] <- tcx.optimized_mir(def_id));
+ record!(self.tables.cross_crate_inlinable[def_id.to_def_id()] <- self.tcx.cross_crate_inlinable(def_id));
  record!(self.tables.closure_saved_names_of_captured_variables[def_id.to_def_id()]
  <- tcx.closure_saved_names_of_captured_variables(def_id));
 

compiler/rustc_metadata/src/rmeta/mod.rs

@@ -427,6 +427,7 @@ define_tables! {
  object_lifetime_default: Table<DefIndex, LazyValue<ObjectLifetimeDefault>>,
  optimized_mir: Table<DefIndex, LazyValue<mir::Body<'static>>>,
  mir_for_ctfe: Table<DefIndex, LazyValue<mir::Body<'static>>>,
+ cross_crate_inlinable: Table<DefIndex, LazyValue<bool>>,
  closure_saved_names_of_captured_variables: Table<DefIndex, LazyValue<IndexVec<FieldIdx, Symbol>>>,
  mir_generator_witnesses: Table<DefIndex, LazyValue<mir::GeneratorLayout<'static>>>,
  promoted_mir: Table<DefIndex, LazyValue<IndexVec<mir::Promoted, mir::Body<'static>>>>,

compiler/rustc_metadata/src/rmeta/table.rs

@@ -299,6 +299,29 @@ impl FixedSizeEncoding for bool {
  }
 }
 
+impl FixedSizeEncoding for Option<bool> {
+ type ByteArray = [u8; 1];
+
+ #[inline]
+ fn from_bytes(b: &[u8; 1]) -> Self {
+ match b[0] {
+ 0 => Some(false),
+ 1 => Some(true),
+ _ => None,
+ }
+ }
+
+ #[inline]
+ fn write_to_bytes(self, b: &mut [u8; 1]) {
+ debug_assert!(!self.is_default());
+ b[0] = match self {
+ Some(false) => 0,
+ Some(true) => 1,
+ None => 2,
+ };
+ }
+}
+
 impl FixedSizeEncoding for UnusedGenericParams {
  type ByteArray = [u8; 4];
 

compiler/rustc_middle/src/query/mod.rs

@@ -2202,6 +2202,11 @@ rustc_queries! {
  query generics_require_sized_self(def_id: DefId) -> bool {
  desc { "check whether the item has a `where Self: Sized` bound" }
  }
+
+ query cross_crate_inlinable(def_id: DefId) -> bool {
+ desc { "whether the item should be made inlinable across crates" }
+ separate_provide_extern
+ }
 }
 
 rustc_query_append! { define_callbacks! }

compiler/rustc_middle/src/ty/instance.rs

@@ -245,16 +245,15 @@ impl<'tcx> InstanceDef<'tcx> {
  // drops of `Option::None` before LTO. We also respect the intent of
  // `#[inline]` on `Drop::drop` implementations.
  return ty.ty_adt_def().map_or(true, |adt_def| {
- adt_def.destructor(tcx).map_or_else(
- || adt_def.is_enum(),
- |dtor| tcx.codegen_fn_attrs(dtor.did).requests_inline(),
- )
+ adt_def
+ .destructor(tcx)
+ .map_or_else(|| adt_def.is_enum(), |dtor| tcx.cross_crate_inlinable(dtor.did))
  });
  }
  if let ty::InstanceDef::ThreadLocalShim(..) = *self {
  return false;
  }
- tcx.codegen_fn_attrs(self.def_id()).requests_inline()
+ tcx.cross_crate_inlinable(self.def_id())
  }
 
  pub fn requires_caller_location(&self, tcx: TyCtxt<'_>) -> bool {

compiler/rustc_mir_transform/src/cross_crate_inline.rs

@@ -0,0 +1,97 @@
+use rustc_attr::InlineAttr;
+use rustc_hir::def_id::LocalDefId;
+use rustc_middle::query::Providers;
+use rustc_middle::ty::TyCtxt;
+use rustc_session::config::OptLevel;
+
+pub fn provide(providers: &mut Providers) {
+ providers.cross_crate_inlinable = cross_crate_inlinable;
+}
+
+fn cross_crate_inlinable(tcx: TyCtxt<'_>, def_id: LocalDefId) -> bool {
+ match tcx.codegen_fn_attrs(def_id).inline {
+ InlineAttr::Never => return false,
+ InlineAttr::Hint | InlineAttr::Always => return true,
+ _ => {}
+ }
+
+ if matches!(tcx.sess.opts.optimize, OptLevel::No | OptLevel::Default) {
+ return false;
+ }
+
+ match tcx.hir().body_const_context(def_id) {
+ Some(rustc_hir::ConstContext::ConstFn) | None => {}
+ _ => return false,
+ }
+
+ if tcx.lang_items().iter().any(|(_, lang_def_id)| lang_def_id == def_id.into()) {
+ return false;
+ }
+
+ let mir = tcx.optimized_mir(def_id);
+ let mut checker = CostChecker { tcx, cost: 0, callee_body: mir };
+ checker.visit_body(mir);
+ checker.cost < 200
+}
+
+use rustc_middle::mir::visit::Visitor;
+use rustc_middle::mir::*;
+
+const INSTR_COST: usize = 5;
+const CALL_PENALTY: usize = 25;
+const LANDINGPAD_PENALTY: usize = 50;
+const RESUME_PENALTY: usize = 45;
+
+struct CostChecker<'b, 'tcx> {
+ tcx: TyCtxt<'tcx>,
+ cost: usize,
+ callee_body: &'b Body<'tcx>,
+}
+
+impl<'tcx> Visitor<'tcx> for CostChecker<'_, 'tcx> {
+ fn visit_statement(&mut self, statement: &Statement<'tcx>, _: Location) {
+ // Don't count StorageLive/StorageDead in the inlining cost.
+ match statement.kind {
+ StatementKind::StorageLive(_)
+ | StatementKind::StorageDead(_)
+ | StatementKind::Deinit(_)
+ | StatementKind::Nop => {}
+ _ => self.cost += INSTR_COST,
+ }
+ }
+
+ fn visit_terminator(&mut self, terminator: &Terminator<'tcx>, _: Location) {
+ let tcx = self.tcx;
+ match terminator.kind {
+ TerminatorKind::Drop { ref place, unwind, .. } => {
+ let ty = place.ty(self.callee_body, tcx).ty;
+ if !ty.is_trivially_pure_clone_copy() {
+ self.cost += CALL_PENALTY;
+ if let UnwindAction::Cleanup(_) = unwind {
+ self.cost += LANDINGPAD_PENALTY;
+ }
+ }
+ }
+ TerminatorKind::Call { unwind, .. } => {
+ self.cost += CALL_PENALTY;
+ if let UnwindAction::Cleanup(_) = unwind {
+ self.cost += LANDINGPAD_PENALTY;
+ }
+ }
+ TerminatorKind::Assert { unwind, .. } => {
+ self.cost += CALL_PENALTY;
+ if let UnwindAction::Cleanup(_) = unwind {
+ self.cost += LANDINGPAD_PENALTY;
+ }
+ }
+ TerminatorKind::UnwindResume => self.cost += RESUME_PENALTY,
+ TerminatorKind::InlineAsm { unwind, .. } => {
+ self.cost += INSTR_COST;
+ if let UnwindAction::Cleanup(_) = unwind {
+ self.cost += LANDINGPAD_PENALTY;
+ }
+ }
+ _ => self.cost += INSTR_COST,
+ }
+ }
+}

compiler/rustc_mir_transform/src/inline.rs

@@ -170,7 +170,13 @@ impl<'tcx> Inliner<'tcx> {
  callsite: &CallSite<'tcx>,
  ) -> Result<std::ops::Range<BasicBlock>, &'static str> {
  let callee_attrs = self.tcx.codegen_fn_attrs(callsite.callee.def_id());
- self.check_codegen_attributes(callsite, callee_attrs)?;
+ let cross_crate_inlinable = if callsite.callee.def_id().is_local() {
+ // Avoid a query cycle, cross_crate_inlinable is based on optimized_mir
+ callee_attrs.requests_inline()
+ } else {
+ self.tcx.cross_crate_inlinable(callsite.callee.def_id())
+ };
+ self.check_codegen_attributes(callsite, callee_attrs, cross_crate_inlinable)?;
 
  let terminator = caller_body[callsite.block].terminator.as_ref().unwrap();
  let TerminatorKind::Call { args, destination, .. } = &terminator.kind else { bug!() };
@@ -185,7 +191,7 @@ impl<'tcx> Inliner<'tcx> {
 
  self.check_mir_is_available(caller_body, &callsite.callee)?;
  let callee_body = try_instance_mir(self.tcx, callsite.callee.def)?;
- self.check_mir_body(callsite, callee_body, callee_attrs)?;
+ self.check_mir_body(callsite, callee_body, callee_attrs, cross_crate_inlinable)?;
 
  if !self.tcx.consider_optimizing(|| {
  format!("Inline {:?} into {:?}", callsite.callee, caller_body.source)
@@ -401,6 +407,7 @@ impl<'tcx> Inliner<'tcx> {
  &self,
  callsite: &CallSite<'tcx>,
  callee_attrs: &CodegenFnAttrs,
+ cross_crate_inlinable: bool,
  ) -> Result<(), &'static str> {
  if let InlineAttr::Never = callee_attrs.inline {
  return Err("never inline hint");
@@ -414,7 +421,7 @@ impl<'tcx> Inliner<'tcx> {
  .non_erasable_generics(self.tcx, callsite.callee.def_id())
  .next()
  .is_some();
- if !is_generic && !callee_attrs.requests_inline() {
+ if !is_generic && !cross_crate_inlinable {
  return Err("not exported");
  }
 
@@ -456,10 +463,11 @@ impl<'tcx> Inliner<'tcx> {
  callsite: &CallSite<'tcx>,
  callee_body: &Body<'tcx>,
  callee_attrs: &CodegenFnAttrs,
+ cross_crate_inlinable: bool,
  ) -> Result<(), &'static str> {
  let tcx = self.tcx;
 
- let mut threshold = if callee_attrs.requests_inline() {
+ let mut threshold = if cross_crate_inlinable {
  self.tcx.sess.opts.unstable_opts.inline_mir_hint_threshold.unwrap_or(100)
  } else {
  self.tcx.sess.opts.unstable_opts.inline_mir_threshold.unwrap_or(50)

compiler/rustc_mir_transform/src/lib.rs

@@ -62,6 +62,7 @@ mod const_prop;
 mod const_prop_lint;
 mod copy_prop;
 mod coverage;
+mod cross_crate_inline;
 mod ctfe_limit;
 mod dataflow_const_prop;
 mod dead_store_elimination;
@@ -123,6 +124,7 @@ pub fn provide(providers: &mut Providers) {
  coverage::query::provide(providers);
  ffi_unwind_calls::provide(providers);
  shim::provide(providers);
+ cross_crate_inline::provide(providers);
  *providers = Providers {
  mir_keys,
  mir_const,

compiler/rustc_passes/src/reachable.rs

@@ -21,8 +21,8 @@ use rustc_target::spec::abi::Abi;
 // Returns true if the given item must be inlined because it may be
 // monomorphized or it was marked with `#[inline]`. This will only return
 // true for functions.
-fn item_might_be_inlined(tcx: TyCtxt<'_>, item: &hir::Item<'_>, attrs: &CodegenFnAttrs) -> bool {
- if attrs.requests_inline() {
+fn item_might_be_inlined(tcx: TyCtxt<'_>, item: &hir::Item<'_>, inlinable: bool) -> bool {
+ if inlinable {
  return true;
  }
 
@@ -41,9 +41,9 @@ fn method_might_be_inlined(
  impl_item: &hir::ImplItem<'_>,
  impl_src: LocalDefId,
 ) -> bool {
- let codegen_fn_attrs = tcx.codegen_fn_attrs(impl_item.hir_id().owner.to_def_id());
+ let inlinable = tcx.cross_crate_inlinable(impl_item.hir_id().owner.to_def_id());
  let generics = tcx.generics_of(impl_item.owner_id);
- if codegen_fn_attrs.requests_inline() || generics.requires_monomorphization(tcx) {
+ if inlinable || generics.requires_monomorphization(tcx) {
  return true;
  }
  if let hir::ImplItemKind::Fn(method_sig, _) = &impl_item.kind {
@@ -52,7 +52,7 @@ fn method_might_be_inlined(
  }
  }
  match tcx.hir().find_by_def_id(impl_src) {
- Some(Node::Item(item)) => item_might_be_inlined(tcx, &item, codegen_fn_attrs),
+ Some(Node::Item(item)) => item_might_be_inlined(tcx, &item, inlinable),
  Some(..) | None => span_bug!(impl_item.span, "impl did is not an item"),
  }
 }
@@ -149,7 +149,7 @@ impl<'tcx> ReachableContext<'tcx> {
  match self.tcx.hir().find_by_def_id(def_id) {
  Some(Node::Item(item)) => match item.kind {
  hir::ItemKind::Fn(..) => {
- item_might_be_inlined(self.tcx, &item, self.tcx.codegen_fn_attrs(def_id))
+ item_might_be_inlined(self.tcx, &item, self.tcx.cross_crate_inlinable(def_id))
  }
  _ => false,
  },
@@ -227,7 +227,7 @@ impl<'tcx> ReachableContext<'tcx> {
  if item_might_be_inlined(
  self.tcx,
  &item,
- self.tcx.codegen_fn_attrs(item.owner_id),
+ self.tcx.cross_crate_inlinable(item.owner_id),
  ) {
  self.visit_nested_body(body);
  }

tests/mir-opt/enum_opt.cand.EnumSizeOpt.32bit.diff

@@ -66,7 +66,7 @@
  }
 + }
 + 
-+ alloc15 (size: 8, align: 4) {
++ alloc14 (size: 8, align: 4) {
 + 02 00 00 00 05 20 00 00 │ ..... ..
  }
 

tests/mir-opt/enum_opt.cand.EnumSizeOpt.64bit.diff

@@ -66,7 +66,7 @@
  }
 + }
 + 
-+ alloc15 (size: 16, align: 8) {
++ alloc14 (size: 16, align: 8) {
 + 02 00 00 00 00 00 00 00 05 20 00 00 00 00 00 00 │ ......... ......
  }
 

tests/mir-opt/enum_opt.unin.EnumSizeOpt.32bit.diff

@@ -66,7 +66,7 @@
  }
 + }
 + 
-+ alloc14 (size: 8, align: 4) {
++ alloc15 (size: 8, align: 4) {
 + 05 20 00 00 01 00 00 00 │ . ......
  }
 

tests/mir-opt/enum_opt.unin.EnumSizeOpt.64bit.diff

@@ -66,7 +66,7 @@
  }
 + }
 + 
-+ alloc14 (size: 16, align: 8) {
++ alloc15 (size: 16, align: 8) {
 + 05 20 00 00 00 00 00 00 01 00 00 00 00 00 00 00 │ . ..............
  }