[WIP] Allow functions to be inlined across crates without an inline attribute

src/bootstrap/builder.rs

@@ -1098,6 +1098,9 @@ impl<'a> Builder<'a> {
  }
 
  if let Mode::Rustc | Mode::Codegen = mode {
+ if stage > 0 {
+ rustflags.arg("-Zcross-crate-inline-threshold=50");
+ }
  rustflags.arg("-Zunstable-options");
  rustflags.arg("-Wrustc::internal");
  }

src/librustc/middle/codegen_fn_attrs.rs

@@ -95,14 +95,6 @@ impl CodegenFnAttrs {
  }
  }
 
- /// Returns `true` if `#[inline]` or `#[inline(always)]` is present.
- pub fn requests_inline(&self) -> bool {
- match self.inline {
- InlineAttr::Hint | InlineAttr::Always => true,
- InlineAttr::None | InlineAttr::Never => false,
- }
- }
-
  /// Returns `true` if it looks like this symbol needs to be exported, for example:
  ///
  /// * `#[no_mangle]` is present

src/librustc/mir/mono.rs

@@ -90,6 +90,18 @@ impl<'tcx> MonoItem<'tcx> {
 
  match *self {
  MonoItem::Fn(ref instance) => {
+ /*use rustc_hir::ItemKind;
+ use rustc_hir::Node;
+ tcx.hir().as_local_hir_id(instance.def_id()).map(|id| match tcx.hir().get(id) {
+ Node::Item(item) => match item.kind {
+ ItemKind::Static(..) | ItemKind::Const(..) => {
+ panic!("STATIC!! {:?}", self);
+ }
+ _ => (),
+ },
+ _ => (),
+ });*/
+
  let entry_def_id = tcx.entry_fn(LOCAL_CRATE).map(|(id, _)| id);
  // If this function isn't inlined or otherwise has explicit
  // linkage, then we'll be creating a globally shared version.

src/librustc/query/mod.rs

@@ -649,6 +649,8 @@ rustc_queries! {
  query codegen_fn_attrs(_: DefId) -> CodegenFnAttrs {
  cache_on_disk_if { true }
  }
+
+ query cross_crate_inlinable(_: DefId) -> bool {}
  }
 
  Other {

src/librustc/ty/instance.rs

@@ -204,12 +204,13 @@ impl<'tcx> InstanceDef<'tcx> {
  // drops of `Option::None` before LTO. We also respect the intent of
  // `#[inline]` on `Drop::drop` implementations.
  return ty.ty_adt_def().map_or(true, |adt_def| {
- adt_def.destructor(tcx).map_or(adt_def.is_enum(), |dtor| {
- tcx.codegen_fn_attrs(dtor.did).requests_inline()
- })
+ adt_def
+ .destructor(tcx)
+  .map_or(adt_def.is_enum(), |dtor| tcx.cross_crate_inlinable(dtor.did))
  });
  }
- tcx.codegen_fn_attrs(self.def_id()).requests_inline()
+
+ tcx.cross_crate_inlinable(self.def_id())
  }
 
  pub fn requires_caller_location(&self, tcx: TyCtxt<'_>) -> bool {

src/librustc_codegen_ssa/back/symbol_export.rs

@@ -85,8 +85,7 @@ fn reachable_non_generics_provider(
  }
 
  // Only consider nodes that actually have exported symbols.
- Node::Item(&hir::Item { kind: hir::ItemKind::Static(..), .. })
- | Node::Item(&hir::Item { kind: hir::ItemKind::Fn(..), .. })
+ Node::Item(&hir::Item { kind: hir::ItemKind::Fn(..), .. })
  | Node::ImplItem(&hir::ImplItem { kind: hir::ImplItemKind::Fn(..), .. }) => {
  let def_id = tcx.hir().local_def_id(hir_id);
  let generics = tcx.generics_of(def_id);
@@ -101,6 +100,15 @@ fn reachable_non_generics_provider(
  }
  }
 
+ Node::Item(&hir::Item { kind: hir::ItemKind::Static(..), .. }) => {
+ let def_id = tcx.hir().local_def_id(hir_id);
+ let generics = tcx.generics_of(def_id);
+ let r =
+ if !generics.requires_monomorphization(tcx) { Some(def_id) } else { None };
+ //assert!(r.is_some());
+ r
+ }
+
  _ => None,
  }
  })

src/librustc_metadata/rmeta/decoder.rs

@@ -1099,6 +1099,15 @@ impl<'a, 'tcx> CrateMetadataRef<'a> {
  !self.is_proc_macro(id) && self.root.per_def.mir.get(self, id).is_some()
  }
 
+ fn is_item_cross_crate_inlinable(&self, tcx: TyCtxt<'tcx>, id: DefIndex) -> bool {
+ self.root
+ .per_def
+ .cross_crate_inlinable
+ .get(self, id)
+ .map(|v| v.decode((self, tcx)))
+ .unwrap_or(false)
+ }
+
  fn get_optimized_mir(&self, tcx: TyCtxt<'tcx>, id: DefIndex) -> BodyAndCache<'tcx> {
  let mut cache = self
  .root

src/librustc_metadata/rmeta/decoder/cstore_impl.rs

@@ -149,6 +149,7 @@ provide! { <'tcx> tcx, def_id, other, cdata,
  impl_parent => { cdata.get_parent_impl(def_id.index) }
  trait_of_item => { cdata.get_trait_of_item(def_id.index) }
  is_mir_available => { cdata.is_item_mir_available(def_id.index) }
+ cross_crate_inlinable => { cdata.is_item_cross_crate_inlinable(tcx, def_id.index) }
 
  dylib_dependency_formats => { cdata.get_dylib_dependency_formats(tcx) }
  is_panic_runtime => { cdata.root.panic_runtime }

src/librustc_metadata/rmeta/encoder.rs

@@ -882,6 +882,9 @@ impl EncodeContext<'tcx> {
  ty::AssocKind::OpaqueTy => unreachable!(),
  }
  if trait_item.kind == ty::AssocKind::Method {
+ if self.tcx.mir_keys(LOCAL_CRATE).contains(&def_id) {
+ record!(self.per_def.cross_crate_inlinable[def_id] <- self.tcx.cross_crate_inlinable(def_id));
+ }
  record!(self.per_def.fn_sig[def_id] <- tcx.fn_sig(def_id));
  self.encode_variances_of(def_id);
  }
@@ -968,9 +971,11 @@ impl EncodeContext<'tcx> {
  let mir = match ast_item.kind {
  hir::ImplItemKind::Const(..) => true,
  hir::ImplItemKind::Fn(ref sig, _) => {
+ record!(self.per_def.cross_crate_inlinable[def_id] <- self.tcx.cross_crate_inlinable(def_id));
+
  let generics = self.tcx.generics_of(def_id);
  let needs_inline = (generics.requires_monomorphization(self.tcx)
- || tcx.codegen_fn_attrs(def_id).requests_inline())
+ || tcx.cross_crate_inlinable(def_id))
  && !self.metadata_output_only();
  let is_const_fn = sig.header.constness == hir::Constness::Const;
  let always_encode_mir = self.tcx.sess.opts.debugging_opts.always_encode_mir;
@@ -1266,9 +1271,11 @@ impl EncodeContext<'tcx> {
  let mir = match item.kind {
  hir::ItemKind::Static(..) | hir::ItemKind::Const(..) => true,
  hir::ItemKind::Fn(ref sig, ..) => {
+ record!(self.per_def.cross_crate_inlinable[def_id] <- self.tcx.cross_crate_inlinable(def_id));
+
  let generics = tcx.generics_of(def_id);
  let needs_inline = (generics.requires_monomorphization(tcx)
- || tcx.codegen_fn_attrs(def_id).requests_inline())
+ || tcx.cross_crate_inlinable(def_id))
  && !self.metadata_output_only();
  let always_encode_mir = self.tcx.sess.opts.debugging_opts.always_encode_mir;
  needs_inline || sig.header.constness == hir::Constness::Const || always_encode_mir
@@ -1743,8 +1750,8 @@ impl<'tcx, 'v> ParItemLikeVisitor<'v> for PrefetchVisitor<'tcx> {
  hir::ItemKind::Fn(ref sig, ..) => {
  let def_id = tcx.hir().local_def_id(item.hir_id);
  let generics = tcx.generics_of(def_id);
- let needs_inline = generics.requires_monomorphization(tcx)
- || tcx.codegen_fn_attrs(def_id).requests_inline();
+ let needs_inline =
+ generics.requires_monomorphization(tcx) || tcx.cross_crate_inlinable(def_id);
  if needs_inline || sig.header.constness == hir::Constness::Const {
  self.prefetch_mir(def_id)
  }
@@ -1768,8 +1775,8 @@ impl<'tcx, 'v> ParItemLikeVisitor<'v> for PrefetchVisitor<'tcx> {
  hir::ImplItemKind::Fn(ref sig, _) => {
  let def_id = tcx.hir().local_def_id(impl_item.hir_id);
  let generics = tcx.generics_of(def_id);
- let needs_inline = generics.requires_monomorphization(tcx)
- || tcx.codegen_fn_attrs(def_id).requests_inline();
+ let needs_inline =
+ generics.requires_monomorphization(tcx) || tcx.cross_crate_inlinable(def_id);
  let is_const_fn = sig.header.constness == hir::Constness::Const;
  if needs_inline || is_const_fn {
  self.prefetch_mir(def_id)

src/librustc_metadata/rmeta/mod.rs

@@ -276,6 +276,7 @@ define_per_def_tables! {
  inferred_outlives: Table<DefIndex, Lazy!(&'tcx [(ty::Predicate<'tcx>, Span)])>,
  super_predicates: Table<DefIndex, Lazy!(ty::GenericPredicates<'tcx>)>,
  mir: Table<DefIndex, Lazy!(mir::BodyAndCache<'tcx>)>,
+ cross_crate_inlinable: Table<DefIndex, Lazy<bool>>,
  promoted_mir: Table<DefIndex, Lazy!(IndexVec<mir::Promoted, mir::BodyAndCache<'tcx>>)>,
 }
 

src/librustc_mir/cross_crate_inline.rs

@@ -0,0 +1,161 @@
+use rustc::middle::codegen_fn_attrs::CodegenFnAttrFlags;
+use rustc::mir::{Body, TerminatorKind};
+use rustc::ty::query::Providers;
+use rustc::ty::TyCtxt;
+use rustc_attr::InlineAttr;
+use rustc_hir::def_id::DefId;
+//use rustc_hir::ItemKind;
+//use rustc_session::config::OptLevel;
+use rustc_session::config::Sanitizer;
+
+pub fn provide(providers: &mut Providers<'_>) {
+ providers.cross_crate_inlinable = cross_crate_inlinable;
+}
+
+fn cross_crate_inlinable(tcx: TyCtxt<'_>, def_id: DefId) -> bool {
+ /*use rustc_hir::Node;
+ tcx.hir().as_local_hir_id(def_id).map(|id| match tcx.hir().get(id) {
+ Node::Item(item) => match item.kind {
+ ItemKind::Static(..) | ItemKind::Const(..) => {
+ panic!("STATIC!! {:?}", def_id);
+ }
+ _ => (),
+ },
+ _ => (),
+ });*/
+
+ let attrs = tcx.codegen_fn_attrs(def_id);
+
+ match attrs.inline {
+ InlineAttr::Hint | InlineAttr::Always => return true,
+ InlineAttr::Never => return false,
+ InlineAttr::None => (),
+ }
+
+ if attrs.contains_extern_indicator() {
+ return false;
+ }
+
+ if tcx.lang_items().start_fn() == Some(def_id) {
+ return false;
+ }
+
+ // FIXME: Share some of this logic with the MIR inlining pass
+
+ // FIXME: Is this needed?
+ if attrs.flags.contains(CodegenFnAttrFlags::TRACK_CALLER) {
+ return false;
+ }
+
+ // Avoid inlining functions marked as no_sanitize if sanitizer is enabled,
+ // since instrumentation might be enabled and performed on the caller.
+ match tcx.sess.opts.debugging_opts.sanitizer {
+ Some(Sanitizer::Address) => {
+ if attrs.flags.contains(CodegenFnAttrFlags::NO_SANITIZE_ADDRESS) {
+ return false;
+ }
+ }
+ Some(Sanitizer::Memory) => {
+ if attrs.flags.contains(CodegenFnAttrFlags::NO_SANITIZE_MEMORY) {
+ return false;
+ }
+ }
+ Some(Sanitizer::Thread) => {
+ if attrs.flags.contains(CodegenFnAttrFlags::NO_SANITIZE_THREAD) {
+ return false;
+ }
+ }
+ Some(Sanitizer::Leak) => {}
+ None => {}
+ }
+
+ if tcx.sess.opts.debugging_opts.cross_crate_inline_threshold.is_none() {
+ return false;
+ }
+ /*
+ if tcx.sess.opts.optimize != OptLevel::Aggressive {
+ return false;
+ }
+ */
+ let r = allow_cross_inline(tcx, def_id, &tcx.optimized_mir(def_id).unwrap_read_only());
+ /*if r {
+ eprintln!("cross_crate_inlinable({:?})", def_id);
+ }*/
+ r
+}
+
+const THRESHOLD: usize = 30;
+
+const INSTR_COST: usize = 2;
+const CALL_PENALTY: usize = 10;
+const LANDINGPAD_PENALTY: usize = 5;
+const RESUME_PENALTY: usize = 5;
+
+const UNKNOWN_SIZE_COST: usize = 2;
+
+fn allow_cross_inline<'tcx>(tcx: TyCtxt<'tcx>, def_id: DefId, body: &Body<'tcx>) -> bool {
+ // Generators should have been transformed here.
+ debug_assert!(body.yield_ty.is_none());
+
+ let mut threshold =
+ tcx.sess.opts.debugging_opts.cross_crate_inline_threshold.unwrap_or(THRESHOLD);
+
+ // Give a bonus functions with a small number of blocks,
+ // We normally have two or three blocks for even
+ // very small functions.
+ if body.basic_blocks().len() <= 3 {
+ threshold += threshold / 4;
+ }
+
+ let mut cost = 0;
+
+ for block in body.basic_blocks() {
+ cost += block.statements.len();
+
+ match block.terminator().kind {
+ TerminatorKind::Drop { unwind, .. } | TerminatorKind::DropAndReplace { unwind, .. } => {
+ cost += CALL_PENALTY;
+ if unwind.is_some() {
+ cost += LANDINGPAD_PENALTY;
+ }
+ }
+ TerminatorKind::Call { cleanup, .. } => {
+ cost += CALL_PENALTY;
+ if cleanup.is_some() {
+ cost += LANDINGPAD_PENALTY;
+ }
+ }
+ TerminatorKind::Assert { cleanup, .. } => {
+ cost += CALL_PENALTY;
+
+ if cleanup.is_some() {
+ cost += LANDINGPAD_PENALTY;
+ }
+ }
+ TerminatorKind::Resume => cost += RESUME_PENALTY,
+ _ => cost += INSTR_COST,
+ }
+ }
+
+ // Count up the cost of local variables and temps, if we know the size
+ // use that, otherwise we use a moderately-large dummy cost.
+
+ let ptr_size = tcx.data_layout.pointer_size.bytes();
+
+ let param_env = tcx.param_env(def_id);
+
+ for v in body.vars_and_temps_iter() {
+ let v = &body.local_decls[v];
+ // Cost of the var is the size in machine-words, if we know
+ // it.
+ if let Some(size) =
+ tcx.layout_of(param_env.and(v.ty)).ok().map(|layout| layout.size.bytes())
+ {
+ cost += (size / ptr_size) as usize;
+ } else {
+ cost += UNKNOWN_SIZE_COST;
+ }
+ }
+
+ cost <= threshold
+}

src/librustc_mir/lib.rs

@@ -34,6 +34,7 @@ extern crate rustc;
 
 mod borrow_check;
 pub mod const_eval;
+mod cross_crate_inline;
 pub mod dataflow;
 pub mod interpret;
 pub mod monomorphize;
@@ -47,6 +48,7 @@ pub fn provide(providers: &mut Providers<'_>) {
  borrow_check::provide(providers);
  const_eval::provide(providers);
  shim::provide(providers);
+ cross_crate_inline::provide(providers);
  transform::provide(providers);
  monomorphize::partitioning::provide(providers);
  providers.const_eval_validated = const_eval::const_eval_validated_provider;