diff --git a/src/bootstrap/builder.rs b/src/bootstrap/builder.rs index 243cd3fa19998..fc8bfbfd4dcf9 100644 --- a/src/bootstrap/builder.rs +++ b/src/bootstrap/builder.rs @@ -1098,6 +1098,9 @@ impl<'a> Builder<'a> { } if let Mode::Rustc | Mode::Codegen = mode { + if stage > 0 { + rustflags.arg("-Zcross-crate-inline-threshold=50"); + } rustflags.arg("-Zunstable-options"); rustflags.arg("-Wrustc::internal"); } diff --git a/src/librustc/middle/codegen_fn_attrs.rs b/src/librustc/middle/codegen_fn_attrs.rs index 61b25cc486446..14d2d118989fb 100644 --- a/src/librustc/middle/codegen_fn_attrs.rs +++ b/src/librustc/middle/codegen_fn_attrs.rs @@ -95,14 +95,6 @@ impl CodegenFnAttrs { } } - /// Returns `true` if `#[inline]` or `#[inline(always)]` is present. - pub fn requests_inline(&self) -> bool { - match self.inline { - InlineAttr::Hint | InlineAttr::Always => true, - InlineAttr::None | InlineAttr::Never => false, - } - } - /// Returns `true` if it looks like this symbol needs to be exported, for example: /// /// * `#[no_mangle]` is present diff --git a/src/librustc/mir/mono.rs b/src/librustc/mir/mono.rs index 0b64cb479d559..de4567f3e13cc 100644 --- a/src/librustc/mir/mono.rs +++ b/src/librustc/mir/mono.rs @@ -90,6 +90,18 @@ impl<'tcx> MonoItem<'tcx> { match *self { MonoItem::Fn(ref instance) => { + /*use rustc_hir::ItemKind; + use rustc_hir::Node; + tcx.hir().as_local_hir_id(instance.def_id()).map(|id| match tcx.hir().get(id) { + Node::Item(item) => match item.kind { + ItemKind::Static(..) | ItemKind::Const(..) => { + panic!("STATIC!! {:?}", self); + } + _ => (), + }, + _ => (), + });*/ + let entry_def_id = tcx.entry_fn(LOCAL_CRATE).map(|(id, _)| id); // If this function isn't inlined or otherwise has explicit // linkage, then we'll be creating a globally shared version. diff --git a/src/librustc/query/mod.rs b/src/librustc/query/mod.rs index c1ece6275092d..cc1e19b504f97 100644 --- a/src/librustc/query/mod.rs +++ b/src/librustc/query/mod.rs @@ -649,6 +649,8 @@ rustc_queries! { query codegen_fn_attrs(_: DefId) -> CodegenFnAttrs { cache_on_disk_if { true } } + + query cross_crate_inlinable(_: DefId) -> bool {} } Other { diff --git a/src/librustc/ty/instance.rs b/src/librustc/ty/instance.rs index fcfee0fea85af..38a534cd89cae 100644 --- a/src/librustc/ty/instance.rs +++ b/src/librustc/ty/instance.rs @@ -204,12 +204,13 @@ impl<'tcx> InstanceDef<'tcx> { // drops of `Option::None` before LTO. We also respect the intent of // `#[inline]` on `Drop::drop` implementations. return ty.ty_adt_def().map_or(true, |adt_def| { - adt_def.destructor(tcx).map_or(adt_def.is_enum(), |dtor| { - tcx.codegen_fn_attrs(dtor.did).requests_inline() - }) + adt_def + .destructor(tcx) + .map_or(adt_def.is_enum(), |dtor| tcx.cross_crate_inlinable(dtor.did)) }); } - tcx.codegen_fn_attrs(self.def_id()).requests_inline() + + tcx.cross_crate_inlinable(self.def_id()) } pub fn requires_caller_location(&self, tcx: TyCtxt<'_>) -> bool { diff --git a/src/librustc_codegen_ssa/back/symbol_export.rs b/src/librustc_codegen_ssa/back/symbol_export.rs index 8a2503ce16730..9bf04c805784a 100644 --- a/src/librustc_codegen_ssa/back/symbol_export.rs +++ b/src/librustc_codegen_ssa/back/symbol_export.rs @@ -85,8 +85,7 @@ fn reachable_non_generics_provider( } // Only consider nodes that actually have exported symbols. - Node::Item(&hir::Item { kind: hir::ItemKind::Static(..), .. }) - | Node::Item(&hir::Item { kind: hir::ItemKind::Fn(..), .. }) + Node::Item(&hir::Item { kind: hir::ItemKind::Fn(..), .. }) | Node::ImplItem(&hir::ImplItem { kind: hir::ImplItemKind::Fn(..), .. }) => { let def_id = tcx.hir().local_def_id(hir_id); let generics = tcx.generics_of(def_id); @@ -101,6 +100,15 @@ fn reachable_non_generics_provider( } } + Node::Item(&hir::Item { kind: hir::ItemKind::Static(..), .. }) => { + let def_id = tcx.hir().local_def_id(hir_id); + let generics = tcx.generics_of(def_id); + let r = + if !generics.requires_monomorphization(tcx) { Some(def_id) } else { None }; + //assert!(r.is_some()); + r + } + _ => None, } }) diff --git a/src/librustc_metadata/rmeta/decoder.rs b/src/librustc_metadata/rmeta/decoder.rs index 4520df588996e..2ab7a53507a89 100644 --- a/src/librustc_metadata/rmeta/decoder.rs +++ b/src/librustc_metadata/rmeta/decoder.rs @@ -1099,6 +1099,15 @@ impl<'a, 'tcx> CrateMetadataRef<'a> { !self.is_proc_macro(id) && self.root.per_def.mir.get(self, id).is_some() } + fn is_item_cross_crate_inlinable(&self, tcx: TyCtxt<'tcx>, id: DefIndex) -> bool { + self.root + .per_def + .cross_crate_inlinable + .get(self, id) + .map(|v| v.decode((self, tcx))) + .unwrap_or(false) + } + fn get_optimized_mir(&self, tcx: TyCtxt<'tcx>, id: DefIndex) -> BodyAndCache<'tcx> { let mut cache = self .root diff --git a/src/librustc_metadata/rmeta/decoder/cstore_impl.rs b/src/librustc_metadata/rmeta/decoder/cstore_impl.rs index 7a1ac9e0a60ec..53467991130c9 100644 --- a/src/librustc_metadata/rmeta/decoder/cstore_impl.rs +++ b/src/librustc_metadata/rmeta/decoder/cstore_impl.rs @@ -149,6 +149,7 @@ provide! { <'tcx> tcx, def_id, other, cdata, impl_parent => { cdata.get_parent_impl(def_id.index) } trait_of_item => { cdata.get_trait_of_item(def_id.index) } is_mir_available => { cdata.is_item_mir_available(def_id.index) } + cross_crate_inlinable => { cdata.is_item_cross_crate_inlinable(tcx, def_id.index) } dylib_dependency_formats => { cdata.get_dylib_dependency_formats(tcx) } is_panic_runtime => { cdata.root.panic_runtime } diff --git a/src/librustc_metadata/rmeta/encoder.rs b/src/librustc_metadata/rmeta/encoder.rs index 3686b2f20487b..22702b51e5d92 100644 --- a/src/librustc_metadata/rmeta/encoder.rs +++ b/src/librustc_metadata/rmeta/encoder.rs @@ -882,6 +882,9 @@ impl EncodeContext<'tcx> { ty::AssocKind::OpaqueTy => unreachable!(), } if trait_item.kind == ty::AssocKind::Method { + if self.tcx.mir_keys(LOCAL_CRATE).contains(&def_id) { + record!(self.per_def.cross_crate_inlinable[def_id] <- self.tcx.cross_crate_inlinable(def_id)); + } record!(self.per_def.fn_sig[def_id] <- tcx.fn_sig(def_id)); self.encode_variances_of(def_id); } @@ -968,9 +971,11 @@ impl EncodeContext<'tcx> { let mir = match ast_item.kind { hir::ImplItemKind::Const(..) => true, hir::ImplItemKind::Fn(ref sig, _) => { + record!(self.per_def.cross_crate_inlinable[def_id] <- self.tcx.cross_crate_inlinable(def_id)); + let generics = self.tcx.generics_of(def_id); let needs_inline = (generics.requires_monomorphization(self.tcx) - || tcx.codegen_fn_attrs(def_id).requests_inline()) + || tcx.cross_crate_inlinable(def_id)) && !self.metadata_output_only(); let is_const_fn = sig.header.constness == hir::Constness::Const; let always_encode_mir = self.tcx.sess.opts.debugging_opts.always_encode_mir; @@ -1266,9 +1271,11 @@ impl EncodeContext<'tcx> { let mir = match item.kind { hir::ItemKind::Static(..) | hir::ItemKind::Const(..) => true, hir::ItemKind::Fn(ref sig, ..) => { + record!(self.per_def.cross_crate_inlinable[def_id] <- self.tcx.cross_crate_inlinable(def_id)); + let generics = tcx.generics_of(def_id); let needs_inline = (generics.requires_monomorphization(tcx) - || tcx.codegen_fn_attrs(def_id).requests_inline()) + || tcx.cross_crate_inlinable(def_id)) && !self.metadata_output_only(); let always_encode_mir = self.tcx.sess.opts.debugging_opts.always_encode_mir; needs_inline || sig.header.constness == hir::Constness::Const || always_encode_mir @@ -1743,8 +1750,8 @@ impl<'tcx, 'v> ParItemLikeVisitor<'v> for PrefetchVisitor<'tcx> { hir::ItemKind::Fn(ref sig, ..) => { let def_id = tcx.hir().local_def_id(item.hir_id); let generics = tcx.generics_of(def_id); - let needs_inline = generics.requires_monomorphization(tcx) - || tcx.codegen_fn_attrs(def_id).requests_inline(); + let needs_inline = + generics.requires_monomorphization(tcx) || tcx.cross_crate_inlinable(def_id); if needs_inline || sig.header.constness == hir::Constness::Const { self.prefetch_mir(def_id) } @@ -1768,8 +1775,8 @@ impl<'tcx, 'v> ParItemLikeVisitor<'v> for PrefetchVisitor<'tcx> { hir::ImplItemKind::Fn(ref sig, _) => { let def_id = tcx.hir().local_def_id(impl_item.hir_id); let generics = tcx.generics_of(def_id); - let needs_inline = generics.requires_monomorphization(tcx) - || tcx.codegen_fn_attrs(def_id).requests_inline(); + let needs_inline = + generics.requires_monomorphization(tcx) || tcx.cross_crate_inlinable(def_id); let is_const_fn = sig.header.constness == hir::Constness::Const; if needs_inline || is_const_fn { self.prefetch_mir(def_id) diff --git a/src/librustc_metadata/rmeta/mod.rs b/src/librustc_metadata/rmeta/mod.rs index 57415c0a6070d..8ec734f866d1c 100644 --- a/src/librustc_metadata/rmeta/mod.rs +++ b/src/librustc_metadata/rmeta/mod.rs @@ -276,6 +276,7 @@ define_per_def_tables! { inferred_outlives: Table, Span)])>, super_predicates: Table)>, mir: Table)>, + cross_crate_inlinable: Table>, promoted_mir: Table>)>, } diff --git a/src/librustc_mir/cross_crate_inline.rs b/src/librustc_mir/cross_crate_inline.rs new file mode 100644 index 0000000000000..4b6274e467327 --- /dev/null +++ b/src/librustc_mir/cross_crate_inline.rs @@ -0,0 +1,161 @@ +use rustc::middle::codegen_fn_attrs::CodegenFnAttrFlags; +use rustc::mir::{Body, TerminatorKind}; +use rustc::ty::query::Providers; +use rustc::ty::TyCtxt; +use rustc_attr::InlineAttr; +use rustc_hir::def_id::DefId; +//use rustc_hir::ItemKind; +//use rustc_session::config::OptLevel; +use rustc_session::config::Sanitizer; + +pub fn provide(providers: &mut Providers<'_>) { + providers.cross_crate_inlinable = cross_crate_inlinable; +} + +fn cross_crate_inlinable(tcx: TyCtxt<'_>, def_id: DefId) -> bool { + /*use rustc_hir::Node; + tcx.hir().as_local_hir_id(def_id).map(|id| match tcx.hir().get(id) { + Node::Item(item) => match item.kind { + ItemKind::Static(..) | ItemKind::Const(..) => { + panic!("STATIC!! {:?}", def_id); + } + _ => (), + }, + _ => (), + });*/ + + let attrs = tcx.codegen_fn_attrs(def_id); + + match attrs.inline { + InlineAttr::Hint | InlineAttr::Always => return true, + InlineAttr::Never => return false, + InlineAttr::None => (), + } + + if attrs.contains_extern_indicator() { + return false; + } + + if tcx.lang_items().start_fn() == Some(def_id) { + return false; + } + + // FIXME: Share some of this logic with the MIR inlining pass + + // FIXME: Is this needed? + if attrs.flags.contains(CodegenFnAttrFlags::TRACK_CALLER) { + return false; + } + + // Avoid inlining functions marked as no_sanitize if sanitizer is enabled, + // since instrumentation might be enabled and performed on the caller. + match tcx.sess.opts.debugging_opts.sanitizer { + Some(Sanitizer::Address) => { + if attrs.flags.contains(CodegenFnAttrFlags::NO_SANITIZE_ADDRESS) { + return false; + } + } + Some(Sanitizer::Memory) => { + if attrs.flags.contains(CodegenFnAttrFlags::NO_SANITIZE_MEMORY) { + return false; + } + } + Some(Sanitizer::Thread) => { + if attrs.flags.contains(CodegenFnAttrFlags::NO_SANITIZE_THREAD) { + return false; + } + } + Some(Sanitizer::Leak) => {} + None => {} + } + + if tcx.sess.opts.debugging_opts.cross_crate_inline_threshold.is_none() { + return false; + } + /* + if tcx.sess.opts.optimize != OptLevel::Aggressive { + return false; + } + */ + let r = allow_cross_inline(tcx, def_id, &tcx.optimized_mir(def_id).unwrap_read_only()); + /*if r { + eprintln!("cross_crate_inlinable({:?})", def_id); + }*/ + r +} + +const THRESHOLD: usize = 30; + +const INSTR_COST: usize = 2; +const CALL_PENALTY: usize = 10; +const LANDINGPAD_PENALTY: usize = 5; +const RESUME_PENALTY: usize = 5; + +const UNKNOWN_SIZE_COST: usize = 2; + +fn allow_cross_inline<'tcx>(tcx: TyCtxt<'tcx>, def_id: DefId, body: &Body<'tcx>) -> bool { + // Generators should have been transformed here. + debug_assert!(body.yield_ty.is_none()); + + let mut threshold = + tcx.sess.opts.debugging_opts.cross_crate_inline_threshold.unwrap_or(THRESHOLD); + + // Give a bonus functions with a small number of blocks, + // We normally have two or three blocks for even + // very small functions. + if body.basic_blocks().len() <= 3 { + threshold += threshold / 4; + } + + let mut cost = 0; + + for block in body.basic_blocks() { + cost += block.statements.len(); + + match block.terminator().kind { + TerminatorKind::Drop { unwind, .. } | TerminatorKind::DropAndReplace { unwind, .. } => { + cost += CALL_PENALTY; + if unwind.is_some() { + cost += LANDINGPAD_PENALTY; + } + } + TerminatorKind::Call { cleanup, .. } => { + cost += CALL_PENALTY; + if cleanup.is_some() { + cost += LANDINGPAD_PENALTY; + } + } + TerminatorKind::Assert { cleanup, .. } => { + cost += CALL_PENALTY; + + if cleanup.is_some() { + cost += LANDINGPAD_PENALTY; + } + } + TerminatorKind::Resume => cost += RESUME_PENALTY, + _ => cost += INSTR_COST, + } + } + + // Count up the cost of local variables and temps, if we know the size + // use that, otherwise we use a moderately-large dummy cost. + + let ptr_size = tcx.data_layout.pointer_size.bytes(); + + let param_env = tcx.param_env(def_id); + + for v in body.vars_and_temps_iter() { + let v = &body.local_decls[v]; + // Cost of the var is the size in machine-words, if we know + // it. + if let Some(size) = + tcx.layout_of(param_env.and(v.ty)).ok().map(|layout| layout.size.bytes()) + { + cost += (size / ptr_size) as usize; + } else { + cost += UNKNOWN_SIZE_COST; + } + } + + cost <= threshold +} diff --git a/src/librustc_mir/lib.rs b/src/librustc_mir/lib.rs index 85e44adc30b86..e613d5d619de3 100644 --- a/src/librustc_mir/lib.rs +++ b/src/librustc_mir/lib.rs @@ -34,6 +34,7 @@ extern crate rustc; mod borrow_check; pub mod const_eval; +mod cross_crate_inline; pub mod dataflow; pub mod interpret; pub mod monomorphize; @@ -47,6 +48,7 @@ pub fn provide(providers: &mut Providers<'_>) { borrow_check::provide(providers); const_eval::provide(providers); shim::provide(providers); + cross_crate_inline::provide(providers); transform::provide(providers); monomorphize::partitioning::provide(providers); providers.const_eval_validated = const_eval::const_eval_validated_provider; diff --git a/src/librustc_passes/reachable.rs b/src/librustc_passes/reachable.rs index 1e9c6c91d385f..f1ddda86c1285 100644 --- a/src/librustc_passes/reachable.rs +++ b/src/librustc_passes/reachable.rs @@ -5,7 +5,7 @@ // makes all other generics or inline functions that it references // reachable as well. -use rustc::middle::codegen_fn_attrs::{CodegenFnAttrFlags, CodegenFnAttrs}; +use rustc::middle::codegen_fn_attrs::CodegenFnAttrFlags; use rustc::middle::privacy; use rustc::ty::query::Providers; use rustc::ty::{self, TyCtxt}; @@ -24,8 +24,8 @@ use rustc_target::spec::abi::Abi; // Returns true if the given item must be inlined because it may be // monomorphized or it was marked with `#[inline]`. This will only return // true for functions. -fn item_might_be_inlined(tcx: TyCtxt<'tcx>, item: &hir::Item<'_>, attrs: CodegenFnAttrs) -> bool { - if attrs.requests_inline() { +fn item_might_be_inlined(tcx: TyCtxt<'tcx>, item: &hir::Item<'_>, inlinable: bool) -> bool { + if inlinable { return true; } @@ -44,9 +44,9 @@ fn method_might_be_inlined( impl_item: &hir::ImplItem<'_>, impl_src: DefId, ) -> bool { - let codegen_fn_attrs = tcx.codegen_fn_attrs(impl_item.hir_id.owner.to_def_id()); + let inlinable = tcx.cross_crate_inlinable(impl_item.hir_id.owner.to_def_id()); let generics = tcx.generics_of(tcx.hir().local_def_id(impl_item.hir_id)); - if codegen_fn_attrs.requests_inline() || generics.requires_monomorphization(tcx) { + if inlinable || generics.requires_monomorphization(tcx) { return true; } if let hir::ImplItemKind::Fn(method_sig, _) = &impl_item.kind { @@ -56,7 +56,7 @@ fn method_might_be_inlined( } if let Some(impl_hir_id) = tcx.hir().as_local_hir_id(impl_src) { match tcx.hir().find(impl_hir_id) { - Some(Node::Item(item)) => item_might_be_inlined(tcx, &item, codegen_fn_attrs), + Some(Node::Item(item)) => item_might_be_inlined(tcx, &item, inlinable), Some(..) | None => span_bug!(impl_item.span, "impl did is not an item"), } } else { @@ -152,7 +152,7 @@ impl<'a, 'tcx> ReachableContext<'a, 'tcx> { match self.tcx.hir().find(hir_id) { Some(Node::Item(item)) => match item.kind { hir::ItemKind::Fn(..) => { - item_might_be_inlined(self.tcx, &item, self.tcx.codegen_fn_attrs(def_id)) + item_might_be_inlined(self.tcx, &item, self.tcx.cross_crate_inlinable(def_id)) } _ => false, }, @@ -166,9 +166,9 @@ impl<'a, 'tcx> ReachableContext<'a, 'tcx> { match impl_item.kind { hir::ImplItemKind::Const(..) => true, hir::ImplItemKind::Fn(..) => { - let attrs = self.tcx.codegen_fn_attrs(def_id); + let inlinable = self.tcx.cross_crate_inlinable(def_id); let generics = self.tcx.generics_of(def_id); - if generics.requires_monomorphization(self.tcx) || attrs.requests_inline() { + if generics.requires_monomorphization(self.tcx) || inlinable { true } else { let impl_did = self.tcx.hir().get_parent_did(hir_id); @@ -239,8 +239,11 @@ impl<'a, 'tcx> ReachableContext<'a, 'tcx> { match item.kind { hir::ItemKind::Fn(.., body) => { let def_id = self.tcx.hir().local_def_id(item.hir_id); - if item_might_be_inlined(self.tcx, &item, self.tcx.codegen_fn_attrs(def_id)) - { + if item_might_be_inlined( + self.tcx, + &item, + self.tcx.cross_crate_inlinable(def_id), + ) { self.visit_nested_body(body); } } diff --git a/src/librustc_session/options.rs b/src/librustc_session/options.rs index e43908a79143b..e060d75025a52 100644 --- a/src/librustc_session/options.rs +++ b/src/librustc_session/options.rs @@ -775,6 +775,8 @@ options! {DebuggingOptions, DebuggingSetter, basic_debugging_options, "run all passes except codegen; no output"), treat_err_as_bug: Option = (None, parse_treat_err_as_bug, [TRACKED], "treat error number `val` that occurs as bug"), + cross_crate_inline_threshold: Option = (None, parse_treat_err_as_bug, [TRACKED], + "threshold to allow cross crate inlining of functions"), report_delayed_bugs: bool = (false, parse_bool, [TRACKED], "immediately print bugs registered with `delay_span_bug`"), macro_backtrace: bool = (false, parse_bool, [UNTRACKED], diff --git a/src/librustc_symbol_mangling/lib.rs b/src/librustc_symbol_mangling/lib.rs index 26cb341050027..1329446bbe1da 100644 --- a/src/librustc_symbol_mangling/lib.rs +++ b/src/librustc_symbol_mangling/lib.rs @@ -101,6 +101,7 @@ use rustc::mir::mono::{InstantiationMode, MonoItem}; use rustc::ty::query::Providers; use rustc::ty::subst::SubstsRef; use rustc::ty::{self, Instance, TyCtxt}; +use rustc_hir::def::DefKind; use rustc_hir::def_id::{CrateNum, LOCAL_CRATE}; use rustc_hir::Node; use rustc_session::config::SymbolManglingVersion; @@ -233,6 +234,8 @@ fn compute_symbol_name( // project. is_generic(substs) || + (tcx.def_kind(instance.def_id()) != Some(DefKind::Static) && + // If we're dealing with an instance of a function that's inlined from // another crate but we're marking it as globally shared to our // compliation (aka we're not making an internal copy in each of our @@ -242,7 +245,7 @@ fn compute_symbol_name( match MonoItem::Fn(instance).instantiation_mode(tcx) { InstantiationMode::GloballyShared { may_conflict: true } => true, _ => false, - }; + }); let instantiating_crate = if avoid_cross_crate_conflicts { Some(compute_instantiating_crate()) } else { None }; diff --git a/src/libstd/thread/local.rs b/src/libstd/thread/local.rs index 1dd942e252f6b..55bfbdb956bdf 100644 --- a/src/libstd/thread/local.rs +++ b/src/libstd/thread/local.rs @@ -152,6 +152,9 @@ macro_rules! __thread_local_inner { #[inline] fn __init() -> $t { $init } + // This cannot be inlined due to #[thread_local] not working correctly across + // dylibs. + #[inline(never)] unsafe fn __getit() -> $crate::option::Option<&'static $t> { #[cfg(all(target_arch = "wasm32", not(target_feature = "atomics")))] static __KEY: $crate::thread::__StaticLocalKeyInner<$t> =