diff --git a/compiler/rustc_interface/src/tests.rs b/compiler/rustc_interface/src/tests.rs index 8e66083a390fc..d308169552317 100644 --- a/compiler/rustc_interface/src/tests.rs +++ b/compiler/rustc_interface/src/tests.rs @@ -4,11 +4,11 @@ use rustc_data_structures::profiling::TimePassesFormat; use rustc_errors::{emitter::HumanReadableErrorType, registry, ColorConfig}; use rustc_session::config::{ build_configuration, build_session_options, rustc_optgroups, BranchProtection, CFGuard, Cfg, - DebugInfo, DumpMonoStatsFormat, ErrorOutputType, ExternEntry, ExternLocation, Externs, Input, - InstrumentCoverage, InstrumentXRay, LinkSelfContained, LinkerPluginLto, LocationDetail, LtoCli, - MirSpanview, OomStrategy, Options, OutFileName, OutputType, OutputTypes, PAuthKey, PacRet, - Passes, Polonius, ProcMacroExecutionStrategy, Strip, SwitchWithOptPath, SymbolManglingVersion, - TraitSolver, WasiExecModel, + DebugInfo, DumpMonoStatsFormat, ErrorOutputType, ExternEntry, ExternLocation, Externs, + InliningThreshold, Input, InstrumentCoverage, InstrumentXRay, LinkSelfContained, + LinkerPluginLto, LocationDetail, LtoCli, MirSpanview, OomStrategy, Options, OutFileName, + OutputType, OutputTypes, PAuthKey, PacRet, Passes, Polonius, ProcMacroExecutionStrategy, Strip, + SwitchWithOptPath, SymbolManglingVersion, TraitSolver, WasiExecModel, }; use rustc_session::lint::Level; use rustc_session::search_paths::SearchPath; @@ -748,7 +748,7 @@ fn test_unstable_options_tracking_hash() { ); tracked!(codegen_backend, Some("abc".to_string())); tracked!(crate_attr, vec!["abc".to_string()]); - tracked!(cross_crate_inline_threshold, Some(200)); + tracked!(cross_crate_inline_threshold, InliningThreshold::Always); tracked!(debug_info_for_profiling, true); tracked!(debug_macros, true); tracked!(dep_info_omit_d_target, true); diff --git a/compiler/rustc_mir_transform/src/cross_crate_inline.rs b/compiler/rustc_mir_transform/src/cross_crate_inline.rs index 4d0e261ed1fd7..261d9dd448d47 100644 --- a/compiler/rustc_mir_transform/src/cross_crate_inline.rs +++ b/compiler/rustc_mir_transform/src/cross_crate_inline.rs @@ -7,6 +7,7 @@ use rustc_middle::mir::visit::Visitor; use rustc_middle::mir::*; use rustc_middle::query::Providers; use rustc_middle::ty::TyCtxt; +use rustc_session::config::InliningThreshold; use rustc_session::config::OptLevel; pub fn provide(providers: &mut Providers) { @@ -54,6 +55,12 @@ fn cross_crate_inlinable(tcx: TyCtxt<'_>, def_id: LocalDefId) -> bool { return false; } + let threshold = match tcx.sess.opts.unstable_opts.cross_crate_inline_threshold { + InliningThreshold::Always => return true, + InliningThreshold::Sometimes(threshold) => threshold, + InliningThreshold::Never => return false, + }; + let mir = tcx.optimized_mir(def_id); let mut checker = CostChecker { tcx, callee_body: mir, calls: 0, statements: 0, landing_pads: 0, resumes: 0 }; @@ -61,8 +68,7 @@ fn cross_crate_inlinable(tcx: TyCtxt<'_>, def_id: LocalDefId) -> bool { checker.calls == 0 && checker.resumes == 0 && checker.landing_pads == 0 - && checker.statements - <= tcx.sess.opts.unstable_opts.cross_crate_inline_threshold.unwrap_or(100) + && checker.statements <= threshold } struct CostChecker<'b, 'tcx> { diff --git a/compiler/rustc_session/src/config.rs b/compiler/rustc_session/src/config.rs index 930fb693eb6d2..f745bc390cadc 100644 --- a/compiler/rustc_session/src/config.rs +++ b/compiler/rustc_session/src/config.rs @@ -3161,10 +3161,10 @@ impl PpMode { pub(crate) mod dep_tracking { use super::{ BranchProtection, CFGuard, CFProtection, CrateType, DebugInfo, DebugInfoCompression, - ErrorOutputType, InstrumentCoverage, InstrumentXRay, LinkerPluginLto, LocationDetail, - LtoCli, OomStrategy, OptLevel, OutFileName, OutputType, OutputTypes, Polonius, - RemapPathScopeComponents, ResolveDocLinks, SourceFileHashAlgorithm, SplitDwarfKind, - SwitchWithOptPath, SymbolManglingVersion, TraitSolver, TrimmedDefPaths, + ErrorOutputType, InliningThreshold, InstrumentCoverage, InstrumentXRay, LinkerPluginLto, + LocationDetail, LtoCli, OomStrategy, OptLevel, OutFileName, OutputType, OutputTypes, + Polonius, RemapPathScopeComponents, ResolveDocLinks, SourceFileHashAlgorithm, + SplitDwarfKind, SwitchWithOptPath, SymbolManglingVersion, TraitSolver, TrimmedDefPaths, }; use crate::lint; use crate::options::WasiExecModel; @@ -3270,6 +3270,7 @@ pub(crate) mod dep_tracking { LanguageIdentifier, TraitSolver, Polonius, + InliningThreshold, ); impl DepTrackingHash for (T1, T2) @@ -3435,3 +3436,16 @@ impl Polonius { matches!(self, Polonius::Next) } } + +#[derive(Clone, Copy, PartialEq, Hash, Debug)] +pub enum InliningThreshold { + Always, + Sometimes(usize), + Never, +} + +impl Default for InliningThreshold { + fn default() -> Self { + Self::Sometimes(100) + } +} diff --git a/compiler/rustc_session/src/options.rs b/compiler/rustc_session/src/options.rs index 7510a41485af6..964a26e94fed9 100644 --- a/compiler/rustc_session/src/options.rs +++ b/compiler/rustc_session/src/options.rs @@ -428,6 +428,8 @@ mod desc { "one of supported execution strategies (`same-thread`, or `cross-thread`)"; pub const parse_dump_solver_proof_tree: &str = "one of: `always`, `on-request`, `on-error`"; pub const parse_remap_path_scope: &str = "comma separated list of scopes: `macro`, `diagnostics`, `unsplit-debuginfo`, `split-debuginfo`, `split-debuginfo-path`, `object`, `all`"; + pub const parse_inlining_threshold: &str = + "either a boolean (`yes`, `no`, `on`, `off`, etc), or a non-negative number"; } mod parse { @@ -1310,6 +1312,26 @@ mod parse { }; true } + + pub(crate) fn parse_inlining_threshold(slot: &mut InliningThreshold, v: Option<&str>) -> bool { + match v { + Some("always" | "yes") => { + *slot = InliningThreshold::Always; + } + Some("never") => { + *slot = InliningThreshold::Never; + } + Some(v) => { + if let Ok(threshold) = v.parse() { + *slot = InliningThreshold::Sometimes(threshold); + } else { + return false; + } + } + None => return false, + } + true + } } options! { @@ -1479,7 +1501,7 @@ options! { "combine CGUs into a single one"), crate_attr: Vec = (Vec::new(), parse_string_push, [TRACKED], "inject the given attribute in the crate"), - cross_crate_inline_threshold: Option = (None, parse_opt_number, [TRACKED], + cross_crate_inline_threshold: InliningThreshold = (InliningThreshold::Sometimes(100), parse_inlining_threshold, [TRACKED], "threshold to allow cross crate inlining of functions"), debug_info_for_profiling: bool = (false, parse_bool, [TRACKED], "emit discriminators and other data necessary for AutoFDO"), diff --git a/tests/codegen/cross-crate-inlining/always-inline.rs b/tests/codegen/cross-crate-inlining/always-inline.rs new file mode 100644 index 0000000000000..f3f08bf116aee --- /dev/null +++ b/tests/codegen/cross-crate-inlining/always-inline.rs @@ -0,0 +1,13 @@ +// compile-flags: -O +// aux-build:always.rs + +#![crate_type = "lib"] + +extern crate always; + +// Check that we inline a cross-crate call, even though it isn't a leaf +#[no_mangle] +pub fn outer() -> String { + // CHECK-NOT: call {{.*}}stem_fn + always::stem_fn() +} diff --git a/tests/codegen/cross-crate-inlining/auxiliary/always.rs b/tests/codegen/cross-crate-inlining/auxiliary/always.rs new file mode 100644 index 0000000000000..3670307ec8134 --- /dev/null +++ b/tests/codegen/cross-crate-inlining/auxiliary/always.rs @@ -0,0 +1,20 @@ +// compile-flags: -O -Zcross-crate-inline-threshold=always + +#![crate_type = "lib"] + +// This function *looks* like it contains a call, but that call will be optimized out by MIR +// optimizations. +pub fn leaf_fn() -> String { + String::new() +} + +// This function contains a call, even after MIR optimizations. It is only eligible for +// cross-crate-inlining with "always". +pub fn stem_fn() -> String { + inner() +} + +#[inline(never)] +fn inner() -> String { + String::from("test") +} diff --git a/tests/codegen/cross-crate-inlining/auxiliary/leaf.rs b/tests/codegen/cross-crate-inlining/auxiliary/leaf.rs new file mode 100644 index 0000000000000..963f087f22d4d --- /dev/null +++ b/tests/codegen/cross-crate-inlining/auxiliary/leaf.rs @@ -0,0 +1,20 @@ +// compile-flags: -O + +#![crate_type = "lib"] + +// This function *looks* like it contains a call, but that call will be optimized out by MIR +// optimizations. +pub fn leaf_fn() -> String { + String::new() +} + +// This function contains a call, even after MIR optimizations. It is only eligible for +// cross-crate-inlining with "always". +pub fn stem_fn() -> String { + inner() +} + +#[inline(never)] +fn inner() -> String { + String::from("test") +} diff --git a/tests/codegen/cross-crate-inlining/auxiliary/never.rs b/tests/codegen/cross-crate-inlining/auxiliary/never.rs new file mode 100644 index 0000000000000..e222a6dea3871 --- /dev/null +++ b/tests/codegen/cross-crate-inlining/auxiliary/never.rs @@ -0,0 +1,20 @@ +// compile-flags: -O -Zcross-crate-inline-threshold=never + +#![crate_type = "lib"] + +// This function *looks* like it contains a call, but that call will be optimized out by MIR +// optimizations. +pub fn leaf_fn() -> String { + String::new() +} + +// This function contains a call, even after MIR optimizations. It is only eligible for +// cross-crate-inlining with "always". +pub fn stem_fn() -> String { + inner() +} + +#[inline(never)] +fn inner() -> String { + String::from("test") +} diff --git a/tests/codegen/cross-crate-inlining/leaf-inlining.rs b/tests/codegen/cross-crate-inlining/leaf-inlining.rs new file mode 100644 index 0000000000000..73b1a520b06cf --- /dev/null +++ b/tests/codegen/cross-crate-inlining/leaf-inlining.rs @@ -0,0 +1,20 @@ +// compile-flags: -O -Zcross-crate-inline-threshold=yes +// aux-build:leaf.rs + +#![crate_type = "lib"] + +extern crate leaf; + +// Check that we inline a leaf cross-crate call +#[no_mangle] +pub fn leaf_outer() -> String { + // CHECK-NOT: call {{.*}}leaf_fn + leaf::leaf_fn() +} + +// Check that we do not inline a non-leaf cross-crate call +#[no_mangle] +pub fn stem_outer() -> String { + // CHECK: call {{.*}}stem_fn + leaf::stem_fn() +} diff --git a/tests/codegen/cross-crate-inlining/never-inline.rs b/tests/codegen/cross-crate-inlining/never-inline.rs new file mode 100644 index 0000000000000..4e7bc3e5154cb --- /dev/null +++ b/tests/codegen/cross-crate-inlining/never-inline.rs @@ -0,0 +1,13 @@ +// compile-flags: -O +// aux-build:never.rs + +#![crate_type = "lib"] + +extern crate never; + +// Check that we do not inline a cross-crate call, even though it is a leaf +#[no_mangle] +pub fn outer() -> String { + // CHECK: call {{.*}}leaf_fn + never::leaf_fn() +}