Rollup merge of #117650 - saethlin:inline-me-please, r=davidtwco

Add -Zcross-crate-inline-threshold=yes

``@thomcc`` says this would be useful for

>  seeing if it makes a difference in some code if i do it when building the sysroot, since -Zbuild-std + lto helps more than it seems like it should

And I've changed the possible values as a reference to ``@Manishearth`` saying

> LLVM's inlining heuristic is "yes".

compiler/rustc_interface/src/tests.rs

@@ -4,11 +4,11 @@ use rustc_data_structures::profiling::TimePassesFormat;
 use rustc_errors::{emitter::HumanReadableErrorType, registry, ColorConfig};
 use rustc_session::config::{
  build_configuration, build_session_options, rustc_optgroups, BranchProtection, CFGuard, Cfg,
- DebugInfo, DumpMonoStatsFormat, ErrorOutputType, ExternEntry, ExternLocation, Externs, Input,
- InstrumentCoverage, InstrumentXRay, LinkSelfContained, LinkerPluginLto, LocationDetail, LtoCli,
- MirSpanview, OomStrategy, Options, OutFileName, OutputType, OutputTypes, PAuthKey, PacRet,
- Passes, Polonius, ProcMacroExecutionStrategy, Strip, SwitchWithOptPath, SymbolManglingVersion,
- TraitSolver, WasiExecModel,
+ DebugInfo, DumpMonoStatsFormat, ErrorOutputType, ExternEntry, ExternLocation, Externs,
+ InliningThreshold, Input, InstrumentCoverage, InstrumentXRay, LinkSelfContained,
+ LinkerPluginLto, LocationDetail, LtoCli, MirSpanview, OomStrategy, Options, OutFileName,
+ OutputType, OutputTypes, PAuthKey, PacRet, Passes, Polonius, ProcMacroExecutionStrategy, Strip,
+ SwitchWithOptPath, SymbolManglingVersion, TraitSolver, WasiExecModel,
 };
 use rustc_session::lint::Level;
 use rustc_session::search_paths::SearchPath;
@@ -748,7 +748,7 @@ fn test_unstable_options_tracking_hash() {
  );
  tracked!(codegen_backend, Some("abc".to_string()));
  tracked!(crate_attr, vec!["abc".to_string()]);
- tracked!(cross_crate_inline_threshold, Some(200));
+ tracked!(cross_crate_inline_threshold, InliningThreshold::Always);
  tracked!(debug_info_for_profiling, true);
  tracked!(debug_macros, true);
  tracked!(dep_info_omit_d_target, true);

compiler/rustc_mir_transform/src/cross_crate_inline.rs

@@ -7,6 +7,7 @@ use rustc_middle::mir::visit::Visitor;
 use rustc_middle::mir::*;
 use rustc_middle::query::Providers;
 use rustc_middle::ty::TyCtxt;
+use rustc_session::config::InliningThreshold;
 use rustc_session::config::OptLevel;
 
 pub fn provide(providers: &mut Providers) {
@@ -54,15 +55,20 @@ fn cross_crate_inlinable(tcx: TyCtxt<'_>, def_id: LocalDefId) -> bool {
  return false;
  }
 
+ let threshold = match tcx.sess.opts.unstable_opts.cross_crate_inline_threshold {
+ InliningThreshold::Always => return true,
+ InliningThreshold::Sometimes(threshold) => threshold,
+ InliningThreshold::Never => return false,
+ };
+
  let mir = tcx.optimized_mir(def_id);
  let mut checker =
  CostChecker { tcx, callee_body: mir, calls: 0, statements: 0, landing_pads: 0, resumes: 0 };
  checker.visit_body(mir);
  checker.calls == 0
  && checker.resumes == 0
  && checker.landing_pads == 0
- && checker.statements
- <= tcx.sess.opts.unstable_opts.cross_crate_inline_threshold.unwrap_or(100)
+ && checker.statements <= threshold
 }
 
 struct CostChecker<'b, 'tcx> {

compiler/rustc_session/src/config.rs

@@ -3161,10 +3161,10 @@ impl PpMode {
 pub(crate) mod dep_tracking {
  use super::{
  BranchProtection, CFGuard, CFProtection, CrateType, DebugInfo, DebugInfoCompression,
- ErrorOutputType, InstrumentCoverage, InstrumentXRay, LinkerPluginLto, LocationDetail,
- LtoCli, OomStrategy, OptLevel, OutFileName, OutputType, OutputTypes, Polonius,
- RemapPathScopeComponents, ResolveDocLinks, SourceFileHashAlgorithm, SplitDwarfKind,
- SwitchWithOptPath, SymbolManglingVersion, TraitSolver, TrimmedDefPaths,
+ ErrorOutputType, InliningThreshold, InstrumentCoverage, InstrumentXRay, LinkerPluginLto,
+ LocationDetail, LtoCli, OomStrategy, OptLevel, OutFileName, OutputType, OutputTypes,
+ Polonius, RemapPathScopeComponents, ResolveDocLinks, SourceFileHashAlgorithm,
+ SplitDwarfKind, SwitchWithOptPath, SymbolManglingVersion, TraitSolver, TrimmedDefPaths,
  };
  use crate::lint;
  use crate::options::WasiExecModel;
@@ -3270,6 +3270,7 @@ pub(crate) mod dep_tracking {
  LanguageIdentifier,
  TraitSolver,
  Polonius,
+ InliningThreshold,
  );
 
  impl<T1, T2> DepTrackingHash for (T1, T2)
@@ -3435,3 +3436,16 @@ impl Polonius {
  matches!(self, Polonius::Next)
  }
 }
+
+#[derive(Clone, Copy, PartialEq, Hash, Debug)]
+pub enum InliningThreshold {
+ Always,
+ Sometimes(usize),
+ Never,
+}
+
+impl Default for InliningThreshold {
+ fn default() -> Self {
+ Self::Sometimes(100)
+ }
+}

compiler/rustc_session/src/options.rs

@@ -428,6 +428,8 @@ mod desc {
  "one of supported execution strategies (`same-thread`, or `cross-thread`)";
  pub const parse_dump_solver_proof_tree: &str = "one of: `always`, `on-request`, `on-error`";
  pub const parse_remap_path_scope: &str = "comma separated list of scopes: `macro`, `diagnostics`, `unsplit-debuginfo`, `split-debuginfo`, `split-debuginfo-path`, `object`, `all`";
+ pub const parse_inlining_threshold: &str =
+ "either a boolean (`yes`, `no`, `on`, `off`, etc), or a non-negative number";
 }
 
 mod parse {
@@ -1310,6 +1312,26 @@ mod parse {
  };
  true
  }
+
+ pub(crate) fn parse_inlining_threshold(slot: &mut InliningThreshold, v: Option<&str>) -> bool {
+ match v {
+ Some("always" | "yes") => {
+ *slot = InliningThreshold::Always;
+ }
+ Some("never") => {
+ *slot = InliningThreshold::Never;
+ }
+ Some(v) => {
+ if let Ok(threshold) = v.parse() {
+ *slot = InliningThreshold::Sometimes(threshold);
+ } else {
+ return false;
+ }
+ }
+ None => return false,
+ }
+ true
+ }
 }
 
 options! {
@@ -1479,7 +1501,7 @@ options! {
  "combine CGUs into a single one"),
  crate_attr: Vec<String> = (Vec::new(), parse_string_push, [TRACKED],
  "inject the given attribute in the crate"),
- cross_crate_inline_threshold: Option<usize> = (None, parse_opt_number, [TRACKED],
+ cross_crate_inline_threshold: InliningThreshold = (InliningThreshold::Sometimes(100), parse_inlining_threshold, [TRACKED],
  "threshold to allow cross crate inlining of functions"),
  debug_info_for_profiling: bool = (false, parse_bool, [TRACKED],
  "emit discriminators and other data necessary for AutoFDO"),

tests/codegen/cross-crate-inlining/always-inline.rs

@@ -0,0 +1,13 @@
+// compile-flags: -O
+// aux-build:always.rs
+
+#![crate_type = "lib"]
+
+extern crate always;
+
+// Check that we inline a cross-crate call, even though it isn't a leaf
+#[no_mangle]
+pub fn outer() -> String {
+ // CHECK-NOT: call {{.*}}stem_fn
+ always::stem_fn()
+}

tests/codegen/cross-crate-inlining/auxiliary/always.rs

@@ -0,0 +1,20 @@
+// compile-flags: -O -Zcross-crate-inline-threshold=always
+
+#![crate_type = "lib"]
+
+// This function *looks* like it contains a call, but that call will be optimized out by MIR
+// optimizations.
+pub fn leaf_fn() -> String {
+ String::new()
+}
+
+// This function contains a call, even after MIR optimizations. It is only eligible for
+// cross-crate-inlining with "always".
+pub fn stem_fn() -> String {
+ inner()
+}
+
+#[inline(never)]
+fn inner() -> String {
+ String::from("test")
+}

tests/codegen/cross-crate-inlining/auxiliary/leaf.rs

@@ -0,0 +1,20 @@
+// compile-flags: -O
+
+#![crate_type = "lib"]
+
+// This function *looks* like it contains a call, but that call will be optimized out by MIR
+// optimizations.
+pub fn leaf_fn() -> String {
+ String::new()
+}
+
+// This function contains a call, even after MIR optimizations. It is only eligible for
+// cross-crate-inlining with "always".
+pub fn stem_fn() -> String {
+ inner()
+}
+
+#[inline(never)]
+fn inner() -> String {
+ String::from("test")
+}

tests/codegen/cross-crate-inlining/auxiliary/never.rs

@@ -0,0 +1,20 @@
+// compile-flags: -O -Zcross-crate-inline-threshold=never
+
+#![crate_type = "lib"]
+
+// This function *looks* like it contains a call, but that call will be optimized out by MIR
+// optimizations.
+pub fn leaf_fn() -> String {
+ String::new()
+}
+
+// This function contains a call, even after MIR optimizations. It is only eligible for
+// cross-crate-inlining with "always".
+pub fn stem_fn() -> String {
+ inner()
+}
+
+#[inline(never)]
+fn inner() -> String {
+ String::from("test")
+}

tests/codegen/cross-crate-inlining/leaf-inlining.rs

@@ -0,0 +1,20 @@
+// compile-flags: -O -Zcross-crate-inline-threshold=yes
+// aux-build:leaf.rs
+
+#![crate_type = "lib"]
+
+extern crate leaf;
+
+// Check that we inline a leaf cross-crate call
+#[no_mangle]
+pub fn leaf_outer() -> String {
+ // CHECK-NOT: call {{.*}}leaf_fn
+ leaf::leaf_fn()
+}
+
+// Check that we do not inline a non-leaf cross-crate call
+#[no_mangle]
+pub fn stem_outer() -> String {
+ // CHECK: call {{.*}}stem_fn
+ leaf::stem_fn()
+}

tests/codegen/cross-crate-inlining/never-inline.rs

@@ -0,0 +1,13 @@
+// compile-flags: -O
+// aux-build:never.rs
+
+#![crate_type = "lib"]
+
+extern crate never;
+
+// Check that we do not inline a cross-crate call, even though it is a leaf
+#[no_mangle]
+pub fn outer() -> String {
+ // CHECK: call {{.*}}leaf_fn
+ never::leaf_fn()
+}