Skip to content

Commit

Permalink
Auto merge of #121309 - Nilstrieb:inline-all-the-fallbacks, r=oli-obk
Browse files Browse the repository at this point in the history
Make intrinsic fallback bodies cross-crate inlineable

This change was prompted by the stage1 compiler spending 4% of its time when compiling the polymorphic-recursion MIR opt test in `unlikely`.

Intrinsic fallback bodies like `unlikely` should always be inlined, it's very silly if they are not. To do this, we enable the fallback bodies to be cross-crate inlineable. Not that this matters for our workloads since the compiler never actually _uses_ the "fallback bodies", it just uses whatever was cfg(bootstrap)ped, so I've also added `#[inline]` to those.

See the comments for more information.

r? oli-obk
  • Loading branch information
bors committed Feb 22, 2024
2 parents 52dba5f + 0f4925e commit 1bb3a9f
Show file tree
Hide file tree
Showing 4 changed files with 25 additions and 0 deletions.
9 changes: 9 additions & 0 deletions compiler/rustc_mir_transform/src/cross_crate_inline.rs
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ use rustc_middle::query::Providers;
use rustc_middle::ty::TyCtxt;
use rustc_session::config::InliningThreshold;
use rustc_session::config::OptLevel;
use rustc_span::sym;

pub fn provide(providers: &mut Providers) {
providers.cross_crate_inlinable = cross_crate_inlinable;
Expand All @@ -34,6 +35,14 @@ fn cross_crate_inlinable(tcx: TyCtxt<'_>, def_id: LocalDefId) -> bool {
return true;
}

if tcx.has_attr(def_id, sym::rustc_intrinsic) {
// Intrinsic fallback bodies are always cross-crate inlineable.
// To ensure that the MIR inliner doesn't cluelessly try to inline fallback
// bodies even when the backend would implement something better, we stop
// the MIR inliner from ever inlining an intrinsic.
return true;
}

// Obey source annotations first; this is important because it means we can use
// #[inline(never)] to force code generation.
match codegen_fn_attrs.inline {
Expand Down
8 changes: 8 additions & 0 deletions compiler/rustc_mir_transform/src/inline.rs
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ use rustc_middle::ty::TypeVisitableExt;
use rustc_middle::ty::{self, Instance, InstanceDef, ParamEnv, Ty, TyCtxt};
use rustc_session::config::OptLevel;
use rustc_span::source_map::Spanned;
use rustc_span::sym;
use rustc_target::abi::FieldIdx;
use rustc_target::spec::abi::Abi;

Expand Down Expand Up @@ -170,6 +171,13 @@ impl<'tcx> Inliner<'tcx> {
let cross_crate_inlinable = self.tcx.cross_crate_inlinable(callsite.callee.def_id());
self.check_codegen_attributes(callsite, callee_attrs, cross_crate_inlinable)?;

// Intrinsic fallback bodies are automatically made cross-crate inlineable,
// but at this stage we don't know whether codegen knows the intrinsic,
// so just conservatively don't inline it.
if self.tcx.has_attr(callsite.callee.def_id(), sym::rustc_intrinsic) {
return Err("Callee is an intrinsic, do not inline fallback bodies");
}

let terminator = caller_body[callsite.block].terminator.as_ref().unwrap();
let TerminatorKind::Call { args, destination, .. } = &terminator.kind else { bug!() };
let destination_ty = destination.ty(&caller_body.local_decls, self.tcx).ty;
Expand Down
6 changes: 6 additions & 0 deletions library/core/src/intrinsics.rs
Original file line number Diff line number Diff line change
Expand Up @@ -953,6 +953,7 @@ extern "rust-intrinsic" {
#[rustc_nounwind]
#[unstable(feature = "core_intrinsics", issue = "none")]
#[cfg_attr(not(bootstrap), rustc_intrinsic)]
#[cfg_attr(bootstrap, inline)]
pub const unsafe fn assume(b: bool) {
if !b {
// SAFETY: the caller must guarantee the argument is never `false`
Expand All @@ -975,6 +976,7 @@ pub const unsafe fn assume(b: bool) {
#[unstable(feature = "core_intrinsics", issue = "none")]
#[cfg_attr(not(bootstrap), rustc_intrinsic)]
#[rustc_nounwind]
#[cfg_attr(bootstrap, inline)]
pub const fn likely(b: bool) -> bool {
b
}
Expand All @@ -994,6 +996,7 @@ pub const fn likely(b: bool) -> bool {
#[unstable(feature = "core_intrinsics", issue = "none")]
#[cfg_attr(not(bootstrap), rustc_intrinsic)]
#[rustc_nounwind]
#[cfg_attr(bootstrap, inline)]
pub const fn unlikely(b: bool) -> bool {
b
}
Expand Down Expand Up @@ -2596,6 +2599,7 @@ extern "rust-intrinsic" {
#[rustc_nounwind]
#[unstable(feature = "core_intrinsics", issue = "none")]
#[cfg_attr(not(bootstrap), rustc_intrinsic)]
#[cfg_attr(bootstrap, inline)]
pub const fn is_val_statically_known<T: Copy>(_arg: T) -> bool {
false
}
Expand Down Expand Up @@ -2633,6 +2637,7 @@ pub(crate) const fn debug_assertions() -> bool {
#[unstable(feature = "core_intrinsics", issue = "none")]
#[rustc_nounwind]
#[cfg_attr(not(bootstrap), rustc_intrinsic)]
#[cfg_attr(bootstrap, inline)]
pub const unsafe fn const_allocate(_size: usize, _align: usize) -> *mut u8 {
// const eval overrides this function, but runtime code should always just return null pointers.
crate::ptr::null_mut()
Expand All @@ -2652,6 +2657,7 @@ pub const unsafe fn const_allocate(_size: usize, _align: usize) -> *mut u8 {
#[unstable(feature = "core_intrinsics", issue = "none")]
#[rustc_nounwind]
#[cfg_attr(not(bootstrap), rustc_intrinsic)]
#[cfg_attr(bootstrap, inline)]
pub const unsafe fn const_deallocate(_ptr: *mut u8, _size: usize, _align: usize) {}

// Some functions are defined here because they accidentally got made
Expand Down
2 changes: 2 additions & 0 deletions src/doc/unstable-book/src/language-features/intrinsics.md
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,8 @@ the intrinsic directly when you can.
Many intrinsics can be written in pure rust, albeit inefficiently or without supporting
some features that only exist on some backends. Backends can simply not implement those
intrinsics without causing any code miscompilations or failures to compile.
All intrinsic fallback bodies are automatically made cross-crate inlineable (like `#[inline]`)
by the codegen backend, but not the MIR inliner.

```rust
#![feature(rustc_attrs, effects)]
Expand Down

0 comments on commit 1bb3a9f

Please sign in to comment.