diff --git a/compiler/rustc_arena/src/lib.rs b/compiler/rustc_arena/src/lib.rs index 6217bf46a942d..97dd21db07e7d 100644 --- a/compiler/rustc_arena/src/lib.rs +++ b/compiler/rustc_arena/src/lib.rs @@ -13,7 +13,6 @@ #![cfg_attr(test, feature(test))] #![deny(unsafe_op_in_unsafe_fn)] #![doc(test(no_crate_inject, attr(deny(warnings), allow(internal_features))))] -#![feature(core_intrinsics)] #![feature(decl_macro)] #![feature(dropck_eyepatch)] #![feature(never_type)] @@ -26,7 +25,7 @@ use std::cell::{Cell, RefCell}; use std::marker::PhantomData; use std::mem::{self, MaybeUninit}; use std::ptr::{self, NonNull}; -use std::{cmp, intrinsics, slice}; +use std::{cmp, hint, slice}; use smallvec::SmallVec; @@ -452,7 +451,7 @@ impl DroplessArena { let bytes = align_up(layout.size(), DROPLESS_ALIGNMENT); // Tell LLVM that `end` is aligned to DROPLESS_ALIGNMENT. - unsafe { intrinsics::assume(end == align_down(end, DROPLESS_ALIGNMENT)) }; + unsafe { hint::assert_unchecked(end == align_down(end, DROPLESS_ALIGNMENT)) }; if let Some(sub) = end.checked_sub(bytes) { let new_end = align_down(sub, layout.align()); diff --git a/compiler/rustc_data_structures/src/lib.rs b/compiler/rustc_data_structures/src/lib.rs index 4467a28111815..bef19d77d1e73 100644 --- a/compiler/rustc_data_structures/src/lib.rs +++ b/compiler/rustc_data_structures/src/lib.rs @@ -10,6 +10,7 @@ #![allow(internal_features)] #![allow(rustc::default_hash_types)] #![allow(rustc::potential_query_instability)] +#![cfg_attr(bootstrap, feature(cold_path))] #![deny(unsafe_op_in_unsafe_fn)] #![feature(allocator_api)] #![feature(ascii_char)] @@ -19,7 +20,6 @@ #![feature(cfg_select)] #![feature(const_default)] #![feature(const_trait_impl)] -#![feature(core_intrinsics)] #![feature(dropck_eyepatch)] #![feature(extend_one)] #![feature(file_buffered)] diff --git a/compiler/rustc_data_structures/src/profiling.rs b/compiler/rustc_data_structures/src/profiling.rs index b04caa69adfbc..cab57edb89461 100644 --- a/compiler/rustc_data_structures/src/profiling.rs +++ b/compiler/rustc_data_structures/src/profiling.rs @@ -85,12 +85,11 @@ use std::borrow::Borrow; use std::collections::hash_map::Entry; use std::error::Error; use std::fmt::Display; -use std::intrinsics::unlikely; use std::path::Path; use std::sync::Arc; use std::sync::atomic::Ordering; use std::time::{Duration, Instant}; -use std::{fs, process}; +use std::{fs, hint, process}; pub use measureme::EventId; use measureme::{EventIdBuilder, Profiler, SerializableString, StringId}; @@ -427,7 +426,8 @@ impl SelfProfilerRef { .unwrap() .increment_query_cache_hit_counters(QueryInvocationId(query_invocation_id.0)); } - if unlikely(profiler_ref.event_filter_mask.contains(EventFilter::QUERY_CACHE_HITS)) { + if profiler_ref.event_filter_mask.contains(EventFilter::QUERY_CACHE_HITS) { + hint::cold_path(); profiler_ref.instant_query_event( |profiler| profiler.query_cache_hit_event_kind, query_invocation_id, @@ -437,7 +437,8 @@ impl SelfProfilerRef { // We check both kinds of query cache hit events at once, to reduce overhead in the // common case (with self-profile disabled). - if unlikely(self.event_filter_mask.intersects(EventFilter::QUERY_CACHE_HIT_COMBINED)) { + if self.event_filter_mask.intersects(EventFilter::QUERY_CACHE_HIT_COMBINED) { + hint::cold_path(); cold_call(self, query_invocation_id); } } diff --git a/compiler/rustc_data_structures/src/sync/freeze.rs b/compiler/rustc_data_structures/src/sync/freeze.rs index 6338afb92c34b..fb2dcb25556a5 100644 --- a/compiler/rustc_data_structures/src/sync/freeze.rs +++ b/compiler/rustc_data_structures/src/sync/freeze.rs @@ -1,5 +1,5 @@ use std::cell::UnsafeCell; -use std::intrinsics::likely; +use std::hint; use std::marker::PhantomData; use std::ops::{Deref, DerefMut}; use std::ptr::NonNull; @@ -60,10 +60,11 @@ impl FreezeLock { /// Get the inner value if frozen. #[inline] pub fn get(&self) -> Option<&T> { - if likely(self.frozen.load(Ordering::Acquire)) { + if self.frozen.load(Ordering::Acquire) { // SAFETY: This is frozen so the data cannot be modified. unsafe { Some(&*self.data.get()) } } else { + hint::cold_path(); None } } diff --git a/compiler/rustc_data_structures/src/sync/lock.rs b/compiler/rustc_data_structures/src/sync/lock.rs index a8161c5151156..f183af0c0dabd 100644 --- a/compiler/rustc_data_structures/src/sync/lock.rs +++ b/compiler/rustc_data_structures/src/sync/lock.rs @@ -1,7 +1,7 @@ //! This module implements a lock which only uses synchronization if `might_be_dyn_thread_safe` is true. //! It implements `DynSend` and `DynSync` instead of the typical `Send` and `Sync` traits. -use std::fmt; +use std::{fmt, hint}; #[derive(Clone, Copy, PartialEq)] pub enum Mode { @@ -10,7 +10,6 @@ pub enum Mode { } use std::cell::{Cell, UnsafeCell}; -use std::intrinsics::unlikely; use std::marker::PhantomData; use std::mem::ManuallyDrop; use std::ops::{Deref, DerefMut}; @@ -92,7 +91,8 @@ pub struct Lock { impl Lock { #[inline(always)] pub fn new(inner: T) -> Self { - let (mode, mode_union) = if unlikely(mode::might_be_dyn_thread_safe()) { + let (mode, mode_union) = if mode::might_be_dyn_thread_safe() { + hint::cold_path(); // Create the lock with synchronization enabled using the `RawMutex` type. (Mode::Sync, ModeUnion { sync: ManuallyDrop::new(RawMutex::INIT) }) } else { @@ -150,7 +150,8 @@ impl Lock { unsafe { match mode { Mode::NoSync => { - if unlikely(self.mode_union.no_sync.replace(LOCKED) == LOCKED) { + if self.mode_union.no_sync.replace(LOCKED) == LOCKED { + hint::cold_path(); lock_held() } }