Skip to content
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.

Commit 550c19c

Browse files
authoredMar 11, 2025
Rollup merge of rust-lang#137701 - cuviper:sharded-hashtable, r=fmease
Convert `ShardedHashMap` to use `hashbrown::HashTable` The `hash_raw_entry` feature (rust-lang#56167) has finished fcp-close, so the compiler should stop using it to allow its removal. Several `Sharded` maps were using raw entries to avoid re-hashing between shard and map lookup, and we can do that with `hashbrown::HashTable` instead.
2 parents 4c3d606 + 3b0c258 commit 550c19c

File tree

10 files changed

+109
-66
lines changed

10 files changed

+109
-66
lines changed
 

‎Cargo.lock

+2
Original file line numberDiff line numberDiff line change
@@ -1491,6 +1491,7 @@ version = "0.15.2"
14911491
source = "registry+https://github.com/rust-lang/crates.io-index"
14921492
checksum = "bf151400ff0baff5465007dd2f3e717f3fe502074ca563069ce3a6629d07b289"
14931493
dependencies = [
1494+
"allocator-api2",
14941495
"foldhash",
14951496
"serde",
14961497
]
@@ -3485,6 +3486,7 @@ dependencies = [
34853486
"either",
34863487
"elsa",
34873488
"ena",
3489+
"hashbrown 0.15.2",
34883490
"indexmap",
34893491
"jobserver",
34903492
"libc",

‎compiler/rustc_data_structures/Cargo.toml

+5
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,11 @@ thin-vec = "0.2.12"
2929
tracing = "0.1"
3030
# tidy-alphabetical-end
3131

32+
[dependencies.hashbrown]
33+
version = "0.15.2"
34+
default-features = false
35+
features = ["nightly"] # for may_dangle
36+
3237
[dependencies.parking_lot]
3338
version = "0.12"
3439

‎compiler/rustc_data_structures/src/lib.rs

-1
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,6 @@
2424
#![feature(dropck_eyepatch)]
2525
#![feature(extend_one)]
2626
#![feature(file_buffered)]
27-
#![feature(hash_raw_entry)]
2827
#![feature(macro_metavar_expr)]
2928
#![feature(map_try_insert)]
3029
#![feature(min_specialization)]

‎compiler/rustc_data_structures/src/marker.rs

+2
Original file line numberDiff line numberDiff line change
@@ -76,6 +76,7 @@ impl_dyn_send!(
7676
[crate::sync::RwLock<T> where T: DynSend]
7777
[crate::tagged_ptr::TaggedRef<'a, P, T> where 'a, P: Sync, T: Send + crate::tagged_ptr::Tag]
7878
[rustc_arena::TypedArena<T> where T: DynSend]
79+
[hashbrown::HashTable<T> where T: DynSend]
7980
[indexmap::IndexSet<V, S> where V: DynSend, S: DynSend]
8081
[indexmap::IndexMap<K, V, S> where K: DynSend, V: DynSend, S: DynSend]
8182
[thin_vec::ThinVec<T> where T: DynSend]
@@ -153,6 +154,7 @@ impl_dyn_sync!(
153154
[crate::tagged_ptr::TaggedRef<'a, P, T> where 'a, P: Sync, T: Sync + crate::tagged_ptr::Tag]
154155
[parking_lot::lock_api::Mutex<R, T> where R: DynSync, T: ?Sized + DynSend]
155156
[parking_lot::lock_api::RwLock<R, T> where R: DynSync, T: ?Sized + DynSend + DynSync]
157+
[hashbrown::HashTable<T> where T: DynSync]
156158
[indexmap::IndexSet<V, S> where V: DynSync, S: DynSync]
157159
[indexmap::IndexMap<K, V, S> where K: DynSync, V: DynSync, S: DynSync]
158160
[smallvec::SmallVec<A> where A: smallvec::Array + DynSync]

‎compiler/rustc_data_structures/src/sharded.rs

+78-17
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,11 @@
11
use std::borrow::Borrow;
2-
use std::collections::hash_map::RawEntryMut;
32
use std::hash::{Hash, Hasher};
4-
use std::iter;
3+
use std::{iter, mem};
54

65
use either::Either;
6+
use hashbrown::hash_table::{Entry, HashTable};
77

8-
use crate::fx::{FxHashMap, FxHasher};
8+
use crate::fx::FxHasher;
99
use crate::sync::{CacheAligned, Lock, LockGuard, Mode, is_dyn_thread_safe};
1010

1111
// 32 shards is sufficient to reduce contention on an 8-core Ryzen 7 1700,
@@ -140,17 +140,67 @@ pub fn shards() -> usize {
140140
1
141141
}
142142

143-
pub type ShardedHashMap<K, V> = Sharded<FxHashMap<K, V>>;
143+
pub type ShardedHashMap<K, V> = Sharded<HashTable<(K, V)>>;
144144

145145
impl<K: Eq, V> ShardedHashMap<K, V> {
146146
pub fn with_capacity(cap: usize) -> Self {
147-
Self::new(|| FxHashMap::with_capacity_and_hasher(cap, rustc_hash::FxBuildHasher::default()))
147+
Self::new(|| HashTable::with_capacity(cap))
148148
}
149149
pub fn len(&self) -> usize {
150150
self.lock_shards().map(|shard| shard.len()).sum()
151151
}
152152
}
153153

154+
impl<K: Eq + Hash, V> ShardedHashMap<K, V> {
155+
#[inline]
156+
pub fn get<Q>(&self, key: &Q) -> Option<V>
157+
where
158+
K: Borrow<Q>,
159+
Q: Hash + Eq,
160+
V: Clone,
161+
{
162+
let hash = make_hash(key);
163+
let shard = self.lock_shard_by_hash(hash);
164+
let (_, value) = shard.find(hash, |(k, _)| k.borrow() == key)?;
165+
Some(value.clone())
166+
}
167+
168+
#[inline]
169+
pub fn get_or_insert_with(&self, key: K, default: impl FnOnce() -> V) -> V
170+
where
171+
V: Copy,
172+
{
173+
let hash = make_hash(&key);
174+
let mut shard = self.lock_shard_by_hash(hash);
175+
176+
match table_entry(&mut shard, hash, &key) {
177+
Entry::Occupied(e) => e.get().1,
178+
Entry::Vacant(e) => {
179+
let value = default();
180+
e.insert((key, value));
181+
value
182+
}
183+
}
184+
}
185+
186+
#[inline]
187+
pub fn insert(&self, key: K, value: V) -> Option<V> {
188+
let hash = make_hash(&key);
189+
let mut shard = self.lock_shard_by_hash(hash);
190+
191+
match table_entry(&mut shard, hash, &key) {
192+
Entry::Occupied(e) => {
193+
let previous = mem::replace(&mut e.into_mut().1, value);
194+
Some(previous)
195+
}
196+
Entry::Vacant(e) => {
197+
e.insert((key, value));
198+
None
199+
}
200+
}
201+
}
202+
}
203+
154204
impl<K: Eq + Hash + Copy> ShardedHashMap<K, ()> {
155205
#[inline]
156206
pub fn intern_ref<Q: ?Sized>(&self, value: &Q, make: impl FnOnce() -> K) -> K
@@ -160,13 +210,12 @@ impl<K: Eq + Hash + Copy> ShardedHashMap<K, ()> {
160210
{
161211
let hash = make_hash(value);
162212
let mut shard = self.lock_shard_by_hash(hash);
163-
let entry = shard.raw_entry_mut().from_key_hashed_nocheck(hash, value);
164213

165-
match entry {
166-
RawEntryMut::Occupied(e) => *e.key(),
167-
RawEntryMut::Vacant(e) => {
214+
match table_entry(&mut shard, hash, value) {
215+
Entry::Occupied(e) => e.get().0,
216+
Entry::Vacant(e) => {
168217
let v = make();
169-
e.insert_hashed_nocheck(hash, v, ());
218+
e.insert((v, ()));
170219
v
171220
}
172221
}
@@ -180,13 +229,12 @@ impl<K: Eq + Hash + Copy> ShardedHashMap<K, ()> {
180229
{
181230
let hash = make_hash(&value);
182231
let mut shard = self.lock_shard_by_hash(hash);
183-
let entry = shard.raw_entry_mut().from_key_hashed_nocheck(hash, &value);
184232

185-
match entry {
186-
RawEntryMut::Occupied(e) => *e.key(),
187-
RawEntryMut::Vacant(e) => {
233+
match table_entry(&mut shard, hash, &value) {
234+
Entry::Occupied(e) => e.get().0,
235+
Entry::Vacant(e) => {
188236
let v = make(value);
189-
e.insert_hashed_nocheck(hash, v, ());
237+
e.insert((v, ()));
190238
v
191239
}
192240
}
@@ -203,17 +251,30 @@ impl<K: Eq + Hash + Copy + IntoPointer> ShardedHashMap<K, ()> {
203251
let hash = make_hash(&value);
204252
let shard = self.lock_shard_by_hash(hash);
205253
let value = value.into_pointer();
206-
shard.raw_entry().from_hash(hash, |entry| entry.into_pointer() == value).is_some()
254+
shard.find(hash, |(k, ())| k.into_pointer() == value).is_some()
207255
}
208256
}
209257

210258
#[inline]
211-
pub fn make_hash<K: Hash + ?Sized>(val: &K) -> u64 {
259+
fn make_hash<K: Hash + ?Sized>(val: &K) -> u64 {
212260
let mut state = FxHasher::default();
213261
val.hash(&mut state);
214262
state.finish()
215263
}
216264

265+
#[inline]
266+
fn table_entry<'a, K, V, Q>(
267+
table: &'a mut HashTable<(K, V)>,
268+
hash: u64,
269+
key: &Q,
270+
) -> Entry<'a, (K, V)>
271+
where
272+
K: Hash + Borrow<Q>,
273+
Q: ?Sized + Eq,
274+
{
275+
table.entry(hash, move |(k, _)| k.borrow() == key, |(k, _)| make_hash(k))
276+
}
277+
217278
/// Get a shard with a pre-computed hash value. If `get_shard_by_value` is
218279
/// ever used in combination with `get_shard_by_hash` on a single `Sharded`
219280
/// instance, then `hash` must be computed with `FxHasher`. Otherwise,

‎compiler/rustc_middle/src/mir/interpret/mod.rs

+5-15
Original file line numberDiff line numberDiff line change
@@ -452,12 +452,7 @@ impl<'tcx> TyCtxt<'tcx> {
452452
}
453453
let id = self.alloc_map.reserve();
454454
debug!("creating alloc {:?} with id {id:?}", alloc_salt.0);
455-
let had_previous = self
456-
.alloc_map
457-
.to_alloc
458-
.lock_shard_by_value(&id)
459-
.insert(id, alloc_salt.0.clone())
460-
.is_some();
455+
let had_previous = self.alloc_map.to_alloc.insert(id, alloc_salt.0.clone()).is_some();
461456
// We just reserved, so should always be unique.
462457
assert!(!had_previous);
463458
dedup.insert(alloc_salt, id);
@@ -510,7 +505,7 @@ impl<'tcx> TyCtxt<'tcx> {
510505
/// local dangling pointers and allocations in constants/statics.
511506
#[inline]
512507
pub fn try_get_global_alloc(self, id: AllocId) -> Option<GlobalAlloc<'tcx>> {
513-
self.alloc_map.to_alloc.lock_shard_by_value(&id).get(&id).cloned()
508+
self.alloc_map.to_alloc.get(&id)
514509
}
515510

516511
#[inline]
@@ -529,21 +524,16 @@ impl<'tcx> TyCtxt<'tcx> {
529524
/// Freezes an `AllocId` created with `reserve` by pointing it at an `Allocation`. Trying to
530525
/// call this function twice, even with the same `Allocation` will ICE the compiler.
531526
pub fn set_alloc_id_memory(self, id: AllocId, mem: ConstAllocation<'tcx>) {
532-
if let Some(old) =
533-
self.alloc_map.to_alloc.lock_shard_by_value(&id).insert(id, GlobalAlloc::Memory(mem))
534-
{
527+
if let Some(old) = self.alloc_map.to_alloc.insert(id, GlobalAlloc::Memory(mem)) {
535528
bug!("tried to set allocation ID {id:?}, but it was already existing as {old:#?}");
536529
}
537530
}
538531

539532
/// Freezes an `AllocId` created with `reserve` by pointing it at a static item. Trying to
540533
/// call this function twice, even with the same `DefId` will ICE the compiler.
541534
pub fn set_nested_alloc_id_static(self, id: AllocId, def_id: LocalDefId) {
542-
if let Some(old) = self
543-
.alloc_map
544-
.to_alloc
545-
.lock_shard_by_value(&id)
546-
.insert(id, GlobalAlloc::Static(def_id.to_def_id()))
535+
if let Some(old) =
536+
self.alloc_map.to_alloc.insert(id, GlobalAlloc::Static(def_id.to_def_id()))
547537
{
548538
bug!("tried to set allocation ID {id:?}, but it was already existing as {old:#?}");
549539
}

‎compiler/rustc_middle/src/ty/context.rs

+2-2
Original file line numberDiff line numberDiff line change
@@ -2336,8 +2336,8 @@ macro_rules! sty_debug_print {
23362336
$(let mut $variant = total;)*
23372337

23382338
for shard in tcx.interners.type_.lock_shards() {
2339-
let types = shard.keys();
2340-
for &InternedInSet(t) in types {
2339+
let types = shard.iter();
2340+
for &(InternedInSet(t), ()) in types {
23412341
let variant = match t.internee {
23422342
ty::Bool | ty::Char | ty::Int(..) | ty::Uint(..) |
23432343
ty::Float(..) | ty::Str | ty::Never => continue,

‎compiler/rustc_query_system/src/dep_graph/graph.rs

+11-20
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,4 @@
11
use std::assert_matches::assert_matches;
2-
use std::collections::hash_map::Entry;
32
use std::fmt::Debug;
43
use std::hash::Hash;
54
use std::marker::PhantomData;
@@ -9,7 +8,7 @@ use std::sync::atomic::{AtomicU32, Ordering};
98
use rustc_data_structures::fingerprint::Fingerprint;
109
use rustc_data_structures::fx::{FxHashMap, FxHashSet};
1110
use rustc_data_structures::profiling::{QueryInvocationId, SelfProfilerRef};
12-
use rustc_data_structures::sharded::{self, Sharded};
11+
use rustc_data_structures::sharded::{self, ShardedHashMap};
1312
use rustc_data_structures::stable_hasher::{HashStable, StableHasher};
1413
use rustc_data_structures::sync::{AtomicU64, Lock};
1514
use rustc_data_structures::unord::UnordMap;
@@ -619,7 +618,7 @@ impl<D: Deps> DepGraphData<D> {
619618
if let Some(prev_index) = self.previous.node_to_index_opt(dep_node) {
620619
self.current.prev_index_to_index.lock()[prev_index]
621620
} else {
622-
self.current.new_node_to_index.lock_shard_by_value(dep_node).get(dep_node).copied()
621+
self.current.new_node_to_index.get(dep_node)
623622
}
624623
}
625624

@@ -1048,7 +1047,7 @@ rustc_index::newtype_index! {
10481047
/// first, and `data` second.
10491048
pub(super) struct CurrentDepGraph<D: Deps> {
10501049
encoder: GraphEncoder<D>,
1051-
new_node_to_index: Sharded<FxHashMap<DepNode, DepNodeIndex>>,
1050+
new_node_to_index: ShardedHashMap<DepNode, DepNodeIndex>,
10521051
prev_index_to_index: Lock<IndexVec<SerializedDepNodeIndex, Option<DepNodeIndex>>>,
10531052

10541053
/// This is used to verify that fingerprints do not change between the creation of a node
@@ -1117,12 +1116,9 @@ impl<D: Deps> CurrentDepGraph<D> {
11171116
profiler,
11181117
previous,
11191118
),
1120-
new_node_to_index: Sharded::new(|| {
1121-
FxHashMap::with_capacity_and_hasher(
1122-
new_node_count_estimate / sharded::shards(),
1123-
Default::default(),
1124-
)
1125-
}),
1119+
new_node_to_index: ShardedHashMap::with_capacity(
1120+
new_node_count_estimate / sharded::shards(),
1121+
),
11261122
prev_index_to_index: Lock::new(IndexVec::from_elem_n(None, prev_graph_node_count)),
11271123
anon_id_seed,
11281124
#[cfg(debug_assertions)]
@@ -1152,14 +1148,9 @@ impl<D: Deps> CurrentDepGraph<D> {
11521148
edges: EdgesVec,
11531149
current_fingerprint: Fingerprint,
11541150
) -> DepNodeIndex {
1155-
let dep_node_index = match self.new_node_to_index.lock_shard_by_value(&key).entry(key) {
1156-
Entry::Occupied(entry) => *entry.get(),
1157-
Entry::Vacant(entry) => {
1158-
let dep_node_index = self.encoder.send(key, current_fingerprint, edges);
1159-
entry.insert(dep_node_index);
1160-
dep_node_index
1161-
}
1162-
};
1151+
let dep_node_index = self
1152+
.new_node_to_index
1153+
.get_or_insert_with(key, || self.encoder.send(key, current_fingerprint, edges));
11631154

11641155
#[cfg(debug_assertions)]
11651156
self.record_edge(dep_node_index, key, current_fingerprint);
@@ -1257,7 +1248,7 @@ impl<D: Deps> CurrentDepGraph<D> {
12571248
) {
12581249
let node = &prev_graph.index_to_node(prev_index);
12591250
debug_assert!(
1260-
!self.new_node_to_index.lock_shard_by_value(node).contains_key(node),
1251+
!self.new_node_to_index.get(node).is_some(),
12611252
"node from previous graph present in new node collection"
12621253
);
12631254
}
@@ -1382,7 +1373,7 @@ fn panic_on_forbidden_read<D: Deps>(data: &DepGraphData<D>, dep_node_index: DepN
13821373
if dep_node.is_none() {
13831374
// Try to find it among the new nodes
13841375
for shard in data.current.new_node_to_index.lock_shards() {
1385-
if let Some((node, _)) = shard.iter().find(|(_, index)| **index == dep_node_index) {
1376+
if let Some((node, _)) = shard.iter().find(|(_, index)| *index == dep_node_index) {
13861377
dep_node = Some(*node);
13871378
break;
13881379
}

‎compiler/rustc_query_system/src/lib.rs

-1
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,6 @@
33
#![feature(assert_matches)]
44
#![feature(core_intrinsics)]
55
#![feature(dropck_eyepatch)]
6-
#![feature(hash_raw_entry)]
76
#![feature(let_chains)]
87
#![feature(min_specialization)]
98
#![warn(unreachable_pub)]

‎compiler/rustc_query_system/src/query/caches.rs

+4-10
Original file line numberDiff line numberDiff line change
@@ -2,8 +2,7 @@ use std::fmt::Debug;
22
use std::hash::Hash;
33
use std::sync::OnceLock;
44

5-
use rustc_data_structures::fx::FxHashMap;
6-
use rustc_data_structures::sharded::{self, Sharded};
5+
use rustc_data_structures::sharded::ShardedHashMap;
76
pub use rustc_data_structures::vec_cache::VecCache;
87
use rustc_hir::def_id::LOCAL_CRATE;
98
use rustc_index::Idx;
@@ -36,7 +35,7 @@ pub trait QueryCache: Sized {
3635
/// In-memory cache for queries whose keys aren't suitable for any of the
3736
/// more specialized kinds of cache. Backed by a sharded hashmap.
3837
pub struct DefaultCache<K, V> {
39-
cache: Sharded<FxHashMap<K, (V, DepNodeIndex)>>,
38+
cache: ShardedHashMap<K, (V, DepNodeIndex)>,
4039
}
4140

4241
impl<K, V> Default for DefaultCache<K, V> {
@@ -55,19 +54,14 @@ where
5554

5655
#[inline(always)]
5756
fn lookup(&self, key: &K) -> Option<(V, DepNodeIndex)> {
58-
let key_hash = sharded::make_hash(key);
59-
let lock = self.cache.lock_shard_by_hash(key_hash);
60-
let result = lock.raw_entry().from_key_hashed_nocheck(key_hash, key);
61-
62-
if let Some((_, value)) = result { Some(*value) } else { None }
57+
self.cache.get(key)
6358
}
6459

6560
#[inline]
6661
fn complete(&self, key: K, value: V, index: DepNodeIndex) {
67-
let mut lock = self.cache.lock_shard_by_value(&key);
6862
// We may be overwriting another value. This is all right, since the dep-graph
6963
// will check that the fingerprint matches.
70-
lock.insert(key, (value, index));
64+
self.cache.insert(key, (value, index));
7165
}
7266

7367
fn iter(&self, f: &mut dyn FnMut(&Self::Key, &Self::Value, DepNodeIndex)) {

0 commit comments

Comments
 (0)
Please sign in to comment.