diff --git a/Cargo.lock b/Cargo.lock index a804638f7026..93075cd03633 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4016,6 +4016,7 @@ dependencies = [ "either", "gsgdt", "polonius-engine", + "rustc-hash 2.1.1", "rustc-rayon-core", "rustc_abi", "rustc_apfloat", diff --git a/compiler/rustc_data_structures/src/lib.rs b/compiler/rustc_data_structures/src/lib.rs index a3b62b469196..5e0027af403d 100644 --- a/compiler/rustc_data_structures/src/lib.rs +++ b/compiler/rustc_data_structures/src/lib.rs @@ -13,6 +13,7 @@ #![deny(unsafe_op_in_unsafe_fn)] #![doc(html_root_url = "https://doc.rust-lang.org/nightly/nightly-rustc/")] #![doc(rust_logo)] +#![feature(alloc_layout_extra)] #![feature(allocator_api)] #![feature(array_windows)] #![feature(ascii_char)] diff --git a/compiler/rustc_data_structures/src/sync.rs b/compiler/rustc_data_structures/src/sync.rs index a1cc75c49850..16b9a3ed036f 100644 --- a/compiler/rustc_data_structures/src/sync.rs +++ b/compiler/rustc_data_structures/src/sync.rs @@ -45,6 +45,7 @@ pub use self::mode::{is_dyn_thread_safe, set_dyn_thread_safe_mode}; pub use self::parallel::{ join, par_for_each_in, par_map, parallel_guard, scope, try_par_for_each_in, }; +pub use self::table::*; pub use self::vec::{AppendOnlyIndexVec, AppendOnlyVec}; pub use self::worker_local::{Registry, WorkerLocal}; pub use crate::marker::*; @@ -52,6 +53,7 @@ pub use crate::marker::*; mod freeze; mod lock; mod parallel; +mod table; mod vec; mod worker_local; diff --git a/compiler/rustc_data_structures/src/sync/table/LICENSE-APACHE b/compiler/rustc_data_structures/src/sync/table/LICENSE-APACHE new file mode 100644 index 000000000000..16fe87b06e80 --- /dev/null +++ b/compiler/rustc_data_structures/src/sync/table/LICENSE-APACHE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + +2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + +3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + +4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + +5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + +6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + +8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + +END OF TERMS AND CONDITIONS + +APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + +Copyright [yyyy] [name of copyright owner] + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. diff --git a/compiler/rustc_data_structures/src/sync/table/LICENSE-MIT b/compiler/rustc_data_structures/src/sync/table/LICENSE-MIT new file mode 100644 index 000000000000..62f72704eba3 --- /dev/null +++ b/compiler/rustc_data_structures/src/sync/table/LICENSE-MIT @@ -0,0 +1,26 @@ +Copyright (c) 2016 Amanieu d'Antras +Copyright (c) 2021 Zoxc + +Permission is hereby granted, free of charge, to any +person obtaining a copy of this software and associated +documentation files (the "Software"), to deal in the +Software without restriction, including without +limitation the rights to use, copy, modify, merge, +publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software +is furnished to do so, subject to the following +conditions: + +The above copyright notice and this permission notice +shall be included in all copies or substantial portions +of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF +ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED +TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A +PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR +IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. diff --git a/compiler/rustc_data_structures/src/sync/table/collect.rs b/compiler/rustc_data_structures/src/sync/table/collect.rs new file mode 100644 index 000000000000..4b635e140101 --- /dev/null +++ b/compiler/rustc_data_structures/src/sync/table/collect.rs @@ -0,0 +1,310 @@ +//! An API for quiescent state based reclamation. + +use std::arch::asm; +use std::cell::Cell; +use std::collections::HashMap; +use std::intrinsics::unlikely; +use std::marker::PhantomData; +use std::mem; +use std::sync::LazyLock; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::thread::{self, ThreadId}; + +use parking_lot::Mutex; + +use super::scopeguard::guard; +use super::util::cold_path; + +mod code; + +// TODO: Use a reference count of pins and a PinRef ZST with a destructor that drops the reference +// so a closure with the `pin` function isn't required. + +static EVENTS: AtomicUsize = AtomicUsize::new(0); + +/// Represents a proof that no deferred methods will run for the lifetime `'a`. +/// It can be used to access data structures in a lock-free manner. +#[derive(Clone, Copy)] +pub struct Pin<'a> { + _private: PhantomData<&'a ()>, +} + +// FIXME: Prevent pin calls inside the callback? +/// This schedules a closure to run at some point after all threads are outside their current pinned +/// regions. +/// +/// The closure will be called by the [collect] method. +/// +/// # Safety +/// This method is unsafe since the closure is not required to be `'static`. +/// It's up to the caller to ensure the closure does not access freed memory. +/// A `move` closure is recommended to avoid accidental references to stack variables. +pub unsafe fn defer_unchecked(f: F) +where + F: FnOnce(), + F: Send, +{ + unsafe { + let f: Box = Box::new(f); + let f: Box = mem::transmute(f); + + COLLECTOR.lock().defer(f); + + EVENTS.fetch_add(1, Ordering::Release); + } +} + +thread_local! { + static DATA: Data = const { + Data { + pinned: Cell::new(false), + registered: Cell::new(false), + seen_events: Cell::new(0), + } + }; +} + +struct Data { + pinned: Cell, + registered: Cell, + seen_events: Cell, +} + +#[inline(never)] +#[cold] +fn panic_pinned() { + panic!("The current thread was pinned"); +} + +impl Data { + #[inline] + fn assert_unpinned(&self) { + if self.pinned.get() { + panic_pinned() + } + } + + #[inline(never)] + #[cold] + fn register(&self) { + COLLECTOR.lock().register(); + self.registered.set(true); + } +} + +cfg_if! { + if #[cfg(all( + any(target_arch = "x86", target_arch = "x86_64"), + not(miri) + ))] { + #[inline] + fn hide(mut data: *const Data) -> *const Data { + // Hide the `data` value from LLVM to prevent it from generating multiple TLS accesses + unsafe { + asm!("/* {} */", inout(reg) data, options(pure, nomem, nostack, preserves_flags)); + } + + data + } + } else { + #[inline] + fn hide(data: *const Data) -> *const Data { + data + } + } +} + +fn data() -> *const Data { + DATA.with(|data| data as *const Data) +} + +fn data_init() -> *const Data { + let data = hide(DATA.with(|data| data as *const Data)); + + { + let data = unsafe { &*data }; + if unlikely(!data.registered.get()) { + data.register(); + } + } + + data +} + +// TODO: Can this be made into an OwnedPin type which restores the state when dropped? +// Would need to check that only one of them is around? +// Can we keep `pin` in that case? Could drop `OwnedPin` inside `pin`? +// Using seperate flags for both of them might work. +// Won't be optimized away like `pin` +// Use a reference count? + +/// Marks the current thread as pinned and returns a proof of that to the closure. +/// +/// This adds the current thread to the set of threads that needs to regularly call [collect] +/// before memory can be freed. [release] can be called if a thread no longer needs +/// access to lock-free data structures for an extended period of time. +#[inline] +pub fn pin(f: impl FnOnce(Pin<'_>) -> R) -> R { + let data = unsafe { &*(hide(data_init())) }; + let old_pinned = data.pinned.get(); + data.pinned.set(true); + guard(old_pinned, |pin| data.pinned.set(*pin)); + f(Pin { _private: PhantomData }) +} + +/// Removes the current thread from the threads allowed to access lock-free data structures. +/// +/// This allows memory to be freed without waiting for [collect] calls from the current thread. +/// [pin] can be called to after to continue accessing lock-free data structures. +/// +/// This will not free any garbage so [collect] should be called first before the last thread +/// terminates to avoid memory leaks. +/// +/// This will panic if called while the current thread is pinned. +pub fn release() { + let data = unsafe { &*(hide(data())) }; + if cfg!(debug_assertions) { + data.assert_unpinned(); + } + if data.registered.get() { + data.assert_unpinned(); + data.registered.set(false); + COLLECTOR.lock().unregister(); + } +} + +/// Signals a quiescent state where garbage may be collected. +/// +/// This may collect garbage using the callbacks registered in [Pin::defer_unchecked](struct.Pin.html#method.defer_unchecked). +/// +/// This will panic if called while a thread is pinned. +pub fn collect() { + let data = unsafe { &*(hide(data())) }; + if cfg!(debug_assertions) { + data.assert_unpinned(); + } + let new = EVENTS.load(Ordering::Acquire); + if unlikely(new != data.seen_events.get()) { + data.seen_events.set(new); + cold_path(|| { + data.assert_unpinned(); + + let callbacks = { + let mut collector = COLLECTOR.lock(); + + // Check if we could block any deferred methods + if data.registered.get() { + collector.quiet() + } else { + collector.collect_unregistered() + } + }; + + callbacks.into_iter().for_each(|callback| callback()); + }); + } +} + +static COLLECTOR: LazyLock> = LazyLock::new(|| Mutex::new(Collector::new())); + +type Callbacks = Vec>; + +struct Collector { + pending: bool, + busy_count: usize, + threads: HashMap, + current_deferred: Callbacks, + previous_deferred: Callbacks, +} + +impl Collector { + fn new() -> Self { + Self { + pending: false, + busy_count: 0, + threads: HashMap::new(), + current_deferred: Vec::new(), + previous_deferred: Vec::new(), + } + } + + fn register(&mut self) { + debug_assert!(!self.threads.contains_key(&thread::current().id())); + + self.busy_count += 1; + self.threads.insert(thread::current().id(), false); + } + + fn unregister(&mut self) { + debug_assert!(self.threads.contains_key(&thread::current().id())); + + let thread = &thread::current().id(); + if *self.threads.get(&thread).unwrap() { + self.busy_count -= 1; + + if self.busy_count == 0 + && (!self.previous_deferred.is_empty() || !self.current_deferred.is_empty()) + { + // Don't collect garbage here, but let the other threads know that there's + // garbage to be collected. + self.pending = true; + EVENTS.fetch_add(1, Ordering::Release); + } + } + + self.threads.remove(&thread); + } + + fn collect_unregistered(&mut self) -> Callbacks { + debug_assert!(!self.threads.contains_key(&thread::current().id())); + + if self.threads.is_empty() { + let mut callbacks = mem::take(&mut self.previous_deferred); + callbacks.extend(mem::take(&mut self.current_deferred)); + callbacks + } else { + Vec::new() + } + } + + fn quiet(&mut self) -> Callbacks { + let quiet = self.threads.get_mut(&thread::current().id()).unwrap(); + if !*quiet || self.pending { + if !*quiet { + self.busy_count -= 1; + *quiet = true; + } + + if self.busy_count == 0 { + // All threads are quiet + self.pending = false; + + self.busy_count = self.threads.len(); + self.threads.values_mut().for_each(|value| { + *value = false; + }); + + let mut callbacks = mem::take(&mut self.previous_deferred); + self.previous_deferred = mem::take(&mut self.current_deferred); + + if !self.previous_deferred.is_empty() { + // Mark ourselves as quiet again + callbacks.extend(self.quiet()); + + // Signal other threads to check in + EVENTS.fetch_add(1, Ordering::Release); + } + + callbacks + } else { + Vec::new() + } + } else { + Vec::new() + } + } + + fn defer(&mut self, callback: Box) { + self.current_deferred.push(callback); + } +} diff --git a/compiler/rustc_data_structures/src/sync/table/collect/code.rs b/compiler/rustc_data_structures/src/sync/table/collect/code.rs new file mode 100644 index 000000000000..5d92ae3e1111 --- /dev/null +++ b/compiler/rustc_data_structures/src/sync/table/collect/code.rs @@ -0,0 +1,26 @@ +#![cfg(code)] + +use crate::collect; + +#[inline(never)] +#[no_mangle] +unsafe fn dummy() { + if *(5345 as *const bool) { + panic!("whoops") + } +} + +#[no_mangle] +unsafe fn pin_test() { + collect::pin(|_| dummy()); +} + +#[no_mangle] +unsafe fn collect_test() { + collect::collect(); +} + +#[no_mangle] +unsafe fn release_test() { + collect::release(); +} diff --git a/compiler/rustc_data_structures/src/sync/table/macros.rs b/compiler/rustc_data_structures/src/sync/table/macros.rs new file mode 100644 index 000000000000..e743094768fc --- /dev/null +++ b/compiler/rustc_data_structures/src/sync/table/macros.rs @@ -0,0 +1,54 @@ +// See the cfg-if crate. +macro_rules! cfg_if { + // match if/else chains with a final `else` + ($( + if #[cfg($($meta:meta),*)] { $($it:item)* } + ) else * else { + $($it2:item)* + }) => { + cfg_if! { + @__items + () ; + $( ( ($($meta),*) ($($it)*) ), )* + ( () ($($it2)*) ), + } + }; + + // match if/else chains lacking a final `else` + ( + if #[cfg($($i_met:meta),*)] { $($i_it:item)* } + $( + else if #[cfg($($e_met:meta),*)] { $($e_it:item)* } + )* + ) => { + cfg_if! { + @__items + () ; + ( ($($i_met),*) ($($i_it)*) ), + $( ( ($($e_met),*) ($($e_it)*) ), )* + ( () () ), + } + }; + + // Internal and recursive macro to emit all the items + // + // Collects all the negated cfgs in a list at the beginning and after the + // semicolon is all the remaining items + (@__items ($($not:meta,)*) ; ) => {}; + (@__items ($($not:meta,)*) ; ( ($($m:meta),*) ($($it:item)*) ), $($rest:tt)*) => { + // Emit all items within one block, applying an approprate #[cfg]. The + // #[cfg] will require all `$m` matchers specified and must also negate + // all previous matchers. + cfg_if! { @__apply cfg(all($($m,)* not(any($($not),*)))), $($it)* } + + // Recurse to emit all other items in `$rest`, and when we do so add all + // our `$m` matchers to the list of `$not` matchers as future emissions + // will have to negate everything we just matched as well. + cfg_if! { @__items ($($not,)* $($m,)*) ; $($rest)* } + }; + + // Internal macro to Apply a cfg attribute to a list of items + (@__apply $m:meta, $($it:item)*) => { + $(#[$m] $it)* + }; +} diff --git a/compiler/rustc_data_structures/src/sync/table/mod.rs b/compiler/rustc_data_structures/src/sync/table/mod.rs new file mode 100644 index 000000000000..0a47743be47d --- /dev/null +++ b/compiler/rustc_data_structures/src/sync/table/mod.rs @@ -0,0 +1,18 @@ +//! This module contains [SyncTable] and [SyncPushVec] which offers lock-free reads and uses +//! quiescent state based reclamation for which an API is available in the [collect] module. + +#![allow(unexpected_cfgs, clippy::len_without_is_empty, clippy::type_complexity)] + +#[macro_use] +mod macros; + +pub mod collect; +mod raw; +mod scopeguard; +mod util; + +pub mod sync_push_vec; +pub mod sync_table; + +pub use sync_push_vec::SyncPushVec; +pub use sync_table::SyncTable; diff --git a/compiler/rustc_data_structures/src/sync/table/raw/bitmask.rs b/compiler/rustc_data_structures/src/sync/table/raw/bitmask.rs new file mode 100644 index 000000000000..e22b33e4da3b --- /dev/null +++ b/compiler/rustc_data_structures/src/sync/table/raw/bitmask.rs @@ -0,0 +1,77 @@ +use core::intrinsics; + +use super::imp::{BITMASK_MASK, BITMASK_STRIDE, BitMaskWord}; + +/// A bit mask which contains the result of a `Match` operation on a `Group` and +/// allows iterating through them. +/// +/// The bit mask is arranged so that low-order bits represent lower memory +/// addresses for group match results. +/// +/// For implementation reasons, the bits in the set may be sparsely packed, so +/// that there is only one bit-per-byte used (the high bit, 7). If this is the +/// case, `BITMASK_STRIDE` will be 8 to indicate a divide-by-8 should be +/// performed on counts/indices to normalize this difference. `BITMASK_MASK` is +/// similarly a mask of all the actually-used bits. +#[derive(Copy, Clone)] +pub struct BitMask(pub BitMaskWord); + +#[allow(clippy::use_self)] +impl BitMask { + /// Returns a new `BitMask` with all bits inverted. + #[inline] + #[must_use] + pub fn invert(self) -> Self { + BitMask(self.0 ^ BITMASK_MASK) + } + + /// Returns a new `BitMask` with the lowest bit removed. + #[inline] + #[must_use] + pub fn remove_lowest_bit(self) -> Self { + BitMask(self.0 & (self.0 - 1)) + } + /// Returns whether the `BitMask` has at least one set bit. + #[inline] + pub fn any_bit_set(self) -> bool { + self.0 != 0 + } + + /// Returns the first set bit in the `BitMask`, if there is one. + #[inline] + pub fn lowest_set_bit(self) -> Option { + if self.0 == 0 { None } else { Some(unsafe { self.lowest_set_bit_nonzero() }) } + } + + /// Returns the first set bit in the `BitMask`, if there is one. The + /// bitmask must not be empty. + #[inline] + pub unsafe fn lowest_set_bit_nonzero(self) -> usize { + unsafe { intrinsics::cttz_nonzero(self.0) as usize / BITMASK_STRIDE } + } +} + +impl IntoIterator for BitMask { + type Item = usize; + type IntoIter = BitMaskIter; + + #[inline] + fn into_iter(self) -> BitMaskIter { + BitMaskIter(self) + } +} + +/// Iterator over the contents of a `BitMask`, returning the indicies of set +/// bits. +pub struct BitMaskIter(BitMask); + +impl Iterator for BitMaskIter { + type Item = usize; + + #[inline] + fn next(&mut self) -> Option { + let bit = self.0.lowest_set_bit()?; + self.0 = self.0.remove_lowest_bit(); + Some(bit) + } +} diff --git a/compiler/rustc_data_structures/src/sync/table/raw/generic.rs b/compiler/rustc_data_structures/src/sync/table/raw/generic.rs new file mode 100644 index 000000000000..b6a31ef78b93 --- /dev/null +++ b/compiler/rustc_data_structures/src/sync/table/raw/generic.rs @@ -0,0 +1,117 @@ +use core::intrinsics::atomic_load_acq; +use core::mem; + +use super::EMPTY; +use super::bitmask::BitMask; + +// Use the native word size as the group size. Using a 64-bit group size on +// a 32-bit architecture will just end up being more expensive because +// shifts and multiplies will need to be emulated. +#[cfg(any(target_pointer_width = "64", target_arch = "aarch64", target_arch = "x86_64",))] +type GroupWord = u64; +#[cfg(all(target_pointer_width = "32", not(target_arch = "aarch64"), not(target_arch = "x86_64"),))] +type GroupWord = u32; + +pub type BitMaskWord = GroupWord; +pub const BITMASK_STRIDE: usize = 8; +// We only care about the highest bit of each byte for the mask. +#[allow(clippy::cast_possible_truncation, clippy::unnecessary_cast)] +pub const BITMASK_MASK: BitMaskWord = 0x8080_8080_8080_8080_u64 as GroupWord; + +/// Helper function to replicate a byte across a `GroupWord`. +#[inline] +fn repeat(byte: u8) -> GroupWord { + GroupWord::from_ne_bytes([byte; Group::WIDTH]) +} + +/// Abstraction over a group of control bytes which can be scanned in +/// parallel. +/// +/// This implementation uses a word-sized integer. +#[derive(Copy, Clone)] +pub struct Group(GroupWord); + +// We perform all operations in the native endianess, and convert to +// little-endian just before creating a BitMask. The can potentially +// enable the compiler to eliminate unnecessary byte swaps if we are +// only checking whether a BitMask is empty. +#[allow(clippy::use_self)] +impl Group { + /// Number of bytes in the group. + pub const WIDTH: usize = mem::size_of::(); + + /// Returns a full group of empty bytes, suitable for use as the initial + /// value for an empty hash table. + /// + /// This is guaranteed to be aligned to the group size. + pub const EMPTY: Group = { + #[repr(C)] + struct AlignedBytes { + _align: [Group; 0], + bytes: [u8; Group::WIDTH], + } + const ALIGNED_BYTES: AlignedBytes = + AlignedBytes { _align: [], bytes: [EMPTY; Group::WIDTH] }; + unsafe { mem::transmute(ALIGNED_BYTES) } + }; + + /// Loads a group of bytes starting at the given address. + #[inline] + #[allow(clippy::cast_ptr_alignment)] // unaligned load + pub unsafe fn load(ptr: *const u8) -> Self { + let mut bytes = [0u8; Group::WIDTH]; + for (i, byte) in bytes.iter_mut().enumerate() { + *byte = atomic_load_acq(ptr.add(i)); + } + mem::transmute(bytes) + } + + /// Loads a group of bytes starting at the given address, which must be + /// aligned to `mem::align_of::()`. + #[inline] + #[allow(clippy::cast_ptr_alignment)] + pub unsafe fn load_aligned(ptr: *const u8) -> Self { + // FIXME: use align_offset once it stabilizes + debug_assert_eq!(ptr as usize & (mem::align_of::() - 1), 0); + Group::load(ptr) + } + + /// Returns a `BitMask` indicating all bytes in the group which *may* + /// have the given value. + /// + /// This function may return a false positive in certain cases where + /// the byte in the group differs from the searched value only in its + /// lowest bit. This is fine because: + /// - This never happens for `EMPTY` and `DELETED`, only full entries. + /// - The check for key equality will catch these. + /// - This only happens if there is at least 1 true match. + /// - The chance of this happening is very low (< 1% chance per byte). + #[inline] + pub fn match_byte(self, byte: u8) -> BitMask { + // This algorithm is derived from + // http://graphics.stanford.edu/~seander/bithacks.html##ValueInWord + let cmp = self.0 ^ repeat(byte); + BitMask((cmp.wrapping_sub(repeat(0x01)) & !cmp & repeat(0x80)).to_le()) + } + + /// Returns a `BitMask` indicating all bytes in the group which are + /// `EMPTY`. + #[inline] + pub fn match_empty(self) -> BitMask { + self.match_byte(EMPTY) + } + + /// Returns a `BitMask` indicating all bytes in the group which are + /// `EMPTY` or `DELETED`. + #[inline] + pub fn match_empty_or_deleted(self) -> BitMask { + // A byte is EMPTY or DELETED iff the high bit is set + BitMask((self.0 & repeat(0x80)).to_le()) + } + + /// Returns a `BitMask` indicating all bytes in the group which are full. + #[inline] + pub fn match_full(self) -> BitMask { + self.match_empty_or_deleted().invert() + } +} diff --git a/compiler/rustc_data_structures/src/sync/table/raw/mod.rs b/compiler/rustc_data_structures/src/sync/table/raw/mod.rs new file mode 100644 index 000000000000..575211fd90ae --- /dev/null +++ b/compiler/rustc_data_structures/src/sync/table/raw/mod.rs @@ -0,0 +1,27 @@ +cfg_if! { + // Use the SSE2 implementation if possible: it allows us to scan 16 buckets + // at once instead of 8. We don't bother with AVX since it would require + // runtime dispatch and wouldn't gain us much anyways: the probability of + // finding a match drops off drastically after the first few buckets. + // + // I attempted an implementation on ARM using NEON instructions, but it + // turns out that most NEON instructions have multi-cycle latency, which in + // the end outweighs any gains over the generic implementation. + if #[cfg(all( + target_feature = "sse2", + any(target_arch = "x86", target_arch = "x86_64"), + not(miri) + ))] { + pub mod sse2; + pub use sse2 as imp; + } else { + #[path = "generic.rs"] + pub mod generic; + pub use generic as imp; + } +} + +pub mod bitmask; + +/// Control byte value for an empty bucket. +const EMPTY: u8 = 0b1111_1111; diff --git a/compiler/rustc_data_structures/src/sync/table/raw/sse2.rs b/compiler/rustc_data_structures/src/sync/table/raw/sse2.rs new file mode 100644 index 000000000000..60dc1d65a3c6 --- /dev/null +++ b/compiler/rustc_data_structures/src/sync/table/raw/sse2.rs @@ -0,0 +1,127 @@ +use core::arch::asm; +#[cfg(target_arch = "x86")] +use core::arch::x86; +#[cfg(target_arch = "x86_64")] +use core::arch::x86_64 as x86; +use core::mem; + +use super::EMPTY; +use super::bitmask::BitMask; + +pub type BitMaskWord = u16; +pub const BITMASK_STRIDE: usize = 1; +pub const BITMASK_MASK: BitMaskWord = 0xffff; + +/// Abstraction over a group of control bytes which can be scanned in +/// parallel. +/// +/// This implementation uses a 128-bit SSE value. +#[derive(Copy, Clone)] +#[repr(transparent)] +pub struct Group(x86::__m128i); + +// FIXME: https://github.com/rust-lang/rust-clippy/issues/3859 +#[allow(clippy::use_self)] +impl Group { + /// Number of bytes in the group. + pub const WIDTH: usize = mem::size_of::(); + + /// Returns a full group of empty bytes, suitable for use as the initial + /// value for an empty hash table. + /// + /// This is guaranteed to be aligned to the group size. + pub const EMPTY: Group = { + #[repr(C)] + struct AlignedBytes { + _align: [Group; 0], + bytes: [u8; Group::WIDTH], + } + const ALIGNED_BYTES: AlignedBytes = + AlignedBytes { _align: [], bytes: [EMPTY; Group::WIDTH] }; + unsafe { mem::transmute(ALIGNED_BYTES) } + }; + + /// Loads a group of bytes starting at the given address. + #[inline] + #[allow(clippy::cast_ptr_alignment)] // unaligned load + pub unsafe fn load(ptr: *const u8) -> Self { + unsafe { + let mut result = x86::_mm_loadu_si128(ptr.cast()); + + // Hide the `result` value since we need the backend to assume it could be the hardware + // instruction `movdqu` which has Ordering::Acquire semantics. + asm!("/* {} */", inout(xmm_reg) result, options(pure, readonly, nostack, preserves_flags)); + + Group(result) + } + } + + /// Loads a group of bytes starting at the given address, which must be + /// aligned to `mem::align_of::()`. + #[inline] + #[allow(clippy::cast_ptr_alignment)] + pub unsafe fn load_aligned(ptr: *const u8) -> Self { + unsafe { + // TODO: Use a volatile read here instead for better code generation. + // Find out if compiler fences are enough to make that atomic. + // FIXME: use align_offset once it stabilizes + debug_assert_eq!(ptr as usize & (mem::align_of::() - 1), 0); + + let mut result = x86::_mm_load_si128(ptr.cast()); + + // Hide the `result` value since we need the backend to assume it could be the hardware + // instruction `movdqa` which has Ordering::Acquire semantics. + asm!("/* {} */", inout(xmm_reg) result, options(pure, readonly, nostack, preserves_flags)); + + Group(result) + } + } + + /// Returns a `BitMask` indicating all bytes in the group which have + /// the given value. + #[inline] + pub fn match_byte(self, byte: u8) -> BitMask { + #[allow( + clippy::cast_possible_wrap, // byte: u8 as i8 + // byte: i32 as u16 + // note: _mm_movemask_epi8 returns a 16-bit mask in a i32, the + // upper 16-bits of the i32 are zeroed: + clippy::cast_sign_loss, + clippy::cast_possible_truncation + )] + unsafe { + let cmp = x86::_mm_cmpeq_epi8(self.0, x86::_mm_set1_epi8(byte as i8)); + BitMask(x86::_mm_movemask_epi8(cmp) as u16) + } + } + + /// Returns a `BitMask` indicating all bytes in the group which are + /// `EMPTY`. + #[inline] + pub fn match_empty(self) -> BitMask { + self.match_byte(EMPTY) + } + + /// Returns a `BitMask` indicating all bytes in the group which are + /// `EMPTY` or `DELETED`. + #[inline] + pub fn match_empty_or_deleted(self) -> BitMask { + #[allow( + // byte: i32 as u16 + // note: _mm_movemask_epi8 returns a 16-bit mask in a i32, the + // upper 16-bits of the i32 are zeroed: + clippy::cast_sign_loss, + clippy::cast_possible_truncation + )] + unsafe { + // A byte is EMPTY or DELETED iff the high bit is set + BitMask(x86::_mm_movemask_epi8(self.0) as u16) + } + } + + /// Returns a `BitMask` indicating all bytes in the group which are full. + #[inline] + pub fn match_full(&self) -> BitMask { + self.match_empty_or_deleted().invert() + } +} diff --git a/compiler/rustc_data_structures/src/sync/table/scopeguard.rs b/compiler/rustc_data_structures/src/sync/table/scopeguard.rs new file mode 100644 index 000000000000..4e9bf045adbd --- /dev/null +++ b/compiler/rustc_data_structures/src/sync/table/scopeguard.rs @@ -0,0 +1,49 @@ +// Extracted from the scopeguard crate +use core::ops::{Deref, DerefMut}; + +pub struct ScopeGuard +where + F: FnMut(&mut T), +{ + dropfn: F, + value: T, +} + +#[inline] +pub fn guard(value: T, dropfn: F) -> ScopeGuard +where + F: FnMut(&mut T), +{ + ScopeGuard { dropfn, value } +} + +impl Deref for ScopeGuard +where + F: FnMut(&mut T), +{ + type Target = T; + #[inline] + fn deref(&self) -> &T { + &self.value + } +} + +impl DerefMut for ScopeGuard +where + F: FnMut(&mut T), +{ + #[inline] + fn deref_mut(&mut self) -> &mut T { + &mut self.value + } +} + +impl Drop for ScopeGuard +where + F: FnMut(&mut T), +{ + #[inline] + fn drop(&mut self) { + (self.dropfn)(&mut self.value) + } +} diff --git a/compiler/rustc_data_structures/src/sync/table/sync_push_vec.rs b/compiler/rustc_data_structures/src/sync/table/sync_push_vec.rs new file mode 100644 index 000000000000..afefad8980cd --- /dev/null +++ b/compiler/rustc_data_structures/src/sync/table/sync_push_vec.rs @@ -0,0 +1,578 @@ +//! A contiguous push-only array type with lock-free reads. + +use core::ptr::NonNull; +use std::alloc::{Allocator, Global, Layout, LayoutError, handle_alloc_error}; +use std::cell::UnsafeCell; +use std::intrinsics::unlikely; +use std::iter::FromIterator; +use std::marker::PhantomData; +use std::ops::{Deref, DerefMut}; +use std::ptr::slice_from_raw_parts; +use std::sync::Arc; +use std::sync::atomic::{AtomicPtr, AtomicUsize, Ordering}; +use std::{cmp, mem}; + +use parking_lot::{Mutex, MutexGuard}; + +use super::collect::{self, Pin}; +use super::scopeguard::guard; + +mod code; +mod tests; + +/// A handle to a [SyncPushVec] with read access. +/// +/// It is acquired either by a pin, or by exclusive access to the vector. +pub struct Read<'a, T> { + table: &'a SyncPushVec, +} + +impl Copy for Read<'_, T> {} +impl Clone for Read<'_, T> { + fn clone(&self) -> Self { + Self { table: self.table } + } +} + +/// A handle to a [SyncPushVec] with write access. +pub struct Write<'a, T> { + table: &'a SyncPushVec, +} + +/// A handle to a [SyncPushVec] with write access protected by a lock. +pub struct LockedWrite<'a, T> { + table: Write<'a, T>, + _guard: MutexGuard<'a, ()>, +} + +impl<'a, T> Deref for LockedWrite<'a, T> { + type Target = Write<'a, T>; + + #[inline] + fn deref(&self) -> &Self::Target { + &self.table + } +} + +impl<'a, T> DerefMut for LockedWrite<'a, T> { + #[inline] + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.table + } +} + +/// A contiguous push-only array type with lock-free reads. +pub struct SyncPushVec { + current: AtomicPtr, + + lock: Mutex<()>, + + old: UnsafeCell>>>, + + // Tell dropck that we own instances of T. + marker: PhantomData, +} + +struct TableInfo { + items: AtomicUsize, + capacity: usize, +} + +#[repr(transparent)] +struct TableRef { + data: NonNull, + + marker: PhantomData<*mut T>, +} + +impl Copy for TableRef {} +impl Clone for TableRef { + #[inline] + fn clone(&self) -> Self { + Self { data: self.data, marker: self.marker } + } +} + +impl TableRef { + #[inline] + fn empty() -> Self { + if cfg!(debug_assertions) { + let real = Self::layout(0).unwrap().0; + let dummy = Layout::new::().align_to(real.align()).unwrap(); + debug_assert_eq!(real, dummy); + } + + #[repr(C, align(64))] + struct EmptyTable { + info: TableInfo, + } + + static EMPTY: EmptyTable = + EmptyTable { info: TableInfo { capacity: 0, items: AtomicUsize::new(0) } }; + + Self { + data: unsafe { + NonNull::new_unchecked(&EMPTY.info as *const TableInfo as *mut TableInfo) + }, + marker: PhantomData, + } + } + + #[inline] + fn layout(capacity: usize) -> Result<(Layout, usize), LayoutError> { + let data = Layout::new::().repeat(capacity)?.0; + let info = Layout::new::(); + data.extend(info) + } + + #[inline] + fn allocate(capacity: usize) -> Self { + let (layout, info_offset) = Self::layout(capacity).expect("capacity overflow"); + + let ptr: NonNull = Global + .allocate(layout) + .map(|ptr| ptr.cast()) + .unwrap_or_else(|_| handle_alloc_error(layout)); + + let info = + unsafe { NonNull::new_unchecked(ptr.as_ptr().add(info_offset) as *mut TableInfo) }; + + let mut result = Self { data: info, marker: PhantomData }; + + unsafe { + *result.info_mut() = TableInfo { capacity, items: AtomicUsize::new(0) }; + } + + result + } + + #[inline] + unsafe fn free(self) { + unsafe { + let items = self.info().items.load(Ordering::Relaxed); + if items > 0 { + if mem::needs_drop::() { + for i in 0..items { + self.data(i).drop_in_place(); + } + } + + let (layout, info_offset) = Self::layout(self.info().capacity).unwrap_unchecked(); + + Global.deallocate( + NonNull::new_unchecked((self.data.as_ptr() as *mut u8).sub(info_offset)), + layout, + ) + } + } + } + + fn from_maybe_empty_iter, const CHECK_LEN: bool>( + iter: I, + iter_size: usize, + capacity: usize, + ) -> TableRef { + if iter_size == 0 { + TableRef::empty() + } else { + let capacity = cmp::max(iter_size, capacity); + unsafe { TableRef::from_iter::<_, CHECK_LEN>(iter, capacity) } + } + } + + /// Allocates a new table and fills it with the content of an iterator + unsafe fn from_iter, const CHECK_LEN: bool>( + iter: I, + new_capacity: usize, + ) -> TableRef { + unsafe { + debug_assert!(new_capacity > 0); + + let mut new_table = TableRef::::allocate(new_capacity); + + let mut guard = guard(Some(new_table), |new_table| { + new_table.map(|new_table| new_table.free()); + }); + + // Copy all elements to the new table. + for (index, item) in iter.enumerate() { + debug_assert!(index < new_capacity); + if CHECK_LEN && index >= new_capacity { + break; + } + + new_table.first().add(index).write(item); + + // Write items per iteration in case `next` on the iterator panics. + + *new_table.info_mut().items.get_mut() = index + 1; + } + + *guard = None; + + new_table + } + } + + unsafe fn info(&self) -> &TableInfo { + unsafe { self.data.as_ref() } + } + + unsafe fn info_mut(&mut self) -> &mut TableInfo { + unsafe { self.data.as_mut() } + } + + #[inline] + unsafe fn first(&self) -> *mut T { + unsafe { (self.data.as_ptr() as *mut T).sub(self.info().capacity) } + } + + /// Returns a pointer to an element in the table. + #[inline] + unsafe fn slice(&self) -> *const [T] { + unsafe { + let items = self.info().items.load(Ordering::Acquire); + let base = if items == 0 && mem::align_of::() > 64 { + // Need a special case here since our empty allocation isn't aligned to T. + // It only has an alignment of 64. + mem::align_of::() as *const T + } else { + self.first() as *const T + }; + slice_from_raw_parts(base, items) + } + } + + /// Returns a pointer to an element in the table. + #[inline] + unsafe fn data(&self, index: usize) -> *mut T { + unsafe { + debug_assert!(index < self.info().items.load(Ordering::Acquire)); + + self.first().add(index) + } + } +} + +impl TableRef { + /// Allocates a new table of a different size and moves the contents of the + /// current table into it. + unsafe fn clone(&self, new_capacity: usize) -> TableRef { + unsafe { + debug_assert!(new_capacity >= self.info().capacity); + + TableRef::from_iter::<_, false>((*self.slice()).iter().cloned(), new_capacity) + } + } +} + +struct DestroyTable { + table: TableRef, + lock: Mutex, +} + +unsafe impl Sync for DestroyTable {} +unsafe impl Send for DestroyTable {} + +impl DestroyTable { + unsafe fn run(&self) { + unsafe { + let mut status = self.lock.lock(); + if !*status { + *status = true; + self.table.free(); + } + } + } +} + +unsafe impl<#[may_dangle] T> Drop for SyncPushVec { + #[inline] + fn drop(&mut self) { + unsafe { + self.current().free(); + for table in self.old.get_mut() { + table.run(); + } + } + } +} + +unsafe impl Send for SyncPushVec {} +unsafe impl Sync for SyncPushVec {} + +impl Default for SyncPushVec { + #[inline] + fn default() -> Self { + Self::new() + } +} + +impl SyncPushVec { + /// Constructs a new, empty vector with zero capacity. + /// + /// The vector will not allocate until elements are pushed onto it. + #[inline] + pub fn new() -> Self { + Self::with_capacity(0) + } + + /// Constructs a new, empty vector with the specified capacity. + /// + /// The vector will be able to hold exactly `capacity` elements without reallocating. If `capacity` is 0, the vector will not allocate. + #[inline] + pub fn with_capacity(capacity: usize) -> Self { + Self { + current: AtomicPtr::new( + if capacity > 0 { TableRef::::allocate(capacity) } else { TableRef::empty() } + .data + .as_ptr(), + ), + old: UnsafeCell::new(Vec::new()), + marker: PhantomData, + lock: Mutex::new(()), + } + } + + /// Gets a reference to the underlying mutex that protects writes. + #[inline] + pub fn mutex(&self) -> &Mutex<()> { + &self.lock + } + + /// Creates a [Read] handle from a pinned region. + /// + /// Use [crate::collect::pin] to get a `Pin` instance. + #[inline] + pub fn read<'a>(&'a self, pin: Pin<'a>) -> Read<'a, T> { + let _pin = pin; + Read { table: self } + } + + /// Creates a [Write] handle without checking for exclusive access. + /// + /// # Safety + /// It's up to the caller to ensure only one thread writes to the vector at a time. + #[inline] + pub unsafe fn unsafe_write(&self) -> Write<'_, T> { + Write { table: self } + } + + /// Creates a [Write] handle from a mutable reference. + #[inline] + pub fn write(&mut self) -> Write<'_, T> { + Write { table: self } + } + + /// Creates a [LockedWrite] handle by taking the underlying mutex that protects writes. + #[inline] + pub fn lock(&self) -> LockedWrite<'_, T> { + LockedWrite { table: Write { table: self }, _guard: self.lock.lock() } + } + + /// Creates a [LockedWrite] handle from a guard protecting the underlying mutex that protects writes. + #[inline] + pub fn lock_from_guard<'a>(&'a self, guard: MutexGuard<'a, ()>) -> LockedWrite<'a, T> { + // Verify that we are target of the guard + assert_eq!(&self.lock as *const _, MutexGuard::mutex(&guard) as *const _); + + LockedWrite { table: Write { table: self }, _guard: guard } + } + + /// Extracts a mutable slice of the entire vector. + #[inline] + pub fn as_mut_slice(&mut self) -> &mut [T] { + unsafe { &mut *(self.current().slice() as *mut [T]) } + } + + #[inline] + fn current(&self) -> TableRef { + TableRef { + data: unsafe { NonNull::new_unchecked(self.current.load(Ordering::Acquire)) }, + marker: PhantomData, + } + } +} + +impl<'a, T> Read<'a, T> { + /// Returns the number of elements the map can hold without reallocating. + #[inline] + pub fn capacity(self) -> usize { + unsafe { self.table.current().info().capacity } + } + + /// Returns the number of elements in the table. + #[inline] + pub fn len(self) -> usize { + unsafe { self.table.current().info().items.load(Ordering::Acquire) } + } + + /// Extracts a slice containing the entire vector. + #[inline] + pub fn as_slice(self) -> &'a [T] { + let table = self.table.current(); + unsafe { &*table.slice() } + } +} + +impl Write<'_, T> { + /// Creates a [Read] handle which gives access to read operations. + #[inline] + pub fn read(&self) -> Read<'_, T> { + Read { table: self.table } + } +} + +impl<'a, T: Send + Clone> Write<'a, T> { + /// Inserts a new element into the end of the table, and returns a refernce to it along + /// with its index. + #[inline] + pub fn push(&mut self, value: T) -> (&'a T, usize) { + let mut table = self.table.current(); + unsafe { + let items = table.info().items.load(Ordering::Relaxed); + + if unlikely(items == table.info().capacity) { + table = self.expand_by_one(); + } + + let result = table.first().add(items); + + result.write(value); + + table.info().items.store(items + 1, Ordering::Release); + + (&*result, items) + } + } + + /// Reserves capacity for at least `additional` more elements to be inserted + /// in the given vector. The collection may reserve more space to avoid + /// frequent reallocations. Does nothing if the capacity is already sufficient. + #[inline] + pub fn reserve(&mut self, additional: usize) { + let table = self.table.current(); + unsafe { + let required = table + .info() + .items + .load(Ordering::Relaxed) + .checked_add(additional) + .expect("capacity overflow"); + + if table.info().capacity < required { + self.expand_by(additional); + } + } + } + + #[cold] + #[inline(never)] + fn expand_by_one(&mut self) -> TableRef { + self.expand_by(1) + } + + // Tiny Vecs are dumb. Skip to: + // - 8 if the element size is 1, because any heap allocators is likely + // to round up a request of less than 8 bytes to at least 8 bytes. + // - 4 if elements are moderate-sized (<= 1 KiB). + // - 1 otherwise, to avoid wasting too much space for very short Vecs. + const MIN_NON_ZERO_CAP: usize = if mem::size_of::() == 1 { + 8 + } else if mem::size_of::() <= 1024 { + 4 + } else { + 1 + }; + + fn expand_by(&mut self, additional: usize) -> TableRef { + let table = self.table.current(); + + let items = unsafe { table.info().items.load(Ordering::Relaxed) }; + let capacity = unsafe { table.info().capacity }; + + // Avoid `Option::ok_or_else` because it bloats LLVM IR. + let required_cap = match items.checked_add(additional) { + Some(required_cap) => required_cap, + None => panic!("capacity overflow"), + }; + + // This guarantees exponential growth. The doubling cannot overflow + // because `cap <= isize::MAX` and the type of `cap` is `usize`. + let cap = cmp::max(capacity * 2, required_cap); + let cap = cmp::max(Self::MIN_NON_ZERO_CAP, cap); + + let new_table = unsafe { table.clone(cap) }; + + self.replace_table(new_table); + + new_table + } +} + +impl Write<'_, T> { + fn replace_table(&mut self, new_table: TableRef) { + let table = self.table.current(); + + self.table.current.store(new_table.data.as_ptr(), Ordering::Release); + + let destroy = Arc::new(DestroyTable { table, lock: Mutex::new(false) }); + + unsafe { + (*self.table.old.get()).push(destroy.clone()); + + collect::defer_unchecked(move || destroy.run()); + } + } + + /// Replaces the content of the vector with the content of the iterator. + /// `capacity` specifies the new capacity if it's greater than the length of the iterator. + #[inline] + pub fn replace>(&mut self, iter: I, capacity: usize) { + let iter = iter.into_iter(); + + let table = if let Some(max) = iter.size_hint().1 { + TableRef::from_maybe_empty_iter::<_, true>(iter, max, capacity) + } else { + let elements: Vec<_> = iter.collect(); + let len = elements.len(); + TableRef::from_maybe_empty_iter::<_, false>(elements.into_iter(), len, capacity) + }; + + self.replace_table(table); + } +} + +impl Extend for Write<'_, T> { + #[inline] + fn extend>(&mut self, iter: I) { + let iter = iter.into_iter(); + self.reserve(iter.size_hint().0); + iter.for_each(|v| { + self.push(v); + }); + } + + #[inline] + fn extend_one(&mut self, item: T) { + self.push(item); + } + + #[inline] + fn extend_reserve(&mut self, additional: usize) { + self.reserve(additional); + } +} + +impl FromIterator for SyncPushVec { + #[inline] + fn from_iter>(iter: I) -> Self { + let iter = iter.into_iter(); + let mut map = Self::with_capacity(iter.size_hint().0); + let mut write = map.write(); + iter.for_each(|v| { + write.push(v); + }); + map + } +} diff --git a/compiler/rustc_data_structures/src/sync/table/sync_push_vec/code.rs b/compiler/rustc_data_structures/src/sync/table/sync_push_vec/code.rs new file mode 100644 index 000000000000..b76d1c63e63d --- /dev/null +++ b/compiler/rustc_data_structures/src/sync/table/sync_push_vec/code.rs @@ -0,0 +1,18 @@ +#![cfg(code)] + +use super::{SyncPushVec, Write}; + +#[no_mangle] +unsafe fn get(a: &SyncPushVec) -> Option { + a.unsafe_write().read().as_slice().get(2).cloned() +} + +#[no_mangle] +unsafe fn push(a: &SyncPushVec) { + a.unsafe_write().push(4000); +} + +#[no_mangle] +unsafe fn push2(a: &mut Write<'_, usize>) { + a.push(4000); +} diff --git a/compiler/rustc_data_structures/src/sync/table/sync_push_vec/tests.rs b/compiler/rustc_data_structures/src/sync/table/sync_push_vec/tests.rs new file mode 100644 index 000000000000..924db3384a75 --- /dev/null +++ b/compiler/rustc_data_structures/src/sync/table/sync_push_vec/tests.rs @@ -0,0 +1,74 @@ +#![cfg(test)] + +use crate::collect::release; +use crate::sync_push_vec::SyncPushVec; + +#[test] +fn test_iter() { + let mut m = SyncPushVec::new(); + m.write().push(1); + m.write().push(2); + assert_eq!(m.write().read().as_slice().iter().copied().collect::>(), vec![1, 2]); +} + +#[test] +fn test_high_align() { + #[repr(align(128))] + #[derive(Clone)] + struct A(u8); + let mut m = SyncPushVec::::new(); + for _a in m.write().read().as_slice() {} + m.write().push(A(1)); + for _a in m.write().read().as_slice() {} +} + +#[test] +fn test_low_align() { + let mut m = SyncPushVec::::with_capacity(1); + m.write().push(1); +} + +#[test] +fn test_insert() { + let m = SyncPushVec::new(); + assert_eq!(m.lock().read().len(), 0); + m.lock().push(2); + assert_eq!(m.lock().read().len(), 1); + m.lock().push(5); + assert_eq!(m.lock().read().len(), 2); + assert_eq!(m.lock().read().as_slice()[0], 2); + assert_eq!(m.lock().read().as_slice()[1], 5); + + release(); +} + +#[test] +fn test_replace() { + let m = SyncPushVec::new(); + m.lock().push(2); + m.lock().push(5); + assert_eq!(m.lock().read().as_slice(), [2, 5]); + m.lock().replace(vec![3].into_iter(), 0); + assert_eq!(m.lock().read().as_slice(), [3]); + m.lock().replace(vec![].into_iter(), 0); + assert_eq!(m.lock().read().as_slice(), []); + release(); +} + +#[test] +fn test_expand() { + let m = SyncPushVec::new(); + + assert_eq!(m.lock().read().len(), 0); + + let mut i = 0; + let old_raw_cap = m.lock().read().capacity(); + while old_raw_cap == m.lock().read().capacity() { + m.lock().push(i); + i += 1; + } + + assert_eq!(m.lock().read().len(), i); + + release(); +} diff --git a/compiler/rustc_data_structures/src/sync/table/sync_table.rs b/compiler/rustc_data_structures/src/sync/table/sync_table.rs new file mode 100644 index 000000000000..1bb82c0d6149 --- /dev/null +++ b/compiler/rustc_data_structures/src/sync/table/sync_table.rs @@ -0,0 +1,1569 @@ +//! A hash table with lock-free reads. +//! +//! It is based on the table from the `hashbrown` crate. + +use core::ptr::NonNull; +use std::alloc::{Allocator, Global, Layout, LayoutError, handle_alloc_error}; +use std::borrow::Borrow; +use std::cell::UnsafeCell; +use std::collections::hash_map::RandomState; +use std::hash::{BuildHasher, Hash}; +use std::intrinsics::{likely, unlikely}; +use std::iter::{FromIterator, FusedIterator}; +use std::marker::PhantomData; +use std::ops::{Deref, DerefMut}; +use std::sync::Arc; +use std::sync::atomic::{AtomicPtr, AtomicU8, AtomicUsize, Ordering}; +use std::{cmp, fmt, mem}; + +use parking_lot::Mutex; + +use super::collect::{self, Pin, pin}; +use super::raw::bitmask::BitMask; +use super::raw::imp::Group; +use super::scopeguard::guard; +use super::util::{cold_path, make_insert_hash}; +use crate::sync::{DynSend, DynSync, Lock, LockGuard}; + +mod code; +mod tests; + +#[inline] +fn hasher(hash_builder: &S, val: &(K, V)) -> u64 { + make_insert_hash(hash_builder, &val.0) +} + +#[inline] +fn eq(key: &Q) -> impl Fn(&(K, V)) -> bool + '_ +where + K: Borrow, + Q: ?Sized + Eq, +{ + move |x| key.eq(x.0.borrow()) +} + +/// A reference to a hash table bucket containing a `T`. +/// +/// This is usually just a pointer to the element itself. However if the element +/// is a ZST, then we instead track the index of the element in the table so +/// that `erase` works properly. +struct Bucket { + // Actually it is pointer to next element than element itself + // this is needed to maintain pointer arithmetic invariants + // keeping direct pointer to element introduces difficulty. + // Using `NonNull` for variance and niche layout + ptr: NonNull, +} + +impl Clone for Bucket { + #[inline] + fn clone(&self) -> Self { + Self { ptr: self.ptr } + } +} + +impl Bucket { + #[inline] + fn as_ptr(&self) -> *mut T { + if mem::size_of::() == 0 { + // Just return an arbitrary ZST pointer which is properly aligned + mem::align_of::() as *mut T + } else { + unsafe { self.ptr.as_ptr().sub(1) } + } + } + #[inline] + unsafe fn next_n(&self, offset: usize) -> Self { + unsafe { + let ptr = if mem::size_of::() == 0 { + (self.ptr.as_ptr() as usize + offset) as *mut T + } else { + self.ptr.as_ptr().sub(offset) + }; + Self { ptr: NonNull::new_unchecked(ptr) } + } + } + #[inline] + unsafe fn drop(&self) { + unsafe { + self.as_ptr().drop_in_place(); + } + } + #[inline] + unsafe fn write(&self, val: T) { + unsafe { + self.as_ptr().write(val); + } + } + #[inline] + unsafe fn as_ref<'a>(&self) -> &'a T { + unsafe { &*self.as_ptr() } + } + #[inline] + unsafe fn as_mut<'a>(&self) -> &'a mut T { + unsafe { &mut *self.as_ptr() } + } +} + +impl Bucket<(K, V)> { + #[inline] + pub unsafe fn as_pair_ref<'a>(&self) -> (&'a K, &'a V) { + unsafe { + let pair = &*self.as_ptr(); + (&pair.0, &pair.1) + } + } +} + +/// A handle to a [SyncTable] with read access. +/// +/// It is acquired either by a pin, or by exclusive access to the table. +pub struct Read<'a, K, V, S = DefaultHashBuilder> { + table: &'a SyncTable, +} + +impl Copy for Read<'_, K, V, S> {} +impl Clone for Read<'_, K, V, S> { + fn clone(&self) -> Self { + Self { table: self.table } + } +} + +/// A handle to a [SyncTable] with write access. +pub struct Write<'a, K, V, S = DefaultHashBuilder> { + table: &'a SyncTable, +} + +/// A handle to a [SyncTable] with write access protected by a lock. +pub struct LockedWrite<'a, K, V, S = DefaultHashBuilder> { + table: Write<'a, K, V, S>, + _guard: LockGuard<'a, ()>, +} + +impl<'a, K, V, S> Deref for LockedWrite<'a, K, V, S> { + type Target = Write<'a, K, V, S>; + + #[inline] + fn deref(&self) -> &Self::Target { + &self.table + } +} + +impl<'a, K, V, S> DerefMut for LockedWrite<'a, K, V, S> { + #[inline] + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.table + } +} + +/// Default hash builder for [SyncTable]. +pub type DefaultHashBuilder = RandomState; + +/// A hash table with lock-free reads. +/// +/// It is based on the table from the `hashbrown` crate. +pub struct SyncTable { + hash_builder: S, + + current: AtomicPtr, + + lock: Lock<()>, + + old: UnsafeCell>>>, + + // Tell dropck that we own instances of K, V. + marker: PhantomData<(K, V)>, +} + +struct TableInfo { + // Mask to get an index from a hash value. The value is one less than the + // number of buckets in the table. + bucket_mask: usize, + + // Number of elements that can be inserted before we need to grow the table + growth_left: AtomicUsize, + + // Number of elements that has been removed from the table + tombstones: AtomicUsize, +} + +impl TableInfo { + #[inline] + fn num_ctrl_bytes(&self) -> usize { + self.buckets() + Group::WIDTH + } + + /// Returns the number of buckets in the table. + #[inline] + fn buckets(&self) -> usize { + self.bucket_mask + 1 + } + + #[inline] + fn items(&self) -> usize { + // FIXME: May overflow and return wrong value. + // TODO: Or will they synchronize due to the lock? + // NO: A concurrent write / remove may happen which puts them out of sync? + bucket_mask_to_capacity(self.bucket_mask) + - self.growth_left.load(Ordering::Acquire) + - self.tombstones.load(Ordering::Acquire) + } + + /// Returns a pointer to a control byte. + #[inline] + unsafe fn ctrl(&self, index: usize) -> *mut u8 { + unsafe { + debug_assert!(index < self.num_ctrl_bytes()); + + let info = Layout::new::(); + let control = Layout::new::(); + let offset = info.extend(control).unwrap().1; + + let ctrl = (self as *const TableInfo as *mut u8).add(offset); + + ctrl.add(index) + } + } + + /// Sets a control byte, and possibly also the replicated control byte at + /// the end of the array. + #[inline] + unsafe fn set_ctrl(&self, index: usize, ctrl: u8) { + unsafe { + // Replicate the first Group::WIDTH control bytes at the end of + // the array without using a branch: + // - If index >= Group::WIDTH then index == index2. + // - Otherwise index2 == self.bucket_mask + 1 + index. + // + // The very last replicated control byte is never actually read because + // we mask the initial index for unaligned loads, but we write it + // anyways because it makes the set_ctrl implementation simpler. + // + // If there are fewer buckets than Group::WIDTH then this code will + // replicate the buckets at the end of the trailing group. For example + // with 2 buckets and a group size of 4, the control bytes will look + // like this: + // + // Real | Replicated + // --------------------------------------------- + // | [A] | [B] | [EMPTY] | [EMPTY] | [A] | [B] | + // --------------------------------------------- + let index2 = ((index.wrapping_sub(Group::WIDTH)) & self.bucket_mask) + Group::WIDTH; + + *self.ctrl(index) = ctrl; + *self.ctrl(index2) = ctrl; + } + } + + /// Sets a control byte, and possibly also the replicated control byte at + /// the end of the array. Same as set_ctrl, but uses release stores. + #[inline] + unsafe fn set_ctrl_release(&self, index: usize, ctrl: u8) { + unsafe { + let index2 = ((index.wrapping_sub(Group::WIDTH)) & self.bucket_mask) + Group::WIDTH; + + (*(self.ctrl(index) as *mut AtomicU8)).store(ctrl, Ordering::Release); + (*(self.ctrl(index2) as *mut AtomicU8)).store(ctrl, Ordering::Release); + } + } + + /// Sets a control byte to the hash, and possibly also the replicated control byte at + /// the end of the array. + #[inline] + unsafe fn set_ctrl_h2(&self, index: usize, hash: u64) { + unsafe { self.set_ctrl(index, h2(hash)) } + } + + #[inline] + unsafe fn record_item_insert_at(&self, index: usize, hash: u64) { + unsafe { + self.growth_left.store(self.growth_left.load(Ordering::Relaxed) - 1, Ordering::Release); + self.set_ctrl_release(index, h2(hash)); + } + } + + /// Searches for an empty or deleted bucket which is suitable for inserting + /// a new element and sets the hash for that slot. + /// + /// There must be at least 1 empty bucket in the table. + #[inline] + unsafe fn prepare_insert_slot(&self, hash: u64) -> (usize, u8) { + unsafe { + let index = self.find_insert_slot(hash); + let old_ctrl = *self.ctrl(index); + self.set_ctrl_h2(index, hash); + (index, old_ctrl) + } + } + + /// Searches for an empty or deleted bucket which is suitable for inserting + /// a new element. + /// + /// There must be at least 1 empty bucket in the table. + #[inline] + unsafe fn find_insert_slot(&self, hash: u64) -> usize { + unsafe { + let mut probe_seq = self.probe_seq(hash); + loop { + let group = Group::load(self.ctrl(probe_seq.pos)); + if let Some(bit) = group.match_empty().lowest_set_bit() { + let result = (probe_seq.pos + bit) & self.bucket_mask; + + return result; + } + probe_seq.move_next(self.bucket_mask); + } + } + } + + /// Returns an iterator-like object for a probe sequence on the table. + /// + /// This iterator never terminates, but is guaranteed to visit each bucket + /// group exactly once. The loop using `probe_seq` must terminate upon + /// reaching a group containing an empty bucket. + #[inline] + unsafe fn probe_seq(&self, hash: u64) -> ProbeSeq { + ProbeSeq { pos: h1(hash) & self.bucket_mask, stride: 0 } + } +} + +#[repr(transparent)] +struct TableRef { + data: NonNull, + + marker: PhantomData<*mut T>, +} + +impl Copy for TableRef {} +impl Clone for TableRef { + #[inline] + fn clone(&self) -> Self { + Self { data: self.data, marker: self.marker } + } +} + +impl TableRef { + #[inline] + fn empty() -> Self { + #[repr(C)] + struct EmptyTable { + info: TableInfo, + control_bytes: [Group; 1], + } + + static EMPTY: EmptyTable = EmptyTable { + info: TableInfo { + bucket_mask: 0, + growth_left: AtomicUsize::new(0), + tombstones: AtomicUsize::new(0), + }, + control_bytes: [Group::EMPTY; 1], + }; + + Self { + data: unsafe { NonNull::new_unchecked(&EMPTY as *const EmptyTable as *mut TableInfo) }, + marker: PhantomData, + } + } + + #[inline] + fn layout(bucket_count: usize) -> Result<(Layout, usize), LayoutError> { + let buckets = Layout::new::().repeat(bucket_count)?.0; + let info = Layout::new::(); + let control = + Layout::array::(bucket_count + Group::WIDTH)?.align_to(mem::align_of::())?; + let (total, info_offset) = buckets.extend(info)?; + Ok((total.extend(control)?.0, info_offset)) + } + + #[inline] + fn allocate(bucket_count: usize) -> Self { + let (layout, info_offset) = Self::layout(bucket_count).expect("capacity overflow"); + + let ptr: NonNull = Global + .allocate(layout) + .map(|ptr| ptr.cast()) + .unwrap_or_else(|_| handle_alloc_error(layout)); + + let info = + unsafe { NonNull::new_unchecked(ptr.as_ptr().add(info_offset) as *mut TableInfo) }; + + let mut result = Self { data: info, marker: PhantomData }; + + unsafe { + *result.info_mut() = TableInfo { + bucket_mask: bucket_count - 1, + growth_left: AtomicUsize::new(bucket_mask_to_capacity(bucket_count - 1)), + tombstones: AtomicUsize::new(0), + }; + + result.info().ctrl(0).write_bytes(EMPTY, result.info().num_ctrl_bytes()); + } + + result + } + + #[inline] + unsafe fn free(self) { + unsafe { + if self.info().bucket_mask > 0 { + if mem::needs_drop::() { + for item in self.iter() { + item.drop(); + } + } + + // TODO: Document why we don't need to account for padding when adjusting + // the pointer. Sizes allowed can't result in padding? + Global.deallocate( + NonNull::new_unchecked(self.bucket_before_first() as *mut u8), + Self::layout(self.info().buckets()).unwrap_unchecked().0, + ) + } + } + } + + fn from_maybe_empty_iter< + S, + I: Iterator, + H: Fn(&S, &T) -> u64, + const CHECK_LEN: bool, + >( + iter: I, + iter_size: usize, + capacity: usize, + hash_builder: &S, + hasher: H, + ) -> TableRef { + if iter_size == 0 { + TableRef::empty() + } else { + let buckets = + capacity_to_buckets(cmp::max(iter_size, capacity)).expect("capacity overflow"); + unsafe { + TableRef::from_iter::<_, _, _, CHECK_LEN>(iter, buckets, hash_builder, hasher) + } + } + } + + /// Allocates a new table and fills it with the content of an iterator + unsafe fn from_iter, H: Fn(&S, &T) -> u64, const CHECK_LEN: bool>( + iter: I, + buckets: usize, + hash_builder: &S, + hasher: H, + ) -> TableRef { + unsafe { + let mut new_table = TableRef::allocate(buckets); + + let mut guard = guard(Some(new_table), |new_table| { + new_table.map(|new_table| new_table.free()); + }); + + let mut growth_left = *new_table.info_mut().growth_left.get_mut(); + + // Copy all elements to the new table. + for item in iter { + if CHECK_LEN && growth_left == 0 { + break; + } + + // This may panic. + let hash = hasher(hash_builder, &item); + + // We can use a simpler version of insert() here since: + // - we know there is enough space in the table. + // - all elements are unique. + let (index, _) = new_table.info().prepare_insert_slot(hash); + + new_table.bucket(index).write(item); + + growth_left -= 1; + } + + *new_table.info_mut().growth_left.get_mut() = growth_left; + + *guard = None; + + new_table + } + } + + unsafe fn info(&self) -> &TableInfo { + unsafe { self.data.as_ref() } + } + + unsafe fn info_mut(&mut self) -> &mut TableInfo { + unsafe { self.data.as_mut() } + } + + #[inline] + unsafe fn bucket_before_first(&self) -> *mut T { + unsafe { self.bucket_past_last().sub(self.info().buckets()) } + } + + #[inline] + unsafe fn bucket_past_last(&self) -> *mut T { + self.data.as_ptr() as *mut T + } + + /// Returns a pointer to an element in the table. + #[inline] + unsafe fn bucket(&self, index: usize) -> Bucket { + unsafe { + debug_assert!(index < self.info().buckets()); + + Bucket { ptr: NonNull::new_unchecked(self.bucket_past_last().sub(index)) } + } + } + + /// Returns an iterator over every element in the table. It is up to + /// the caller to ensure that the table outlives the `RawIterRange`. + /// Because we cannot make the `next` method unsafe on the `RawIterRange` + /// struct, we have to make the `iter` method unsafe. + #[inline] + unsafe fn iter(&self) -> RawIterRange { + unsafe { + let data = Bucket { ptr: NonNull::new_unchecked(self.bucket_past_last()) }; + RawIterRange::new(self.info().ctrl(0), data, self.info().buckets()) + } + } + + /// Searches for an element in the table. + #[inline] + unsafe fn search( + &self, + hash: u64, + mut eq: impl FnMut(&T) -> bool, + mut stop: impl FnMut(&Group, &ProbeSeq) -> Option, + ) -> Result<(usize, Bucket), R> { + unsafe { + let h2_hash = h2(hash); + let mut probe_seq = self.info().probe_seq(hash); + let mut group = Group::load(self.info().ctrl(probe_seq.pos)); + let mut bitmask = group.match_byte(h2_hash).into_iter(); + + loop { + if let Some(bit) = bitmask.next() { + let index = (probe_seq.pos + bit) & self.info().bucket_mask; + + let bucket = self.bucket(index); + let elm = self.bucket(index).as_ref(); + if likely(eq(elm)) { + return Ok((index, bucket)); + } + + // Look at the next bit + continue; + } + + if let Some(stop) = stop(&group, &probe_seq) { + return Err(stop); + } + + probe_seq.move_next(self.info().bucket_mask); + group = Group::load(self.info().ctrl(probe_seq.pos)); + bitmask = group.match_byte(h2_hash).into_iter(); + } + } + } + + /// Searches for an element in the table. + #[inline] + unsafe fn find(&self, hash: u64, eq: impl FnMut(&T) -> bool) -> Option<(usize, Bucket)> { + unsafe { + self.search(hash, eq, |group, _| { + if likely(group.match_empty().any_bit_set()) { Some(()) } else { None } + }) + .ok() + } + } + + /// Searches for an element in the table. + #[inline] + unsafe fn find_potential( + &self, + hash: u64, + eq: impl FnMut(&T) -> bool, + ) -> Result<(usize, Bucket), PotentialSlot<'static>> { + unsafe { + self.search(hash, eq, |group, probe_seq| { + let bit = group.match_empty().lowest_set_bit(); + if likely(bit.is_some()) { + let index = (probe_seq.pos + bit.unwrap_unchecked()) & self.info().bucket_mask; + Some(PotentialSlot { table_info: &*self.data.as_ptr(), index }) + } else { + None + } + }) + } + } +} + +impl TableRef { + /// Allocates a new table of a different size and moves the contents of the + /// current table into it. + unsafe fn clone_table( + &self, + hash_builder: &S, + buckets: usize, + hasher: impl Fn(&S, &T) -> u64, + ) -> TableRef { + unsafe { + debug_assert!(buckets >= self.info().buckets()); + + TableRef::from_iter::<_, _, _, false>( + self.iter().map(|bucket| bucket.as_ref().clone()), + buckets, + hash_builder, + hasher, + ) + } + } +} + +struct DestroyTable { + table: TableRef, + lock: Mutex, +} + +unsafe impl Sync for DestroyTable {} + +// FIXME: Unsound +//unsafe impl Send for DestroyTable {} +unsafe impl Send for DestroyTable {} + +impl DestroyTable { + unsafe fn run(&self) { + unsafe { + let mut status = self.lock.lock(); + if !*status { + *status = true; + self.table.free(); + } + } + } +} + +unsafe impl<#[may_dangle] K, #[may_dangle] V, S> Drop for SyncTable { + #[inline] + fn drop(&mut self) { + unsafe { + self.current().free(); + for table in self.old.get_mut() { + table.run(); + } + } + } +} + +unsafe impl DynSend for SyncTable {} +unsafe impl Sync for SyncTable {} +unsafe impl DynSync for SyncTable {} + +impl Default for SyncTable { + #[inline] + fn default() -> Self { + Self::new_with(Default::default(), 0) + } +} + +impl SyncTable { + /// Creates an empty [SyncTable]. + /// + /// The hash map is initially created with a capacity of 0, so it will not allocate until it + /// is first inserted into. + #[inline] + pub fn new() -> Self { + Self::default() + } +} + +impl SyncTable { + /// Creates an empty [SyncTable] with the specified capacity, using `hash_builder` + /// to hash the elements or keys. + /// + /// The hash map will be able to hold at least `capacity` elements without + /// reallocating. If `capacity` is 0, the hash map will not allocate. + #[inline] + pub fn new_with(hash_builder: S, capacity: usize) -> Self { + Self { + hash_builder, + current: AtomicPtr::new( + if capacity > 0 { + TableRef::<(K, V)>::allocate( + capacity_to_buckets(capacity).expect("capacity overflow"), + ) + } else { + TableRef::empty() + } + .data + .as_ptr(), + ), + old: UnsafeCell::new(Vec::new()), + marker: PhantomData, + lock: Lock::new(()), + } + } + + /// Returns a reference to the table's `BuildHasher`. + #[inline] + pub fn hasher(&self) -> &S { + &self.hash_builder + } + + /// Gets a reference to the underlying mutex that protects writes. + #[inline] + pub fn mutex(&self) -> &Lock<()> { + &self.lock + } + + /// Creates a [Read] handle from a pinned region. + /// + /// Use [crate::collect::pin] to get a `Pin` instance. + #[inline] + pub fn read<'a>(&'a self, pin: Pin<'a>) -> Read<'a, K, V, S> { + let _pin = pin; + Read { table: self } + } + + /// Creates a [Write] handle without checking for exclusive access. + /// + /// # Safety + /// It's up to the caller to ensure only one thread writes to the vector at a time. + #[inline] + pub unsafe fn unsafe_write(&self) -> Write<'_, K, V, S> { + Write { table: self } + } + + /// Creates a [Write] handle from a mutable reference. + #[inline] + pub fn write(&mut self) -> Write<'_, K, V, S> { + Write { table: self } + } + + /// Creates a [LockedWrite] handle by taking the underlying mutex that protects writes. + #[inline] + pub fn lock(&self) -> LockedWrite<'_, K, V, S> { + LockedWrite { table: Write { table: self }, _guard: self.lock.lock() } + } + /* + /// Creates a [LockedWrite] handle from a guard protecting the underlying mutex that protects writes. + #[inline] + pub fn lock_from_guard<'a>(&'a self, guard: LockGuard<'a, ()>) -> LockedWrite<'a, K, V, S> { + // Verify that we are target of the guard + assert_eq!(&self.lock as *const _, LockGuard::mutex(&guard) as *const _); + + LockedWrite { table: Write { table: self }, _guard: guard } + } + */ + #[inline] + fn current(&self) -> TableRef<(K, V)> { + TableRef { + data: unsafe { NonNull::new_unchecked(self.current.load(Ordering::Acquire)) }, + marker: PhantomData, + } + } +} + +impl SyncTable { + #[inline] + fn unwrap_hash(&self, key: &Q, hash: Option) -> u64 + where + K: Borrow, + Q: ?Sized + Hash, + { + hash.unwrap_or_else(|| self.hash_key(key)) + } + + /// Hashes a key with the table's hasher. + #[inline] + pub fn hash_key(&self, key: &Q) -> u64 + where + K: Borrow, + Q: ?Sized + Hash, + { + make_insert_hash(&self.hash_builder, key) + } + + /// Gets a mutable reference to an element in the table. + #[inline] + pub fn get_mut(&mut self, key: &Q, hash: Option) -> Option<(&mut K, &mut V)> + where + K: Borrow, + Q: ?Sized + Eq + Hash, + { + let hash = self.unwrap_hash(key, hash); + + unsafe { + self.current().find(hash, eq(key)).map(|(_, bucket)| { + let pair = bucket.as_mut(); + (&mut pair.0, &mut pair.1) + }) + } + } +} + +impl<'a, K, V, S: BuildHasher> Read<'a, K, V, S> { + /// Gets a reference to an element in the table or a potential location + /// where that element could be. + #[inline] + pub fn get_potential( + self, + key: &Q, + hash: Option, + ) -> Result<(&'a K, &'a V), PotentialSlot<'a>> + where + K: Borrow, + Q: ?Sized + Eq + Hash, + { + let hash = self.table.unwrap_hash(key, hash); + + unsafe { + match self.table.current().find_potential(hash, eq(key)) { + Ok((_, bucket)) => Ok(bucket.as_pair_ref()), + Err(slot) => Err(slot), + } + } + } + + /// Gets a reference to an element in the table. + #[inline] + pub fn get(self, key: &Q, hash: Option) -> Option<(&'a K, &'a V)> + where + K: Borrow, + Q: ?Sized + Eq + Hash, + { + let hash = self.table.unwrap_hash(key, hash); + + unsafe { self.table.current().find(hash, eq(key)).map(|(_, bucket)| bucket.as_pair_ref()) } + } + + /// Gets a reference to an element in the table, using a closure to find the match. + #[inline] + pub fn get_from_hash( + self, + hash: u64, + mut is_match: impl FnMut(&K) -> bool, + ) -> Option<(&'a K, &'a V)> { + unsafe { + self.table + .current() + .find(hash, move |(k, _)| is_match(k)) + .map(|(_, bucket)| bucket.as_pair_ref()) + } + } +} + +impl<'a, K, V, S> Read<'a, K, V, S> { + /// Gets a reference to an element in the table with a custom equality function. + #[inline] + pub fn get_with_eq( + self, + hash: u64, + mut eq: impl FnMut(&K, &V) -> bool, + ) -> Option<(&'a K, &'a V)> { + unsafe { + self.table + .current() + .find(hash, |(k, v)| eq(k, v)) + .map(|(_, bucket)| bucket.as_pair_ref()) + } + } + + /// Returns the number of elements the map can hold without reallocating. + #[inline] + pub fn capacity(self) -> usize { + unsafe { bucket_mask_to_capacity(self.table.current().info().bucket_mask) } + } + + /// Returns the number of elements in the table. + #[inline] + pub fn len(self) -> usize { + unsafe { self.table.current().info().items() } + } + + /// An iterator visiting all key-value pairs in arbitrary order. + /// The iterator element type is `(&'a K, &'a V)`. + #[inline] + pub fn iter(self) -> Iter<'a, K, V> { + let table = self.table.current(); + + // Here we tie the lifetime of self to the iter. + unsafe { Iter { inner: table.iter(), marker: PhantomData } } + } + + #[allow(dead_code)] + fn dump(self) + where + K: std::fmt::Debug, + V: std::fmt::Debug, + { + let table = self.table.current(); + + println!("Table dump:"); + + unsafe { + for i in 0..table.info().buckets() { + if *table.info().ctrl(i) == EMPTY { + println!("[#{:x}]", i); + } else { + println!( + "[#{:x}, ${:x}, {:?}]", + i, + *table.info().ctrl(i), + table.bucket(i).as_ref() + ); + } + } + } + + println!("--------"); + } +} + +impl Clone for SyncTable { + fn clone(&self) -> SyncTable { + pin(|_pin| { + let table = self.current(); + + unsafe { + let buckets = table.info().buckets(); + + SyncTable { + hash_builder: self.hash_builder.clone(), + current: AtomicPtr::new( + if buckets > 0 { + table.clone_table(&self.hash_builder, buckets, hasher) + } else { + TableRef::empty() + } + .data + .as_ptr(), + ), + old: UnsafeCell::new(Vec::new()), + marker: PhantomData, + lock: Lock::new(()), + } + } + }) + } +} + +impl<'a, K, V, S> Write<'a, K, V, S> { + /// Creates a [Read] handle which gives access to read operations. + #[inline] + pub fn read(&self) -> Read<'_, K, V, S> { + Read { table: self.table } + } + + /// Returns a reference to the table's `BuildHasher`. + #[inline] + pub fn hasher(&self) -> &'a S { + &self.table.hash_builder + } +} + +impl<'a, K: DynSend, V: DynSend + Clone, S: BuildHasher> Write<'a, K, V, S> { + /// Removes an element from the table, and returns a reference to it if was present. + #[inline] + pub fn remove(&mut self, key: &Q, hash: Option) -> Option<(&'a K, &'a V)> + where + K: Borrow, + Q: ?Sized + Eq + Hash, + { + let hash = self.table.unwrap_hash(key, hash); + + let table = self.table.current(); + + unsafe { + table.find(hash, eq(key)).map(|(index, bucket)| { + debug_assert!(is_full(*table.info().ctrl(index))); + table.info().set_ctrl_release(index, DELETED); + table + .info() + .tombstones + .store(table.info().tombstones.load(Ordering::Relaxed) + 1, Ordering::Release); + bucket.as_pair_ref() + }) + } + } +} + +impl<'a, K: Hash + Eq + DynSend + Clone, V: DynSend + Clone, S: BuildHasher> Write<'a, K, V, S> { + /// Inserts a element into the table. + /// Returns `false` if it already exists and doesn't update the value. + #[inline] + pub fn insert(&mut self, key: K, value: V, hash: Option) -> bool { + let hash = self.table.unwrap_hash(&key, hash); + + let mut table = self.table.current(); + + unsafe { + if unlikely(table.info().growth_left.load(Ordering::Relaxed) == 0) { + table = self.expand_by_one(); + } + + match table.find_potential(hash, eq(&key)) { + Ok(_) => false, + Err(slot) => { + slot.insert_new_unchecked(self, key, value, Some(hash)); + true + } + } + } + } +} + +impl<'a, K: Hash + DynSend + Clone, V: DynSend + Clone, S: BuildHasher> Write<'a, K, V, S> { + /// Inserts a new element into the table, and returns a reference to it. + /// + /// This does not check if the given element already exists in the table. + #[inline] + pub fn insert_new(&mut self, key: K, value: V, hash: Option) -> (&'a K, &'a V) { + let hash = self.table.unwrap_hash(&key, hash); + + let mut table = self.table.current(); + + unsafe { + if unlikely(table.info().growth_left.load(Ordering::Relaxed) == 0) { + table = self.expand_by_one(); + } + + let index = table.info().find_insert_slot(hash); + + let bucket = table.bucket(index); + bucket.write((key, value)); + + table.info().record_item_insert_at(index, hash); + + bucket.as_pair_ref() + } + } + + /// Reserve room for one more element. + #[inline] + pub fn reserve_one(&mut self) { + let table = self.table.current(); + + if unlikely(unsafe { table.info().growth_left.load(Ordering::Relaxed) } == 0) { + self.expand_by_one(); + } + } + + #[cold] + #[inline(never)] + fn expand_by_one(&mut self) -> TableRef<(K, V)> { + self.expand_by(1) + } + + /// Out-of-line slow path for `reserve` and `try_reserve`. + fn expand_by(&mut self, additional: usize) -> TableRef<(K, V)> { + let table = self.table.current(); + + // Avoid `Option::ok_or_else` because it bloats LLVM IR. + let new_items = match unsafe { table.info() }.items().checked_add(additional) { + Some(new_items) => new_items, + None => panic!("capacity overflow"), + }; + + let full_capacity = bucket_mask_to_capacity(unsafe { table.info().bucket_mask }); + + let new_capacity = usize::max(new_items, full_capacity + 1); + + unsafe { + debug_assert!(table.info().items() <= new_capacity); + } + + let buckets = capacity_to_buckets(new_capacity).expect("capacity overflow"); + + let new_table = unsafe { table.clone_table(&self.table.hash_builder, buckets, hasher) }; + + self.replace_table(new_table); + + new_table + } +} + +impl Write<'_, K, V, S> { + fn replace_table(&mut self, new_table: TableRef<(K, V)>) { + let table = self.table.current(); + + self.table.current.store(new_table.data.as_ptr(), Ordering::Release); + + let destroy = Arc::new(DestroyTable { table, lock: Mutex::new(false) }); + + unsafe { + (*self.table.old.get()).push(destroy.clone()); + + collect::defer_unchecked(move || destroy.run()); + } + } + + /// Replaces the content of the table with the content of the iterator. + /// All the elements must be unique. + /// `capacity` specifies the new capacity if it's greater than the length of the iterator. + #[inline] + pub fn replace>(&mut self, iter: I, capacity: usize) { + let iter = iter.into_iter(); + + let table = if let Some(max) = iter.size_hint().1 { + TableRef::from_maybe_empty_iter::<_, _, _, true>( + iter, + max, + capacity, + &self.table.hash_builder, + hasher, + ) + } else { + let elements: Vec<_> = iter.collect(); + let len = elements.len(); + TableRef::from_maybe_empty_iter::<_, _, _, false>( + elements.into_iter(), + len, + capacity, + &self.table.hash_builder, + hasher, + ) + }; + + self.replace_table(table); + } +} + +impl + FromIterator<(K, V)> for SyncTable +{ + fn from_iter>(iter: I) -> Self { + let iter = iter.into_iter(); + let mut map = Self::new_with(S::default(), iter.size_hint().0); + { + let mut write = map.write(); + iter.for_each(|(k, v)| { + write.insert(k, v, None); + }); + } + map + } +} + +/// Represents where a value would be if inserted. +/// +/// Created by calling `get_potential` on [Read]. All methods on this type takes a table handle +/// and this must be a handle to the same table `get_potential` was called on. Operations also must +/// be on the same element as given to `get_potential`. The operations have a fast path for when +/// the element is still missing. +#[derive(Copy, Clone)] +pub struct PotentialSlot<'a> { + table_info: &'a TableInfo, + index: usize, +} + +impl<'a> PotentialSlot<'a> { + #[inline] + unsafe fn is_empty(self, table: TableRef) -> bool { + unsafe { + let table_info = table.info() as *const TableInfo; + let index = self.index; + + // Check that we are still looking at the same table, + // otherwise our index could be out of date due to expansion + // or a `replace` call. + table_info == (self.table_info as *const TableInfo) + && *self.table_info.ctrl(index) == EMPTY + } + } + + /// Gets a reference to an element in the table. + #[inline] + pub fn get<'b, Q, K, V, S: BuildHasher>( + self, + table: Read<'b, K, V, S>, + key: &Q, + hash: Option, + ) -> Option<(&'b K, &'b V)> + where + K: Borrow, + Q: ?Sized + Eq + Hash, + { + unsafe { + if likely(self.is_empty(table.table.current())) { + return None; + } + } + + cold_path(|| table.get(key, hash)) + } + + /// Returns a new up-to-date potential slot. + /// This can be useful if there could have been insertions since the slot was derived + /// and you want to use `try_insert_new` or `insert_new_unchecked`. + #[inline] + pub fn refresh( + self, + table: Read<'a, K, V, S>, + key: &Q, + hash: Option, + ) -> Result<(&'a K, &'a V), PotentialSlot<'a>> + where + K: Borrow, + Q: ?Sized + Eq + Hash, + { + unsafe { + if likely(self.is_empty(table.table.current())) { + return Err(self); + } + } + + cold_path(|| table.get_potential(key, hash)) + } + + #[inline] + unsafe fn insert<'b, K, V>( + self, + table: TableRef<(K, V)>, + value: (K, V), + hash: u64, + ) -> (&'b K, &'b V) { + unsafe { + let index = self.index; + let bucket = table.bucket(index); + bucket.write(value); + + table.info().record_item_insert_at(index, hash); + + let pair = bucket.as_ref(); + (&pair.0, &pair.1) + } + } + + /// Inserts a new element into the table, and returns a reference to it. + /// + /// This does not check if the given element already exists in the table. + #[inline] + pub fn insert_new<'b, K: Hash + DynSend + Clone, V: DynSend + Clone, S: BuildHasher>( + self, + table: &mut Write<'b, K, V, S>, + key: K, + value: V, + hash: Option, + ) -> (&'b K, &'b V) { + let hash = table.table.unwrap_hash(&key, hash); + + unsafe { + let table = table.table.current(); + + if likely(self.is_empty(table) && table.info().growth_left.load(Ordering::Relaxed) > 0) + { + debug_assert_eq!(self.index, table.info().find_insert_slot(hash)); + + return self.insert(table, (key, value), hash); + } + } + + cold_path(|| table.insert_new(key, value, Some(hash))) + } + + /// Inserts a new element into the table, and returns a reference to it. + /// Returns [None] if the potential slot is taken by other insertions or if + /// there's no spare capacity in the table. + /// + /// This does not check if the given element already exists in the table. + #[inline] + pub fn try_insert_new<'b, K: Hash, V, S: BuildHasher>( + self, + table: &mut Write<'b, K, V, S>, + key: K, + value: V, + hash: Option, + ) -> Option<(&'b K, &'b V)> { + let hash = table.table.unwrap_hash(&key, hash); + + unsafe { + let table = table.table.current(); + + if likely(self.is_empty(table) && table.info().growth_left.load(Ordering::Relaxed) > 0) + { + Some(self.insert(table, (key, value), hash)) + } else { + None + } + } + } + + /// Inserts a new element into the table, and returns a reference to it. + /// + /// This does not check if the given element already exists in the table. + /// + /// # Safety + /// Derived refers here to either a value returned by `get_potential` or by a `refresh` call. + /// + /// The following conditions must hold for this function to be safe: + /// - `table` must be the same table that `self` is derived from. + /// - `hash`, `key` and `value` must match the value used when `self` was derived. + /// - There must not have been any insertions or `replace` calls to the table since `self` + /// was derived. + #[inline] + pub unsafe fn insert_new_unchecked<'b, K: Hash, V, S: BuildHasher>( + self, + table: &mut Write<'b, K, V, S>, + key: K, + value: V, + hash: Option, + ) -> (&'b K, &'b V) { + unsafe { + let hash = table.table.unwrap_hash(&key, hash); + + let table = table.table.current(); + + debug_assert!(self.is_empty(table)); + debug_assert!(table.info().growth_left.load(Ordering::Relaxed) > 0); + + self.insert(table, (key, value), hash) + } + } +} + +/// An iterator over the entries of a `HashMap`. +/// +/// This `struct` is created by the [`iter`] method on [`HashMap`]. See its +/// documentation for more. +/// +/// [`iter`]: struct.HashMap.html#method.iter +/// [`HashMap`]: struct.HashMap.html +pub struct Iter<'a, K, V> { + inner: RawIterRange<(K, V)>, + marker: PhantomData<&'a (K, V)>, +} + +impl Clone for Iter<'_, K, V> { + #[inline] + fn clone(&self) -> Self { + Iter { inner: self.inner.clone(), marker: PhantomData } + } +} + +impl fmt::Debug for Iter<'_, K, V> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_list().entries(self.clone()).finish() + } +} + +impl<'a, K, V> Iterator for Iter<'a, K, V> { + type Item = (&'a K, &'a V); + + #[inline] + fn next(&mut self) -> Option<(&'a K, &'a V)> { + self.inner.next().map(|bucket| unsafe { bucket.as_pair_ref() }) + } +} + +impl FusedIterator for Iter<'_, K, V> {} + +/// Returns the maximum effective capacity for the given bucket mask, taking +/// the maximum load factor into account. +#[inline] +fn bucket_mask_to_capacity(bucket_mask: usize) -> usize { + if bucket_mask < 8 { + // For tables with 1/2/4/8 buckets, we always reserve one empty slot. + // Keep in mind that the bucket mask is one less than the bucket count. + bucket_mask + } else { + // For larger tables we reserve 12.5% of the slots as empty. + ((bucket_mask + 1) / 8) * 7 + } +} + +/// Returns the number of buckets needed to hold the given number of items, +/// taking the maximum load factor into account. +/// +/// Returns `None` if an overflow occurs. +// Workaround for emscripten bug emscripten-core/emscripten-fastcomp#258 +#[cfg_attr(target_os = "emscripten", inline(never))] +#[cfg_attr(not(target_os = "emscripten"), inline)] +fn capacity_to_buckets(cap: usize) -> Option { + debug_assert_ne!(cap, 0); + + // For small tables we require at least 1 empty bucket so that lookups are + // guaranteed to terminate if an element doesn't exist in the table. + let result = if cap < 8 { + // We don't bother with a table size of 2 buckets since that can only + // hold a single element. Instead we skip directly to a 4 bucket table + // which can hold 3 elements. + if cap < 4 { 4 } else { 8 } + } else { + // Otherwise require 1/8 buckets to be empty (87.5% load) + // + // Be careful when modifying this, calculate_layout relies on the + // overflow check here. + let adjusted_cap = cap.checked_mul(8)? / 7; + + // Any overflows will have been caught by the checked_mul. Also, any + // rounding errors from the division above will be cleaned up by + // next_power_of_two (which can't overflow because of the previous divison). + adjusted_cap.next_power_of_two() + }; + + // Have at least the number of buckets required to fill a group. + // This avoids logic to deal with control bytes not associated with a bucket + // when batch processing a group. + Some(usize::max(result, Group::WIDTH)) +} + +/// Primary hash function, used to select the initial bucket to probe from. +#[inline] +#[allow(clippy::cast_possible_truncation)] +fn h1(hash: u64) -> usize { + // On 32-bit platforms we simply ignore the higher hash bits. + hash as usize +} + +/// Secondary hash function, saved in the low 7 bits of the control byte. +#[inline] +#[allow(clippy::cast_possible_truncation)] +fn h2(hash: u64) -> u8 { + // Grab the top 7 bits of the hash. While the hash is normally a full 64-bit + // value, some hash functions (such as FxHash) produce a usize result + // instead, which means that the top 32 bits are 0 on 32-bit platforms. + let hash_len = usize::min(mem::size_of::(), mem::size_of::()); + let top7 = hash >> (hash_len * 8 - 7); + (top7 & 0x7f) as u8 // truncation +} + +/// Control byte value for an empty bucket. +const EMPTY: u8 = 0b1111_1111; + +/// Control byte value for a deleted bucket. +const DELETED: u8 = 0b1000_0000; + +/// Checks whether a control byte represents a full bucket (top bit is clear). +#[inline] +fn is_full(ctrl: u8) -> bool { + ctrl & 0x80 == 0 +} + +/// Probe sequence based on triangular numbers, which is guaranteed (since our +/// table size is a power of two) to visit every group of elements exactly once. +/// +/// A triangular probe has us jump by 1 more group every time. So first we +/// jump by 1 group (meaning we just continue our linear scan), then 2 groups +/// (skipping over 1 group), then 3 groups (skipping over 2 groups), and so on. +/// +/// Proof that the probe will visit every group in the table: +/// +struct ProbeSeq { + pos: usize, + stride: usize, +} + +impl ProbeSeq { + #[inline] + fn move_next(&mut self, bucket_mask: usize) { + // We should have found an empty bucket by now and ended the probe. + debug_assert!(self.stride <= bucket_mask, "Went past end of probe sequence"); + + self.stride += Group::WIDTH; + self.pos += self.stride; + self.pos &= bucket_mask; + } +} + +/// Iterator over a sub-range of a table. +struct RawIterRange { + // Mask of full buckets in the current group. Bits are cleared from this + // mask as each element is processed. + current_group: BitMask, + + // Pointer to the buckets for the current group. + data: Bucket, + + // Pointer to the next group of control bytes, + // Must be aligned to the group size. + next_ctrl: *const u8, + + // Pointer one past the last control byte of this range. + end: *const u8, +} + +impl RawIterRange { + /// Returns a `RawIterRange` covering a subset of a table. + /// + /// The control byte address must be aligned to the group size. + #[inline] + unsafe fn new(ctrl: *const u8, data: Bucket, len: usize) -> Self { + unsafe { + debug_assert_ne!(len, 0); + debug_assert_eq!(ctrl as usize % Group::WIDTH, 0); + let end = ctrl.add(len); + + // Load the first group and advance ctrl to point to the next group + let current_group = Group::load_aligned(ctrl).match_full(); + let next_ctrl = ctrl.add(Group::WIDTH); + + Self { current_group, data, next_ctrl, end } + } + } +} + +// We make raw iterators unconditionally DynSend and Sync, and let the PhantomData +// in the actual iterator implementations determine the real DynSend/Sync bounds. +unsafe impl DynSend for RawIterRange {} +unsafe impl Sync for RawIterRange {} + +impl Clone for RawIterRange { + #[inline] + fn clone(&self) -> Self { + Self { + data: self.data.clone(), + next_ctrl: self.next_ctrl, + current_group: self.current_group, + end: self.end, + } + } +} + +impl Iterator for RawIterRange { + type Item = Bucket; + + #[inline] + fn next(&mut self) -> Option> { + unsafe { + loop { + if let Some(index) = self.current_group.lowest_set_bit() { + self.current_group = self.current_group.remove_lowest_bit(); + return Some(self.data.next_n(index)); + } + + if self.next_ctrl >= self.end { + return None; + } + + // We might read past self.end up to the next group boundary, + // but this is fine because it only occurs on tables smaller + // than the group size where the trailing control bytes are all + // EMPTY. On larger tables self.end is guaranteed to be aligned + // to the group size (since tables are power-of-two sized). + self.current_group = Group::load_aligned(self.next_ctrl).match_full(); + self.data = self.data.next_n(Group::WIDTH); + self.next_ctrl = self.next_ctrl.add(Group::WIDTH); + } + } + } +} + +impl FusedIterator for RawIterRange {} + +/// Get a suitable shard index from a hash when sharding the hash table. +/// +/// `shards` must be a power of 2. +#[inline] +pub fn shard_index_by_hash(hash: u64, shards: usize) -> usize { + assert!(shards.is_power_of_two()); + let shard_bits = shards - 1; + + let hash_len = mem::size_of::(); + // Ignore the top 7 bits as hashbrown uses these and get the next SHARD_BITS highest bits. + // hashbrown also uses the lowest bits, so we can't use those + let bits = (hash >> (hash_len * 8 - 7 - shard_bits)) as usize; + bits % shards +} diff --git a/compiler/rustc_data_structures/src/sync/table/sync_table/code.rs b/compiler/rustc_data_structures/src/sync/table/sync_table/code.rs new file mode 100644 index 000000000000..7c64f65aad4a --- /dev/null +++ b/compiler/rustc_data_structures/src/sync/table/sync_table/code.rs @@ -0,0 +1,169 @@ +#![cfg(code)] + +use std::collections::HashMap; + +use super::{PotentialSlot, Read, SyncTable, TableRef, Write}; +use crate::collect::Pin; + +#[no_mangle] +unsafe fn bucket_index(t: TableRef, i: usize) -> *mut usize { + t.bucket(i).as_ptr() +} + +#[no_mangle] +unsafe fn ctrl_index(t: TableRef, i: usize) -> *mut u8 { + t.info().ctrl(i) +} + +#[no_mangle] +unsafe fn first_bucket(t: TableRef) -> *mut usize { + t.bucket_before_first() +} + +#[no_mangle] +unsafe fn last_bucket(t: TableRef) -> *mut usize { + t.bucket_past_last() +} + +#[no_mangle] +fn alloc_test(bucket_count: usize) -> TableRef { + TableRef::allocate(bucket_count) +} + +#[no_mangle] +fn len_test(table: &mut SyncTable) { + table.write().read().len(); +} + +#[no_mangle] +fn get_potential_test(table: &SyncTable, pin: Pin<'_>) -> Result { + table.read(pin).get_potential(5, |a| *a == 5).map(|b| *b) +} + +#[no_mangle] +fn potential_get_test(potential: PotentialSlot, table: Read<'_, usize>) -> Option { + potential.get(table, 5, |a| *a == 5).map(|b| *b) +} + +#[no_mangle] +fn potential_insert(potential: PotentialSlot, mut table: Write<'_, usize>) { + potential.insert_new(&mut table, 5, 5, |_, h| *h as u64); +} + +#[no_mangle] +unsafe fn potential_insert_opt(mut table: Write<'_, usize>, index: usize) { + let potential = PotentialSlot { bucket_mask: table.table.current().info().bucket_mask, index }; + potential.insert_new(&mut table, 5, 5, |_, h| *h as u64); +} + +#[no_mangle] +fn find_test(table: &SyncTable, val: usize, hash: u64) -> Option { + unsafe { table.find(hash, |a| *a == val).map(|b| *b.as_ref()) } +} + +#[no_mangle] +fn insert_test(table: &SyncTable) { + table.lock().insert_new(5, 5, |_, a| *a); +} + +#[no_mangle] +fn map_insert_test(table: &mut SyncTable<(u64, u64)>) { + table.write().map_insert(5, 5); +} + +#[no_mangle] +unsafe fn insert_test2(table: &SyncTable) { + table.unsafe_write().insert_new(5, 5, |_, a| *a); +} + +#[no_mangle] +unsafe fn insert_slot_test(table: TableRef, hash: u64) -> usize { + table.info().find_insert_slot(hash) +} + +#[no_mangle] +fn find_test2(table: &HashMap) -> Option { + table.get_key_value(&5).map(|b| *b.0) +} + +#[no_mangle] +fn intern_triple_test(table: &SyncTable<(u64, u64)>, k: u64, v: u64, pin: Pin<'_>) -> u64 { + let hash = table.hash_any(&k); + match table.read(pin).get(hash, |v| v.0 == k) { + Some(v) => return v.1, + None => (), + }; + + let mut write = table.lock(); + match write.read().get(hash, |v| v.0 == k) { + Some(v) => v.1, + None => { + write.insert_new(hash, (k, v), SyncTable::hasher); + v + } + } +} + +#[no_mangle] +fn intern_try_test(table: &SyncTable<(u64, u64)>, k: u64, v: u64, pin: Pin<'_>) -> u64 { + let hash = table.hash_any(&k); + let p = match table.read(pin).get_potential(hash, |v| v.0 == k) { + Ok(v) => return v.1, + Err(p) => p, + }; + + let mut write = table.lock(); + match p.get(write.read(), hash, |v| v.0 == k) { + Some(v) => v.1, + None => { + p.try_insert_new(&mut write, hash, (k, v)); + v + } + } +} + +#[no_mangle] +fn intern_test(table: &SyncTable<(u64, u64)>, k: u64, v: u64, pin: Pin<'_>) -> u64 { + let hash = table.hash_any(&k); + let p = match table.read(pin).get_potential(hash, |v| v.0 == k) { + Ok(v) => return v.1, + Err(p) => p, + }; + + let mut write = table.lock(); + match p.get(write.read(), hash, |v| v.0 == k) { + Some(v) => v.1, + None => { + p.insert_new(&mut write, hash, (k, v), SyncTable::hasher); + v + } + } +} + +#[no_mangle] +fn intern_refresh_test( + table: &SyncTable<(u64, u64)>, + k: u64, + v: u64, + hash: u64, + pin: Pin<'_>, +) -> u64 { + let p = match table.read(pin).get_potential(hash, |v| v.0 == k) { + Ok(v) => return v.1, + Err(p) => p, + }; + + let mut write = table.lock(); + + write.reserve_one(SyncTable::hasher); + + let p = p.refresh(write.read(), hash, |v| v.0 == k); + + match p { + Ok(v) => v.1, + Err(p) => { + p.try_insert_new(&mut write, hash, (k, v)); + v + } + } +} diff --git a/compiler/rustc_data_structures/src/sync/table/sync_table/tests.rs b/compiler/rustc_data_structures/src/sync/table/sync_table/tests.rs new file mode 100644 index 000000000000..f8b13798ce15 --- /dev/null +++ b/compiler/rustc_data_structures/src/sync/table/sync_table/tests.rs @@ -0,0 +1,298 @@ +#![cfg(test)] + +use std::collections::HashMap; +use std::collections::hash_map::{DefaultHasher, RandomState}; +use std::hash::Hasher; + +use super::SyncTable; +use crate::collect::{Pin, pin, release}; + +#[test] +fn high_align() { + #[repr(align(64))] + #[derive(Clone, Eq, PartialEq, Hash)] + struct A(u64); + + let mut table = SyncTable::new(); + + table.get_mut(&A(1), None); + + table.lock().insert_new(A(1), 1, None); + + release(); +} + +#[test] +fn test_create_capacity_zero() { + let m = SyncTable::new_with(RandomState::new(), 0); + + assert!(m.lock().insert(1, 1, None)); + + assert!(m.lock().read().get(&1, None).is_some()); + assert!(m.lock().read().get(&0, None).is_none()); + + release(); +} + +#[test] +fn test_replace() { + let m = SyncTable::new(); + m.lock().insert(2, 7, None); + m.lock().insert(5, 3, None); + m.lock().replace(vec![(3, 4)], 0); + assert_eq!(*m.lock().read().get(&3, None).unwrap().1, 4); + assert_eq!(m.lock().read().get(&2, None), None); + assert_eq!(m.lock().read().get(&5, None), None); + m.lock().replace(vec![], 0); + assert_eq!(m.lock().read().get(&3, None), None); + assert_eq!(m.lock().read().get(&2, None), None); + assert_eq!(m.lock().read().get(&5, None), None); + release(); +} + +#[test] +fn test_remove() { + let m = SyncTable::new(); + m.lock().insert(2, 7, None); + m.lock().insert(5, 3, None); + m.lock().remove(&2, None); + m.lock().remove(&5, None); + assert_eq!(m.lock().read().get(&2, None), None); + assert_eq!(m.lock().read().get(&5, None), None); + assert_eq!(m.lock().read().len(), 0); + release(); +} + +#[test] +fn test_insert() { + let m = SyncTable::new(); + assert_eq!(m.lock().read().len(), 0); + assert!(m.lock().insert(1, 2, None)); + assert_eq!(m.lock().read().len(), 1); + assert!(m.lock().insert(2, 4, None)); + assert_eq!(m.lock().read().len(), 2); + assert_eq!(*m.lock().read().get(&1, None).unwrap().1, 2); + assert_eq!(*m.lock().read().get(&2, None).unwrap().1, 4); + + release(); +} + +#[test] +fn test_iter() { + let m = SyncTable::new(); + assert!(m.lock().insert(1, 2, None)); + assert!(m.lock().insert(5, 3, None)); + assert!(m.lock().insert(2, 4, None)); + assert!(m.lock().insert(9, 4, None)); + + pin(|pin| { + let mut v: Vec<(i32, i32)> = m.read(pin).iter().map(|i| (*i.0, *i.1)).collect(); + v.sort_by_key(|k| k.0); + + assert_eq!(v, vec![(1, 2), (2, 4), (5, 3), (9, 4)]); + }); + + release(); +} + +#[test] +fn test_insert_conflicts() { + let m = SyncTable::new_with(RandomState::default(), 4); + assert!(m.lock().insert(1, 2, None)); + assert!(m.lock().insert(5, 3, None)); + assert!(m.lock().insert(9, 4, None)); + assert_eq!(*m.lock().read().get(&9, None).unwrap().1, 4); + assert_eq!(*m.lock().read().get(&5, None).unwrap().1, 3); + assert_eq!(*m.lock().read().get(&1, None).unwrap().1, 2); + + release(); +} + +#[test] +fn test_expand() { + let m = SyncTable::new(); + + assert_eq!(m.lock().read().len(), 0); + + let mut i = 0; + let old_raw_cap = unsafe { m.current().info().buckets() }; + while old_raw_cap == unsafe { m.current().info().buckets() } { + m.lock().insert(i, i, None); + i += 1; + } + + assert_eq!(m.lock().read().len(), i); + + release(); +} + +#[test] +fn test_find() { + let m = SyncTable::new(); + assert!(m.lock().read().get(&1, None).is_none()); + m.lock().insert(1, 2, None); + match m.lock().read().get(&1, None) { + None => panic!(), + Some(v) => assert_eq!(*v.1, 2), + } + + release(); +} + +#[test] +fn test_capacity_not_less_than_len() { + let a: SyncTable = SyncTable::new(); + let mut item = 0; + + for _ in 0..116 { + a.lock().insert(item, 0, None); + item += 1; + } + + pin(|pin| { + assert!(a.read(pin).capacity() > a.read(pin).len()); + + let free = a.read(pin).capacity() - a.read(pin).len(); + for _ in 0..free { + a.lock().insert(item, 0, None); + item += 1; + } + + assert_eq!(a.read(pin).len(), a.read(pin).capacity()); + + // Insert at capacity should cause allocation. + a.lock().insert(item, 0, None); + assert!(a.read(pin).capacity() > a.read(pin).len()); + }); + + release(); +} + +#[test] +fn rehash() { + let table = SyncTable::new(); + for i in 0..100 { + table.lock().insert_new(i, (), None); + } + + pin(|pin| { + for i in 0..100 { + assert_eq!(table.read(pin).get(&i, None).map(|b| *b.0), Some(i)); + assert!(table.read(pin).get(&(i + 100), None).is_none()); + } + }); + + release(); +} + +const INTERN_SIZE: u64 = if cfg!(miri) { 35 } else { 26334 }; +const HIT_RATE: u64 = 84; + +fn assert_equal(a: &mut SyncTable, b: &HashMap) { + let mut ca: Vec<_> = b.iter().map(|v| (*v.0, *v.1)).collect(); + ca.sort(); + let mut cb: Vec<_> = a.write().read().iter().map(|v| (*v.0, *v.1)).collect(); + cb.sort(); + assert_eq!(ca, cb); +} + +fn test_interning(intern: impl Fn(&SyncTable, u64, u64, Pin<'_>) -> bool) { + let mut control = HashMap::new(); + let mut test = SyncTable::new(); + + for i in 0..INTERN_SIZE { + let mut s = DefaultHasher::new(); + s.write_u64(i); + let s = s.finish(); + if s % 100 > (100 - HIT_RATE) { + test.lock().insert(i, i * 2, None); + control.insert(i, i * 2); + } + } + + assert_equal(&mut test, &control); + + pin(|pin| { + for i in 0..INTERN_SIZE { + assert_eq!(intern(&test, i, i * 2, pin), control.insert(i, i * 2).is_some()) + } + }); + + assert_equal(&mut test, &control); + + release(); +} + +#[test] +fn intern_potential() { + fn intern(table: &SyncTable, k: u64, v: u64, pin: Pin<'_>) -> bool { + let hash = table.hash_key(&k); + let p = match table.read(pin).get_potential(&k, Some(hash)) { + Ok(_) => return true, + Err(p) => p, + }; + + let mut write = table.lock(); + match p.get(write.read(), &k, Some(hash)) { + Some(v) => { + v.1; + true + } + None => { + p.insert_new(&mut write, k, v, Some(hash)); + false + } + } + } + + test_interning(intern); +} + +#[test] +fn intern_get_insert() { + fn intern(table: &SyncTable, k: u64, v: u64, pin: Pin<'_>) -> bool { + let hash = table.hash_key(&k); + match table.read(pin).get(&k, Some(hash)) { + Some(_) => return true, + None => (), + }; + + let mut write = table.lock(); + match write.read().get(&k, Some(hash)) { + Some(_) => true, + None => { + write.insert_new(k, v, Some(hash)); + false + } + } + } + + test_interning(intern); +} + +#[test] +fn intern_potential_try() { + fn intern(table: &SyncTable, k: u64, v: u64, pin: Pin<'_>) -> bool { + let hash = table.hash_key(&k); + let p = match table.read(pin).get_potential(&k, Some(hash)) { + Ok(_) => return true, + Err(p) => p, + }; + + let mut write = table.lock(); + + write.reserve_one(); + + let p = p.refresh(table.read(pin), &k, Some(hash)); + + match p { + Ok(_) => true, + Err(p) => { + p.try_insert_new(&mut write, k, v, Some(hash)).unwrap(); + false + } + } + } + + test_interning(intern); +} diff --git a/compiler/rustc_data_structures/src/sync/table/util.rs b/compiler/rustc_data_structures/src/sync/table/util.rs new file mode 100644 index 000000000000..3b2645d2043f --- /dev/null +++ b/compiler/rustc_data_structures/src/sync/table/util.rs @@ -0,0 +1,19 @@ +use core::hash::{BuildHasher, Hash}; + +#[inline(never)] +#[cold] +pub(crate) fn cold_path R, R>(f: F) -> R { + f() +} + +#[inline] +pub(crate) fn make_insert_hash(hash_builder: &S, val: &K) -> u64 +where + K: Hash, + S: BuildHasher, +{ + use core::hash::Hasher; + let mut state = hash_builder.build_hasher(); + val.hash(&mut state); + state.finish() +} diff --git a/compiler/rustc_middle/Cargo.toml b/compiler/rustc_middle/Cargo.toml index 43c1af642dd5..a74e511dfd15 100644 --- a/compiler/rustc_middle/Cargo.toml +++ b/compiler/rustc_middle/Cargo.toml @@ -9,6 +9,7 @@ bitflags = "2.4.1" either = "1.5.0" gsgdt = "0.1.2" polonius-engine = "0.13.0" +rustc-hash = "2.0.0" rustc-rayon-core = { version = "0.5.0" } rustc_abi = { path = "../rustc_abi" } rustc_apfloat = "0.2.0" diff --git a/compiler/rustc_middle/src/ty/context.rs b/compiler/rustc_middle/src/ty/context.rs index edba2a2530f6..94853bc6760d 100644 --- a/compiler/rustc_middle/src/ty/context.rs +++ b/compiler/rustc_middle/src/ty/context.rs @@ -7,7 +7,7 @@ pub mod tls; use std::assert_matches::{assert_matches, debug_assert_matches}; use std::borrow::Borrow; use std::cmp::Ordering; -use std::hash::{Hash, Hasher}; +use std::hash::{BuildHasher, Hash, Hasher}; use std::marker::PhantomData; use std::ops::{Bound, Deref}; use std::sync::{Arc, OnceLock}; @@ -20,16 +20,18 @@ use rustc_data_structures::fingerprint::Fingerprint; use rustc_data_structures::fx::FxHashMap; use rustc_data_structures::intern::Interned; use rustc_data_structures::profiling::SelfProfilerRef; -use rustc_data_structures::sharded::{IntoPointer, ShardedHashMap}; +use rustc_data_structures::sharded::IntoPointer; use rustc_data_structures::stable_hasher::{HashStable, StableHasher}; use rustc_data_structures::steal::Steal; +use rustc_data_structures::sync::collect::pin; use rustc_data_structures::sync::{ - self, DynSend, DynSync, FreezeReadGuard, Lock, RwLock, WorkerLocal, + self, DynSend, DynSync, FreezeReadGuard, Lock, RwLock, SyncTable, WorkerLocal, }; use rustc_data_structures::unord::UnordSet; use rustc_errors::{ Applicability, Diag, DiagCtxtHandle, ErrorGuaranteed, LintDiagnostic, MultiSpan, }; +use rustc_hash::FxBuildHasher; use rustc_hir::def::{CtorKind, DefKind}; use rustc_hir::def_id::{CrateNum, DefId, LOCAL_CRATE, LocalDefId}; use rustc_hir::definitions::Definitions; @@ -780,7 +782,106 @@ impl<'tcx> rustc_type_ir::inherent::Span> for Span { } } -type InternedSet<'tcx, T> = ShardedHashMap, ()>; +trait SyncInsertTableExt { + fn len(&self) -> usize; + fn with_capacity(cap: usize) -> Self; + fn contains_pointer_to(&self, value: &T) -> bool + where + K: IntoPointer; + + fn intern_ref(&self, value: &Q, make: impl FnOnce() -> K) -> K + where + K: Borrow, + Q: Hash + Eq; + + fn intern(&self, value: Q, make: impl FnOnce(Q) -> K) -> K + where + K: Borrow, + Q: Hash + Eq; +} + +impl SyncInsertTableExt for SyncTable { + fn len(&self) -> usize { + pin(|pin| self.read(pin).len()) + } + + fn with_capacity(cap: usize) -> Self { + Self::new_with(FxBuildHasher::default(), cap) + } + + fn contains_pointer_to(&self, value: &T) -> bool + where + K: IntoPointer, + { + pin(|pin| { + let mut state = self.hasher().build_hasher(); + value.hash(&mut state); + let hash = state.finish(); + let value = value.into_pointer(); + self.read(pin).get_from_hash(hash, |entry| entry.into_pointer() == value).is_some() + }) + } + + #[inline] + fn intern_ref(&self, value: &Q, make: impl FnOnce() -> K) -> K + where + K: Borrow, + Q: Hash + Eq, + { + pin(|pin| { + let hash = self.hash_key(value); + + let entry = self.read(pin).get(value, Some(hash)); + if let Some(entry) = entry { + return *entry.0; + } + + let mut write = self.lock(); + + let entry = self.read(pin).get(value, Some(hash)); + if let Some(entry) = entry { + return *entry.0; + } + + let result = make(); + + write.insert_new(result, (), Some(hash)); + + result + }) + } + + #[inline] + fn intern(&self, value: Q, make: impl FnOnce(Q) -> K) -> K + where + K: Borrow, + Q: Hash + Eq, + { + pin(|pin| { + let hash = self.hash_key(&value); + + let entry = self.read(pin).get(&value, Some(hash)); + if let Some(entry) = entry { + return *entry.0; + } + + let mut write = self.lock(); + + let entry = self.read(pin).get(&value, Some(hash)); + if let Some(entry) = entry { + return *entry.0; + } + + let result = make(value); + + write.insert_new(result, (), Some(hash)); + + result + }) + } +} + +type InternedSet<'tcx, T> = SyncTable, (), FxBuildHasher>; pub struct CtxtInterners<'tcx> { /// The arena that types, regions, etc. are allocated from. @@ -2315,6 +2416,7 @@ macro_rules! sty_debug_print { mod inner { use crate::ty::{self, TyCtxt}; use crate::ty::context::InternedInSet; + use rustc_data_structures::sync::collect::pin; #[derive(Copy, Clone)] struct DebugStat { @@ -2335,9 +2437,9 @@ macro_rules! sty_debug_print { }; $(let mut $variant = total;)* - for shard in tcx.interners.type_.lock_shards() { - let types = shard.keys(); - for &InternedInSet(t) in types { + pin(|pin| { + let types = tcx.interners.type_.read(pin); + for (&InternedInSet(t), _) in types.iter() { let variant = match t.internee { ty::Bool | ty::Char | ty::Int(..) | ty::Uint(..) | ty::Float(..) | ty::Str | ty::Never => continue, @@ -2355,7 +2457,7 @@ macro_rules! sty_debug_print { if ct { total.ct_infer += 1; variant.ct_infer += 1 } if lt && ty && ct { total.all_infer += 1; variant.all_infer += 1 } } - } + }); writeln!(fmt, "Ty interner total ty lt ct all")?; $(writeln!(fmt, " {:18}: {uses:6} {usespc:4.1}%, \ {ty:4.1}% {lt:5.1}% {ct:4.1}% {all:4.1}%", diff --git a/compiler/rustc_query_system/src/query/caches.rs b/compiler/rustc_query_system/src/query/caches.rs index 3b47e7eba0ff..a82ef3a4d603 100644 --- a/compiler/rustc_query_system/src/query/caches.rs +++ b/compiler/rustc_query_system/src/query/caches.rs @@ -1,9 +1,10 @@ use std::fmt::Debug; -use std::hash::Hash; +use std::hash::{BuildHasherDefault, Hash}; use std::sync::OnceLock; -use rustc_data_structures::fx::FxHashMap; -use rustc_data_structures::sharded::{self, Sharded}; +use rustc_data_structures::fx::FxHasher; +use rustc_data_structures::sync::collect::{self, pin}; +use rustc_data_structures::sync::{DynSend, SyncTable}; pub use rustc_data_structures::vec_cache::VecCache; use rustc_hir::def_id::LOCAL_CRATE; use rustc_index::Idx; @@ -36,46 +37,43 @@ pub trait QueryCache: Sized { /// In-memory cache for queries whose keys aren't suitable for any of the /// more specialized kinds of cache. Backed by a sharded hashmap. pub struct DefaultCache { - cache: Sharded>, + cache: SyncTable>, } impl Default for DefaultCache { fn default() -> Self { - DefaultCache { cache: Default::default() } + DefaultCache { cache: SyncTable::new_with(BuildHasherDefault::default(), 0) } } } impl QueryCache for DefaultCache where - K: Eq + Hash + Copy + Debug, - V: Copy, + K: Eq + Hash + Copy + Debug + DynSend, + V: Copy + DynSend, { type Key = K; type Value = V; #[inline(always)] fn lookup(&self, key: &K) -> Option<(V, DepNodeIndex)> { - let key_hash = sharded::make_hash(key); - let lock = self.cache.lock_shard_by_hash(key_hash); - let result = lock.raw_entry().from_key_hashed_nocheck(key_hash, key); - - if let Some((_, value)) = result { Some(*value) } else { None } + pin(|pin| { + let result = self.cache.read(pin).get(key, None); + if let Some((_, value)) = result { Some(*value) } else { None } + }) } #[inline] fn complete(&self, key: K, value: V, index: DepNodeIndex) { - let mut lock = self.cache.lock_shard_by_value(&key); - // We may be overwriting another value. This is all right, since the dep-graph - // will check that the fingerprint matches. - lock.insert(key, (value, index)); + self.cache.lock().insert_new(key, (value, index), None); + collect::collect(); } fn iter(&self, f: &mut dyn FnMut(&Self::Key, &Self::Value, DepNodeIndex)) { - for shard in self.cache.lock_shards() { - for (k, v) in shard.iter() { + pin(|pin| { + for (k, v) in self.cache.read(pin).iter() { f(k, &v.0, v.1); } - } + }) } } @@ -134,7 +132,7 @@ impl Default for DefIdCache { impl QueryCache for DefIdCache where - V: Copy, + V: Copy + DynSend, { type Key = DefId; type Value = V;