From 09f5b119aebfea0adfdda9998f57319be180946b Mon Sep 17 00:00:00 2001 From: greg7mdp Date: Sun, 19 Nov 2023 18:16:40 -0500 Subject: [PATCH] Add support for Windows `phmap::srwlock` --- examples/phmap/lazy_emplace_l.cpp | 11 +- include/gtl/phmap.hpp | 252 +++++++++++++++++------------- 2 files changed, 147 insertions(+), 116 deletions(-) diff --git a/examples/phmap/lazy_emplace_l.cpp b/examples/phmap/lazy_emplace_l.cpp index 7119b0c..71f746d 100644 --- a/examples/phmap/lazy_emplace_l.cpp +++ b/examples/phmap/lazy_emplace_l.cpp @@ -9,22 +9,13 @@ #include #include -class srwlock { - SRWLOCK _lock; - -public: - srwlock() { InitializeSRWLock(&_lock); } - void lock() { AcquireSRWLockExclusive(&_lock); } - void unlock() { ReleaseSRWLockExclusive(&_lock); } -}; - using Map = gtl::parallel_flat_hash_map, gtl::priv::hash_default_eq, std::allocator>, 8, - srwlock>; + gtl::srwlock>; class Dict { Map m_stringsMap; diff --git a/include/gtl/phmap.hpp b/include/gtl/phmap.hpp index 69494dd..8a73c90 100644 --- a/include/gtl/phmap.hpp +++ b/include/gtl/phmap.hpp @@ -1181,9 +1181,9 @@ class raw_hash_set { using node_type = node_handle, Alloc>; using insert_return_type = InsertReturnType; - raw_hash_set() noexcept(std::is_nothrow_default_constructible_v && - std::is_nothrow_default_constructible_v && - std::is_nothrow_default_constructible_v) {} + raw_hash_set() noexcept( + std::is_nothrow_default_constructible_v&& std::is_nothrow_default_constructible_v&& + std::is_nothrow_default_constructible_v) {} explicit raw_hash_set(size_t bucket_cnt, const hasher& hashfn = hasher(), @@ -1309,9 +1309,9 @@ class raw_hash_set { growth_left() -= that.size(); } - raw_hash_set(raw_hash_set&& that) noexcept(std::is_nothrow_copy_constructible_v && - std::is_nothrow_copy_constructible_v && - std::is_nothrow_copy_constructible_v) + raw_hash_set(raw_hash_set&& that) noexcept( + std::is_nothrow_copy_constructible_v&& std::is_nothrow_copy_constructible_v&& + std::is_nothrow_copy_constructible_v) : ctrl_(std::exchange(that.ctrl_, EmptyGroup())) , slots_(std::exchange(that.slots_, nullptr)) , size_(std::exchange(that.size_, 0)) @@ -1356,8 +1356,8 @@ class raw_hash_set { } raw_hash_set& operator=(raw_hash_set&& that) noexcept( - std::allocator_traits::is_always_equal::value && std::is_nothrow_move_assignable_v && - std::is_nothrow_move_assignable_v) { + std::allocator_traits::is_always_equal::value&& std::is_nothrow_move_assignable_v&& + std::is_nothrow_move_assignable_v) { // TODO(sbenza): We should only use the operations from the noexcept clause // to make sure we actually adhere to that contract. return move_assign(std::move(that), typename AllocTraits::propagate_on_container_move_assignment()); @@ -1393,8 +1393,7 @@ class raw_hash_set { return; if (capacity_) { if constexpr (!std::is_trivially_destructible::value || - std::is_same::value) - { + std::is_same::value) { // node map or not trivially destructible... we need to iterate and destroy values one by one for (size_t i = 0; i != capacity_; ++i) { if (IsFull(ctrl_[i])) { @@ -1761,8 +1760,7 @@ class raw_hash_set { assert(this != &src); for (auto it = src.begin(), e = src.end(); it != e; ++it) { if (PolicyTraits::apply(InsertSlot{ *this, std::move(*it.slot_) }, PolicyTraits::element(it.slot_)) - .second) - { + .second) { src.erase_meta_only(it); } } @@ -1785,10 +1783,9 @@ class raw_hash_set { return it == end() ? node_type() : extract(const_iterator{ it }); } - void swap(raw_hash_set& that) noexcept(std::is_nothrow_swappable_v && - std::is_nothrow_swappable_v && - (!AllocTraits::propagate_on_container_swap::value || - std::is_nothrow_swappable_v)) { + void swap(raw_hash_set& that) noexcept( + std::is_nothrow_swappable_v&& std::is_nothrow_swappable_v && + (!AllocTraits::propagate_on_container_swap::value || std::is_nothrow_swappable_v)) { using std::swap; swap(ctrl_, that.ctrl_); swap(slots_, that.slots_); @@ -1936,9 +1933,9 @@ class raw_hash_set { } size_t bucket_count() const { return capacity_; } - float load_factor() const { return capacity_ ? static_cast(static_cast(size()) / capacity_) : 0.0f; } - float max_load_factor() const { return 1.0f; } - void max_load_factor(float) { + float load_factor() const { return capacity_ ? static_cast(static_cast(size()) / capacity_) : 0.0f; } + float max_load_factor() const { return 1.0f; } + void max_load_factor(float) { // Does nothing. } @@ -2024,9 +2021,9 @@ class raw_hash_set { offset = prepare_insert(hashval); emplace_at(offset, std::forward(args)...); this->set_ctrl(offset, H2(hashval)); - return {iterator_at(offset), true}; + return { iterator_at(offset), true }; } - return {iterator_at(offset), false}; + return { iterator_at(offset), false }; } struct EmplaceDecomposable { @@ -2111,8 +2108,7 @@ class raw_hash_set { void initialize_slots(size_t new_capacity) { assert(new_capacity); - if (std::is_same_v> && slots_ == nullptr) { - } + if (std::is_same_v> && slots_ == nullptr) {} auto layout = MakeLayout(new_capacity); char* mem = static_cast(Allocate(&alloc_ref(), layout.AllocSize())); @@ -2127,8 +2123,7 @@ class raw_hash_set { return; if constexpr (!std::is_trivially_destructible::value || - std::is_same::value) - { + std::is_same::value) { // node map or not trivially destructible... we need to iterate and destroy values one by one for (size_t i = 0; i != capacity_; ++i) { if (IsFull(ctrl_[i])) { @@ -2309,32 +2304,32 @@ class raw_hash_set { } protected: - template + template size_t _find_key(const K& key, size_t hashval) { auto seq = probe(hashval); while (true) { - Group g{ctrl_ + seq.offset()}; + Group g{ ctrl_ + seq.offset() }; for (uint32_t i : g.Match((h2_t)H2(hashval))) { - if (GTL_PREDICT_TRUE(PolicyTraits::apply( - EqualElement{key, eq_ref()}, - PolicyTraits::element(slots_ + seq.offset((size_t)i))))) + if (GTL_PREDICT_TRUE(PolicyTraits::apply(EqualElement{ key, eq_ref() }, + PolicyTraits::element(slots_ + seq.offset((size_t)i))))) return seq.offset((size_t)i); } - if (GTL_PREDICT_TRUE(g.MatchEmpty())) break; + if (GTL_PREDICT_TRUE(g.MatchEmpty())) + break; seq.next(); } return (size_t)-1; } - template + template std::pair find_or_prepare_insert(const K& key, size_t hashval) { size_t offset = _find_key(key, hashval); if (offset == (size_t)-1) - return {prepare_insert(hashval), true}; - return {offset, false}; + return { prepare_insert(hashval), true }; + return { offset, false }; } - size_t prepare_insert(size_t hashval) GTL_ATTRIBUTE_NOINLINE { + size_t prepare_insert(size_t hashval) GTL_ATTRIBUTE_NOINLINE { auto target = find_first_non_full(hashval); if (GTL_PREDICT_FALSE(growth_left() == 0 && !IsDeleted(ctrl_[target.offset]))) { rehash_and_grow_if_necessary(); @@ -2604,30 +2599,31 @@ class raw_hash_map : public raw_hash_set { template std::pair insert_or_assign_impl(K&& k, V&& v) { size_t hashval = this->hash(k); - size_t offset = this->_find_key(k, hashval); + size_t offset = this->_find_key(k, hashval); if (offset == (size_t)-1) { offset = this->prepare_insert(hashval); this->emplace_at(offset, std::forward(k), std::forward(v)); this->set_ctrl(offset, H2(hashval)); - return {this->iterator_at(offset), true}; - } + return { this->iterator_at(offset), true }; + } Policy::value(&*this->iterator_at(offset)) = std::forward(v); - return {this->iterator_at(offset), false}; + return { this->iterator_at(offset), false }; } template std::pair try_emplace_impl(K&& k, Args&&... args) { size_t hashval = this->hash(k); - size_t offset = this->_find_key(k, hashval); + size_t offset = this->_find_key(k, hashval); if (offset == (size_t)-1) { offset = this->prepare_insert(hashval); - this->emplace_at(offset, std::piecewise_construct, + this->emplace_at(offset, + std::piecewise_construct, std::forward_as_tuple(std::forward(k)), std::forward_as_tuple(std::forward(args)...)); this->set_ctrl(offset, H2(hashval)); - return {this->iterator_at(offset), true}; + return { this->iterator_at(offset), true }; } - return {this->iterator_at(offset), false}; + return { this->iterator_at(offset), false }; } }; @@ -2851,23 +2847,31 @@ class LockableBaseImpl { }; // ---------------------------------------------------- - class ReadWriteLock - { + class ReadWriteLock { public: using mutex_type = MutexType; - ReadWriteLock() : m_(nullptr), locked_(false), locked_shared_(false) {} + ReadWriteLock() + : m_(nullptr) + , locked_(false) + , locked_shared_(false) {} - explicit ReadWriteLock(mutex_type &m) : m_(&m), locked_(false), locked_shared_(true) { - m_->lock_shared(); + explicit ReadWriteLock(mutex_type& m) + : m_(&m) + , locked_(false) + , locked_shared_(true) { + m_->lock_shared(); } - ReadWriteLock(mutex_type& m, defer_lock_t) noexcept : - m_(&m), locked_(false), locked_shared_(false) - {} + ReadWriteLock(mutex_type& m, defer_lock_t) noexcept + : m_(&m) + , locked_(false) + , locked_shared_(false) {} - ReadWriteLock(ReadWriteLock &&o) noexcept : - m_(std::move(o.m_)), locked_(o.locked_), locked_shared_(o.locked_shared_) { + ReadWriteLock(ReadWriteLock&& o) noexcept + : m_(std::move(o.m_)) + , locked_(o.locked_) + , locked_shared_(o.locked_shared_) { o.locked_ = false; o.locked_shared_ = false; o.m_ = nullptr; @@ -2880,50 +2884,50 @@ class LockableBaseImpl { } ~ReadWriteLock() { - if (locked_shared_) + if (locked_shared_) m_->unlock_shared(); - else if (locked_) + else if (locked_) m_->unlock(); } - void lock_shared() { - if (!locked_shared_) { - m_->lock_shared(); - locked_shared_ = true; + void lock_shared() { + if (!locked_shared_) { + m_->lock_shared(); + locked_shared_ = true; } } - void unlock_shared() { + void unlock_shared() { if (locked_shared_) { - m_->unlock_shared(); + m_->unlock_shared(); locked_shared_ = false; } - } + } - void lock() { - if (!locked_) { - m_->lock(); - locked_ = true; + void lock() { + if (!locked_) { + m_->lock(); + locked_ = true; } } - void unlock() { + void unlock() { if (locked_) { - m_->unlock(); + m_->unlock(); locked_ = false; } - } + } bool owns_lock() const noexcept { return locked_; } bool owns_shared_lock() const noexcept { return locked_shared_; } - void swap(ReadWriteLock &o) noexcept { + void swap(ReadWriteLock& o) noexcept { std::swap(m_, o.m_); std::swap(locked_, o.locked_); std::swap(locked_shared_, o.locked_shared_); } - mutex_type *mutex() const noexcept { return m_; } + mutex_type* mutex() const noexcept { return m_; } bool switch_to_unique() { assert(locked_shared_); @@ -2933,7 +2937,7 @@ class LockableBaseImpl { } private: - mutex_type *m_; + mutex_type* m_; bool locked_shared_; bool locked_; }; @@ -3048,15 +3052,16 @@ class LockableImpl : public NullMutex { // -------------------------------------------------------------------------- // Abseil Mutex support (read and write lock support) +// use: `phmap::AbslMutex` instead of `std::mutex` // -------------------------------------------------------------------------- #ifdef ABSL_SYNCHRONIZATION_MUTEX_H_ struct AbslMutex : protected absl::Mutex { - void lock() ABSL_EXCLUSIVE_LOCK_FUNCTION() { this->Lock(); } - void unlock() ABSL_UNLOCK_FUNCTION() { this->Unlock(); } - void try_lock() ABSL_EXCLUSIVE_TRYLOCK_FUNCTION(true) { this->TryLock(); } - void lock_shared() ABSL_SHARED_LOCK_FUNCTION() { this->ReaderLock(); } - void unlock_shared() ABSL_UNLOCK_FUNCTION() { this->ReaderUnlock(); } + void lock() ABSL_EXCLUSIVE_LOCK_FUNCTION() { this->Lock(); } + void unlock() ABSL_UNLOCK_FUNCTION() { this->Unlock(); } + void try_lock() ABSL_EXCLUSIVE_TRYLOCK_FUNCTION(true) { this->TryLock(); } + void lock_shared() ABSL_SHARED_LOCK_FUNCTION() { this->ReaderLock(); } + void unlock_shared() ABSL_UNLOCK_FUNCTION() { this->ReaderUnlock(); } void try_lock_shared() ABSL_SHARED_TRYLOCK_FUNCTION(true) { this->ReaderTryLock(); } }; @@ -3076,6 +3081,42 @@ class LockableImpl : public AbslMutex { #endif +// -------------------------------------------------------------------------- +// Microsoft SRWLOCK support (read and write lock support) +// use: `phmap::srwlock` instead of `std::mutex` +// -------------------------------------------------------------------------- +#if defined(_MSVC_LANG) && (defined(SRWLOCK_INIT) || defined(_SYNCHAPI_H_)) + +class srwlock { + SRWLOCK _lock; + +public: + srwlock() { InitializeSRWLock(&_lock); } + void lock() { AcquireSRWLockExclusive(&_lock); } + void unlock() { ReleaseSRWLockExclusive(&_lock); } + bool try_lock() { return !!TryAcquireSRWLockExclusive(&_lock); } + void lock_shared() { AcquireSRWLockShared(&_lock); } + void unlock_shared() { ReleaseSRWLockShared(&_lock); } + bool try_lock_shared() { return !!TryAcquireSRWLockShared(&_lock); } +}; + + +template<> +class LockableImpl : public srwlock { +public: + using mutex_type = srwlock; + using Base = LockableBaseImpl; + using SharedLock = typename Base::ReadLock; + using ReadWriteLock = typename Base::ReadWriteLock; + using UpgradeLock = typename Base::WriteLock; + using UniqueLock = typename Base::WriteLock; + using SharedLocks = typename Base::ReadLocks; + using UniqueLocks = typename Base::WriteLocks; + using UpgradeToUnique = typename Base::DoNothing; // we already have unique ownership +}; + +#endif + // -------------------------------------------------------------------------- // Boost shared_mutex support (read and write lock support) // -------------------------------------------------------------------------- @@ -3332,9 +3373,9 @@ class parallel_hash_set { // ------------------------- c o n s t r u c t o r s ------------------ - parallel_hash_set() noexcept(std::is_nothrow_default_constructible_v && - std::is_nothrow_default_constructible_v && - std::is_nothrow_default_constructible_v) {} + parallel_hash_set() noexcept( + std::is_nothrow_default_constructible_v&& std::is_nothrow_default_constructible_v&& + std::is_nothrow_default_constructible_v) {} explicit parallel_hash_set(size_t bucket_cnt, const hasher& hash_param = hasher(), @@ -3454,9 +3495,9 @@ class parallel_hash_set { sets_[i].set_ = { that.sets_[i].set_, a }; } - parallel_hash_set(parallel_hash_set&& that) noexcept(std::is_nothrow_copy_constructible_v && - std::is_nothrow_copy_constructible_v && - std::is_nothrow_copy_constructible_v) + parallel_hash_set(parallel_hash_set&& that) noexcept( + std::is_nothrow_copy_constructible_v&& std::is_nothrow_copy_constructible_v&& + std::is_nothrow_copy_constructible_v) : parallel_hash_set(std::move(that), that.alloc_ref()) {} parallel_hash_set(parallel_hash_set&& that, const allocator_type& a) { @@ -3471,8 +3512,8 @@ class parallel_hash_set { } parallel_hash_set& operator=(parallel_hash_set&& that) noexcept( - std::allocator_traits::is_always_equal::value && std::is_nothrow_move_assignable_v && - std::is_nothrow_move_assignable_v) { + std::allocator_traits::is_always_equal::value&& std::is_nothrow_move_assignable_v&& + std::is_nothrow_move_assignable_v) { for (size_t i = 0; i < num_tables; ++i) sets_[i].set_ = std::move(that.sets_[i].set_); return *this; @@ -3763,12 +3804,12 @@ class parallel_hash_set { // lazy_emplace // ------------ - template + template iterator lazy_emplace_with_hash(const key_arg& key, size_t hashval, F&& f) { - Inner& inner = sets_[subidx(hashval)]; - auto& set = inner.set_; + Inner& inner = sets_[subidx(hashval)]; + auto& set = inner.set_; ReadWriteLock m(inner); - size_t offset = set._find_key(key, hashval); + size_t offset = set._find_key(key, hashval); if (offset == (size_t)-1 && m.switch_to_unique()) { // we did an unlock/lock, and another thread could have inserted the same key, so we need to // do a find() again. @@ -3782,7 +3823,7 @@ class parallel_hash_set { return iterator_at(offset); } - template + template iterator lazy_emplace(const key_arg& key, F&& f) { return lazy_emplace_with_hash(key, this->hash(key), std::forward(f)); } @@ -3857,14 +3898,14 @@ class parallel_hash_set { return erase_if_impl(key, std::forward(f)); } - template + template bool erase_if_impl(const key_arg& key, F&& f) { static_assert(std::is_invocable::value); - auto hashval = this->hash(key); - Inner& inner = sets_[subidx(hashval)]; - auto& set = inner.set_; - L m(inner); - auto it = set.find(key, hashval); + auto hashval = this->hash(key); + Inner& inner = sets_[subidx(hashval)]; + auto& set = inner.set_; + L m(inner); + auto it = set.find(key, hashval); if (it == set.end()) return false; if (m.switch_to_unique()) { @@ -3873,8 +3914,7 @@ class parallel_hash_set { if (it == set.end()) return false; } - if (std::forward(f)(const_cast(*it))) - { + if (std::forward(f)(const_cast(*it))) { set._erase(it); return true; } @@ -4344,12 +4384,12 @@ class parallel_hash_set { } template - std::tuple find_or_prepare_insert_with_hash(size_t hashval, - const K& key, + std::tuple find_or_prepare_insert_with_hash(size_t hashval, + const K& key, ReadWriteLock& mutexlock) { - Inner& inner = sets_[subidx(hashval)]; - auto& set = inner.set_; - mutexlock = std::move(ReadWriteLock(inner)); + Inner& inner = sets_[subidx(hashval)]; + auto& set = inner.set_; + mutexlock = std::move(ReadWriteLock(inner)); size_t offset = set._find_key(key, hashval); if (offset == (size_t)-1 && mutexlock.switch_to_unique()) { // we did an unlock/lock, and another thread could have inserted the same key, so we need to @@ -4423,9 +4463,9 @@ class parallel_hash_map : public parallel_hash_set::value && IsTransparent::value>; - using Base = typename parallel_hash_map::parallel_hash_set; - using Lockable = LockableImpl; - using UniqueLock = typename Lockable::UniqueLock; + using Base = typename parallel_hash_map::parallel_hash_set; + using Lockable = LockableImpl; + using UniqueLock = typename Lockable::UniqueLock; using ReadWriteLock = typename Lockable::ReadWriteLock; public: