From a92d74f9ba6507509d4e876c7eb1aa9beef072e1 Mon Sep 17 00:00:00 2001 From: jiangrujie Date: Sun, 5 Mar 2023 21:37:12 +0800 Subject: [PATCH 1/2] Add _Alloc template parameters for FlatMap and FlatSet --- src/butil/containers/flat_map.h | 23 +++++-- src/butil/containers/flat_map_inl.h | 103 +++++++++++++++------------- src/butil/single_threaded_pool.h | 17 +++-- 3 files changed, 83 insertions(+), 60 deletions(-) diff --git a/src/butil/containers/flat_map.h b/src/butil/containers/flat_map.h index 6789c3c7fe..99d21337b9 100644 --- a/src/butil/containers/flat_map.h +++ b/src/butil/containers/flat_map.h @@ -126,11 +126,13 @@ template , - bool _Sparse = false> + bool _Sparse = false, + typename _Alloc = PtAllocator> class FlatMap { public: typedef _K key_type; typedef _T mapped_type; + typedef _Alloc allocator_type; typedef FlatMapElement<_K, _T> Element; typedef typename Element::value_type value_type; typedef typename conditional< @@ -148,7 +150,9 @@ class FlatMap { key_type key; }; - FlatMap(const hasher& hashfn = hasher(), const key_equal& eql = key_equal()); + FlatMap(const hasher& hashfn = hasher(), + const key_equal& eql = key_equal(), + const allocator_type& alloc = allocator_type()); ~FlatMap(); FlatMap(const FlatMap& rhs); void operator=(const FlatMap& rhs); @@ -279,16 +283,18 @@ template friend class SparseFlatMapIterator; u_int _load_factor; hasher _hashfn; key_equal _eql; - SingleThreadedPool _pool; + allocator_type _allocator; + SingleThreadedPool _pool; }; template , typename _Equal = DefaultEqualTo<_K>, - bool _Sparse = false> + bool _Sparse = false, + typename _Alloc = PtAllocator> class FlatSet { public: - typedef FlatMap<_K, FlatMapVoid, _Hash, _Equal, _Sparse> Map; + typedef FlatMap<_K, FlatMapVoid, _Hash, _Equal, _Sparse, _Alloc> Map; typedef typename Map::key_type key_type; typedef typename Map::value_type value_type; typedef typename Map::Bucket Bucket; @@ -296,9 +302,12 @@ class FlatSet { typedef typename Map::const_iterator const_iterator; typedef typename Map::hasher hasher; typedef typename Map::key_equal key_equal; + typedef typename Map::allocator_type allocator_type; - FlatSet(const hasher& hashfn = hasher(), const key_equal& eql = key_equal()) - : _map(hashfn, eql) {} + FlatSet(const hasher& hashfn = hasher(), + const key_equal& eql = key_equal(), + const allocator_type& alloc = allocator_type()) + : _map(hashfn, eql, alloc) {} void swap(FlatSet & rhs) { _map.swap(rhs._map); } int init(size_t nbucket, u_int load_factor = 80) diff --git a/src/butil/containers/flat_map_inl.h b/src/butil/containers/flat_map_inl.h index e98c88b5e1..51246e266a 100644 --- a/src/butil/containers/flat_map_inl.h +++ b/src/butil/containers/flat_map_inl.h @@ -134,7 +134,8 @@ template class FlatMapIterator { private: friend class FlatMapIterator; friend class FlatMap; + typename Map::hasher, typename Map::key_equal, + false, typename Map::allocator_type>; void find_and_set_valid_node() { for (; !_entry->is_valid(); ++_entry); @@ -222,8 +223,8 @@ friend class SparseFlatMapIterator; }; -template -FlatMap<_K, _T, _H, _E, _S>::FlatMap(const hasher& hashfn, const key_equal& eql) +template +FlatMap<_K, _T, _H, _E, _S, _A>::FlatMap(const hasher& hashfn, const key_equal& eql, const allocator_type& alloc) : _size(0) , _nbucket(0) , _buckets(NULL) @@ -231,12 +232,14 @@ FlatMap<_K, _T, _H, _E, _S>::FlatMap(const hasher& hashfn, const key_equal& eql) , _load_factor(0) , _hashfn(hashfn) , _eql(eql) + , _allocator(alloc) + , _pool(alloc) {} -template -FlatMap<_K, _T, _H, _E, _S>::~FlatMap() { +template +FlatMap<_K, _T, _H, _E, _S, _A>::~FlatMap() { clear(); - free(_buckets); + _allocator.Free(_buckets); _buckets = NULL; free(_thumbnail); _thumbnail = NULL; @@ -244,21 +247,22 @@ FlatMap<_K, _T, _H, _E, _S>::~FlatMap() { _load_factor = 0; } -template -FlatMap<_K, _T, _H, _E, _S>::FlatMap(const FlatMap& rhs) +template +FlatMap<_K, _T, _H, _E, _S, _A>::FlatMap(const FlatMap& rhs) : _size(0) , _nbucket(0) , _buckets(NULL) , _thumbnail(NULL) , _load_factor(rhs._load_factor) , _hashfn(rhs._hashfn) - , _eql(rhs._eql) { + , _eql(rhs._eql) + , _allocator(rhs._allocator) { operator=(rhs); } -template +template void -FlatMap<_K, _T, _H, _E, _S>::operator=(const FlatMap<_K, _T, _H, _E, _S>& rhs) { +FlatMap<_K, _T, _H, _E, _S, _A>::operator=(const FlatMap<_K, _T, _H, _E, _S, _A>& rhs) { if (this == &rhs) { return; } @@ -272,10 +276,10 @@ FlatMap<_K, _T, _H, _E, _S>::operator=(const FlatMap<_K, _T, _H, _E, _S>& rhs) { _load_factor = rhs._load_factor; } if (_buckets == NULL || is_too_crowded(rhs._size)) { - free(_buckets); + _allocator.Free(_buckets); _nbucket = rhs._nbucket; // note: need an extra bucket to let iterator know where buckets end - _buckets = (Bucket*)malloc(sizeof(Bucket) * (_nbucket + 1/*note*/)); + _buckets = (Bucket*)_allocator.Alloc(sizeof(Bucket) * (_nbucket + 1/*note*/)); if (NULL == _buckets) { LOG(ERROR) << "Fail to new _buckets"; return; @@ -320,8 +324,8 @@ FlatMap<_K, _T, _H, _E, _S>::operator=(const FlatMap<_K, _T, _H, _E, _S>& rhs) { } } -template -int FlatMap<_K, _T, _H, _E, _S>::init(size_t nbucket, u_int load_factor) { +template +int FlatMap<_K, _T, _H, _E, _S, _A>::init(size_t nbucket, u_int load_factor) { if (initialized()) { LOG(ERROR) << "Already initialized"; return -1; @@ -334,7 +338,7 @@ int FlatMap<_K, _T, _H, _E, _S>::init(size_t nbucket, u_int load_factor) { _nbucket = flatmap_round(nbucket); _load_factor = load_factor; - _buckets = (Bucket*)malloc(sizeof(Bucket) * (_nbucket + 1)); + _buckets = (Bucket*)_allocator.Alloc(sizeof(Bucket) * (_nbucket + 1)); if (NULL == _buckets) { LOG(ERROR) << "Fail to new _buckets"; return -1; @@ -355,8 +359,8 @@ int FlatMap<_K, _T, _H, _E, _S>::init(size_t nbucket, u_int load_factor) { return 0; } -template -void FlatMap<_K, _T, _H, _E, _S>::swap(FlatMap<_K, _T, _H, _E, _S> & rhs) { +template +void FlatMap<_K, _T, _H, _E, _S, _A>::swap(FlatMap<_K, _T, _H, _E, _S, _A> & rhs) { std::swap(rhs._size, _size); std::swap(rhs._nbucket, _nbucket); std::swap(rhs._buckets, _buckets); @@ -364,25 +368,26 @@ void FlatMap<_K, _T, _H, _E, _S>::swap(FlatMap<_K, _T, _H, _E, _S> & rhs) { std::swap(rhs._load_factor, _load_factor); std::swap(rhs._hashfn, _hashfn); std::swap(rhs._eql, _eql); + std::swap(rhs._allocator, _allocator); rhs._pool.swap(_pool); } -template -_T* FlatMap<_K, _T, _H, _E, _S>::insert(const key_type& key, +template +_T* FlatMap<_K, _T, _H, _E, _S, _A>::insert(const key_type& key, const mapped_type& value) { mapped_type *p = &operator[](key); *p = value; return p; } -template -_T* FlatMap<_K, _T, _H, _E, _S>::insert(const std::pair& kv) { +template +_T* FlatMap<_K, _T, _H, _E, _S, _A>::insert(const std::pair& kv) { return insert(kv.first, kv.second); } -template +template template -size_t FlatMap<_K, _T, _H, _E, _S>::erase(const K2& key, _T* old_value) { +size_t FlatMap<_K, _T, _H, _E, _S, _A>::erase(const K2& key, _T* old_value) { if (!initialized()) { return 0; } @@ -446,8 +451,8 @@ size_t FlatMap<_K, _T, _H, _E, _S>::erase(const K2& key, _T* old_value) { return 0; } -template -void FlatMap<_K, _T, _H, _E, _S>::clear() { +template +void FlatMap<_K, _T, _H, _E, _S, _A>::clear() { if (0 == _size) { return; } @@ -473,15 +478,15 @@ void FlatMap<_K, _T, _H, _E, _S>::clear() { } } -template -void FlatMap<_K, _T, _H, _E, _S>::clear_and_reset_pool() { +template +void FlatMap<_K, _T, _H, _E, _S, _A>::clear_and_reset_pool() { clear(); _pool.reset(); } -template +template template -_T* FlatMap<_K, _T, _H, _E, _S>::seek(const K2& key) const { +_T* FlatMap<_K, _T, _H, _E, _S, _A>::seek(const K2& key) const { if (!initialized()) { return NULL; } @@ -502,8 +507,8 @@ _T* FlatMap<_K, _T, _H, _E, _S>::seek(const K2& key) const { return NULL; } -template -_T& FlatMap<_K, _T, _H, _E, _S>::operator[](const key_type& key) { +template +_T& FlatMap<_K, _T, _H, _E, _S, _A>::operator[](const key_type& key) { const size_t index = flatmap_mod(_hashfn(key), _nbucket); Bucket& first_node = _buckets[index]; if (!first_node.is_valid()) { @@ -550,8 +555,8 @@ _T& FlatMap<_K, _T, _H, _E, _S>::operator[](const key_type& key) { } } -template -void FlatMap<_K, _T, _H, _E, _S>::save_iterator( +template +void FlatMap<_K, _T, _H, _E, _S, _A>::save_iterator( const const_iterator& it, PositionHint* hint) const { hint->nbucket = _nbucket; hint->offset = it._entry - _buckets; @@ -564,9 +569,9 @@ void FlatMap<_K, _T, _H, _E, _S>::save_iterator( } } -template -typename FlatMap<_K, _T, _H, _E, _S>::const_iterator -FlatMap<_K, _T, _H, _E, _S>::restore_iterator(const PositionHint& hint) const { +template +typename FlatMap<_K, _T, _H, _E, _S, _A>::const_iterator +FlatMap<_K, _T, _H, _E, _S, _A>::restore_iterator(const PositionHint& hint) const { if (hint.nbucket != _nbucket) // resized return begin(); // restart @@ -598,8 +603,8 @@ FlatMap<_K, _T, _H, _E, _S>::restore_iterator(const PositionHint& hint) const { return const_iterator(this, hint.offset); } -template -bool FlatMap<_K, _T, _H, _E, _S>::resize(size_t nbucket2) { +template +bool FlatMap<_K, _T, _H, _E, _S, _A>::resize(size_t nbucket2) { nbucket2 = flatmap_round(nbucket2); if (_nbucket == nbucket2) { return false; @@ -622,8 +627,8 @@ bool FlatMap<_K, _T, _H, _E, _S>::resize(size_t nbucket2) { return true; } -template -BucketInfo FlatMap<_K, _T, _H, _E, _S>::bucket_info() const { +template +BucketInfo FlatMap<_K, _T, _H, _E, _S, _A>::bucket_info() const { size_t max_n = 0; size_t nentry = 0; for (size_t i = 0; i < _nbucket; ++i) { @@ -643,23 +648,23 @@ inline std::ostream& operator<<(std::ostream& os, const BucketInfo& info) { << " avgb=" << info.average_length << '}'; } -template -typename FlatMap<_K, _T, _H, _E, _S>::iterator FlatMap<_K, _T, _H, _E, _S>::begin() { +template +typename FlatMap<_K, _T, _H, _E, _S, _A>::iterator FlatMap<_K, _T, _H, _E, _S, _A>::begin() { return iterator(this, 0); } -template -typename FlatMap<_K, _T, _H, _E, _S>::iterator FlatMap<_K, _T, _H, _E, _S>::end() { +template +typename FlatMap<_K, _T, _H, _E, _S, _A>::iterator FlatMap<_K, _T, _H, _E, _S, _A>::end() { return iterator(this, _nbucket); } -template -typename FlatMap<_K, _T, _H, _E, _S>::const_iterator FlatMap<_K, _T, _H, _E, _S>::begin() const { +template +typename FlatMap<_K, _T, _H, _E, _S, _A>::const_iterator FlatMap<_K, _T, _H, _E, _S, _A>::begin() const { return const_iterator(this, 0); } -template -typename FlatMap<_K, _T, _H, _E, _S>::const_iterator FlatMap<_K, _T, _H, _E, _S>::end() const { +template +typename FlatMap<_K, _T, _H, _E, _S, _A>::const_iterator FlatMap<_K, _T, _H, _E, _S, _A>::end() const { return const_iterator(this, _nbucket); } diff --git a/src/butil/single_threaded_pool.h b/src/butil/single_threaded_pool.h index e6d90fedb5..4dd4123658 100644 --- a/src/butil/single_threaded_pool.h +++ b/src/butil/single_threaded_pool.h @@ -30,9 +30,16 @@ namespace butil { // void* mem = pool.get(); // pool.back(mem); +class PtAllocator { +public: + void* Alloc(size_t n) { return malloc(n); } + void Free(void* p) { free(p); } +}; + template // minimum number of items in one block + size_t MIN_NITEM = 1, + typename Allocator = PtAllocator> // minimum number of items in one block class SingleThreadedPool { public: // Note: this is a union. The next pointer is set iff when spaces is free, @@ -54,7 +61,8 @@ class SingleThreadedPool { static const size_t NITEM = Block::NITEM; static const size_t ITEM_SIZE = ITEM_SIZE_IN; - SingleThreadedPool() : _free_nodes(NULL), _blocks(NULL) {} + SingleThreadedPool(const Allocator& alloc = Allocator()) + : _free_nodes(NULL), _blocks(NULL), _allocator(alloc) {} ~SingleThreadedPool() { reset(); } void swap(SingleThreadedPool & other) { @@ -71,7 +79,7 @@ class SingleThreadedPool { return spaces; } if (_blocks == NULL || _blocks->nalloc >= Block::NITEM) { - Block* new_block = (Block*)malloc(sizeof(Block)); + Block* new_block = (Block*)_allocator.Alloc(sizeof(Block)); if (new_block == NULL) { return NULL; } @@ -98,7 +106,7 @@ class SingleThreadedPool { _free_nodes = NULL; while (_blocks) { Block* next = _blocks->next; - free(_blocks); + _allocator.Free(_blocks); _blocks = next; } } @@ -129,6 +137,7 @@ class SingleThreadedPool { Node* _free_nodes; Block* _blocks; + Allocator _allocator; }; } // namespace butil From 6c93f7ae92266a6a9aa6175e6c8279932f2c82c1 Mon Sep 17 00:00:00 2001 From: oldbear Date: Thu, 9 Mar 2023 17:00:14 +0800 Subject: [PATCH 2/2] + Remove allocator inside FlatMap and reuse the one in _pool --- src/butil/containers/flat_map.h | 3 ++- src/butil/containers/flat_map_inl.h | 13 +++++-------- src/butil/single_threaded_pool.h | 2 ++ 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/butil/containers/flat_map.h b/src/butil/containers/flat_map.h index 99d21337b9..9e6eec1f27 100644 --- a/src/butil/containers/flat_map.h +++ b/src/butil/containers/flat_map.h @@ -269,6 +269,8 @@ class FlatMap { char element_spaces[sizeof(Element)]; }; + allocator_type& get_allocator() { return _pool.get_allocator(); } + private: template friend class FlatMapIterator; template friend class SparseFlatMapIterator; @@ -283,7 +285,6 @@ template friend class SparseFlatMapIterator; u_int _load_factor; hasher _hashfn; key_equal _eql; - allocator_type _allocator; SingleThreadedPool _pool; }; diff --git a/src/butil/containers/flat_map_inl.h b/src/butil/containers/flat_map_inl.h index 51246e266a..8c87bb17e1 100644 --- a/src/butil/containers/flat_map_inl.h +++ b/src/butil/containers/flat_map_inl.h @@ -232,14 +232,13 @@ FlatMap<_K, _T, _H, _E, _S, _A>::FlatMap(const hasher& hashfn, const key_equal& , _load_factor(0) , _hashfn(hashfn) , _eql(eql) - , _allocator(alloc) , _pool(alloc) {} template FlatMap<_K, _T, _H, _E, _S, _A>::~FlatMap() { clear(); - _allocator.Free(_buckets); + get_allocator().Free(_buckets); _buckets = NULL; free(_thumbnail); _thumbnail = NULL; @@ -255,8 +254,7 @@ FlatMap<_K, _T, _H, _E, _S, _A>::FlatMap(const FlatMap& rhs) , _thumbnail(NULL) , _load_factor(rhs._load_factor) , _hashfn(rhs._hashfn) - , _eql(rhs._eql) - , _allocator(rhs._allocator) { + , _eql(rhs._eql) { operator=(rhs); } @@ -276,10 +274,10 @@ FlatMap<_K, _T, _H, _E, _S, _A>::operator=(const FlatMap<_K, _T, _H, _E, _S, _A> _load_factor = rhs._load_factor; } if (_buckets == NULL || is_too_crowded(rhs._size)) { - _allocator.Free(_buckets); + get_allocator().Free(_buckets); _nbucket = rhs._nbucket; // note: need an extra bucket to let iterator know where buckets end - _buckets = (Bucket*)_allocator.Alloc(sizeof(Bucket) * (_nbucket + 1/*note*/)); + _buckets = (Bucket*)get_allocator().Alloc(sizeof(Bucket) * (_nbucket + 1/*note*/)); if (NULL == _buckets) { LOG(ERROR) << "Fail to new _buckets"; return; @@ -338,7 +336,7 @@ int FlatMap<_K, _T, _H, _E, _S, _A>::init(size_t nbucket, u_int load_factor) { _nbucket = flatmap_round(nbucket); _load_factor = load_factor; - _buckets = (Bucket*)_allocator.Alloc(sizeof(Bucket) * (_nbucket + 1)); + _buckets = (Bucket*)get_allocator().Alloc(sizeof(Bucket) * (_nbucket + 1)); if (NULL == _buckets) { LOG(ERROR) << "Fail to new _buckets"; return -1; @@ -368,7 +366,6 @@ void FlatMap<_K, _T, _H, _E, _S, _A>::swap(FlatMap<_K, _T, _H, _E, _S, _A> & rhs std::swap(rhs._load_factor, _load_factor); std::swap(rhs._hashfn, _hashfn); std::swap(rhs._eql, _eql); - std::swap(rhs._allocator, _allocator); rhs._pool.swap(_pool); } diff --git a/src/butil/single_threaded_pool.h b/src/butil/single_threaded_pool.h index 4dd4123658..f1b28e0305 100644 --- a/src/butil/single_threaded_pool.h +++ b/src/butil/single_threaded_pool.h @@ -130,6 +130,8 @@ class SingleThreadedPool { return count_allocated() - count_free(); } + Allocator& get_allocator() { return _allocator; } + private: // You should not copy a pool. SingleThreadedPool(const SingleThreadedPool&);