diff --git a/include/qt_addrstat.h b/include/qt_addrstat.h index 92772deb4..a64dbc5c6 100644 --- a/include/qt_addrstat.h +++ b/include/qt_addrstat.h @@ -7,19 +7,19 @@ * keeping track of the FEB status of an address. It expects a shepherd pointer * to use to find the right memory pool to use. */ static QINLINE qthread_addrstat_t *qthread_addrstat_new(void) -{ /*{{{ */ +{ /*{{{ */ qthread_addrstat_t *ret = ALLOC_ADDRSTAT(); + QTHREAD_FASTLOCK_INIT_PTR(&ret->lock); + QTHREAD_FASTLOCK_LOCK(&ret->lock); + ret->full = 1; + ret->valid = 1; + ret->EFQ = NULL; + ret->FEQ = NULL; + ret->FFQ = NULL; + ret->FFWQ = NULL; + QTHREAD_EMPTY_TIMER_INIT(ret); + QTHREAD_FASTLOCK_UNLOCK(&ret->lock); - if (ret != NULL) { - QTHREAD_FASTLOCK_INIT(ret->lock); - ret->full = 1; - ret->valid = 1; - ret->EFQ = NULL; - ret->FEQ = NULL; - ret->FFQ = NULL; - ret->FFWQ = NULL; - QTHREAD_EMPTY_TIMER_INIT(ret); - } return ret; } /*}}} */ diff --git a/include/qt_atomics.h b/include/qt_atomics.h index 26c7d9f7e..3b7192e7f 100644 --- a/include/qt_atomics.h +++ b/include/qt_atomics.h @@ -68,20 +68,16 @@ # define QTHREAD_FASTLOCK_SETUP() do { } while (0) # define QTHREAD_FASTLOCK_ATTRVAR typedef struct qt_spin_exclusive_s { /* added to allow fast critical section ordering */ - aligned_t enter; /* and not call pthreads spin_lock -- hard to debug */ - aligned_t exit; /* near the lock under gdb -- 4/1/11 akp */ + aligned_t _Atomic enter; /* and not call pthreads spin_lock -- hard to debug */ + aligned_t _Atomic exit; /* near the lock under gdb -- 4/1/11 akp */ } qt_spin_exclusive_t; void qt_spin_exclusive_lock(qt_spin_exclusive_t *); void qt_spin_exclusive_unlock(qt_spin_exclusive_t *); -# define QTHREAD_FASTLOCK_INIT(x) { (x).enter = 0; (x).exit = 0; } -# define QTHREAD_FASTLOCK_INIT_PTR(x) { (x)->enter = 0; (x)->exit = 0; } -# define QTHREAD_FASTLOCK_LOCK(x) { aligned_t val = qthread_incr(&(x)->enter, 1); \ - while (val != (x)->exit) SPINLOCK_BODY(); \ - THREAD_FENCE_MEM_ACQUIRE; /* spin waiting for my turn */ } -# define QTHREAD_FASTLOCK_UNLOCK(x) do { COMPILER_FENCE; \ - THREAD_FENCE_MEM_RELEASE; \ - (x)->exit++; /* allow next guy's turn */ \ - } while (0) +# define QTHREAD_FASTLOCK_INIT(x) { atomic_store_explicit(&(x).enter, 0u, memory_order_release); atomic_store_explicit(&(x).exit, 0u, memory_order_release); } +# define QTHREAD_FASTLOCK_INIT_PTR(x) { atomic_store_explicit(&(x)->enter, 0u, memory_order_release); atomic_store_explicit(&(x)->exit, 0u, memory_order_release); } +# define QTHREAD_FASTLOCK_LOCK(x) { aligned_t val = atomic_fetch_add_explicit(&(x)->enter, 1u, memory_order_relaxed); \ + while (val != atomic_load_explicit(&(x)->exit, memory_order_acquire)) SPINLOCK_BODY(); /* spin waiting for turn */ } +# define QTHREAD_FASTLOCK_UNLOCK(x) do { atomic_fetch_add_explicit(&(x)->exit, 1u, memory_order_release); /* notify next waiting thread */} while (0) # define QTHREAD_FASTLOCK_DESTROY(x) # define QTHREAD_FASTLOCK_DESTROY_PTR(x) # define QTHREAD_FASTLOCK_TYPE qt_spin_exclusive_t diff --git a/src/hashmap.c b/src/hashmap.c index 1c9117c90..63cb324ea 100644 --- a/src/hashmap.c +++ b/src/hashmap.c @@ -147,10 +147,13 @@ qt_hash INTERNAL qt_hash_create(int needSync) if (needSync) { ret->lock = MALLOC(sizeof(QTHREAD_FASTLOCK_TYPE)); QTHREAD_FASTLOCK_INIT_PTR(ret->lock); + QTHREAD_FASTLOCK_LOCK(ret->lock); + qt_hash_internal_create(ret, 100); + QTHREAD_FASTLOCK_UNLOCK(ret->lock); } else { ret->lock = NULL; + qt_hash_internal_create(ret, 100); } - qt_hash_internal_create(ret, 100); } return ret; } /*}}}*/ diff --git a/src/mpool.c b/src/mpool.c index efe7ed0fa..84f480416 100644 --- a/src/mpool.c +++ b/src/mpool.c @@ -64,7 +64,7 @@ struct qt_mpool_s { qt_mpool_threadlocal_cache_t *_Atomic caches; // for cleanup QTHREAD_FASTLOCK_TYPE reuse_lock; - void *reuse_pool; + void *_Atomic reuse_pool; QTHREAD_FASTLOCK_TYPE pool_lock; void **alloc_list; @@ -200,11 +200,14 @@ qt_mpool INTERNAL qt_mpool_create_aligned(size_t item_size, alloc_size *= 2; } } - pool->alloc_size = alloc_size; - pool->items_per_alloc = alloc_size / item_size; - pool->reuse_pool = NULL; QTHREAD_FASTLOCK_INIT(pool->reuse_lock); QTHREAD_FASTLOCK_INIT(pool->pool_lock); + QTHREAD_FASTLOCK_LOCK(&pool->reuse_lock); + atomic_store_explicit(&pool->reuse_pool, NULL, memory_order_relaxed); + QTHREAD_FASTLOCK_UNLOCK(&pool->reuse_lock); + QTHREAD_FASTLOCK_LOCK(&pool->pool_lock); + pool->alloc_size = alloc_size; + pool->items_per_alloc = alloc_size / item_size; #ifdef TLS pool->offset = qthread_incr(&pool_cache_global_max, 1); #else @@ -218,6 +221,7 @@ qt_mpool INTERNAL qt_mpool_create_aligned(size_t item_size, atomic_store_explicit(&pool->alloc_list_pos, 0u, memory_order_relaxed); atomic_store_explicit(&pool->caches, NULL, memory_order_relaxed); + QTHREAD_FASTLOCK_UNLOCK(&pool->pool_lock); return pool; qgoto(errexit); @@ -344,14 +348,14 @@ void INTERNAL *qt_mpool_alloc(qt_mpool pool) cnt = 0; /* cache is empty; need to fill it */ - if (pool->reuse_pool) { // global cache + if (atomic_load_explicit(&pool->reuse_pool, memory_order_relaxed)) { // global cache qthread_debug(MPOOL_BEHAVIOR, "->...pull from reuse\n"); QTHREAD_FASTLOCK_LOCK(&pool->reuse_lock); - if (pool->reuse_pool) { - cache = pool->reuse_pool; - pool->reuse_pool = atomic_load_explicit(&cache->block_tail->next, memory_order_relaxed); + if (atomic_load_explicit(&pool->reuse_pool, memory_order_relaxed)) { + cache = atomic_load_explicit(&pool->reuse_pool, memory_order_relaxed); + atomic_store_explicit(&pool->reuse_pool, atomic_load_explicit(&cache->block_tail->next, memory_order_relaxed), memory_order_relaxed); atomic_store_explicit(&cache->block_tail->next, NULL, memory_order_relaxed); - cnt = items_per_alloc; + cnt = items_per_alloc; } QTHREAD_FASTLOCK_UNLOCK(&pool->reuse_lock); } @@ -433,8 +437,8 @@ void INTERNAL qt_mpool_free(qt_mpool pool, assert(toglobal); assert(atomic_load_explicit(&toglobal->block_tail, memory_order_relaxed)); QTHREAD_FASTLOCK_LOCK(&pool->reuse_lock); - atomic_store_explicit(&atomic_load_explicit(&toglobal->block_tail, memory_order_relaxed)->next, pool->reuse_pool, memory_order_relaxed); - pool->reuse_pool = toglobal; + atomic_store_explicit(&atomic_load_explicit(&toglobal->block_tail, memory_order_relaxed)->next, atomic_load_explicit(&pool->reuse_pool, memory_order_relaxed), memory_order_relaxed); + atomic_store_explicit(&pool->reuse_pool, toglobal, memory_order_relaxed); QTHREAD_FASTLOCK_UNLOCK(&pool->reuse_lock); cnt -= items_per_alloc; } else if (cnt == items_per_alloc + 1) {