|
| 1 | +/* |
| 2 | + * Implementation of safe memory reclamation scheme using |
| 3 | + * quiescent states. |
| 4 | + * |
| 5 | + * This is dervied from the "GUS" safe memory reclamation technique |
| 6 | + * in FreeBSD written by Jeffrey Roberson. It is heavily modified. Any bugs |
| 7 | + * in this code are likely due to the modifications. |
| 8 | + * |
| 9 | + * The original copyright is preserved below. |
| 10 | + * |
| 11 | + * Copyright (c) 2019,2020 Jeffrey Roberson <jeff@FreeBSD.org> |
| 12 | + * |
| 13 | + * Redistribution and use in source and binary forms, with or without |
| 14 | + * modification, are permitted provided that the following conditions |
| 15 | + * are met: |
| 16 | + * 1. Redistributions of source code must retain the above copyright |
| 17 | + * notice unmodified, this list of conditions, and the following |
| 18 | + * disclaimer. |
| 19 | + * 2. Redistributions in binary form must reproduce the above copyright |
| 20 | + * notice, this list of conditions and the following disclaimer in the |
| 21 | + * documentation and/or other materials provided with the distribution. |
| 22 | + * |
| 23 | + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR |
| 24 | + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES |
| 25 | + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. |
| 26 | + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, |
| 27 | + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT |
| 28 | + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, |
| 29 | + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY |
| 30 | + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT |
| 31 | + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF |
| 32 | + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
| 33 | + */ |
| 34 | +#include "Python.h" |
| 35 | +#include "pycore_initconfig.h" // _PyStatus_NO_MEMORY() |
| 36 | +#include "pycore_lock.h" // PyMutex_Lock() |
| 37 | +#include "pycore_qsbr.h" |
| 38 | +#include "pycore_pystate.h" // _PyThreadState_GET() |
| 39 | + |
| 40 | + |
| 41 | +// Wrap-around safe comparison |
| 42 | +#define QSBR_LT(a, b) ((int64_t)((a)-(b)) < 0) |
| 43 | +#define QSBR_LEQ(a, b) ((int64_t)((a)-(b)) <= 0) |
| 44 | + |
| 45 | +// Starting size of the array of qsbr thread states |
| 46 | +#define MIN_ARRAY_SIZE 8 |
| 47 | + |
| 48 | +// The shared write sequence is always odd and incremented by two. Detached |
| 49 | +// threads are indicated by a read sequence of zero. |
| 50 | +#define QSBR_OFFLINE 0 |
| 51 | +#define QSBR_INITIAL 1 |
| 52 | +#define QSBR_INCR 2 |
| 53 | + |
| 54 | +// For _Py_qsbr_deferred_advance(): the number of deferrals before advancing |
| 55 | +// the write sequence. |
| 56 | +#define QSBR_DEFERRED_LIMIT 10 |
| 57 | + |
| 58 | +// Allocate a QSBR thread state from the freelist |
| 59 | +struct _qsbr_thread_state * |
| 60 | +qsbr_allocate(struct _qsbr_shared *shared) |
| 61 | +{ |
| 62 | + struct _qsbr_thread_state *qsbr = shared->freelist; |
| 63 | + if (qsbr == NULL) { |
| 64 | + return NULL; |
| 65 | + } |
| 66 | + shared->freelist = qsbr->freelist_next; |
| 67 | + qsbr->freelist_next = NULL; |
| 68 | + qsbr->shared = shared; |
| 69 | + qsbr->allocated = true; |
| 70 | + return qsbr; |
| 71 | +} |
| 72 | + |
| 73 | +// Initialize (or reintialize) the freelist of QSBR thread states |
| 74 | +static void |
| 75 | +initialize_freelist(struct _qsbr_shared *shared) |
| 76 | +{ |
| 77 | + for (Py_ssize_t i = 0; i != shared->size; i++) { |
| 78 | + struct _qsbr_thread_state *qsbr = &shared->array[i].qsbr; |
| 79 | + if (qsbr->tstate != NULL) { |
| 80 | + // Update the thread state pointer to its QSBR state |
| 81 | + _PyThreadStateImpl *tstate = (_PyThreadStateImpl *)qsbr->tstate; |
| 82 | + tstate->qsbr = qsbr; |
| 83 | + } |
| 84 | + if (!qsbr->allocated) { |
| 85 | + // Push to freelist |
| 86 | + qsbr->freelist_next = shared->freelist; |
| 87 | + shared->freelist = qsbr; |
| 88 | + } |
| 89 | + } |
| 90 | +} |
| 91 | + |
| 92 | +// Grow the array of QSBR thread states. Returns 0 on success, -1 on failure. |
| 93 | +static int |
| 94 | +grow_thread_array(struct _qsbr_shared *shared) |
| 95 | +{ |
| 96 | + Py_ssize_t new_size = shared->size * 2; |
| 97 | + if (new_size < MIN_ARRAY_SIZE) { |
| 98 | + new_size = MIN_ARRAY_SIZE; |
| 99 | + } |
| 100 | + |
| 101 | + struct _qsbr_pad *array = PyMem_RawCalloc(new_size, sizeof(*array)); |
| 102 | + if (array == NULL) { |
| 103 | + return -1; |
| 104 | + } |
| 105 | + |
| 106 | + struct _qsbr_pad *old = shared->array; |
| 107 | + if (old != NULL) { |
| 108 | + memcpy(array, shared->array, shared->size * sizeof(*array)); |
| 109 | + } |
| 110 | + |
| 111 | + shared->array = array; |
| 112 | + shared->size = new_size; |
| 113 | + shared->freelist = NULL; |
| 114 | + initialize_freelist(shared); |
| 115 | + |
| 116 | + PyMem_RawFree(old); |
| 117 | + return 0; |
| 118 | +} |
| 119 | + |
| 120 | +uint64_t |
| 121 | +_Py_qsbr_advance(struct _qsbr_shared *shared) |
| 122 | +{ |
| 123 | + return _Py_atomic_add_uint64(&shared->wr_seq, QSBR_INCR) + QSBR_INCR; |
| 124 | +} |
| 125 | + |
| 126 | +uint64_t |
| 127 | +_Py_qsbr_deferred_advance(struct _qsbr_thread_state *qsbr) |
| 128 | +{ |
| 129 | + if (++qsbr->deferrals < QSBR_DEFERRED_LIMIT) { |
| 130 | + return _Py_qsbr_shared_current(qsbr->shared) + QSBR_INCR; |
| 131 | + } |
| 132 | + qsbr->deferrals = 0; |
| 133 | + return _Py_qsbr_advance(qsbr->shared); |
| 134 | +} |
| 135 | + |
| 136 | +static uint64_t |
| 137 | +qsbr_poll_scan(struct _qsbr_shared *shared) |
| 138 | +{ |
| 139 | + // Compute the minimum sequence number of all attached threads |
| 140 | + uint64_t min_seq = _Py_atomic_load_uint64(&shared->wr_seq); |
| 141 | + struct _qsbr_pad *array = shared->array; |
| 142 | + for (Py_ssize_t i = 0, size = shared->size; i != size; i++) { |
| 143 | + struct _qsbr_thread_state *qsbr = &array[i].qsbr; |
| 144 | + |
| 145 | + uint64_t seq = _Py_atomic_load_uint64(&qsbr->seq); |
| 146 | + if (seq != QSBR_OFFLINE && QSBR_LT(seq, min_seq)) { |
| 147 | + min_seq = seq; |
| 148 | + } |
| 149 | + } |
| 150 | + |
| 151 | + // Update the shared read sequence |
| 152 | + uint64_t rd_seq = _Py_atomic_load_uint64(&shared->rd_seq); |
| 153 | + if (QSBR_LT(rd_seq, min_seq)) { |
| 154 | + // It's okay if the compare-exchange failed: another thread updated it |
| 155 | + (void)_Py_atomic_compare_exchange_uint64(&shared->rd_seq, &rd_seq, min_seq); |
| 156 | + rd_seq = min_seq; |
| 157 | + } |
| 158 | + |
| 159 | + return rd_seq; |
| 160 | +} |
| 161 | + |
| 162 | +bool |
| 163 | +_Py_qsbr_poll(struct _qsbr_thread_state *qsbr, uint64_t goal) |
| 164 | +{ |
| 165 | + assert(_PyThreadState_GET()->state == _Py_THREAD_ATTACHED); |
| 166 | + |
| 167 | + uint64_t rd_seq = _Py_atomic_load_uint64(&qsbr->shared->rd_seq); |
| 168 | + if (QSBR_LEQ(goal, rd_seq)) { |
| 169 | + return true; |
| 170 | + } |
| 171 | + |
| 172 | + rd_seq = qsbr_poll_scan(qsbr->shared); |
| 173 | + return QSBR_LEQ(goal, rd_seq); |
| 174 | +} |
| 175 | + |
| 176 | +void |
| 177 | +_Py_qsbr_attach(struct _qsbr_thread_state *qsbr) |
| 178 | +{ |
| 179 | + assert(qsbr->seq == 0 && "already attached"); |
| 180 | + |
| 181 | + uint64_t seq = _Py_qsbr_shared_current(qsbr->shared); |
| 182 | + _Py_atomic_store_uint64_relaxed(&qsbr->seq, seq); |
| 183 | + |
| 184 | + // ensure update to local counter is visible |
| 185 | + _Py_atomic_fence_seq_cst(); |
| 186 | +} |
| 187 | + |
| 188 | +void |
| 189 | +_Py_qsbr_detach(struct _qsbr_thread_state *qsbr) |
| 190 | +{ |
| 191 | + assert(qsbr->seq != 0 && "already detached"); |
| 192 | + |
| 193 | + _Py_atomic_fence_release(); |
| 194 | + _Py_atomic_store_uint64_relaxed(&qsbr->seq, QSBR_OFFLINE); |
| 195 | +} |
| 196 | + |
| 197 | +Py_ssize_t |
| 198 | +_Py_qsbr_reserve(PyInterpreterState *interp) |
| 199 | +{ |
| 200 | + struct _qsbr_shared *shared = &interp->qsbr; |
| 201 | + |
| 202 | + PyMutex_LockFlags(&shared->mutex, _Py_LOCK_DONT_DETACH); |
| 203 | + struct _qsbr_thread_state *qsbr = qsbr_allocate(shared); |
| 204 | + |
| 205 | + if (qsbr == NULL) { |
| 206 | + _PyEval_StopTheWorld(interp); |
| 207 | + if (grow_thread_array(shared) == 0) { |
| 208 | + qsbr = qsbr_allocate(shared); |
| 209 | + } |
| 210 | + _PyEval_StartTheWorld(interp); |
| 211 | + } |
| 212 | + PyMutex_Unlock(&shared->mutex); |
| 213 | + |
| 214 | + if (qsbr == NULL) { |
| 215 | + return -1; |
| 216 | + } |
| 217 | + |
| 218 | + // Compute index in the shared array from the pointer |
| 219 | + return (struct _qsbr_pad *)qsbr - shared->array; |
| 220 | +} |
| 221 | + |
| 222 | +void |
| 223 | +_Py_qsbr_register(_PyThreadStateImpl *tstate, PyInterpreterState *interp, |
| 224 | + Py_ssize_t index) |
| 225 | +{ |
| 226 | + struct _qsbr_shared *shared = &interp->qsbr; |
| 227 | + |
| 228 | + // NOTE: this function is called with runtime locked, so we don't detach |
| 229 | + // while waiting for the lock. This prevents a stop-the-world pause |
| 230 | + // while the runtime lock is held, which could lead to deadlock. |
| 231 | + PyMutex_LockFlags(&shared->mutex, _Py_LOCK_DONT_DETACH); |
| 232 | + struct _qsbr_thread_state *qsbr = &interp->qsbr.array[index].qsbr; |
| 233 | + assert(qsbr->allocated); |
| 234 | + assert(qsbr->tstate == NULL); |
| 235 | + qsbr->tstate = (PyThreadState *)tstate; |
| 236 | + tstate->qsbr = qsbr; |
| 237 | + PyMutex_Unlock(&shared->mutex); |
| 238 | +} |
| 239 | + |
| 240 | +void |
| 241 | +_Py_qsbr_unregister(_PyThreadStateImpl *tstate) |
| 242 | +{ |
| 243 | + struct _qsbr_thread_state *qsbr = tstate->qsbr; |
| 244 | + struct _qsbr_shared *shared = qsbr->shared; |
| 245 | + |
| 246 | + assert(qsbr->seq == 0 && "thread state must be detached"); |
| 247 | + |
| 248 | + PyMutex_LockFlags(&shared->mutex, _Py_LOCK_DONT_DETACH); |
| 249 | + qsbr->tstate = NULL; |
| 250 | + qsbr->allocated = false; |
| 251 | + qsbr->freelist_next = shared->freelist; |
| 252 | + shared->freelist = qsbr; |
| 253 | + PyMutex_Unlock(&shared->mutex); |
| 254 | +} |
| 255 | + |
| 256 | +void |
| 257 | +_Py_qsbr_fini(PyInterpreterState *interp) |
| 258 | +{ |
| 259 | + struct _qsbr_shared *shared = &interp->qsbr; |
| 260 | + PyMem_RawFree(shared->array); |
| 261 | + shared->array = NULL; |
| 262 | + shared->size = 0; |
| 263 | + shared->freelist = NULL; |
| 264 | +} |
| 265 | + |
| 266 | +void |
| 267 | +_Py_qsbr_after_fork(struct _qsbr_shared *shared, struct _qsbr_thread_state *this_qsbr) |
| 268 | +{ |
| 269 | + _PyMutex_at_fork_reinit(&shared->mutex); |
| 270 | + |
| 271 | + for (Py_ssize_t i = 0; i != shared->size; i++) { |
| 272 | + struct _qsbr_thread_state *qsbr = &shared->array[i].qsbr; |
| 273 | + if (qsbr != this_qsbr && qsbr->tstate != NULL) { |
| 274 | + qsbr->tstate = NULL; |
| 275 | + qsbr->allocated = false; |
| 276 | + qsbr->freelist_next = shared->freelist; |
| 277 | + shared->freelist = qsbr; |
| 278 | + } |
| 279 | + } |
| 280 | +} |
0 commit comments