Skip to content

Commit

Permalink
gh-115999: Specialize LOAD_GLOBAL in free-threaded builds (#126607)
Browse files Browse the repository at this point in the history 
Enable specialization of LOAD_GLOBAL in free-threaded builds.

Thread-safety of specialization in free-threaded builds is provided by the following:

A critical section is held on both the globals and builtins objects during specialization. This ensures we get an atomic view of both builtins and globals during specialization.
Generation of new keys versions is made atomic in free-threaded builds.
Existing helpers are used to atomically modify the opcode.
Thread-safety of specialized instructions in free-threaded builds is provided by the following:

Relaxed atomics are used when loading and storing dict keys versions. This avoids potential data races as the dict keys versions are read without holding the dictionary's per-object lock in version guards.
Dicts keys objects are passed from keys version guards to the downstream uops. This ensures that we are loading from the correct offset in the keys object. Once a unicode key has been stored in a keys object for a combined dictionary in free-threaded builds, the offset that it is stored in will never be reused for a different key. Once the version guard passes, we know that we are reading from the correct offset.
The dictionary read fast-path is used to read values from the dictionary once we know the correct offset.
  • Loading branch information
@mpage
mpage authored Nov 21, 2024
1 parent 9dabace commit 09c240f
Show file tree
Hide file tree
Showing 11 changed files with 222 additions and 69 deletions.
11 changes: 11 additions & 0 deletions Include/internal/pycore_dict.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -90,6 +90,17 @@ extern PyObject *_PyDict_FromKeys(PyObject *, PyObject *, PyObject *);
extern uint32_t _PyDictKeys_GetVersionForCurrentState(
PyInterpreterState *interp, PyDictKeysObject *dictkeys);

/* Gets a version number unique to the current state of the keys of dict, if possible.
*
* In free-threaded builds ensures that the dict can be used for lock-free
* reads if a version was assigned.
*
* The caller must hold the per-object lock on dict.
*
* Returns the version number, or zero if it was not possible to get a version number. */
extern uint32_t _PyDict_GetKeysVersionForCurrentState(
PyInterpreterState *interp, PyDictObject *dict);

extern size_t _PyDict_KeysSize(PyDictKeysObject *keys);

extern void _PyDictKeys_DecRef(PyDictKeysObject *keys);
Expand Down
15 changes: 15 additions & 0 deletions Include/internal/pycore_object.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ extern "C" {
#include "pycore_interp.h" // PyInterpreterState.gc
#include "pycore_pyatomic_ft_wrappers.h" // FT_ATOMIC_STORE_PTR_RELAXED
#include "pycore_pystate.h" // _PyInterpreterState_GET()
#include "pycore_stackref.h"
#include "pycore_uniqueid.h" // _PyObject_ThreadIncrefSlow()

// This value is added to `ob_ref_shared` for objects that use deferred
Expand Down Expand Up @@ -595,6 +596,20 @@ _Py_TryIncrefCompare(PyObject **src, PyObject *op)
return 1;
}

static inline int
_Py_TryIncrefCompareStackRef(PyObject **src, PyObject *op, _PyStackRef *out)
{
if (_Py_IsImmortal(op) || _PyObject_HasDeferredRefcount(op)) {
*out = (_PyStackRef){ .bits = (intptr_t)op | Py_TAG_DEFERRED };
return 1;
}
if (_Py_TryIncrefCompare(src, op)) {
*out = PyStackRef_FromPyObjectSteal(op);
return 1;
}
return 0;
}

/* Loads and increfs an object from ptr, which may contain a NULL value.
Safe with concurrent (atomic) updates to ptr.
NOTE: The writer must set maybe-weakref on the stored object! */
Expand Down
19 changes: 18 additions & 1 deletion Lib/test/test_opcache.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -546,7 +546,6 @@ def count_args(self, *args):


@threading_helper.requires_working_threading()
@requires_specialization
class TestRacesDoNotCrash(TestBase):
# Careful with these. Bigger numbers have a higher chance of catching bugs,
# but you can also burn through a *ton* of type/dict/function versions:
Expand Down Expand Up @@ -588,6 +587,7 @@ def assert_races_do_not_crash(
for writer in writers:
writer.join()

@requires_specialization
def test_binary_subscr_getitem(self):
def get_items():
class C:
Expand Down Expand Up @@ -617,6 +617,7 @@ def write(items):
opname = "BINARY_SUBSCR_GETITEM"
self.assert_races_do_not_crash(opname, get_items, read, write)

@requires_specialization
def test_binary_subscr_list_int(self):
def get_items():
items = []
Expand All @@ -640,6 +641,7 @@ def write(items):
opname = "BINARY_SUBSCR_LIST_INT"
self.assert_races_do_not_crash(opname, get_items, read, write)

@requires_specialization
def test_for_iter_gen(self):
def get_items():
def g():
Expand Down Expand Up @@ -671,6 +673,7 @@ def write(items):
opname = "FOR_ITER_GEN"
self.assert_races_do_not_crash(opname, get_items, read, write)

@requires_specialization
def test_for_iter_list(self):
def get_items():
items = []
Expand All @@ -692,6 +695,7 @@ def write(items):
opname = "FOR_ITER_LIST"
self.assert_races_do_not_crash(opname, get_items, read, write)

@requires_specialization
def test_load_attr_class(self):
def get_items():
class C:
Expand Down Expand Up @@ -721,6 +725,7 @@ def write(items):
opname = "LOAD_ATTR_CLASS"
self.assert_races_do_not_crash(opname, get_items, read, write)

@requires_specialization
def test_load_attr_getattribute_overridden(self):
def get_items():
class C:
Expand Down Expand Up @@ -750,6 +755,7 @@ def write(items):
opname = "LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN"
self.assert_races_do_not_crash(opname, get_items, read, write)

@requires_specialization
def test_load_attr_instance_value(self):
def get_items():
class C:
Expand All @@ -773,6 +779,7 @@ def write(items):
opname = "LOAD_ATTR_INSTANCE_VALUE"
self.assert_races_do_not_crash(opname, get_items, read, write)

@requires_specialization
def test_load_attr_method_lazy_dict(self):
def get_items():
class C(Exception):
Expand Down Expand Up @@ -802,6 +809,7 @@ def write(items):
opname = "LOAD_ATTR_METHOD_LAZY_DICT"
self.assert_races_do_not_crash(opname, get_items, read, write)

@requires_specialization
def test_load_attr_method_no_dict(self):
def get_items():
class C:
Expand Down Expand Up @@ -832,6 +840,7 @@ def write(items):
opname = "LOAD_ATTR_METHOD_NO_DICT"
self.assert_races_do_not_crash(opname, get_items, read, write)

@requires_specialization
def test_load_attr_method_with_values(self):
def get_items():
class C:
Expand Down Expand Up @@ -861,6 +870,7 @@ def write(items):
opname = "LOAD_ATTR_METHOD_WITH_VALUES"
self.assert_races_do_not_crash(opname, get_items, read, write)

@requires_specialization
def test_load_attr_module(self):
def get_items():
items = []
Expand All @@ -885,6 +895,7 @@ def write(items):
opname = "LOAD_ATTR_MODULE"
self.assert_races_do_not_crash(opname, get_items, read, write)

@requires_specialization
def test_load_attr_property(self):
def get_items():
class C:
Expand Down Expand Up @@ -914,6 +925,7 @@ def write(items):
opname = "LOAD_ATTR_PROPERTY"
self.assert_races_do_not_crash(opname, get_items, read, write)

@requires_specialization
def test_load_attr_with_hint(self):
def get_items():
class C:
Expand All @@ -940,6 +952,7 @@ def write(items):
opname = "LOAD_ATTR_WITH_HINT"
self.assert_races_do_not_crash(opname, get_items, read, write)

@requires_specialization_ft
def test_load_global_module(self):
def get_items():
items = []
Expand All @@ -961,6 +974,7 @@ def write(items):
opname, get_items, read, write, check_items=True
)

@requires_specialization
def test_store_attr_instance_value(self):
def get_items():
class C:
Expand All @@ -983,6 +997,7 @@ def write(items):
opname = "STORE_ATTR_INSTANCE_VALUE"
self.assert_races_do_not_crash(opname, get_items, read, write)

@requires_specialization
def test_store_attr_with_hint(self):
def get_items():
class C:
Expand All @@ -1008,6 +1023,7 @@ def write(items):
opname = "STORE_ATTR_WITH_HINT"
self.assert_races_do_not_crash(opname, get_items, read, write)

@requires_specialization
def test_store_subscr_list_int(self):
def get_items():
items = []
Expand All @@ -1031,6 +1047,7 @@ def write(items):
opname = "STORE_SUBSCR_LIST_INT"
self.assert_races_do_not_crash(opname, get_items, read, write)

@requires_specialization
def test_unpack_sequence_list(self):
def get_items():
items = []
Expand Down
68 changes: 58 additions & 10 deletions Objects/dictobject.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1285,6 +1285,20 @@ ensure_shared_on_resize(PyDictObject *mp)
#endif
}

static inline void
ensure_shared_on_keys_version_assignment(PyDictObject *mp)
{
ASSERT_DICT_LOCKED((PyObject *) mp);
#ifdef Py_GIL_DISABLED
if (!IS_DICT_SHARED(mp)) {
// This ensures that a concurrent resize operation will delay
// freeing the old keys or values using QSBR, which is necessary to
// safely allow concurrent reads without locking.
SET_DICT_SHARED(mp);
}
#endif
}

#ifdef Py_GIL_DISABLED

static inline Py_ALWAYS_INLINE int
Expand Down Expand Up @@ -1644,7 +1658,7 @@ insert_combined_dict(PyInterpreterState *interp, PyDictObject *mp,
}

_PyDict_NotifyEvent(interp, PyDict_EVENT_ADDED, mp, key, value);
mp->ma_keys->dk_version = 0;
FT_ATOMIC_STORE_UINT32_RELAXED(mp->ma_keys->dk_version, 0);

Py_ssize_t hashpos = find_empty_slot(mp->ma_keys, hash);
dictkeys_set_index(mp->ma_keys, hashpos, mp->ma_keys->dk_nentries);
Expand Down Expand Up @@ -1686,7 +1700,7 @@ insert_split_key(PyDictKeysObject *keys, PyObject *key, Py_hash_t hash)
ix = unicodekeys_lookup_unicode(keys, key, hash);
if (ix == DKIX_EMPTY && keys->dk_usable > 0) {
// Insert into new slot
keys->dk_version = 0;
FT_ATOMIC_STORE_UINT32_RELAXED(keys->dk_version, 0);
Py_ssize_t hashpos = find_empty_slot(keys, hash);
ix = keys->dk_nentries;
dictkeys_set_index(keys, hashpos, ix);
Expand Down Expand Up @@ -2617,7 +2631,7 @@ delitem_common(PyDictObject *mp, Py_hash_t hash, Py_ssize_t ix,
ASSERT_CONSISTENT(mp);
}
else {
mp->ma_keys->dk_version = 0;
FT_ATOMIC_STORE_UINT32_RELAXED(mp->ma_keys->dk_version, 0);
dictkeys_set_index(mp->ma_keys, hashpos, DKIX_DUMMY);
if (DK_IS_UNICODE(mp->ma_keys)) {
PyDictUnicodeEntry *ep = &DK_UNICODE_ENTRIES(mp->ma_keys)[ix];
Expand Down Expand Up @@ -4429,7 +4443,7 @@ dict_popitem_impl(PyDictObject *self)
return NULL;
}
}
self->ma_keys->dk_version = 0;
FT_ATOMIC_STORE_UINT32_RELAXED(self->ma_keys->dk_version, 0);

/* Pop last item */
PyObject *key, *value;
Expand Down Expand Up @@ -7417,20 +7431,54 @@ _PyDictKeys_DecRef(PyDictKeysObject *keys)
dictkeys_decref(interp, keys, false);
}

uint32_t _PyDictKeys_GetVersionForCurrentState(PyInterpreterState *interp,
PyDictKeysObject *dictkeys)
static inline uint32_t
get_next_dict_keys_version(PyInterpreterState *interp)
{
if (dictkeys->dk_version != 0) {
return dictkeys->dk_version;
}
#ifdef Py_GIL_DISABLED
uint32_t v;
do {
v = _Py_atomic_load_uint32_relaxed(
&interp->dict_state.next_keys_version);
if (v == 0) {
return 0;
}
} while (!_Py_atomic_compare_exchange_uint32(
&interp->dict_state.next_keys_version, &v, v + 1));
#else
if (interp->dict_state.next_keys_version == 0) {
return 0;
}
uint32_t v = interp->dict_state.next_keys_version++;
dictkeys->dk_version = v;
#endif
return v;
}

// In free-threaded builds the caller must ensure that the keys object is not
// being mutated concurrently by another thread.
uint32_t
_PyDictKeys_GetVersionForCurrentState(PyInterpreterState *interp,
PyDictKeysObject *dictkeys)
{
uint32_t dk_version = FT_ATOMIC_LOAD_UINT32_RELAXED(dictkeys->dk_version);
if (dk_version != 0) {
return dk_version;
}
dk_version = get_next_dict_keys_version(interp);
FT_ATOMIC_STORE_UINT32_RELAXED(dictkeys->dk_version, dk_version);
return dk_version;
}

uint32_t
_PyDict_GetKeysVersionForCurrentState(PyInterpreterState *interp,
PyDictObject *dict)
{
ASSERT_DICT_LOCKED((PyObject *) dict);
uint32_t dk_version =
_PyDictKeys_GetVersionForCurrentState(interp, dict->ma_keys);
ensure_shared_on_keys_version_assignment(dict);
return dk_version;
}

static inline int
validate_watcher_id(PyInterpreterState *interp, int watcher_id)
{
Expand Down
2 changes: 2 additions & 0 deletions Objects/funcobject.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -289,12 +289,14 @@ functions is running.
*/

#ifndef Py_GIL_DISABLED
static inline struct _func_version_cache_item *
get_cache_item(PyInterpreterState *interp, uint32_t version)
{
return interp->func_state.func_version_cache +
(version % FUNC_VERSION_CACHE_SIZE);
}
#endif

void
_PyFunction_SetVersion(PyFunctionObject *func, uint32_t version)
Expand Down
Loading

0 comments on commit 09c240f

Please sign in to comment.