Lock on allocation, use version as sync point

python · Mar 27, 2024 · ac91d8f · ac91d8f
1 parent be10b2d
commit ac91d8f
Show file tree

Hide file tree

Showing 12 changed files with 180 additions and 118 deletions.
diff --git a/Include/cpython/pyatomic.h b/Include/cpython/pyatomic.h
@@ -465,10 +465,16 @@ _Py_atomic_store_ullong_relaxed(unsigned long long *obj,
 static inline void *
 _Py_atomic_load_ptr_acquire(const void *obj);
 
+static inline uintptr_t
+_Py_atomic_load_uintptr_acquire(const uintptr_t *obj);
+
 // Stores `*obj = value` (release operation)
 static inline void
 _Py_atomic_store_ptr_release(void *obj, void *value);
 
+static inline void
+_Py_atomic_store_uintptr_release(uintptr_t *obj, uintptr_t value);
+
 static inline void
 _Py_atomic_store_ssize_release(Py_ssize_t *obj, Py_ssize_t value);
 
@@ -491,6 +497,8 @@ static inline Py_ssize_t
 _Py_atomic_load_ssize_acquire(const Py_ssize_t *obj);
 
 
+
+
 // --- _Py_atomic_fence ------------------------------------------------------
 
 // Sequential consistency fence. C11 fences have complex semantics. When

diff --git a/Include/cpython/pyatomic_gcc.h b/Include/cpython/pyatomic_gcc.h
@@ -492,10 +492,18 @@ static inline void *
 _Py_atomic_load_ptr_acquire(const void *obj)
 { return (void *)__atomic_load_n((void **)obj, __ATOMIC_ACQUIRE); }
 
+static inline uintptr_t
+_Py_atomic_load_uintptr_acquire(const uintptr_t *obj)
+{ return (uintptr_t)__atomic_load_n((uintptr_t *)obj, __ATOMIC_ACQUIRE); }
+
 static inline void
 _Py_atomic_store_ptr_release(void *obj, void *value)
 { __atomic_store_n((void **)obj, value, __ATOMIC_RELEASE); }
 
+static inline void
+_Py_atomic_store_uintptr_release(uintptr_t *obj, uintptr_t value)
+{ __atomic_store_n(obj, value, __ATOMIC_RELEASE); }
+
 static inline void
 _Py_atomic_store_int_release(int *obj, int value)
 { __atomic_store_n(obj, value, __ATOMIC_RELEASE); }

diff --git a/Include/cpython/pyatomic_msc.h b/Include/cpython/pyatomic_msc.h
@@ -914,6 +914,18 @@ _Py_atomic_load_ptr_acquire(const void *obj)
 #endif
 }
 
+static inline uintptr_t
+_Py_atomic_load_uintptr_acquire(const uintptr_t *obj)
+{
+#if defined(_M_X64) || defined(_M_IX86)
+ return *(uintptr_t volatile *)obj;
+#elif defined(_M_ARM64)
+ return (uintptr_t)__ldar64((unsigned __int64 volatile *)obj);
+#else
+# error "no implementation of _Py_atomic_load_ptr_acquire"
+#endif
+}
+
 static inline void
 _Py_atomic_store_ptr_release(void *obj, void *value)
 {
@@ -926,6 +938,19 @@ _Py_atomic_store_ptr_release(void *obj, void *value)
 #endif
 }
 
+static inline void
+_Py_atomic_store_uintptr_release(uintptr_t *obj, uintptr_t value)
+{
+#if defined(_M_X64) || defined(_M_IX86)
+ *(uintptr_t volatile *)obj = value;
+#elif defined(_M_ARM64)
+ _Py_atomic_ASSERT_ARG_TYPE(unsigned __int64);
+ __stlr32((unsigned __int64 volatile *)obj, (unsigned __int64)value);
+#else
+# error "no implementation of _Py_atomic_store_int_release"
+#endif
+}
+
 static inline void
 _Py_atomic_store_int_release(int *obj, int value)
 {

diff --git a/Include/cpython/pyatomic_std.h b/Include/cpython/pyatomic_std.h
@@ -863,6 +863,14 @@ _Py_atomic_load_ptr_acquire(const void *obj)
  memory_order_acquire);
 }
 
+static inline uintptr_t
+_Py_atomic_load_uintptr_acquire(const uintptr_t *obj)
+{
+ _Py_USING_STD;
+ return atomic_load_explicit((const _Atomic(uintptr_t)*)obj,
+ memory_order_acquire);
+}
+
 static inline void
 _Py_atomic_store_ptr_release(void *obj, void *value)
 {
@@ -871,6 +879,14 @@ _Py_atomic_store_ptr_release(void *obj, void *value)
  memory_order_release);
 }
 
+static inline void
+_Py_atomic_store_uintptr_release(uintptr_t *obj, uintptr_t value)
+{
+ _Py_USING_STD;
+ atomic_store_explicit((_Atomic(uintptr_t)*)obj, value,
+ memory_order_release);
+}
+
 static inline void
 _Py_atomic_store_int_release(int *obj, int value)
 {

diff --git a/Include/internal/pycore_gc.h b/Include/internal/pycore_gc.h
@@ -92,7 +92,7 @@ static inline void _PyObject_GC_SET_SHARED(PyObject *op) {
  * threads and needs special purpose when freeing due to
  * the possibility of in-flight lock-free reads occurring.
  * Objects with this bit that are GC objects will automatically
- * delay-freed by PyObject_GC_Del.  */
+ * delay-freed by PyObject_GC_Del. */
 static inline int _PyObject_GC_IS_SHARED_INLINE(PyObject *op) {
  return (op->ob_gc_bits & _PyGC_BITS_SHARED_INLINE) != 0;
 }

diff --git a/Include/internal/pycore_pyatomic_ft_wrappers.h b/Include/internal/pycore_pyatomic_ft_wrappers.h
@@ -25,10 +25,14 @@ extern "C" {
  _Py_atomic_load_ssize_relaxed(&value)
 #define FT_ATOMIC_LOAD_PTR_ACQUIRE(value) \
  _Py_atomic_load_ptr_acquire(&value)
+#define FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(value) \
+ _Py_atomic_load_uintptr_acquire(&value)
 #define FT_ATOMIC_STORE_PTR_RELAXED(value, new_value) \
  _Py_atomic_store_ptr_relaxed(&value, new_value)
 #define FT_ATOMIC_STORE_PTR_RELEASE(value, new_value) \
  _Py_atomic_store_ptr_release(&value, new_value)
+#define FT_ATOMIC_STORE_UINTPTR_RELEASE(value, new_value) \
+ _Py_atomic_store_uintptr_release(&value, new_value)
 #define FT_ATOMIC_STORE_SSIZE_RELAXED(value, new_value) \
  _Py_atomic_store_ssize_relaxed(&value, new_value)
 #define FT_ATOMIC_STORE_UINT8_RELAXED(value, new_value) \
@@ -37,8 +41,10 @@ extern "C" {
 #define FT_ATOMIC_LOAD_SSIZE(value) value
 #define FT_ATOMIC_LOAD_SSIZE_RELAXED(value) value
 #define FT_ATOMIC_LOAD_PTR_ACQUIRE(value) value
+#define FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(value) value
 #define FT_ATOMIC_STORE_PTR_RELAXED(value, new_value) value = new_value
 #define FT_ATOMIC_STORE_PTR_RELEASE(value, new_value) value = new_value
+#define FT_ATOMIC_STORE_UINTPTR_RELEASE(value, new_value) value = new_value
 #define FT_ATOMIC_STORE_SSIZE_RELAXED(value, new_value) value = new_value
 #define FT_ATOMIC_STORE_UINT8_RELAXED(value, new_value) value = new_value
 

diff --git a/Lib/test/test_free_threading/test_monitoring.py b/Lib/test/test_free_threading/test_monitoring.py
@@ -49,7 +49,7 @@ def after_test(self):
  """Runs once after the test is done"""
  pass
 
- def test_instrumention(self):
+ def test_instrumentation(self):
  # Setup a bunch of functions which will need instrumentation...
  funcs = []
  for i in range(self.func_count):

diff --git a/Python/bytecodes.c b/Python/bytecodes.c
@@ -18,6 +18,7 @@
 #include "pycore_object.h" // _PyObject_GC_TRACK()
 #include "pycore_opcode_metadata.h" // uop names
 #include "pycore_opcode_utils.h" // MAKE_FUNCTION_*
+#include "pycore_pyatomic_ft_wrappers.h" // FT_ATOMIC_LOAD_PTR_ACQUIRE
 #include "pycore_pyerrors.h" // _PyErr_GetRaisedException()
 #include "pycore_pystate.h" // _PyInterpreterState_GET()
 #include "pycore_range.h" // _PyRangeIterObject
@@ -148,10 +149,11 @@ dummy_func(
  uintptr_t global_version =
  _Py_atomic_load_uintptr_relaxed(&tstate->eval_breaker) &
  ~_PY_EVAL_EVENTS_MASK;
- uintptr_t code_version = _PyFrame_GetCode(frame)->_co_instrumentation_version;
+ PyCodeObject *code = _PyFrame_GetCode(frame);
+ uintptr_t code_version = FT_ATOMIC_LOAD_UINTPTR_ACQUIRE(code->_co_instrumentation_version);
  assert((code_version & 255) == 0);
  if (code_version != global_version) {
- int err = _Py_Instrument(_PyFrame_GetCode(frame), tstate->interp);
+ int err = _Py_Instrument(code, tstate->interp);
  ERROR_IF(err, error);
  next_instr = this_instr;
  }

diff --git a/Python/ceval.c b/Python/ceval.c
@@ -18,6 +18,7 @@
 #include "pycore_opcode_metadata.h" // EXTRA_CASES
 #include "pycore_optimizer.h" // _PyUOpExecutor_Type
 #include "pycore_opcode_utils.h" // MAKE_FUNCTION_*
+#include "pycore_pyatomic_ft_wrappers.h" // FT_ATOMIC_LOAD_PTR_ACQUIRE
 #include "pycore_pyerrors.h" // _PyErr_GetRaisedException()
 #include "pycore_pystate.h" // _PyInterpreterState_GET()
 #include "pycore_range.h" // _PyRangeIterObject

diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h