WIP

Include/cpython/code.h

@@ -128,6 +128,7 @@ typedef struct {
  _PyCoCached *_co_cached; /* cached co_* attributes */ \
  uintptr_t _co_instrumentation_version; /* current instrumentation version */ \
  _PyCoMonitoringData *_co_monitoring; /* Monitoring data */ \
+ Py_ssize_t _co_unique_id; /* ID used for thread-local refcounting */ \
  int _co_firsttraceable; /* index of first traceable instruction */ \
  /* Scratch space for extra data relating to the code object. \
  Type is a void* to keep the format private in codeobject.c to force \

Include/internal/pycore_interp.h

@@ -221,7 +221,7 @@ struct _is {
 #if defined(Py_GIL_DISABLED)
  struct _mimalloc_interp_state mimalloc;
  struct _brc_state brc; // biased reference counting state
- struct _Py_type_id_pool type_ids;
+ struct _Py_unique_id_pool type_ids;
  PyMutex weakref_locks[NUM_WEAKREF_LIST_LOCKS];
 #endif
 

Include/internal/pycore_object.h

@@ -14,7 +14,7 @@ extern "C" {
 #include "pycore_interp.h" // PyInterpreterState.gc
 #include "pycore_pyatomic_ft_wrappers.h" // FT_ATOMIC_STORE_PTR_RELAXED
 #include "pycore_pystate.h" // _PyInterpreterState_GET()
-#include "pycore_typeid.h" // _PyType_IncrefSlow
+#include "pycore_typeid.h" // _PyObject_ThreadIncrefSlow()
 
 
 #define _Py_IMMORTAL_REFCNT_LOOSE ((_Py_IMMORTAL_REFCNT >> 1) + 1)
@@ -311,7 +311,31 @@ extern bool _PyRefchain_IsTraced(PyInterpreterState *interp, PyObject *obj);
 #ifndef Py_GIL_DISABLED
 # define _Py_INCREF_TYPE Py_INCREF
 # define _Py_DECREF_TYPE Py_DECREF
+# define _Py_INCREF_CODE Py_INCREF
+# define _Py_DECREF_CODE Py_DECREF
 #else
+static inline void
+_Py_THREAD_INCREF_OBJECT(PyObject *obj, Py_ssize_t unique_id)
+{
+ _PyThreadStateImpl *tstate = (_PyThreadStateImpl *)_PyThreadState_GET();
+
+ // Unsigned comparison so that `unique_id=-1`, which indicates that
+ // per-thread refcounting has been disabled on this object, is handled by
+ // the "else".
+ if ((size_t)unique_id < (size_t)tstate->types.size) {
+# ifdef Py_REF_DEBUG
+ _Py_INCREF_IncRefTotal();
+# endif
+ _Py_INCREF_STAT_INC();
+ tstate->types.refcounts[unique_id]++;
+ }
+ else {
+ // The slow path resizes the thread-local refcount array if necessary.
+ // It handles the unique_id=-1 case to keep the inlinable function smaller.
+ _PyObject_ThreadIncrefSlow(obj, unique_id);
+ }
+}
+
 static inline void
 _Py_INCREF_TYPE(PyTypeObject *type)
 {
@@ -328,29 +352,38 @@ _Py_INCREF_TYPE(PyTypeObject *type)
 # pragma GCC diagnostic push
 # pragma GCC diagnostic ignored "-Warray-bounds"
 #endif
+ _Py_THREAD_INCREF_OBJECT((PyObject *)type, ((PyHeapTypeObject *)type)->unique_id);
+#if defined(__GNUC__) && __GNUC__ >= 11
+# pragma GCC diagnostic pop
+#endif
+}
 
+static inline void
+_Py_INCREF_CODE(PyCodeObject *co)
+{
+ _Py_THREAD_INCREF_OBJECT((PyObject *)co, co->_co_unique_id);
+}
+
+static inline void
+_Py_THREAD_DECREF_OBJECT(PyObject *obj, Py_ssize_t unique_id)
+{
  _PyThreadStateImpl *tstate = (_PyThreadStateImpl *)_PyThreadState_GET();
- PyHeapTypeObject *ht = (PyHeapTypeObject *)type;
 
  // Unsigned comparison so that `unique_id=-1`, which indicates that
- // per-thread refcounting has been disabled on this type, is handled by
+ // per-thread refcounting has been disabled on this object, is handled by
  // the "else".
- if ((size_t)ht->unique_id < (size_t)tstate->types.size) {
+ if ((size_t)unique_id < (size_t)tstate->types.size) {
 # ifdef Py_REF_DEBUG
- _Py_INCREF_IncRefTotal();
+ _Py_DECREF_DecRefTotal();
 # endif
- _Py_INCREF_STAT_INC();
- tstate->types.refcounts[ht->unique_id]++;
+ _Py_DECREF_STAT_INC();
+ tstate->types.refcounts[unique_id]--;
  }
  else {
- // The slow path resizes the thread-local refcount array if necessary.
- // It handles the unique_id=-1 case to keep the inlinable function smaller.
- _PyType_IncrefSlow(ht);
+ // Directly decref the type if the type id is not assigned or if
+ // per-thread refcounting has been disabled on this type.
+ Py_DECREF(obj);
  }
-
-#if defined(__GNUC__) && __GNUC__ >= 11
-# pragma GCC diagnostic pop
-#endif
 }
 
 static inline void
@@ -361,25 +394,14 @@ _Py_DECREF_TYPE(PyTypeObject *type)
  _Py_DECREF_IMMORTAL_STAT_INC();
  return;
  }
-
- _PyThreadStateImpl *tstate = (_PyThreadStateImpl *)_PyThreadState_GET();
  PyHeapTypeObject *ht = (PyHeapTypeObject *)type;
+ _Py_THREAD_DECREF_OBJECT((PyObject *)type, ht->unique_id);
+}
 
- // Unsigned comparison so that `unique_id=-1`, which indicates that
- // per-thread refcounting has been disabled on this type, is handled by
- // the "else".
- if ((size_t)ht->unique_id < (size_t)tstate->types.size) {
-# ifdef Py_REF_DEBUG
- _Py_DECREF_DecRefTotal();
-# endif
- _Py_DECREF_STAT_INC();
- tstate->types.refcounts[ht->unique_id]--;
- }
- else {
- // Directly decref the type if the type id is not assigned or if
- // per-thread refcounting has been disabled on this type.
- Py_DECREF(type);
- }
+static inline void
+_Py_DECREF_CODE(PyCodeObject *co)
+{
+ _Py_THREAD_DECREF_OBJECT((PyObject *)co, co->_co_unique_id);
 }
 #endif
 

Include/internal/pycore_uniqueid.h

@@ -0,0 +1,76 @@
+#ifndef Py_INTERNAL_TYPEID_H
+#define Py_INTERNAL_TYPEID_H
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#ifndef Py_BUILD_CORE
+# error "this header requires Py_BUILD_CORE define"
+#endif
+
+#ifdef Py_GIL_DISABLED
+
+// This contains code for allocating unique ids for objects
+// and re-using those ids when the object is deallocated.
+//
+// The unique ids are used to implement per-thread reference counts of
+// heap types and code objects to avoid contention on the reference count
+// fields. Static type objects are immortal, so contention is not an issue for
+// those types.
+//
+// An id of -1 is used to indicate that the object doesn't use per-thread
+// refcounting. This value is used when a type object is finalized by the GC
+// and during interpreter shutdown to allow the type object to be
+// deallocated promptly when the object's refcount reaches zero.
+//
+// Each entry implicitly represents a unique id based on it's offset in the
+// table. Non-allocated entries form a free-list via the 'next' pointer.
+// Allocated entries store the corresponding PyObject.
+typedef union _Py_unique_id_entry {
+ // Points to the next free type id, when part of the freelist
+ union _Py_unique_id_entry *next;
+
+ // Stores object when the id is assigned
+ PyObject *object;
+} _Py_unique_id_entry;
+
+struct _Py_unique_id_pool {
+ PyMutex mutex;
+
+ // combined table of types with allocated type ids and unallocated
+ // type ids.
+ _Py_unique_id_entry *table;
+
+ // Next entry to allocate inside 'table' or NULL
+ _Py_unique_id_entry *freelist;
+
+ // size of 'table'
+ Py_ssize_t size;
+};
+
+// Assigns the next id from the pool of type ids.
+extern Py_ssize_t _PyObject_AssignUniqueId(PyObject *obj);
+
+// Releases the allocated id back to the pool.
+extern void _PyObject_ReleaseUniqueId(Py_ssize_t id);
+
+// Merges the thread-local reference counts into the corresponding types.
+extern void _PyObject_MergeThreadLocalRefcounts(_PyThreadStateImpl *tstate);
+
+// Like _PyObject_MergeThreadLocalRefcounts, but also frees the thread-local
+// array of refcounts.
+extern void _PyObject_FinalizeThreadLocalRefcounts(_PyThreadStateImpl *tstate);
+
+// Frees the interpreter's pool of type ids.
+extern void _PyObject_FinalizeIdPool(PyInterpreterState *interp);
+
+// Increfs the object, resizing the thread-local refcount array if necessary.
+PyAPI_FUNC(void) _PyObject_ThreadIncrefSlow(PyObject *obj, Py_ssize_t unique_id);
+
+
+#endif /* Py_GIL_DISABLED */
+
+#ifdef __cplusplus
+}
+#endif
+#endif /* !Py_INTERNAL_TYPEID_H */

Objects/codeobject.c

@@ -14,6 +14,7 @@
 #include "pycore_pystate.h" // _PyInterpreterState_GET()
 #include "pycore_setobject.h" // _PySet_NextEntry()
 #include "pycore_tuple.h" // _PyTuple_ITEMS()
+#include "pycore_typeid.h" // _PyObject_AssignRefcountId()
 #include "clinic/codeobject.c.h"
 
 static const char *
@@ -676,7 +677,7 @@ _PyCode_New(struct _PyCodeConstructor *con)
  }
  init_code(co, con);
 #ifdef Py_GIL_DISABLED
- _PyObject_SetDeferredRefcount((PyObject *)co);
+ co->_co_unique_id = _PyObject_AssignUniqueId((PyObject *)co);
  _PyObject_GC_TRACK(co);
 #endif
  Py_XDECREF(replacement_locations);
@@ -1859,6 +1860,9 @@ code_dealloc(PyCodeObject *co)
  Py_XDECREF(co->co_qualname);
  Py_XDECREF(co->co_linetable);
  Py_XDECREF(co->co_exceptiontable);
+ if (co->_co_unique_id != -1) {
+ _PyObject_ReleaseUniqueId(co->_co_unique_id);
+ }
  if (co->_co_cached != NULL) {
  Py_XDECREF(co->_co_cached->_co_code);
  Py_XDECREF(co->_co_cached->_co_cellvars);

Objects/typeobject.c

@@ -3932,7 +3932,7 @@ type_new_alloc(type_new_ctx *ctx)
  et->ht_token = NULL;
 
 #ifdef Py_GIL_DISABLED
- _PyType_AssignId(et);
+ et->unique_id = _PyObject_AssignUniqueId((PyObject *)et);
 #endif
 
  return type;
@@ -5025,8 +5025,8 @@ PyType_FromMetaclass(
  type->tp_dictoffset = dictoffset;
 
 #ifdef Py_GIL_DISABLED
- // Assign a type id to enable thread-local refcounting
- _PyType_AssignId(res);
+ // Assign a unique id to enable per-thread refcounting
+ res->unique_id = _PyObject_AssignUniqueId((PyObject *)res);
 #endif
 
  /* Ready the type (which includes inheritance).
@@ -6080,7 +6080,7 @@ type_dealloc(PyObject *self)
  Py_XDECREF(et->ht_module);
  PyMem_Free(et->_ht_tpname);
 #ifdef Py_GIL_DISABLED
- _PyType_ReleaseId(et);
+ _PyObject_ReleaseUniqueId(et->unique_id);
 #endif
  et->ht_token = NULL;
  Py_TYPE(type)->tp_free((PyObject *)type);

Python/gc_free_threading.c

@@ -15,7 +15,7 @@
 #include "pycore_tstate.h" // _PyThreadStateImpl
 #include "pycore_weakref.h" // _PyWeakref_ClearRef()
 #include "pydtrace.h"
-#include "pycore_typeid.h" // _PyType_MergeThreadLocalRefcounts
+#include "pycore_typeid.h" // _PyObject_MergeThreadLocalRefcounts
 
 #ifdef Py_GIL_DISABLED
 
@@ -218,12 +218,10 @@ disable_deferred_refcounting(PyObject *op)
  }
 
  // Heap types also use thread-local refcounting -- disable it here.
- if (PyType_Check(op)) {
- // Disable thread-local refcounting for heap types
- PyTypeObject *type = (PyTypeObject *)op;
- if (PyType_HasFeature(type, Py_TPFLAGS_HEAPTYPE)) {
- _PyType_ReleaseId((PyHeapTypeObject *)op);
- }
+ if (PyType_Check(op) && PyType_HasFeature((PyTypeObject *)op, Py_TPFLAGS_HEAPTYPE)) {
+ PyHeapTypeObject *ht = (PyHeapTypeObject *)op;
+ _PyObject_ReleaseUniqueId(ht->unique_id);
+ ht->unique_id = -1;
  }
 
  // Generators and frame objects may contain deferred references to other
@@ -1221,7 +1219,7 @@ gc_collect_internal(PyInterpreterState *interp, struct collection_state *state,
  _PyThreadStateImpl *tstate = (_PyThreadStateImpl *)p;
 
  // merge per-thread refcount for types into the type's actual refcount
- _PyType_MergeThreadLocalRefcounts(tstate);
+ _PyObject_MergeThreadLocalRefcounts(tstate);
 
  // merge refcounts for all queued objects
  merge_queued_objects(tstate, state);