pythongh-124218: Refactor per-thread reference counting

Currently, we only use per-thread reference counting for heap type
objects and the naming reflects that. In an upcoming change we will
extend it to code objects and certain dictionaries used in
`PyFunctionObject` to avoid scaling bottlenecks when creating nested
functions.

Rename some of the files and functions in preparation for this change.

Include/internal/pycore_interp.h

@@ -35,7 +35,7 @@ extern "C" {
 #include "pycore_qsbr.h" // struct _qsbr_state
 #include "pycore_tstate.h" // _PyThreadStateImpl
 #include "pycore_tuple.h" // struct _Py_tuple_state
-#include "pycore_typeid.h"  // struct _Py_type_id_pool
+#include "pycore_uniqueid.h" // struct _Py_unique_id_pool
 #include "pycore_typeobject.h" // struct types_state
 #include "pycore_unicodeobject.h" // struct _Py_unicode_state
 #include "pycore_warnings.h" // struct _warnings_runtime_state
@@ -221,7 +221,7 @@ struct _is {
 #if defined(Py_GIL_DISABLED)
  struct _mimalloc_interp_state mimalloc;
  struct _brc_state brc; // biased reference counting state
- struct _Py_type_id_pool type_ids;
+ struct _Py_unique_id_pool unique_ids; // object ids for per-thread refcounts
  PyMutex weakref_locks[NUM_WEAKREF_LIST_LOCKS];
 #endif
 

Include/internal/pycore_object.h

@@ -14,7 +14,7 @@ extern "C" {
 #include "pycore_interp.h" // PyInterpreterState.gc
 #include "pycore_pyatomic_ft_wrappers.h" // FT_ATOMIC_STORE_PTR_RELAXED
 #include "pycore_pystate.h" // _PyInterpreterState_GET()
-#include "pycore_typeid.h"  // _PyType_IncrefSlow
+#include "pycore_uniqueid.h" // _PyType_IncrefSlow
 
 
 #define _Py_IMMORTAL_REFCNT_LOOSE ((_Py_IMMORTAL_REFCNT >> 1) + 1)
@@ -335,12 +335,12 @@ _Py_INCREF_TYPE(PyTypeObject *type)
  // Unsigned comparison so that `unique_id=-1`, which indicates that
  // per-thread refcounting has been disabled on this type, is handled by
  // the "else".
- if ((size_t)ht->unique_id < (size_t)tstate->types.size) {
+ if ((size_t)ht->unique_id < (size_t)tstate->refcounts.size) {
 # ifdef Py_REF_DEBUG
  _Py_INCREF_IncRefTotal();
 # endif
  _Py_INCREF_STAT_INC();
- tstate->types.refcounts[ht->unique_id]++;
+ tstate->refcounts.values[ht->unique_id]++;
  }
  else {
  // The slow path resizes the thread-local refcount array if necessary.
@@ -368,12 +368,12 @@ _Py_DECREF_TYPE(PyTypeObject *type)
  // Unsigned comparison so that `unique_id=-1`, which indicates that
  // per-thread refcounting has been disabled on this type, is handled by
  // the "else".
- if ((size_t)ht->unique_id < (size_t)tstate->types.size) {
+ if ((size_t)ht->unique_id < (size_t)tstate->refcounts.size) {
 # ifdef Py_REF_DEBUG
  _Py_DECREF_DecRefTotal();
 # endif
  _Py_DECREF_STAT_INC();
- tstate->types.refcounts[ht->unique_id]--;
+ tstate->refcounts.values[ht->unique_id]--;
  }
  else {
  // Directly decref the type if the type id is not assigned or if

Include/internal/pycore_tstate.h

@@ -32,15 +32,15 @@ typedef struct _PyThreadStateImpl {
  struct _Py_freelists freelists;
  struct _brc_thread_state brc;
  struct {
- // The thread-local refcounts for heap type objects
- Py_ssize_t *refcounts;
+ // The per-thread refcounts
+ Py_ssize_t *values;
 
  // Size of the refcounts array.
  Py_ssize_t size;
 
- // If set, don't use thread-local refcounts
+ // If set, don't use per-thread refcounts
  int is_finalized;
- } types;
+ } refcounts;
 #endif
 
 #if defined(Py_REF_DEBUG) && defined(Py_GIL_DISABLED)

Include/internal/pycore_uniqueid.h

@@ -0,0 +1,72 @@
+#ifndef Py_INTERNAL_UNIQUEID_H
+#define Py_INTERNAL_UNIQUEID_H
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#ifndef Py_BUILD_CORE
+# error "this header requires Py_BUILD_CORE define"
+#endif
+
+#ifdef Py_GIL_DISABLED
+
+// This contains code for allocating and reusing unique object ids, which are
+// used to implement per-thread reference counting.
+// heap type objects to avoid contention on the reference count fields
+// of heap type objects. Static type objects are immortal, so contention
+// is not an issue for those types.
+//
+// An id of -1 is used to indicate that an object doesn't use per-thread
+// refcounting. This value is used when a type object is finalized by the GC
+// and during interpreter shutdown to allow the type object to be
+// deallocated promptly when the object's refcount reaches zero.
+//
+// Each entry implicitly represents a unique id based on its offset in the
+// table. Non-allocated entries form a free-list via the 'next' pointer.
+// Allocated entries store the corresponding PyObject.
+typedef union _Py_unique_id_entry {
+ // Points to the next free type id, when part of the freelist
+ union _Py_unique_id_entry *next;
+
+ // Stores the object when the id is assigned
+ PyObject *obj;
+} _Py_unique_id_entry;
+
+struct _Py_unique_id_pool {
+ PyMutex mutex;
+
+ // combined table of object with allocated unique ids and unallocated ids.
+ _Py_unique_id_entry *table;
+
+ // Next entry to allocate inside 'table' or NULL
+ _Py_unique_id_entry *freelist;
+
+ // size of 'table'
+ Py_ssize_t size;
+};
+
+// Assigns the next id from the pool of ids.
+extern Py_ssize_t _PyObject_AssignUniqueId(PyObject *obj);
+
+// Releases the allocated id back to the pool.
+extern void _PyObject_ReleaseUniqueId(Py_ssize_t unique_id);
+
+// Merges the per-thread reference counts into the corresponding objects.
+extern void _PyObject_MergePerThreadRefcounts(_PyThreadStateImpl *tstate);
+
+// Like _PyObject_MergePerThreadRefcounts, but also frees the per-thread
+// array of refcounts.
+extern void _PyObject_FinalizePerThreadRefcounts(_PyThreadStateImpl *tstate);
+
+// Frees the interpreter's pool of type ids.
+extern void _PyObject_FinalizeUniqueIdPool(PyInterpreterState *interp);
+
+// Increfs the type, resizing the per-thread refcount array if necessary.
+PyAPI_FUNC(void) _PyType_IncrefSlow(PyHeapTypeObject *type);
+
+#endif /* Py_GIL_DISABLED */
+
+#ifdef __cplusplus
+}
+#endif
+#endif /* !Py_INTERNAL_UNIQUEID_H */

Makefile.pre.in

@@ -490,7 +490,7 @@ PYTHON_OBJS= \
  Python/thread.o \
  Python/traceback.o \
  Python/tracemalloc.o \
- Python/typeid.o \
+ Python/uniqueid.o \
  Python/getopt.o \
  Python/pystrcmp.o \
  Python/pystrtod.o \
@@ -1279,7 +1279,7 @@ PYTHON_HEADERS= \
  $(srcdir)/Include/internal/pycore_tracemalloc.h \
  $(srcdir)/Include/internal/pycore_tstate.h \
  $(srcdir)/Include/internal/pycore_tuple.h \
- $(srcdir)/Include/internal/pycore_typeid.h \
+ $(srcdir)/Include/internal/pycore_uniqueid.h \
  $(srcdir)/Include/internal/pycore_typeobject.h \
  $(srcdir)/Include/internal/pycore_typevarobject.h \
  $(srcdir)/Include/internal/pycore_ucnhash.h \

Objects/typeobject.c

@@ -3932,7 +3932,7 @@ type_new_alloc(type_new_ctx *ctx)
  et->ht_token = NULL;
 
 #ifdef Py_GIL_DISABLED
- _PyType_AssignId(et);
+ et->unique_id = _PyObject_AssignUniqueId((PyObject *)et);
 #endif
 
  return type;
@@ -5026,7 +5026,7 @@ PyType_FromMetaclass(
 
 #ifdef Py_GIL_DISABLED
  // Assign a type id to enable thread-local refcounting
- _PyType_AssignId(res);
+ res->unique_id = _PyObject_AssignUniqueId((PyObject *)res);
 #endif
 
  /* Ready the type (which includes inheritance).
@@ -6080,7 +6080,7 @@ type_dealloc(PyObject *self)
  Py_XDECREF(et->ht_module);
  PyMem_Free(et->_ht_tpname);
 #ifdef Py_GIL_DISABLED
- _PyType_ReleaseId(et);
+ _PyObject_ReleaseUniqueId(et->unique_id);
 #endif
  et->ht_token = NULL;
  Py_TYPE(type)->tp_free((PyObject *)type);

PCbuild/_freeze_module.vcxproj

@@ -268,7 +268,7 @@
  <ClCompile Include="..\Python\thread.c" />
  <ClCompile Include="..\Python\traceback.c" />
  <ClCompile Include="..\Python\tracemalloc.c" />
- <ClCompile Include="..\Python\typeid.c" />
+ <ClCompile Include="..\Python\uniqueid.c" />
  </ItemGroup>
  <ItemGroup>
  <ClInclude Include="..\PC\pyconfig.h.in" />

PCbuild/_freeze_module.vcxproj.filters

@@ -467,7 +467,7 @@
  <ClCompile Include="..\Python\tracemalloc.c">
  <Filter>Source Files</Filter>
  </ClCompile>
- <ClCompile Include="..\Python\typeid.c">
+ <ClCompile Include="..\Python\uniqueid.c">
  <Filter>Source Files</Filter>
  </ClCompile>
  <ClCompile Include="..\Objects\tupleobject.c">

PCbuild/pythoncore.vcxproj

@@ -304,13 +304,13 @@
  <ClInclude Include="..\Include\internal\pycore_tracemalloc.h" />
  <ClInclude Include="..\Include\internal\pycore_tstate.h" />
  <ClInclude Include="..\Include\internal\pycore_tuple.h" />
- <ClInclude Include="..\Include\internal\pycore_typeid.h" />
  <ClInclude Include="..\Include\internal\pycore_typeobject.h" />
  <ClInclude Include="..\Include\internal\pycore_typevarobject.h" />
  <ClInclude Include="..\Include\internal\pycore_ucnhash.h" />
  <ClInclude Include="..\Include\internal\pycore_unionobject.h" />
  <ClInclude Include="..\Include\internal\pycore_unicodeobject.h" />
  <ClInclude Include="..\Include\internal\pycore_unicodeobject_generated.h" />
+ <ClInclude Include="..\Include\internal\pycore_uniqueid.h" />
  <ClInclude Include="..\Include\internal\pycore_warnings.h" />
  <ClInclude Include="..\Include\internal\pycore_weakref.h" />
  <ClInclude Include="..\Include\intrcheck.h" />
@@ -657,7 +657,7 @@
  <ClCompile Include="..\Python\thread.c" />
  <ClCompile Include="..\Python\traceback.c" />
  <ClCompile Include="..\Python\tracemalloc.c" />
- <ClCompile Include="..\Python\typeid.c" />
+ <ClCompile Include="..\Python\uniqueid.c" />
  </ItemGroup>
  <ItemGroup Condition="$(IncludeExternals)">
  <ClCompile Include="..\Modules\zlibmodule.c" />

PCbuild/pythoncore.vcxproj.filters

@@ -831,9 +831,6 @@
  <ClInclude Include="..\Include\internal\pycore_tuple.h">
  <Filter>Include\internal</Filter>
  </ClInclude>
- <ClInclude Include="..\Include\internal\pycore_typeid.h">
- <Filter>Include\internal</Filter>
- </ClInclude>
  <ClInclude Include="..\Include\internal\pycore_typeobject.h">
  <Filter>Include\internal</Filter>
  </ClInclude>
@@ -846,6 +843,9 @@
  <ClInclude Include="..\Include\internal\pycore_unionobject.h">
  <Filter>Include\internal</Filter>
  </ClInclude>
+ <ClInclude Include="..\Include\internal\pycore_uniqueid.h">
+ <Filter>Include\internal</Filter>
+ </ClInclude>
  <ClInclude Include="..\Include\internal\mimalloc\mimalloc.h">
  <Filter>Include\internal\mimalloc</Filter>
  </ClInclude>
@@ -1499,7 +1499,7 @@
  <ClCompile Include="..\Python\tracemalloc.c">
  <Filter>Python</Filter>
  </ClCompile>
- <ClCompile Include="..\Python\typeid.c">
+ <ClCompile Include="..\Python\uniqueid.c">
  <Filter>Python</Filter>
  </ClCompile>
  <ClCompile Include="..\Python\bootstrap_hash.c">

Python/gc_free_threading.c

@@ -15,7 +15,7 @@
 #include "pycore_tstate.h" // _PyThreadStateImpl
 #include "pycore_weakref.h" // _PyWeakref_ClearRef()
 #include "pydtrace.h"
-#include "pycore_typeid.h"  // _PyType_MergeThreadLocalRefcounts
+#include "pycore_uniqueid.h" // _PyType_MergeThreadLocalRefcounts
 
 #ifdef Py_GIL_DISABLED
 
@@ -217,12 +217,12 @@ disable_deferred_refcounting(PyObject *op)
  merge_refcount(op, 0);
  }
 
- // Heap types also use thread-local refcounting -- disable it here.
+ // Heap types also use per-thread refcounting -- disable it here.
  if (PyType_Check(op)) {
- // Disable thread-local refcounting for heap types
- PyTypeObject *type = (PyTypeObject *)op;
- if (PyType_HasFeature(type, Py_TPFLAGS_HEAPTYPE)) {
- _PyType_ReleaseId((PyHeapTypeObject *)op);
+ if (PyType_HasFeature((PyTypeObject *)op, Py_TPFLAGS_HEAPTYPE)) {
+  PyHeapTypeObject *ht = (PyHeapTypeObject *)op;
+  _PyObject_ReleaseUniqueId(ht->unique_id);
+ ht->unique_id = -1;
  }
  }
 
@@ -1221,7 +1221,7 @@ gc_collect_internal(PyInterpreterState *interp, struct collection_state *state,
  _PyThreadStateImpl *tstate = (_PyThreadStateImpl *)p;
 
  // merge per-thread refcount for types into the type's actual refcount
- _PyType_MergeThreadLocalRefcounts(tstate);
+ _PyObject_MergePerThreadRefcounts(tstate);
 
  // merge refcounts for all queued objects
  merge_queued_objects(tstate, state);

Python/pylifecycle.c

@@ -28,7 +28,7 @@
 #include "pycore_sliceobject.h" // _PySlice_Fini()
 #include "pycore_sysmodule.h" // _PySys_ClearAuditHooks()
 #include "pycore_traceback.h" // _Py_DumpTracebackThreads()
-#include "pycore_typeid.h"  // _PyType_FinalizeIdPool()
+#include "pycore_uniqueid.h" // _PyType_FinalizeIdPool()
 #include "pycore_typeobject.h" // _PyTypes_InitTypes()
 #include "pycore_typevarobject.h" // _Py_clear_generic_types()
 #include "pycore_unicodeobject.h" // _PyUnicode_InitTypes()
@@ -1834,7 +1834,7 @@ finalize_interp_types(PyInterpreterState *interp)
 
  _PyTypes_Fini(interp);
 #ifdef Py_GIL_DISABLED
- _PyType_FinalizeIdPool(interp);
+ _PyObject_FinalizeUniqueIdPool(interp);
 #endif
 
  _PyCode_Fini(interp);