Deferred reference counting

Initial partial implementation of deferred reference counting.

Include/internal/pycore_object.h

@@ -360,6 +360,34 @@ _PyObject_SetMaybeWeakref(PyObject *op)
  }
 }
 
+/* Marks the object as support deferred reference counting.
+ *
+ * The object's type must be GC-enabled. This function is not thread-safe with
+ * respect to concurrent modifications; it must be called before the object
+ * becomes visible to other threads.
+ *
+ * Deferred refcounted objects are marked as "queued" to prevent merging
+ * reference count fields outside the garbage collector.
+ */
+static inline void
+_PyObject_SetDeferredRefcount(PyObject *op)
+{
+ assert(_Py_ThreadLocal(op) && "non thread-safe");
+ assert(!_PyObject_HasDeferredRefcount(op) && "already uses deferred refcounting");
+ assert(PyType_IS_GC(Py_TYPE(op)));
+ op->ob_ref_local += _Py_REF_DEFERRED_MASK + 1;
+ op->ob_ref_shared = (op->ob_ref_shared & ~_Py_REF_SHARED_FLAG_MASK) | _Py_REF_QUEUED;
+}
+
+#define _PyObject_SET_DEFERRED_REFCOUNT(op) _PyObject_SetDeferredRefcount(_PyObject_CAST(op))
+
+// Check is refcount is deferred or immortal
+static inline int
+_Py_REF_NON_IMMEDIATE(uint32_t local)
+{
+ return _Py_STATIC_CAST(int32_t, local) <= Py_REF_IMMORTAL;
+}
+
 #ifdef Py_REF_DEBUG
 extern void _PyDebug_PrintTotalRefs(void);
 #endif

Include/object.h

@@ -135,7 +135,7 @@ PyAPI_FUNC(int) Py_Is(PyObject *x, PyObject *y);
 #define Py_Is(x, y) ((x) == (y))
 
 static inline void
-_PyRef_UnpackLocal(uint32_t bits, Py_ssize_t *refcount, int *immortal);
+_PyRef_UnpackLocal(uint32_t bits, Py_ssize_t *refcount, int *immortal, int *deferred);
 
 static inline void
 _PyRef_UnpackShared(uint32_t bits, Py_ssize_t *refcount, int *queued, int *merged);
@@ -148,7 +148,7 @@ static inline Py_ssize_t Py_REFCNT(PyObject *ob) {
  int immortal;
 
  uint32_t local = _Py_atomic_load_uint32_relaxed(&ob->ob_ref_local);
- _PyRef_UnpackLocal(local, &local_refcount, &immortal);
+ _PyRef_UnpackLocal(local, &local_refcount, &immortal, NULL);
 
  if (immortal) {
  return 999;
@@ -617,7 +617,7 @@ _Py_ThreadLocal(PyObject *op)
 #define _Py_REF_LOCAL_SHIFT 0
 #define _Py_REF_LOCAL_INIT 1
 #define Py_REF_IMMORTAL -1
-#define _Py_REF_DEFERRED_MASK  0x40000000
+#define _Py_REF_DEFERRED_MASK 0xC0000000
 
 // 30 2
 // [refcount] [bits]
@@ -884,10 +884,17 @@ static inline PyObject* _Py_XNewRef(PyObject *obj)
 #endif
 
 static inline void
-_PyRef_UnpackLocal(uint32_t bits, Py_ssize_t *refcount, int *immortal)
+_PyRef_UnpackLocal(uint32_t bits, Py_ssize_t *refcount, int *immortal, int *deferred)
 {
  *refcount = bits >> _Py_REF_LOCAL_SHIFT;
  *immortal = _Py_REF_IS_IMMORTAL(bits);
+ int is_deferred = _Py_STATIC_CAST(int32_t, bits) < Py_REF_IMMORTAL;
+ if (is_deferred) {
+ *refcount -= _Py_REF_DEFERRED_MASK;
+ }
+ if (deferred) {
+ *deferred = is_deferred;
+ }
 }
 
 static inline void
@@ -925,7 +932,7 @@ _PyObject_IsReferened(PyObject *op)
  int immortal;
 
  uint32_t local = _Py_atomic_load_uint32_relaxed(&op->ob_ref_local);
- _PyRef_UnpackLocal(local, &local_refcount, &immortal);
+ _PyRef_UnpackLocal(local, &local_refcount, &immortal, NULL);
 
  if (immortal) {
  return 1;
@@ -940,6 +947,13 @@ _PyObject_IsReferened(PyObject *op)
  return (local_refcount + shared_refcount) > 0;
 }
 
+static inline int
+_PyObject_HasDeferredRefcount(PyObject *op)
+{
+ uint32_t local = _Py_atomic_load_uint32_relaxed(&op->ob_ref_local);
+ return _Py_STATIC_CAST(int32_t, local) < Py_REF_IMMORTAL;
+}
+
 static inline void
 _Py_RESURRECT(PyObject *op, Py_ssize_t refcnt)
 {

Modules/gcmodule.c

@@ -318,13 +318,13 @@ static Py_ssize_t
 _Py_GC_REFCNT(PyObject *op)
 {
  Py_ssize_t local, shared;
- int immortal;
+ int immortal, deferred;
 
- _PyRef_UnpackLocal(op->ob_ref_local, &local, &immortal);
+ _PyRef_UnpackLocal(op->ob_ref_local, &local, &immortal, &deferred);
  _PyRef_UnpackShared(op->ob_ref_shared, &shared, NULL, NULL);
  assert(!immortal);
 
- return local + shared;
+ return local + shared - deferred;
 }
 
 typedef int (gc_visit_fn)(PyGC_Head* gc, void *arg);
@@ -682,9 +682,35 @@ find_dead_shared_keys(_PyObjectQueue **queue, int *num_unmarked)
  }
 }
 
+static void
+merge_refcount(PyObject *op, Py_ssize_t extra)
+{
+ Py_ssize_t local_refcount, shared_refcount;
+ int immortal, deferred;
+
+ assert(_PyRuntime.stop_the_world);
+
+ _PyRef_UnpackLocal(op->ob_ref_local, &local_refcount, &immortal, &deferred);
+ _PyRef_UnpackShared(op->ob_ref_shared, &shared_refcount, NULL, NULL);
+ assert(!immortal && "immortal objects should not be in garbage");
+
+ Py_ssize_t refcount = local_refcount + shared_refcount;
+ refcount += extra;
+ refcount -= deferred;
+
+#ifdef Py_REF_DEBUG
+ _Py_IncRefTotalN(extra);
+#endif
+
+ op->ob_tid = 0;
+ op->ob_ref_local = 0;
+ op->ob_ref_shared = _Py_REF_PACK_SHARED(refcount, _Py_REF_MERGED);
+}
+
 struct update_refs_args {
  PyGC_Head *list;
  int split_keys_marked;
+ _PyGC_Reason gc_reason;
 };
 
 // Compute the number of external references to objects in the heap
@@ -730,6 +756,15 @@ update_refs(const mi_heap_t* heap, const mi_heap_area_t* area, void* block, size
  }
  }
 
+ if (arg->gc_reason == GC_REASON_SHUTDOWN) {
+ if (_PyObject_HasDeferredRefcount(op)) {
+ // Disable deferred reference counting when we're shutting down.
+ // This is useful for interp->sysdict because the last reference
+ // to it is cleared after the last GC cycle.
+ merge_refcount(op, 0);
+ }
+ }
+
  // Add the actual refcount to gc_refs.
  Py_ssize_t refcount = _Py_GC_REFCNT(op);
  _PyObject_ASSERT(op, refcount >= 0);
@@ -1008,23 +1043,7 @@ move_legacy_finalizer_reachable(PyGC_Head *finalizers)
 static void
 incref_merge(PyObject *op)
 {
- Py_ssize_t local_refcount, shared_refcount;
- int immortal;
-
- assert(_PyRuntime.stop_the_world);
-
-#ifdef Py_REF_DEBUG
- _Py_IncRefTotal();
-#endif
-
- _PyRef_UnpackLocal(op->ob_ref_local, &local_refcount, &immortal);
- _PyRef_UnpackShared(op->ob_ref_shared, &shared_refcount, NULL, NULL);
- assert(!immortal && "immortal objects should not be in garbage");
-
- Py_ssize_t refcount = local_refcount + shared_refcount + 1;
- op->ob_tid = 0;
- op->ob_ref_local = 0;
- op->ob_ref_shared = _Py_REF_PACK_SHARED(refcount, _Py_REF_MERGED);
+ merge_refcount(op, 1);
 }
 
 /* Subtracts one from the refcount field. */
@@ -1594,7 +1613,11 @@ gc_collect_main(PyThreadState *tstate, int generation, _PyGC_Reason reason)
  validate_tracked_heap(_PyGC_PREV_MASK|_PyGC_PREV_MASK_UNREACHABLE, 0);
 
  gc_list_init(&young);
- struct update_refs_args args = { .list = &young, .split_keys_marked = 0 };
+ struct update_refs_args args = {
+ .list = &young,
+ .split_keys_marked = 0,
+ .gc_reason = reason,
+ };
  visit_heaps2(mi_heap_tag_gc, update_refs, &args);
 
  _PyObjectQueue *dead_keys = NULL;

Objects/descrobject.c

@@ -903,6 +903,7 @@ descr_new(PyTypeObject *descrtype, PyTypeObject *type, const char *name)
 
  descr = (PyDescrObject *)PyType_GenericAlloc(descrtype, 0);
  if (descr != NULL) {
+ _PyObject_SET_DEFERRED_REFCOUNT(descr);
  descr->d_type = (PyTypeObject*)Py_XNewRef(type);
  descr->d_name = PyUnicode_InternFromString(name);
  if (descr->d_name == NULL) {

Objects/dictobject.c

@@ -1371,6 +1371,10 @@ _PyDict_MaybeUntrack(PyObject *op)
  if (!PyDict_CheckExact(op) || !_PyObject_GC_IS_TRACKED(op))
  return;
 
+ if (_PyObject_HasDeferredRefcount(op)) {
+ return;
+ }
+
  mp = (PyDictObject *) op;
  numentries = mp->ma_keys->dk_nentries;
  if (_PyDict_HasSplitTable(mp)) {

Objects/funcobject.c

@@ -77,7 +77,9 @@ PyFunction_ClearWatcher(int watcher_id)
 PyFunctionObject *
 _PyFunction_FromConstructor(PyFrameConstructor *constr)
 {
-
+ // NOTE(sgross): functions created via FrameConstructor do *not* use
+ // deferred reference counting because they are typically not part of cycles
+ // nor accessed by multiple threads.
  PyFunctionObject *op = PyObject_GC_New(PyFunctionObject, &PyFunction_Type);
  if (op == NULL) {
  return NULL;
@@ -172,6 +174,9 @@ PyFunction_NewWithQualName(PyObject *code, PyObject *globals, PyObject *qualname
  op->func_annotations = NULL;
  op->vectorcall = _PyFunction_Vectorcall;
  op->func_version = 0;
+ if ((code_obj->co_flags & CO_NESTED) == 0) {
+ _PyObject_SET_DEFERRED_REFCOUNT(op);
+ }
  _PyObject_GC_TRACK(op);
  handle_func_event(PyFunction_EVENT_CREATE, op, NULL);
  return (PyObject *)op;

Objects/moduleobject.c

@@ -107,6 +107,9 @@ new_module_notrack(PyTypeObject *mt)
  m->md_name = NULL;
  m->md_dict = PyDict_New();
  if (m->md_dict != NULL) {
+ PyObject_GC_Track(m->md_dict);
+ _PyObject_SET_DEFERRED_REFCOUNT(m->md_dict);
+ _PyObject_SET_DEFERRED_REFCOUNT(m);
  return m;
  }
  Py_DECREF(m);

Objects/typeobject.c

@@ -2887,6 +2887,7 @@ type_new_alloc(type_new_ctx *ctx)
  if (type == NULL) {
  return NULL;
  }
+ _PyObject_SET_DEFERRED_REFCOUNT(type);
  PyHeapTypeObject *et = (PyHeapTypeObject *)type;
 
  // Initialize tp_flags.
@@ -3775,6 +3776,7 @@ PyType_FromMetaclass(PyTypeObject *metaclass, PyObject *module,
  if (res == NULL) {
  goto finally;
  }
+ _PyObject_SET_DEFERRED_REFCOUNT(res);
  res_start = (char*)res;
 
  type = &res->ht_type;

Python/sysmodule.c

@@ -1876,16 +1876,17 @@ sys_getfullrefcount(PyObject *module, PyObject *object)
  uintptr_t tid = _PyObject_ThreadId(object);
 
  Py_ssize_t local, shared;
- int immortal, queued, merged;
+ int immortal, deferred, queued, merged;
 
- _PyRef_UnpackLocal(object->ob_ref_local, &local, &immortal);
+ _PyRef_UnpackLocal(object->ob_ref_local, &local, &immortal, &deferred);
  _PyRef_UnpackShared(object->ob_ref_shared, &shared, &queued, &merged);
 
  PyDict_SetItemString(res, "local", PyLong_FromSsize_t(local));
  PyDict_SetItemString(res, "shared", PyLong_FromSsize_t(shared));
  PyDict_SetItemString(res, "merged", PyBool_FromLong(merged));
  PyDict_SetItemString(res, "queued", PyBool_FromLong(queued));
  PyDict_SetItemString(res, "immortal", PyBool_FromLong(immortal));
+ PyDict_SetItemString(res, "deferred", PyBool_FromLong(deferred));
  if (queued) {
  Py_INCREF(Py_None);
  PyDict_SetItemString(res, "tid", Py_None);