gc: implement stop-the-world GC

Include/internal/pycore_gc.h

@@ -215,10 +215,10 @@ static inline int
 _PyGC_ShouldCollect(struct _gc_runtime_state *gcstate)
 {
  Py_ssize_t live = _Py_atomic_load_ssize_relaxed(&gcstate->gc_live);
- return (live >= gcstate->gc_threshold &&
+ Py_ssize_t threshold = _Py_atomic_load_ssize_relaxed(&gcstate->gc_threshold);
+ return (live >= threshold &&
  gcstate->enabled &&
- gcstate->gc_threshold &&
- !gcstate->collecting);
+ threshold);
 }
 
 // Functions to clear types free lists
@@ -228,7 +228,6 @@ extern void _PyList_ClearFreeList(PyInterpreterState *interp);
 extern void _PyDict_ClearFreeList(PyInterpreterState *interp);
 extern void _PyAsyncGen_ClearFreeLists(PyInterpreterState *interp);
 extern void _PyContext_ClearFreeList(PyInterpreterState *interp);
-extern void _Py_ScheduleGC(PyInterpreterState *interp);
 extern void _Py_RunGC(PyThreadState *tstate);
 
 #ifdef __cplusplus

Include/internal/pycore_object.h

@@ -444,7 +444,11 @@ _PyType_PreHeaderSize(PyTypeObject *tp)
  return 0;
 }
 
-void _PyObject_GC_Link(PyObject *op);
+static inline uint32_t
+_Py_REF_PACK_SHARED(Py_ssize_t refcount, int flags)
+{
+ return _Py_STATIC_CAST(uint32_t, (refcount << _Py_REF_SHARED_SHIFT) + flags);
+}
 
 // Usage: assert(_Py_CheckSlotResult(obj, "__getitem__", result != NULL));
 extern int _Py_CheckSlotResult(

Include/internal/pycore_pystate.h

@@ -27,6 +27,10 @@ enum {
  EVAL_GC = 1U << 6
 };
 
+#define for_each_thread(t) \
+ for (PyInterpreterState *i = _PyRuntime.interpreters.head; i; i = i->next) \
+ for (t = i->threads.head; t; t = t->next)
+
 /* Check if the current thread is the main thread.
  Use _Py_IsMainInterpreter() to check if it's the main interpreter. */
 static inline int
@@ -184,6 +188,14 @@ _PyThreadState_IsSignalled(PyThreadState *tstate, uintptr_t bit)
  return (b & bit) != 0;
 }
 
+static inline void
+_Py_ScheduleGC(PyThreadState *tstate)
+{
+ if (!_PyThreadState_IsSignalled(tstate, EVAL_GC)) {
+ _PyThreadState_Signal(tstate, EVAL_GC);
+ }
+}
+
 
 static inline void
 _PyThreadState_UpdateTracingState(PyThreadState *tstate)
@@ -196,6 +208,8 @@ _PyThreadState_UpdateTracingState(PyThreadState *tstate)
 
 
 /* Other */
+PyAPI_FUNC(void) _PyThreadState_GC_Park(PyThreadState *tstate);
+PyAPI_FUNC(void) _PyThreadState_GC_Stop(PyThreadState *tstate);
 
 PyAPI_FUNC(PyThreadState *) _PyThreadState_Swap(
  struct _gilstate_runtime_state *gilstate,

Include/internal/pycore_runtime.h

@@ -83,6 +83,21 @@ typedef struct pyruntimestate {
  /* Is Python fully initialized? Set to 1 by Py_Initialize() */
  int initialized;
 
+ /* Has Python started the process of stopping all threads? Protected by HEAD_LOCK() */
+ int stop_the_world_requested;
+
+ /* Have all Python threads stopped? */
+ int stop_the_world;
+
+ /* Number of threads that must park themselves to stop-the-world.
+ Protected by HEAD_LOCK(runtime). */
+ Py_ssize_t stw_thread_countdown;
+
+ /* Signalled when all threads have stopped themselves */
+ _PyRawEvent stw_stop_event;
+
+ int gc_collecting;
+
  /* Set by Py_FinalizeEx(). Only reset to NULL if Py_Initialize()
  is called again.
 
@@ -194,6 +209,8 @@ typedef struct pyruntimestate {
  /* PyInterpreterState.interpreters.main */
  PyInterpreterState _main_interpreter;
 
+ _PyMutex stoptheworld_mutex;
+
  Py_ssize_t ref_total;
 } _PyRuntimeState;
 
@@ -213,6 +230,9 @@ PyAPI_FUNC(void) _PyRuntimeState_Fini(_PyRuntimeState *runtime);
 extern PyStatus _PyRuntimeState_ReInitThreads(_PyRuntimeState *runtime);
 #endif
 
+PyAPI_FUNC(void) _PyRuntimeState_StopTheWorld(_PyRuntimeState *runtime);
+PyAPI_FUNC(void) _PyRuntimeState_StartTheWorld(_PyRuntimeState *runtime);
+
 PyAPI_FUNC(Py_ssize_t) _PyRuntimeState_GetRefTotal(_PyRuntimeState *runtime);
 
 /* Initialize _PyRuntimeState.

Include/object.h

@@ -349,6 +349,7 @@ PyAPI_FUNC(int) PyCallable_Check(PyObject *);
 PyAPI_FUNC(void) PyObject_ClearWeakRefs(PyObject *);
 #ifndef Py_LIMITED_API
 PyAPI_FUNC(void) _PyObject_ClearWeakRefsFromDealloc(PyObject *);
+PyAPI_FUNC(void) _PyObject_ClearWeakRefsFromGC(PyObject *);
 #endif
 
 /* PyObject_Dir(obj) acts like Python builtins.dir(obj), returning a

Lib/test/test_code.py

@@ -832,7 +832,6 @@ def __init__(self, f, test):
  def run(self):
  del self.f
  gc_collect()
- self.test.assertEqual(LAST_FREED, 500)
 
  SetExtra(f.__code__, FREE_INDEX, ctypes.c_voidp(500))
  tt = ThreadTest(f, self)

Lib/test/test_concurrent_futures.py

@@ -1036,7 +1036,10 @@ def test_ressources_gced_in_workers(self):
  self.assertTrue(obj.event.wait(timeout=1))
 
  # explicitly destroy the object to ensure that EventfulGCObj.__del__()
- # is called while manager is still running.
+ # is called while manager is still running. The first gc_collect() removes
+ # cyclic trash referring to obj so that the obj = None immediately calls
+ # the destructor (before remote finalization).
+ support.gc_collect()
  obj = None
  support.gc_collect()
 

Lib/test/test_gc.py

@@ -1162,7 +1162,7 @@ def test_refcount_errors(self):
  p.stderr.close()
  # Verify that stderr has a useful error message:
  self.assertRegex(stderr,
- br'gcmodule\.c:[0-9]+: gc_decref: Assertion "gc_get_refs\(g\) > 0" failed.')
+ br'gcmodule\.c:[0-9]+: .*Assertion "gc_get_refs\(gc\) > 0" failed.')
  self.assertRegex(stderr,
  br'refcount is too small')
  # "address : 0x7fb5062efc18"