gh-100227: Make the Global PyModuleDef Cache Safe for Isolated Interpreters

Include/internal/pycore_global_objects.h

@@ -27,7 +27,21 @@ extern "C" {
     _PyRuntime.cached_objects.NAME
 
 struct _Py_cached_objects {
+    /* A thread state tied to the main interpreter,
+       used exclusively for when a global object (e.g. interned strings)
+       is resized (i.e. deallocated + allocated) from an arbitrary thread. */
+    PyThreadState main_tstate;
+
+    /* The dict of interned strings. */
     PyObject *interned_strings;
+
+    /* A dict mapping (filename, name) to PyModuleDef for modules.
+       Only legacy (single-phase init) extension modules are added
+       and only if they support multiple initialization (m_size >- 0)
+       or are imported in the main interpreter.
+       This is initialized lazily in _PyImport_FixupExtensionObject().
+       Modules are added there and looked up in _imp.find_extension(). */
+    PyObject *extensions;
 };
 
 #define _Py_GLOBAL_OBJECT(NAME) \

Include/internal/pycore_import.h

@@ -14,15 +14,10 @@ struct _import_runtime_state {
        which is just about every time an extension module is imported.
        See PyInterpreterState.modules_by_index for more info. */
     Py_ssize_t last_module_index;
-    /* A dict mapping (filename, name) to PyModuleDef for modules.
-       Only legacy (single-phase init) extension modules are added
-       and only if they support multiple initialization (m_size >- 0)
-       or are imported in the main interpreter.
-       This is initialized lazily in _PyImport_FixupExtensionObject().
-       Modules are added there and looked up in _imp.find_extension(). */
-    PyObject *extensions;
     /* Package context -- the full module name for package imports */
     const char * pkgcontext;
+    /* The dict of cached module defs is over at
+       _PyRuntime.cached_objects.extensions. */
 };
 
 struct _import_state {

Include/internal/pycore_pystate.h

@@ -127,6 +127,13 @@ PyAPI_FUNC(void) _PyThreadState_Init(
     PyThreadState *tstate);
 PyAPI_FUNC(void) _PyThreadState_DeleteExcept(PyThreadState *tstate);
 
+extern void _PyThreadState_InitDetached(PyThreadState *, PyInterpreterState *);
+extern void _PyThreadState_ClearDetached(PyThreadState *);
+
+extern PyObject * _Py_GetFromGlobalDict(PyObject *, PyObject *);
+extern PyObject * _Py_AddToGlobalDict(PyObject *, PyObject *, PyObject *);
+extern PyObject * _Py_PopFromGlobalDict(PyObject *, PyObject *);
+
 
 static inline void
 _PyThreadState_UpdateTracingState(PyThreadState *tstate)

Include/internal/pycore_runtime_init.h

@@ -58,6 +58,9 @@ extern PyTypeObject _PyExc_MemoryError;
         .types = { \
             .next_version_tag = 1, \
         }, \
+        .cached_objects = { \
+            .main_tstate = _PyThreadState_INIT, \
+        }, \
         .static_objects = { \
             .singletons = { \
                 .small_ints = _Py_small_ints_INIT, \

Include/internal/pycore_unicodeobject.h

@@ -34,6 +34,7 @@ struct _Py_unicode_runtime_ids {
 
 struct _Py_unicode_runtime_state {
     struct _Py_unicode_runtime_ids ids;
+    /* The interned dict is at _PyRuntime.cached_objects.interned_strings. */
 };
 
 /* fs_codec.encoding is initialized to NULL.

Objects/unicodeobject.c

@@ -1534,7 +1534,8 @@ unicode_dealloc(PyObject *unicode)
            PyDict_DelItem(). */
         assert(Py_REFCNT(unicode) == 0);
         Py_SET_REFCNT(unicode, 3);
-        if (PyDict_DelItem(interned, unicode) != 0) {
+        if (_Py_PopFromGlobalDict(interned, unicode) == NULL
+                && PyErr_Occurred()) {
             _PyErr_WriteUnraisableMsg("deletion of interned string failed",
                                       NULL);
         }
@@ -14609,16 +14610,11 @@ PyUnicode_InternInPlace(PyObject **p)
     }
 
     PyObject *interned = get_interned_dict();
-    assert(interned != NULL);
-
-    PyObject *t = PyDict_SetDefault(interned, s, s);
-    if (t == NULL) {
-        PyErr_Clear();
-        return;
-    }
-
+    PyObject *t = _Py_AddToGlobalDict(interned, s, s);
     if (t != s) {
-        Py_SETREF(*p, Py_NewRef(t));
+        if (t != NULL) {
+            Py_SETREF(*p, Py_NewRef(t));
+        }
         return;
     }
 

Python/import.c

@@ -54,7 +54,7 @@ static struct _inittab *inittab_copy = NULL;
 
 #define INITTAB _PyRuntime.imports.inittab
 #define LAST_MODULE_INDEX _PyRuntime.imports.last_module_index
-#define EXTENSIONS _PyRuntime.imports.extensions
+#define EXTENSIONS _Py_CACHED_OBJECT(extensions)
 
 #define PKGCONTEXT (_PyRuntime.imports.pkgcontext)
 
@@ -889,7 +889,7 @@ _extensions_cache_get(PyObject *filename, PyObject *name)
     if (key == NULL) {
         return NULL;
     }
-    PyModuleDef *def = (PyModuleDef *)PyDict_GetItemWithError(extensions, key);
+    PyModuleDef *def = (PyModuleDef *)_Py_GetFromGlobalDict(extensions, key);
     Py_DECREF(key);
     return def;
 }
@@ -909,9 +909,15 @@ _extensions_cache_set(PyObject *filename, PyObject *name, PyModuleDef *def)
     if (key == NULL) {
         return -1;
     }
-    int res = PyDict_SetItem(extensions, key, (PyObject *)def);
+    PyObject *existing = _Py_AddToGlobalDict(extensions, key, (PyObject *)def);
     Py_DECREF(key);
-    if (res < 0) {
+    if (existing == NULL) {
+        return -1;
+    }
+    if (existing != (PyObject *)def) {
+        assert(!PyErr_Occurred());
+        PyErr_Format(PyExc_ImportError,
+                     "could not add moduledef for %s to the cache", name);
         return -1;
     }
     return 0;
@@ -928,14 +934,11 @@ _extensions_cache_delete(PyObject *filename, PyObject *name)
     if (key == NULL) {
         return -1;
     }
-    if (PyDict_DelItem(extensions, key) < 0) {
-        if (!PyErr_ExceptionMatches(PyExc_KeyError)) {
-            Py_DECREF(key);
-            return -1;
-        }
-        PyErr_Clear();
-    }
+    PyObject *value = _Py_PopFromGlobalDict(extensions, key);
     Py_DECREF(key);
+    if (value == NULL && PyErr_Occurred()) {
+        return -1;
+    }
     return 0;
 }
 

Python/pylifecycle.c

@@ -635,6 +635,8 @@ pycore_create_interpreter(_PyRuntimeState *runtime,
         return status;
     }
 
+    _PyThreadState_InitDetached(&runtime->cached_objects.main_tstate, interp);
+
     *tstate_p = tstate;
     return _PyStatus_OK();
 }
@@ -1928,6 +1930,8 @@ Py_FinalizeEx(void)
     // XXX Do this sooner during finalization.
     // XXX Ensure finalizer errors are handled properly.
 
+    _PyThreadState_ClearDetached(&runtime->cached_objects.main_tstate);
+
     finalize_interp_clear(tstate);
     finalize_interp_delete(tstate->interp);