From e21e012a22b20f35c13826b15ecfcf8f81f8c60f Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Thu, 10 Nov 2022 15:30:25 -0700 Subject: [PATCH 1/6] Add a comment describing PyInterpreterState.modules_by_index. --- Include/internal/pycore_interp.h | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/Include/internal/pycore_interp.h b/Include/internal/pycore_interp.h index ae2a3d3b13cfa9..068b0a700af536 100644 --- a/Include/internal/pycore_interp.h +++ b/Include/internal/pycore_interp.h @@ -123,6 +123,25 @@ struct _is { // sys.modules dictionary PyObject *modules; + /* This is the list of module objects for all legacy (single-phase init) + extension modules ever loaded in this process (i.e. imported + in this interpreter or in any other). Py_None stands in for + modules that haven't actually been imported in this interpreter. + + A module's index (PyModuleDef.m_base.m_index) is used to look up + the corresponding module object for this interpreter, if any. + (See PyState_FindModule().) When any extension module + is initialized during import, its moduledef gets initialized by + PyModuleDef_Init(), and the first time that happens for each + PyModuleDef, its index gets set to the current value of + a global counter (see _PyRuntimeState.imports.last_module_index). + The entry for that index in this interpreter remains unset until + the module is actually imported here. (Py_None is used as + a placeholder.) Note that multi-phase init modules always get + an index for which there will never be a module set. + + This is initialized lazily in _PyState_AddModule(), which is also + where modules get added. */ PyObject *modules_by_index; // Dictionary of the sys module PyObject *sysdict; From 453ca0e102e7aef42ec608deddc7e526a159ed9a Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Thu, 10 Nov 2022 13:43:05 -0700 Subject: [PATCH 2/6] Add comments describing the PyModuleDef_Base fields. --- Include/moduleobject.h | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/Include/moduleobject.h b/Include/moduleobject.h index fbb2c5ae79444b..84fbcd47558541 100644 --- a/Include/moduleobject.h +++ b/Include/moduleobject.h @@ -43,8 +43,21 @@ PyAPI_DATA(PyTypeObject) PyModuleDef_Type; typedef struct PyModuleDef_Base { PyObject_HEAD + /* The function used to re-initialize the module. + This is only set for legacy (single-phase init) extension modules + and only used for those that support multiple initializations + (m_size >= 0). + It is set by _PyImport_LoadDynamicModuleWithSpec() + and _imp.create_builtin(). */ PyObject* (*m_init)(void); + /* The module's index into its interpreter's modules_by_index cache. + This is set for all extension modules but only used for legacy ones. + It is set by PyModuleDef_Init(). */ Py_ssize_t m_index; + /* A copy of the module's __dict__ after the first time it was loaded. + This is only set/used for legacy modules that do not support + multiple initializations. + It is set by _PyImport_FixupExtensionObject(). */ PyObject* m_copy; } PyModuleDef_Base; From 3b93ab91d59d968ac6b319b2223230dcf2bc7e01 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Thu, 10 Nov 2022 12:20:27 -0700 Subject: [PATCH 3/6] Add _PyRuntimeState.imports.extensions. --- Include/internal/pycore_import.h | 12 +++++++ Include/internal/pycore_runtime.h | 2 ++ Python/import.c | 38 ++++++++++++++++----- Tools/c-analyzer/cpython/globals-to-fix.tsv | 1 - 4 files changed, 44 insertions(+), 9 deletions(-) diff --git a/Include/internal/pycore_import.h b/Include/internal/pycore_import.h index aee1f66a3ea171..73750651aece65 100644 --- a/Include/internal/pycore_import.h +++ b/Include/internal/pycore_import.h @@ -5,6 +5,18 @@ extern "C" { #endif + +struct _import_runtime_state { + /* A dict mapping (filename, name) to PyModuleDef for modules. + Only legacy (single-phase init) extension modules are added + and only if they support multiple initialization (m_size >- 0) + or are imported in the main interpreter. + This is initialized lazily in _PyImport_FixupExtensionObject(). + Modules are added there and looked up in _imp.find_extension(). */ + PyObject *extensions; +}; + + #ifdef HAVE_FORK extern PyStatus _PyImport_ReInitLock(void); #endif diff --git a/Include/internal/pycore_runtime.h b/Include/internal/pycore_runtime.h index d1fbc09f1ea206..df35e34291afc2 100644 --- a/Include/internal/pycore_runtime.h +++ b/Include/internal/pycore_runtime.h @@ -11,6 +11,7 @@ extern "C" { #include "pycore_atomic.h" /* _Py_atomic_address */ #include "pycore_gil.h" // struct _gil_runtime_state #include "pycore_global_objects.h" // struct _Py_global_objects +#include "pycore_import.h" // struct _import_runtime_state #include "pycore_interp.h" // PyInterpreterState #include "pycore_unicodeobject.h" // struct _Py_unicode_runtime_ids @@ -115,6 +116,7 @@ typedef struct pyruntimestate { void (*exitfuncs[NEXITFUNCS])(void); int nexitfuncs; + struct _import_runtime_state imports; struct _ceval_runtime_state ceval; struct _gilstate_runtime_state gilstate; struct _getargs_runtime_state getargs; diff --git a/Python/import.c b/Python/import.c index 2fd2d1b6b89d1c..a1b2e8b1725f2f 100644 --- a/Python/import.c +++ b/Python/import.c @@ -27,9 +27,6 @@ extern "C" { /* Forward references */ static PyObject *import_add_module(PyThreadState *tstate, PyObject *name); -/* See _PyImport_FixupExtensionObject() below */ -static PyObject *extensions = NULL; - /* This table is defined in config.c: */ extern struct _inittab _PyImport_Inittab[]; @@ -221,10 +218,12 @@ _imp_release_lock_impl(PyObject *module) Py_RETURN_NONE; } +static inline void _clear_extensions_cache(void); + void _PyImport_Fini(void) { - Py_CLEAR(extensions); + _clear_extensions_cache(); if (import_lock != NULL) { PyThread_free_lock(import_lock); import_lock = NULL; @@ -398,6 +397,30 @@ PyImport_GetMagicTag(void) dictionary, to avoid loading shared libraries twice. */ +static inline PyObject * +_get_extensions_cache(void) +{ + return _PyRuntime.imports.extensions; +} + +static inline PyObject * +_ensure_extensions_cache(void) +{ + if (_PyRuntime.imports.extensions == NULL) { + _PyRuntime.imports.extensions = PyDict_New(); + if (_PyRuntime.imports.extensions == NULL) { + return NULL; + } + } + return _PyRuntime.imports.extensions; +} + +static inline void +_clear_extensions_cache(void) +{ + Py_CLEAR(_PyRuntime.imports.extensions); +} + int _PyImport_FixupExtensionObject(PyObject *mod, PyObject *name, PyObject *filename, PyObject *modules) @@ -442,11 +465,9 @@ _PyImport_FixupExtensionObject(PyObject *mod, PyObject *name, } } + PyObject *extensions = _ensure_extensions_cache(); if (extensions == NULL) { - extensions = PyDict_New(); - if (extensions == NULL) { - return -1; - } + return -1; } PyObject *key = PyTuple_Pack(2, filename, name); @@ -480,6 +501,7 @@ static PyObject * import_find_extension(PyThreadState *tstate, PyObject *name, PyObject *filename) { + PyObject *extensions = _get_extensions_cache(); if (extensions == NULL) { return NULL; } diff --git a/Tools/c-analyzer/cpython/globals-to-fix.tsv b/Tools/c-analyzer/cpython/globals-to-fix.tsv index 4cd29a8a0b0cb6..dd6709e3bfd996 100644 --- a/Tools/c-analyzer/cpython/globals-to-fix.tsv +++ b/Tools/c-analyzer/cpython/globals-to-fix.tsv @@ -317,7 +317,6 @@ Python/hamt.c - _empty_hamt - # state Objects/typeobject.c resolve_slotdups pname - -Python/import.c - extensions - ################################## From b3e3f45e79b5cef518e68a132544215f78d3aead Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Thu, 10 Nov 2022 15:57:50 -0700 Subject: [PATCH 4/6] Switch to _extensions_cache_*(). --- Python/import.c | 74 ++++++++++++++++++++++++------------------------- 1 file changed, 37 insertions(+), 37 deletions(-) diff --git a/Python/import.c b/Python/import.c index a1b2e8b1725f2f..d16161381a97a8 100644 --- a/Python/import.c +++ b/Python/import.c @@ -218,12 +218,12 @@ _imp_release_lock_impl(PyObject *module) Py_RETURN_NONE; } -static inline void _clear_extensions_cache(void); +static inline void _extensions_cache_clear(void); void _PyImport_Fini(void) { - _clear_extensions_cache(); + _extensions_cache_clear(); if (import_lock != NULL) { PyThread_free_lock(import_lock); import_lock = NULL; @@ -397,26 +397,47 @@ PyImport_GetMagicTag(void) dictionary, to avoid loading shared libraries twice. */ -static inline PyObject * -_get_extensions_cache(void) +static PyModuleDef * +_extensions_cache_get(PyObject *filename, PyObject *name) { - return _PyRuntime.imports.extensions; + PyObject *extensions = _PyRuntime.imports.extensions; + if (extensions == NULL) { + return NULL; + } + PyObject *key = PyTuple_Pack(2, filename, name); + if (key == NULL) { + return NULL; + } + PyModuleDef *def = (PyModuleDef *)PyDict_GetItemWithError(extensions, key); + Py_DECREF(key); + return def; } -static inline PyObject * -_ensure_extensions_cache(void) +static int +_extensions_cache_set(PyObject *filename, PyObject *name, PyModuleDef *def) { - if (_PyRuntime.imports.extensions == NULL) { - _PyRuntime.imports.extensions = PyDict_New(); - if (_PyRuntime.imports.extensions == NULL) { - return NULL; + PyObject *extensions = _PyRuntime.imports.extensions; + if (extensions == NULL) { + extensions = PyDict_New(); + if (extensions == NULL) { + return -1; } + _PyRuntime.imports.extensions = extensions; + } + PyObject *key = PyTuple_Pack(2, filename, name); + if (key == NULL) { + return -1; + } + int res = PyDict_SetItem(extensions, key, (PyObject *)def); + Py_DECREF(key); + if (res < 0) { + return -1; } - return _PyRuntime.imports.extensions; + return 0; } -static inline void -_clear_extensions_cache(void) +static void +_extensions_cache_clear(void) { Py_CLEAR(_PyRuntime.imports.extensions); } @@ -465,18 +486,7 @@ _PyImport_FixupExtensionObject(PyObject *mod, PyObject *name, } } - PyObject *extensions = _ensure_extensions_cache(); - if (extensions == NULL) { - return -1; - } - - PyObject *key = PyTuple_Pack(2, filename, name); - if (key == NULL) { - return -1; - } - int res = PyDict_SetItem(extensions, key, (PyObject *)def); - Py_DECREF(key); - if (res < 0) { + if (_extensions_cache_set(filename, name, def) < 0) { return -1; } } @@ -501,17 +511,7 @@ static PyObject * import_find_extension(PyThreadState *tstate, PyObject *name, PyObject *filename) { - PyObject *extensions = _get_extensions_cache(); - if (extensions == NULL) { - return NULL; - } - - PyObject *key = PyTuple_Pack(2, filename, name); - if (key == NULL) { - return NULL; - } - PyModuleDef* def = (PyModuleDef *)PyDict_GetItemWithError(extensions, key); - Py_DECREF(key); + PyModuleDef *def = _extensions_cache_get(filename, name); if (def == NULL) { return NULL; } From 1663bfce07f58387aee49386eb0c569a58f213a1 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Thu, 10 Nov 2022 15:06:16 -0700 Subject: [PATCH 5/6] Add _PyRuntimeState.imports.last_module_index. --- Include/internal/pycore_import.h | 5 +++++ Objects/moduleobject.c | 5 ++--- Tools/c-analyzer/cpython/globals-to-fix.tsv | 1 - 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/Include/internal/pycore_import.h b/Include/internal/pycore_import.h index 73750651aece65..7f1240f7c9db3c 100644 --- a/Include/internal/pycore_import.h +++ b/Include/internal/pycore_import.h @@ -7,6 +7,11 @@ extern "C" { struct _import_runtime_state { + /* The most recent value assigned to a PyModuleDef.m_base.m_index. + This is incremented each time PyModuleDef_Init() is called, + which is just about every time an extension module is imported. + See PyInterpreterState.modules_by_index for more info. */ + Py_ssize_t last_module_index; /* A dict mapping (filename, name) to PyModuleDef for modules. Only legacy (single-phase init) extension modules are added and only if they support multiple initialization (m_size >- 0) diff --git a/Objects/moduleobject.c b/Objects/moduleobject.c index ee8ef7f5b4a5dc..9a13303a076296 100644 --- a/Objects/moduleobject.c +++ b/Objects/moduleobject.c @@ -9,7 +9,6 @@ #include "pycore_moduleobject.h" // _PyModule_GetDef() #include "structmember.h" // PyMemberDef -static Py_ssize_t max_module_number; static PyMemberDef module_members[] = { {"__dict__", T_OBJECT, offsetof(PyModuleObject, md_dict), READONLY}, @@ -43,10 +42,10 @@ PyModuleDef_Init(PyModuleDef* def) { assert(PyModuleDef_Type.tp_flags & Py_TPFLAGS_READY); if (def->m_base.m_index == 0) { - max_module_number++; + _PyRuntime.imports.last_module_index++; Py_SET_REFCNT(def, 1); Py_SET_TYPE(def, &PyModuleDef_Type); - def->m_base.m_index = max_module_number; + def->m_base.m_index = _PyRuntime.imports.last_module_index; } return (PyObject*)def; } diff --git a/Tools/c-analyzer/cpython/globals-to-fix.tsv b/Tools/c-analyzer/cpython/globals-to-fix.tsv index dd6709e3bfd996..bb05a2c469bd9a 100644 --- a/Tools/c-analyzer/cpython/globals-to-fix.tsv +++ b/Tools/c-analyzer/cpython/globals-to-fix.tsv @@ -448,7 +448,6 @@ Python/getargs.c - static_arg_parsers - Objects/dictobject.c - _pydict_global_version - Objects/dictobject.c - next_dict_keys_version - Objects/funcobject.c - next_func_version - -Objects/moduleobject.c - max_module_number - Objects/object.c - _Py_RefTotal - Python/perf_trampoline.c - perf_status - Python/perf_trampoline.c - extra_code_index - From 67dff4dcfc75be453fc06ec54896f7522faa0865 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Fri, 11 Nov 2022 10:21:41 -0700 Subject: [PATCH 6/6] Refer to the modules_by_index comment. --- Include/moduleobject.h | 1 + 1 file changed, 1 insertion(+) diff --git a/Include/moduleobject.h b/Include/moduleobject.h index 84fbcd47558541..555564ec73b4a2 100644 --- a/Include/moduleobject.h +++ b/Include/moduleobject.h @@ -52,6 +52,7 @@ typedef struct PyModuleDef_Base { PyObject* (*m_init)(void); /* The module's index into its interpreter's modules_by_index cache. This is set for all extension modules but only used for legacy ones. + (See PyInterpreterState.modules_by_index for more info.) It is set by PyModuleDef_Init(). */ Py_ssize_t m_index; /* A copy of the module's __dict__ after the first time it was loaded.