From 2be855c8358e3f382c2fa18ac85263449a7c1ef7 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Fri, 17 Feb 2023 09:57:56 -0700 Subject: [PATCH 01/17] Stop skipping the tests. --- Lib/test/test_imp.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/Lib/test/test_imp.py b/Lib/test/test_imp.py index 2292bb20939599..f82e1c8ad643a1 100644 --- a/Lib/test/test_imp.py +++ b/Lib/test/test_imp.py @@ -263,7 +263,6 @@ def test_issue16421_multiple_modules_in_one_dll(self): with self.assertRaises(ImportError): imp.load_dynamic('nonexistent', pathname) - @unittest.skip('known refleak (temporarily skipping)') @requires_subinterpreters @requires_load_dynamic def test_singlephase_multiple_interpreters(self): @@ -330,7 +329,6 @@ def clean_up(): # However, globals are still shared. _interpreters.run_string(interp2, script % 2) - @unittest.skip('known refleak (temporarily skipping)') @requires_load_dynamic def test_singlephase_variants(self): # Exercise the most meaningful variants described in Python/import.c. From 9bbeddadba9071737cf3b3fec988e0e25c58911a Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Fri, 17 Feb 2023 09:58:28 -0700 Subject: [PATCH 02/17] _modules_by_index_clear() -> _modules_by_index_clear_one(). --- Python/import.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Python/import.c b/Python/import.c index fabf03b1c5d698..8989965ee39610 100644 --- a/Python/import.c +++ b/Python/import.c @@ -465,7 +465,7 @@ _modules_by_index_set(PyInterpreterState *interp, } static int -_modules_by_index_clear(PyInterpreterState *interp, PyModuleDef *def) +_modules_by_index_clear_one(PyInterpreterState *interp, PyModuleDef *def) { Py_ssize_t index = def->m_base.m_index; const char *err = _modules_by_index_check(interp, index); @@ -546,7 +546,7 @@ PyState_RemoveModule(PyModuleDef* def) "PyState_RemoveModule called on module with slots"); return -1; } - return _modules_by_index_clear(tstate->interp, def); + return _modules_by_index_clear_one(tstate->interp, def); } @@ -1007,7 +1007,7 @@ clear_singlephase_extension(PyInterpreterState *interp, /* Clear the PyState_*Module() cache entry. */ if (_modules_by_index_check(interp, def->m_base.m_index) == NULL) { - if (_modules_by_index_clear(interp, def) < 0) { + if (_modules_by_index_clear_one(interp, def) < 0) { return -1; } } From 83f42d62307d66cf97eb24806f5e0f376a855c32 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Fri, 17 Feb 2023 13:06:40 -0700 Subject: [PATCH 03/17] Add an explanation of what happens when extensions are loaded. --- Python/import.c | 103 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 103 insertions(+) diff --git a/Python/import.c b/Python/import.c index 8989965ee39610..2ed43d79a779dc 100644 --- a/Python/import.c +++ b/Python/import.c @@ -584,6 +584,109 @@ _PyImport_ClearModulesByIndex(PyInterpreterState *interp) /* extension modules */ /*********************/ +/* + It may help to have a big picture view of what happens + when an extension is loaded. This includes when it is imported + for the first time or via imp.load_dynamic(). + + Here's a summary, using imp.load_dynamic() as the starting point: + + 1. imp.load_dynamic() -> importlib._bootstrap._load() + 2. _load(): acquire import lock + 3. _load() -> importlib._bootstrap._load_unlocked() + 4. _load_unlocked() -> importlib._bootstrap.module_from_spec() + 5. module_from_spec() -> ExtensionFileLoader.create_module() + 6. create_module() -> _imp.create_dynamic() + (see below) + 7. module_from_spec() -> importlib._bootstrap._init_module_attrs() + 8. _load_unlocked(): sys.modules[name] = module + 9. _load_unlocked() -> ExtensionFileLoader.exec_module() + 10. exec_module() -> _imp.exec_dynamic() + (see below) + 11. _load(): release import lock + + + ...for single-phase init modules, where m_size == -1: + + (6). first time (not found in _PyRuntime.imports.extensions): + 1. _imp_create_dynamic_impl() -> import_find_extension() + 2. _imp_create_dynamic_impl() -> _PyImport_LoadDynamicModuleWithSpec() + 3. _PyImport_LoadDynamicModuleWithSpec(): load + 4. _PyImport_LoadDynamicModuleWithSpec(): call + 5. -> PyModule_Create() -> PyModule_Create2() -> PyModule_CreateInitialized() + 6. PyModule_CreateInitialized() -> PyModule_New() + 7. PyModule_CreateInitialized(): allocate mod->md_state + 8. PyModule_CreateInitialized() -> PyModule_AddFunctions() + 9. PyModule_CreateInitialized() -> PyModule_SetDocString() + 10. PyModule_CreateInitialized(): set mod->md_def + 11. : initialize the module + 12. _PyImport_LoadDynamicModuleWithSpec() -> _PyImport_CheckSubinterpIncompatibleExtensionAllowed() + 13. _PyImport_LoadDynamicModuleWithSpec(): set def->m_base.m_init + 14. _PyImport_LoadDynamicModuleWithSpec(): set __file__ + 15. _PyImport_LoadDynamicModuleWithSpec() -> _PyImport_FixupExtensionObject() + 16. _PyImport_FixupExtensionObject(): add it to interp->imports.modules_by_index + 17. _PyImport_FixupExtensionObject(): copy __dict__ into def->m_base.m_copy + 18. _PyImport_FixupExtensionObject(): add it to _PyRuntime.imports.extensions + + (6). subsequent times (found in _PyRuntime.imports.extensions): + 1. _imp_create_dynamic_impl() -> import_find_extension() + 2. import_find_extension() -> import_add_module() + 3. if name in sys.modules: use that module + 4. else: + 1. import_add_module() -> PyModule_NewObject() + 2. import_add_module(): set it on sys.modules + 5. import_find_extension(): copy the "m_copy" dict into __dict__ + 6. _imp_create_dynamic_impl() -> _PyImport_CheckSubinterpIncompatibleExtensionAllowed() + + (10). (every time): + 1. noop + + + ...for single-phase init modules, where m_size >= 0: + + (6). not main interpreter and never loaded there - every time (not found in _PyRuntime.imports.extensions): + 1-16. (same as for m_size == -1) + + (6). main interpreter - first time (not found in _PyRuntime.imports.extensions): + 1-16. (same as for m_size == -1) + 17. _PyImport_FixupExtensionObject(): add it to _PyRuntime.imports.extensions + + (6). previously loaded in main interpreter (found in _PyRuntime.imports.extensions): + 1. _imp_create_dynamic_impl() -> import_find_extension() + 2. import_find_extension(): call def->m_base.m_init + 3. import_find_extension(): add the module to sys.modules + + (10). every time: + 1. noop + + + ...for multi-phase init modules: + + (6). every time: + 1. _imp_create_dynamic_impl() -> import_find_extension() (not found) + 2. _imp_create_dynamic_impl() -> _PyImport_LoadDynamicModuleWithSpec() + 3. _PyImport_LoadDynamicModuleWithSpec(): load module init func + 4. _PyImport_LoadDynamicModuleWithSpec(): call module init func + 5. _PyImport_LoadDynamicModuleWithSpec() -> PyModule_FromDefAndSpec() + 6. PyModule_FromDefAndSpec(): gather/check moduledef slots + 7. if there's a Py_mod_create slot: + 1. PyModule_FromDefAndSpec(): call its function + 8. else: + 1. PyModule_FromDefAndSpec() -> PyModule_NewObject() + 9: PyModule_FromDefAndSpec(): set mod->md_def + 10. PyModule_FromDefAndSpec() -> _add_methods_to_object() + 11. PyModule_FromDefAndSpec() -> PyModule_SetDocString() + + (10). every time: + 1. _imp_exec_dynamic_impl() -> exec_builtin_or_dynamic() + 2. if mod->md_state == NULL (including if m_size == 0): + 1. exec_builtin_or_dynamic() -> PyModule_ExecDef() + 2. PyModule_ExecDef(): allocate mod->md_state + 3. if there's a Py_mod_exec slot: + 1. PyModule_ExecDef(): call its function + */ + + /* Make sure name is fully qualified. This is a bit of a hack: when the shared library is loaded, From 828a734364ccfe94281f1c960396cd79f417cfa9 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Fri, 17 Feb 2023 14:52:01 -0700 Subject: [PATCH 04/17] Add _forget_extension(). --- Lib/test/test_imp.py | 131 ++++++++++++++++++++++++++++--------------- 1 file changed, 87 insertions(+), 44 deletions(-) diff --git a/Lib/test/test_imp.py b/Lib/test/test_imp.py index f82e1c8ad643a1..d2778b98d1f46d 100644 --- a/Lib/test/test_imp.py +++ b/Lib/test/test_imp.py @@ -39,6 +39,38 @@ def requires_load_dynamic(meth): 'imp.load_dynamic() required')(meth) +def _forget_extension(mod): + """Clear all internally cached data for the extension. + + This mostly applies only to single-phase init modules. + """ + if isinstance(mod, str): + name = mod + mod = None + else: + name = mod.__name__ + + try: + del sys.modules[name] + except KeyError: + pass + sys.modules[name] = None + + try: + if mod is None: + fileobj, filename, _ = imp.find_module(name) + fileobj.close() + mod = imp.load_dynamic(name, filename) + else: + filename = mod.__file__ + if hasattr(mod, '_clear_globals'): + mod._clear_globals() + del mod + _testinternalcapi.clear_extension(name, filename) + finally: + del sys.modules[name] + + class LockTests(unittest.TestCase): """Very basic test of import lock functions.""" @@ -263,6 +295,16 @@ def test_issue16421_multiple_modules_in_one_dll(self): with self.assertRaises(ImportError): imp.load_dynamic('nonexistent', pathname) + @requires_load_dynamic + def test_singlephase_clears_globals(self): + import _testsinglephase + self.addCleanup(_forget_extension, _testsinglephase) + + _testsinglephase._clear_globals() + init_count = _testsinglephase.initialized_count() + + self.assertEqual(init_count, -1) + @requires_subinterpreters @requires_load_dynamic def test_singlephase_multiple_interpreters(self): @@ -271,63 +313,65 @@ def test_singlephase_multiple_interpreters(self): # PyModuleDef for that object, which can be a problem. # This single-phase module has global state, which is shared - # by the interpreters. + # by all interpreters. import _testsinglephase - name = _testsinglephase.__name__ - filename = _testsinglephase.__file__ - del sys.modules[name] - _testsinglephase._clear_globals() - _testinternalcapi.clear_extension(name, filename) + # Start and end clean. + _forget_extension(_testsinglephase) + import _testsinglephase + self.addCleanup(_forget_extension, _testsinglephase) + init_count = _testsinglephase.initialized_count() - assert init_count == -1, (init_count,) + lookedup = _testsinglephase.look_up_self() + _initialized = _testsinglephase._initialized + initialized = _testsinglephase.initialized() - def clean_up(): - _testsinglephase._clear_globals() - _testinternalcapi.clear_extension(name, filename) - self.addCleanup(clean_up) + self.assertEqual(init_count, 1) + self.assertIs(lookedup, _testsinglephase) + self.assertEqual(_initialized, initialized) interp1 = _interpreters.create(isolated=False) self.addCleanup(_interpreters.destroy, interp1) interp2 = _interpreters.create(isolated=False) self.addCleanup(_interpreters.destroy, interp2) - script = textwrap.dedent(f''' - import _testsinglephase + with self.subTest('without resetting'): + script = textwrap.dedent(f''' + import _testsinglephase - expected = %d - init_count = _testsinglephase.initialized_count() - if init_count != expected: - raise Exception(init_count) + expected = %d + init_count = _testsinglephase.initialized_count() + if init_count != expected: + raise Exception(init_count) - lookedup = _testsinglephase.look_up_self() - if lookedup is not _testsinglephase: - raise Exception((_testsinglephase, lookedup)) + lookedup = _testsinglephase.look_up_self() + if lookedup is not _testsinglephase: + raise Exception((_testsinglephase, lookedup)) - # Attrs set in the module init func are in m_copy. - _initialized = _testsinglephase._initialized - initialized = _testsinglephase.initialized() - if _initialized != initialized: - raise Exception((_initialized, initialized)) + # Attrs set in the module init func are in m_copy. + _initialized = _testsinglephase._initialized + initialized = _testsinglephase.initialized() + if _initialized != initialized: + raise Exception((_initialized, initialized)) - # Attrs set after loading are not in m_copy. - if hasattr(_testsinglephase, 'spam'): - raise Exception(_testsinglephase.spam) - _testsinglephase.spam = expected - ''') + # Attrs set after loading are not in m_copy. + if hasattr(_testsinglephase, 'spam'): + raise Exception(_testsinglephase.spam) + _testsinglephase.spam = expected + ''') - # Use an interpreter that gets destroyed right away. - ret = support.run_in_subinterp(script % 1) - self.assertEqual(ret, 0) + # Use an interpreter that gets destroyed right away. + ret = support.run_in_subinterp(script % 1) + self.assertEqual(ret, 0) - # The module's init func gets run again. - # The module's globals did not get destroyed. - _interpreters.run_string(interp1, script % 2) + # The module's init func gets run again. + # The module's globals did not get destroyed. + _interpreters.run_string(interp1, script % 1) - # The module's init func is not run again. - # The second interpreter copies the module's m_copy. - # However, globals are still shared. - _interpreters.run_string(interp2, script % 2) + # The module's init func is not run again. + # The second interpreter copies the module's m_copy. + # However, globals are still shared. + _interpreters.run_string(interp2, script % 1) @requires_load_dynamic def test_singlephase_variants(self): @@ -338,10 +382,9 @@ def test_singlephase_variants(self): fileobj, pathname, _ = imp.find_module(basename) fileobj.close() - def clean_up(): - import _testsinglephase - _testsinglephase._clear_globals() - self.addCleanup(clean_up) + # Start and end clean. + _forget_extension(basename) + self.addCleanup(_forget_extension, basename) def add_ext_cleanup(name): def clean_up(): From e9096cd7839395a5dedcbe63c9ecc5ad9a211198 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Fri, 17 Feb 2023 15:13:08 -0700 Subject: [PATCH 05/17] Clean up test_singlephase_multiple_interpreters. --- Lib/test/test_imp.py | 37 +++++++++++++++++++++++-------------- Modules/_testsinglephase.c | 22 +++++++++++----------- 2 files changed, 34 insertions(+), 25 deletions(-) diff --git a/Lib/test/test_imp.py b/Lib/test/test_imp.py index d2778b98d1f46d..0747a75fadce37 100644 --- a/Lib/test/test_imp.py +++ b/Lib/test/test_imp.py @@ -323,8 +323,8 @@ def test_singlephase_multiple_interpreters(self): init_count = _testsinglephase.initialized_count() lookedup = _testsinglephase.look_up_self() - _initialized = _testsinglephase._initialized - initialized = _testsinglephase.initialized() + _initialized = _testsinglephase._module_initialized + initialized = _testsinglephase.state_initialized() self.assertEqual(init_count, 1) self.assertIs(lookedup, _testsinglephase) @@ -335,43 +335,51 @@ def test_singlephase_multiple_interpreters(self): interp2 = _interpreters.create(isolated=False) self.addCleanup(_interpreters.destroy, interp2) - with self.subTest('without resetting'): + with self.subTest('without resetting; ' + 'already loaded in main interpreter'): + # Attrs set after loading are not in m_copy. + _testsinglephase.spam = 'spam, spam, spam, spam, eggs, and spam' + objid = id(_testsinglephase) + script = textwrap.dedent(f''' import _testsinglephase - expected = %d init_count = _testsinglephase.initialized_count() - if init_count != expected: + if init_count != {init_count}: raise Exception(init_count) + # The "looked up" module is interpreter-specific. lookedup = _testsinglephase.look_up_self() if lookedup is not _testsinglephase: raise Exception((_testsinglephase, lookedup)) # Attrs set in the module init func are in m_copy. - _initialized = _testsinglephase._initialized - initialized = _testsinglephase.initialized() + # Both of the following were set in module init, + # which didn't happen in this interpreter + # (unfortunately). + _initialized = _testsinglephase._module_initialized + initialized = _testsinglephase.state_initialized() if _initialized != initialized: raise Exception((_initialized, initialized)) # Attrs set after loading are not in m_copy. if hasattr(_testsinglephase, 'spam'): raise Exception(_testsinglephase.spam) - _testsinglephase.spam = expected + _testsinglephase.spam = 'spam, spam, spam, spam, ...' ''') # Use an interpreter that gets destroyed right away. - ret = support.run_in_subinterp(script % 1) + ret = support.run_in_subinterp(script) self.assertEqual(ret, 0) # The module's init func gets run again. # The module's globals did not get destroyed. - _interpreters.run_string(interp1, script % 1) + _interpreters.run_string(interp1, script) # The module's init func is not run again. # The second interpreter copies the module's m_copy. # However, globals are still shared. - _interpreters.run_string(interp2, script % 1) + _interpreters.run_string(interp2, script) @requires_load_dynamic def test_singlephase_variants(self): @@ -411,7 +419,7 @@ def re_load(name, module): def check_common(name, module): summed = module.sum(1, 2) lookedup = module.look_up_self() - initialized = module.initialized() + initialized = module.state_initialized() cached = sys.modules[name] # module.__name__ might not match, but the spec will. @@ -459,7 +467,7 @@ def check_common_reloaded(name, module, cached, before, reloaded): def check_basic_reloaded(module, lookedup, initialized, init_count, before, reloaded): relookedup = reloaded.look_up_self() - reinitialized = reloaded.initialized() + reinitialized = reloaded.state_initialized() reinit_count = reloaded.initialized_count() self.assertIs(reloaded, module) @@ -474,7 +482,7 @@ def check_basic_reloaded(module, lookedup, initialized, init_count, def check_with_reinit_reloaded(module, lookedup, initialized, before, reloaded): relookedup = reloaded.look_up_self() - reinitialized = reloaded.initialized() + reinitialized = reloaded.state_initialized() self.assertIsNot(reloaded, module) self.assertIsNot(reloaded, module) @@ -502,6 +510,7 @@ def check_with_reinit_reloaded(module, lookedup, initialized, check_basic_reloaded(mod, lookedup, initialized, init_count, before, reloaded) basic = mod + return # Check its indirect variants. diff --git a/Modules/_testsinglephase.c b/Modules/_testsinglephase.c index 565221c887e5ae..a16157702ae789 100644 --- a/Modules/_testsinglephase.c +++ b/Modules/_testsinglephase.c @@ -140,7 +140,7 @@ init_module(PyObject *module, module_state *state) if (initialized == NULL) { return -1; } - if (PyModule_AddObjectRef(module, "_initialized", initialized) != 0) { + if (PyModule_AddObjectRef(module, "_module_initialized", initialized) != 0) { return -1; } @@ -148,13 +148,13 @@ init_module(PyObject *module, module_state *state) } -PyDoc_STRVAR(common_initialized_doc, -"initialized()\n\ +PyDoc_STRVAR(common_state_initialized_doc, +"state_initialized()\n\ \n\ -Return the seconds-since-epoch when the module was initialized."); +Return the seconds-since-epoch when the module state was initialized."); static PyObject * -common_initialized(PyObject *self, PyObject *Py_UNUSED(ignored)) +common_state_initialized(PyObject *self, PyObject *Py_UNUSED(ignored)) { module_state *state = get_module_state(self); if (state == NULL) { @@ -164,9 +164,9 @@ common_initialized(PyObject *self, PyObject *Py_UNUSED(ignored)) return PyFloat_FromDouble(d); } -#define INITIALIZED_METHODDEF \ - {"initialized", common_initialized, METH_NOARGS, \ - common_initialized_doc} +#define STATE_INITIALIZED_METHODDEF \ + {"state_initialized", common_state_initialized, METH_NOARGS, \ + common_state_initialized_doc} PyDoc_STRVAR(common_look_up_self_doc, @@ -265,7 +265,7 @@ basic__clear_globals(PyObject *self, PyObject *Py_UNUSED(ignored)) static PyMethodDef TestMethods_Basic[] = { LOOK_UP_SELF_METHODDEF, SUM_METHODDEF, - INITIALIZED_METHODDEF, + STATE_INITIALIZED_METHODDEF, INITIALIZED_COUNT_METHODDEF, _CLEAR_GLOBALS_METHODDEF, {NULL, NULL} /* sentinel */ @@ -360,7 +360,7 @@ PyInit__testsinglephase_basic_copy(void) static PyMethodDef TestMethods_Reinit[] = { LOOK_UP_SELF_METHODDEF, SUM_METHODDEF, - INITIALIZED_METHODDEF, + STATE_INITIALIZED_METHODDEF, {NULL, NULL} /* sentinel */ }; @@ -421,7 +421,7 @@ PyInit__testsinglephase_with_reinit(void) static PyMethodDef TestMethods_WithState[] = { LOOK_UP_SELF_METHODDEF, SUM_METHODDEF, - INITIALIZED_METHODDEF, + STATE_INITIALIZED_METHODDEF, {NULL, NULL} /* sentinel */ }; From c993f970782a823519e74e554087eba1f720ab4f Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Fri, 17 Feb 2023 15:41:15 -0700 Subject: [PATCH 06/17] Add another test case. --- Lib/test/test_imp.py | 79 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 79 insertions(+) diff --git a/Lib/test/test_imp.py b/Lib/test/test_imp.py index 0747a75fadce37..aef85a21377717 100644 --- a/Lib/test/test_imp.py +++ b/Lib/test/test_imp.py @@ -315,6 +315,8 @@ def test_singlephase_multiple_interpreters(self): # This single-phase module has global state, which is shared # by all interpreters. import _testsinglephase + name = _testsinglephase.__name__ + filename = _testsinglephase.__file__ # Start and end clean. _forget_extension(_testsinglephase) @@ -334,6 +336,15 @@ def test_singlephase_multiple_interpreters(self): self.addCleanup(_interpreters.destroy, interp1) interp2 = _interpreters.create(isolated=False) self.addCleanup(_interpreters.destroy, interp2) + for interpid in [interp1, interp2]: + _interpreters.run_string(interpid, 'import _testinternalcapi, sys') + + def clear_subinterp(interpid): + _interpreters.run_string(interpid, textwrap.dedent(f''' + del sys.modules[{name!r}] + _testsinglephase._clear_globals() + _testinternalcapi.clear_extension({name!r}, {filename!r}) + ''')) with self.subTest('without resetting; ' 'already loaded in main interpreter'): @@ -361,6 +372,10 @@ def test_singlephase_multiple_interpreters(self): initialized = _testsinglephase.state_initialized() if _initialized != initialized: raise Exception((_initialized, initialized)) + if _initialized != {initialized}: + raise Exception((_initialized, {initialized})) + if initialized != {initialized}: + raise Exception((initialized, {initialized})) # Attrs set after loading are not in m_copy. if hasattr(_testsinglephase, 'spam'): @@ -381,6 +396,70 @@ def test_singlephase_multiple_interpreters(self): # However, globals are still shared. _interpreters.run_string(interp2, script) + _forget_extension(_testsinglephase) + for interpid in [interp1, interp2]: + clear_subinterp(interpid) + + with self.subTest('without resetting; ' + 'already loaded in deleted interpreter'): + + # Use an interpreter that gets destroyed right away. + ret = support.run_in_subinterp(textwrap.dedent(f''' + import _testsinglephase + + # This is the first time loaded since reset. + init_count = _testsinglephase.initialized_count() + if init_count != 1: + raise Exception(init_count) + + # Attrs set in the module init func are in m_copy. + _initialized = _testsinglephase._module_initialized + initialized = _testsinglephase.state_initialized() + if _initialized != initialized: + raise Exception((_initialized, initialized)) + if _initialized == {initialized}: + raise Exception((_initialized, {initialized})) + if initialized == {initialized}: + raise Exception((initialized, {initialized})) + + # Attrs set after loading are not in m_copy. + if hasattr(_testsinglephase, 'spam'): + raise Exception(_testsinglephase.spam) + _testsinglephase.spam = 'spam, spam, mash, spam, eggs, and spam' + ''')) + self.assertEqual(ret, 0) + + script = textwrap.dedent(f''' + import _testsinglephase + + init_count = _testsinglephase.initialized_count() + if init_count != 2: + raise Exception(init_count) + + # Attrs set in the module init func are in m_copy. + # Both of the following were set in module init, + # which didn't happen in this interpreter + # (unfortunately). + _initialized = _testsinglephase._module_initialized + initialized = _testsinglephase.state_initialized() + if _initialized != initialized: + raise Exception((_initialized, initialized)) + + # Attrs set after loading are not in m_copy. + if hasattr(_testsinglephase, 'spam'): + raise Exception(_testsinglephase.spam) + _testsinglephase.spam = 'spam, spam, spam, spam, ...' + ''') + + # The module's init func gets run again. + # The module's globals did not get destroyed. + _interpreters.run_string(interp1, script) + + # The module's init func is not run again. + # The second interpreter copies the module's m_copy. + # However, globals are still shared. + _interpreters.run_string(interp2, script) + @requires_load_dynamic def test_singlephase_variants(self): # Exercise the most meaningful variants described in Python/import.c. From cc5c771a441ac2c0b7df22e9d7a0afa768087b6f Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Fri, 17 Feb 2023 18:18:02 -0700 Subject: [PATCH 07/17] Expand test_singlephase_multiple_interpreters. --- Lib/test/test_imp.py | 277 +++++++++++++++++++++++++++---------------- 1 file changed, 178 insertions(+), 99 deletions(-) diff --git a/Lib/test/test_imp.py b/Lib/test/test_imp.py index aef85a21377717..d9fdaddc96efae 100644 --- a/Lib/test/test_imp.py +++ b/Lib/test/test_imp.py @@ -1,4 +1,5 @@ import gc +import json import importlib import importlib.util import os @@ -312,6 +313,78 @@ def test_singlephase_multiple_interpreters(self): # in multiple interpreters, those interpreters share a # PyModuleDef for that object, which can be a problem. + def parse_snapshot(data): + if isinstance(data, str): + data = json.loads(data.decode()) + elif isinstance(data, bytes): + data = json.loads(data) + spam = data.pop('spam', None) + has_spam = data.pop('has_spam', False) + snapshot = type(sys.implementation)(**data) + if has_spam: + snapshot.spam = spam + return snapshot + + def check_common(snapshot): + # The "looked up" module is interpreter-specific + # (interp->imports.modules_by_index was set for the module). + self.assertEqual(snapshot.lookedup, snapshot.objid) + with self.assertRaises(AttributeError): + snapshot.spam + + def check_fresh(snapshot): + """ + The module had not been loaded before (at least since fully reset). + """ + # The module's init func was run. + # A copy of the module's __dict__ was stored in def->m_base.m_copy. + # The previous m_copy was deleted first. + # _PyRuntime.imports.extensions was set. + self.assertEqual(snapshot.init_count, 1) + # The global state was initialized. + # The module attrs were initialized from that state. + self.assertEqual(snapshot.module_initialized, + snapshot.state_initialized) + + def check_semi_fresh(snapshot, base, prev): + """ + The module had been loaded before and then reset + (but the module global state wasn't). + """ + # The module's init func was run again. + # A copy of the module's __dict__ was stored in def->m_base.m_copy. + # The previous m_copy was deleted first. + # The module globals did not get reset. + self.assertNotEqual(snapshot.objid, base.objid) + self.assertNotEqual(snapshot.objid, prev.objid) + self.assertEqual(snapshot.init_count, prev.init_count + 1) + # The global state was updated. + # The module attrs were initialized from that state. + self.assertEqual(snapshot.module_initialized, + snapshot.state_initialized) + self.assertNotEqual(snapshot.state_initialized, + base.state_initialized) + self.assertNotEqual(snapshot.state_initialized, + prev.state_initialized) + + def check_copied(snapshot, base): + """ + The module had been loaded before and never reset. + """ + # The module's init func was not run again. + # The interpreter copied m_copy, as set by the other interpreter, + # with objects owned by the other interpreter. + # The module globals did not get reset. + self.assertNotEqual(snapshot.objid, base.objid) + self.assertEqual(snapshot.init_count, base.init_count) + # The global state was not updated since the init func did not run. + # The module attrs were not directly initialized from that state. + # The state and module attrs still match the previous loading. + self.assertEqual(snapshot.module_initialized, + snapshot.state_initialized) + self.assertEqual(snapshot.state_initialized, + base.state_initialized) + # This single-phase module has global state, which is shared # by all interpreters. import _testsinglephase @@ -323,78 +396,87 @@ def test_singlephase_multiple_interpreters(self): import _testsinglephase self.addCleanup(_forget_extension, _testsinglephase) - init_count = _testsinglephase.initialized_count() - lookedup = _testsinglephase.look_up_self() - _initialized = _testsinglephase._module_initialized - initialized = _testsinglephase.state_initialized() - - self.assertEqual(init_count, 1) - self.assertIs(lookedup, _testsinglephase) - self.assertEqual(_initialized, initialized) - + # Check the main interpreter. + main_snap = parse_snapshot(dict( + objid=id(_testsinglephase), + init_count=_testsinglephase.initialized_count(), + lookedup=id(_testsinglephase.look_up_self()), + state_initialized=_testsinglephase.state_initialized(), + module_initialized=_testsinglephase._module_initialized, + )) + check_common(main_snap) + check_fresh(main_snap) + + # Set up the interpreters. + setup_script = textwrap.dedent(''' + import sys + import _testinternalcapi + ''') interp1 = _interpreters.create(isolated=False) self.addCleanup(_interpreters.destroy, interp1) interp2 = _interpreters.create(isolated=False) self.addCleanup(_interpreters.destroy, interp2) for interpid in [interp1, interp2]: - _interpreters.run_string(interpid, 'import _testinternalcapi, sys') + _interpreters.run_string(interpid, setup_script) + cleanup_script = textwrap.dedent(f''' + sys.modules.pop({name!r}, None) + _testinternalcapi.clear_extension({name!r}, {filename!r}) + ''') def clear_subinterp(interpid): + _interpreters.run_string(interpid, cleanup_script) _interpreters.run_string(interpid, textwrap.dedent(f''' - del sys.modules[{name!r}] _testsinglephase._clear_globals() - _testinternalcapi.clear_extension({name!r}, {filename!r}) ''')) + r, w = os.pipe() + self.addCleanup(os.close, r) + self.addCleanup(os.close, w) + + script = textwrap.dedent(f''' + import json + import os + + import _testsinglephase + + data = dict( + objid=id(_testsinglephase), + init_count=_testsinglephase.initialized_count(), + lookedup=id(_testsinglephase.look_up_self()), + state_initialized=_testsinglephase.state_initialized(), + module_initialized=_testsinglephase._module_initialized, + has_spam=hasattr(_testsinglephase, 'spam'), + spam=getattr(_testsinglephase, 'spam', None), + ) + os.write({w}, json.dumps(data).encode()) + ''') + + def read_data(): + text = os.read(r, 500) + return parse_snapshot(text) + with self.subTest('without resetting; ' 'already loaded in main interpreter'): # Attrs set after loading are not in m_copy. _testsinglephase.spam = 'spam, spam, spam, spam, eggs, and spam' - objid = id(_testsinglephase) - - script = textwrap.dedent(f''' - import _testsinglephase - - init_count = _testsinglephase.initialized_count() - if init_count != {init_count}: - raise Exception(init_count) - - # The "looked up" module is interpreter-specific. - lookedup = _testsinglephase.look_up_self() - if lookedup is not _testsinglephase: - raise Exception((_testsinglephase, lookedup)) - - # Attrs set in the module init func are in m_copy. - # Both of the following were set in module init, - # which didn't happen in this interpreter - # (unfortunately). - _initialized = _testsinglephase._module_initialized - initialized = _testsinglephase.state_initialized() - if _initialized != initialized: - raise Exception((_initialized, initialized)) - if _initialized != {initialized}: - raise Exception((_initialized, {initialized})) - if initialized != {initialized}: - raise Exception((initialized, {initialized})) - - # Attrs set after loading are not in m_copy. - if hasattr(_testsinglephase, 'spam'): - raise Exception(_testsinglephase.spam) - _testsinglephase.spam = 'spam, spam, spam, spam, ...' - ''') # Use an interpreter that gets destroyed right away. ret = support.run_in_subinterp(script) self.assertEqual(ret, 0) + snap = read_data() + check_common(snap) + check_copied(snap, main_snap) - # The module's init func gets run again. - # The module's globals did not get destroyed. + # Use several interpreters that overlap. _interpreters.run_string(interp1, script) + snap = read_data() + check_common(snap) + check_copied(snap, main_snap) - # The module's init func is not run again. - # The second interpreter copies the module's m_copy. - # However, globals are still shared. _interpreters.run_string(interp2, script) + snap = read_data() + check_common(snap) + check_copied(snap, main_snap) _forget_extension(_testsinglephase) for interpid in [interp1, interp2]: @@ -402,63 +484,60 @@ def clear_subinterp(interpid): with self.subTest('without resetting; ' 'already loaded in deleted interpreter'): + # Use an interpreter that gets destroyed right away. + ret = support.run_in_subinterp(os.linesep.join([ + script, + textwrap.dedent(''' + # Attrs set after loading are not in m_copy. + _testsinglephase.spam = 'spam, spam, mash, spam, eggs, and spam' + ''')])) + self.assertEqual(ret, 0) + base = read_data() + check_common(base) + check_fresh(base) + + # Use several interpreters that overlap. + _interpreters.run_string(interp1, script) + interp1_snap = read_data() + check_common(interp1_snap) + check_semi_fresh(interp1_snap, main_snap, base) + + _interpreters.run_string(interp2, script) + snap = read_data() + check_common(snap) + check_copied(snap, interp1_snap) + + with self.subTest('resetting between each interpreter'): + _testsinglephase._clear_globals() # Use an interpreter that gets destroyed right away. - ret = support.run_in_subinterp(textwrap.dedent(f''' - import _testsinglephase - - # This is the first time loaded since reset. - init_count = _testsinglephase.initialized_count() - if init_count != 1: - raise Exception(init_count) - - # Attrs set in the module init func are in m_copy. - _initialized = _testsinglephase._module_initialized - initialized = _testsinglephase.state_initialized() - if _initialized != initialized: - raise Exception((_initialized, initialized)) - if _initialized == {initialized}: - raise Exception((_initialized, {initialized})) - if initialized == {initialized}: - raise Exception((initialized, {initialized})) - - # Attrs set after loading are not in m_copy. - if hasattr(_testsinglephase, 'spam'): - raise Exception(_testsinglephase.spam) - _testsinglephase.spam = 'spam, spam, mash, spam, eggs, and spam' - ''')) + ret = support.run_in_subinterp(os.linesep.join([ + setup_script, + cleanup_script, + script, + textwrap.dedent(''' + # Attrs set after loading are not in m_copy. + _testsinglephase.spam = 'spam, spam, mash, spam, eggs, and spam' + ''')])) self.assertEqual(ret, 0) + base = read_data() + check_common(base) + check_fresh(base) - script = textwrap.dedent(f''' - import _testsinglephase - - init_count = _testsinglephase.initialized_count() - if init_count != 2: - raise Exception(init_count) - - # Attrs set in the module init func are in m_copy. - # Both of the following were set in module init, - # which didn't happen in this interpreter - # (unfortunately). - _initialized = _testsinglephase._module_initialized - initialized = _testsinglephase.state_initialized() - if _initialized != initialized: - raise Exception((_initialized, initialized)) - - # Attrs set after loading are not in m_copy. - if hasattr(_testsinglephase, 'spam'): - raise Exception(_testsinglephase.spam) - _testsinglephase.spam = 'spam, spam, spam, spam, ...' - ''') - - # The module's init func gets run again. - # The module's globals did not get destroyed. + # Use several interpreters that overlap. + clear_subinterp(interp1) + #_interpreters.run_string(interpid, cleanup_script) _interpreters.run_string(interp1, script) + snap = read_data() + check_common(snap) + check_fresh(snap) - # The module's init func is not run again. - # The second interpreter copies the module's m_copy. - # However, globals are still shared. + clear_subinterp(interp2) + #_interpreters.run_string(interpid, cleanup_script) _interpreters.run_string(interp2, script) + snap = read_data() + check_common(snap) + check_fresh(snap) @requires_load_dynamic def test_singlephase_variants(self): From 08741b85c74921a4d4ee2c296da704a80f18f62c Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Tue, 21 Feb 2023 18:04:34 -0700 Subject: [PATCH 08/17] Clean up the tests. --- Lib/test/test_imp.py | 79 ++++++++++++++++++++------------------------ 1 file changed, 36 insertions(+), 43 deletions(-) diff --git a/Lib/test/test_imp.py b/Lib/test/test_imp.py index d9fdaddc96efae..33e09674cb1e14 100644 --- a/Lib/test/test_imp.py +++ b/Lib/test/test_imp.py @@ -40,36 +40,17 @@ def requires_load_dynamic(meth): 'imp.load_dynamic() required')(meth) -def _forget_extension(mod): +def _forget_extension(name, filename): """Clear all internally cached data for the extension. This mostly applies only to single-phase init modules. """ - if isinstance(mod, str): - name = mod - mod = None - else: - name = mod.__name__ - - try: - del sys.modules[name] - except KeyError: - pass - sys.modules[name] = None - - try: - if mod is None: - fileobj, filename, _ = imp.find_module(name) - fileobj.close() - mod = imp.load_dynamic(name, filename) - else: - filename = mod.__file__ - if hasattr(mod, '_clear_globals'): - mod._clear_globals() - del mod - _testinternalcapi.clear_extension(name, filename) - finally: + if name in sys.modules: + if hasattr(sys.modules[name], '_clear_globals'): + assert sys.modules[name].__file__ == filename + sys.modules[name]._clear_globals() del sys.modules[name] + _testinternalcapi.clear_extension(name, filename) class LockTests(unittest.TestCase): @@ -298,12 +279,21 @@ def test_issue16421_multiple_modules_in_one_dll(self): @requires_load_dynamic def test_singlephase_clears_globals(self): - import _testsinglephase - self.addCleanup(_forget_extension, _testsinglephase) + name = '_testsinglephase' + fileobj, filename, _ = imp.find_module(name) + fileobj.close() + + # Start and end fresh. + _forget_extension(name, filename) + self.addCleanup(_forget_extension, name, filename) + + _testsinglephase = imp.load_dynamic(name, filename) + initialized = _testsinglephase.state_initialized() _testsinglephase._clear_globals() init_count = _testsinglephase.initialized_count() + self.assertGreater(initialized, 0) self.assertEqual(init_count, -1) @requires_subinterpreters @@ -385,16 +375,18 @@ def check_copied(snapshot, base): self.assertEqual(snapshot.state_initialized, base.state_initialized) + # Find the module's file. + name = '_testsinglephase' + fileobj, filename, _ = imp.find_module(name) + fileobj.close() + + # Start and end fresh. + _forget_extension(name, filename) + self.addCleanup(_forget_extension, name, filename) + # This single-phase module has global state, which is shared # by all interpreters. - import _testsinglephase - name = _testsinglephase.__name__ - filename = _testsinglephase.__file__ - - # Start and end clean. - _forget_extension(_testsinglephase) - import _testsinglephase - self.addCleanup(_forget_extension, _testsinglephase) + _testsinglephase = imp.load_dynamic(name, filename) # Check the main interpreter. main_snap = parse_snapshot(dict( @@ -478,9 +470,10 @@ def read_data(): check_common(snap) check_copied(snap, main_snap) - _forget_extension(_testsinglephase) for interpid in [interp1, interp2]: clear_subinterp(interpid) + _testsinglephase._clear_globals() + _forget_extension(name, filename) with self.subTest('without resetting; ' 'already loaded in deleted interpreter'): @@ -545,22 +538,22 @@ def test_singlephase_variants(self): self.maxDiff = None basename = '_testsinglephase' - fileobj, pathname, _ = imp.find_module(basename) + fileobj, filename, _ = imp.find_module(basename) fileobj.close() - # Start and end clean. - _forget_extension(basename) - self.addCleanup(_forget_extension, basename) + # Start and end fresh. + _forget_extension(name, filename) + self.addCleanup(_forget_extension, name, filename) def add_ext_cleanup(name): def clean_up(): - _testinternalcapi.clear_extension(name, pathname) + _testinternalcapi.clear_extension(name, filename) self.addCleanup(clean_up) modules = {} def load(name): assert name not in modules - module = imp.load_dynamic(name, pathname) + module = imp.load_dynamic(name, filename) self.assertNotIn(module, modules.values()) modules[name] = module return module @@ -570,7 +563,7 @@ def re_load(name, module): before = type(module)(module.__name__) before.__dict__.update(vars(module)) - reloaded = imp.load_dynamic(name, pathname) + reloaded = imp.load_dynamic(name, filename) return before, reloaded From 31716e917aa65294ac6cf12b5eb844ea483f3e8c Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Tue, 21 Feb 2023 18:05:05 -0700 Subject: [PATCH 09/17] Clean up clear_singlephase_extension(). --- Python/import.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/Python/import.c b/Python/import.c index 2ed43d79a779dc..57d4eea148810f 100644 --- a/Python/import.c +++ b/Python/import.c @@ -1116,7 +1116,11 @@ clear_singlephase_extension(PyInterpreterState *interp, } /* Clear the cached module def. */ - return _extensions_cache_delete(filename, name); + if (_extensions_cache_delete(filename, name) < 0) { + return -1; + } + + return 0; } From 2c3dd42754e719975c59160d1725ff4bda928f9f Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Tue, 21 Feb 2023 18:10:50 -0700 Subject: [PATCH 10/17] Add a check. --- Lib/test/test_imp.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_imp.py b/Lib/test/test_imp.py index 33e09674cb1e14..c022efd3618e04 100644 --- a/Lib/test/test_imp.py +++ b/Lib/test/test_imp.py @@ -288,12 +288,14 @@ def test_singlephase_clears_globals(self): self.addCleanup(_forget_extension, name, filename) _testsinglephase = imp.load_dynamic(name, filename) - initialized = _testsinglephase.state_initialized() + init_before = _testsinglephase.state_initialized() _testsinglephase._clear_globals() + init_after = _testsinglephase.state_initialized() init_count = _testsinglephase.initialized_count() - self.assertGreater(initialized, 0) + self.assertGreater(init_before, 0) + self.assertEqual(init_after, 0) self.assertEqual(init_count, -1) @requires_subinterpreters From a353e7e9640c38e7a11e68749e38095b1ba3fa11 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Wed, 22 Feb 2023 09:07:45 -0700 Subject: [PATCH 11/17] Fix a typo. --- Lib/test/test_imp.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_imp.py b/Lib/test/test_imp.py index c022efd3618e04..cb1239c28e5a59 100644 --- a/Lib/test/test_imp.py +++ b/Lib/test/test_imp.py @@ -544,8 +544,8 @@ def test_singlephase_variants(self): fileobj.close() # Start and end fresh. - _forget_extension(name, filename) - self.addCleanup(_forget_extension, name, filename) + _forget_extension(basename, filename) + self.addCleanup(_forget_extension, basename, filename) def add_ext_cleanup(name): def clean_up(): From c6a39e38d423bf7549d96f382fc776be849adffe Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Wed, 22 Feb 2023 09:13:24 -0700 Subject: [PATCH 12/17] script -> snapshot_script. --- Lib/test/test_imp.py | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/Lib/test/test_imp.py b/Lib/test/test_imp.py index cb1239c28e5a59..067a590258b134 100644 --- a/Lib/test/test_imp.py +++ b/Lib/test/test_imp.py @@ -427,7 +427,7 @@ def clear_subinterp(interpid): self.addCleanup(os.close, r) self.addCleanup(os.close, w) - script = textwrap.dedent(f''' + snapshot_script = textwrap.dedent(f''' import json import os @@ -455,19 +455,19 @@ def read_data(): _testsinglephase.spam = 'spam, spam, spam, spam, eggs, and spam' # Use an interpreter that gets destroyed right away. - ret = support.run_in_subinterp(script) + ret = support.run_in_subinterp(snapshot_script) self.assertEqual(ret, 0) snap = read_data() check_common(snap) check_copied(snap, main_snap) # Use several interpreters that overlap. - _interpreters.run_string(interp1, script) + _interpreters.run_string(interp1, snapshot_script) snap = read_data() check_common(snap) check_copied(snap, main_snap) - _interpreters.run_string(interp2, script) + _interpreters.run_string(interp2, snapshot_script) snap = read_data() check_common(snap) check_copied(snap, main_snap) @@ -481,7 +481,7 @@ def read_data(): 'already loaded in deleted interpreter'): # Use an interpreter that gets destroyed right away. ret = support.run_in_subinterp(os.linesep.join([ - script, + snapshot_script, textwrap.dedent(''' # Attrs set after loading are not in m_copy. _testsinglephase.spam = 'spam, spam, mash, spam, eggs, and spam' @@ -492,12 +492,12 @@ def read_data(): check_fresh(base) # Use several interpreters that overlap. - _interpreters.run_string(interp1, script) + _interpreters.run_string(interp1, snapshot_script) interp1_snap = read_data() check_common(interp1_snap) check_semi_fresh(interp1_snap, main_snap, base) - _interpreters.run_string(interp2, script) + _interpreters.run_string(interp2, snapshot_script) snap = read_data() check_common(snap) check_copied(snap, interp1_snap) @@ -506,14 +506,15 @@ def read_data(): _testsinglephase._clear_globals() # Use an interpreter that gets destroyed right away. - ret = support.run_in_subinterp(os.linesep.join([ + script = os.linesep.join([ setup_script, cleanup_script, - script, + snapshot_script, textwrap.dedent(''' # Attrs set after loading are not in m_copy. _testsinglephase.spam = 'spam, spam, mash, spam, eggs, and spam' - ''')])) + ''')]) + ret = support.run_in_subinterp(script) self.assertEqual(ret, 0) base = read_data() check_common(base) @@ -522,14 +523,14 @@ def read_data(): # Use several interpreters that overlap. clear_subinterp(interp1) #_interpreters.run_string(interpid, cleanup_script) - _interpreters.run_string(interp1, script) + _interpreters.run_string(interp1, snapshot_script) snap = read_data() check_common(snap) check_fresh(snap) clear_subinterp(interp2) #_interpreters.run_string(interpid, cleanup_script) - _interpreters.run_string(interp2, script) + _interpreters.run_string(interp2, snapshot_script) snap = read_data() check_common(snap) check_fresh(snap) From 0c65700909bf91843938c2cb9a6add0caa891b65 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Wed, 22 Feb 2023 15:41:10 -0700 Subject: [PATCH 13/17] Use spec_from_file_location() in imp.load_dynamic(). --- Lib/imp.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/imp.py b/Lib/imp.py index fc42c15765852e..fe850f6a001814 100644 --- a/Lib/imp.py +++ b/Lib/imp.py @@ -338,8 +338,8 @@ def load_dynamic(name, path, file=None): # Issue #24748: Skip the sys.modules check in _load_module_shim; # always load new extension - spec = importlib.machinery.ModuleSpec( - name=name, loader=loader, origin=path) + spec = importlib.util.spec_from_file_location( + name, path, loader=loader) return _load(spec) else: From 993484e9ca769d7308e515020e923ddd0467f645 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Thu, 23 Feb 2023 16:06:52 -0700 Subject: [PATCH 14/17] Update single-phase-init tests. --- Lib/test/test_imp.py | 1301 ++++++++++++++++++++++++++---------------- 1 file changed, 820 insertions(+), 481 deletions(-) diff --git a/Lib/test/test_imp.py b/Lib/test/test_imp.py index 067a590258b134..ecc34b7b68d051 100644 --- a/Lib/test/test_imp.py +++ b/Lib/test/test_imp.py @@ -12,6 +12,7 @@ from test.support import script_helper from test.support import warnings_helper import textwrap +import types import unittest import warnings imp = warnings_helper.import_deprecated('imp') @@ -40,17 +41,167 @@ def requires_load_dynamic(meth): 'imp.load_dynamic() required')(meth) -def _forget_extension(name, filename): - """Clear all internally cached data for the extension. +class ModuleSnapshot(types.SimpleNamespace): + """A representation of a module for testing. - This mostly applies only to single-phase init modules. + Fields: + + * id - the module's object ID + * module - the actual module or an adequate substitute + * __file__ + * __spec__ + * name + * origin + * ns - a copy (dict) of the module's __dict__ (or None) + * ns_id - the object ID of the module's __dict__ + * cached - the sys.modules[mod.__spec__.name] entry (or None) + * cached_id - the object ID of the sys.modules entry (or None) + + In cases where the value is not available (e.g. due to serialization), + the value will be None. """ - if name in sys.modules: - if hasattr(sys.modules[name], '_clear_globals'): - assert sys.modules[name].__file__ == filename - sys.modules[name]._clear_globals() - del sys.modules[name] - _testinternalcapi.clear_extension(name, filename) + _fields = tuple('id module ns ns_id cached cached_id'.split()) + + @classmethod + def from_module(cls, mod): + name = mod.__spec__.name + cached = sys.modules.get(name) + return cls( + id=id(mod), + module=mod, + ns=types.SimpleNamespace(**mod.__dict__), + ns_id=id(mod.__dict__), + cached=cached, + cached_id=id(cached), + ) + + SCRIPT = textwrap.dedent(''' + {imports} + + name = {name!r} + + {prescript} + + mod = {name} + + {body} + + {postscript} + ''') + IMPORTS = textwrap.dedent(''' + import sys + ''').strip() + SCRIPT_BODY = textwrap.dedent(''' + # Capture the snapshot data. + cached = sys.modules.get(name) + snapshot = dict( + id=id(mod), + module=dict( + __file__=mod.__file__, + __spec__=dict( + name=mod.__spec__.name, + origin=mod.__spec__.origin, + ), + ), + ns=None, + ns_id=id(mod.__dict__), + cached=None, + cached_id=id(cached) if cached else None, + ) + ''').strip() + CLEANUP_SCRIPT = textwrap.dedent(''' + # Clean up the module. + sys.modules.pop(name, None) + ''').strip() + + @classmethod + def build_script(cls, name, *, + prescript=None, + import_first=False, + postscript=None, + postcleanup=False, + ): + if postcleanup is True: + postcleanup = cls.CLEANUP_SCRIPT + elif isinstance(postcleanup, str): + postcleanup = textwrap.dedent(postcleanup).strip() + postcleanup = cls.CLEANUP_SCRIPT + os.linesep + postcleanup + else: + postcleanup = '' + prescript = textwrap.dedent(prescript).strip() if prescript else '' + postscript = textwrap.dedent(postscript).strip() if postscript else '' + + if postcleanup: + if postscript: + postscript = postscript + os.linesep * 2 + postcleanup + else: + postscript = postcleanup + + if import_first: + prescript += textwrap.dedent(f''' + + # Now import the module. + assert name not in sys.modules + import {name}''') + + return cls.SCRIPT.format( + imports=cls.IMPORTS.strip(), + name=name, + prescript=prescript.strip(), + body=cls.SCRIPT_BODY.strip(), + postscript=postscript, + ) + + @classmethod + def parse(cls, text): + raw = json.loads(text) + mod = raw['module'] + mod['__spec__'] = types.SimpleNamespace(**mod['__spec__']) + raw['module'] = types.SimpleNamespace(**mod) + return cls(**raw) + + @classmethod + def from_subinterp(cls, name, interpid=None, *, pipe=None, **script_kwds): + if pipe is not None: + return cls._from_subinterp(name, interpid, pipe, script_kwds) + pipe = os.pipe() + try: + return cls._from_subinterp(name, interpid, pipe, script_kwds) + finally: + r, w = pipe + os.close(r) + os.close(w) + + @classmethod + def _from_subinterp(cls, name, interpid, pipe, script_kwargs): + r, w = pipe + + # Build the script. + postscript = textwrap.dedent(f''' + # Send the result over the pipe. + import json + import os + os.write({w}, json.dumps(snapshot).encode()) + + ''') + _postscript = script_kwargs.get('postscript') + if _postscript: + _postscript = textwrap.dedent(_postscript).lstrip() + postscript += _postscript + script_kwargs['postscript'] = postscript.strip() + script = cls.build_script(name, **script_kwargs) + + # Run the script. + if interpid is None: + ret = support.run_in_subinterp(script) + if ret != 0: + raise AssertionError(f'{ret} != 0') + else: + _interpreters.run_string(interpid, script) + + # Parse the results. + text = os.read(r, 1000) + return cls.parse(text.decode()) class LockTests(unittest.TestCase): @@ -277,478 +428,6 @@ def test_issue16421_multiple_modules_in_one_dll(self): with self.assertRaises(ImportError): imp.load_dynamic('nonexistent', pathname) - @requires_load_dynamic - def test_singlephase_clears_globals(self): - name = '_testsinglephase' - fileobj, filename, _ = imp.find_module(name) - fileobj.close() - - # Start and end fresh. - _forget_extension(name, filename) - self.addCleanup(_forget_extension, name, filename) - - _testsinglephase = imp.load_dynamic(name, filename) - init_before = _testsinglephase.state_initialized() - - _testsinglephase._clear_globals() - init_after = _testsinglephase.state_initialized() - init_count = _testsinglephase.initialized_count() - - self.assertGreater(init_before, 0) - self.assertEqual(init_after, 0) - self.assertEqual(init_count, -1) - - @requires_subinterpreters - @requires_load_dynamic - def test_singlephase_multiple_interpreters(self): - # Currently, for every single-phrase init module loaded - # in multiple interpreters, those interpreters share a - # PyModuleDef for that object, which can be a problem. - - def parse_snapshot(data): - if isinstance(data, str): - data = json.loads(data.decode()) - elif isinstance(data, bytes): - data = json.loads(data) - spam = data.pop('spam', None) - has_spam = data.pop('has_spam', False) - snapshot = type(sys.implementation)(**data) - if has_spam: - snapshot.spam = spam - return snapshot - - def check_common(snapshot): - # The "looked up" module is interpreter-specific - # (interp->imports.modules_by_index was set for the module). - self.assertEqual(snapshot.lookedup, snapshot.objid) - with self.assertRaises(AttributeError): - snapshot.spam - - def check_fresh(snapshot): - """ - The module had not been loaded before (at least since fully reset). - """ - # The module's init func was run. - # A copy of the module's __dict__ was stored in def->m_base.m_copy. - # The previous m_copy was deleted first. - # _PyRuntime.imports.extensions was set. - self.assertEqual(snapshot.init_count, 1) - # The global state was initialized. - # The module attrs were initialized from that state. - self.assertEqual(snapshot.module_initialized, - snapshot.state_initialized) - - def check_semi_fresh(snapshot, base, prev): - """ - The module had been loaded before and then reset - (but the module global state wasn't). - """ - # The module's init func was run again. - # A copy of the module's __dict__ was stored in def->m_base.m_copy. - # The previous m_copy was deleted first. - # The module globals did not get reset. - self.assertNotEqual(snapshot.objid, base.objid) - self.assertNotEqual(snapshot.objid, prev.objid) - self.assertEqual(snapshot.init_count, prev.init_count + 1) - # The global state was updated. - # The module attrs were initialized from that state. - self.assertEqual(snapshot.module_initialized, - snapshot.state_initialized) - self.assertNotEqual(snapshot.state_initialized, - base.state_initialized) - self.assertNotEqual(snapshot.state_initialized, - prev.state_initialized) - - def check_copied(snapshot, base): - """ - The module had been loaded before and never reset. - """ - # The module's init func was not run again. - # The interpreter copied m_copy, as set by the other interpreter, - # with objects owned by the other interpreter. - # The module globals did not get reset. - self.assertNotEqual(snapshot.objid, base.objid) - self.assertEqual(snapshot.init_count, base.init_count) - # The global state was not updated since the init func did not run. - # The module attrs were not directly initialized from that state. - # The state and module attrs still match the previous loading. - self.assertEqual(snapshot.module_initialized, - snapshot.state_initialized) - self.assertEqual(snapshot.state_initialized, - base.state_initialized) - - # Find the module's file. - name = '_testsinglephase' - fileobj, filename, _ = imp.find_module(name) - fileobj.close() - - # Start and end fresh. - _forget_extension(name, filename) - self.addCleanup(_forget_extension, name, filename) - - # This single-phase module has global state, which is shared - # by all interpreters. - _testsinglephase = imp.load_dynamic(name, filename) - - # Check the main interpreter. - main_snap = parse_snapshot(dict( - objid=id(_testsinglephase), - init_count=_testsinglephase.initialized_count(), - lookedup=id(_testsinglephase.look_up_self()), - state_initialized=_testsinglephase.state_initialized(), - module_initialized=_testsinglephase._module_initialized, - )) - check_common(main_snap) - check_fresh(main_snap) - - # Set up the interpreters. - setup_script = textwrap.dedent(''' - import sys - import _testinternalcapi - ''') - interp1 = _interpreters.create(isolated=False) - self.addCleanup(_interpreters.destroy, interp1) - interp2 = _interpreters.create(isolated=False) - self.addCleanup(_interpreters.destroy, interp2) - for interpid in [interp1, interp2]: - _interpreters.run_string(interpid, setup_script) - - cleanup_script = textwrap.dedent(f''' - sys.modules.pop({name!r}, None) - _testinternalcapi.clear_extension({name!r}, {filename!r}) - ''') - def clear_subinterp(interpid): - _interpreters.run_string(interpid, cleanup_script) - _interpreters.run_string(interpid, textwrap.dedent(f''' - _testsinglephase._clear_globals() - ''')) - - r, w = os.pipe() - self.addCleanup(os.close, r) - self.addCleanup(os.close, w) - - snapshot_script = textwrap.dedent(f''' - import json - import os - - import _testsinglephase - - data = dict( - objid=id(_testsinglephase), - init_count=_testsinglephase.initialized_count(), - lookedup=id(_testsinglephase.look_up_self()), - state_initialized=_testsinglephase.state_initialized(), - module_initialized=_testsinglephase._module_initialized, - has_spam=hasattr(_testsinglephase, 'spam'), - spam=getattr(_testsinglephase, 'spam', None), - ) - os.write({w}, json.dumps(data).encode()) - ''') - - def read_data(): - text = os.read(r, 500) - return parse_snapshot(text) - - with self.subTest('without resetting; ' - 'already loaded in main interpreter'): - # Attrs set after loading are not in m_copy. - _testsinglephase.spam = 'spam, spam, spam, spam, eggs, and spam' - - # Use an interpreter that gets destroyed right away. - ret = support.run_in_subinterp(snapshot_script) - self.assertEqual(ret, 0) - snap = read_data() - check_common(snap) - check_copied(snap, main_snap) - - # Use several interpreters that overlap. - _interpreters.run_string(interp1, snapshot_script) - snap = read_data() - check_common(snap) - check_copied(snap, main_snap) - - _interpreters.run_string(interp2, snapshot_script) - snap = read_data() - check_common(snap) - check_copied(snap, main_snap) - - for interpid in [interp1, interp2]: - clear_subinterp(interpid) - _testsinglephase._clear_globals() - _forget_extension(name, filename) - - with self.subTest('without resetting; ' - 'already loaded in deleted interpreter'): - # Use an interpreter that gets destroyed right away. - ret = support.run_in_subinterp(os.linesep.join([ - snapshot_script, - textwrap.dedent(''' - # Attrs set after loading are not in m_copy. - _testsinglephase.spam = 'spam, spam, mash, spam, eggs, and spam' - ''')])) - self.assertEqual(ret, 0) - base = read_data() - check_common(base) - check_fresh(base) - - # Use several interpreters that overlap. - _interpreters.run_string(interp1, snapshot_script) - interp1_snap = read_data() - check_common(interp1_snap) - check_semi_fresh(interp1_snap, main_snap, base) - - _interpreters.run_string(interp2, snapshot_script) - snap = read_data() - check_common(snap) - check_copied(snap, interp1_snap) - - with self.subTest('resetting between each interpreter'): - _testsinglephase._clear_globals() - - # Use an interpreter that gets destroyed right away. - script = os.linesep.join([ - setup_script, - cleanup_script, - snapshot_script, - textwrap.dedent(''' - # Attrs set after loading are not in m_copy. - _testsinglephase.spam = 'spam, spam, mash, spam, eggs, and spam' - ''')]) - ret = support.run_in_subinterp(script) - self.assertEqual(ret, 0) - base = read_data() - check_common(base) - check_fresh(base) - - # Use several interpreters that overlap. - clear_subinterp(interp1) - #_interpreters.run_string(interpid, cleanup_script) - _interpreters.run_string(interp1, snapshot_script) - snap = read_data() - check_common(snap) - check_fresh(snap) - - clear_subinterp(interp2) - #_interpreters.run_string(interpid, cleanup_script) - _interpreters.run_string(interp2, snapshot_script) - snap = read_data() - check_common(snap) - check_fresh(snap) - - @requires_load_dynamic - def test_singlephase_variants(self): - # Exercise the most meaningful variants described in Python/import.c. - self.maxDiff = None - - basename = '_testsinglephase' - fileobj, filename, _ = imp.find_module(basename) - fileobj.close() - - # Start and end fresh. - _forget_extension(basename, filename) - self.addCleanup(_forget_extension, basename, filename) - - def add_ext_cleanup(name): - def clean_up(): - _testinternalcapi.clear_extension(name, filename) - self.addCleanup(clean_up) - - modules = {} - def load(name): - assert name not in modules - module = imp.load_dynamic(name, filename) - self.assertNotIn(module, modules.values()) - modules[name] = module - return module - - def re_load(name, module): - assert sys.modules[name] is module - before = type(module)(module.__name__) - before.__dict__.update(vars(module)) - - reloaded = imp.load_dynamic(name, filename) - - return before, reloaded - - def check_common(name, module): - summed = module.sum(1, 2) - lookedup = module.look_up_self() - initialized = module.state_initialized() - cached = sys.modules[name] - - # module.__name__ might not match, but the spec will. - self.assertEqual(module.__spec__.name, name) - if initialized is not None: - self.assertIsInstance(initialized, float) - self.assertGreater(initialized, 0) - self.assertEqual(summed, 3) - self.assertTrue(issubclass(module.error, Exception)) - self.assertEqual(module.int_const, 1969) - self.assertEqual(module.str_const, 'something different') - self.assertIs(cached, module) - - return lookedup, initialized, cached - - def check_direct(name, module, lookedup): - # The module has its own PyModuleDef, with a matching name. - self.assertEqual(module.__name__, name) - self.assertIs(lookedup, module) - - def check_indirect(name, module, lookedup, orig): - # The module re-uses another's PyModuleDef, with a different name. - assert orig is not module - assert orig.__name__ != name - self.assertNotEqual(module.__name__, name) - self.assertIs(lookedup, module) - - def check_basic(module, initialized): - init_count = module.initialized_count() - - self.assertIsNot(initialized, None) - self.assertIsInstance(init_count, int) - self.assertGreater(init_count, 0) - - return init_count - - def check_common_reloaded(name, module, cached, before, reloaded): - recached = sys.modules[name] - - self.assertEqual(reloaded.__spec__.name, name) - self.assertEqual(reloaded.__name__, before.__name__) - self.assertEqual(before.__dict__, module.__dict__) - self.assertIs(recached, reloaded) - - def check_basic_reloaded(module, lookedup, initialized, init_count, - before, reloaded): - relookedup = reloaded.look_up_self() - reinitialized = reloaded.state_initialized() - reinit_count = reloaded.initialized_count() - - self.assertIs(reloaded, module) - self.assertIs(reloaded.__dict__, module.__dict__) - # It only happens to be the same but that's good enough here. - # We really just want to verify that the re-loaded attrs - # didn't change. - self.assertIs(relookedup, lookedup) - self.assertEqual(reinitialized, initialized) - self.assertEqual(reinit_count, init_count) - - def check_with_reinit_reloaded(module, lookedup, initialized, - before, reloaded): - relookedup = reloaded.look_up_self() - reinitialized = reloaded.state_initialized() - - self.assertIsNot(reloaded, module) - self.assertIsNot(reloaded, module) - self.assertNotEqual(reloaded.__dict__, module.__dict__) - self.assertIs(relookedup, reloaded) - if initialized is None: - self.assertIs(reinitialized, None) - else: - self.assertGreater(reinitialized, initialized) - - # Check the "basic" module. - - name = basename - add_ext_cleanup(name) - expected_init_count = 1 - with self.subTest(name): - mod = load(name) - lookedup, initialized, cached = check_common(name, mod) - check_direct(name, mod, lookedup) - init_count = check_basic(mod, initialized) - self.assertEqual(init_count, expected_init_count) - - before, reloaded = re_load(name, mod) - check_common_reloaded(name, mod, cached, before, reloaded) - check_basic_reloaded(mod, lookedup, initialized, init_count, - before, reloaded) - basic = mod - return - - # Check its indirect variants. - - name = f'{basename}_basic_wrapper' - add_ext_cleanup(name) - expected_init_count += 1 - with self.subTest(name): - mod = load(name) - lookedup, initialized, cached = check_common(name, mod) - check_indirect(name, mod, lookedup, basic) - init_count = check_basic(mod, initialized) - self.assertEqual(init_count, expected_init_count) - - before, reloaded = re_load(name, mod) - check_common_reloaded(name, mod, cached, before, reloaded) - check_basic_reloaded(mod, lookedup, initialized, init_count, - before, reloaded) - - # Currently PyState_AddModule() always replaces the cached module. - self.assertIs(basic.look_up_self(), mod) - self.assertEqual(basic.initialized_count(), expected_init_count) - - # The cached module shouldn't be changed after this point. - basic_lookedup = mod - - # Check its direct variant. - - name = f'{basename}_basic_copy' - add_ext_cleanup(name) - expected_init_count += 1 - with self.subTest(name): - mod = load(name) - lookedup, initialized, cached = check_common(name, mod) - check_direct(name, mod, lookedup) - init_count = check_basic(mod, initialized) - self.assertEqual(init_count, expected_init_count) - - before, reloaded = re_load(name, mod) - check_common_reloaded(name, mod, cached, before, reloaded) - check_basic_reloaded(mod, lookedup, initialized, init_count, - before, reloaded) - - # This should change the cached module for _testsinglephase. - self.assertIs(basic.look_up_self(), basic_lookedup) - self.assertEqual(basic.initialized_count(), expected_init_count) - - # Check the non-basic variant that has no state. - - name = f'{basename}_with_reinit' - add_ext_cleanup(name) - with self.subTest(name): - mod = load(name) - lookedup, initialized, cached = check_common(name, mod) - self.assertIs(initialized, None) - check_direct(name, mod, lookedup) - - before, reloaded = re_load(name, mod) - check_common_reloaded(name, mod, cached, before, reloaded) - check_with_reinit_reloaded(mod, lookedup, initialized, - before, reloaded) - - # This should change the cached module for _testsinglephase. - self.assertIs(basic.look_up_self(), basic_lookedup) - self.assertEqual(basic.initialized_count(), expected_init_count) - - # Check the basic variant that has state. - - name = f'{basename}_with_state' - add_ext_cleanup(name) - with self.subTest(name): - mod = load(name) - lookedup, initialized, cached = check_common(name, mod) - self.assertIsNot(initialized, None) - check_direct(name, mod, lookedup) - - before, reloaded = re_load(name, mod) - check_common_reloaded(name, mod, cached, before, reloaded) - check_with_reinit_reloaded(mod, lookedup, initialized, - before, reloaded) - - # This should change the cached module for _testsinglephase. - self.assertIs(basic.look_up_self(), basic_lookedup) - self.assertEqual(basic.initialized_count(), expected_init_count) - @requires_load_dynamic def test_load_dynamic_ImportError_path(self): # Issue #1559549 added `name` and `path` attributes to ImportError @@ -941,6 +620,666 @@ def check_get_builtins(): check_get_builtins() +class TestSinglePhaseSnapshot(ModuleSnapshot): + + @classmethod + def from_module(cls, mod): + self = super().from_module(mod) + self.summed = mod.sum(1, 2) + self.lookedup = mod.look_up_self() + self.lookedup_id = id(self.lookedup) + self.state_initialized = mod.state_initialized() + if hasattr(mod, 'initialized_count'): + self.init_count = mod.initialized_count() + return self + + SCRIPT_BODY = ModuleSnapshot.SCRIPT_BODY + textwrap.dedent(f''' + snapshot['module'].update(dict( + int_const=mod.int_const, + str_const=mod.str_const, + _module_initialized=mod._module_initialized, + )) + snapshot.update(dict( + summed=mod.sum(1, 2), + lookedup_id=id(mod.look_up_self()), + state_initialized=mod.state_initialized(), + init_count=mod.initialized_count(), + has_spam=hasattr(mod, 'spam'), + spam=getattr(mod, 'spam', None), + )) + ''').rstrip() + + @classmethod + def parse(cls, text): + self = super().parse(text) + if not self.has_spam: + del self.spam + del self.has_spam + return self + + +class SinglephaseInitTests(unittest.TestCase): + + NAME = '_testsinglephase' + + @classmethod + def setUpClass(cls): + fileobj, filename, _ = imp.find_module(cls.NAME) + fileobj.close() + cls.FILE = filename + + def setUp(self): + # Start and end fresh. + def clean_up(): + name = self.NAME + if name in sys.modules: + if hasattr(sys.modules[name], '_clear_globals'): + assert sys.modules[name].__file__ == self.FILE + sys.modules[name]._clear_globals() + del sys.modules[name] + # Clear all internally cached data for the extension. + _testinternalcapi.clear_extension(name, self.FILE) + clean_up() + self.addCleanup(clean_up) + + ######################### + # helpers + + def add_module_cleanup(self, name): + def clean_up(): + # Clear all internally cached data for the extension. + _testinternalcapi.clear_extension(name, self.FILE) + self.addCleanup(clean_up) + + def load(self, name): + try: + already_loaded = self.already_loaded + except AttributeError: + already_loaded = self.already_loaded = {} + assert name not in already_loaded + mod = imp.load_dynamic(name, self.FILE) + self.assertNotIn(mod, already_loaded.values()) + already_loaded[name] = mod + return types.SimpleNamespace( + name=name, + module=mod, + snapshot=TestSinglePhaseSnapshot.from_module(mod), + ) + + def re_load(self, name, mod): + assert sys.modules[name] is mod + assert mod.__dict__ == mod.__dict__ + reloaded = imp.load_dynamic(name, self.FILE) + return types.SimpleNamespace( + name=name, + module=reloaded, + snapshot=TestSinglePhaseSnapshot.from_module(reloaded), + ) + + # subinterpreters + + def add_subinterpreter(self): + interpid = _interpreters.create(isolated=False) + _interpreters.run_string(interpid, textwrap.dedent(''' + import sys + import _testinternalcapi + ''')) + def clean_up(): + _interpreters.run_string(interpid, textwrap.dedent(f''' + name = {self.NAME!r} + if name in sys.modules: + sys.modules[name]._clear_globals() + _testinternalcapi.clear_extension(name, {self.FILE!r}) + ''')) + _interpreters.destroy(interpid) + self.addCleanup(clean_up) + return interpid + + def import_in_subinterp(self, interpid=None, *, + postscript=None, + postcleanup=False, + ): + name = self.NAME + + if postcleanup: + import_ = 'import _testinternalcapi' if interpid is None else '' + postcleanup = f''' + {import_} + mod._clear_globals() + _testinternalcapi.clear_extension(name, {self.FILE!r}) + ''' + + try: + pipe = self._pipe + except AttributeError: + r, w = pipe = self._pipe = os.pipe() + self.addCleanup(os.close, r) + self.addCleanup(os.close, w) + + snapshot = TestSinglePhaseSnapshot.from_subinterp( + name, + interpid, + pipe=pipe, + import_first=True, + postscript=postscript, + postcleanup=postcleanup, + ) + + return types.SimpleNamespace( + name=name, + module=None, + snapshot=snapshot, + ) + + # checks + + def check_common(self, loaded): + isolated = False + + mod = loaded.module + if not mod: + # It came from a subinterpreter. + isolated = True + mod = loaded.snapshot.module + # mod.__name__ might not match, but the spec will. + self.assertEqual(mod.__spec__.name, loaded.name) + self.assertEqual(mod.__file__, self.FILE) + self.assertEqual(mod.__spec__.origin, self.FILE) + if not isolated: + self.assertTrue(issubclass(mod.error, Exception)) + self.assertEqual(mod.int_const, 1969) + self.assertEqual(mod.str_const, 'something different') + self.assertIsInstance(mod._module_initialized, float) + self.assertGreater(mod._module_initialized, 0) + + snap = loaded.snapshot + self.assertEqual(snap.summed, 3) + if snap.state_initialized is not None: + self.assertIsInstance(snap.state_initialized, float) + self.assertGreater(snap.state_initialized, 0) + if isolated: + # The "looked up" module is interpreter-specific + # (interp->imports.modules_by_index was set for the module). + self.assertEqual(snap.lookedup_id, snap.id) + self.assertEqual(snap.cached_id, snap.id) + with self.assertRaises(AttributeError): + snap.spam + else: + self.assertIs(snap.lookedup, mod) + self.assertIs(snap.cached, mod) + + def check_direct(self, loaded): + # The module has its own PyModuleDef, with a matching name. + self.assertEqual(loaded.module.__name__, loaded.name) + self.assertIs(loaded.snapshot.lookedup, loaded.module) + + def check_indirect(self, loaded, orig): + # The module re-uses another's PyModuleDef, with a different name. + assert orig is not loaded.module + assert orig.__name__ != loaded.name + self.assertNotEqual(loaded.module.__name__, loaded.name) + self.assertIs(loaded.snapshot.lookedup, loaded.module) + + def check_basic(self, loaded, expected_init_count): + # m_size == -1 + # The module loads fresh the first time and copies m_copy after. + snap = loaded.snapshot + self.assertIsNot(snap.state_initialized, None) + self.assertIsInstance(snap.init_count, int) + self.assertGreater(snap.init_count, 0) + self.assertEqual(snap.init_count, expected_init_count) + + def check_with_reinit(self, loaded): + # m_size >= 0 + # The module loads fresh every time. + pass + + def check_fresh(self, loaded): + """ + The module had not been loaded before (at least since fully reset). + """ + snap = loaded.snapshot + # The module's init func was run. + # A copy of the module's __dict__ was stored in def->m_base.m_copy. + # The previous m_copy was deleted first. + # _PyRuntime.imports.extensions was set. + self.assertEqual(snap.init_count, 1) + # The global state was initialized. + # The module attrs were initialized from that state. + self.assertEqual(snap.module._module_initialized, + snap.state_initialized) + + def check_semi_fresh(self, loaded, base, prev): + """ + The module had been loaded before and then reset + (but the module global state wasn't). + """ + snap = loaded.snapshot + # The module's init func was run again. + # A copy of the module's __dict__ was stored in def->m_base.m_copy. + # The previous m_copy was deleted first. + # The module globals did not get reset. + self.assertNotEqual(snap.id, base.snapshot.id) + self.assertNotEqual(snap.id, prev.snapshot.id) + self.assertEqual(snap.init_count, prev.snapshot.init_count + 1) + # The global state was updated. + # The module attrs were initialized from that state. + self.assertEqual(snap.module._module_initialized, + snap.state_initialized) + self.assertNotEqual(snap.state_initialized, + base.snapshot.state_initialized) + self.assertNotEqual(snap.state_initialized, + prev.snapshot.state_initialized) + + def check_copied(self, loaded, base): + """ + The module had been loaded before and never reset. + """ + snap = loaded.snapshot + # The module's init func was not run again. + # The interpreter copied m_copy, as set by the other interpreter, + # with objects owned by the other interpreter. + # The module globals did not get reset. + self.assertNotEqual(snap.id, base.snapshot.id) + self.assertEqual(snap.init_count, base.snapshot.init_count) + # The global state was not updated since the init func did not run. + # The module attrs were not directly initialized from that state. + # The state and module attrs still match the previous loading. + self.assertEqual(snap.module._module_initialized, + snap.state_initialized) + self.assertEqual(snap.state_initialized, + base.snapshot.state_initialized) + + ######################### + # the tests + + @requires_load_dynamic + def test_cleared_globals(self): + loaded = self.load(self.NAME) + _testsinglephase = loaded.module + init_before = _testsinglephase.state_initialized() + + _testsinglephase._clear_globals() + init_after = _testsinglephase.state_initialized() + init_count = _testsinglephase.initialized_count() + + self.assertGreater(init_before, 0) + self.assertEqual(init_after, 0) + self.assertEqual(init_count, -1) + + @requires_load_dynamic + def test_variants(self): + # Exercise the most meaningful variants described in Python/import.c. + self.maxDiff = None + + # Check the "basic" module. + + name = self.NAME + expected_init_count = 1 + with self.subTest(name): + loaded = self.load(name) + + self.check_common(loaded) + self.check_direct(loaded) + self.check_basic(loaded, expected_init_count) + basic = loaded.module + + # Check its indirect variants. + + name = f'{self.NAME}_basic_wrapper' + self.add_module_cleanup(name) + expected_init_count += 1 + with self.subTest(name): + loaded = self.load(name) + + self.check_common(loaded) + self.check_indirect(loaded, basic) + self.check_basic(loaded, expected_init_count) + + # Currently PyState_AddModule() always replaces the cached module. + self.assertIs(basic.look_up_self(), loaded.module) + self.assertEqual(basic.initialized_count(), expected_init_count) + + # The cached module shouldn't change after this point. + basic_lookedup = loaded.module + + # Check its direct variant. + + name = f'{self.NAME}_basic_copy' + self.add_module_cleanup(name) + expected_init_count += 1 + with self.subTest(name): + loaded = self.load(name) + + self.check_common(loaded) + self.check_direct(loaded) + self.check_basic(loaded, expected_init_count) + + # This should change the cached module for _testsinglephase. + self.assertIs(basic.look_up_self(), basic_lookedup) + self.assertEqual(basic.initialized_count(), expected_init_count) + + # Check the non-basic variant that has no state. + + name = f'{self.NAME}_with_reinit' + self.add_module_cleanup(name) + with self.subTest(name): + loaded = self.load(name) + + self.check_common(loaded) + self.assertIs(loaded.snapshot.state_initialized, None) + self.check_direct(loaded) + self.check_with_reinit(loaded) + + # This should change the cached module for _testsinglephase. + self.assertIs(basic.look_up_self(), basic_lookedup) + self.assertEqual(basic.initialized_count(), expected_init_count) + + # Check the basic variant that has state. + + name = f'{self.NAME}_with_state' + self.add_module_cleanup(name) + with self.subTest(name): + loaded = self.load(name) + + self.check_common(loaded) + self.assertIsNot(loaded.snapshot.state_initialized, None) + self.check_direct(loaded) + self.check_with_reinit(loaded) + + # This should change the cached module for _testsinglephase. + self.assertIs(basic.look_up_self(), basic_lookedup) + self.assertEqual(basic.initialized_count(), expected_init_count) + + @requires_load_dynamic + def test_basic_reloaded(self): + # m_copy is copied into the existing module object. + # Global state is not changed. + self.maxDiff = None + + for name in [ + self.NAME, # the "basic" module + f'{self.NAME}_basic_wrapper', # the indirect variant + f'{self.NAME}_basic_copy', # the direct variant + ]: + self.add_module_cleanup(name) + with self.subTest(name): + loaded = self.load(name) + reloaded = self.re_load(name, loaded.module) + + self.check_common(loaded) + self.check_common(reloaded) + + # Make sure the original __dict__ did not get replaced. + self.assertEqual(id(loaded.module.__dict__), + loaded.snapshot.ns_id) + self.assertEqual(loaded.snapshot.ns.__dict__, + loaded.module.__dict__) + + self.assertEqual(reloaded.module.__spec__.name, reloaded.name) + self.assertEqual(reloaded.module.__name__, + reloaded.snapshot.ns.__name__) + + self.assertIs(reloaded.module, loaded.module) + self.assertIs(reloaded.module.__dict__, loaded.module.__dict__) + # It only happens to be the same but that's good enough here. + # We really just want to verify that the re-loaded attrs + # didn't change. + self.assertIs(reloaded.snapshot.lookedup, + loaded.snapshot.lookedup) + self.assertEqual(reloaded.snapshot.state_initialized, + loaded.snapshot.state_initialized) + self.assertEqual(reloaded.snapshot.init_count, + loaded.snapshot.init_count) + + self.assertIs(reloaded.snapshot.cached, reloaded.module) + + @requires_load_dynamic + def test_with_reinit_reloaded(self): + # The module's m_init func is run again. + self.maxDiff = None + + # Keep a reference around. + basic = self.load(self.NAME) + + for name in [ + f'{self.NAME}_with_reinit', # m_size == 0 + f'{self.NAME}_with_state', # m_size > 0 + ]: + self.add_module_cleanup(name) + with self.subTest(name): + loaded = self.load(name) + reloaded = self.re_load(name, loaded.module) + + self.check_common(loaded) + self.check_common(reloaded) + + # Make sure the original __dict__ did not get replaced. + self.assertEqual(id(loaded.module.__dict__), + loaded.snapshot.ns_id) + self.assertEqual(loaded.snapshot.ns.__dict__, + loaded.module.__dict__) + + self.assertEqual(reloaded.module.__spec__.name, reloaded.name) + self.assertEqual(reloaded.module.__name__, + reloaded.snapshot.ns.__name__) + + self.assertIsNot(reloaded.module, loaded.module) + self.assertNotEqual(reloaded.module.__dict__, + loaded.module.__dict__) + self.assertIs(reloaded.snapshot.lookedup, reloaded.module) + if loaded.snapshot.state_initialized is None: + self.assertIs(reloaded.snapshot.state_initialized, None) + else: + self.assertGreater(reloaded.snapshot.state_initialized, + loaded.snapshot.state_initialized) + + self.assertIs(reloaded.snapshot.cached, reloaded.module) + + # Currently, for every single-phrase init module loaded + # in multiple interpreters, those interpreters share a + # PyModuleDef for that object, which can be a problem. + # Also, we test with a single-phase module that has global state, + # which is shared by all interpreters. + + @requires_subinterpreters + @requires_load_dynamic + def test_basic_multiple_interpreters_main_no_reset(self): + # without resetting; already loaded in main interpreter + + # At this point: + # * alive in 0 interpreters + # * module def may or may not be loaded already + # * module def not in _PyRuntime.imports.extensions + # * mod init func has not run yet (since reset, at least) + # * m_copy not set (hasn't been loaded yet or already cleared) + # * module's global state has not been initialized yet + # (or already cleared) + + main_loaded = self.load(self.NAME) + _testsinglephase = main_loaded.module + # Attrs set after loading are not in m_copy. + _testsinglephase.spam = 'spam, spam, spam, spam, eggs, and spam' + + self.check_common(main_loaded) + self.check_fresh(main_loaded) + + interpid1 = self.add_subinterpreter() + interpid2 = self.add_subinterpreter() + + # At this point: + # * alive in 1 interpreter (main) + # * module def in _PyRuntime.imports.extensions + # * mod init func ran for the first time (since reset, at least) + # * m_copy was copied from the main interpreter (was NULL) + # * module's global state was initialized + + # Use an interpreter that gets destroyed right away. + loaded = self.import_in_subinterp() + self.check_common(loaded) + self.check_copied(loaded, main_loaded) + + # At this point: + # * alive in 1 interpreter (main) + # * module def still in _PyRuntime.imports.extensions + # * mod init func ran again + # * m_copy is NULL (claered when the interpreter was destroyed) + # (was from main interpreter) + # * module's global state was updated, not reset + + # Use a subinterpreter that sticks around. + loaded = self.import_in_subinterp(interpid1) + self.check_common(loaded) + self.check_copied(loaded, main_loaded) + + # At this point: + # * alive in 2 interpreters (main, interp1) + # * module def still in _PyRuntime.imports.extensions + # * mod init func ran again + # * m_copy was copied from interp1 + # * module's global state was updated, not reset + + # Use a subinterpreter while the previous one is still alive. + loaded = self.import_in_subinterp(interpid2) + self.check_common(loaded) + self.check_copied(loaded, main_loaded) + + # At this point: + # * alive in 3 interpreters (main, interp1, interp2) + # * module def still in _PyRuntime.imports.extensions + # * mod init func ran again + # * m_copy was copied from interp2 (was from interp1) + # * module's global state was updated, not reset + + @requires_subinterpreters + @requires_load_dynamic + def test_basic_multiple_interpreters_deleted_no_reset(self): + # without resetting; already loaded in a deleted interpreter + + # At this point: + # * alive in 0 interpreters + # * module def may or may not be loaded already + # * module def not in _PyRuntime.imports.extensions + # * mod init func has not run yet (since reset, at least) + # * m_copy not set (hasn't been loaded yet or already cleared) + # * module's global state has not been initialized yet + # (or already cleared) + + interpid1 = self.add_subinterpreter() + interpid2 = self.add_subinterpreter() + + # First, load in the main interpreter but then completely clear it. + loaded_main = self.load(self.NAME) + loaded_main.module._clear_globals() + _testinternalcapi.clear_extension(self.NAME, self.FILE) + + # At this point: + # * alive in 0 interpreters + # * module def loaded already + # * module def was in _PyRuntime.imports.extensions, but cleared + # * mod init func ran for the first time (since reset, at least) + # * m_copy was set, but cleared (was NULL) + # * module's global state was initialized but cleared + + # Start with an interpreter that gets destroyed right away. + base = self.import_in_subinterp(postscript=''' + # Attrs set after loading are not in m_copy. + mod.spam = 'spam, spam, mash, spam, eggs, and spam' + ''') + self.check_common(base) + self.check_fresh(base) + + # At this point: + # * alive in 0 interpreters + # * module def in _PyRuntime.imports.extensions + # * mod init func ran again + # * m_copy is NULL (claered when the interpreter was destroyed) + # * module's global state was initialized, not reset + + # Use a subinterpreter that sticks around. + loaded_interp1 = self.import_in_subinterp(interpid1) + self.check_common(loaded_interp1) + self.check_semi_fresh(loaded_interp1, loaded_main, base) + + # At this point: + # * alive in 1 interpreter (interp1) + # * module def still in _PyRuntime.imports.extensions + # * mod init func ran again + # * m_copy was copied from interp1 (was NULL) + # * module's global state was updated, not reset + + # Use a subinterpreter while the previous one is still alive. + loaded_interp2 = self.import_in_subinterp(interpid2) + self.check_common(loaded_interp2) + self.check_copied(loaded_interp2, loaded_interp1) + + # At this point: + # * alive in 2 interpreters (interp1, interp2) + # * module def still in _PyRuntime.imports.extensions + # * mod init func ran again + # * m_copy was copied from interp2 (was from interp1) + # * module's global state was updated, not reset + + @requires_subinterpreters + @requires_load_dynamic + def test_basic_multiple_interpreters_reset_each(self): + # resetting between each interpreter + + # At this point: + # * alive in 0 interpreters + # * module def may or may not be loaded already + # * module def not in _PyRuntime.imports.extensions + # * mod init func has not run yet (since reset, at least) + # * m_copy not set (hasn't been loaded yet or already cleared) + # * module's global state has not been initialized yet + # (or already cleared) + + interpid1 = self.add_subinterpreter() + interpid2 = self.add_subinterpreter() + + # Use an interpreter that gets destroyed right away. + loaded = self.import_in_subinterp( + postscript=''' + # Attrs set after loading are not in m_copy. + mod.spam = 'spam, spam, mash, spam, eggs, and spam' + ''', + postcleanup=True, + ) + self.check_common(loaded) + self.check_fresh(loaded) + + # At this point: + # * alive in 0 interpreters + # * module def in _PyRuntime.imports.extensions + # * mod init func ran for the first time (since reset, at least) + # * m_copy is NULL (claered when the interpreter was destroyed) + # * module's global state was initialized, not reset + + # Use a subinterpreter that sticks around. + loaded = self.import_in_subinterp(interpid1, postcleanup=True) + self.check_common(loaded) + self.check_fresh(loaded) + + # At this point: + # * alive in 1 interpreter (interp1) + # * module def still in _PyRuntime.imports.extensions + # * mod init func ran again + # * m_copy was copied from interp1 (was NULL) + # * module's global state was initialized, not reset + + # Use a subinterpreter while the previous one is still alive. + loaded = self.import_in_subinterp(interpid2, postcleanup=True) + self.check_common(loaded) + self.check_fresh(loaded) + + # At this point: + # * alive in 2 interpreters (interp2, interp2) + # * module def still in _PyRuntime.imports.extensions + # * mod init func ran again + # * m_copy was copied from interp2 (was from interp1) + # * module's global state was initialized, not reset + + class ReloadTests(unittest.TestCase): """Very basic tests to make sure that imp.reload() operates just like From fc940603fc0b6cc6b0ccf05b4740f971b2df7f01 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Fri, 24 Feb 2023 12:59:52 -0700 Subject: [PATCH 15/17] Do the initial cleanup in setUpClass(). --- Lib/test/test_imp.py | 31 ++++++++++++++++++------------- 1 file changed, 18 insertions(+), 13 deletions(-) diff --git a/Lib/test/test_imp.py b/Lib/test/test_imp.py index ecc34b7b68d051..1a3bf56e2aff89 100644 --- a/Lib/test/test_imp.py +++ b/Lib/test/test_imp.py @@ -668,19 +668,24 @@ def setUpClass(cls): fileobj.close() cls.FILE = filename - def setUp(self): - # Start and end fresh. - def clean_up(): - name = self.NAME - if name in sys.modules: - if hasattr(sys.modules[name], '_clear_globals'): - assert sys.modules[name].__file__ == self.FILE - sys.modules[name]._clear_globals() - del sys.modules[name] - # Clear all internally cached data for the extension. - _testinternalcapi.clear_extension(name, self.FILE) - clean_up() - self.addCleanup(clean_up) + # Start fresh. + cls.clean_up() + + def tearDown(self): + # Clean up the module. + self.clean_up() + + @classmethod + def clean_up(cls): + name = cls.NAME + filename = cls.FILE + if name in sys.modules: + if hasattr(sys.modules[name], '_clear_globals'): + assert sys.modules[name].__file__ == filename + sys.modules[name]._clear_globals() + del sys.modules[name] + # Clear all internally cached data for the extension. + _testinternalcapi.clear_extension(name, filename) ######################### # helpers From 3e2a1ddb1f5dc9022c4e1b8dc9f5976d56c7d431 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Sat, 25 Feb 2023 11:40:37 -0700 Subject: [PATCH 16/17] Move the requires_load_dynamic decorator to the class. --- Lib/test/test_imp.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/Lib/test/test_imp.py b/Lib/test/test_imp.py index 1a3bf56e2aff89..7c83748dbda3c3 100644 --- a/Lib/test/test_imp.py +++ b/Lib/test/test_imp.py @@ -658,6 +658,7 @@ def parse(cls, text): return self +@requires_load_dynamic class SinglephaseInitTests(unittest.TestCase): NAME = '_testsinglephase' @@ -898,7 +899,6 @@ def check_copied(self, loaded, base): ######################### # the tests - @requires_load_dynamic def test_cleared_globals(self): loaded = self.load(self.NAME) _testsinglephase = loaded.module @@ -912,7 +912,6 @@ def test_cleared_globals(self): self.assertEqual(init_after, 0) self.assertEqual(init_count, -1) - @requires_load_dynamic def test_variants(self): # Exercise the most meaningful variants described in Python/import.c. self.maxDiff = None @@ -996,7 +995,6 @@ def test_variants(self): self.assertIs(basic.look_up_self(), basic_lookedup) self.assertEqual(basic.initialized_count(), expected_init_count) - @requires_load_dynamic def test_basic_reloaded(self): # m_copy is copied into the existing module object. # Global state is not changed. @@ -1039,7 +1037,6 @@ def test_basic_reloaded(self): self.assertIs(reloaded.snapshot.cached, reloaded.module) - @requires_load_dynamic def test_with_reinit_reloaded(self): # The module's m_init func is run again. self.maxDiff = None @@ -1088,7 +1085,6 @@ def test_with_reinit_reloaded(self): # which is shared by all interpreters. @requires_subinterpreters - @requires_load_dynamic def test_basic_multiple_interpreters_main_no_reset(self): # without resetting; already loaded in main interpreter @@ -1157,7 +1153,6 @@ def test_basic_multiple_interpreters_main_no_reset(self): # * module's global state was updated, not reset @requires_subinterpreters - @requires_load_dynamic def test_basic_multiple_interpreters_deleted_no_reset(self): # without resetting; already loaded in a deleted interpreter From f0ce6077b4441de1b54cc951f86eec64f45ebda1 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Sat, 25 Feb 2023 11:42:46 -0700 Subject: [PATCH 17/17] For now, skip the tests during refleak detection. --- Lib/test/test_imp.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Lib/test/test_imp.py b/Lib/test/test_imp.py index 7c83748dbda3c3..03e3adba221e57 100644 --- a/Lib/test/test_imp.py +++ b/Lib/test/test_imp.py @@ -665,6 +665,9 @@ class SinglephaseInitTests(unittest.TestCase): @classmethod def setUpClass(cls): + if '-R' in sys.argv or '--huntrleaks' in sys.argv: + # https://github.com/python/cpython/issues/102251 + raise unittest.SkipTest('unresolved refleaks (see gh-102251)') fileobj, filename, _ = imp.find_module(cls.NAME) fileobj.close() cls.FILE = filename