gh-104212: Add importlib.util.load_source_path() function

Author: vstinner

Doc/library/importlib.rst

@@ -1240,6 +1240,20 @@ an :term:`importer`.
    .. versionchanged:: 3.6
       Accepts a :term:`path-like object`.
 
+.. function:: load_source_path(module_name, filename)
+
+   Load a module from a filename: execute the module and cache it to
+   :data:`sys.modules`.
+
+   *module_name* must not contain dots. A package cannot be imported by its
+   directory path, whereas its ``__init__.py`` file (ex:
+   ``package/__init__.py``) can be imported.
+
+   The module is always executed even if it's already cached in
+   :data:`sys.modules`.
+
+   .. versionadded:: 3.12
+
 .. function:: source_hash(source_bytes)
 
    Return the hash of *source_bytes* as bytes. A hash-based ``.pyc`` file embeds

Doc/whatsnew/3.12.rst

@@ -570,6 +570,12 @@ fractions
 * Objects of type :class:`fractions.Fraction` now support float-style
   formatting. (Contributed by Mark Dickinson in :gh:`100161`.)
 
+importlib
+---------
+
+* Add :func:`importlib.util.load_source_path` to load a module from a filename.
+  (Contributed by Victor Stinner in :gh:`104212`.)
+
 inspect
 -------
 
@@ -1371,6 +1377,9 @@ Removed
 
   * Replace ``imp.new_module(name)`` with ``types.ModuleType(name)``.
 
+  * Replace ``imp.load_source(module_name, filename)``
+    with ``importlib.util.load_source_path(module_name, filename)``.
+
 * Removed the ``suspicious`` rule from the documentation Makefile, and
   removed ``Doc/tools/rstlint.py``, both in favor of `sphinx-lint
   <https://github.com/sphinx-contrib/sphinx-lint>`_.

Lib/importlib/util.py

@@ -10,6 +10,7 @@
 from ._bootstrap_external import decode_source
 from ._bootstrap_external import source_from_cache
 from ._bootstrap_external import spec_from_file_location
+from ._bootstrap_external import SourceFileLoader
 
 import _imp
 import sys
@@ -246,3 +247,21 @@ def exec_module(self, module):
         loader_state['__class__'] = module.__class__
         module.__spec__.loader_state = loader_state
         module.__class__ = _LazyModule
+
+
+def load_source_path(module_name, filename):
+    """Load a module from a filename."""
+    if '.' in module_name:
+        raise ValueError(f"module name must not contain dots: {module_name!r}")
+
+    loader = SourceFileLoader(module_name, filename)
+    # use spec_from_file_location() to always set the __file__ attribute,
+    # even if the filename does not end with ".py"
+    spec = spec_from_file_location(module_name, filename,
+                                   loader=loader,
+                                   submodule_search_locations=[])
+
+    module = module_from_spec(spec)
+    sys.modules[module.__name__] = module
+    loader.exec_module(module)
+    return module

Lib/test/test_importlib/test_util.py

@@ -1,3 +1,4 @@
+from test.test_importlib import fixtures
 from test.test_importlib import util
 
 abc = util.import_importlib('importlib.abc')
@@ -12,6 +13,7 @@
 import string
 import sys
 from test import support
+from test.support import import_helper, os_helper
 import textwrap
 import types
 import unittest
@@ -758,5 +760,101 @@ def test_complete_multi_phase_init_module(self):
             self.run_with_own_gil(script)
 
 
+class LoadSourcePathTests(unittest.TestCase):
+    def check_module(self, mod, modname, filename, is_package=False):
+        abs_filename = os.path.abspath(filename)
+
+        self.assertIsInstance(mod, types.ModuleType)
+        self.assertEqual(mod.__name__, modname)
+        self.assertEqual(mod.__file__, abs_filename)
+        self.assertIn(modname, sys.modules)
+        self.assertIs(sys.modules[modname], mod)
+        self.assertEqual(mod.__path__, [os.path.dirname(abs_filename)])
+
+        loader = mod.__loader__
+        self.assertEqual(loader.is_package(modname), is_package)
+
+        spec = mod.__spec__
+        self.assertEqual(spec.name, modname)
+        self.assertEqual(spec.origin, abs_filename)
+
+    def test_filename(self):
+        modname = 'test_load_source_path_mod'
+        # Filename doesn't have to end with ".py" suffix
+        filename = 'load_source_path_filename'
+        side_effect = 'load_source_path_side_effect'
+
+        def delete_side_effect():
+            try:
+                delattr(sys, side_effect)
+            except AttributeError:
+                pass
+
+        self.assertNotIn(modname, sys.modules)
+        self.addCleanup(import_helper.unload, modname)
+
+        self.assertFalse(hasattr(sys, side_effect))
+        self.addCleanup(delete_side_effect)
+
+        # Use a temporary directory to remove __pycache__/ subdirectory
+        with fixtures.tempdir_as_cwd():
+            with open(filename, "w", encoding="utf8") as fp:
+                print("attr = 'load_source_path_attr'", file=fp)
+                print(f"import sys; sys.{side_effect} = 1", file=fp)
+
+            mod = importlib.util.load_source_path(modname, filename)
+
+            self.check_module(mod, modname, filename)
+            self.assertEqual(mod.attr, 'load_source_path_attr')
+            self.assertEqual(getattr(sys, side_effect), 1)
+
+            # reload cached in sys.modules: the module is executed again
+            self.assertIn(modname, sys.modules)
+            setattr(sys, side_effect, 0)
+            mod = importlib.util.load_source_path(modname, filename)
+            self.assertEqual(getattr(sys, side_effect), 1)
+
+            # reload uncached in sys.modules: the module is executed again
+            del sys.modules[modname]
+            setattr(sys, side_effect, 0)
+            mod = importlib.util.load_source_path(modname, filename)
+            self.assertEqual(getattr(sys, side_effect), 1)
+
+    def test_dots(self):
+        modname = 'package.submodule'
+        filename = __file__
+        with self.assertRaises(ValueError) as cm:
+            importlib.util.load_source_path(modname, filename)
+
+        err_msg = str(cm.exception)
+        self.assertIn("module name must not contain dots", err_msg)
+        self.assertIn(repr(modname), err_msg)
+
+    def test_package(self):
+        modname = 'test_load_source_path_package'
+        dirname = 'load_source_path_dir'
+        filename = os.path.join('load_source_path_dir', '__init__.py')
+
+        self.assertNotIn(modname, sys.modules)
+        self.addCleanup(import_helper.unload, modname)
+
+        # Use a temporary directory to remove __pycache__/ subdirectory
+        with fixtures.tempdir_as_cwd():
+            os.mkdir(dirname)
+            with open(filename, "w", encoding="utf8") as fp:
+                print("attr = 'load_source_path_pkg'", file=fp)
+
+            # Package cannot be imported from a directory. It can with
+            # IsADirectoryError on Unix and PermissionError on Windows.
+            with self.assertRaises(OSError):
+                importlib.util.load_source_path(modname, dirname)
+
+            # whereas loading a package __init__.py file is ok
+            mod = importlib.util.load_source_path(modname, filename)
+
+            self.check_module(mod, modname, filename, is_package=True)
+            self.assertEqual(mod.attr, 'load_source_path_pkg')
+
+
 if __name__ == '__main__':
     unittest.main()

Misc/NEWS.d/next/Library/2023-06-14-00-54-11.gh-issue-104212.4IJaKp.rst

@@ -0,0 +1,2 @@
+Add :func:`importlib.util.load_source_path` to load a module from a filename.
+Patch by Victor Stinner.