From c40b26d5e23537aca1d266592df04fa15e338902 Mon Sep 17 00:00:00 2001 From: "Gregory P. Smith" Date: Thu, 7 Nov 2024 00:56:09 -0800 Subject: [PATCH] gh-117378: Fix multiprocessing forkserver preload sys.path inheritance. `sys.path` was not properly being sent from the parent process when launching the multiprocessing forkserver process to preload imports. This bug has been there since the forkserver start method was introduced in Python ~3.4. It was always _supposed_ to inherit `sys.path` the same way the spawn method does. Observable behavior change: A `''` value in `sys.path` will now be replaced in the forkserver's `sys.path` with an absolute pathname `os.path.abspath(os.getcwd())` saved at the time that `multiprocessing` was imported in the parent process as it already was when using the spawn start method. A workaround for the bug this fixes was to set PYTHONPATH in the environment before the forkserver process was started. --- Lib/multiprocessing/forkserver.py | 2 + .../test_forkserver_main.py | 57 +++++++++++++++++++ 2 files changed, 59 insertions(+) create mode 100644 Lib/test/test_multiprocessing_forkserver/test_forkserver_main.py diff --git a/Lib/multiprocessing/forkserver.py b/Lib/multiprocessing/forkserver.py index 53b8c492675878..90cc04a8346379 100644 --- a/Lib/multiprocessing/forkserver.py +++ b/Lib/multiprocessing/forkserver.py @@ -174,6 +174,8 @@ def main(listener_fd, alive_r, preload, main_path=None, sys_path=None): spawn.import_main_path(main_path) finally: del process.current_process()._inheriting + if sys_path is not None: + sys.path[:] = sys_path for modname in preload: try: __import__(modname) diff --git a/Lib/test/test_multiprocessing_forkserver/test_forkserver_main.py b/Lib/test/test_multiprocessing_forkserver/test_forkserver_main.py new file mode 100644 index 00000000000000..1f4ee3ab47214c --- /dev/null +++ b/Lib/test/test_multiprocessing_forkserver/test_forkserver_main.py @@ -0,0 +1,57 @@ +import os +import sys +import unittest +from unittest import mock + +from multiprocessing import forkserver + + +class TestForkserverMain(unittest.TestCase): + + def setUp(self): + self._orig_sys_path = list(sys.path) + + def tearDown(self): + sys.path[:] = self._orig_sys_path + + @mock.patch("multiprocessing.process.current_process") + @mock.patch("multiprocessing.spawn.import_main_path") + @mock.patch("multiprocessing.util._close_stdin") + def test_preload_kwargs( + self, + mock_close_stdin, + mock_import_main_path, + mock_current_process, + ): + # Very much a whitebox test of the first stanza of main before + # we start diddling with file descriptors and pipes. + mock_close_stdin.side_effect = RuntimeError("stop test") + self.assertNotIn( + "colorsys", + sys.modules.keys(), + msg="Thie test requires a module that has not yet been imported.", + ) + + with self.assertRaisesRegex(RuntimeError, "stop test"): + forkserver.main(None, None, ["sys", "colorsys"]) + mock_current_process.assert_not_called() + mock_import_main_path.assert_not_called() + self.assertIn("colorsys", sys.modules.keys()) + self.assertEqual(sys.path, self._orig_sys_path) # unmodified + + del sys.modules["colorsys"] # unimport + fake_path = os.path.dirname(__file__) + with self.assertRaisesRegex(RuntimeError, "stop test"): + forkserver.main(None, None, ["sys", "colorsys"], sys_path=[fake_path]) + self.assertEqual( + sys.path, [fake_path], msg="sys.path should have been overridden" + ) + self.assertNotIn( + "colorsys", + sys.modules.keys(), + msg="import of colorsys should have failed with unusual sys.path", + ) + + +if __name__ == "__main__": + unittest.main()