Skip to content

Commit

Permalink
pythongh-117378: Fix multiprocessing forkserver preload sys.path inhe…
Browse files Browse the repository at this point in the history
…ritance.

`sys.path` was not properly being sent from the parent process when launching
the multiprocessing forkserver process to preload imports.  This bug has been
there since the forkserver start method was introduced in Python ~3.4.  It was
always _supposed_ to inherit `sys.path` the same way the spawn method does.

Observable behavior change: A `''` value in `sys.path` will now be replaced in
the forkserver's `sys.path` with an absolute pathname
`os.path.abspath(os.getcwd())` saved at the time that `multiprocessing` was
imported in the parent process as it already was when using the spawn start
method.

A workaround for the bug this fixes was to set PYTHONPATH in the environment
before the forkserver process was started.
  • Loading branch information
gpshead committed Nov 7, 2024
1 parent dbb6e22 commit c40b26d
Show file tree
Hide file tree
Showing 2 changed files with 59 additions and 0 deletions.
2 changes: 2 additions & 0 deletions Lib/multiprocessing/forkserver.py
Original file line number Diff line number Diff line change
Expand Up @@ -174,6 +174,8 @@ def main(listener_fd, alive_r, preload, main_path=None, sys_path=None):
spawn.import_main_path(main_path)
finally:
del process.current_process()._inheriting
if sys_path is not None:
sys.path[:] = sys_path
for modname in preload:
try:
__import__(modname)
Expand Down
57 changes: 57 additions & 0 deletions Lib/test/test_multiprocessing_forkserver/test_forkserver_main.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,57 @@
import os
import sys
import unittest
from unittest import mock

from multiprocessing import forkserver


class TestForkserverMain(unittest.TestCase):

def setUp(self):
self._orig_sys_path = list(sys.path)

def tearDown(self):
sys.path[:] = self._orig_sys_path

@mock.patch("multiprocessing.process.current_process")
@mock.patch("multiprocessing.spawn.import_main_path")
@mock.patch("multiprocessing.util._close_stdin")
def test_preload_kwargs(
self,
mock_close_stdin,
mock_import_main_path,
mock_current_process,
):
# Very much a whitebox test of the first stanza of main before
# we start diddling with file descriptors and pipes.
mock_close_stdin.side_effect = RuntimeError("stop test")
self.assertNotIn(
"colorsys",
sys.modules.keys(),
msg="Thie test requires a module that has not yet been imported.",
)

with self.assertRaisesRegex(RuntimeError, "stop test"):
forkserver.main(None, None, ["sys", "colorsys"])
mock_current_process.assert_not_called()
mock_import_main_path.assert_not_called()
self.assertIn("colorsys", sys.modules.keys())
self.assertEqual(sys.path, self._orig_sys_path) # unmodified

del sys.modules["colorsys"] # unimport
fake_path = os.path.dirname(__file__)
with self.assertRaisesRegex(RuntimeError, "stop test"):
forkserver.main(None, None, ["sys", "colorsys"], sys_path=[fake_path])
self.assertEqual(
sys.path, [fake_path], msg="sys.path should have been overridden"
)
self.assertNotIn(
"colorsys",
sys.modules.keys(),
msg="import of colorsys should have failed with unusual sys.path",
)


if __name__ == "__main__":
unittest.main()

0 comments on commit c40b26d

Please sign in to comment.