From e8d2d2b48e491c387bf8ac519fbc0d4fdbc746b8 Mon Sep 17 00:00:00 2001 From: tommoral Date: Wed, 4 Jan 2023 19:21:18 +0100 Subject: [PATCH 01/50] FIX support for frozen executable on all platforms --- loky/__init__.py | 31 ++++++++++++--- loky/backend/fork_exec.py | 2 +- loky/backend/popen_loky_posix.py | 53 ++++--------------------- loky/backend/popen_loky_win32.py | 57 +++------------------------ loky/backend/resource_tracker.py | 30 ++++++++------ loky/backend/spawn.py | 67 ++++++++++++++++++++++++++++++++ 6 files changed, 124 insertions(+), 116 deletions(-) diff --git a/loky/__init__.py b/loky/__init__.py index e9fc121f..33a6891b 100644 --- a/loky/__init__.py +++ b/loky/__init__.py @@ -16,17 +16,38 @@ from ._base import Future from .backend.context import cpu_count +from .backend.spawn import freeze_support from .backend.reduction import set_loky_pickler from .reusable_executor import get_reusable_executor from .cloudpickle_wrapper import wrap_non_picklable_objects from .process_executor import BrokenProcessPool, ProcessPoolExecutor -__all__ = ["get_reusable_executor", "cpu_count", "wait", "as_completed", - "Future", "Executor", "ProcessPoolExecutor", - "BrokenProcessPool", "CancelledError", "TimeoutError", - "FIRST_COMPLETED", "FIRST_EXCEPTION", "ALL_COMPLETED", - "wrap_non_picklable_objects", "set_loky_pickler"] +__all__ = [ + # Constants + "ALL_COMPLETED", + "FIRST_COMPLETED", + "FIRST_EXCEPTION", + + # Classes + "Executor", + "Future", + "ProcessPoolExecutor", + + # Functions + "as_completed", + "cpu_count", + "freeze_support", + "get_reusable_executor", + "set_loky_pickler", + "wait", + "wrap_non_picklable_objects", + + # Errors + "BrokenProcessPool", + "CancelledError", + "TimeoutError", +] __version__ = '3.4.0.dev0' diff --git a/loky/backend/fork_exec.py b/loky/backend/fork_exec.py index 211d1835..9cad383b 100644 --- a/loky/backend/fork_exec.py +++ b/loky/backend/fork_exec.py @@ -37,6 +37,6 @@ def fork_exec(cmd, keep_fds, env=None): pid = os.fork() if pid == 0: # pragma: no cover close_fds(keep_fds) - os.execve(sys.executable, cmd, child_env) + os.execve(cmd[0], cmd, child_env) else: return pid diff --git a/loky/backend/popen_loky_posix.py b/loky/backend/popen_loky_posix.py index 37a73172..1be993b5 100644 --- a/loky/backend/popen_loky_posix.py +++ b/loky/backend/popen_loky_posix.py @@ -6,9 +6,8 @@ import os import sys import signal -import pickle from io import BytesIO -from multiprocessing import util, process +from multiprocessing import util from multiprocessing.connection import wait from multiprocessing.context import set_spawning_popen @@ -96,7 +95,8 @@ def _launch(self, process_obj): try: prep_data = spawn.get_preparation_data( process_obj._name, - getattr(process_obj, "init_main_module", True)) + getattr(process_obj, "init_main_module", True) + ) reduction.dump(prep_data, fp) reduction.dump(process_obj, fp) @@ -106,15 +106,13 @@ def _launch(self, process_obj): try: parent_r, child_w = os.pipe() child_r, parent_w = os.pipe() - # for fd in self._fds: - # _mk_inheritable(fd) - cmd_python = [sys.executable] - cmd_python += ['-m', self.__module__] - cmd_python += ['--process-name', str(process_obj.name)] - cmd_python += ['--pipe', str(reduction._mk_inheritable(child_r))] + reduction._mk_inheritable(child_r) reduction._mk_inheritable(child_w) reduction._mk_inheritable(tracker_fd) + cmd_python = spawn.get_command_line( + fd=child_r, process_name=process_obj.name + ) self._fds += [child_r, child_w, tracker_fd] if sys.version_info >= (3, 8) and os.name == 'posix': mp_tracker_fd = prep_data['mp_tracker_args']['fd'] @@ -143,40 +141,3 @@ def _launch(self, process_obj): @staticmethod def thread_is_spawning(): return True - - -if __name__ == '__main__': - import argparse - parser = argparse.ArgumentParser('Command line parser') - parser.add_argument('--pipe', type=int, required=True, - help='File handle for the pipe') - parser.add_argument('--process-name', type=str, default=None, - help='Identifier for debugging purpose') - - args = parser.parse_args() - - info = {} - exitcode = 1 - try: - with os.fdopen(args.pipe, 'rb') as from_parent: - process.current_process()._inheriting = True - try: - prep_data = pickle.load(from_parent) - spawn.prepare(prep_data) - process_obj = pickle.load(from_parent) - finally: - del process.current_process()._inheriting - - exitcode = process_obj._bootstrap() - except Exception: - print('\n\n' + '-' * 80) - print(f'{args.process_name} failed with traceback: ') - print('-' * 80) - import traceback - print(traceback.format_exc()) - print('\n' + '-' * 80) - finally: - if from_parent is not None: - from_parent.close() - - sys.exit(exitcode) diff --git a/loky/backend/popen_loky_win32.py b/loky/backend/popen_loky_win32.py index c07fb46d..034ab151 100644 --- a/loky/backend/popen_loky_win32.py +++ b/loky/backend/popen_loky_win32.py @@ -2,8 +2,7 @@ import sys import msvcrt import _winapi -from pickle import load -from multiprocessing import process, util +from multiprocessing import util from multiprocessing.context import get_spawning_popen, set_spawning_popen from multiprocessing.popen_spawn_win32 import Popen as _Popen from multiprocessing.reduction import duplicate @@ -47,11 +46,10 @@ def __init__(self, process_obj): rhandle = duplicate(msvcrt.get_osfhandle(rfd), inheritable=True) os.close(rfd) - cmd = get_command_line(parent_pid=os.getpid(), pipe_handle=rhandle) + cmd = spawn.get_command_line(fd=rhandle) + python_exe = cmd[0] cmd = ' '.join(f'"{x}"' for x in cmd) - python_exe = spawn.get_executable() - # copy the environment variables to set in the child process child_env = {**os.environ, **process_obj.env} @@ -76,7 +74,8 @@ def __init__(self, process_obj): hp, ht, pid, _ = _winapi.CreateProcess( python_exe, cmd, None, None, inherit, 0, - child_env, None, None) + child_env, None, None + ) _winapi.CloseHandle(ht) except BaseException: _winapi.CloseHandle(rhandle) @@ -108,49 +107,3 @@ def __init__(self, process_obj): def duplicate_for_child(self, handle): assert self is get_spawning_popen() return duplicate(handle, self.sentinel) - - -def get_command_line(pipe_handle, **kwds): - ''' - Returns prefix of command line used for spawning a child process - ''' - if getattr(sys, 'frozen', False): - return [sys.executable, '--multiprocessing-fork', pipe_handle] - else: - prog = 'from loky.backend.popen_loky_win32 import main; main()' - opts = util._args_from_interpreter_flags() - return [spawn.get_executable(), *opts, - '-c', prog, '--multiprocessing-fork', pipe_handle] - - -def is_forking(argv): - ''' - Return whether commandline indicates we are forking - ''' - if len(argv) >= 2 and argv[1] == '--multiprocessing-fork': - assert len(argv) == 3 - return True - else: - return False - - -def main(): - ''' - Run code specified by data received over pipe - ''' - assert is_forking(sys.argv) - - handle = int(sys.argv[-1]) - fd = msvcrt.open_osfhandle(handle, os.O_RDONLY) - from_parent = os.fdopen(fd, 'rb') - - process.current_process()._inheriting = True - preparation_data = load(from_parent) - spawn.prepare(preparation_data) - self = load(from_parent) - process.current_process()._inheriting = False - - from_parent.close() - - exitcode = self._bootstrap() - sys.exit(exitcode) diff --git a/loky/backend/resource_tracker.py b/loky/backend/resource_tracker.py index d84504e1..c091a677 100644 --- a/loky/backend/resource_tracker.py +++ b/loky/backend/resource_tracker.py @@ -49,8 +49,8 @@ import signal import warnings import threading -from _multiprocessing import sem_unlink from multiprocessing import util +from _multiprocessing import sem_unlink from . import spawn @@ -130,13 +130,13 @@ def ensure_running(self): os.close(r) r = _r - cmd = f'from {main.__module__} import main; main({r}, {VERBOSE})' try: fds_to_pass.append(r) # process will out live us, so no need to wait on pid - exe = spawn.get_executable() - args = [exe, *util._args_from_interpreter_flags(), '-c', cmd] - util.debug(f"launching resource tracker: {args}") + cmd = spawn.get_command_line( + main_prog=main, fd=r, verbose=int(VERBOSE) + ) + util.debug(f"launching resource tracker: {cmd}") # bpo-33613: Register a signal mask that will block the # signals. This signal mask will be inherited by the child # that is going to be spawned and will protect the child from a @@ -147,7 +147,7 @@ def ensure_running(self): if _HAVE_SIGMASK: signal.pthread_sigmask(signal.SIG_BLOCK, _IGNORED_SIGNALS) - pid = spawnv_passfds(exe, args, fds_to_pass) + pid = spawnv_passfds(cmd, fds_to_pass) finally: if _HAVE_SIGMASK: signal.pthread_sigmask(signal.SIG_UNBLOCK, @@ -208,6 +208,11 @@ def _send(self, cmd, name, rtype): def main(fd, verbose=0): '''Run resource tracker.''' + # Mak sure the arguments have the right type as theyr are + # passed as strings through the command line. + fd = int(fd) + verbose = int(verbose) + # protect the process from ^C and "killall python" etc if verbose: util.log_to_stderr(level=util.DEBUG) @@ -271,8 +276,8 @@ def main(fd, verbose=0): del registry[rtype][name] if verbose: util.debug( - f"[ResourceTracker] unregister {name} {rtype}: " - f"registry({len(registry)})" + f"[ResourceTracker] unregister {name} {rtype}:" + f" registry({len(registry)})" ) elif cmd == 'MAYBE_UNLINK': registry[rtype][name] -= 1 @@ -348,7 +353,7 @@ def _unlink_resources(rtype_registry, rtype): # Start a program with only specified fds kept open # -def spawnv_passfds(path, args, passfds): +def spawnv_passfds(cmd, passfds): passfds = sorted(passfds) if sys.platform != "win32": errpipe_read, errpipe_write = os.pipe() @@ -356,15 +361,16 @@ def spawnv_passfds(path, args, passfds): from .reduction import _mk_inheritable from .fork_exec import fork_exec _pass = [_mk_inheritable(fd) for fd in passfds] - return fork_exec(args, _pass) + return fork_exec(cmd, _pass) finally: os.close(errpipe_read) os.close(errpipe_write) else: - cmd = ' '.join(f'"{x}"' for x in args) + exe = cmd[0] + cmd = ' '.join(f'"{x}"' for x in cmd) try: _, ht, pid, _ = _winapi.CreateProcess( - path, cmd, None, None, True, 0, None, None, None) + exe, cmd, None, None, True, 0, None, None, None) _winapi.CloseHandle(ht) except BaseException: pass diff --git a/loky/backend/spawn.py b/loky/backend/spawn.py index 3a9cc2dd..8e9b3d5c 100644 --- a/loky/backend/spawn.py +++ b/loky/backend/spawn.py @@ -10,6 +10,8 @@ import sys import runpy import types +import pickle +import importlib from multiprocessing import process, util @@ -240,3 +242,68 @@ def _fixup_main_from_path(main_path): run_name="__mp_main__") main_module.__dict__.update(main_content) sys.modules['__main__'] = sys.modules['__mp_main__'] = main_module + + +def main(fd, process_name): + fd = int(fd) + if sys.platform == "win32": + fd = msvcrt.open_osfhandle(fd, os.O_RDONLY) + + exitcode = 1 + try: + with os.fdopen(fd, 'rb') as from_parent: + process.current_process()._inheriting = True + try: + prep_data = pickle.load(from_parent) + prepare(prep_data) + process_obj = pickle.load(from_parent) + finally: + del process.current_process()._inheriting + + exitcode = process_obj._bootstrap() + except Exception: + print('\n\n' + '-' * 80) + print(f'{process_name} failed with traceback: ') + print('-' * 80) + import traceback + print(traceback.format_exc()) + print('\n' + '-' * 80) + finally: + if from_parent is not None: + from_parent.close() + + sys.exit(exitcode) + + +def get_command_line(main_prog=main, **kwargs): + ''' + Returns prefix of command line used for spawning a child process + ''' + + if getattr(sys, 'frozen', False): + list_kwargs = [f'{k}={v}' for k, v in kwargs.items()] + argv = [ + sys.executable, '--multiprocessing-fork', main_prog.__module__, + *list_kwargs + ] + else: + list_kwargs = [f'{k}="{v}"' for k, v in kwargs.items()] + prog = ( + f'from {main_prog.__module__} import main; ' + f'main({", ".join(list_kwargs)})' + ) + opts = util._args_from_interpreter_flags() + argv = [get_executable(), *opts, '-c', prog] + return argv + + +def freeze_support(): + if len(sys.argv) >= 2 and sys.argv[1] == "--multiprocessing-fork": + module_main = sys.argv[2] + main = importlib.import_module(module_main).main + kwargs = {} + for p in sys.argv[3:]: + k, v = p.split("=") + kwargs[k] = v + exitcode = main(**kwargs) + sys.exit(exitcode) From 3441a87c00a0669b531da34e0a6264f3713d646e Mon Sep 17 00:00:00 2001 From: tommoral Date: Wed, 4 Jan 2023 19:52:33 +0100 Subject: [PATCH 02/50] CI fix tox.ini --- tox.ini | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 983fd5f1..297fd294 100644 --- a/tox.ini +++ b/tox.ini @@ -4,7 +4,9 @@ envlist = py37, py38, py39, py310, pypy3 skip_missing_interpreters=True [testenv] -passenv = NUMBER_OF_PROCESSORS LOKY_MAX_CPU_COUNT +passenv = + NUMBER_OF_PROCESSORS + LOKY_MAX_CPU_COUNT usedevelop = True # Do not install psutil on Python 3.8 to have some CI runs that # tests that loky has no hard dependency on psutil. From 95ae81c62ec42accefa1fec0063694b9d0602a31 Mon Sep 17 00:00:00 2001 From: tommoral Date: Wed, 4 Jan 2023 19:55:33 +0100 Subject: [PATCH 03/50] FIX spawn.main for window --- loky/backend/spawn.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/loky/backend/spawn.py b/loky/backend/spawn.py index 8e9b3d5c..95d6028b 100644 --- a/loky/backend/spawn.py +++ b/loky/backend/spawn.py @@ -244,7 +244,7 @@ def _fixup_main_from_path(main_path): sys.modules['__main__'] = sys.modules['__mp_main__'] = main_module -def main(fd, process_name): +def main(fd, process_name=None): fd = int(fd) if sys.platform == "win32": fd = msvcrt.open_osfhandle(fd, os.O_RDONLY) From bb05e4abdd200aafbebdeb11ada93327671826d4 Mon Sep 17 00:00:00 2001 From: Olivier Grisel Date: Tue, 21 Feb 2023 12:19:19 +0100 Subject: [PATCH 04/50] Blind attempt to fix broken test on Windows --- tests/test_loky_backend.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/tests/test_loky_backend.py b/tests/test_loky_backend.py index 7f5b626a..15f4a229 100644 --- a/tests/test_loky_backend.py +++ b/tests/test_loky_backend.py @@ -647,10 +647,8 @@ def test_interactively_define_process_fail_main(self): stdout, stderr = check_subprocess_call( [sys.executable, filename], timeout=10 ) - if sys.platform == "win32": - assert "RuntimeError:" in stderr - else: - assert "RuntimeError:" in stdout + all_outputs = f"stdout:\n{stdout}\nstderr:\n{stderr}" + assert "RuntimeError:" in all_outputs, all_outputs finally: os.unlink(filename) From eb5a665fc1a40939ec74cba1d5e2cdc8564284b1 Mon Sep 17 00:00:00 2001 From: tommoral Date: Tue, 21 Feb 2023 13:45:35 +0100 Subject: [PATCH 05/50] FIX freeze support for multiprocessing resource_tracker --- loky/backend/spawn.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/loky/backend/spawn.py b/loky/backend/spawn.py index a0e7245e..37da28d4 100644 --- a/loky/backend/spawn.py +++ b/loky/backend/spawn.py @@ -13,7 +13,9 @@ import types import pickle import importlib + from multiprocessing import process, util +from multiprocessing import freeze_support as _freeze_support_mp if sys.platform != "win32": @@ -320,3 +322,4 @@ def freeze_support(): kwargs[k] = v exitcode = main(**kwargs) sys.exit(exitcode) + _freeze_support_mp() From 9ffac110307462bc580a2acd8b63d6d4fcb51b4e Mon Sep 17 00:00:00 2001 From: Olivier Grisel Date: Tue, 21 Feb 2023 14:33:53 +0100 Subject: [PATCH 06/50] Add new integration test for freeze_support with pyinstaller --- tests/test_loky_module.py | 36 +++++++++++++++++++++++++++++++++++- tox.ini | 1 + 2 files changed, 36 insertions(+), 1 deletion(-) diff --git a/tests/test_loky_module.py b/tests/test_loky_module.py index 474f49dc..77604c4d 100644 --- a/tests/test_loky_module.py +++ b/tests/test_loky_module.py @@ -3,8 +3,9 @@ import sys import shutil import tempfile +import textwrap import warnings -from subprocess import check_output +from subprocess import check_call, check_output import pytest @@ -210,3 +211,36 @@ def test_only_physical_cores_with_user_limitation(): if cpu_count_user < cpu_count_mp: assert cpu_count() == cpu_count_user assert cpu_count(only_physical_cores=True) == cpu_count_user + + +def test_freeze_support_with_pyinstaller(tmpdir): + pyinstaller = shutil.which("pyinstaller") + + if pyinstaller is None: + raise pytest.skip("pyinstaller is not installed") + + frozen_source_code = textwrap.dedent( + """ + import loky + + if __name__ == "__main__": + loky.freeze_support() + e = loky.get_reusable_executor(max_workers=2) + print(sum(e.map(lambda x: x ** 2, range(10)))) + + """ + ) + python_source_path = tmpdir / "frozen_loky.py" + python_source_path.write_text(frozen_source_code, encoding="utf-8") + + original_cwd = os.getcwd() + try: + os.chdir(tmpdir) + check_call([pyinstaller, python_source_path]) + result = check_output( + [tmpdir / "dist" / "frozen_loky" / "frozen_loky"], text=True + ).strip() + finally: + os.chdir(original_cwd) + + assert result == "285" diff --git a/tox.ini b/tox.ini index 650b7dcf..3dd76c8d 100644 --- a/tox.ini +++ b/tox.ini @@ -18,6 +18,7 @@ deps = viztracer ; python_version >= '3.8' and python_version <= '3.10' numpy ; implementation_name == 'cpython' tblib + pyinstaller whitelist_externals= bash setenv = From 6bad7f1ada6d0f93e72184ce2bcf5c5421c0ec1f Mon Sep 17 00:00:00 2001 From: Thomas Moreau Date: Tue, 21 Feb 2023 18:27:01 +0100 Subject: [PATCH 07/50] CLN comment to clarify the code on freeze_support --- loky/backend/spawn.py | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/loky/backend/spawn.py b/loky/backend/spawn.py index 37da28d4..79ac03d2 100644 --- a/loky/backend/spawn.py +++ b/loky/backend/spawn.py @@ -257,6 +257,7 @@ def _fixup_main_from_path(main_path): def main(fd, process_name=None): + # arguments are passed as strings, convert them back to int. fd = int(fd) if sys.platform == "win32": fd = msvcrt.open_osfhandle(fd, os.O_RDONLY) @@ -290,10 +291,18 @@ def main(fd, process_name=None): def get_command_line(main_prog=main, **kwargs): """ - Returns prefix of command line used for spawning a child process + Returns a command line used for spawning a child process. + + This command provides supports for frozen executables and + only works with main_prog named main. """ + assert main_prog.__name__ == 'main' + if getattr(sys, "frozen", False): + # For frozen executables, add flag '--multiprocessin-fork' to notify, + # the `freeze_support` function and pass the arguments as 'key=value' + # so they can be used to call main. list_kwargs = [f"{k}={v}" for k, v in kwargs.items()] argv = [ sys.executable, @@ -302,6 +311,8 @@ def get_command_line(main_prog=main, **kwargs): *list_kwargs, ] else: + # For non-frozen executables, directly call `main_prog` with + # the arguments passed as strings. list_kwargs = [f'{k}="{v}"' for k, v in kwargs.items()] prog = ( f"from {main_prog.__module__} import main; " @@ -313,6 +324,13 @@ def get_command_line(main_prog=main, **kwargs): def freeze_support(): + """Run code for the child workers when necessary. + + This helper allows the frozen executable to call the code for the child + workers when not in the main process. + It should be called right after the beginning of the programme, to + avoid recursive process spawning. + """ if len(sys.argv) >= 2 and sys.argv[1] == "--multiprocessing-fork": module_main = sys.argv[2] main = importlib.import_module(module_main).main From 6cf235b57648fd87dbb1218c18777d3d59abee40 Mon Sep 17 00:00:00 2001 From: Thomas Moreau Date: Tue, 21 Feb 2023 18:31:18 +0100 Subject: [PATCH 08/50] Update loky/backend/resource_tracker.py --- loky/backend/resource_tracker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/loky/backend/resource_tracker.py b/loky/backend/resource_tracker.py index 4a9d391a..5814fe85 100644 --- a/loky/backend/resource_tracker.py +++ b/loky/backend/resource_tracker.py @@ -208,7 +208,7 @@ def _send(self, cmd, name, rtype): def main(fd, verbose=0): """Run resource tracker.""" - # Mak sure the arguments have the right type as theyr are + # Make sure the arguments have the right type as they are # passed as strings through the command line. fd = int(fd) verbose = int(verbose) From a6e88a1c7c26853f2016c3e3771a855afd7e63e0 Mon Sep 17 00:00:00 2001 From: Thomas Moreau Date: Tue, 21 Feb 2023 18:38:38 +0100 Subject: [PATCH 09/50] FIX linter black... --- loky/backend/spawn.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/loky/backend/spawn.py b/loky/backend/spawn.py index 79ac03d2..669c800e 100644 --- a/loky/backend/spawn.py +++ b/loky/backend/spawn.py @@ -292,12 +292,11 @@ def main(fd, process_name=None): def get_command_line(main_prog=main, **kwargs): """ Returns a command line used for spawning a child process. - This command provides supports for frozen executables and only works with main_prog named main. """ - assert main_prog.__name__ == 'main' + assert main_prog.__name__ == "main" if getattr(sys, "frozen", False): # For frozen executables, add flag '--multiprocessin-fork' to notify, @@ -325,7 +324,6 @@ def get_command_line(main_prog=main, **kwargs): def freeze_support(): """Run code for the child workers when necessary. - This helper allows the frozen executable to call the code for the child workers when not in the main process. It should be called right after the beginning of the programme, to From 32870a1db8069c90d60da9b7f17ee142c6cb51e7 Mon Sep 17 00:00:00 2001 From: Olivier Grisel Date: Tue, 21 Feb 2023 19:03:03 +0100 Subject: [PATCH 10/50] Install loky in tox --- tox.ini | 1 - 1 file changed, 1 deletion(-) diff --git a/tox.ini b/tox.ini index 3dd76c8d..d9790508 100644 --- a/tox.ini +++ b/tox.ini @@ -7,7 +7,6 @@ skip_missing_interpreters=True passenv = NUMBER_OF_PROCESSORS LOKY_MAX_CPU_COUNT -usedevelop = True # Do not install psutil on Python 3.8 to have some CI runs that # tests that loky has no hard dependency on psutil. deps = From 06f36b4e8c9da33ad721702eaab9412d1ea4966d Mon Sep 17 00:00:00 2001 From: Olivier Grisel Date: Tue, 21 Feb 2023 19:03:20 +0100 Subject: [PATCH 11/50] Do not use chdir in pyinstaller test --- tests/test_loky_module.py | 32 ++++++++++++++++++++++---------- 1 file changed, 22 insertions(+), 10 deletions(-) diff --git a/tests/test_loky_module.py b/tests/test_loky_module.py index 77604c4d..de02a431 100644 --- a/tests/test_loky_module.py +++ b/tests/test_loky_module.py @@ -233,14 +233,26 @@ def test_freeze_support_with_pyinstaller(tmpdir): python_source_path = tmpdir / "frozen_loky.py" python_source_path.write_text(frozen_source_code, encoding="utf-8") - original_cwd = os.getcwd() - try: - os.chdir(tmpdir) - check_call([pyinstaller, python_source_path]) - result = check_output( - [tmpdir / "dist" / "frozen_loky" / "frozen_loky"], text=True - ).strip() - finally: - os.chdir(original_cwd) + # Run the Python script directly: + non_frozen_result = check_output( + [sys.executable, python_source_path], + text=True, + ) + + # Call pyinstaller to generate the frozen_loky executable. + check_call( + [ + pyinstaller, + "--onefile", + "--distpath", + tmpdir, + "--specpath", + tmpdir, + python_source_path, + ] + ) + frozen_loky = tmpdir / "frozen_loky" + assert frozen_loky.exists() - assert result == "285" + frozen_result = check_output(frozen_loky, text=True) + assert frozen_result == non_frozen_result From 67397b7d980dcc1c0c2eb55123c1b21ce11b3b76 Mon Sep 17 00:00:00 2001 From: Olivier Grisel Date: Tue, 21 Feb 2023 19:08:37 +0100 Subject: [PATCH 12/50] Cosmetics --- tests/test_loky_module.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_loky_module.py b/tests/test_loky_module.py index de02a431..91944a80 100644 --- a/tests/test_loky_module.py +++ b/tests/test_loky_module.py @@ -227,7 +227,6 @@ def test_freeze_support_with_pyinstaller(tmpdir): loky.freeze_support() e = loky.get_reusable_executor(max_workers=2) print(sum(e.map(lambda x: x ** 2, range(10)))) - """ ) python_source_path = tmpdir / "frozen_loky.py" From f8ec3448efde431f618ab643fc508990c92a1321 Mon Sep 17 00:00:00 2001 From: Olivier Grisel Date: Tue, 21 Feb 2023 19:31:00 +0100 Subject: [PATCH 13/50] Missing enclosing [] in when calling check_output with a single Path argument --- tests/test_loky_module.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_loky_module.py b/tests/test_loky_module.py index 91944a80..bad95ac6 100644 --- a/tests/test_loky_module.py +++ b/tests/test_loky_module.py @@ -253,5 +253,5 @@ def test_freeze_support_with_pyinstaller(tmpdir): frozen_loky = tmpdir / "frozen_loky" assert frozen_loky.exists() - frozen_result = check_output(frozen_loky, text=True) + frozen_result = check_output([frozen_loky], text=True) assert frozen_result == non_frozen_result From 8d424bda46ea80aacc43a8ea8e2614d0c13d810b Mon Sep 17 00:00:00 2001 From: Olivier Grisel Date: Wed, 22 Feb 2023 11:02:38 +0100 Subject: [PATCH 14/50] Do not attempt to run pyinstaller with non-CPython implementations --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index d9790508..a7ba8d11 100644 --- a/tox.ini +++ b/tox.ini @@ -17,7 +17,7 @@ deps = viztracer ; python_version >= '3.8' and python_version <= '3.10' numpy ; implementation_name == 'cpython' tblib - pyinstaller + pyinstaller ; implementation_name == 'cpython' whitelist_externals= bash setenv = From 4a676e08757eba6d8e9c3d8b6ce1f3b752d3e807 Mon Sep 17 00:00:00 2001 From: Olivier Grisel Date: Wed, 22 Feb 2023 11:12:27 +0100 Subject: [PATCH 15/50] Restore previoous' tox.ini to isolate pyinstaller integration test in another env --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index a7ba8d11..650b7dcf 100644 --- a/tox.ini +++ b/tox.ini @@ -7,6 +7,7 @@ skip_missing_interpreters=True passenv = NUMBER_OF_PROCESSORS LOKY_MAX_CPU_COUNT +usedevelop = True # Do not install psutil on Python 3.8 to have some CI runs that # tests that loky has no hard dependency on psutil. deps = @@ -17,7 +18,6 @@ deps = viztracer ; python_version >= '3.8' and python_version <= '3.10' numpy ; implementation_name == 'cpython' tblib - pyinstaller ; implementation_name == 'cpython' whitelist_externals= bash setenv = From 221610e3348b3ae725b120c6a0acd2d11f4f0237 Mon Sep 17 00:00:00 2001 From: Olivier Grisel Date: Wed, 22 Feb 2023 11:26:08 +0100 Subject: [PATCH 16/50] Tentative CI config for the pyinstaller test --- .azure_pipeline.yml | 48 ++++++++++++++++++++++++++++++ continuous_integration/runtests.sh | 14 +++++++++ 2 files changed, 62 insertions(+) diff --git a/.azure_pipeline.yml b/.azure_pipeline.yml index 47178e46..6315cd84 100644 --- a/.azure_pipeline.yml +++ b/.azure_pipeline.yml @@ -109,3 +109,51 @@ jobs: curl -s https://codecov.io/bash | bash displayName: 'Upload to codecov' condition: and(succeeded(), ne(variables['joblib.tests'], 'true')) + + +- job: 'test_frozen_loky' + strategy: + matrix: + + windows-py310: + imageName: windows-latest + python.version: "3.10" + macos-py310: + imageName: "macos-latest" + python.version: "3.10" + linux-py310: + imageName: "ubuntu-latest" + python.version: "3.10" + pool: + vmImage: $(imageName) + variables: + JUNITXML: 'test-data.xml' + PYINSTALLER_TESTS: "true" + steps: + - task: UsePythonVersion@0 + inputs: + versionSpec: '$(python.version)' + displayName: 'Use Python $(python.version)' + + # azure-pipelines unpredictably switches between Git\bin\bash and + # Git\usr\bin\bash when running a bash script inside Windows environments. + # The latter may use wrong bash commands, resulting in errors when codecov + # tries to upload the coverage results. + - bash: echo "##vso[task.prependpath]C:/Program Files/Git/bin" + displayName: 'Override Git bash shell for Windows' + condition: eq(variables['Agent.OS'], 'Windows_NT') + + - script: | + bash continuous_integration/runtests.sh + displayName: 'Test loky with PyInstaller' + + - task: PublishTestResults@2 + inputs: + testResultsFiles: '$(JUNITXML)' + displayName: 'Publish Test Results' + condition: succeededOrFailed() + + - bash: | + curl -s https://codecov.io/bash | bash + displayName: 'Upload to codecov' + condition: and(succeeded(), ne(variables['joblib.tests'], 'true')) diff --git a/continuous_integration/runtests.sh b/continuous_integration/runtests.sh index 3e478a9a..9dea9ba7 100755 --- a/continuous_integration/runtests.sh +++ b/continuous_integration/runtests.sh @@ -23,6 +23,20 @@ if [ "$JOBLIB_TESTS" = "true" ]; then cp "$BUILD_SOURCESDIRECTORY"/continuous_integration/copy_loky.sh $JOBLIB/externals (cd $JOBLIB/externals && bash copy_loky.sh "$BUILD_SOURCESDIRECTORY") pytest -vl --ignore $JOBLIB/externals --pyargs joblib +if [ "$PYINSTALLER_TESTS" = "true" ]; then + python -m venv venv/ + source ./venv/bin/activate + which python + pip install pytest pytest-timeout psutil pyinstaller + pip install -e . + python -c "import loky; print('loky.cpu_count():', loky.cpu_count())" + python -c "import os; print('os.cpu_count():', os.cpu_count())" + export COVERAGE_PROCESS_START=`pwd`/.coveragerc + python continuous_integration/install_coverage_subprocess_pth.py + pytest -vl --maxfail=5 --timeout=60 -k pyinstaller --junitxml="${JUNITXML}" + coverage combine --quiet --append + coverage xml -i # language agnostic report for the codecov upload script + coverage report # display the report as text on stdout else # Make sure that we have the python docker image cached locally to avoid # a timeout in a test that needs it. From 25f8eeb7f4e63415ff0c796c9ecb5a8a121901eb Mon Sep 17 00:00:00 2001 From: Olivier Grisel Date: Wed, 22 Feb 2023 11:28:07 +0100 Subject: [PATCH 17/50] Typo in runtests.sh --- continuous_integration/runtests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/continuous_integration/runtests.sh b/continuous_integration/runtests.sh index 9dea9ba7..e90a1f3f 100755 --- a/continuous_integration/runtests.sh +++ b/continuous_integration/runtests.sh @@ -23,7 +23,7 @@ if [ "$JOBLIB_TESTS" = "true" ]; then cp "$BUILD_SOURCESDIRECTORY"/continuous_integration/copy_loky.sh $JOBLIB/externals (cd $JOBLIB/externals && bash copy_loky.sh "$BUILD_SOURCESDIRECTORY") pytest -vl --ignore $JOBLIB/externals --pyargs joblib -if [ "$PYINSTALLER_TESTS" = "true" ]; then +elif [ "$PYINSTALLER_TESTS" = "true" ]; then python -m venv venv/ source ./venv/bin/activate which python From 78c4a8af86440b6d819a2bb0a7133e4f9c8d7724 Mon Sep 17 00:00:00 2001 From: Olivier Grisel Date: Wed, 22 Feb 2023 11:31:08 +0100 Subject: [PATCH 18/50] Install loky in non-editable mode and coverage --- continuous_integration/runtests.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/continuous_integration/runtests.sh b/continuous_integration/runtests.sh index e90a1f3f..bead216a 100755 --- a/continuous_integration/runtests.sh +++ b/continuous_integration/runtests.sh @@ -27,8 +27,8 @@ elif [ "$PYINSTALLER_TESTS" = "true" ]; then python -m venv venv/ source ./venv/bin/activate which python - pip install pytest pytest-timeout psutil pyinstaller - pip install -e . + pip install pytest pytest-timeout psutil coverage pyinstaller + pip install . python -c "import loky; print('loky.cpu_count():', loky.cpu_count())" python -c "import os; print('os.cpu_count():', os.cpu_count())" export COVERAGE_PROCESS_START=`pwd`/.coveragerc From d68413b3ce2770f947e4e84b1194f0d7f9d457f0 Mon Sep 17 00:00:00 2001 From: Olivier Grisel Date: Wed, 22 Feb 2023 11:38:58 +0100 Subject: [PATCH 19/50] Windows specific folder --- continuous_integration/runtests.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/continuous_integration/runtests.sh b/continuous_integration/runtests.sh index bead216a..617392d1 100755 --- a/continuous_integration/runtests.sh +++ b/continuous_integration/runtests.sh @@ -25,7 +25,11 @@ if [ "$JOBLIB_TESTS" = "true" ]; then pytest -vl --ignore $JOBLIB/externals --pyargs joblib elif [ "$PYINSTALLER_TESTS" = "true" ]; then python -m venv venv/ - source ./venv/bin/activate + if [ -d "./venv/Scripts" ]; then + source ./venv/Scripts/activate + else + source ./venv/bin/activate + fi which python pip install pytest pytest-timeout psutil coverage pyinstaller pip install . From e0949edb7d61746f26136f16700180726c83749c Mon Sep 17 00:00:00 2001 From: Olivier Grisel Date: Wed, 22 Feb 2023 12:00:05 +0100 Subject: [PATCH 20/50] Adjust expected executable filename for windows --- tests/test_loky_module.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/test_loky_module.py b/tests/test_loky_module.py index bad95ac6..64a9de19 100644 --- a/tests/test_loky_module.py +++ b/tests/test_loky_module.py @@ -250,7 +250,10 @@ def test_freeze_support_with_pyinstaller(tmpdir): python_source_path, ] ) - frozen_loky = tmpdir / "frozen_loky" + if sys.platform == "win32": + frozen_loky = tmpdir / "frozen_loky.exe" + else: + frozen_loky = tmpdir / "frozen_loky" assert frozen_loky.exists() frozen_result = check_output([frozen_loky], text=True) From 27e0ad7769e7a2fad88eff4b396b15ce3eecdfd6 Mon Sep 17 00:00:00 2001 From: Olivier Grisel Date: Wed, 22 Feb 2023 14:02:39 +0100 Subject: [PATCH 21/50] DEBUG: run test with the standard library only --- tests/test_loky_module.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/tests/test_loky_module.py b/tests/test_loky_module.py index 64a9de19..cd0c5e5a 100644 --- a/tests/test_loky_module.py +++ b/tests/test_loky_module.py @@ -221,12 +221,16 @@ def test_freeze_support_with_pyinstaller(tmpdir): frozen_source_code = textwrap.dedent( """ - import loky + from concurrent.futures import ProcessPoolExecutor + from multiprocessing import freeze_support + # import loky if __name__ == "__main__": - loky.freeze_support() - e = loky.get_reusable_executor(max_workers=2) - print(sum(e.map(lambda x: x ** 2, range(10)))) + # loky.freeze_support() + freeze_support() + e = ProcessPoolExecutor() + # e = loky.get_reusable_executor(max_workers=2) + print(sum(e.map(int, range(10)))) """ ) python_source_path = tmpdir / "frozen_loky.py" From 0fd30f2a921022cba428b09d3f4b4479483ce512 Mon Sep 17 00:00:00 2001 From: Olivier Grisel Date: Wed, 22 Feb 2023 14:16:17 +0100 Subject: [PATCH 22/50] DEBUG try again with loky's ProcessPoolExecutor --- tests/test_loky_module.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_loky_module.py b/tests/test_loky_module.py index cd0c5e5a..315cc248 100644 --- a/tests/test_loky_module.py +++ b/tests/test_loky_module.py @@ -221,7 +221,8 @@ def test_freeze_support_with_pyinstaller(tmpdir): frozen_source_code = textwrap.dedent( """ - from concurrent.futures import ProcessPoolExecutor + # from concurrent.futures import ProcessPoolExecutor + from loky import ProcessPoolExecutor from multiprocessing import freeze_support # import loky From 58ec71f3a9a683ba7ce84020312559e00adc84ac Mon Sep 17 00:00:00 2001 From: Olivier Grisel Date: Wed, 22 Feb 2023 14:24:07 +0100 Subject: [PATCH 23/50] DEBUG: try loky.freeze_support with loky.ProcessPoolExecutor --- tests/test_loky_module.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/tests/test_loky_module.py b/tests/test_loky_module.py index 315cc248..5b2adaeb 100644 --- a/tests/test_loky_module.py +++ b/tests/test_loky_module.py @@ -221,15 +221,11 @@ def test_freeze_support_with_pyinstaller(tmpdir): frozen_source_code = textwrap.dedent( """ - # from concurrent.futures import ProcessPoolExecutor - from loky import ProcessPoolExecutor - from multiprocessing import freeze_support - # import loky + import loky if __name__ == "__main__": - # loky.freeze_support() - freeze_support() - e = ProcessPoolExecutor() + loky.freeze_support() + e = loky.ProcessPoolExecutor() # e = loky.get_reusable_executor(max_workers=2) print(sum(e.map(int, range(10)))) """ From a31af24544cd256163cef40b16104a0a23bf5227 Mon Sep 17 00:00:00 2001 From: Olivier Grisel Date: Wed, 22 Feb 2023 14:32:09 +0100 Subject: [PATCH 24/50] Back to the original test_freeze_support_with_pyinstaller --- tests/test_loky_module.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/test_loky_module.py b/tests/test_loky_module.py index 5b2adaeb..64584b7d 100644 --- a/tests/test_loky_module.py +++ b/tests/test_loky_module.py @@ -225,8 +225,7 @@ def test_freeze_support_with_pyinstaller(tmpdir): if __name__ == "__main__": loky.freeze_support() - e = loky.ProcessPoolExecutor() - # e = loky.get_reusable_executor(max_workers=2) + e = loky.get_reusable_executor(max_workers=2) print(sum(e.map(int, range(10)))) """ ) From 77933ff63eb1784029e300ae8c54b015bf25213c Mon Sep 17 00:00:00 2001 From: Olivier Grisel Date: Wed, 22 Feb 2023 14:41:53 +0100 Subject: [PATCH 25/50] Tentative fix for windows: use get_executable instead of sys.executable --- loky/backend/spawn.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/loky/backend/spawn.py b/loky/backend/spawn.py index 669c800e..cf466df0 100644 --- a/loky/backend/spawn.py +++ b/loky/backend/spawn.py @@ -304,7 +304,7 @@ def get_command_line(main_prog=main, **kwargs): # so they can be used to call main. list_kwargs = [f"{k}={v}" for k, v in kwargs.items()] argv = [ - sys.executable, + get_executable(), "--multiprocessing-fork", main_prog.__module__, *list_kwargs, From f32c8d951e6fc96d753c1abd4ae0137f167a5924 Mon Sep 17 00:00:00 2001 From: Olivier Grisel Date: Wed, 22 Feb 2023 15:26:25 +0100 Subject: [PATCH 26/50] Revert "Tentative fix for windows: use get_executable instead of sys.executable" This reverts commit 77933ff63eb1784029e300ae8c54b015bf25213c. --- loky/backend/spawn.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/loky/backend/spawn.py b/loky/backend/spawn.py index cf466df0..669c800e 100644 --- a/loky/backend/spawn.py +++ b/loky/backend/spawn.py @@ -304,7 +304,7 @@ def get_command_line(main_prog=main, **kwargs): # so they can be used to call main. list_kwargs = [f"{k}={v}" for k, v in kwargs.items()] argv = [ - get_executable(), + sys.executable, "--multiprocessing-fork", main_prog.__module__, *list_kwargs, From 08e1c1718e5bd362a109ba1bccf7bb3c735e1707 Mon Sep 17 00:00:00 2001 From: Olivier Grisel Date: Wed, 22 Feb 2023 15:42:25 +0100 Subject: [PATCH 27/50] Blind attempt to sync Popen.__init__ with cpython main --- loky/backend/popen_loky_win32.py | 34 +++++++++++++++++++++----------- 1 file changed, 22 insertions(+), 12 deletions(-) diff --git a/loky/backend/popen_loky_win32.py b/loky/backend/popen_loky_win32.py index 3401234f..1ff8208f 100644 --- a/loky/backend/popen_loky_win32.py +++ b/loky/backend/popen_loky_win32.py @@ -43,24 +43,34 @@ def __init__(self, process_obj): process_obj._name, getattr(process_obj, "init_main_module", True) ) - # read end of pipe will be "stolen" by the child process + # read end of pipe will be duplicated by the child process # -- see spawn_main() in spawn.py. - rfd, wfd = os.pipe() - rhandle = duplicate(msvcrt.get_osfhandle(rfd), inheritable=True) - os.close(rfd) + # + # bpo-33929: Previously, the read end of pipe was "stolen" by the child + # process, but it leaked a handle if the child process had been + # terminated before it could steal the handle from the parent process. + rhandle, whandle = _winapi.CreatePipe(None, 0) + wfd = msvcrt.open_osfhandle(whandle, 0) + cmd = spawn.get_command_line(parent_pid=os.getpid(), + pipe_handle=rhandle) - cmd = spawn.get_command_line(fd=rhandle) - python_exe = cmd[0] - cmd = " ".join(f'"{x}"' for x in cmd) - - # copy the environment variables to set in the child process - child_env = {**os.environ, **process_obj.env} + python_exe = spawn.get_executable() # bpo-35797: When running in a venv, we bypass the redirect # executor and launch our base Python. if WINENV and _path_eq(python_exe, sys.executable): - python_exe = sys._base_executable - child_env["__PYVENV_LAUNCHER__"] = sys.executable + cmd[0] = python_exe = sys._base_executable + env = os.environ.copy() + env["__PYVENV_LAUNCHER__"] = sys.executable + else: + env = None + + cmd = ' '.join('"%s"' % x for x in cmd) + + # XXX: remove debug logs + util.debug(f"{WINENV = }") + util.debug(f"{python_exe = }") + util.debug(f"{sys.executable = }") try: with open(wfd, "wb") as to_child: From 106251014f658c49e731b45e38b33e5b966ed122 Mon Sep 17 00:00:00 2001 From: Olivier Grisel Date: Wed, 22 Feb 2023 16:06:05 +0100 Subject: [PATCH 28/50] Fix typo in variable name in last commit code sync --- loky/backend/popen_loky_win32.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/loky/backend/popen_loky_win32.py b/loky/backend/popen_loky_win32.py index 1ff8208f..7d00c43f 100644 --- a/loky/backend/popen_loky_win32.py +++ b/loky/backend/popen_loky_win32.py @@ -91,7 +91,7 @@ def __init__(self, process_obj): None, inherit, 0, - child_env, + env, None, None, ) From 4bd88358adddaf09506f37104ddb4243bdf3b66d Mon Sep 17 00:00:00 2001 From: Olivier Grisel Date: Wed, 22 Feb 2023 16:10:18 +0100 Subject: [PATCH 29/50] Revert "Fix typo in variable name in last commit code sync" This reverts commit 106251014f658c49e731b45e38b33e5b966ed122. --- loky/backend/popen_loky_win32.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/loky/backend/popen_loky_win32.py b/loky/backend/popen_loky_win32.py index 7d00c43f..1ff8208f 100644 --- a/loky/backend/popen_loky_win32.py +++ b/loky/backend/popen_loky_win32.py @@ -91,7 +91,7 @@ def __init__(self, process_obj): None, inherit, 0, - env, + child_env, None, None, ) From ee1b326eab0cef562c90f473d4e3811531976c46 Mon Sep 17 00:00:00 2001 From: Olivier Grisel Date: Wed, 22 Feb 2023 16:10:25 +0100 Subject: [PATCH 30/50] Revert "Blind attempt to sync Popen.__init__ with cpython main" This reverts commit 08e1c1718e5bd362a109ba1bccf7bb3c735e1707. --- loky/backend/popen_loky_win32.py | 34 +++++++++++--------------------- 1 file changed, 12 insertions(+), 22 deletions(-) diff --git a/loky/backend/popen_loky_win32.py b/loky/backend/popen_loky_win32.py index 1ff8208f..3401234f 100644 --- a/loky/backend/popen_loky_win32.py +++ b/loky/backend/popen_loky_win32.py @@ -43,34 +43,24 @@ def __init__(self, process_obj): process_obj._name, getattr(process_obj, "init_main_module", True) ) - # read end of pipe will be duplicated by the child process + # read end of pipe will be "stolen" by the child process # -- see spawn_main() in spawn.py. - # - # bpo-33929: Previously, the read end of pipe was "stolen" by the child - # process, but it leaked a handle if the child process had been - # terminated before it could steal the handle from the parent process. - rhandle, whandle = _winapi.CreatePipe(None, 0) - wfd = msvcrt.open_osfhandle(whandle, 0) - cmd = spawn.get_command_line(parent_pid=os.getpid(), - pipe_handle=rhandle) + rfd, wfd = os.pipe() + rhandle = duplicate(msvcrt.get_osfhandle(rfd), inheritable=True) + os.close(rfd) - python_exe = spawn.get_executable() + cmd = spawn.get_command_line(fd=rhandle) + python_exe = cmd[0] + cmd = " ".join(f'"{x}"' for x in cmd) + + # copy the environment variables to set in the child process + child_env = {**os.environ, **process_obj.env} # bpo-35797: When running in a venv, we bypass the redirect # executor and launch our base Python. if WINENV and _path_eq(python_exe, sys.executable): - cmd[0] = python_exe = sys._base_executable - env = os.environ.copy() - env["__PYVENV_LAUNCHER__"] = sys.executable - else: - env = None - - cmd = ' '.join('"%s"' % x for x in cmd) - - # XXX: remove debug logs - util.debug(f"{WINENV = }") - util.debug(f"{python_exe = }") - util.debug(f"{sys.executable = }") + python_exe = sys._base_executable + child_env["__PYVENV_LAUNCHER__"] = sys.executable try: with open(wfd, "wb") as to_child: From 15c001ed72e6a6a57b436ccaf75b45b4dc4f4084 Mon Sep 17 00:00:00 2001 From: tommoral Date: Thu, 23 Feb 2023 10:55:32 +0100 Subject: [PATCH 31/50] FIX try to make loky closer too mp --- loky/backend/_win_reduction.py | 56 +++----------------------------- loky/backend/popen_loky_win32.py | 11 ++++++- loky/backend/spawn.py | 4 +-- loky/backend/synchronize.py | 10 ++++-- 4 files changed, 23 insertions(+), 58 deletions(-) diff --git a/loky/backend/_win_reduction.py b/loky/backend/_win_reduction.py index 99fcb78c..85fccb7b 100644 --- a/loky/backend/_win_reduction.py +++ b/loky/backend/_win_reduction.py @@ -6,62 +6,14 @@ # adapted from multiprocessing/reduction.py (17/02/2017) # * Add adapted reduction for LokyProcesses and socket/PipeConnection # -import os import socket -import _winapi -from multiprocessing.connection import PipeConnection +from multiprocessing import connection from multiprocessing.reduction import _reduce_socket from .reduction import register -class DupHandle: - def __init__(self, handle, access, pid=None): - # duplicate handle for process with given pid - if pid is None: - pid = os.getpid() - proc = _winapi.OpenProcess(_winapi.PROCESS_DUP_HANDLE, False, pid) - try: - self._handle = _winapi.DuplicateHandle( - _winapi.GetCurrentProcess(), handle, proc, access, False, 0 - ) - finally: - _winapi.CloseHandle(proc) - self._access = access - self._pid = pid - - def detach(self): - # retrieve handle from process which currently owns it - if self._pid == os.getpid(): - return self._handle - proc = _winapi.OpenProcess( - _winapi.PROCESS_DUP_HANDLE, False, self._pid - ) - try: - return _winapi.DuplicateHandle( - proc, - self._handle, - _winapi.GetCurrentProcess(), - self._access, - False, - _winapi.DUPLICATE_CLOSE_SOURCE, - ) - finally: - _winapi.CloseHandle(proc) - - -def rebuild_pipe_connection(dh, readable, writable): - handle = dh.detach() - return PipeConnection(handle, readable, writable) - - -def reduce_pipe_connection(conn): - access = (_winapi.FILE_GENERIC_READ if conn.readable else 0) | ( - _winapi.FILE_GENERIC_WRITE if conn.writable else 0 - ) - dh = DupHandle(conn.fileno(), access) - return rebuild_pipe_connection, (dh, conn.readable, conn.writable) - - -register(PipeConnection, reduce_pipe_connection) +# register reduction for win32 communication objects register(socket.socket, _reduce_socket) +register(connection.Connection, connection.reduce_connection) +register(connection.PipeConnection, connection.reduce_pipe_connection) diff --git a/loky/backend/popen_loky_win32.py b/loky/backend/popen_loky_win32.py index 3401234f..98ac0b12 100644 --- a/loky/backend/popen_loky_win32.py +++ b/loky/backend/popen_loky_win32.py @@ -25,6 +25,11 @@ def _path_eq(p1, p2): sys.executable, sys._base_executable ) + +def _close_handles(*handles): + for handle in handles: + _winapi.CloseHandle(handle) + # # We define a Popen class similar to the one from subprocess, but # whose constructor takes a process object as its argument. @@ -95,7 +100,11 @@ def __init__(self, process_obj): self.returncode = None self._handle = hp self.sentinel = int(hp) - util.Finalize(self, _winapi.CloseHandle, (self.sentinel,)) + self.finalizer = util.Finalize( + self, + _close_handles, + (self.sentinel, int(rhandle)) + ) # send information to child set_spawning_popen(self) diff --git a/loky/backend/spawn.py b/loky/backend/spawn.py index 669c800e..35e998eb 100644 --- a/loky/backend/spawn.py +++ b/loky/backend/spawn.py @@ -305,7 +305,7 @@ def get_command_line(main_prog=main, **kwargs): list_kwargs = [f"{k}={v}" for k, v in kwargs.items()] argv = [ sys.executable, - "--multiprocessing-fork", + "--loky-fork", main_prog.__module__, *list_kwargs, ] @@ -329,7 +329,7 @@ def freeze_support(): It should be called right after the beginning of the programme, to avoid recursive process spawning. """ - if len(sys.argv) >= 2 and sys.argv[1] == "--multiprocessing-fork": + if len(sys.argv) >= 2 and sys.argv[1] == "--loky-fork": module_main = sys.argv[2] main = importlib.import_module(module_main).main kwargs = {} diff --git a/loky/backend/synchronize.py b/loky/backend/synchronize.py index 18db3e34..c802da40 100644 --- a/loky/backend/synchronize.py +++ b/loky/backend/synchronize.py @@ -19,7 +19,7 @@ import _multiprocessing from time import time as _time from multiprocessing import process, util -from multiprocessing.context import assert_spawning +from multiprocessing import context from . import resource_tracker @@ -122,9 +122,13 @@ def __exit__(self, *args): return self._semlock.release() def __getstate__(self): - assert_spawning(self) + context.assert_spawning(self) sl = self._semlock h = sl.handle + if sys.platform == 'win32': + h = context.get_spawning_popen().duplicate_for_child(sl.handle) + else: + h = sl.handle return (h, sl.kind, sl.maxvalue, sl.name) def __setstate__(self, state): @@ -249,7 +253,7 @@ def __init__(self, lock=None): self._make_methods() def __getstate__(self): - assert_spawning(self) + context.assert_spawning(self) return ( self._lock, self._sleeping_count, From 46a6c09f4ee52f40be78a08be804ec1c467b6a39 Mon Sep 17 00:00:00 2001 From: tommoral Date: Mon, 27 Feb 2023 10:34:50 +0100 Subject: [PATCH 32/50] FIX black formatting --- loky/backend/popen_loky_win32.py | 5 ++--- loky/backend/synchronize.py | 2 +- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/loky/backend/popen_loky_win32.py b/loky/backend/popen_loky_win32.py index 98ac0b12..d31366d2 100644 --- a/loky/backend/popen_loky_win32.py +++ b/loky/backend/popen_loky_win32.py @@ -30,6 +30,7 @@ def _close_handles(*handles): for handle in handles: _winapi.CloseHandle(handle) + # # We define a Popen class similar to the one from subprocess, but # whose constructor takes a process object as its argument. @@ -101,9 +102,7 @@ def __init__(self, process_obj): self._handle = hp self.sentinel = int(hp) self.finalizer = util.Finalize( - self, - _close_handles, - (self.sentinel, int(rhandle)) + self, _close_handles, (self.sentinel, int(rhandle)) ) # send information to child diff --git a/loky/backend/synchronize.py b/loky/backend/synchronize.py index c802da40..db5e0505 100644 --- a/loky/backend/synchronize.py +++ b/loky/backend/synchronize.py @@ -125,7 +125,7 @@ def __getstate__(self): context.assert_spawning(self) sl = self._semlock h = sl.handle - if sys.platform == 'win32': + if sys.platform == "win32": h = context.get_spawning_popen().duplicate_for_child(sl.handle) else: h = sl.handle From 643f37791fa4a0c4dbac9e4a8f0f3e8bd5b760a5 Mon Sep 17 00:00:00 2001 From: tommoral Date: Mon, 27 Feb 2023 13:37:26 +0100 Subject: [PATCH 33/50] FIX tentative to set the right permission in win processes --- loky/backend/popen_loky_posix.py | 14 ++++++--- loky/backend/popen_loky_win32.py | 43 ++++++++++----------------- loky/backend/spawn.py | 50 +++++++++++++++++++++----------- tests/test_loky_backend.py | 5 ++-- 4 files changed, 61 insertions(+), 51 deletions(-) diff --git a/loky/backend/popen_loky_posix.py b/loky/backend/popen_loky_posix.py index a30b683e..7a114954 100644 --- a/loky/backend/popen_loky_posix.py +++ b/loky/backend/popen_loky_posix.py @@ -112,7 +112,9 @@ def _launch(self, process_obj): reduction._mk_inheritable(child_w) reduction._mk_inheritable(tracker_fd) cmd_python = spawn.get_command_line( - fd=child_r, process_name=process_obj.name + pipe_handle=child_r, + parent_pid=os.getpid(), + process_name=process_obj.name ) self._fds += [child_r, child_w, tracker_fd] if sys.version_info >= (3, 8) and os.name == "posix": @@ -130,12 +132,16 @@ def _launch(self, process_obj): method = "getbuffer" if not hasattr(fp, method): method = "getvalue" - with os.fdopen(parent_w, "wb") as f: + with os.fdopen(parent_w, "wb", closefd=False) as f: f.write(getattr(fp, method)()) self.pid = pid finally: - if parent_r is not None: - util.Finalize(self, os.close, (parent_r,)) + fds_to_close = [] + for fd in (parent_r, parent_w): + if fd is not None: + fds_to_close.append(fd) + self.finalizer = util.Finalize(self, util.close_fds, fds_to_close) + for fd in (child_r, child_w): if fd is not None: os.close(fd) diff --git a/loky/backend/popen_loky_win32.py b/loky/backend/popen_loky_win32.py index d31366d2..f236ae35 100644 --- a/loky/backend/popen_loky_win32.py +++ b/loky/backend/popen_loky_win32.py @@ -3,9 +3,9 @@ import msvcrt import _winapi from multiprocessing import util -from multiprocessing.context import get_spawning_popen, set_spawning_popen +from multiprocessing.context import set_spawning_popen +from multiprocessing.popen_spawn_win32 import _close_handles from multiprocessing.popen_spawn_win32 import Popen as _Popen -from multiprocessing.reduction import duplicate from . import reduction, spawn @@ -25,12 +25,6 @@ def _path_eq(p1, p2): sys.executable, sys._base_executable ) - -def _close_handles(*handles): - for handle in handles: - _winapi.CloseHandle(handle) - - # # We define a Popen class similar to the one from subprocess, but # whose constructor takes a process object as its argument. @@ -49,13 +43,18 @@ def __init__(self, process_obj): process_obj._name, getattr(process_obj, "init_main_module", True) ) - # read end of pipe will be "stolen" by the child process + # read end of pipe will be duplicated by the child process # -- see spawn_main() in spawn.py. - rfd, wfd = os.pipe() - rhandle = duplicate(msvcrt.get_osfhandle(rfd), inheritable=True) - os.close(rfd) - - cmd = spawn.get_command_line(fd=rhandle) + # + # bpo-33929: Previously, the read end of pipe was "stolen" by the child + # process, but it leaked a handle if the child process had been + # terminated before it could steal the handle from the parent process. + rhandle, whandle = _winapi.CreatePipe(None, 0) + wfd = msvcrt.open_osfhandle(whandle, 0) + + cmd = spawn.get_command_line( + pipe_handle=rhandle, parent_pid=os.getpid() + ) python_exe = cmd[0] cmd = " ".join(f'"{x}"' for x in cmd) @@ -69,23 +68,15 @@ def __init__(self, process_obj): child_env["__PYVENV_LAUNCHER__"] = sys.executable try: - with open(wfd, "wb") as to_child: + with open(wfd, "wb", closefd=True) as to_child: # start process try: - # This flag allows to pass inheritable handles from the - # parent to the child process in a python2-3 compatible way - # (see - # https://github.com/tomMoral/loky/pull/204#discussion_r290719629 - # for more detail). When support for Python 2 is dropped, - # the cleaner multiprocessing.reduction.steal_handle should - # be used instead. - inherit = True hp, ht, pid, _ = _winapi.CreateProcess( python_exe, cmd, None, None, - inherit, + False, 0, child_env, None, @@ -120,7 +111,3 @@ def __init__(self, process_obj): util.debug( f"While starting {process_obj._name}, ignored a IOError 22" ) - - def duplicate_for_child(self, handle): - assert self is get_spawning_popen() - return duplicate(handle, self.sentinel) diff --git a/loky/backend/spawn.py b/loky/backend/spawn.py index 35e998eb..d1205bc9 100644 --- a/loky/backend/spawn.py +++ b/loky/backend/spawn.py @@ -256,24 +256,31 @@ def _fixup_main_from_path(main_path): sys.modules["__main__"] = sys.modules["__mp_main__"] = main_module -def main(fd, process_name=None): +def main(pipe_handle, parent_pid, process_name=None): # arguments are passed as strings, convert them back to int. - fd = int(fd) - if sys.platform == "win32": - fd = msvcrt.open_osfhandle(fd, os.O_RDONLY) + pipe_handle, parent_pid = int(pipe_handle), int(parent_pid) + if sys.platform == 'win32': + import msvcrt + import _winapi + + if parent_pid is not None: + source_process = _winapi.OpenProcess( + _winapi.SYNCHRONIZE | _winapi.PROCESS_DUP_HANDLE, + False, + parent_pid + ) + else: + source_process = None + new_handle = duplicate(pipe_handle, source_process=source_process) + fd = msvcrt.open_osfhandle(new_handle, os.O_RDONLY) + parent_sentinel = source_process + else: + fd = pipe_handle + parent_sentinel = os.dup(pipe_handle) exitcode = 1 try: - with os.fdopen(fd, "rb") as from_parent: - process.current_process()._inheriting = True - try: - prep_data = pickle.load(from_parent) - prepare(prep_data) - process_obj = pickle.load(from_parent) - finally: - del process.current_process()._inheriting - - exitcode = process_obj._bootstrap() + exitcode = _main(fd, parent_sentinel=parent_sentinel) except Exception: print("\n\n" + "-" * 80) print(f"{process_name} failed with traceback: ") @@ -283,12 +290,21 @@ def main(fd, process_name=None): print(traceback.format_exc()) print("\n" + "-" * 80) finally: - if from_parent is not None: - from_parent.close() - sys.exit(exitcode) +def _main(fd, parent_sentinel): + with os.fdopen(fd, 'rb', closefd=True) as from_parent: + process.current_process()._inheriting = True + try: + preparation_data = pickle.load(from_parent) + prepare(preparation_data) + self = pickle.load(from_parent) + finally: + del process.current_process()._inheriting + return self._bootstrap(parent_sentinel) + + def get_command_line(main_prog=main, **kwargs): """ Returns a command line used for spawning a child process. diff --git a/tests/test_loky_backend.py b/tests/test_loky_backend.py index 15f4a229..27696514 100644 --- a/tests/test_loky_backend.py +++ b/tests/test_loky_backend.py @@ -509,10 +509,11 @@ def _check_fds(self, pid, w): # - one pipe for communication with main process # - loky's resource_tracker pipe # - the Connection pipe + # - the pipe used for the parent_sentinel # - additionally, on posix + Python 3.8: multiprocessing's # resource_tracker pipe - if sys.version_info >= (3, 8) and os.name == "posix": - n_expected_pipes = 4 + if sys.version_info >= (3, 8): + n_expected_pipes = 5 if os.name == "posix" else 4 else: n_expected_pipes = 3 msg = ( From aac59a58b7a67d7dbbc5130e9cae7a4f479aec11 Mon Sep 17 00:00:00 2001 From: tommoral Date: Mon, 27 Feb 2023 13:56:05 +0100 Subject: [PATCH 34/50] FIX compat with python<=3.7 and pypy --- loky/backend/popen_loky_posix.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/loky/backend/popen_loky_posix.py b/loky/backend/popen_loky_posix.py index 7a114954..40840ca2 100644 --- a/loky/backend/popen_loky_posix.py +++ b/loky/backend/popen_loky_posix.py @@ -29,6 +29,18 @@ def __init__(self, fd): def detach(self): return self.fd +# +# Backward compat for pypy and python<=3.7 +# + + +if not hasattr(util, "close_fds"): + def _close_fds(*fds): + for fd in fds: + os.close(fd) + + util.close_fds = _close_fds() + # # Start child process using subprocess.Popen From 88449d224ae17d42ef061eb128d2340da2765ca7 Mon Sep 17 00:00:00 2001 From: tommoral Date: Mon, 27 Feb 2023 17:22:49 +0100 Subject: [PATCH 35/50] FIX compat for pypy --- loky/backend/process.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/loky/backend/process.py b/loky/backend/process.py index 35625509..ef94c253 100644 --- a/loky/backend/process.py +++ b/loky/backend/process.py @@ -44,6 +44,13 @@ def _Popen(process_obj): from .popen_loky_posix import Popen return Popen(process_obj) + def _bootstrap(self, parent_sentinel=None): + try: + super()._bootstrap(parent_sentinel=parent_sentinel) + except TypeError: + # Compat for pypy that doesn't accept the parent_sentinel argument + super()._bootstrap() + class LokyInitMainProcess(LokyProcess): _start_method = "loky_init_main" From 283d9b8984413b2c8231f667c3f3370b8468a854 Mon Sep 17 00:00:00 2001 From: tommoral Date: Mon, 27 Feb 2023 17:32:50 +0100 Subject: [PATCH 36/50] FIX black formatting --- loky/backend/popen_loky_posix.py | 4 +++- loky/backend/spawn.py | 6 +++--- loky/backend/synchronize.py | 1 - loky/process_executor.py | 3 --- tests/_test_process_executor.py | 1 - tests/test_loky_backend.py | 5 ----- tests/test_reusable_executor.py | 1 - tests/test_synchronize.py | 1 - tests/test_worker_timeout.py | 1 - 9 files changed, 6 insertions(+), 17 deletions(-) diff --git a/loky/backend/popen_loky_posix.py b/loky/backend/popen_loky_posix.py index 40840ca2..e0cdf639 100644 --- a/loky/backend/popen_loky_posix.py +++ b/loky/backend/popen_loky_posix.py @@ -29,12 +29,14 @@ def __init__(self, fd): def detach(self): return self.fd + # # Backward compat for pypy and python<=3.7 # if not hasattr(util, "close_fds"): + def _close_fds(*fds): for fd in fds: os.close(fd) @@ -126,7 +128,7 @@ def _launch(self, process_obj): cmd_python = spawn.get_command_line( pipe_handle=child_r, parent_pid=os.getpid(), - process_name=process_obj.name + process_name=process_obj.name, ) self._fds += [child_r, child_w, tracker_fd] if sys.version_info >= (3, 8) and os.name == "posix": diff --git a/loky/backend/spawn.py b/loky/backend/spawn.py index d1205bc9..c4d091c5 100644 --- a/loky/backend/spawn.py +++ b/loky/backend/spawn.py @@ -259,7 +259,7 @@ def _fixup_main_from_path(main_path): def main(pipe_handle, parent_pid, process_name=None): # arguments are passed as strings, convert them back to int. pipe_handle, parent_pid = int(pipe_handle), int(parent_pid) - if sys.platform == 'win32': + if sys.platform == "win32": import msvcrt import _winapi @@ -267,7 +267,7 @@ def main(pipe_handle, parent_pid, process_name=None): source_process = _winapi.OpenProcess( _winapi.SYNCHRONIZE | _winapi.PROCESS_DUP_HANDLE, False, - parent_pid + parent_pid, ) else: source_process = None @@ -294,7 +294,7 @@ def main(pipe_handle, parent_pid, process_name=None): def _main(fd, parent_sentinel): - with os.fdopen(fd, 'rb', closefd=True) as from_parent: + with os.fdopen(fd, "rb", closefd=True) as from_parent: process.current_process()._inheriting = True try: preparation_data = pickle.load(from_parent) diff --git a/loky/backend/synchronize.py b/loky/backend/synchronize.py index db5e0505..f7cfe0fa 100644 --- a/loky/backend/synchronize.py +++ b/loky/backend/synchronize.py @@ -59,7 +59,6 @@ class SemLock: - _rand = tempfile._RandomNameSequence() def __init__(self, kind, value, maxvalue, name=None): diff --git a/loky/process_executor.py b/loky/process_executor.py index 3986ea7a..409ce593 100644 --- a/loky/process_executor.py +++ b/loky/process_executor.py @@ -146,7 +146,6 @@ class _ExecutorFlags: """ def __init__(self, shutdown_lock): - self.shutdown = False self.broken = None self.kill_workers = False @@ -246,7 +245,6 @@ def _rebuild_exc(exc, tb): class _WorkItem: - __slots__ = ["future", "fn", "args", "kwargs"] def __init__(self, future, fn, args, kwargs): @@ -1004,7 +1002,6 @@ class ShutdownExecutorError(RuntimeError): class ProcessPoolExecutor(Executor): - _at_exit = None def __init__( diff --git a/tests/_test_process_executor.py b/tests/_test_process_executor.py index f58af9f8..15eb54c3 100644 --- a/tests/_test_process_executor.py +++ b/tests/_test_process_executor.py @@ -192,7 +192,6 @@ def test_processes_terminate(self): p.join() def test_processes_terminate_on_executor_gc(self): - results = self.executor.map(sleep_and_return, [0.1] * 10, range(10)) assert len(self.executor._processes) == self.worker_count processes = self.executor._processes diff --git a/tests/test_loky_backend.py b/tests/test_loky_backend.py index 27696514..8b8f3000 100644 --- a/tests/test_loky_backend.py +++ b/tests/test_loky_backend.py @@ -71,7 +71,6 @@ def teardown_class(cls): kill_process_tree(child_process) def test_current(self): - current = self.current_process() authkey = current.authkey @@ -83,7 +82,6 @@ def test_current(self): assert current.exitcode is None def test_daemon_argument(self): - # By default uses the current process's daemon flag. proc0 = self.Process(target=self._test_process) assert proc0.daemon == self.current_process().daemon @@ -296,7 +294,6 @@ def _test_terminate(cls, event): time.sleep(100) def test_terminate(self): - manager = self.Manager() event = manager.Event() @@ -481,7 +478,6 @@ def _check_fds(self, pid, w): n_pipe = 0 named_sem = [] for fd, t, name in zip(lines[::3], lines[1::3], lines[2::3]): - # Check if fd is a standard IO file. For python 3.x stdin # should be closed. is_std = fd in ["f1", "f2"] @@ -553,7 +549,6 @@ def test_sync_object_handling(self): ) named_sem = [] try: - p.start() assert started.wait(5), "The process took too long to start" r.close() diff --git a/tests/test_reusable_executor.py b/tests/test_reusable_executor.py index 10a19dbc..01a9520e 100644 --- a/tests/test_reusable_executor.py +++ b/tests/test_reusable_executor.py @@ -219,7 +219,6 @@ def __del__(self): class TestExecutorDeadLock(ReusableExecutorMixin): - crash_cases = [ # Check problem occuring while pickling a task in (id, (ExitAtPickle(),), PicklingError, None), diff --git a/tests/test_synchronize.py b/tests/test_synchronize.py index 1ceaa01c..cefd8225 100644 --- a/tests/test_synchronize.py +++ b/tests/test_synchronize.py @@ -113,7 +113,6 @@ def test_bounded_semaphore(self): assert_sem_value_equal(sem, 2) def test_timeout(self): - sem = loky_context.Semaphore(0) acquire = TimingWrapper(sem.acquire) diff --git a/tests/test_worker_timeout.py b/tests/test_worker_timeout.py index 740bd171..68872cbe 100644 --- a/tests/test_worker_timeout.py +++ b/tests/test_worker_timeout.py @@ -57,7 +57,6 @@ def close(self): @staticmethod def _feed(readlock, reader, writer, delay): - PICKLE_NONE = dumps(None) while True: From 80a31a1659fd9a80d7766b47b61f3572648b4858 Mon Sep 17 00:00:00 2001 From: tommoral Date: Mon, 27 Feb 2023 18:29:32 +0100 Subject: [PATCH 37/50] FIX resource_tracker use handle duplication too --- loky/backend/resource_tracker.py | 54 +++++++++++++++++++++----------- loky/backend/spawn.py | 45 +++++++++++++++----------- 2 files changed, 62 insertions(+), 37 deletions(-) diff --git a/loky/backend/resource_tracker.py b/loky/backend/resource_tracker.py index 5814fe85..01ceadd3 100644 --- a/loky/backend/resource_tracker.py +++ b/loky/backend/resource_tracker.py @@ -57,7 +57,7 @@ if sys.platform == "win32": import _winapi import msvcrt - from multiprocessing.reduction import duplicate + from .spawn import duplicate_in_child_process __all__ = ["ensure_running", "register", "unregister"] @@ -116,23 +116,25 @@ def ensure_running(self): "leak." ) - fds_to_pass = [] - try: - fds_to_pass.append(sys.stderr.fileno()) - except Exception: - pass - - r, w = os.pipe() if sys.platform == "win32": - _r = duplicate(msvcrt.get_osfhandle(r), inheritable=True) - os.close(r) - r = _r + r, whandle = _winapi.CreatePipe(None, 0) + w = msvcrt.open_osfhandle(whandle, 0) + fds_to_pass = [r] + else: + r, w = os.pipe() + fds_to_pass = [r] + try: + fds_to_pass.append(sys.stderr.fileno()) + except Exception: + pass try: - fds_to_pass.append(r) # process will out live us, so no need to wait on pid cmd = spawn.get_command_line( - main_prog=main, fd=r, verbose=int(VERBOSE) + main_prog=main, + pipe_handle=r, + parent_pid=os.getpid(), + verbose=int(VERBOSE), ) util.debug(f"launching resource tracker: {cmd}") # bpo-33613: Register a signal mask that will block the @@ -206,12 +208,19 @@ def _send(self, cmd, name, rtype): getfd = _resource_tracker.getfd -def main(fd, verbose=0): +def main(pipe_handle, parent_pid, verbose=0): """Run resource tracker.""" # Make sure the arguments have the right type as they are # passed as strings through the command line. - fd = int(fd) + pipe_handle, parent_pid = int(pipe_handle), int(parent_pid) verbose = int(verbose) + if sys.platform == "win32": + handle, parent_sentinel = duplicate_in_child_process( + pipe_handle, parent_pid + ) + fd = msvcrt.open_osfhandle(handle, os.O_RDONLY) + else: + fd = pipe_handle # protect the process from ^C and "killall python" etc if verbose: @@ -235,9 +244,7 @@ def main(fd, verbose=0): registry = {rtype: {} for rtype in _CLEANUP_FUNCS.keys()} try: # keep track of registered/unregistered resources - if sys.platform == "win32": - fd = msvcrt.open_osfhandle(fd, os.O_RDONLY) - with open(fd, "rb") as f: + with open(fd, "rb", closefd=True) as f: while True: line = f.readline() if line == b"": # EOF @@ -359,6 +366,14 @@ def _unlink_resources(rtype_registry, rtype): def spawnv_passfds(cmd, passfds): + """Spawn the resource tracker in a platform specific way. + + For posix platforms, make the passfds inheritable and use fork_exec. + + For windows platforms, passfds is only used to clean up the handles in + case of failure, the inheritance of the handles will be taken care in the + child process through _winapi.OpenProcess. + """ passfds = sorted(passfds) if sys.platform != "win32": errpipe_read, errpipe_write = os.pipe() @@ -376,9 +391,10 @@ def spawnv_passfds(cmd, passfds): cmd = " ".join(f'"{x}"' for x in cmd) try: _, ht, pid, _ = _winapi.CreateProcess( - exe, cmd, None, None, True, 0, None, None, None + exe, cmd, None, None, False, 0, None, None, None ) _winapi.CloseHandle(ht) except BaseException: + _winapi.CloseHandle(passfds[0]) pass return pid diff --git a/loky/backend/spawn.py b/loky/backend/spawn.py index c4d091c5..0f1656be 100644 --- a/loky/backend/spawn.py +++ b/loky/backend/spawn.py @@ -28,6 +28,25 @@ WINEXE = sys.platform == "win32" and getattr(sys, "frozen", False) WINSERVICE = sys.executable.lower().endswith("pythonservice.exe") + def duplicate_in_child_process(handle, parent_pid=None): + """Duplicate a handle in child process given its parent pid. + + Returns a file descriptor for the handle and the parent process. + """ + import _winapi + + if parent_pid is not None: + source_process = _winapi.OpenProcess( + _winapi.SYNCHRONIZE | _winapi.PROCESS_DUP_HANDLE, + False, + parent_pid, + ) + else: + source_process = None + new_handle = duplicate(handle, source_process=source_process) + return new_handle, source_process + + if WINSERVICE: _python_exe = os.path.join(sys.exec_prefix, "python.exe") else: @@ -88,10 +107,8 @@ def get_preparation_data(name, init_main_module=True): _resource_tracker.ensure_running() d["tracker_args"] = {"pid": _resource_tracker._pid} if sys.platform == "win32": - child_w = duplicate( - msvcrt.get_osfhandle(_resource_tracker._fd), inheritable=True - ) - d["tracker_args"]["fh"] = child_w + d["tracker_args"]["parent_pid"] = os.getpid() + d["tracker_args"]["fh"] = msvcrt.get_osfhandle(_resource_tracker._fd) else: d["tracker_args"]["fd"] = _resource_tracker._fd @@ -192,6 +209,8 @@ def prepare(data): _resource_tracker._pid = data["tracker_args"]["pid"] if sys.platform == "win32": handle = data["tracker_args"]["fh"] + parent_pid = data["tracker_args"]["parent_pid"] + handle, _ = duplicate_in_child_process(handle, parent_pid) _resource_tracker._fd = msvcrt.open_osfhandle(handle, 0) else: _resource_tracker._fd = data["tracker_args"]["fd"] @@ -260,20 +279,10 @@ def main(pipe_handle, parent_pid, process_name=None): # arguments are passed as strings, convert them back to int. pipe_handle, parent_pid = int(pipe_handle), int(parent_pid) if sys.platform == "win32": - import msvcrt - import _winapi - - if parent_pid is not None: - source_process = _winapi.OpenProcess( - _winapi.SYNCHRONIZE | _winapi.PROCESS_DUP_HANDLE, - False, - parent_pid, - ) - else: - source_process = None - new_handle = duplicate(pipe_handle, source_process=source_process) - fd = msvcrt.open_osfhandle(new_handle, os.O_RDONLY) - parent_sentinel = source_process + handle, parent_sentinel = duplicate_in_child_process( + pipe_handle, parent_pid + ) + fd = msvcrt.open_osfhandle(handle, os.O_RDONLY) else: fd = pipe_handle parent_sentinel = os.dup(pipe_handle) From 4f738e7c0db006a4771899651d955fc5c8068808 Mon Sep 17 00:00:00 2001 From: tommoral Date: Mon, 27 Feb 2023 18:46:54 +0100 Subject: [PATCH 38/50] DBG blind test for resource tracker pipe open --- loky/backend/resource_tracker.py | 4 +++- loky/backend/spawn.py | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/loky/backend/resource_tracker.py b/loky/backend/resource_tracker.py index 01ceadd3..7a4a7195 100644 --- a/loky/backend/resource_tracker.py +++ b/loky/backend/resource_tracker.py @@ -118,7 +118,9 @@ def ensure_running(self): if sys.platform == "win32": r, whandle = _winapi.CreatePipe(None, 0) - w = msvcrt.open_osfhandle(whandle, 0) + w = os.open( + msvcrt.open_osfhandle(whandle, 0), os.O_BINARY, mode="wb" + ) fds_to_pass = [r] else: r, w = os.pipe() diff --git a/loky/backend/spawn.py b/loky/backend/spawn.py index 0f1656be..ca93a681 100644 --- a/loky/backend/spawn.py +++ b/loky/backend/spawn.py @@ -282,7 +282,9 @@ def main(pipe_handle, parent_pid, process_name=None): handle, parent_sentinel = duplicate_in_child_process( pipe_handle, parent_pid ) - fd = msvcrt.open_osfhandle(handle, os.O_RDONLY) + fd = os.open( + msvcrt.open_osfhandle(handle, os.O_RDONLY), os.O_BINARY, mode="wb" + ) else: fd = pipe_handle parent_sentinel = os.dup(pipe_handle) From 453f178e25cadfb477afbc91670a3620d8f432c7 Mon Sep 17 00:00:00 2001 From: tommoral Date: Mon, 27 Feb 2023 23:56:25 +0100 Subject: [PATCH 39/50] FIX close_fds compat 3.7 --- loky/backend/popen_loky_posix.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/loky/backend/popen_loky_posix.py b/loky/backend/popen_loky_posix.py index e0cdf639..44975559 100644 --- a/loky/backend/popen_loky_posix.py +++ b/loky/backend/popen_loky_posix.py @@ -32,16 +32,15 @@ def detach(self): # # Backward compat for pypy and python<=3.7 +# XXX: to remove once 3.7 is not supported anymore. # - - if not hasattr(util, "close_fds"): def _close_fds(*fds): for fd in fds: os.close(fd) - util.close_fds = _close_fds() + util.close_fds = _close_fds # From 50526af1d98ab50d5e6ff0535db1cd95afec92a6 Mon Sep 17 00:00:00 2001 From: tommoral Date: Tue, 28 Feb 2023 00:00:51 +0100 Subject: [PATCH 40/50] FIX use fdopen and not os.open --- loky/backend/resource_tracker.py | 4 +--- loky/backend/spawn.py | 4 +--- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/loky/backend/resource_tracker.py b/loky/backend/resource_tracker.py index 7a4a7195..4fbcb495 100644 --- a/loky/backend/resource_tracker.py +++ b/loky/backend/resource_tracker.py @@ -118,9 +118,7 @@ def ensure_running(self): if sys.platform == "win32": r, whandle = _winapi.CreatePipe(None, 0) - w = os.open( - msvcrt.open_osfhandle(whandle, 0), os.O_BINARY, mode="wb" - ) + w = os.fdopen(msvcrt.open_osfhandle(whandle, 0), mode="wb") fds_to_pass = [r] else: r, w = os.pipe() diff --git a/loky/backend/spawn.py b/loky/backend/spawn.py index ca93a681..912ecc47 100644 --- a/loky/backend/spawn.py +++ b/loky/backend/spawn.py @@ -282,9 +282,7 @@ def main(pipe_handle, parent_pid, process_name=None): handle, parent_sentinel = duplicate_in_child_process( pipe_handle, parent_pid ) - fd = os.open( - msvcrt.open_osfhandle(handle, os.O_RDONLY), os.O_BINARY, mode="wb" - ) + fd = os.fdopen(msvcrt.open_osfhandle(handle, os.O_RDONLY), mode="rb") else: fd = pipe_handle parent_sentinel = os.dup(pipe_handle) From 579ade50e2aa140b6b7ea0df14c7299c64ea7cef Mon Sep 17 00:00:00 2001 From: tommoral Date: Tue, 28 Feb 2023 00:10:03 +0100 Subject: [PATCH 41/50] FIX correct fd from ressource_tracker on win32 --- loky/backend/spawn.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/loky/backend/spawn.py b/loky/backend/spawn.py index 912ecc47..2e5f7a24 100644 --- a/loky/backend/spawn.py +++ b/loky/backend/spawn.py @@ -108,7 +108,9 @@ def get_preparation_data(name, init_main_module=True): d["tracker_args"] = {"pid": _resource_tracker._pid} if sys.platform == "win32": d["tracker_args"]["parent_pid"] = os.getpid() - d["tracker_args"]["fh"] = msvcrt.get_osfhandle(_resource_tracker._fd) + d["tracker_args"]["fh"] = msvcrt.get_osfhandle( + _resource_tracker._fd.fileno() + ) else: d["tracker_args"]["fd"] = _resource_tracker._fd From 2065f3cebe0a92a55821ddf5a31958130d6061be Mon Sep 17 00:00:00 2001 From: tommoral Date: Tue, 28 Feb 2023 00:19:36 +0100 Subject: [PATCH 42/50] FIX remove bad opening --- loky/backend/resource_tracker.py | 2 +- loky/backend/spawn.py | 6 ++---- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/loky/backend/resource_tracker.py b/loky/backend/resource_tracker.py index 4fbcb495..01ceadd3 100644 --- a/loky/backend/resource_tracker.py +++ b/loky/backend/resource_tracker.py @@ -118,7 +118,7 @@ def ensure_running(self): if sys.platform == "win32": r, whandle = _winapi.CreatePipe(None, 0) - w = os.fdopen(msvcrt.open_osfhandle(whandle, 0), mode="wb") + w = msvcrt.open_osfhandle(whandle, 0) fds_to_pass = [r] else: r, w = os.pipe() diff --git a/loky/backend/spawn.py b/loky/backend/spawn.py index 2e5f7a24..0f1656be 100644 --- a/loky/backend/spawn.py +++ b/loky/backend/spawn.py @@ -108,9 +108,7 @@ def get_preparation_data(name, init_main_module=True): d["tracker_args"] = {"pid": _resource_tracker._pid} if sys.platform == "win32": d["tracker_args"]["parent_pid"] = os.getpid() - d["tracker_args"]["fh"] = msvcrt.get_osfhandle( - _resource_tracker._fd.fileno() - ) + d["tracker_args"]["fh"] = msvcrt.get_osfhandle(_resource_tracker._fd) else: d["tracker_args"]["fd"] = _resource_tracker._fd @@ -284,7 +282,7 @@ def main(pipe_handle, parent_pid, process_name=None): handle, parent_sentinel = duplicate_in_child_process( pipe_handle, parent_pid ) - fd = os.fdopen(msvcrt.open_osfhandle(handle, os.O_RDONLY), mode="rb") + fd = msvcrt.open_osfhandle(handle, os.O_RDONLY) else: fd = pipe_handle parent_sentinel = os.dup(pipe_handle) From 4281a2dc2b9f8caba121c64489cf445547bb30a5 Mon Sep 17 00:00:00 2001 From: tommoral Date: Tue, 28 Feb 2023 00:30:46 +0100 Subject: [PATCH 43/50] FIX handles in resource_tracker --- loky/backend/resource_tracker.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/loky/backend/resource_tracker.py b/loky/backend/resource_tracker.py index 01ceadd3..ff4cc2f9 100644 --- a/loky/backend/resource_tracker.py +++ b/loky/backend/resource_tracker.py @@ -116,13 +116,11 @@ def ensure_running(self): "leak." ) + r, w = os.pipe() + fds_to_pass = [r] if sys.platform == "win32": - r, whandle = _winapi.CreatePipe(None, 0) - w = msvcrt.open_osfhandle(whandle, 0) - fds_to_pass = [r] + r = msvcrt.get_osfhandle(r) else: - r, w = os.pipe() - fds_to_pass = [r] try: fds_to_pass.append(sys.stderr.fileno()) except Exception: From 7f24a18a3e49fdf3dd34fb84988cec7ae6f424ed Mon Sep 17 00:00:00 2001 From: tommoral Date: Tue, 28 Feb 2023 00:55:05 +0100 Subject: [PATCH 44/50] FIX duplicate resource_tracker pipe.r+debug logs --- loky/backend/resource_tracker.py | 24 +++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/loky/backend/resource_tracker.py b/loky/backend/resource_tracker.py index ff4cc2f9..0d4e7626 100644 --- a/loky/backend/resource_tracker.py +++ b/loky/backend/resource_tracker.py @@ -57,6 +57,7 @@ if sys.platform == "win32": import _winapi import msvcrt + from .spawn import duplicate from .spawn import duplicate_in_child_process @@ -119,7 +120,9 @@ def ensure_running(self): r, w = os.pipe() fds_to_pass = [r] if sys.platform == "win32": - r = msvcrt.get_osfhandle(r) + _r = duplicate(msvcrt.get_osfhandle(r)) + os.close(r) + r = _r else: try: fds_to_pass.append(sys.stderr.fileno()) @@ -213,10 +216,17 @@ def main(pipe_handle, parent_pid, verbose=0): pipe_handle, parent_pid = int(pipe_handle), int(parent_pid) verbose = int(verbose) if sys.platform == "win32": - handle, parent_sentinel = duplicate_in_child_process( - pipe_handle, parent_pid - ) - fd = msvcrt.open_osfhandle(handle, os.O_RDONLY) + try: + handle, parent_sentinel = duplicate_in_child_process( + pipe_handle, parent_pid + ) + fd = msvcrt.open_osfhandle(handle, os.O_RDONLY) + except BaseException: + print("ERRORRR") + import traceback + + traceback.print_exc() + raise else: fd = pipe_handle @@ -394,5 +404,9 @@ def spawnv_passfds(cmd, passfds): _winapi.CloseHandle(ht) except BaseException: _winapi.CloseHandle(passfds[0]) + + import traceback + + traceback.print_exc() pass return pid From c7d914221d1a300c34b040ab964c0202e4649e2f Mon Sep 17 00:00:00 2001 From: tommoral Date: Tue, 28 Feb 2023 07:56:01 +0100 Subject: [PATCH 45/50] FIX working implem for win32 resource tracker --- loky/backend/popen_loky_win32.py | 2 +- loky/backend/resource_tracker.py | 26 +++++++------------------- 2 files changed, 8 insertions(+), 20 deletions(-) diff --git a/loky/backend/popen_loky_win32.py b/loky/backend/popen_loky_win32.py index f236ae35..329cb96e 100644 --- a/loky/backend/popen_loky_win32.py +++ b/loky/backend/popen_loky_win32.py @@ -68,7 +68,7 @@ def __init__(self, process_obj): child_env["__PYVENV_LAUNCHER__"] = sys.executable try: - with open(wfd, "wb", closefd=True) as to_child: + with os.fdopen(wfd, "wb", closefd=True) as to_child: # start process try: hp, ht, pid, _ = _winapi.CreateProcess( diff --git a/loky/backend/resource_tracker.py b/loky/backend/resource_tracker.py index 0d4e7626..12dbb8cc 100644 --- a/loky/backend/resource_tracker.py +++ b/loky/backend/resource_tracker.py @@ -120,9 +120,9 @@ def ensure_running(self): r, w = os.pipe() fds_to_pass = [r] if sys.platform == "win32": - _r = duplicate(msvcrt.get_osfhandle(r)) + _r = duplicate(msvcrt.get_osfhandle(r), inheritable=True) os.close(r) - r = _r + fds_to_pass[0] = r = _r else: try: fds_to_pass.append(sys.stderr.fileno()) @@ -216,17 +216,10 @@ def main(pipe_handle, parent_pid, verbose=0): pipe_handle, parent_pid = int(pipe_handle), int(parent_pid) verbose = int(verbose) if sys.platform == "win32": - try: - handle, parent_sentinel = duplicate_in_child_process( - pipe_handle, parent_pid - ) - fd = msvcrt.open_osfhandle(handle, os.O_RDONLY) - except BaseException: - print("ERRORRR") - import traceback - - traceback.print_exc() - raise + # handle, parent_sentinel = duplicate_in_child_process( + # pipe_handle, parent_pid + # ) + fd = msvcrt.open_osfhandle(pipe_handle, os.O_RDONLY) else: fd = pipe_handle @@ -399,14 +392,9 @@ def spawnv_passfds(cmd, passfds): cmd = " ".join(f'"{x}"' for x in cmd) try: _, ht, pid, _ = _winapi.CreateProcess( - exe, cmd, None, None, False, 0, None, None, None + exe, cmd, None, None, True, 0, None, None, None ) _winapi.CloseHandle(ht) except BaseException: _winapi.CloseHandle(passfds[0]) - - import traceback - - traceback.print_exc() - pass return pid From f8b3d8bf44a0ec80f95b9e18801c7d588e5de10d Mon Sep 17 00:00:00 2001 From: tommoral Date: Tue, 28 Feb 2023 09:58:18 +0100 Subject: [PATCH 46/50] FIX working win32 with no inheritance --- loky/backend/resource_tracker.py | 28 +++++++++++++--------------- 1 file changed, 13 insertions(+), 15 deletions(-) diff --git a/loky/backend/resource_tracker.py b/loky/backend/resource_tracker.py index 12dbb8cc..b3c7c554 100644 --- a/loky/backend/resource_tracker.py +++ b/loky/backend/resource_tracker.py @@ -117,13 +117,13 @@ def ensure_running(self): "leak." ) - r, w = os.pipe() - fds_to_pass = [r] if sys.platform == "win32": - _r = duplicate(msvcrt.get_osfhandle(r), inheritable=True) - os.close(r) - fds_to_pass[0] = r = _r + r, whandle = _winapi.CreatePipe(None, 0) + w = msvcrt.open_osfhandle(whandle, 0) + fds_to_pass = [r] else: + r, w = os.pipe() + fds_to_pass = [r] try: fds_to_pass.append(sys.stderr.fileno()) except Exception: @@ -162,9 +162,7 @@ def ensure_running(self): self._fd = w self._pid = pid finally: - if sys.platform == "win32": - _winapi.CloseHandle(r) - else: + if sys.platform != "win32": os.close(r) def _check_alive(self): @@ -216,10 +214,10 @@ def main(pipe_handle, parent_pid, verbose=0): pipe_handle, parent_pid = int(pipe_handle), int(parent_pid) verbose = int(verbose) if sys.platform == "win32": - # handle, parent_sentinel = duplicate_in_child_process( - # pipe_handle, parent_pid - # ) - fd = msvcrt.open_osfhandle(pipe_handle, os.O_RDONLY) + handle, parent_sentinel = duplicate_in_child_process( + pipe_handle, parent_pid + ) + fd = msvcrt.open_osfhandle(handle, os.O_RDONLY) else: fd = pipe_handle @@ -383,7 +381,7 @@ def spawnv_passfds(cmd, passfds): from .fork_exec import fork_exec _pass = [_mk_inheritable(fd) for fd in passfds] - return fork_exec(cmd, _pass) + return fork_exec(cmd, passfds) finally: os.close(errpipe_read) os.close(errpipe_write) @@ -392,9 +390,9 @@ def spawnv_passfds(cmd, passfds): cmd = " ".join(f'"{x}"' for x in cmd) try: _, ht, pid, _ = _winapi.CreateProcess( - exe, cmd, None, None, True, 0, None, None, None + exe, cmd, None, None, False, 0, None, None, None ) _winapi.CloseHandle(ht) + return pid except BaseException: _winapi.CloseHandle(passfds[0]) - return pid From 8e4a3ca2d0035900ea1874738fd2987129f258ce Mon Sep 17 00:00:00 2001 From: tommoral Date: Tue, 28 Feb 2023 17:45:46 +0100 Subject: [PATCH 47/50] CLN simplify fork_exec calls+add way to get workers in separate consoles --- loky/backend/fork_exec.py | 3 +++ loky/backend/popen_loky_posix.py | 12 +++++++----- loky/backend/popen_loky_win32.py | 8 +++++++- loky/backend/resource_tracker.py | 13 +++---------- loky/backend/spawn.py | 6 ++++++ 5 files changed, 26 insertions(+), 16 deletions(-) diff --git a/loky/backend/fork_exec.py b/loky/backend/fork_exec.py index 333be6f5..bfb38a8e 100644 --- a/loky/backend/fork_exec.py +++ b/loky/backend/fork_exec.py @@ -35,6 +35,9 @@ def fork_exec(cmd, keep_fds, env=None): env = env or {} child_env = {**os.environ, **env} + # make sure fds are inheritable + [os.set_inheritable(fd, True) for fd in keep_fds] + pid = os.fork() if pid == 0: # pragma: no cover close_fds(keep_fds) diff --git a/loky/backend/popen_loky_posix.py b/loky/backend/popen_loky_posix.py index 44975559..1f312d20 100644 --- a/loky/backend/popen_loky_posix.py +++ b/loky/backend/popen_loky_posix.py @@ -61,7 +61,7 @@ def __init__(self, process_obj): def duplicate_for_child(self, fd): self._fds.append(fd) - return reduction._mk_inheritable(fd) + return fd def poll(self, flag=os.WNOHANG): if self.returncode is None: @@ -121,9 +121,6 @@ def _launch(self, process_obj): parent_r, child_w = os.pipe() child_r, parent_w = os.pipe() - reduction._mk_inheritable(child_r) - reduction._mk_inheritable(child_w) - reduction._mk_inheritable(tracker_fd) cmd_python = spawn.get_command_line( pipe_handle=child_r, parent_pid=os.getpid(), @@ -140,14 +137,19 @@ def _launch(self, process_obj): util.debug( f"launched python with pid {pid} and cmd:\n{cmd_python}" ) - self.sentinel = parent_r + # Write the preparation data in the queue in a backward compatible + # way. + # XXX: can this be simplify now that we only support python3.7+ method = "getbuffer" if not hasattr(fp, method): method = "getvalue" with os.fdopen(parent_w, "wb", closefd=False) as f: f.write(getattr(fp, method)()) + + # Store the process's information self.pid = pid + self.sentinel = parent_r finally: fds_to_close = [] for fd in (parent_r, parent_w): diff --git a/loky/backend/popen_loky_win32.py b/loky/backend/popen_loky_win32.py index 329cb96e..7193dba4 100644 --- a/loky/backend/popen_loky_win32.py +++ b/loky/backend/popen_loky_win32.py @@ -2,6 +2,7 @@ import sys import msvcrt import _winapi + from multiprocessing import util from multiprocessing.context import set_spawning_popen from multiprocessing.popen_spawn_win32 import _close_handles @@ -12,6 +13,11 @@ __all__ = ["Popen"] +POPEN_FLAG = 0 +if spawn.OPEN_CONSOLE_FOR_SUBPROCESSES: + POPEN_FLAG = _winapi.CREATE_NEW_CONSOLE + + # # # @@ -77,7 +83,7 @@ def __init__(self, process_obj): None, None, False, - 0, + POPEN_FLAG, child_env, None, None, diff --git a/loky/backend/resource_tracker.py b/loky/backend/resource_tracker.py index b3c7c554..f5644a22 100644 --- a/loky/backend/resource_tracker.py +++ b/loky/backend/resource_tracker.py @@ -375,16 +375,9 @@ def spawnv_passfds(cmd, passfds): """ passfds = sorted(passfds) if sys.platform != "win32": - errpipe_read, errpipe_write = os.pipe() - try: - from .reduction import _mk_inheritable - from .fork_exec import fork_exec - - _pass = [_mk_inheritable(fd) for fd in passfds] - return fork_exec(cmd, passfds) - finally: - os.close(errpipe_read) - os.close(errpipe_write) + from .fork_exec import fork_exec + + return fork_exec(cmd, passfds) else: exe = cmd[0] cmd = " ".join(f'"{x}"' for x in cmd) diff --git a/loky/backend/spawn.py b/loky/backend/spawn.py index 0f1656be..eac88a43 100644 --- a/loky/backend/spawn.py +++ b/loky/backend/spawn.py @@ -18,6 +18,12 @@ from multiprocessing import freeze_support as _freeze_support_mp +# If set to True, the child process will open a console that can be used to +# get access to debugger. This is useful for debugging the child process +# step-by-step. +OPEN_CONSOLE_FOR_SUBPROCESSES = False + + if sys.platform != "win32": WINEXE = False WINSERVICE = False From 159a52b6e2a35f4088c2a5276eac1d108570ba66 Mon Sep 17 00:00:00 2001 From: tommoral Date: Thu, 2 Mar 2023 14:40:23 +0100 Subject: [PATCH 48/50] CI trigger From b63bec37e5f0a5848daacf421159f0d5ba626e3c Mon Sep 17 00:00:00 2001 From: tommoral Date: Sun, 9 Apr 2023 16:26:57 +0200 Subject: [PATCH 49/50] CI trigger From 5bfd31e02a239564d70ec73d7c4a8469634181e4 Mon Sep 17 00:00:00 2001 From: tommoral Date: Fri, 14 Apr 2023 12:49:02 +0200 Subject: [PATCH 50/50] FIX bad merge --- loky/backend/popen_loky_win32.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/loky/backend/popen_loky_win32.py b/loky/backend/popen_loky_win32.py index aa7f56c0..26027232 100644 --- a/loky/backend/popen_loky_win32.py +++ b/loky/backend/popen_loky_win32.py @@ -76,7 +76,6 @@ def __init__(self, process_obj): pipe_handle=rhandle, parent_pid=os.getpid() ) python_exe = cmd[0] - cmd = " ".join(f'"{x}"' for x in cmd) # copy the environment variables to set in the child process child_env = {**os.environ, **process_obj.env} @@ -87,9 +86,7 @@ def __init__(self, process_obj): cmd[0] = python_exe = sys._base_executable child_env["__PYVENV_LAUNCHER__"] = sys.executable - cmd = " ".join(f'"{x}"' for x in cmd) - with open(wfd, "wb") as to_child: # start process try: