Skip to content

Commit 4e73a3d

Browse files
committed
Fix EM_PYTHON_MULTIPROCESSING under windows
This mirrors the upstream bugfix in python 3.8.0 which limits the size of the worker pool to 61: python/cpython#13132 Fixes: #13785
1 parent 9227a01 commit 4e73a3d

File tree

1 file changed

+5
-1
lines changed

1 file changed

+5
-1
lines changed

tools/shared.py

+5-1
Original file line numberDiff line numberDiff line change
@@ -156,9 +156,13 @@ def run_multiple_processes(commands,
156156
# Use EM_PYTHON_MULTIPROCESSING=1 environment variable to enable it. It can be faster, but may not work on Windows.
157157
if int(os.getenv('EM_PYTHON_MULTIPROCESSING', '0')):
158158
import multiprocessing
159+
max_workers = get_num_cores()
159160
global multiprocessing_pool
160161
if not multiprocessing_pool:
161-
multiprocessing_pool = multiprocessing.Pool(processes=get_num_cores())
162+
if WINDOWS:
163+
# Fix for python < 3.8 on windows. See: https://github.com/python/cpython/pull/13132
164+
max_workers = min(max_workers, 61)
165+
multiprocessing_pool = multiprocessing.Pool(processes=max_workers)
162166
return multiprocessing_pool.map(mp_run_process, [(cmd, env, route_stdout_to_temp_files_suffix, pipe_stdout, check, cwd) for cmd in commands], chunksize=1)
163167

164168
std_outs = []

0 commit comments

Comments
 (0)