Parallel build (#1741)

.ci/travis/run.sh

@@ -22,7 +22,7 @@ python setup.py develop
 if [[ $PYVER == '2.7' ]] && [[ "$(uname -s)" != 'Darwin' ]]; then
     PSUTIL_TESTING=1 python -Wa -m coverage run psutil/tests/runner.py
 else
-    PSUTIL_TESTING=1 python -Wa psutil/tests/runner.py --parallel
+    PSUTIL_TESTING=1 python -Wa psutil/tests/runner.py
 fi
 
 if [ "$PYVER" == "2.7" ] || [ "$PYVER" == "3.6" ]; then

.cirrus.yml

@@ -9,7 +9,7 @@ freebsd_13_py3_task:
     - python3 -m pip install --user setuptools concurrencytest
     - make clean
     - make install
-    - make test-parallel
+    - make test
     - make test-memleaks
     - make print-access-denied
     - make print-api-speed
@@ -25,7 +25,7 @@ freebsd_11_py2_task:
     - python2.7 -m pip install --user setuptools ipaddress mock concurrencytest
     - make clean
     - make install
-    - make test-parallel
+    - make test
     - make test-memleaks
     - make print-access-denied
     - make print-api-speed

HISTORY.rst

@@ -10,6 +10,8 @@ XXXX-XX-XX
 - 1729_: parallel tests on UNIX (make test-parallel).
 - 1736_: psutil.Popen now inherits from subprocess.Popen instead of
   psutil.Process. Also, wait(timeout=...) parameter is backported to Python 2.7.
+- 1741_: "make build/install" is now run in parallel and it's about 15% faster
+  on UNIX.
 
 **Bug fixes**
 

Makefile

@@ -26,6 +26,12 @@ PY2_DEPS = \
 	unittest2
 DEPS += `$(PYTHON) -c \
 	"import sys; print('$(PY2_DEPS)' if sys.version_info[0] == 2 else '')"`
+# "python3 setup.py build" can be parallelized on Python >= 3.6.
+BUILD_OPTS = `$(PYTHON) -c \
+	"import sys, os; \
+	py36 = sys.version_info[:2] >= (3, 6); \
+	cpus = os.cpu_count() or 1 if py36 else 1; \
+	print('--parallel %s' % cpus if cpus > 1 else '')"`
 # In not in a virtualenv, add --user options for install commands.
 INSTALL_OPTS = `$(PYTHON) -c \
 	"import sys; print('' if hasattr(sys, 'real_prefix') else '--user')"`
@@ -62,14 +68,13 @@ clean:  ## Remove all build files.
 
 _:
 
-build: _  ## Compile without installing.
+build: _  ## Compile (in parallel) without installing.
 	# make sure setuptools is installed (needed for 'develop' / edit mode)
 	$(PYTHON) -c "import setuptools"
-	PYTHONWARNINGS=all $(PYTHON) setup.py build
-	@# copies compiled *.so files in ./psutil directory in order to allow
-	@# "import psutil" when using the interactive interpreter from within
-	@# this directory.
-	PYTHONWARNINGS=all $(PYTHON) setup.py build_ext -i
+	@# "build_ext -i" copies compiled *.so files in ./psutil directory in order
+	@# to allow "import psutil" when using the interactive interpreter from
+	@# within  this directory.
+	PYTHONWARNINGS=all $(PYTHON) setup.py build_ext -i $(BUILD_OPTS)
 	$(PYTHON) -c "import psutil"  # make sure it actually worked
 
 install:  ## Install this package as current user in "edit" mode.

psutil/tests/test_testutils.py

@@ -267,11 +267,12 @@ def test_terminate(self):
         assert not psutil.pid_exists(pid)
         terminate(pid)
         # zombie
-        parent, zombie = self.create_zombie_proc()
-        terminate(parent)
-        terminate(zombie)
-        assert not psutil.pid_exists(parent.pid)
-        assert not psutil.pid_exists(zombie.pid)
+        if POSIX:
+            parent, zombie = self.create_zombie_proc()
+            terminate(parent)
+            terminate(zombie)
+            assert not psutil.pid_exists(parent.pid)
+            assert not psutil.pid_exists(zombie.pid)
 
 
 class TestNetUtils(unittest.TestCase):

scripts/internal/winmake.py

@@ -227,8 +227,13 @@ def build():
     # edit mode).
     sh('%s -c "import setuptools"' % PYTHON)
 
+    # "build_ext -i" copies compiled *.pyd files in ./psutil directory in
+    # order to allow "import psutil" when using the interactive interpreter
+    # from within psutil root directory.
+    cmd = [PYTHON, "setup.py", "build_ext", "-i"]
+    if sys.version_info[:2] >= (3, 6) and os.cpu_count() or 1 > 1:
+        cmd += ['--parallel', str(os.cpu_count())]
     # Print coloured warnings in real time.
-    cmd = [PYTHON, "setup.py", "build"]
     p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
     try:
         for line in iter(p.stdout.readline, b''):
@@ -250,10 +255,6 @@ def build():
         p.terminate()
         p.wait()
 
-    # Copies compiled *.pyd files in ./psutil directory in order to
-    # allow "import psutil" when using the interactive interpreter
-    # from within this directory.
-    sh("%s setup.py build_ext -i" % PYTHON)
     # Make sure it actually worked.
     sh('%s -c "import psutil"' % PYTHON)
     win_colorprint("build + import successful", GREEN)