bpo-38502: regrtest uses process groups if available (GH-16829)

test.regrtest now uses process groups in the multiprocessing mode
(-jN command line option) if process groups are available: if
os.setsid() and os.killpg() functions are available.

Author: vstinner

Lib/test/libregrtest/runtest_mp.py

@@ -3,6 +3,7 @@
 import json
 import os
 import queue
+import signal
 import subprocess
 import sys
 import threading
@@ -31,6 +32,8 @@
 # Time to wait until a worker completes: should be immediate
 JOIN_TIMEOUT = 30.0   # seconds
 
+USE_PROCESS_GROUP = (hasattr(os, "setsid") and hasattr(os, "killpg"))
+
 
 def must_stop(result, ns):
     if result.result == INTERRUPTED:
@@ -59,12 +62,16 @@ def run_test_in_subprocess(testname, ns):
     # Running the child from the same working directory as regrtest's original
     # invocation ensures that TEMPDIR for the child is the same when
     # sysconfig.is_python_build() is true. See issue 15300.
+    kw = {}
+    if USE_PROCESS_GROUP:
+        kw['start_new_session'] = True
     return subprocess.Popen(cmd,
                             stdout=subprocess.PIPE,
                             stderr=subprocess.PIPE,
                             universal_newlines=True,
                             close_fds=(os.name != 'nt'),
-                            cwd=support.SAVEDCWD)
+                            cwd=support.SAVEDCWD,
+                            **kw)
 
 
 def run_tests_worker(ns, test_name):
@@ -149,16 +156,24 @@ def _kill(self):
             return
         self._killed = True
 
-        print(f"Kill {self}", file=sys.stderr, flush=True)
+        if USE_PROCESS_GROUP:
+            what = f"{self} process group"
+        else:
+            what = f"{self}"
+
+        print(f"Kill {what}", file=sys.stderr, flush=True)
         try:
-            popen.kill()
+            if USE_PROCESS_GROUP:
+                os.killpg(popen.pid, signal.SIGKILL)
+            else:
+                popen.kill()
         except ProcessLookupError:
-            # Process completed, the TestWorkerProcess thread read its exit
-            # status, but Popen.send_signal() read the returncode just before
-            # Popen.wait() set returncode.
+            # popen.kill(): the process completed, the TestWorkerProcess thread
+            # read its exit status, but Popen.send_signal() read the returncode
+            # just before Popen.wait() set returncode.
             pass
         except OSError as exc:
-            print_warning(f"Failed to kill {self}: {exc!r}")
+            print_warning(f"Failed to kill {what}: {exc!r}")
 
     def stop(self):
         # Method called from a different thread to stop this thread

Misc/NEWS.d/next/Tests/2019-10-17-00-49-38.bpo-38502.vUEic7.rst

@@ -0,0 +1,3 @@
+test.regrtest now uses process groups in the multiprocessing mode (-jN command
+line option) if process groups are available: if :func:`os.setsid` and
+:func:`os.killpg` functions are available.