gh-93353: regrtest creates worker directory in the parent

Lib/test/libregrtest/main.py

@@ -605,9 +605,9 @@ def set_temp_dir(self):
  self.tmp_dir = self.ns.tempdir
 
  if not self.tmp_dir:
- # When tests are run from the Python build directory, it is best practice
- # to keep the test files in a subfolder. This eases the cleanup of leftover
- # files using the "make distclean" command.
+ # When tests are run from the Python build directory, it is best
+ # practice to keep the test files in a subfolder. This eases the
+ # cleanup of leftover files using the "make distclean" command.
  if sysconfig.is_python_build():
  self.tmp_dir = sysconfig.get_config_var('abs_builddir')
  if self.tmp_dir is None:
@@ -634,11 +634,7 @@ def create_temp_dir(self):
  nounce = random.randint(0, 1_000_000)
  else:
  nounce = os.getpid()
- if self.worker_test_name is not None:
- test_cwd = 'test_python_worker_{}'.format(nounce)
- else:
- test_cwd = 'test_python_{}'.format(nounce)
- test_cwd += os_helper.FS_NONASCII
+ test_cwd = f'test_python_{nounce}_{os_helper.FS_NONASCII}'
  test_cwd = os.path.join(self.tmp_dir, test_cwd)
  return test_cwd
 
@@ -658,25 +654,26 @@ def cleanup(self):
  def main(self, tests=None, **kwargs):
  self.parse_args(kwargs)
 
- self.set_temp_dir()
+ if self.worker_test_name is None:
+ self.set_temp_dir()
+ else:
+ self.tmp_dir = os.getcwd()
 
  if self.ns.cleanup:
  self.cleanup()
  sys.exit(0)
 
- test_cwd = self.create_temp_dir()
-
  try:
- # Run the tests in a context manager that temporarily changes the CWD
- # to a temporary and writable directory. If it's not possible to
- # create or change the CWD, the original CWD will be used.
+ # Run the tests in a context manager that temporarily changes the
+ # CWD to a temporary and writable directory. If it's not possible
+ # to create or change the CWD, the original CWD will be used.
  # The original CWD is available from os_helper.SAVEDCWD.
- with os_helper.temp_cwd(test_cwd, quiet=True):
- # When using multiprocessing, worker processes will use test_cwd
- # as their parent temporary directory. So when the main process
- # exit, it removes also subdirectories of worker processes.
-  self.ns.tempdir = test_cwd
-
+ if self.worker_test_name is None:
+ test_cwd = self.create_temp_dir()
+ with os_helper.temp_cwd(test_cwd, quiet=True):
+  self._main(tests, kwargs)
+ else:
+ # Worker process
  self._main(tests, kwargs)
  except SystemExit as exc:
  # bpo-38203: Python can hang at exit in Py_Finalize(), especially

Lib/test/libregrtest/runtest_mp.py

@@ -2,6 +2,7 @@
 import json
 import os.path
 import queue
+import random
 import shlex
 import signal
 import subprocess
@@ -53,7 +54,8 @@ def parse_worker_args(worker_args) -> tuple[Namespace, str]:
  return (ns, test_name)
 
 
-def run_test_in_subprocess(testname: str, ns: Namespace, tmp_dir: str) -> subprocess.Popen:
+def run_test_in_subprocess(testname: str, ns: Namespace,
+ work_dir: str, tmp_dir: str) -> subprocess.Popen:
  ns_dict = vars(ns)
  worker_args = (ns_dict, testname)
  worker_args = json.dumps(worker_args)
@@ -86,7 +88,7 @@ def run_test_in_subprocess(testname: str, ns: Namespace, tmp_dir: str) -> subpro
  stderr=subprocess.STDOUT,
  universal_newlines=True,
  close_fds=(os.name != 'nt'),
- cwd=os_helper.SAVEDCWD,
+ cwd=work_dir,
  **kw)
 
 
@@ -213,12 +215,12 @@ def mp_result_error(
  test_result.duration_sec = time.monotonic() - self.start_time
  return MultiprocessResult(test_result, stdout, err_msg)
 
- def _run_process(self, test_name: str, tmp_dir: str) -> tuple[int, str, str]:
+ def _run_process(self, test_name: str, work_dir: str, tmp_dir: str) -> tuple[int, str, str]:
  self.start_time = time.monotonic()
 
  self.current_test_name = test_name
  try:
- popen = run_test_in_subprocess(test_name, self.ns, tmp_dir)
+ popen = run_test_in_subprocess(test_name, self.ns, work_dir, tmp_dir)
 
  self._killed = False
  self._popen = popen
@@ -272,22 +274,39 @@ def _run_process(self, test_name: str, tmp_dir: str) -> tuple[int, str, str]:
  self._popen = None
  self.current_test_name = None
 
+ def create_work_dir(self):
+ # Generate an unique directory name: 6 random decimal digits gives
+ # almost 20 bits of entropy. It's rare to run more than 100 processes
+ # in parallel.
+ nounce = random.randint(0, 1_000_000)
+ # create a non-ASCII name to detect issues with non-ASCII characters
+ path = f'test_python_worker_{nounce}_{os_helper.FS_NONASCII}'
+ return os.path.join(os.getcwd(), path)
+
  def _runtest(self, test_name: str) -> MultiprocessResult:
- if self.ns.use_mp == 1:
- # gh-93353: Check for leaked temporary files in the parent process,
- # since the deletion of temporary files can happen late during
- # Python finalization: too late for libregrtest.
- tmp_dir = os.getcwd() + '_tmpdir'
- tmp_dir = os.path.abspath(tmp_dir)
- try:
+ # The working directory of the worker process is a sub-directory of the
+ # main process. So when the main process exits, it removes also
+ # sub-directories of worker processes.
+ work_dir = self.create_work_dir()
+ tmp_dir = None
+ tmp_files = None
+ try:
+ os.mkdir(work_dir)
+ if not support.is_wasi:
+ tmp_dir = work_dir + '_tmpdir'
  os.mkdir(tmp_dir)
- retcode, stdout = self._run_process(test_name, tmp_dir)
- finally:
+
+ retcode, stdout = self._run_process(test_name, work_dir, tmp_dir)
+
+ if tmp_dir is not None:
+ # gh-93353: Check for leaked temporary files in the parent
+ # process, since the deletion of temporary files can happen
+ # late during Python finalization: too late for libregrtest.
  tmp_files = os.listdir(tmp_dir)
+ finally:
+ os_helper.rmtree(work_dir)
+ if tmp_dir is not None:
  os_helper.rmtree(tmp_dir)
- else:
- retcode, stdout = self._run_process(test_name, None)
- tmp_files = ()
 
  if retcode is None:
  return self.mp_result_error(Timeout(test_name), stdout)

Lib/test/test_regrtest.py

@@ -1357,6 +1357,8 @@ def test_cleanup(self):
  for name in names:
  self.assertFalse(os.path.exists(name), name)
 
+ @unittest.skipIf(support.is_wasi,
+ 'checking temp files is not implemented on WASI')
  def test_leak_tmp_file(self):
  code = textwrap.dedent(r"""
  import os.path
@@ -1369,15 +1371,17 @@ def test_leak_tmp_file(self):
  with open(filename, "wb") as fp:
  fp.write(b'content')
  """)
- testname = self.create_test(code=code)
+ testnames = [self.create_test(code=code) for _ in range(3)]
 
- output = self.run_tests("--fail-env-changed", "-v", "-j1", testname, exitcode=3)
- self.check_executed_tests(output, [testname],
- env_changed=[testname],
- fail_env_changed=True)
- self.assertIn(f"Warning -- {testname} leaked temporary "
- f"files (1): mytmpfile",
- output)
+ output = self.run_tests("--fail-env-changed", "-v", "-j2", *testnames, exitcode=3)
+ self.check_executed_tests(output, testnames,
+ env_changed=testnames,
+ fail_env_changed=True,
+ randomize=True)
+ for testname in testnames:
+ self.assertIn(f"Warning -- {testname} leaked temporary "
+ f"files (1): mytmpfile",
+ output)
 
 
 class TestUtils(unittest.TestCase):