From f88511cc18e5b2e56052f98cc78f51051d4dcfe7 Mon Sep 17 00:00:00 2001
From: Christian Heimes <christian@python.org>
Date: Fri, 10 Jun 2022 21:10:16 +0200
Subject: [PATCH 1/3] gh-84461: Use HOSTRUNNER to run regression tests
---
...2-06-10-21-18-14.gh-issue-84461.9TAb26.rst | 2 +
Tools/scripts/run_tests.py | 45 +++++++++++++++++--
2 files changed, 43 insertions(+), 4 deletions(-)
create mode 100644 Misc/NEWS.d/next/Tests/2022-06-10-21-18-14.gh-issue-84461.9TAb26.rst
diff --git a/Misc/NEWS.d/next/Tests/2022-06-10-21-18-14.gh-issue-84461.9TAb26.rst b/Misc/NEWS.d/next/Tests/2022-06-10-21-18-14.gh-issue-84461.9TAb26.rst
new file mode 100644
index 00000000000000..7cdf5bee721870
--- /dev/null
+++ b/Misc/NEWS.d/next/Tests/2022-06-10-21-18-14.gh-issue-84461.9TAb26.rst
@@ -0,0 +1,2 @@
+``run_tests.py`` now handles cross compiling env vars correctly and pass
+``HOSTRUNNER`` to regression tests.
diff --git a/Tools/scripts/run_tests.py b/Tools/scripts/run_tests.py
index 48feb3f778ee8d..b76f1ae58d1ae0 100644
--- a/Tools/scripts/run_tests.py
+++ b/Tools/scripts/run_tests.py
@@ -8,7 +8,9 @@
"""
import os
+import shlex
import sys
+import sysconfig
import test.support
@@ -19,15 +21,36 @@ def is_multiprocess_flag(arg):
def is_resource_use_flag(arg):
return arg.startswith('-u') or arg.startswith('--use')
+def is_python_flag(arg):
+ return arg.startswith('-p') or arg.startswith('--python')
+
def main(regrtest_args):
args = [sys.executable,
'-u', # Unbuffered stdout and stderr
'-W', 'default', # Warnings set to 'default'
'-bb', # Warnings about bytes/bytearray
- '-E', # Ignore environment variables
]
+ cross_compile = '_PYTHON_HOST_PLATFORM' in os.environ
+ hostrunner = sysconfig.get_config_var("HOSTRUNNER")
+ if cross_compile:
+ # emulate -E, but keep PYTHONPATH + cross compile env vars, so
+ # test executable can load correct sysconfigdata file.
+ keep = {
+ '_PYTHON_PROJECT_BASE',
+ '_PYTHON_HOST_PLATFORM',
+ '_PYTHON_SYSCONFIGDATA_NAME',
+ 'PYTHONPATH'
+ }
+ environ = {
+ name: value for name, value in os.environ.items()
+ if not name.startswith(('PYTHON', '_PYTHON')) or name in keep
+ }
+ else:
+ environ = os.environ.copy()
+ args.append("-E")
+
# Allow user-specified interpreter options to override our defaults.
args.extend(test.support.args_from_interpreter_flags())
@@ -38,16 +61,30 @@ def main(regrtest_args):
if sys.platform == 'win32':
args.append('-n') # Silence alerts under Windows
if not any(is_multiprocess_flag(arg) for arg in regrtest_args):
- args.extend(['-j', '0']) # Use all CPU cores
+ if cross_compile and hostrunner:
+ # For now use only one core for cross compiled builds.
+ # hostrunner can be expensive.
+ args.extend(['-j', '1'])
+ else:
+ args.extend(['-j', '0']) # Use all CPU cores
if not any(is_resource_use_flag(arg) for arg in regrtest_args):
args.extend(['-u', 'all,-largefile,-audio,-gui'])
+
+ if cross_compile and hostrunner:
+ # If HOSTRUNNER is set and -p/--python option is not given, then
+ # use hostrunner to execute python binary for tests.
+ if not any(is_python_flag(arg) for arg in regrtest_args):
+ buildpython = sysconfig.get_config_var("BUILDPYTHON")
+ args.extend(["--python", f"{hostrunner} {buildpython}"])
+
args.extend(regrtest_args)
- print(' '.join(args))
+
+ print(shlex.join(args))
if sys.platform == 'win32':
from subprocess import call
sys.exit(call(args))
else:
- os.execv(sys.executable, args)
+ os.execve(sys.executable, args, environ)
if __name__ == '__main__':
From bbca5c258db3d74c93fce6acc7262d6d96d6d067 Mon Sep 17 00:00:00 2001
From: Christian Heimes <christian@python.org>
Date: Fri, 10 Jun 2022 22:40:04 +0200
Subject: [PATCH 2/3] Pass hostrunner as env var so shell expressions are
evaluated
---
Tools/scripts/run_tests.py | 4 +++-
configure | 6 +++++-
configure.ac | 7 ++++++-
3 files changed, 14 insertions(+), 3 deletions(-)
diff --git a/Tools/scripts/run_tests.py b/Tools/scripts/run_tests.py
index b76f1ae58d1ae0..6e57fe855b1d96 100644
--- a/Tools/scripts/run_tests.py
+++ b/Tools/scripts/run_tests.py
@@ -33,7 +33,9 @@ def main(regrtest_args):
]
cross_compile = '_PYTHON_HOST_PLATFORM' in os.environ
- hostrunner = sysconfig.get_config_var("HOSTRUNNER")
+ hostrunner = os.environ.get("_PYTHON_HOSTRUNNER")
+ if hostrunner is None:
+ hostrunner = sysconfig.get_config_var("HOSTRUNNER")
if cross_compile:
# emulate -E, but keep PYTHONPATH + cross compile env vars, so
# test executable can load correct sysconfigdata file.
diff --git a/configure b/configure
index 677de64741128b..fb510267b69fec 100755
--- a/configure
+++ b/configure
@@ -6782,7 +6782,7 @@ else
fi
;; #(
WASI/*) :
- HOSTRUNNER='wasmtime run --env PYTHONPATH=$$(realpath --relative-to $(abs_srcdir) $(abs_builddir))/$$(cat pybuilddir.txt) --mapdir /::$(srcdir) --' ;; #(
+ HOSTRUNNER='wasmtime run --env PYTHONPATH=/$(shell realpath --relative-to $(abs_srcdir) $(abs_builddir))/$(shell cat pybuilddir.txt) --mapdir /::$(srcdir) --' ;; #(
*) :
HOSTRUNNER=''
;;
@@ -6792,6 +6792,10 @@ fi
{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $HOSTRUNNER" >&5
$as_echo "$HOSTRUNNER" >&6; }
+if test -n "$HOSTRUNNER"; then
+ PYTHON_FOR_BUILD="_PYTHON_HOSTRUNNER='$HOSTRUNNER' $PYTHON_FOR_BUILD"
+fi
+
{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $LDLIBRARY" >&5
$as_echo "$LDLIBRARY" >&6; }
diff --git a/configure.ac b/configure.ac
index af437b6d401f8f..ca1b2528f88787 100644
--- a/configure.ac
+++ b/configure.ac
@@ -1492,13 +1492,18 @@ then
dnl TODO: support other WASI runtimes
dnl wasmtime starts the proces with "/" as CWD. For OOT builds add the
dnl directory containing _sysconfigdata to PYTHONPATH.
- [WASI/*], [HOSTRUNNER='wasmtime run --env PYTHONPATH=$$(realpath --relative-to $(abs_srcdir) $(abs_builddir))/$$(cat pybuilddir.txt) --mapdir /::$(srcdir) --'],
+ [WASI/*], [HOSTRUNNER='wasmtime run --env PYTHONPATH=/$(shell realpath --relative-to $(abs_srcdir) $(abs_builddir))/$(shell cat pybuilddir.txt) --mapdir /::$(srcdir) --'],
[HOSTRUNNER='']
)
fi
AC_SUBST([HOSTRUNNER])
AC_MSG_RESULT([$HOSTRUNNER])
+if test -n "$HOSTRUNNER"; then
+ dnl Pass hostrunner variable as env var in order to expand shell expressions.
+ PYTHON_FOR_BUILD="_PYTHON_HOSTRUNNER='$HOSTRUNNER' $PYTHON_FOR_BUILD"
+fi
+
AC_MSG_RESULT($LDLIBRARY)
# LIBRARY_DEPS, LINK_PYTHON_OBJS and LINK_PYTHON_DEPS variable
From 3dba7a61cb160fcb8820e8b42d50163e6ca58f1e Mon Sep 17 00:00:00 2001
From: Christian Heimes <christian@python.org>
Date: Fri, 10 Jun 2022 23:49:40 +0200
Subject: [PATCH 3/3] Apply suggestions from code review
Co-authored-by: Brett Cannon <brett@python.org>
---
Tools/scripts/run_tests.py | 5 ++---
1 file changed, 2 insertions(+), 3 deletions(-)
diff --git a/Tools/scripts/run_tests.py b/Tools/scripts/run_tests.py
index 6e57fe855b1d96..8dbcade9238551 100644
--- a/Tools/scripts/run_tests.py
+++ b/Tools/scripts/run_tests.py
@@ -33,8 +33,7 @@ def main(regrtest_args):
]
cross_compile = '_PYTHON_HOST_PLATFORM' in os.environ
- hostrunner = os.environ.get("_PYTHON_HOSTRUNNER")
- if hostrunner is None:
+ if (hostrunner := os.environ.get("_PYTHON_HOSTRUNNER")) is None:
hostrunner = sysconfig.get_config_var("HOSTRUNNER")
if cross_compile:
# emulate -E, but keep PYTHONPATH + cross compile env vars, so
@@ -64,7 +63,7 @@ def main(regrtest_args):
args.append('-n') # Silence alerts under Windows
if not any(is_multiprocess_flag(arg) for arg in regrtest_args):
if cross_compile and hostrunner:
- # For now use only one core for cross compiled builds.
+ # For now use only one core for cross-compiled builds;
# hostrunner can be expensive.
args.extend(['-j', '1'])
else: