From c9a585e2d884bee78264bf757b6c501798cdadd1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bern=C3=A1t=20G=C3=A1bor?= Date: Mon, 20 Sep 2021 09:49:13 +0100 Subject: [PATCH 1/2] Add support for TOX_LIMITED_SHEBANG MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Bernát Gábor --- docs/changelog/2208.feature.rst | 2 + src/tox/execute/local_sub_process/__init__.py | 9 ++++- src/tox/execute/util.py | 30 ++++++++++++++ .../local_subprocess/test_execute_util.py | 26 +++++++++++++ .../local_subprocess/test_local_subprocess.py | 39 ++++++++++++++++++- whitelist.txt | 4 ++ 6 files changed, 106 insertions(+), 4 deletions(-) create mode 100644 docs/changelog/2208.feature.rst create mode 100644 src/tox/execute/util.py create mode 100644 tests/execute/local_subprocess/test_execute_util.py diff --git a/docs/changelog/2208.feature.rst b/docs/changelog/2208.feature.rst new file mode 100644 index 000000000..0c44fe414 --- /dev/null +++ b/docs/changelog/2208.feature.rst @@ -0,0 +1,2 @@ +Add support for rewriting script invocations that have valid shebang lines when the ``TOX_LIMITED_SHEBANG`` environment +variable is set and not empty - by :user:`gaborbernat`. diff --git a/src/tox/execute/local_sub_process/__init__.py b/src/tox/execute/local_sub_process/__init__.py index 6a8ac8516..f23e27d2c 100644 --- a/src/tox/execute/local_sub_process/__init__.py +++ b/src/tox/execute/local_sub_process/__init__.py @@ -13,6 +13,7 @@ from ..api import Execute, ExecuteInstance, ExecuteOptions, ExecuteStatus from ..request import ExecuteRequest, StdinSource from ..stream import SyncWrite +from ..util import shebang if sys.platform == "win32": # explicit check for mypy # pragma: win32 cover # needs stdin/stdout handlers backed by overlapped IO @@ -171,8 +172,12 @@ def cmd(self) -> Sequence[str]: else: msg = f"{base} (resolves to {executable})" if base == executable else base raise Fail(f"{msg} is not allowed, use allowlist_externals to allow it") - # else use expanded format - cmd = [executable, *self.request.cmd[1:]] + cmd = [executable] + if sys.platform != "win32" and self.request.env.get("TOX_LIMITED_SHEBANG", "").strip(): + shebang_line = shebang(executable) + if shebang_line: + cmd = [*shebang_line, executable] + cmd.extend(self.request.cmd[1:]) self._cmd = cmd return self._cmd diff --git a/src/tox/execute/util.py b/src/tox/execute/util.py new file mode 100644 index 000000000..0c10468e4 --- /dev/null +++ b/src/tox/execute/util.py @@ -0,0 +1,30 @@ +from typing import List, Optional + + +def shebang(exe: str) -> Optional[List[str]]: + """ + :param exe: the executable + :return: the shebang interpreter arguments + """ + # When invoking a command using a shebang line that exceeds the OS shebang limit (e.g. Linux has a limit of 128; + # BINPRM_BUF_SIZE) the invocation will fail. In this case you'd want to replace the shebang invocation with an + # explicit invocation. + # see https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git/tree/fs/binfmt_script.c#n34 + try: + with open(exe, "rb") as file_handler: + marker = file_handler.read(2) + if marker != b"#!": + return None + shebang_line = file_handler.readline() + except OSError: + return None + try: + decoded = shebang_line.decode("UTF-8") + except UnicodeDecodeError: + return None + return [i.strip() for i in decoded.strip().split() if i.strip()] + + +__all__ = [ + "shebang", +] diff --git a/tests/execute/local_subprocess/test_execute_util.py b/tests/execute/local_subprocess/test_execute_util.py new file mode 100644 index 000000000..87a46c3f1 --- /dev/null +++ b/tests/execute/local_subprocess/test_execute_util.py @@ -0,0 +1,26 @@ +from pathlib import Path + +from tox.execute.util import shebang + + +def test_shebang_found(tmp_path: Path) -> None: + script_path = tmp_path / "a" + script_path.write_text("#! /bin/python \t-c\t") + assert shebang(str(script_path)) == ["/bin/python", "-c"] + + +def test_shebang_file_missing(tmp_path: Path) -> None: + script_path = tmp_path / "a" + assert shebang(str(script_path)) is None + + +def test_shebang_no_shebang(tmp_path: Path) -> None: + script_path = tmp_path / "a" + script_path.write_bytes(b"magic") + assert shebang(str(script_path)) is None + + +def test_shebang_non_utf8_file(tmp_path: Path) -> None: + script_path, content = tmp_path / "a", b"#!" + bytearray.fromhex("c0") + script_path.write_bytes(content) + assert shebang(str(script_path)) is None diff --git a/tests/execute/local_subprocess/test_local_subprocess.py b/tests/execute/local_subprocess/test_local_subprocess.py index 551e5a285..02ff63fe3 100644 --- a/tests/execute/local_subprocess/test_local_subprocess.py +++ b/tests/execute/local_subprocess/test_local_subprocess.py @@ -1,12 +1,14 @@ import json import logging import os +import shutil +import stat import subprocess import sys from io import TextIOWrapper from pathlib import Path from typing import Dict, List, Tuple -from unittest.mock import MagicMock +from unittest.mock import MagicMock, create_autospec import psutil import pytest @@ -15,7 +17,7 @@ from psutil import AccessDenied from pytest_mock import MockerFixture -from tox.execute.api import Outcome +from tox.execute.api import ExecuteOptions, Outcome from tox.execute.local_sub_process import SIG_INTERRUPT, LocalSubProcessExecuteInstance, LocalSubProcessExecutor from tox.execute.request import ExecuteRequest, StdinSource from tox.execute.stream import SyncWrite @@ -298,3 +300,36 @@ def test_allow_list_external_ok(fake_exe_on_path: Path, mode: str) -> None: inst = LocalSubProcessExecuteInstance(request, MagicMock(), out=SyncWrite("out", None), err=SyncWrite("err", None)) assert inst.cmd == [exe] + + +def test_shebang_limited_on(tmp_path: Path) -> None: + exe, script, instance = _create_shebang_test(tmp_path, env={"TOX_LIMITED_SHEBANG": "1"}) + if sys.platform == "win32": # pragma: win32 cover + assert instance.cmd == [str(script), "--magic"] + else: + assert instance.cmd == [exe, "-s", str(script), "--magic"] + + +@pytest.mark.parametrize("env", [{}, {"TOX_LIMITED_SHEBANG": ""}]) +def test_shebang_limited_off(tmp_path: Path, env: Dict[str, str]) -> None: + _, script, instance = _create_shebang_test(tmp_path, env=env) + assert instance.cmd == [str(script), "--magic"] + + +def test_shebang_failed_to_parse(tmp_path: Path) -> None: + _, script, instance = _create_shebang_test(tmp_path, env={"TOX_LIMITED_SHEBANG": "yes"}) + script.write_text("") + assert instance.cmd == [str(script), "--magic"] + + +def _create_shebang_test(tmp_path: Path, env: Dict[str, str]) -> Tuple[str, Path, LocalSubProcessExecuteInstance]: + exe = shutil.which("python") + assert exe is not None + script = tmp_path / "s.py" + script.write_text(f"#!{exe} -s") + script.chmod(script.stat().st_mode | stat.S_IEXEC) # mark it executable + env["PATH"] = str(script.parent) + request = create_autospec(ExecuteRequest, cmd=["s.py", "--magic"], env=env, allow=None) + writer = create_autospec(SyncWrite) + instance = LocalSubProcessExecuteInstance(request, create_autospec(ExecuteOptions), writer, writer) + return exe, script, instance diff --git a/whitelist.txt b/whitelist.txt index 22d1fe85e..bf2808553 100644 --- a/whitelist.txt +++ b/whitelist.txt @@ -13,6 +13,8 @@ autoclass autodoc autosectionlabel autouse +binprm +buf bufsize byref cachetools @@ -81,6 +83,7 @@ fixup fmt formatter fromdocname +fromhex fromkeys fs fullmatch @@ -101,6 +104,7 @@ hookimpl hookspec hookspecs ident +iexec ign ignorecase impl From dc97d8790b3f212caa839399e6da1065c1013fb9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bern=C3=A1t=20G=C3=A1bor?= Date: Mon, 20 Sep 2021 10:20:04 +0100 Subject: [PATCH 2/2] Fix Windows MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Bernát Gábor --- tests/execute/local_subprocess/test_local_subprocess.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/execute/local_subprocess/test_local_subprocess.py b/tests/execute/local_subprocess/test_local_subprocess.py index 02ff63fe3..126a724b8 100644 --- a/tests/execute/local_subprocess/test_local_subprocess.py +++ b/tests/execute/local_subprocess/test_local_subprocess.py @@ -325,11 +325,11 @@ def test_shebang_failed_to_parse(tmp_path: Path) -> None: def _create_shebang_test(tmp_path: Path, env: Dict[str, str]) -> Tuple[str, Path, LocalSubProcessExecuteInstance]: exe = shutil.which("python") assert exe is not None - script = tmp_path / "s.py" + script = tmp_path / f"s{'.EXE' if sys.platform == 'win32' else ''}" script.write_text(f"#!{exe} -s") script.chmod(script.stat().st_mode | stat.S_IEXEC) # mark it executable env["PATH"] = str(script.parent) - request = create_autospec(ExecuteRequest, cmd=["s.py", "--magic"], env=env, allow=None) + request = create_autospec(ExecuteRequest, cmd=["s", "--magic"], env=env, allow=None) writer = create_autospec(SyncWrite) instance = LocalSubProcessExecuteInstance(request, create_autospec(ExecuteOptions), writer, writer) return exe, script, instance