From e62bf8b5a56763a80816f9826e12d715e3ec2ecb Mon Sep 17 00:00:00 2001 From: Bernat Gabor Date: Mon, 14 Jan 2019 12:26:46 +0000 Subject: [PATCH 01/12] start rebuild --- src/tox/_pytestplugin.py | 154 ++-- src/tox/action.py | 188 ++++ src/tox/config/__init__.py | 37 +- src/tox/config/reporter.py | 20 + src/tox/exception.py | 3 +- src/tox/hookspecs.py | 5 + src/tox/logs/__init__.py | 4 + src/tox/logs/command.py | 14 + src/tox/logs/env.py | 46 + src/tox/logs/result.py | 43 + src/tox/package/__init__.py | 58 +- src/tox/package/builder/__init__.py | 6 +- src/tox/package/builder/isolated.py | 53 +- src/tox/package/builder/legacy.py | 17 +- src/tox/package/local.py | 64 ++ src/tox/package/view.py | 6 +- src/tox/reporter.py | 143 +++ src/tox/result.py | 77 -- src/tox/session.py | 871 ------------------ src/tox/session/__init__.py | 251 +++++ src/tox/session/commands/__init__.py | 0 src/tox/session/commands/help.py | 13 + src/tox/session/commands/help_ini.py | 14 + src/tox/session/commands/run/__init__.py | 0 src/tox/session/commands/run/parallel.py | 106 +++ src/tox/session/commands/run/sequential.py | 72 ++ src/tox/session/commands/show_config.py | 31 + src/tox/session/commands/show_env.py | 31 + src/tox/util/lock.py | 39 + src/tox/util/path.py | 10 + src/tox/venv.py | 110 ++- tests/unit/config/test_config.py | 19 +- .../builder/test_package_builder_isolated.py | 17 +- tests/unit/session/test_parallel.py | 2 +- tests/unit/session/test_session.py | 11 +- tests/unit/test_result.py | 16 +- tests/unit/test_venv.py | 138 +-- tests/unit/test_z_cmdline.py | 33 +- 38 files changed, 1461 insertions(+), 1261 deletions(-) create mode 100644 src/tox/action.py create mode 100644 src/tox/config/reporter.py create mode 100644 src/tox/logs/__init__.py create mode 100644 src/tox/logs/command.py create mode 100644 src/tox/logs/env.py create mode 100644 src/tox/logs/result.py create mode 100644 src/tox/package/local.py create mode 100644 src/tox/reporter.py delete mode 100644 src/tox/result.py delete mode 100644 src/tox/session.py create mode 100644 src/tox/session/__init__.py create mode 100644 src/tox/session/commands/__init__.py create mode 100644 src/tox/session/commands/help.py create mode 100644 src/tox/session/commands/help_ini.py create mode 100644 src/tox/session/commands/run/__init__.py create mode 100644 src/tox/session/commands/run/parallel.py create mode 100644 src/tox/session/commands/run/sequential.py create mode 100644 src/tox/session/commands/show_config.py create mode 100644 src/tox/session/commands/show_env.py create mode 100644 src/tox/util/lock.py create mode 100644 src/tox/util/path.py diff --git a/src/tox/_pytestplugin.py b/src/tox/_pytestplugin.py index 75d986ec5..c53958282 100644 --- a/src/tox/_pytestplugin.py +++ b/src/tox/_pytestplugin.py @@ -1,6 +1,7 @@ from __future__ import print_function, unicode_literals import os +import subprocess import sys import textwrap import time @@ -14,8 +15,8 @@ import tox from tox import venv from tox.config import parseconfig -from tox.result import ResultLog -from tox.session import Reporter, Session, main +from tox.reporter import update_default_reporter +from tox.session import Session, main from tox.venv import CreationConfig, VirtualEnv, getdigest mark_dont_run_on_windows = pytest.mark.skipif(os.name == "nt", reason="non windows test") @@ -64,7 +65,7 @@ def create_new_config_file_(args, source=None, plugins=()): @pytest.fixture -def cmd(request, capfd, monkeypatch): +def cmd(request, monkeypatch): if request.config.option.no_network: pytest.skip("--no-network was specified, test cannot run") request.addfinalizer(py.path.local().chdir) @@ -74,7 +75,7 @@ def run(*argv): python_paths = (i for i in (os.getcwd(), os.getenv(key)) if i) monkeypatch.setenv(key, os.pathsep.join(python_paths)) - with RunResult(capfd, argv) as result: + with RunResult(argv) as result: prev_run_command = Session.runcommand def run_command(self): @@ -96,8 +97,7 @@ def run_command(self): class RunResult: - def __init__(self, capfd, args): - self._capfd = capfd + def __init__(self, args): self.args = args self.ret = None self.duration = None @@ -107,12 +107,16 @@ def __init__(self, capfd, args): def __enter__(self): self._start = time.time() - self._capfd.readouterr() return self def __exit__(self, exc_type, exc_val, exc_tb): self.duration = time.time() - self._start - self.out, self.err = self._capfd.readouterr() + self.out = self._read(sys.stdout) + self.err = self._read(sys.stderr) + + def _read(self, out): + out.buffer.seek(0) + return out.buffer.read().decode(out.encoding, errors=out.errors) @property def outlines(self): @@ -125,32 +129,22 @@ def __repr__(self): class ReportExpectMock: - def __init__(self, session): - self._calls = [] + def __init__(self): + from tox import reporter + + self.instance = reporter._INSTANCE + self.clear() self._index = -1 - self.session = session - self.orig_reporter = Reporter(session) def clear(self): - self._calls[:] = [] - - def __getattr__(self, name): - if name[0] == "_": - raise AttributeError(name) - elif name == "verbosity": - return self.orig_reporter.verbosity - - def generic_report(*args, **_): - self._calls.append((name,) + args) - print("{}".format(self._calls[-1])) - - return generic_report + self._index = -1 + self.instance.reported_lines.clear() def getnext(self, cat): __tracebackhide__ = True newindex = self._index + 1 - while newindex < len(self._calls): - call = self._calls[newindex] + while newindex < len(self.instance.reported_lines): + call = self.instance.reported_lines[newindex] lcat = call[0] if fnmatch(lcat, cat): self._index = newindex @@ -158,7 +152,7 @@ def getnext(self, cat): newindex += 1 raise LookupError( "looking for {!r}, no reports found at >={:d} in {!r}".format( - cat, self._index + 1, self._calls + cat, self._index + 1, self.instance.reported_lines ) ) @@ -166,7 +160,7 @@ def expect(self, cat, messagepattern="*", invert=False): __tracebackhide__ = True if not messagepattern.startswith("*"): messagepattern = "*{}".format(messagepattern) - while self._index < len(self._calls): + while self._index < len(self.instance.reported_lines): try: call = self.getnext(cat) except LookupError: @@ -182,7 +176,7 @@ def expect(self, cat, messagepattern="*", invert=False): if not invert: raise AssertionError( "looking for {}({!r}), no reports found at >={:d} in {!r}".format( - cat, messagepattern, self._index + 1, self._calls + cat, messagepattern, self._index + 1, self.instance.reported_lines ) ) @@ -193,7 +187,7 @@ def not_expect(self, cat, messagepattern="*"): class pcallMock: def __init__(self, args, cwd, env, stdout, stderr, shell): self.arg0 = args[0] - self.args = args[1:] + self.args = args self.cwd = cwd self.env = env self.stdout = stdout @@ -210,36 +204,37 @@ def wait(self): @pytest.fixture(name="mocksession") def create_mocksession(request): - class MockSession(Session): - def __init__(self): - self._clearmocks() - self.config = request.getfixturevalue("newconfig")([], "") - self.resultlog = ResultLog() - self._actions = [] - - def getenv(self, name): - return VirtualEnv(self.config.envconfigs[name], session=self) + config = request.getfixturevalue("newconfig")([], "") - def _clearmocks(self): + class MockSession(Session): + def __init__(self, config): + update_default_reporter(config.option.quiet_level, config.option.verbose_level) + super(MockSession, self).__init__(config, popen=self.popen) self._pcalls = [] - self._spec2pkg = {} - self.report = ReportExpectMock(self) + self.report = ReportExpectMock() - def make_emptydir(self, path): - pass + def _clearmocks(self): + self._pcalls.clear() + self.report.clear() def popen(self, args, cwd, shell=None, stdout=None, stderr=None, env=None, **_): - pm = pcallMock(args, cwd, env, stdout, stderr, shell) - self._pcalls.append(pm) - return pm + process_call_mock = pcallMock(args, cwd, env, stdout, stderr, shell) + self._pcalls.append(process_call_mock) + return process_call_mock + + def new_config(self, config): + self.config = config + self.venv_dict.clear() + self.existing_venvs.clear() - return MockSession() + return MockSession(config) @pytest.fixture def newmocksession(mocksession, newconfig): def newmocksession_(args, source, plugins=()): - mocksession.config = newconfig(args, source, plugins=plugins) + config = newconfig(args, source, plugins=plugins) + mocksession.__init__(config) return mocksession return newmocksession_ @@ -337,7 +332,6 @@ def initproj_(nameversion, filedefs=None, src_root=".", add_missing_setup_py=Tru "include {}".format(p.relto(base)) for p in base.visit(lambda x: x.check(file=1)) ] create_files(base, {"MANIFEST.in": "\n".join(manifestlines)}) - print("created project in {}".format(base)) base.chdir() return base @@ -410,26 +404,7 @@ def mock_venv(monkeypatch): # object to collect some data during the execution class Result(object): def __init__(self, session): - self.popens = [] - self._popen = session.popen - - # collect all popen calls - def popen(cmd, **kwargs): - # we don't want to perform installation of new packages, - # just replace with an always ok cmd - if "pip" in cmd and "install" in cmd: - cmd = ["python", "-c", "print({!r})".format(cmd)] - activity_id = session._actions[-1].id - activity_name = session._actions[-1].activity - try: - ret = self._popen(cmd, **kwargs) - except tox.exception.InvocationError as exception: # pragma: no cover - ret = exception # pragma: no cover - finally: - self.popens.append((activity_id, activity_name, kwargs.get("env"), ret, cmd)) - return ret - - monkeypatch.setattr(session, "popen", popen) + self.popens = popen_list self.session = session res = OrderedDict() @@ -484,10 +459,25 @@ def tox_runenvreport(venv, action): monkeypatch.setattr(venv, "tox_runenvreport", tox_runenvreport) # intercept the build session to save it and we intercept the popen invocations - prev_build = tox.session.build_session + # collect all popen calls + popen_list = [] + + def popen(cmd, **kwargs): + # we don't want to perform installation of new packages, + # just replace with an always ok cmd + if "pip" in cmd and "install" in cmd: + cmd = ["python", "-c", "print({!r})".format(cmd)] + ret = None + try: + ret = subprocess.Popen(cmd, **kwargs) + except tox.exception.InvocationError as exception: # pragma: no cover + ret = exception # pragma: no cover + finally: + popen_list.append((kwargs.get("env"), ret, cmd)) + return ret def build_session(config): - session = prev_build(config) + session = Session(config, popen=popen) res[id(session)] = Result(session) return session @@ -500,3 +490,21 @@ def current_tox_py(): """generate the current (test runners) python versions key e.g. py37 when running under Python 3.7""" return "py{}".format("".join(str(i) for i in sys.version_info[0:2])) + + +def pytest_runtest_setup(item): + from tox.reporter import _INSTANCE + + _INSTANCE._reset() + + +def pytest_runtest_teardown(item): + from tox.reporter import _INSTANCE + + _INSTANCE._reset() + + +def pytest_pyfunc_call(pyfuncitem): + from tox.reporter import _INSTANCE + + _INSTANCE._reset() diff --git a/src/tox/action.py b/src/tox/action.py new file mode 100644 index 000000000..fe3434da7 --- /dev/null +++ b/src/tox/action.py @@ -0,0 +1,188 @@ +from __future__ import absolute_import, unicode_literals + +import os +import pipes +import subprocess +import sys +import time +from contextlib import contextmanager + +import py + +from tox import reporter +from tox.constants import INFO +from tox.exception import InvocationError +from tox.util.lock import get_unique_file + + +class Action(object): + """Action is an effort to group operations with the same goal (within reporting)""" + + def __init__(self, name, msg, args, log_dir, generate_tox_log, command_log, popen, python): + self.name = name + self.args = args + self.msg = msg + self.activity = self.msg.split(" ", 1)[0] + self.log_dir = log_dir + self.generate_tox_log = generate_tox_log + self.via_popen = popen + self.command_log = command_log + self._timed_report = None + self.python = python + + def __enter__(self): + msg = "{} {}".format(self.msg, " ".join(map(str, self.args))) + self._timed_report = reporter.timed_operation(self.name, msg) + self._timed_report.__enter__() + + return self + + def __exit__(self, type, value, traceback): + self._timed_report.__exit__(type, value, traceback) + + def setactivity(self, name, msg): + self.activity = name + if msg: + reporter.verbosity0("{} {}: {}".format(self.name, name, msg), bold=True) + else: + reporter.verbosity1("{} {}: {}".format(self.name, name, msg), bold=True) + + def info(self, name, msg): + reporter.verbosity1("{} {}: {}".format(self.name, name, msg), bold=True) + + def popen( + self, + args, + cwd=None, + env=None, + redirect=True, + returnout=False, + ignore_ret=False, + capture_err=True, + ): + """this drives an interaction with a subprocess""" + cmd_args = [str(x) for x in args] + cmd_args_shell = " ".join(pipes.quote(i) for i in cmd_args) + stream_getter = self._get_standard_streams( + capture_err, cmd_args_shell, redirect, returnout + ) + with stream_getter as (fin, out_path, stderr, stdout): + try: + process = self.via_popen( + self._rewrite_args(cwd, args), + stdout=stdout, + stderr=stderr, + cwd=cwd, + env=os.environ.copy() if env is None else env, + universal_newlines=True, + shell=False, + ) + except OSError as e: + reporter.error( + "invocation failed (errno {:d}), args: {}, cwd: {}".format( + e.errno, cmd_args_shell, cwd + ) + ) + raise + reporter.log_popen(cwd, out_path, cmd_args_shell) + output = self.feed_stdin(fin, process, redirect) + exit_code = process.wait() + if exit_code and not ignore_ret: + invoked = " ".join(map(str, process.args)) + if out_path: + reporter.error( + "invocation failed (exit code {:d}), logfile: {}".format(exit_code, out_path) + ) + output = out_path.read() + reporter.error(output) + self.command_log.add_command(process.args, output, exit_code) + raise InvocationError(invoked, exit_code, out_path) + else: + raise InvocationError(invoked, exit_code) + if not output and out_path: + output = out_path.read() + self.command_log.add_command(process.args, output, exit_code) + return output + + def feed_stdin(self, fin, process, redirect): + try: + if self.generate_tox_log and not redirect: + if process.stderr is not None: + # prevent deadlock + raise ValueError("stderr must not be piped here") + # we read binary from the process and must write using a binary stream + buf = getattr(sys.stdout, "buffer", sys.stdout) + out = None + last_time = time.time() + while 1: + # we have to read one byte at a time, otherwise there + # might be no output for a long time with slow tests + data = fin.read(1) + if data: + buf.write(data) + if b"\n" in data or (time.time() - last_time) > 1: + # we flush on newlines or after 1 second to + # provide quick enough feedback to the user + # when printing a dot per test + buf.flush() + last_time = time.time() + elif process.poll() is not None: + if process.stdout is not None: + process.stdout.close() + break + else: + time.sleep(0.1) + # the seek updates internal read buffers + fin.seek(0, 1) + fin.close() + else: + out, err = process.communicate() + except KeyboardInterrupt: + reporter.error("KEYBOARDINTERRUPT") + process.wait() + raise + return out + + @contextmanager + def _get_standard_streams(self, capture_err, cmd_args_shell, redirect, returnout): + stdout = out_path = fin = None + stderr = subprocess.STDOUT if capture_err else None + stdout_file = None + if self.generate_tox_log or redirect: + stdout_file = open(self.get_log_path(self.name), "wt") + stdout_file.write( + "actionid: {}\nmsg: {}\ncmdargs: {!r}\n\n".format( + self.name, self.msg, cmd_args_shell + ) + ) + stdout_file.flush() + out_path = py.path.local(stdout_file.name) + fin = out_path.open("rb") + fin.read() # read the header, so it won't be written to stdout + stdout = stdout_file + elif returnout: + stdout = subprocess.PIPE + try: + yield fin, out_path, stderr, stdout + finally: + if stdout_file is not None: + stdout_file.close() + + def get_log_path(self, actionid): + return get_unique_file( + self.log_dir, prefix=actionid, suffix=".logs", report=reporter.verbosity1 + ) + + def _rewrite_args(self, cwd, args): + new_args = [] + for arg in args: + if not INFO.IS_WIN and isinstance(arg, py.path.local): + cwd = py.path.local(cwd) + arg = cwd.bestrelpath(arg) + new_args.append(str(arg)) + # subprocess does not always take kindly to .py scripts so adding the interpreter here + if INFO.IS_WIN: + ext = os.path.splitext(str(new_args[0]))[1].lower() + if ext == ".py": + new_args = [str(self.python)] + new_args + return new_args diff --git a/src/tox/config/__init__.py b/src/tox/config/__init__.py index bf4252e36..a72a7e0ee 100644 --- a/src/tox/config/__init__.py +++ b/src/tox/config/__init__.py @@ -22,6 +22,7 @@ from tox.constants import INFO from tox.interpreters import Interpreters, NoInterpreterInfo from .parallel import add_parallel_flags, ENV_VAR_KEY as PARALLEL_ENV_VAR_KEY, add_parallel_config +from .reporter import add_verbosity_commands hookimpl = tox.hookimpl """DEPRECATED - REMOVE - this is left for compatibility with plugins importing this from here. @@ -364,22 +365,7 @@ def tox_addoption(parser): parser.add_argument( "--help-ini", "--hi", action="store_true", dest="helpini", help="show help about ini-names" ) - parser.add_argument( - "-v", - action="count", - dest="verbose_level", - default=0, - help="increase verbosity of reporting output." - "-vv mode turns off output redirection for package installation, " - "above level two verbosity flags are passed through to pip (with two less level)", - ) - parser.add_argument( - "-q", - action="count", - dest="quiet_level", - default=0, - help="progressively silence reporting output.", - ) + add_verbosity_commands(parser) parser.add_argument( "--showconfig", action="store_true", @@ -987,18 +973,23 @@ def __init__(self, config, ini_path, ini_data): # noqa config.hashseed = hash_seed reader.addsubstitutions(toxinidir=config.toxinidir, homedir=config.homedir) + # As older versions of tox may have bugs or incompatibilities that # prevent parsing of tox.ini this must be the first thing checked. config.minversion = reader.getstring("minversion", None) if config.minversion: - tox_version = pkg_resources.parse_version(tox.__version__) - config_min_version = pkg_resources.parse_version(self.config.minversion) - if config_min_version > tox_version: - raise tox.exception.MinVersionError( - "tox version is {}, required is at least {}".format( - tox.__version__, self.config.minversion + # As older versions of tox may have bugs or incompatibilities that + # prevent parsing of tox.ini this must be the first thing checked. + config.minversion = reader.getstring("minversion", None) + if config.minversion: + tox_version = pkg_resources.parse_version(tox.__version__) + config_min_version = pkg_resources.parse_version(self.config.minversion) + if config_min_version > tox_version: + raise tox.exception.MinVersionError( + "tox version is {}, required is at least {}".format( + tox.__version__, self.config.minversion + ) ) - ) self.ensure_requires_satisfied(reader.getlist("requires")) diff --git a/src/tox/config/reporter.py b/src/tox/config/reporter.py new file mode 100644 index 000000000..3acdf107f --- /dev/null +++ b/src/tox/config/reporter.py @@ -0,0 +1,20 @@ +from __future__ import absolute_import, unicode_literals + + +def add_verbosity_commands(parser): + parser.add_argument( + "-v", + action="count", + dest="verbose_level", + default=0, + help="increase verbosity of reporting output." + "-vv mode turns off output redirection for package installation, " + "above level two verbosity flags are passed through to pip (with two less level)", + ) + parser.add_argument( + "-q", + action="count", + dest="quiet_level", + default=0, + help="progressively silence reporting output.", + ) diff --git a/src/tox/exception.py b/src/tox/exception.py index ace6a31d2..286929d23 100644 --- a/src/tox/exception.py +++ b/src/tox/exception.py @@ -59,10 +59,11 @@ class InterpreterNotFound(Error): class InvocationError(Error): """An error while invoking a script.""" - def __init__(self, command, exit_code=None): + def __init__(self, command, exit_code=None, out=None): super(Error, self).__init__(command, exit_code) self.command = command self.exit_code = exit_code + self.out = out def __str__(self): return exit_code_str(self.__class__.__name__, self.command, self.exit_code) diff --git a/src/tox/hookspecs.py b/src/tox/hookspecs.py index 67adfe57e..90374cfb5 100644 --- a/src/tox/hookspecs.py +++ b/src/tox/hookspecs.py @@ -115,3 +115,8 @@ def tox_runenvreport(venv, action): This could be used for alternative (ie non-pip) package managers, this plugin should return a ``list`` of type ``str`` """ + + +@hookspec +def tox_cleanup(session): + """Called just before the session is destroyed, allowing any final cleanup operation""" diff --git a/src/tox/logs/__init__.py b/src/tox/logs/__init__.py new file mode 100644 index 000000000..ed5490686 --- /dev/null +++ b/src/tox/logs/__init__.py @@ -0,0 +1,4 @@ +"""This module handles collecting and persisting in json format a tox session""" +from .result import ResultLog + +__all__ = ("ResultLog",) diff --git a/src/tox/logs/command.py b/src/tox/logs/command.py new file mode 100644 index 000000000..4d6a7fc52 --- /dev/null +++ b/src/tox/logs/command.py @@ -0,0 +1,14 @@ +from __future__ import absolute_import, unicode_literals + + +class CommandLog(object): + """Report commands interacting with third party tools""" + + def __init__(self, env_log, list): + self.envlog = env_log + self.list = list + + def add_command(self, argv, output, retcode): + data = {"command": argv, "output": output, "retcode": str(retcode)} + self.list.append(data) + return data diff --git a/src/tox/logs/env.py b/src/tox/logs/env.py new file mode 100644 index 000000000..f4ac20595 --- /dev/null +++ b/src/tox/logs/env.py @@ -0,0 +1,46 @@ +from __future__ import absolute_import, unicode_literals + +import json +import subprocess + +from .command import CommandLog + + +class EnvLog(object): + """Report the status of a tox environment""" + + def __init__(self, result_log, name, dict): + self.reportlog = result_log + self.name = name + self.dict = dict + + def set_python_info(self, python_executable): + cmd = [ + str(python_executable), + "-c", + "import sys; import json;" + "print(json.dumps({" + "'executable': sys.executable," + "'version_info': list(sys.version_info)," + "'version': sys.version}))", + ] + result = subprocess.check_output(cmd, universal_newlines=True) + self.dict["python"] = json.loads(result) + + def get_commandlog(self, name): + """get the command log for a given group name""" + data = self.dict.setdefault(name, []) + return CommandLog(self, data) + + def set_installed(self, packages): + self.dict["installed_packages"] = packages + + def set_header(self, installpkg): + """ + :param py.path.local installpkg: Path ot the package. + """ + self.dict["installpkg"] = { + "md5": installpkg.computehash("md5"), + "sha256": installpkg.computehash("sha256"), + "basename": installpkg.basename, + } diff --git a/src/tox/logs/result.py b/src/tox/logs/result.py new file mode 100644 index 000000000..78ff9ab18 --- /dev/null +++ b/src/tox/logs/result.py @@ -0,0 +1,43 @@ +"""Generate json report of a run""" +from __future__ import absolute_import, unicode_literals + +import json +import socket +import sys + +from tox.version import __version__ + +from .command import CommandLog +from .env import EnvLog + + +class ResultLog(object): + """The result of a tox session""" + + def __init__(self,): + command_log = [] + self.command_log = CommandLog(None, command_log) + self.dict = { + "reportversion": "1", + "toxversion": __version__, + "platform": sys.platform, + "host": socket.getfqdn(), + "commands": command_log, + } + + @classmethod + def from_json(cls, data): + result = cls() + result.dict = json.loads(data) + result.command_log = CommandLog(None, result.dict["commands"]) + return result + + def get_envlog(self, name): + """Return the env log of a environment (create on first call)""" + test_envs = self.dict.setdefault("testenvs", {}) + env_data = test_envs.setdefault(name, {}) + return EnvLog(self, name, env_data) + + def dumps_json(self): + """Return the json dump of the current state, indented""" + return json.dumps(self.dict, indent=2) diff --git a/src/tox/package/__init__.py b/src/tox/package/__init__.py index e167e1e0e..ab6fd74e0 100644 --- a/src/tox/package/__init__.py +++ b/src/tox/package/__init__.py @@ -1,9 +1,11 @@ import py -from filelock import FileLock, Timeout import tox -from .view import create_session_view +from tox.util.lock import get as hold_lock from .builder import build_package +from .local import resolve_package +from .view import create_session_view +from tox.reporter import verbosity0, info, error, warning, verbosity2 @tox.hookimpl @@ -16,53 +18,55 @@ def tox_package(session, venv): def get_package(session): """"Perform the package operation""" - config, report = session.config, session.report + config = session.config if config.skipsdist: - report.info("skipping sdist step") + info("skipping sdist step") return None - lock_file = str( - session.config.toxworkdir.join("{}.lock".format(session.config.isolated_build_env)) - ) - lock = FileLock(lock_file) - try: - try: - lock.acquire(0.0001) - except Timeout: - report.verbosity0("lock file {} present, will block until released".format(lock_file)) - lock.acquire() - package = acquire_package(config, report, session) - session_package = create_session_view(package, config.temp_dir, report) + lock_file = session.config.toxworkdir.join("{}.lock".format(session.config.isolated_build_env)) + + with hold_lock(lock_file, verbosity0): + package = acquire_package(config, session) + session_package = create_session_view(package, config.temp_dir) return session_package, package - finally: - lock.release(force=True) -def acquire_package(config, report, session): +def acquire_package(config, session): """acquire a source distribution (either by loading a local file or triggering a build)""" if not config.option.sdistonly and (config.sdistsrc or config.option.installpkg): - path = get_local_package(config, report, session) + path = get_local_package(config) else: try: - path = build_package(config, report, session) + path = build_package(config, session) except tox.exception.InvocationError as exception: - report.error("FAIL could not package project - v = {!r}".format(exception)) + error("FAIL could not package project - v = {!r}".format(exception)) return None sdist_file = config.distshare.join(path.basename) if sdist_file != path: - report.info("copying new sdistfile to {!r}".format(str(sdist_file))) + info("copying new sdistfile to {!r}".format(str(sdist_file))) try: sdist_file.dirpath().ensure(dir=1) except py.error.Error: - report.warning("could not copy distfile to {}".format(sdist_file.dirpath())) + warning("could not copy distfile to {}".format(sdist_file.dirpath())) else: path.copy(sdist_file) return path -def get_local_package(config, report, session): +def get_local_package(config): path = config.option.installpkg if not path: path = config.sdistsrc - py_path = py.path.local(session._resolve_package(path)) - report.info("using package {!r}, skipping 'sdist' activity ".format(str(py_path))) + py_path = py.path.local(resolve_package(path)) + info("using package {!r}, skipping 'sdist' activity ".format(str(py_path))) return py_path + + +@tox.hookimpl +def tox_cleanup(session): + for tox_env in session.venv_dict.values(): + if hasattr(tox_env, "package") and isinstance(tox_env.package, py.path.local): + package = tox_env.package + if package.exists(): + verbosity2("cleanup {}".format(package)) + package.remove() + py.path.local(package.dirname).remove(ignore_errors=True) diff --git a/src/tox/package/builder/__init__.py b/src/tox/package/builder/__init__.py index 16cf3d7e9..d8c593f4a 100644 --- a/src/tox/package/builder/__init__.py +++ b/src/tox/package/builder/__init__.py @@ -2,8 +2,8 @@ from .isolated import build -def build_package(config, report, session): +def build_package(config, session): if not config.isolated_build: - return make_sdist(report, config, session) + return make_sdist(config, session) else: - return build(config, report, session) + return build(config, session) diff --git a/src/tox/package/builder/isolated.py b/src/tox/package/builder/isolated.py index 615ed5b53..ea4d11260 100644 --- a/src/tox/package/builder/isolated.py +++ b/src/tox/package/builder/isolated.py @@ -5,13 +5,14 @@ import pkg_resources import six +from tox import reporter from tox.config import DepConfig, get_py_project_toml BuildInfo = namedtuple("BuildInfo", ["requires", "backend_module", "backend_object"]) -def build(config, report, session): - build_info = get_build_info(config.setupdir, report) +def build(config, session): + build_info = get_build_info(config.setupdir) package_venv = session.getvenv(config.isolated_build_env) package_venv.envconfig.deps_matches_subset = True @@ -21,10 +22,10 @@ def build(config, report, session): package_venv.envconfig.deps = [DepConfig(r, None) for r in build_info.requires] package_venv.envconfig.deps.extend(user_specified_deps) - if session.setupenv(package_venv): - session.finishvenv(package_venv) + if package_venv.setupenv(): + package_venv.finishvenv() - build_requires = get_build_requires(build_info, package_venv, session) + build_requires = get_build_requires(build_info, package_venv, config.setupdir) # we need to filter out requirements already specified in pyproject.toml or user deps base_build_deps = {pkg_resources.Requirement(r.name).key for r in package_venv.envconfig.deps} build_requires_dep = [ @@ -33,25 +34,23 @@ def build(config, report, session): if pkg_resources.Requirement(r).key not in base_build_deps ] if build_requires_dep: - with session.newaction( - package_venv, "build_requires", package_venv.envconfig.envdir - ) as action: + with package_venv.newaction("build_requires", package_venv.envconfig.envdir) as action: package_venv.run_install_command(packages=build_requires_dep, action=action) - session.finishvenv(package_venv) - return perform_isolated_build(build_info, package_venv, session, config, report) + package_venv.finishvenv(package_venv) + return perform_isolated_build(build_info, package_venv, config.distdir, config.setupdir) -def get_build_info(folder, report): +def get_build_info(folder): toml_file = folder.join("pyproject.toml") # as per https://www.python.org/dev/peps/pep-0517/ def abort(message): - report.error("{} inside {}".format(message, toml_file)) + reporter.error("{} inside {}".format(message, toml_file)) raise SystemExit(1) if not toml_file.exists(): - report.error("missing {}".format(toml_file)) + reporter.error("missing {}".format(toml_file)) raise SystemExit(1) config_data = get_py_project_toml(toml_file) @@ -81,9 +80,9 @@ def abort(message): return BuildInfo(requires, module, "{}{}".format(module, obj)) -def perform_isolated_build(build_info, package_venv, session, config, report): - with session.newaction( - package_venv, "perform-isolated-build", package_venv.envconfig.envdir +def perform_isolated_build(build_info, package_venv, dist_dir, setup_dir): + with package_venv.new_action( + "perform-isolated-build", package_venv.envconfig.envdir ) as action: script = textwrap.dedent( """ @@ -91,29 +90,27 @@ def perform_isolated_build(build_info, package_venv, session, config, report): import {} basename = {}.build_{}({!r}, {{ "--global-option": ["--formats=gztar"]}}) print(basename)""".format( - build_info.backend_module, build_info.backend_object, "sdist", str(config.distdir) + build_info.backend_module, build_info.backend_object, "sdist", str(dist_dir) ) ) # need to start with an empty (but existing) source distribution folder - if config.distdir.exists(): - config.distdir.remove(rec=1, ignore_errors=True) - config.distdir.ensure_dir() + if dist_dir.exists(): + dist_dir.remove(rec=1, ignore_errors=True) + dist_dir.ensure_dir() result = package_venv._pcall( [package_venv.envconfig.envpython, "-c", script], returnout=True, action=action, - cwd=session.config.setupdir, + cwd=setup_dir, ) - report.verbosity2(result) - return config.distdir.join(result.split("\n")[-2]) + reporter.verbosity2(result) + return dist_dir.join(result.split("\n")[-2]) -def get_build_requires(build_info, package_venv, session): - with session.newaction( - package_venv, "get-build-requires", package_venv.envconfig.envdir - ) as action: +def get_build_requires(build_info, package_venv, setup_dir): + with package_venv.new_action("get-build-requires", package_venv.envconfig.envdir) as action: script = textwrap.dedent( """ import {} @@ -130,6 +127,6 @@ def get_build_requires(build_info, package_venv, session): [package_venv.envconfig.envpython, "-c", script], returnout=True, action=action, - cwd=session.config.setupdir, + cwd=setup_dir, ) return json.loads(result.split("\n")[-2]) diff --git a/src/tox/package/builder/legacy.py b/src/tox/package/builder/legacy.py index ebd180c44..4ddc03b54 100644 --- a/src/tox/package/builder/legacy.py +++ b/src/tox/package/builder/legacy.py @@ -2,11 +2,14 @@ import py +from tox import reporter +from tox.util.path import ensure_empty_dir -def make_sdist(report, config, session): + +def make_sdist(config, session): setup = config.setupdir.join("setup.py") if not setup.check(): - report.error( + reporter.error( "No setup.py file found. The expected location is:\n" " {}\n" "You can\n" @@ -17,15 +20,15 @@ def make_sdist(report, config, session): "#avoiding-expensive-sdist".format(setup) ) raise SystemExit(1) - with session.newaction(None, "packaging") as action: + with session.newaction("GLOB", "packaging") as action: action.setactivity("sdist-make", setup) - session.make_emptydir(config.distdir) + ensure_empty_dir(config.distdir) build_log = action.popen( [sys.executable, setup, "sdist", "--formats=zip", "--dist-dir", config.distdir], cwd=config.setupdir, returnout=True, ) - report.verbosity2(build_log) + reporter.verbosity2(build_log) try: return config.distdir.listdir()[0] except py.error.ENOENT: @@ -37,9 +40,9 @@ def make_sdist(report, config, session): continue data.append(line) if not "".join(data).strip(): - report.error("setup.py is empty") + reporter.error("setup.py is empty") raise SystemExit(1) - report.error( + reporter.error( "No dist directory found. Please check setup.py, e.g with:\n" " python setup.py sdist" ) diff --git a/src/tox/package/local.py b/src/tox/package/local.py new file mode 100644 index 000000000..b56e31cfa --- /dev/null +++ b/src/tox/package/local.py @@ -0,0 +1,64 @@ +import os +import re + +import pkg_resources +import py + +import tox +from tox import reporter +from tox.exception import MissingDependency + +_SPEC_2_PACKAGE = {} + + +def resolve_package(package_spec): + global _SPEC_2_PACKAGE + try: + return _SPEC_2_PACKAGE[package_spec] + except KeyError: + _SPEC_2_PACKAGE[package_spec] = x = get_latest_version_of_package(package_spec) + return x + + +def get_latest_version_of_package(package_spec): + if not os.path.isabs(str(package_spec)): + return package_spec + p = py.path.local(package_spec) + if p.check(): + return p + if not p.dirpath().check(dir=1): + raise tox.exception.MissingDirectory(p.dirpath()) + reporter.info("determining {}".format(p)) + candidates = p.dirpath().listdir(p.basename) + if len(candidates) == 0: + raise MissingDependency(package_spec) + if len(candidates) > 1: + version_package = [] + for filename in candidates: + version = get_version_from_filename(filename.basename) + if version is not None: + version_package.append((version, filename)) + else: + reporter.warning("could not determine version of: {}".format(str(filename))) + if not version_package: + raise tox.exception.MissingDependency(package_spec) + version_package.sort() + _, package_with_largest_version = version_package[-1] + return package_with_largest_version + else: + return candidates[0] + + +_REGEX_FILE_NAME_WITH_VERSION = re.compile(r"[\w_\-\+\.]+-(.*)\.(zip|tar\.gz)") + + +def get_version_from_filename(basename): + m = _REGEX_FILE_NAME_WITH_VERSION.match(basename) + if m is None: + return None + version = m.group(1) + try: + + return pkg_resources.packaging.version.Version(version) + except pkg_resources.packaging.version.InvalidVersion: + return None diff --git a/src/tox/package/view.py b/src/tox/package/view.py index 226df518f..49935a8fa 100644 --- a/src/tox/package/view.py +++ b/src/tox/package/view.py @@ -3,8 +3,10 @@ import six +from tox.reporter import verbosity1 -def create_session_view(package, temp_dir, report): + +def create_session_view(package, temp_dir): """once we build a package we cannot return that directly, as a subsequent call might delete that package (in order to do its own build); therefore we need to return a view of the file that it's not prone to deletion and can be removed when the @@ -37,7 +39,7 @@ def create_session_view(package, temp_dir, report): package.copy(session_package) operation = "links" if links else "copied" common = session_package.common(package) - report.verbosity1( + verbosity1( "package {} {} to {} ({})".format( common.bestrelpath(session_package), operation, common.bestrelpath(package), common ) diff --git a/src/tox/reporter.py b/src/tox/reporter.py new file mode 100644 index 000000000..adab06b96 --- /dev/null +++ b/src/tox/reporter.py @@ -0,0 +1,143 @@ +"""A progress reporter inspired from the logging modules""" +from __future__ import absolute_import, unicode_literals + +import time +from contextlib import contextmanager + +import py + + +class Verbosity(object): + DEBUG = 2 + INFO = 1 + DEFAULT = 0 + QUIET = -1 + EXTRA_QUIET = -2 + + +class Reporter(object): + def __init__(self, verbose_level=None, quiet_level=None): + kwargs = {} + if verbose_level is not None: + kwargs["verbose_level"] = verbose_level + if quiet_level is not None: + kwargs["quiet_level"] = quiet_level + self._reset(**kwargs) + + def _reset(self, verbose_level=0, quiet_level=0): + self.verbose_level = verbose_level + self.quiet_level = quiet_level + self.reported_lines = [] + self.tw = py.io.TerminalWriter() + + @property + def verbosity(self): + return self.verbose_level - self.quiet_level + + def log_popen(self, cwd, outpath, cmd_args_shell): + """ log information about the action.popen() created process. """ + if outpath: + self.verbosity1(" {}$ {} >{}".format(cwd, cmd_args_shell, outpath)) + else: + self.verbosity1(" {}$ {} ".format(cwd, cmd_args_shell)) + + @property + def messages(self): + return [i for _, i in self.reported_lines] + + @contextmanager + def timed_operation(self, name, msg): + self.verbosity2("{} start: {}".format(name, msg), bold=True) + start = time.time() + yield + duration = time.time() - start + self.verbosity2( + "{} finish: {} after {:.2f} seconds".format(name, msg, duration), bold=True + ) + + def separator(self, of, msg, level): + if self.verbosity >= level: + self.reported_lines.append(("separator", "- summary -")) + self.tw.sep(of, msg) + + def logline_if(self, level, of, msg, key=None, **kwargs): + if self.verbosity >= level: + message = str(msg) if key is None else "{}{}".format(key, msg) + self.logline(of, message, **kwargs) + + def logline(self, of, msg, **opts): + self.reported_lines.append((of, msg)) + self.tw.line("{}".format(msg), **opts) + + def keyvalue(self, name, value): + if name.endswith(":"): + name += " " + self.tw.write(name, bold=True) + self.tw.write(value) + self.tw.line() + + def line(self, msg, **opts): + self.logline("line", msg, **opts) + + def info(self, msg): + self.logline_if(Verbosity.DEBUG, "info", msg) + + def using(self, msg): + self.logline_if(Verbosity.INFO, "using", msg, "using ", bold=True) + + def good(self, msg): + self.logline_if(Verbosity.QUIET, "good", msg, green=True) + + def warning(self, msg): + self.logline_if(Verbosity.QUIET, "warning", msg, "WARNING: ", red=True) + + def error(self, msg): + self.logline_if(Verbosity.QUIET, "error", msg, "ERROR: ", red=True) + + def skip(self, msg): + self.logline_if(Verbosity.QUIET, "skip", msg, "SKIPPED: ", yellow=True) + + def verbosity0(self, msg, **opts): + self.logline_if(Verbosity.DEFAULT, "verbosity0", msg, **opts) + + def verbosity1(self, msg, **opts): + self.logline_if(Verbosity.INFO, "verbosity1", msg, **opts) + + def verbosity2(self, msg, **opts): + self.logline_if(Verbosity.DEBUG, "verbosity2", msg, **opts) + + def quiet(self, msg): + self.logline_if(Verbosity.QUIET, "quiet", msg) + + +_INSTANCE = Reporter() + + +def update_default_reporter(quiet_level, verbose_level): + _INSTANCE.quiet_level = quiet_level + _INSTANCE.verbose_level = verbose_level + + +def has_level(of): + return _INSTANCE.verbosity > of + + +def verbosity(): + return _INSTANCE.verbosity + + +verbosity0 = _INSTANCE.verbosity0 +verbosity1 = _INSTANCE.verbosity1 +verbosity2 = _INSTANCE.verbosity2 +error = _INSTANCE.error +warning = _INSTANCE.warning +good = _INSTANCE.good +using = _INSTANCE.using +skip = _INSTANCE.skip +info = _INSTANCE.info +line = _INSTANCE.line +separator = _INSTANCE.separator +keyvalue = _INSTANCE.keyvalue +quiet = _INSTANCE.quiet +timed_operation = _INSTANCE.timed_operation +log_popen = _INSTANCE.log_popen diff --git a/src/tox/result.py b/src/tox/result.py deleted file mode 100644 index 1083f12e5..000000000 --- a/src/tox/result.py +++ /dev/null @@ -1,77 +0,0 @@ -import json -import socket -import subprocess -import sys - -import tox - - -class ResultLog: - def __init__(self, data=None): - if not data: - self.dict = {} - elif isinstance(data, dict): - self.dict = data - else: - self.dict = json.loads(data) - self.dict.update({"reportversion": "1", "toxversion": tox.__version__}) - self.dict["platform"] = sys.platform - self.dict["host"] = socket.getfqdn() - - def set_header(self, installpkg): - """ - :param py.path.local installpkg: Path ot the package. - """ - self.dict["installpkg"] = { - "md5": installpkg.computehash("md5"), - "sha256": installpkg.computehash("sha256"), - "basename": installpkg.basename, - } - - def get_envlog(self, name): - testenvs = self.dict.setdefault("testenvs", {}) - d = testenvs.setdefault(name, {}) - return EnvLog(self, name, d) - - def dumps_json(self): - return json.dumps(self.dict, indent=2) - - -class EnvLog: - def __init__(self, reportlog, name, dict): - self.reportlog = reportlog - self.name = name - self.dict = dict - - def set_python_info(self, python_executable): - cmd = [ - str(python_executable), - "-c", - "import sys; import json;" - "print(json.dumps({" - "'executable': sys.executable," - "'version_info': list(sys.version_info)," - "'version': sys.version}))", - ] - result = subprocess.check_output(cmd, universal_newlines=True) - self.dict["python"] = json.loads(result) - - def get_commandlog(self, name): - return CommandLog(self, self.dict.setdefault(name, [])) - - def set_installed(self, packages): - self.dict["installed_packages"] = packages - - -class CommandLog: - def __init__(self, envlog, list): - self.envlog = envlog - self.list = list - - def add_command(self, argv, output, retcode): - d = {} - self.list.append(d) - d["command"] = argv - d["output"] = output - d["retcode"] = str(retcode) - return d diff --git a/src/tox/session.py b/src/tox/session.py deleted file mode 100644 index 298bdc557..000000000 --- a/src/tox/session.py +++ /dev/null @@ -1,871 +0,0 @@ -""" -Automatically package and test a Python project against configurable -Python2 and Python3 based virtual environments. Environments are -setup by using virtualenv. Configuration is generally done through an -INI-style "tox.ini" file. -""" - -import os -import pipes -import re -import shutil -import subprocess -import sys -import time -from collections import OrderedDict -from contextlib import contextmanager -from threading import Event, Semaphore, Thread - -import pkg_resources -import py - -import tox -from tox.config import parseconfig -from tox.config.parallel import ENV_VAR_KEY as PARALLEL_ENV_VAR_KEY -from tox.config.parallel import OFF_VALUE as PARALLEL_OFF -from tox.result import ResultLog -from tox.util import set_os_env_var -from tox.util.graph import stable_topological_sort -from tox.util.spinner import Spinner -from tox.venv import VirtualEnv - - -def prepare(args): - config = parseconfig(args) - if config.option.help: - show_help(config) - raise SystemExit(0) - elif config.option.helpini: - show_help_ini(config) - raise SystemExit(0) - return config - - -def cmdline(args=None): - if args is None: - args = sys.argv[1:] - main(args) - - -def main(args): - try: - config = prepare(args) - with set_os_env_var("TOX_WORK_DIR", config.toxworkdir): - retcode = build_session(config).runcommand() - if retcode is None: - retcode = 0 - raise SystemExit(retcode) - except KeyboardInterrupt: - raise SystemExit(2) - except (tox.exception.MinVersionError, tox.exception.MissingRequirement) as e: - r = Reporter(None) - r.error(str(e)) - raise SystemExit(1) - - -def build_session(config): - return Session(config) - - -def show_help(config): - tw = py.io.TerminalWriter() - tw.write(config._parser._format_help()) - tw.line() - tw.line("Environment variables", bold=True) - tw.line("TOXENV: comma separated list of environments (overridable by '-e')") - tw.line("TOX_SKIP_ENV: regular expression to filter down from running tox environments") - tw.line( - "TOX_TESTENV_PASSENV: space-separated list of extra environment variables to be " - "passed into test command environments" - ) - tw.line("PY_COLORS: 0 disable colorized output, 1 enable (default)") - - -def show_help_ini(config): - tw = py.io.TerminalWriter() - tw.sep("-", "per-testenv attributes") - for env_attr in config._testenv_attr: - tw.line( - "{:<15} {:<8} default: {}".format( - env_attr.name, "<{}>".format(env_attr.type), env_attr.default - ), - bold=True, - ) - tw.line(env_attr.help) - tw.line() - - -class Action(object): - def __init__(self, session, venv, msg, args): - self.venv = venv - self.msg = msg - self.activity = msg.split(" ", 1)[0] - self.session = session - self.report = session.report - self.args = args - self.id = venv and venv.envconfig.envname or "tox" - self._popenlist = [] - if self.venv: - self.venvname = self.venv.name - else: - self.venvname = "GLOB" - if msg == "runtests": - cat = "test" - else: - cat = "setup" - envlog = session.resultlog.get_envlog(self.venvname) - self.commandlog = envlog.get_commandlog(cat) - - def __enter__(self): - self.report.logaction_start(self) - return self - - def __exit__(self, *args): - self.report.logaction_finish(self) - - def setactivity(self, name, msg): - self.activity = name - if msg: - self.report.verbosity0("{} {}: {}".format(self.venvname, name, msg), bold=True) - else: - self.report.verbosity1("{} {}: {}".format(self.venvname, name, msg), bold=True) - - def info(self, name, msg): - self.report.verbosity1("{} {}: {}".format(self.venvname, name, msg), bold=True) - - def _initlogpath(self, actionid): - if self.venv: - logdir = self.venv.envconfig.envlogdir - else: - logdir = self.session.config.logdir - try: - log_count = len(logdir.listdir("{}-*".format(actionid))) - except (py.error.ENOENT, py.error.ENOTDIR): - logdir.ensure(dir=1) - log_count = 0 - path = logdir.join("{}-{}.log".format(actionid, log_count)) - f = path.open("w") - f.flush() - return f - - def popen( - self, - args, - cwd=None, - env=None, - redirect=True, - returnout=False, - ignore_ret=False, - capture_err=True, - ): - stdout = outpath = None - resultjson = self.session.config.option.resultjson - - stderr = subprocess.STDOUT if capture_err else None - - cmd_args = [str(x) for x in args] - cmd_args_shell = " ".join(pipes.quote(i) for i in cmd_args) - if resultjson or redirect: - fout = self._initlogpath(self.id) - fout.write( - "actionid: {}\nmsg: {}\ncmdargs: {!r}\n\n".format( - self.id, self.msg, cmd_args_shell - ) - ) - fout.flush() - outpath = py.path.local(fout.name) - fin = outpath.open("rb") - fin.read() # read the header, so it won't be written to stdout - stdout = fout - elif returnout: - stdout = subprocess.PIPE - if cwd is None: - # FIXME XXX cwd = self.session.config.cwd - cwd = py.path.local() - try: - popen = self._popen(args, cwd, env=env, stdout=stdout, stderr=stderr) - except OSError as e: - self.report.error( - "invocation failed (errno {:d}), args: {}, cwd: {}".format( - e.errno, cmd_args_shell, cwd - ) - ) - raise - popen.outpath = outpath - popen.args = cmd_args - popen.cwd = cwd - popen.action = self - self._popenlist.append(popen) - try: - self.report.logpopen(popen, cmd_args_shell) - try: - if resultjson and not redirect: - if popen.stderr is not None: - # prevent deadlock - raise ValueError("stderr must not be piped here") - # we read binary from the process and must write using a - # binary stream - buf = getattr(sys.stdout, "buffer", sys.stdout) - out = None - last_time = time.time() - while 1: - # we have to read one byte at a time, otherwise there - # might be no output for a long time with slow tests - data = fin.read(1) - if data: - buf.write(data) - if b"\n" in data or (time.time() - last_time) > 1: - # we flush on newlines or after 1 second to - # provide quick enough feedback to the user - # when printing a dot per test - buf.flush() - last_time = time.time() - elif popen.poll() is not None: - if popen.stdout is not None: - popen.stdout.close() - break - else: - time.sleep(0.1) - # the seek updates internal read buffers - fin.seek(0, 1) - fin.close() - else: - out, err = popen.communicate() - except KeyboardInterrupt: - self.report.keyboard_interrupt() - popen.wait() - raise - ret = popen.wait() - finally: - self._popenlist.remove(popen) - if ret and not ignore_ret: - invoked = " ".join(map(str, popen.args)) - if outpath: - self.report.error( - "invocation failed (exit code {:d}), logfile: {}".format(ret, outpath) - ) - out = outpath.read() - self.report.error(out) - if hasattr(self, "commandlog"): - self.commandlog.add_command(popen.args, out, ret) - raise tox.exception.InvocationError("{} (see {})".format(invoked, outpath), ret) - else: - raise tox.exception.InvocationError("{!r}".format(invoked), ret) - if not out and outpath: - out = outpath.read() - if hasattr(self, "commandlog"): - self.commandlog.add_command(popen.args, out, ret) - return out - - def _rewriteargs(self, cwd, args): - newargs = [] - for arg in args: - if not tox.INFO.IS_WIN and isinstance(arg, py.path.local): - arg = cwd.bestrelpath(arg) - newargs.append(str(arg)) - # subprocess does not always take kindly to .py scripts so adding the interpreter here - if tox.INFO.IS_WIN: - ext = os.path.splitext(str(newargs[0]))[1].lower() - if ext == ".py" and self.venv: - newargs = [str(self.venv.envconfig.envpython)] + newargs - return newargs - - def _popen(self, args, cwd, stdout, stderr, env=None): - if env is None: - env = os.environ.copy() - return self.session.popen( - self._rewriteargs(cwd, args), - shell=False, - cwd=str(cwd), - universal_newlines=True, - stdout=stdout, - stderr=stderr, - env=env, - ) - - -class Verbosity(object): - DEBUG = 2 - INFO = 1 - DEFAULT = 0 - QUIET = -1 - EXTRA_QUIET = -2 - - -class Reporter(object): - def __init__(self, session): - self.tw = py.io.TerminalWriter() - self.session = session - self.reported_lines = [] - - @property - def verbosity(self): - if self.session: - return ( - self.session.config.option.verbose_level - self.session.config.option.quiet_level - ) - else: - return Verbosity.DEBUG - - def logpopen(self, popen, cmd_args_shell): - """ log information about the action.popen() created process. """ - if popen.outpath: - self.verbosity1(" {}$ {} >{}".format(popen.cwd, cmd_args_shell, popen.outpath)) - else: - self.verbosity1(" {}$ {} ".format(popen.cwd, cmd_args_shell)) - - def logaction_start(self, action): - msg = "{} {}".format(action.msg, " ".join(map(str, action.args))) - self.verbosity2("{} start: {}".format(action.venvname, msg), bold=True) - assert not hasattr(action, "_starttime") - action._starttime = time.time() - - def logaction_finish(self, action): - duration = time.time() - action._starttime - self.verbosity2( - "{} finish: {} after {:.2f} seconds".format(action.venvname, action.msg, duration), - bold=True, - ) - delattr(action, "_starttime") - - def startsummary(self): - if self.verbosity >= Verbosity.QUIET: - self.tw.sep("_", "summary") - - def logline_if(self, level, msg, key=None, **kwargs): - if self.verbosity >= level: - message = str(msg) if key is None else "{}{}".format(key, msg) - self.logline(message, **kwargs) - - def logline(self, msg, **opts): - self.reported_lines.append(msg) - self.tw.line("{}".format(msg), **opts) - - def keyboard_interrupt(self): - self.error("KEYBOARDINTERRUPT") - - def keyvalue(self, name, value): - if name.endswith(":"): - name += " " - self.tw.write(name, bold=True) - self.tw.write(value) - self.tw.line() - - def line(self, msg, **opts): - self.logline(msg, **opts) - - def info(self, msg): - self.logline_if(Verbosity.DEBUG, msg) - - def using(self, msg): - self.logline_if(Verbosity.INFO, msg, "using ", bold=True) - - def good(self, msg): - self.logline_if(Verbosity.QUIET, msg, green=True) - - def warning(self, msg): - self.logline_if(Verbosity.QUIET, msg, "WARNING: ", red=True) - - def error(self, msg): - self.logline_if(Verbosity.QUIET, msg, "ERROR: ", red=True) - - def skip(self, msg): - self.logline_if(Verbosity.QUIET, msg, "SKIPPED: ", yellow=True) - - def verbosity0(self, msg, **opts): - self.logline_if(Verbosity.DEFAULT, msg, **opts) - - def verbosity1(self, msg, **opts): - self.logline_if(Verbosity.INFO, msg, **opts) - - def verbosity2(self, msg, **opts): - self.logline_if(Verbosity.DEBUG, msg, **opts) - - -class Session: - """The session object that ties together configuration, reporting, venv creation, testing.""" - - def __init__(self, config, popen=subprocess.Popen, Report=Reporter): - self.config = config - self.popen = popen - self.resultlog = ResultLog() - self.report = Report(self) - self.make_emptydir(config.logdir) - config.logdir.ensure(dir=1) - self.report.using("tox.ini: {}".format(self.config.toxinipath)) - self._spec2pkg = {} - self._name2venv = {} - try: - self.venvlist = [self.getvenv(x) for x in self.evaluated_env_list()] - except LookupError: - raise SystemExit(1) - except tox.exception.ConfigError as exception: - self.report.error(str(exception)) - raise SystemExit(1) - try: - self.venv_order = stable_topological_sort( - OrderedDict((v.name, v.envconfig.depends) for v in self.venvlist) - ) - except ValueError as exception: - self.report.error("circular dependency detected: {}".format(exception)) - raise SystemExit(1) - self._actions = [] - - def evaluated_env_list(self): - tox_env_filter = os.environ.get("TOX_SKIP_ENV") - tox_env_filter_re = re.compile(tox_env_filter) if tox_env_filter is not None else None - for name in self.config.envlist: - if tox_env_filter_re is not None and tox_env_filter_re.match(name): - msg = "skip environment {}, matches filter {!r}".format( - name, tox_env_filter_re.pattern - ) - self.report.verbosity1(msg) - continue - yield name - - @property - def hook(self): - return self.config.pluginmanager.hook - - def _makevenv(self, name): - envconfig = self.config.envconfigs.get(name, None) - if envconfig is None: - self.report.error("unknown environment {!r}".format(name)) - raise LookupError(name) - elif envconfig.envdir == self.config.toxinidir: - self.report.error( - "venv {!r} in {} would delete project".format(name, envconfig.envdir) - ) - raise tox.exception.ConfigError("envdir must not equal toxinidir") - venv = VirtualEnv(envconfig=envconfig, session=self) - self._name2venv[name] = venv - return venv - - def getvenv(self, name): - """ return a VirtualEnv controler object for the 'name' env. """ - try: - return self._name2venv[name] - except KeyError: - return self._makevenv(name) - - def newaction(self, venv, msg, *args): - action = Action(self, venv, msg, args) - self._actions.append(action) - return action - - def runcommand(self): - self.report.using("tox-{} from {}".format(tox.__version__, tox.__file__)) - verbosity = self.report.verbosity > Verbosity.DEFAULT - if self.config.option.showconfig: - self.showconfig() - elif self.config.option.listenvs: - self.showenvs(all_envs=False, description=verbosity) - elif self.config.option.listenvs_all: - self.showenvs(all_envs=True, description=verbosity) - else: - with self.cleanup(): - return self.subcommand_test() - - @contextmanager - def cleanup(self): - self.config.temp_dir.ensure(dir=True) - try: - yield - finally: - for name in self.venv_order: - tox_env = self.getvenv(name) - if ( - hasattr(tox_env, "package") - and isinstance(tox_env.package, py.path.local) - and tox_env.package.exists() - ): - self.report.verbosity2("cleanup {}".format(tox_env.package)) - tox_env.package.remove() - py.path.local(tox_env.package.dirname).remove(ignore_errors=True) - - def make_emptydir(self, path): - if path.check(): - self.report.info(" removing {}".format(path)) - shutil.rmtree(str(path), ignore_errors=True) - path.ensure(dir=1) - - def setupenv(self, venv): - if venv.envconfig.missing_subs: - venv.status = ( - "unresolvable substitution(s): {}. " - "Environment variables are missing or defined recursively.".format( - ",".join(["'{}'".format(m) for m in venv.envconfig.missing_subs]) - ) - ) - return - if not venv.matching_platform(): - venv.status = "platform mismatch" - return # we simply omit non-matching platforms - with self.newaction(venv, "getenv", venv.envconfig.envdir) as action: - venv.status = 0 - default_ret_code = 1 - envlog = self.resultlog.get_envlog(venv.name) - try: - status = venv.update(action=action) - except IOError as e: - if e.args[0] != 2: - raise - status = ( - "Error creating virtualenv. Note that spaces in paths are " - "not supported by virtualenv. Error details: {!r}".format(e) - ) - except tox.exception.InvocationError as e: - status = ( - "Error creating virtualenv. Note that some special characters (e.g. ':' and " - "unicode symbols) in paths are not supported by virtualenv. Error details: " - "{!r}".format(e) - ) - except tox.exception.InterpreterNotFound as e: - status = e - if self.config.option.skip_missing_interpreters == "true": - default_ret_code = 0 - if status: - str_status = str(status) - commandlog = envlog.get_commandlog("setup") - commandlog.add_command(["setup virtualenv"], str_status, default_ret_code) - venv.status = status - if default_ret_code == 0: - self.report.skip(str_status) - else: - self.report.error(str_status) - return False - commandpath = venv.getcommandpath("python") - envlog.set_python_info(commandpath) - return True - - def finishvenv(self, venv): - with self.newaction(venv, "finishvenv"): - venv.finish() - return True - - def developpkg(self, venv, setupdir): - with self.newaction(venv, "developpkg", setupdir) as action: - try: - venv.developpkg(setupdir, action) - return True - except tox.exception.InvocationError as exception: - venv.status = exception - return False - - def installpkg(self, venv, path): - """Install package in the specified virtual environment. - - :param VenvConfig venv: Destination environment - :param str path: Path to the distribution package. - :return: True if package installed otherwise False. - :rtype: bool - """ - self.resultlog.set_header(installpkg=py.path.local(path)) - with self.newaction(venv, "installpkg", path) as action: - try: - venv.installpkg(path, action) - return True - except tox.exception.InvocationError as exception: - venv.status = exception - return False - - def subcommand_test(self): - if self.config.skipsdist: - self.report.info("skipping sdist step") - else: - for name in self.venv_order: - venv = self.getvenv(name) - if not venv.envconfig.skip_install: - venv.package = self.hook.tox_package(session=self, venv=venv) - if not venv.package: - return 2 - venv.envconfig.setenv[str("TOX_PACKAGE")] = str(venv.package) - if self.config.option.sdistonly: - return - - within_parallel = PARALLEL_ENV_VAR_KEY in os.environ - if not within_parallel and self.config.option.parallel != PARALLEL_OFF: - self.run_parallel() - else: - self.run_sequential() - retcode = self._summary() - return retcode - - def run_sequential(self): - for name in self.venv_order: - venv = self.getvenv(name) - if self.setupenv(venv): - if venv.envconfig.skip_install: - self.finishvenv(venv) - else: - if venv.envconfig.usedevelop: - self.developpkg(venv, self.config.setupdir) - elif self.config.skipsdist: - self.finishvenv(venv) - else: - self.installpkg(venv, venv.package) - - self.runenvreport(venv) - self.runtestenv(venv) - - def run_parallel(self): - """here we'll just start parallel sub-processes""" - live_out = self.config.option.parallel_live - args = [sys.executable, "-m", "tox"] + self.config.args - try: - position = args.index("--") - except ValueError: - position = len(args) - try: - parallel_at = args[0:position].index("--parallel") - del args[parallel_at] - position -= 1 - except ValueError: - pass - - max_parallel = self.config.option.parallel - if max_parallel is None: - max_parallel = len(self.venv_order) - semaphore = Semaphore(max_parallel) - finished = Event() - sink = None if live_out else subprocess.PIPE - - show_progress = not live_out and self.report.verbosity > Verbosity.QUIET - with Spinner(enabled=show_progress) as spinner: - - def run_in_thread(tox_env, os_env): - res = None - env_name = tox_env.envconfig.envname - try: - os_env[str(PARALLEL_ENV_VAR_KEY)] = str(env_name) - args_sub = list(args) - if hasattr(tox_env, "package"): - args_sub.insert(position, str(tox_env.package)) - args_sub.insert(position, "--installpkg") - process = subprocess.Popen( - args_sub, - env=os_env, - stdout=sink, - stderr=sink, - stdin=None, - universal_newlines=True, - ) - res = process.wait() - finally: - semaphore.release() - finished.set() - tox_env.status = ( - "skipped tests" - if self.config.option.notest - else ("parallel child exit code {}".format(res) if res else res) - ) - done.add(env_name) - report = spinner.succeed - if self.config.option.notest: - report = spinner.skip - elif res: - report = spinner.fail - report(env_name) - - if not live_out: - out, err = process.communicate() - if res or tox_env.envconfig.parallel_show_output: - outcome = ( - "Failed {} under process {}, stdout:\n".format(env_name, process.pid) - if res - else "" - ) - message = "{}{}{}".format( - outcome, out, "\nstderr:\n{}".format(err) if err else "" - ).rstrip() - self.report.logline_if(Verbosity.QUIET, message) - - threads = [] - todo_keys = set(self.venv_order) - todo = OrderedDict( - (i, todo_keys & set(self.getvenv(i).envconfig.depends)) for i in self.venv_order - ) - done = set() - while todo: - for name, depends in list(todo.items()): - if depends - done: - # skip if has unfinished dependencies - continue - del todo[name] - venv = self.getvenv(name) - semaphore.acquire(blocking=True) - spinner.add(name) - thread = Thread(target=run_in_thread, args=(venv, os.environ.copy())) - thread.start() - threads.append(thread) - if todo: - # wait until someone finishes and retry queuing jobs - finished.wait() - finished.clear() - - for thread in threads: - thread.join() - - def runenvreport(self, venv): - """ - Run an environment report to show which package - versions are installed in the venv - """ - with self.newaction(venv, "envreport") as action: - packages = self.hook.tox_runenvreport(venv=venv, action=action) - action.setactivity("installed", ",".join(packages)) - envlog = self.resultlog.get_envlog(venv.name) - envlog.set_installed(packages) - - def runtestenv(self, venv, redirect=False): - if venv.status == 0 and self.config.option.notest: - venv.status = "skipped tests" - else: - if venv.status: - return - self.hook.tox_runtest_pre(venv=venv) - if venv.status == 0: - self.hook.tox_runtest(venv=venv, redirect=redirect) - self.hook.tox_runtest_post(venv=venv) - - def _summary(self): - is_parallel_child = PARALLEL_ENV_VAR_KEY in os.environ - if not is_parallel_child: - self.report.startsummary() - exit_code = 0 - for name in self.venv_order: - venv = self.getvenv(name) - reporter = self.report.good - status = venv.status - if isinstance(status, tox.exception.InterpreterNotFound): - msg = " {}: {}".format(venv.envconfig.envname, str(status)) - if self.config.option.skip_missing_interpreters == "true": - reporter = self.report.skip - else: - exit_code = 1 - reporter = self.report.error - elif status == "platform mismatch": - msg = " {}: {}".format(venv.envconfig.envname, str(status)) - reporter = self.report.skip - elif status and status == "ignored failed command": - msg = " {}: {}".format(venv.envconfig.envname, str(status)) - elif status and status != "skipped tests": - msg = " {}: {}".format(venv.envconfig.envname, str(status)) - reporter = self.report.error - exit_code = 1 - else: - if not status: - status = "commands succeeded" - msg = " {}: {}".format(venv.envconfig.envname, status) - if not is_parallel_child: - reporter(msg) - if not exit_code and not is_parallel_child: - self.report.good(" congratulations :)") - if not is_parallel_child: - path = self.config.option.resultjson - if path: - path = py.path.local(path) - path.write(self.resultlog.dumps_json()) - self.report.line("wrote json report at: {}".format(path)) - return exit_code - - def showconfig(self): - self.info_versions() - self.report.keyvalue("config-file:", self.config.option.configfile) - self.report.keyvalue("toxinipath: ", self.config.toxinipath) - self.report.keyvalue("toxinidir: ", self.config.toxinidir) - self.report.keyvalue("toxworkdir: ", self.config.toxworkdir) - self.report.keyvalue("setupdir: ", self.config.setupdir) - self.report.keyvalue("distshare: ", self.config.distshare) - self.report.keyvalue("skipsdist: ", self.config.skipsdist) - self.report.tw.line() - for envconfig in self.config.envconfigs.values(): - self.report.line("[testenv:{}]".format(envconfig.envname), bold=True) - for attr in self.config._parser._testenv_attr: - self.report.line(" {:<15} = {}".format(attr.name, getattr(envconfig, attr.name))) - - def showenvs(self, all_envs=False, description=False): - env_conf = self.config.envconfigs # this contains all environments - default = self.config.envlist # this only the defaults - ignore = {self.config.isolated_build_env}.union(default) - extra = [e for e in env_conf if e not in ignore] if all_envs else [] - - if description: - self.report.line("default environments:") - max_length = max(len(env) for env in (default + extra)) - - def report_env(e): - if description: - text = env_conf[e].description or "[no description]" - msg = "{} -> {}".format(e.ljust(max_length), text).strip() - else: - msg = e - self.report.line(msg) - - for e in default: - report_env(e) - if all_envs and extra: - if description: - self.report.line("") - self.report.line("additional environments:") - for e in extra: - report_env(e) - - def info_versions(self): - versions = ["tox-{}".format(tox.__version__)] - proc = subprocess.Popen( - (sys.executable, "-m", "virtualenv", "--version"), stdout=subprocess.PIPE - ) - out, _ = proc.communicate() - versions.append("virtualenv-{}".format(out.decode("UTF-8").strip())) - self.report.keyvalue("tool-versions:", " ".join(versions)) - - def _resolve_package(self, package_spec): - try: - return self._spec2pkg[package_spec] - except KeyError: - self._spec2pkg[package_spec] = x = self._get_latest_version_of_package(package_spec) - return x - - def _get_latest_version_of_package(self, package_spec): - if not os.path.isabs(str(package_spec)): - return package_spec - p = py.path.local(package_spec) - if p.check(): - return p - if not p.dirpath().check(dir=1): - raise tox.exception.MissingDirectory(p.dirpath()) - self.report.info("determining {}".format(p)) - candidates = p.dirpath().listdir(p.basename) - if len(candidates) == 0: - raise tox.exception.MissingDependency(package_spec) - if len(candidates) > 1: - version_package = [] - for filename in candidates: - version = get_version_from_filename(filename.basename) - if version is not None: - version_package.append((version, filename)) - else: - self.report.warning("could not determine version of: {}".format(str(filename))) - if not version_package: - raise tox.exception.MissingDependency(package_spec) - version_package.sort() - _, package_with_largest_version = version_package[-1] - return package_with_largest_version - else: - return candidates[0] - - -_REGEX_FILE_NAME_WITH_VERSION = re.compile(r"[\w_\-\+\.]+-(.*)\.(zip|tar\.gz)") - - -def get_version_from_filename(basename): - m = _REGEX_FILE_NAME_WITH_VERSION.match(basename) - if m is None: - return None - version = m.group(1) - try: - - return pkg_resources.packaging.version.Version(version) - except pkg_resources.packaging.version.InvalidVersion: - return None diff --git a/src/tox/session/__init__.py b/src/tox/session/__init__.py new file mode 100644 index 000000000..2509f2b7c --- /dev/null +++ b/src/tox/session/__init__.py @@ -0,0 +1,251 @@ +""" +Automatically package and test a Python project against configurable +Python2 and Python3 based virtual environments. Environments are +setup by using virtualenv. Configuration is generally done through an +INI-style "tox.ini" file. +""" + +import os +import re +import subprocess +import sys +from collections import OrderedDict +from contextlib import contextmanager + +import py + +import tox +from tox import reporter +from tox.config import parseconfig +from tox.config.parallel import ENV_VAR_KEY as PARALLEL_ENV_VAR_KEY +from tox.config.parallel import OFF_VALUE as PARALLEL_OFF +from tox.logs.result import ResultLog +from tox.util import set_os_env_var +from tox.util.graph import stable_topological_sort +from tox.util.path import ensure_empty_dir +from tox.venv import VirtualEnv +from tox.action import Action +from .commands.help import show_help +from .commands.help_ini import show_help_ini +from .commands.run.parallel import run_parallel +from .commands.run.sequential import run_sequential +from .commands.show_config import show_config +from .commands.show_env import show_envs +from tox.reporter import update_default_reporter + + +def cmdline(args=None): + if args is None: + args = sys.argv[1:] + main(args) + + +def setup_reporter(args): + from argparse import ArgumentParser + from tox.config.reporter import add_verbosity_commands + + parser = ArgumentParser(add_help=False) + add_verbosity_commands(parser) + options, _ = parser.parse_known_args(args) + update_default_reporter(options.quiet_level, options.verbose_level) + + +def main(args): + setup_reporter(args) + try: + config = load_config(args) + update_default_reporter(config.option.quiet_level, config.option.verbose_level) + reporter.using("tox.ini: {}".format(config.toxinipath)) + config.logdir.ensure(dir=1) + ensure_empty_dir(config.logdir) + with set_os_env_var("TOX_WORK_DIR", config.toxworkdir): + retcode = build_session(config).runcommand() + if retcode is None: + retcode = 0 + raise SystemExit(retcode) + except KeyboardInterrupt: + raise SystemExit(2) + except (tox.exception.MinVersionError, tox.exception.MissingRequirement) as exception: + reporter.error(str(exception)) + raise SystemExit(1) + + +def load_config(args): + config = parseconfig(args) + if config.option.help: + show_help(config) + raise SystemExit(0) + elif config.option.helpini: + show_help_ini(config) + raise SystemExit(0) + return config + + +def build_session(config): + return Session(config) + + +class Session: + """The session object that ties together configuration, reporting, venv creation, testing.""" + + def __init__(self, config, popen=subprocess.Popen): + self.config = config + self.popen = popen + self.resultlog = ResultLog() + self.existing_venvs = OrderedDict() + self.venv_dict = self._build_venvs() + + def _build_venvs(self): + try: + need_to_run = OrderedDict((v, self.getvenv(v)) for v in self._evaluated_env_list) + try: + venv_order = stable_topological_sort( + OrderedDict((name, v.envconfig.depends) for name, v in need_to_run.items()) + ) + + venvs = OrderedDict((v, need_to_run[v]) for v in venv_order) + return venvs + except ValueError as exception: + reporter.error("circular dependency detected: {}".format(exception)) + except LookupError: + pass + except tox.exception.ConfigError as exception: + reporter.error(str(exception)) + raise SystemExit(1) + + def getvenv(self, name): + if name in self.existing_venvs: + return self.existing_venvs[name] + env_config = self.config.envconfigs.get(name, None) + if env_config is None: + reporter.error("unknown environment {!r}".format(name)) + raise LookupError(name) + elif env_config.envdir == self.config.toxinidir: + reporter.error("venv {!r} in {} would delete project".format(name, env_config.envdir)) + raise tox.exception.ConfigError("envdir must not equal toxinidir") + env_log = self.resultlog.get_envlog(name) + venv = VirtualEnv(envconfig=env_config, popen=self.popen, env_log=env_log) + self.existing_venvs[name] = venv + return venv + + @property + def _evaluated_env_list(self): + tox_env_filter = os.environ.get("TOX_SKIP_ENV") + tox_env_filter_re = re.compile(tox_env_filter) if tox_env_filter is not None else None + visited = set() + for name in self.config.envlist: + if name in visited: + continue + visited.add(name) + if tox_env_filter_re is not None and tox_env_filter_re.match(name): + msg = "skip environment {}, matches filter {!r}".format( + name, tox_env_filter_re.pattern + ) + reporter.verbosity1(msg) + continue + yield name + + @property + def hook(self): + return self.config.pluginmanager.hook + + def newaction(self, env, msg, *args): + new_action = Action( + env, + msg, + args, + self.config.logdir, + self.config.option.resultjson, + self.resultlog.command_log, + self.popen, + sys.executable, + ) + return new_action + + def runcommand(self): + reporter.using("tox-{} from {}".format(tox.__version__, tox.__file__)) + show_description = reporter.has_level(reporter.Verbosity.DEFAULT) + if self.config.option.showconfig: + self.showconfig() + elif self.config.option.listenvs: + self.showenvs(all_envs=False, description=show_description) + elif self.config.option.listenvs_all: + self.showenvs(all_envs=True, description=show_description) + else: + with self.cleanup(): + return self.subcommand_test() + + @contextmanager + def cleanup(self): + self.config.temp_dir.ensure(dir=True) + try: + yield + finally: + self.hook.tox_cleanup(session=self) + + def subcommand_test(self): + if self.config.skipsdist: + reporter.info("skipping sdist step") + else: + for venv in self.venv_dict.values(): + if not venv.envconfig.skip_install: + venv.package = self.hook.tox_package(session=self, venv=venv) + if not venv.package: + return 2 + venv.envconfig.setenv[str("TOX_PACKAGE")] = str(venv.package) + if self.config.option.sdistonly: + return + + within_parallel = PARALLEL_ENV_VAR_KEY in os.environ + if not within_parallel and self.config.option.parallel != PARALLEL_OFF: + run_parallel(self.config, self.venv_dict) + else: + run_sequential(self.config, self.venv_dict) + retcode = self._summary() + return retcode + + def _summary(self): + is_parallel_child = PARALLEL_ENV_VAR_KEY in os.environ + if not is_parallel_child: + reporter.separator("_", "summary", reporter.Verbosity.QUIET) + exit_code = 0 + for venv in self.venv_dict.values(): + report = reporter.good + status = venv.status + if isinstance(status, tox.exception.InterpreterNotFound): + msg = " {}: {}".format(venv.envconfig.envname, str(status)) + if self.config.option.skip_missing_interpreters == "true": + report = reporter.skip + else: + exit_code = 1 + report = reporter.error + elif status == "platform mismatch": + msg = " {}: {}".format(venv.envconfig.envname, str(status)) + report = reporter.skip + elif status and status == "ignored failed command": + msg = " {}: {}".format(venv.envconfig.envname, str(status)) + elif status and status != "skipped tests": + msg = " {}: {}".format(venv.envconfig.envname, str(status)) + report = reporter.error + exit_code = 1 + else: + if not status: + status = "commands succeeded" + msg = " {}: {}".format(venv.envconfig.envname, status) + if not is_parallel_child: + report(msg) + if not exit_code and not is_parallel_child: + reporter.good(" congratulations :)") + if not is_parallel_child: + path = self.config.option.resultjson + if path: + path = py.path.local(path) + path.write(self.resultlog.dumps_json()) + reporter.line("wrote json report at: {}".format(path)) + return exit_code + + def showconfig(self): + show_config(self.config) + + def showenvs(self, all_envs=False, description=False): + show_envs(self.config, all_envs=all_envs, description=description) diff --git a/src/tox/session/commands/__init__.py b/src/tox/session/commands/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/tox/session/commands/help.py b/src/tox/session/commands/help.py new file mode 100644 index 000000000..bd9f55848 --- /dev/null +++ b/src/tox/session/commands/help.py @@ -0,0 +1,13 @@ +from tox import reporter + + +def show_help(config): + reporter.line(config._parser._format_help()) + reporter.line("Environment variables", bold=True) + reporter.line("TOXENV: comma separated list of environments (overridable by '-e')") + reporter.line("TOX_SKIP_ENV: regular expression to filter down from running tox environments") + reporter.line( + "TOX_TESTENV_PASSENV: space-separated list of extra environment variables to be " + "passed into test command environments" + ) + reporter.line("PY_COLORS: 0 disable colorized output, 1 enable (default)") diff --git a/src/tox/session/commands/help_ini.py b/src/tox/session/commands/help_ini.py new file mode 100644 index 000000000..eb1f85b34 --- /dev/null +++ b/src/tox/session/commands/help_ini.py @@ -0,0 +1,14 @@ +from tox import reporter + + +def show_help_ini(config): + reporter.separator("-", "per-testenv attributes", reporter.Verbosity.INFO) + for env_attr in config._testenv_attr: + reporter.line( + "{:<15} {:<8} default: {}".format( + env_attr.name, "<{}>".format(env_attr.type), env_attr.default + ), + bold=True, + ) + reporter.line(env_attr.help) + reporter.line("") diff --git a/src/tox/session/commands/run/__init__.py b/src/tox/session/commands/run/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/tox/session/commands/run/parallel.py b/src/tox/session/commands/run/parallel.py new file mode 100644 index 000000000..c9ee9c568 --- /dev/null +++ b/src/tox/session/commands/run/parallel.py @@ -0,0 +1,106 @@ +import os +import subprocess +import sys +from collections import OrderedDict +from threading import Event, Semaphore, Thread + +from tox import reporter +from tox.config.parallel import ENV_VAR_KEY as PARALLEL_ENV_VAR_KEY +from tox.util.spinner import Spinner + + +def run_parallel(config, venv_dict): + """here we'll just start parallel sub-processes""" + live_out = config.option.parallel_live + args = [sys.executable, "-m", "tox"] + config.args + try: + position = args.index("--") + except ValueError: + position = len(args) + try: + parallel_at = args[0:position].index("--parallel") + del args[parallel_at] + position -= 1 + except ValueError: + pass + + max_parallel = config.option.parallel + if max_parallel is None: + max_parallel = len(venv_dict) + semaphore = Semaphore(max_parallel) + finished = Event() + sink = None if live_out else subprocess.PIPE + + show_progress = not live_out and reporter.verbosity() > reporter.Verbosity.QUIET + with Spinner(enabled=show_progress) as spinner: + + def run_in_thread(tox_env, os_env): + res = None + env_name = tox_env.envconfig.envname + try: + os_env[str(PARALLEL_ENV_VAR_KEY)] = str(env_name) + args_sub = list(args) + if hasattr(tox_env, "package"): + args_sub.insert(position, str(tox_env.package)) + args_sub.insert(position, "--installpkg") + process = subprocess.Popen( + args_sub, + env=os_env, + stdout=sink, + stderr=sink, + stdin=None, + universal_newlines=True, + ) + res = process.wait() + finally: + semaphore.release() + finished.set() + tox_env.status = ( + "skipped tests" + if config.option.notest + else ("parallel child exit code {}".format(res) if res else res) + ) + done.add(env_name) + outcome = spinner.succeed + if config.option.notest: + outcome = spinner.skip + elif res: + outcome = spinner.fail + outcome(env_name) + + if not live_out: + out, err = process.communicate() + if res or tox_env.envconfig.parallel_show_output: + outcome = ( + "Failed {} under process {}, stdout:\n".format(env_name, process.pid) + if res + else "" + ) + message = "{}{}{}".format( + outcome, out, "\nstderr:\n{}".format(err) if err else "" + ).rstrip() + reporter.quiet(message) + + threads = [] + todo_keys = set(venv_dict.keys()) + todo = OrderedDict((n, todo_keys & set(v.envconfig.depends)) for n, v in venv_dict.items()) + done = set() + while todo: + for name, depends in list(todo.items()): + if depends - done: + # skip if has unfinished dependencies + continue + del todo[name] + venv = venv_dict[name] + semaphore.acquire(blocking=True) + spinner.add(name) + thread = Thread(target=run_in_thread, args=(venv, os.environ.copy())) + thread.start() + threads.append(thread) + if todo: + # wait until someone finishes and retry queuing jobs + finished.wait() + finished.clear() + + for thread in threads: + thread.join() diff --git a/src/tox/session/commands/run/sequential.py b/src/tox/session/commands/run/sequential.py new file mode 100644 index 000000000..ccb4c7f58 --- /dev/null +++ b/src/tox/session/commands/run/sequential.py @@ -0,0 +1,72 @@ +import py + +import tox +from tox.exception import InvocationError + + +def run_sequential(config, venv_dict): + for venv in venv_dict.values(): + if venv.setupenv(): + if venv.envconfig.skip_install: + venv.finishvenv() + else: + if venv.envconfig.usedevelop: + develop_pkg(venv, config.setupdir) + elif config.skipsdist: + venv.finishvenv() + else: + installpkg(venv, venv.package) + + runenvreport(venv, config) + runtestenv(venv, config) + + +def develop_pkg(venv, setupdir): + with venv.new_action("developpkg", setupdir) as action: + try: + venv.developpkg(setupdir, action) + return True + except InvocationError as exception: + venv.status = exception + return False + + +def installpkg(venv, path): + """Install package in the specified virtual environment. + + :param VenvConfig venv: Destination environment + :param str path: Path to the distribution package. + :return: True if package installed otherwise False. + :rtype: bool + """ + venv.env_log.set_header(installpkg=py.path.local(path)) + with venv.new_action("installpkg", path) as action: + try: + venv.installpkg(path, action) + return True + except tox.exception.InvocationError as exception: + venv.status = exception + return False + + +def runenvreport(venv, config): + """ + Run an environment report to show which package + versions are installed in the venv + """ + with venv.new_action("envreport") as action: + packages = config.pluginmanager.hook.tox_runenvreport(venv=venv, action=action) + action.setactivity("installed", ",".join(packages)) + venv.env_log.set_installed(packages) + + +def runtestenv(venv, config, redirect=False): + if venv.status == 0 and config.option.notest: + venv.status = "skipped tests" + else: + if venv.status: + return + config.pluginmanager.hook.tox_runtest_pre(venv=venv) + if venv.status == 0: + config.pluginmanager.hook.tox_runtest(venv=venv, redirect=redirect) + config.pluginmanager.hook.tox_runtest_post(venv=venv) diff --git a/src/tox/session/commands/show_config.py b/src/tox/session/commands/show_config.py new file mode 100644 index 000000000..d588ac80d --- /dev/null +++ b/src/tox/session/commands/show_config.py @@ -0,0 +1,31 @@ +import subprocess +import sys + +from tox import reporter as report +from tox.version import __version__ + + +def show_config(config): + info_versions() + report.keyvalue("config-file:", config.option.configfile) + report.keyvalue("toxinipath: ", config.toxinipath) + report.keyvalue("toxinidir: ", config.toxinidir) + report.keyvalue("toxworkdir: ", config.toxworkdir) + report.keyvalue("setupdir: ", config.setupdir) + report.keyvalue("distshare: ", config.distshare) + report.keyvalue("skipsdist: ", config.skipsdist) + report.line("") + for envconfig in config.envconfigs.values(): + report.line("[testenv:{}]".format(envconfig.envname), bold=True) + for attr in config._parser._testenv_attr: + report.line(" {:<15} = {}".format(attr.name, getattr(envconfig, attr.name))) + + +def info_versions(): + versions = ["tox-{}".format(__version__)] + proc = subprocess.Popen( + (sys.executable, "-m", "virtualenv", "--version"), stdout=subprocess.PIPE + ) + out, _ = proc.communicate() + versions.append("virtualenv-{}".format(out.decode("UTF-8").strip())) + report.keyvalue("tool-versions:", " ".join(versions)) diff --git a/src/tox/session/commands/show_env.py b/src/tox/session/commands/show_env.py new file mode 100644 index 000000000..02d4d1c61 --- /dev/null +++ b/src/tox/session/commands/show_env.py @@ -0,0 +1,31 @@ +from __future__ import absolute_import, unicode_literals + +from tox import reporter as report + + +def show_envs(config, all_envs=False, description=False): + env_conf = config.envconfigs # this contains all environments + default = config.envlist # this only the defaults + ignore = {config.isolated_build_env}.union(default) + extra = [e for e in env_conf if e not in ignore] if all_envs else [] + + if description: + report.line("default environments:") + max_length = max(len(env) for env in (default + extra)) + + def report_env(e): + if description: + text = env_conf[e].description or "[no description]" + msg = "{} -> {}".format(e.ljust(max_length), text).strip() + else: + msg = e + report.line(msg) + + for e in default: + report_env(e) + if all_envs and extra: + if description: + report.line("") + report.line("additional environments:") + for e in extra: + report_env(e) diff --git a/src/tox/util/lock.py b/src/tox/util/lock.py new file mode 100644 index 000000000..7f7ad7ce1 --- /dev/null +++ b/src/tox/util/lock.py @@ -0,0 +1,39 @@ +"""holds locking functionality that works across processes""" +from __future__ import absolute_import, unicode_literals + +from contextlib import contextmanager + +import py +from filelock import FileLock, Timeout + + +@contextmanager +def get(lock_file, report): + py.path.local(lock_file.dirname).ensure(dir=1) + lock = FileLock(str(lock_file)) + try: + try: + lock.acquire(0.0001) + except Timeout: + report("lock file {} present, will block until released".format(lock_file)) + lock.acquire() + yield + finally: + lock.release(force=True) + + +def get_unique_file(path, prefix, suffix, report): + """get a unique file in a folder having a given prefix and suffix, + with unique number in between""" + lock_file = path.join(".lock") + prefix = "{}-".format(prefix) + with get(lock_file, report): + max_value = -1 + for candidate in path.listdir("{}*{}".format(prefix, suffix)): + try: + max_value = max(max_value, int(candidate.name[len(prefix) : -len(suffix)])) + except ValueError: + continue + winner = path.join("{}-{}.log".format(prefix, max_value + 1)) + winner.ensure(dir=0) + return winner diff --git a/src/tox/util/path.py b/src/tox/util/path.py new file mode 100644 index 000000000..b7a299810 --- /dev/null +++ b/src/tox/util/path.py @@ -0,0 +1,10 @@ +import shutil + +from tox import reporter + + +def ensure_empty_dir(path): + if path.check(): + reporter.info(" removing {}".format(path)) + shutil.rmtree(str(path), ignore_errors=True) + path.ensure(dir=1) diff --git a/src/tox/venv.py b/src/tox/venv.py index 6712531e6..a9022854b 100644 --- a/src/tox/venv.py +++ b/src/tox/venv.py @@ -10,6 +10,10 @@ from pkg_resources import to_filename import tox +from tox import reporter +from tox.action import Action +from tox.package.local import resolve_package +from tox.util.path import ensure_empty_dir from .config import DepConfig @@ -103,9 +107,25 @@ def matches(self, other, deps_matches_subset=False): class VirtualEnv(object): - def __init__(self, envconfig=None, session=None): + def __init__(self, envconfig=None, popen=None, env_log=None): self.envconfig = envconfig - self.session = session + self.popen = popen + self._actions = [] + self.env_log = env_log + + def new_action(self, msg, *args): + config = self.envconfig.config + command_log = self.env_log.get_commandlog("test" if msg == "runtest" else "setup") + return Action( + self.name, + msg, + args, + self.envconfig.envlogdir, + config.option.resultjson, + command_log, + self.popen, + self.envconfig.envpython, + ) @property def hook(self): @@ -169,7 +189,7 @@ def _normal_lookup(self, name): def _check_external_allowed_and_warn(self, path): if not self.is_allowed_external(path): - self.session.report.warning( + reporter.warning( "test command found but not installed in testenv\n" " cmd: {}\n" " env: {}\n" @@ -248,7 +268,7 @@ def get_resolved_dependencies(self): dependencies = [] for dependency in self.envconfig.deps: if dependency.indexserver is None: - package = self.session._resolve_package(package_spec=dependency.name) + package = resolve_package(package_spec=dependency.name) if package != dependency.name: dependency = dependency.__class__(package) dependencies.append(dependency) @@ -265,9 +285,7 @@ def finish(self): live_config = self._getliveconfig() if previous_config is None or not previous_config.matches(live_config): content = live_config.writeconfig(self.path_config) - self.session.report.verbosity1( - "write config to {} as {!r}".format(self.path_config, content) - ) + reporter.verbosity1("write config to {} as {!r}".format(self.path_config, content)) def _needs_reinstall(self, setupdir, action): setup_py = setupdir.join("setup.py") @@ -324,9 +342,9 @@ def install_pkg(self, dir, action, name, is_develop=False): return action.setactivity("{}-nodeps".format(name), dir) pip_flags = ["--no-deps"] + ([] if is_develop else ["-U"]) - pip_flags.extend(["-v"] * min(3, action.report.verbosity - 2)) - if action.venv.envconfig.extras: - dir += "[{}]".format(",".join(action.venv.envconfig.extras)) + pip_flags.extend(["-v"] * min(3, reporter.verbosity() - 2)) + if self.envconfig.extras: + dir += "[{}]".format(",".join(self.envconfig.extras)) target = [dir] if is_develop: target.insert(0, "-e") @@ -369,7 +387,7 @@ def expand(val): cmd, cwd=self.envconfig.config.toxinidir, action=action, - redirect=self.session.report.verbosity < 2, + redirect=reporter.verbosity() < reporter.Verbosity.DEBUG, ) finally: sys.stdout = old_stdout @@ -380,7 +398,7 @@ def ensure_pip_os_environ_ok(self): if "PYTHONPATH" not in self.envconfig.passenv: # If PYTHONPATH not explicitly asked for, remove it. if "PYTHONPATH" in os.environ: - self.session.report.warning( + reporter.warning( "Discarding $PYTHONPATH from environment, to override " "specify PYTHONPATH in 'passenv' in your configuration." ) @@ -450,7 +468,7 @@ def test( ignore_outcome = self.envconfig.ignore_outcome if ignore_errors is None: ignore_errors = self.envconfig.ignore_errors - with self.session.newaction(self, name) as action: + with self.new_action(name) as action: cwd = self.envconfig.changedir if display_hash_seed: env = self._get_os_environ(is_test_command=True) @@ -486,17 +504,17 @@ def test( except tox.exception.InvocationError as err: if ignore_outcome: msg = "command failed but result from testenv is ignored\ncmd:" - self.session.report.warning("{} {}".format(msg, err)) + reporter.warning("{} {}".format(msg, err)) self.status = "ignored failed command" continue # keep processing commands - self.session.report.error(str(err)) + reporter.error(str(err)) self.status = "commands failed" if not ignore_errors: break # Don't process remaining commands except KeyboardInterrupt: self.status = "keyboardinterrupt" - self.session.report.error(self.status) + reporter.error(self.status) raise def _pcall( @@ -515,7 +533,7 @@ def _pcall( env = self._get_os_environ(is_test_command=is_test_command) bin_dir = str(self.envconfig.envbindir) env["PATH"] = os.pathsep.join([bin_dir, os.environ["PATH"]]) - self.session.report.verbosity2("setting PATH={}".format(env["PATH"])) + reporter.verbosity2("setting PATH={}".format(env["PATH"])) # get command args[0] = self.getcommandpath(args[0], venv, cwd) @@ -527,6 +545,60 @@ def _pcall( args, cwd=cwd, env=env, redirect=redirect, ignore_ret=ignore_ret, returnout=returnout ) + def setupenv(self): + if self.envconfig.missing_subs: + self.status = ( + "unresolvable substitution(s): {}. " + "Environment variables are missing or defined recursively.".format( + ",".join(["'{}'".format(m) for m in self.envconfig.missing_subs]) + ) + ) + return + if not self.matching_platform(): + self.status = "platform mismatch" + return # we simply omit non-matching platforms + with self.new_action("getenv", self.envconfig.envdir) as action: + self.status = 0 + default_ret_code = 1 + envlog = self.env_log + try: + status = self.update(action=action) + except IOError as e: + if e.args[0] != 2: + raise + status = ( + "Error creating virtualenv. Note that spaces in paths are " + "not supported by virtualenv. Error details: {!r}".format(e) + ) + except tox.exception.InvocationError as e: + status = ( + "Error creating virtualenv. Note that some special characters (e.g. ':' and " + "unicode symbols) in paths are not supported by virtualenv. Error details: " + "{!r}".format(e) + ) + except tox.exception.InterpreterNotFound as e: + status = e + if self.envconfig.config.option.skip_missing_interpreters == "true": + default_ret_code = 0 + if status: + str_status = str(status) + command_log = envlog.get_commandlog("setup") + command_log.add_command(["setup virtualenv"], str_status, default_ret_code) + self.status = status + if default_ret_code == 0: + reporter.skip(str_status) + else: + reporter.error(str_status) + return False + command_path = self.getcommandpath("python") + envlog.set_python_info(command_path) + return True + + def finishvenv(self): + with self.new_action("finishvenv"): + self.finish() + return True + def getdigest(path): path = py.path.local(path) @@ -574,7 +646,7 @@ def tox_testenv_create(venv, action): args.append("--no-download") # add interpreter explicitly, to prevent using default (virtualenv.ini) args.extend(["--python", str(config_interpreter)]) - venv.session.make_emptydir(venv.path) + ensure_empty_dir(venv.path) basepath = venv.path.dirpath() basepath.ensure(dir=1) args.append(venv.path.basename) @@ -601,7 +673,7 @@ def tox_runtest(venv, redirect): @tox.hookimpl def tox_runtest_pre(venv): venv.status = 0 - venv.session.make_emptydir(venv.envconfig.envtmpdir) + ensure_empty_dir(venv.envconfig.envtmpdir) venv.envconfig.envtmpdir.ensure(dir=1) venv.test( name="run-test-pre", diff --git a/tests/unit/config/test_config.py b/tests/unit/config/test_config.py index 00524bc48..c56aad6a7 100644 --- a/tests/unit/config/test_config.py +++ b/tests/unit/config/test_config.py @@ -20,7 +20,6 @@ parseconfig, ) from tox.config.parallel import ENV_VAR_KEY as PARALLEL_ENV_VAR_KEY -from tox.venv import VirtualEnv class TestVenvConfig: @@ -192,9 +191,9 @@ def test_config_parse_platform_rex(self, newconfig, mocksession, monkeypatch): platform = a123|b123 """, ) + mocksession.config = config assert len(config.envconfigs) == 1 - envconfig = config.envconfigs["py1"] - venv = VirtualEnv(envconfig, session=mocksession) + venv = mocksession.getvenv("py1") assert not venv.matching_platform() monkeypatch.setattr(sys, "platform", "a123") assert venv.matching_platform() @@ -2601,7 +2600,6 @@ def test_help(self, cmd, initproj): initproj("help", filedefs={"tox.ini": ""}) result = cmd("-h") assert not result.ret - assert not result.err assert re.match(r"usage:.*help.*", result.out, re.DOTALL) def test_version_simple(self, cmd, initproj): @@ -2665,9 +2663,8 @@ def test_no_tox_ini(self, cmd, initproj): initproj("noini-0.5") result = cmd() assert result.ret - assert result.out == "" msg = "ERROR: tox config file (either pyproject.toml, tox.ini, setup.cfg) not found\n" - assert result.err == msg + assert result.out == msg def test_override_workdir(self, cmd, initproj): baddir = "badworkdir-123" @@ -2853,17 +2850,19 @@ def test_commands_with_backslash(self, newconfig): def test_plugin_require(newconfig): inisource = """ [tox] - requires = tox + requires = setuptools name[foo,bar]>=2,<3; python_version>"2.0" and os_name=='a' b """ with pytest.raises(tox.exception.MissingRequirement) as exc_info: newconfig([], inisource) - assert exc_info.value.args[0] == ( - r'Packages name[bar,foo]<3,>=2; python_version > "2.0" and os_name == "a", b ' - r"need to be installed alongside tox in {}".format(sys.executable) + expected = ( + 'Packages name[bar,foo]<3,>=2; python_version > "2.0" and os_name == "a", b ' + "need to be installed alongside tox in {}".format(sys.executable) ) + actual = exc_info.value.args[0] + assert actual == expected def test_isolated_build_env_cannot_be_in_envlist(newconfig, capsys): diff --git a/tests/unit/package/builder/test_package_builder_isolated.py b/tests/unit/package/builder/test_package_builder_isolated.py index 5e962911b..b20e28d06 100644 --- a/tests/unit/package/builder/test_package_builder_isolated.py +++ b/tests/unit/package/builder/test_package_builder_isolated.py @@ -4,7 +4,7 @@ import pytest from tox.package.builder.isolated import get_build_info -from tox.session import Reporter +from tox.reporter import _INSTANCE def test_verbose_isolated_build(initproj, mock_venv, cmd): @@ -79,20 +79,19 @@ def toml_file_check(initproj, version, message, toml): "pyproject.toml": toml, }, ) - reporter = Reporter(None) with pytest.raises(SystemExit, message=1): - get_build_info(py.path.local(), reporter) + get_build_info(py.path.local()) toml_file = py.path.local().join("pyproject.toml") msg = "ERROR: {} inside {}".format(message, toml_file) - assert reporter.reported_lines == [msg] + assert _INSTANCE.messages == [msg] -def test_package_isolated_toml_no_build_system(initproj, cmd): +def test_package_isolated_toml_no_build_system(initproj): toml_file_check(initproj, 1, "build-system section missing", "") -def test_package_isolated_toml_no_requires(initproj, cmd): +def test_package_isolated_toml_no_requires(initproj): toml_file_check( initproj, 2, @@ -103,7 +102,7 @@ def test_package_isolated_toml_no_requires(initproj, cmd): ) -def test_package_isolated_toml_no_backend(initproj, cmd): +def test_package_isolated_toml_no_backend(initproj): toml_file_check( initproj, 3, @@ -115,7 +114,7 @@ def test_package_isolated_toml_no_backend(initproj, cmd): ) -def test_package_isolated_toml_bad_requires(initproj, cmd): +def test_package_isolated_toml_bad_requires(initproj): toml_file_check( initproj, 4, @@ -128,7 +127,7 @@ def test_package_isolated_toml_bad_requires(initproj, cmd): ) -def test_package_isolated_toml_bad_backend(initproj, cmd): +def test_package_isolated_toml_bad_backend(initproj): toml_file_check( initproj, 5, diff --git a/tests/unit/session/test_parallel.py b/tests/unit/session/test_parallel.py index bbc7b1c3e..d25b3206e 100644 --- a/tests/unit/session/test_parallel.py +++ b/tests/unit/session/test_parallel.py @@ -94,7 +94,7 @@ def test_parallel_error_report(cmd, initproj): }, ) result = cmd("-p", "all") - msg = "{}{}{}".format(result.err, os.linesep, result.out) + msg = result.out assert result.ret == 1, msg # we print output assert "(exited with code 17)" in result.out, msg diff --git a/tests/unit/session/test_session.py b/tests/unit/session/test_session.py index beeaa7dd0..2ed7d9394 100644 --- a/tests/unit/session/test_session.py +++ b/tests/unit/session/test_session.py @@ -140,7 +140,7 @@ def func(*args): ) result = cmd(*args) assert result.ret == 0 - active = [i.name for i in result.session.venvlist] + active = [i.name for i in result.session.existing_venvs.values()] return active, result yield func @@ -238,7 +238,7 @@ def build_session(config): monkeypatch.setattr(tox.session, "build_session", build_session) def popen(cmd, **kwargs): - activity_id = res.session._actions[-1].id + activity_id = res.session._actions[-1].name activity_name = res.session._actions[-1].activity ret = "NOTSET" try: @@ -314,9 +314,10 @@ def test_command_prev_post_ok(cmd, initproj, mock_venv): """.format( "_" if sys.platform != "win32" else "" ) - ) - actual = result.out.replace(os.linesep, "\n") - assert expected in actual + ).lstrip() + have = result.out.replace(os.linesep, "\n") + actual = have[len(have) - len(expected) :] + assert actual == expected def test_command_prev_fail_command_skip_post_run(cmd, initproj, mock_venv): diff --git a/tests/unit/test_result.py b/tests/unit/test_result.py index 38f02bc15..ed30f5bfb 100644 --- a/tests/unit/test_result.py +++ b/tests/unit/test_result.py @@ -7,7 +7,7 @@ import pytest import tox -from tox.result import ResultLog +from tox.logs import ResultLog @pytest.fixture(name="pkg") @@ -26,14 +26,13 @@ def test_pre_set_header(): assert replog.dict["platform"] == sys.platform assert replog.dict["host"] == socket.getfqdn() data = replog.dumps_json() - replog2 = ResultLog(data) + replog2 = ResultLog.from_json(data) assert replog2.dict == replog.dict def test_set_header(pkg): replog = ResultLog() d = replog.dict - replog.set_header(installpkg=pkg) assert replog.dict == d assert replog.dict["reportversion"] == "1" assert replog.dict["toxversion"] == tox.__version__ @@ -44,17 +43,20 @@ def test_set_header(pkg): "md5": pkg.computehash("md5"), "sha256": pkg.computehash("sha256"), } - assert replog.dict["installpkg"] == expected + env_log = replog.get_envlog("a") + env_log.set_header(installpkg=pkg) + assert env_log.dict["installpkg"] == expected + data = replog.dumps_json() - replog2 = ResultLog(data) + replog2 = ResultLog.from_json(data) assert replog2.dict == replog.dict def test_addenv_setpython(pkg): replog = ResultLog() - replog.set_header(installpkg=pkg) envlog = replog.get_envlog("py36") envlog.set_python_info(py.path.local(sys.executable)) + envlog.set_header(installpkg=pkg) assert envlog.dict["python"]["version_info"] == list(sys.version_info) assert envlog.dict["python"]["version"] == sys.version assert envlog.dict["python"]["executable"] == sys.executable @@ -62,10 +64,10 @@ def test_addenv_setpython(pkg): def test_get_commandlog(pkg): replog = ResultLog() - replog.set_header(installpkg=pkg) envlog = replog.get_envlog("py36") assert "setup" not in envlog.dict setuplog = envlog.get_commandlog("setup") + envlog.set_header(installpkg=pkg) setuplog.add_command(["virtualenv", "..."], "venv created", 0) expected = [{"command": ["virtualenv", "..."], "output": "venv created", "retcode": "0"}] assert setuplog.list == expected diff --git a/tests/unit/test_venv.py b/tests/unit/test_venv.py index 51cb3c08e..7ee6770e6 100644 --- a/tests/unit/test_venv.py +++ b/tests/unit/test_venv.py @@ -6,6 +6,7 @@ import tox from tox.interpreters import NoInterpreterInfo +from tox.session.commands.run.sequential import installpkg, runtestenv from tox.venv import ( CreationConfig, VirtualEnv, @@ -30,7 +31,8 @@ def test_getsupportedinterpreter(monkeypatch, newconfig, mocksession): sys.executable ), ) - venv = VirtualEnv(config.envconfigs["python"], session=mocksession) + mocksession.new_config(config) + venv = mocksession.getvenv("python") interp = venv.getsupportedinterpreter() # realpath needed for debian symlinks assert py.path.local(interp).realpath() == py.path.local(sys.executable).realpath() @@ -60,7 +62,8 @@ def test_create(mocksession, newconfig): """, ) envconfig = config.envconfigs["py123"] - venv = VirtualEnv(envconfig, session=mocksession) + mocksession.new_config(config) + venv = mocksession.getvenv("py123") assert venv.path == envconfig.envdir assert not venv.path.check() action = mocksession.newaction(venv, "getenv") @@ -87,8 +90,9 @@ def test_commandpath_venv_precedence(tmpdir, monkeypatch, mocksession, newconfig [testenv:py123] """, ) - envconfig = config.envconfigs["py123"] - venv = VirtualEnv(envconfig, session=mocksession) + mocksession.new_config(config) + venv = mocksession.getvenv("py123") + envconfig = venv.envconfig tmpdir.ensure("easy_install") monkeypatch.setenv("PATH", str(tmpdir), prepend=os.pathsep) envconfig.envbindir.ensure("easy_install") @@ -107,8 +111,8 @@ def test_create_sitepackages(mocksession, newconfig): sitepackages=False """, ) - envconfig = config.envconfigs["site"] - venv = VirtualEnv(envconfig, session=mocksession) + mocksession.new_config(config) + venv = mocksession.getvenv("site") action = mocksession.newaction(venv, "getenv") tox_testenv_create(action=action, venv=venv) pcalls = mocksession._pcalls @@ -117,8 +121,7 @@ def test_create_sitepackages(mocksession, newconfig): assert "--system-site-packages" in map(str, args) mocksession._clearmocks() - envconfig = config.envconfigs["nosite"] - venv = VirtualEnv(envconfig, session=mocksession) + venv = mocksession.getvenv("nosite") action = mocksession.newaction(venv, "getenv") tox_testenv_create(action=action, venv=venv) pcalls = mocksession._pcalls @@ -139,12 +142,12 @@ def test_install_deps_wildcard(newmocksession): {distshare}/dep1-* """, ) - venv = mocksession.getenv("py123") + venv = mocksession.getvenv("py123") action = mocksession.newaction(venv, "getenv") tox_testenv_create(action=action, venv=venv) pcalls = mocksession._pcalls assert len(pcalls) == 1 - distshare = venv.session.config.distshare + distshare = venv.envconfig.config.distshare distshare.ensure("dep1-1.0.zip") distshare.ensure("dep1-1.1.zip") @@ -175,7 +178,7 @@ def test_install_deps_indexserver(newmocksession): :abc2:dep3 """, ) - venv = mocksession.getenv("py123") + venv = mocksession.getvenv("py123") action = mocksession.newaction(venv, "getenv") tox_testenv_create(action=action, venv=venv) pcalls = mocksession._pcalls @@ -207,7 +210,7 @@ def test_install_deps_pre(newmocksession): dep1 """, ) - venv = mocksession.getenv("python") + venv = mocksession.getvenv("python") action = mocksession.newaction(venv, "getenv") tox_testenv_create(action=action, venv=venv) pcalls = mocksession._pcalls @@ -230,10 +233,10 @@ def test_installpkg_indexserver(newmocksession, tmpdir): default = ABC """, ) - venv = mocksession.getenv("python") + venv = mocksession.getvenv("python") pcalls = mocksession._pcalls p = tmpdir.ensure("distfile.tar.gz") - mocksession.installpkg(venv, p) + installpkg(venv, p) # two different index servers, two calls assert len(pcalls) == 1 args = " ".join(pcalls[0].args) @@ -249,11 +252,11 @@ def test_install_recreate(newmocksession, tmpdir): deps=xyz """, ) - venv = mocksession.getenv("python") + venv = mocksession.getvenv("python") action = mocksession.newaction(venv, "update") venv.update(action) - mocksession.installpkg(venv, pkg) + installpkg(venv, pkg) mocksession.report.expect("verbosity0", "*create*") venv.update(action) mocksession.report.expect("verbosity0", "*recreate*") @@ -268,7 +271,7 @@ def test_install_sdist_extras(newmocksession): development """, ) - venv = mocksession.getenv("python") + venv = mocksession.getvenv("python") action = mocksession.newaction(venv, "getenv") tox_testenv_create(action=action, venv=venv) pcalls = mocksession._pcalls @@ -288,7 +291,7 @@ def test_develop_extras(newmocksession, tmpdir): development """, ) - venv = mocksession.getenv("python") + venv = mocksession.getvenv("python") action = mocksession.newaction(venv, "getenv") tox_testenv_create(action=action, venv=venv) pcalls = mocksession._pcalls @@ -313,8 +316,8 @@ def test_env_variables_added_to_needs_reinstall(tmpdir, mocksession, newconfig, CUSTOM_VAR = 789 """, ) - - venv = VirtualEnv(config.envconfigs["python"], session=mocksession) + mocksession.new_config(config) + venv = mocksession.getvenv("python") action = mocksession.newaction(venv, "hello") venv._needs_reinstall(tmpdir, action) @@ -341,7 +344,7 @@ def test_test_hashseed_is_in_output(newmocksession, monkeypatch): seed = "123456789" monkeypatch.setattr("tox.config.make_hashseed", lambda: seed) mocksession = newmocksession([], "") - venv = mocksession.getenv("python") + venv = mocksession.getvenv("python") action = mocksession.newaction(venv, "update") venv.update(action) tox.venv.tox_runtest_pre(venv) @@ -356,7 +359,7 @@ def test_test_runtests_action_command_is_in_output(newmocksession): commands = echo foo bar """, ) - venv = mocksession.getenv("python") + venv = mocksession.getvenv("python") action = mocksession.newaction(venv, "update") venv.update(action) venv.test() @@ -373,7 +376,7 @@ def test_install_error(newmocksession): qwelkqw """, ) - venv = mocksession.getenv("python") + venv = mocksession.getvenv("python") venv.test() mocksession.report.expect("error", "*not find*qwelkqw*") assert venv.status == "commands failed" @@ -388,7 +391,7 @@ def test_install_command_not_installed(newmocksession): pytest """, ) - venv = mocksession.getenv("python") + venv = mocksession.getvenv("python") venv.status = 0 venv.test() mocksession.report.expect("warning", "*test command found but not*") @@ -407,7 +410,7 @@ def test_install_command_whitelisted(newmocksession): xyz """, ) - venv = mocksession.getenv("python") + venv = mocksession.getvenv("python") venv.test() mocksession.report.expect("warning", "*test command found but not*", invert=True) assert venv.status == "commands failed" @@ -422,7 +425,7 @@ def test_install_command_not_installed_bash(newmocksession): bash """, ) - venv = mocksession.getenv("python") + venv = mocksession.getvenv("python") venv.test() mocksession.report.expect("warning", "*test command found but not*") @@ -440,7 +443,7 @@ def test_install_python3(newmocksession): dep2 """, ) - venv = mocksession.getenv("py123") + venv = mocksession.getvenv("py123") action = mocksession.newaction(venv, "getenv") tox_testenv_create(action=action, venv=venv) pcalls = mocksession._pcalls @@ -461,8 +464,8 @@ def test_install_python3(newmocksession): class TestCreationConfig: def test_basic(self, newconfig, mocksession, tmpdir): config = newconfig([], "") - envconfig = config.envconfigs["python"] - venv = VirtualEnv(envconfig, session=mocksession) + mocksession.new_config(config) + venv = mocksession.getvenv("python") cconfig = venv._getliveconfig() assert cconfig.matches(cconfig) path = tmpdir.join("configdump") @@ -479,8 +482,8 @@ def test_matchingdependencies(self, newconfig, mocksession): deps=abc """, ) - envconfig = config.envconfigs["python"] - venv = VirtualEnv(envconfig, session=mocksession) + mocksession.new_config(config) + venv = mocksession.getvenv("python") cconfig = venv._getliveconfig() config = newconfig( [], @@ -489,8 +492,8 @@ def test_matchingdependencies(self, newconfig, mocksession): deps=xyz """, ) - envconfig = config.envconfigs["python"] - venv = VirtualEnv(envconfig, session=mocksession) + mocksession.new_config(config) + venv = mocksession.getvenv("python") otherconfig = venv._getliveconfig() assert not cconfig.matches(otherconfig) @@ -507,8 +510,8 @@ def test_matchingdependencies_file(self, newconfig, mocksession): ) xyz = config.distshare.join("xyz.zip") xyz.ensure() - envconfig = config.envconfigs["python"] - venv = VirtualEnv(envconfig, session=mocksession) + mocksession.new_config(config) + venv = mocksession.getvenv("python") cconfig = venv._getliveconfig() assert cconfig.matches(cconfig) xyz.write("hello") @@ -527,8 +530,8 @@ def test_matchingdependencies_latest(self, newconfig, mocksession): ) config.distshare.ensure("xyz-1.2.0.zip") xyz2 = config.distshare.ensure("xyz-1.2.1.zip") - envconfig = config.envconfigs["python"] - venv = VirtualEnv(envconfig, session=mocksession) + mocksession.new_config(config) + venv = mocksession.getvenv("python") cconfig = venv._getliveconfig() md5, path = cconfig.deps[0] assert path == xyz2 @@ -537,13 +540,13 @@ def test_matchingdependencies_latest(self, newconfig, mocksession): def test_python_recreation(self, tmpdir, newconfig, mocksession): pkg = tmpdir.ensure("package.tar.gz") config = newconfig([], "") - envconfig = config.envconfigs["python"] - venv = VirtualEnv(envconfig, session=mocksession) + mocksession.new_config(config) + venv = mocksession.getvenv("python") create_config = venv._getliveconfig() action = mocksession.newaction(venv, "update") venv.update(action) assert not venv.path_config.check() - mocksession.installpkg(venv, pkg) + installpkg(venv, pkg) assert venv.path_config.check() assert mocksession._pcalls args1 = map(str, mocksession._pcalls[0].args) @@ -563,8 +566,8 @@ def test_python_recreation(self, tmpdir, newconfig, mocksession): def test_dep_recreation(self, newconfig, mocksession): config = newconfig([], "") - envconfig = config.envconfigs["python"] - venv = VirtualEnv(envconfig, session=mocksession) + mocksession.new_config(config) + venv = mocksession.getvenv("python") action = mocksession.newaction(venv, "update") venv.update(action) cconfig = venv._getliveconfig() @@ -577,8 +580,8 @@ def test_dep_recreation(self, newconfig, mocksession): def test_develop_recreation(self, newconfig, mocksession): config = newconfig([], "") - envconfig = config.envconfigs["python"] - venv = VirtualEnv(envconfig, session=mocksession) + mocksession.new_config(config) + venv = mocksession.getvenv("python") action = mocksession.newaction(venv, "update") venv.update(action) cconfig = venv._getliveconfig() @@ -600,7 +603,7 @@ def test_envbindir_path(self, newmocksession, monkeypatch): commands=abc """, ) - venv = mocksession.getenv("python") + venv = mocksession.getvenv("python") action = mocksession.newaction(venv, "getenv") monkeypatch.setenv("PATH", "xyz") sysfind_calls = [] @@ -629,7 +632,7 @@ def test_envbindir_path(self, newmocksession, monkeypatch): assert os.environ["PIP_USER"] == "0" assert os.environ["PIP_NO_DEPS"] == "0" - def test_pythonpath_usage(self, newmocksession, monkeypatch): + def test_pythonpath_remove(self, newmocksession, monkeypatch, caplog): monkeypatch.setenv("PYTHONPATH", "/my/awesome/library") mocksession = newmocksession( [], @@ -638,7 +641,7 @@ def test_pythonpath_usage(self, newmocksession, monkeypatch): commands=abc """, ) - venv = mocksession.getenv("python") + venv = mocksession.getvenv("python") action = mocksession.newaction(venv, "getenv") venv.run_install_command(["qwe"], action=action) assert "PYTHONPATH" not in os.environ @@ -648,6 +651,7 @@ def test_pythonpath_usage(self, newmocksession, monkeypatch): assert len(pcalls) == 1 assert "PYTHONPATH" not in pcalls[0].env + def test_pythonpath_keep(self, newmocksession, monkeypatch, caplog): # passenv = PYTHONPATH allows PYTHONPATH to stay in environment monkeypatch.setenv("PYTHONPATH", "/my/awesome/library") mocksession = newmocksession( @@ -658,15 +662,15 @@ def test_pythonpath_usage(self, newmocksession, monkeypatch): passenv = PYTHONPATH """, ) - venv = mocksession.getenv("python") + venv = mocksession.getvenv("python") action = mocksession.newaction(venv, "getenv") venv.run_install_command(["qwe"], action=action) - assert "PYTHONPATH" in os.environ mocksession.report.not_expect("warning", "*Discarding $PYTHONPATH from environment*") + assert "PYTHONPATH" in os.environ pcalls = mocksession._pcalls - assert len(pcalls) == 2 - assert pcalls[1].env["PYTHONPATH"] == "/my/awesome/library" + assert len(pcalls) == 1 + assert pcalls[0].env["PYTHONPATH"] == "/my/awesome/library" def test_env_variables_added_to_pcall(tmpdir, mocksession, newconfig, monkeypatch): @@ -686,9 +690,9 @@ def test_env_variables_added_to_pcall(tmpdir, mocksession, newconfig, monkeypatc """, ) mocksession._clearmocks() - - venv = VirtualEnv(config.envconfigs["python"], session=mocksession) - mocksession.installpkg(venv, pkg) + mocksession.new_config(config) + venv = mocksession.getvenv("python") + installpkg(venv, pkg) venv.test() pcalls = mocksession._pcalls @@ -718,10 +722,10 @@ def test_env_variables_added_to_pcall(tmpdir, mocksession, newconfig, monkeypatc def test_installpkg_no_upgrade(tmpdir, newmocksession): pkg = tmpdir.ensure("package.tar.gz") mocksession = newmocksession([], "") - venv = mocksession.getenv("python") + venv = mocksession.getvenv("python") venv.just_created = True venv.envconfig.envdir.ensure(dir=1) - mocksession.installpkg(venv, pkg) + installpkg(venv, pkg) pcalls = mocksession._pcalls assert len(pcalls) == 1 assert pcalls[0].args[1:-1] == ["-m", "pip", "install", "--exists-action", "w"] @@ -731,10 +735,10 @@ def test_installpkg_no_upgrade(tmpdir, newmocksession): def test_install_command_verbosity(tmpdir, newmocksession, count, level): pkg = tmpdir.ensure("package.tar.gz") mock_session = newmocksession(["-{}".format("v" * count)], "") - env = mock_session.getenv("python") + env = mock_session.getvenv("python") env.just_created = True env.envconfig.envdir.ensure(dir=1) - mock_session.installpkg(env, pkg) + installpkg(env, pkg) pcalls = mock_session._pcalls assert len(pcalls) == 1 expected = ["-m", "pip", "install", "--exists-action", "w"] + (["-v"] * level) @@ -744,9 +748,9 @@ def test_install_command_verbosity(tmpdir, newmocksession, count, level): def test_installpkg_upgrade(newmocksession, tmpdir): pkg = tmpdir.ensure("package.tar.gz") mocksession = newmocksession([], "") - venv = mocksession.getenv("python") + venv = mocksession.getvenv("python") assert not hasattr(venv, "just_created") - mocksession.installpkg(venv, pkg) + installpkg(venv, pkg) pcalls = mocksession._pcalls assert len(pcalls) == 1 index = pcalls[0].args.index(str(pkg)) @@ -757,7 +761,7 @@ def test_installpkg_upgrade(newmocksession, tmpdir): def test_run_install_command(newmocksession): mocksession = newmocksession([], "") - venv = mocksession.getenv("python") + venv = mocksession.getvenv("python") venv.just_created = True venv.envconfig.envdir.ensure(dir=1) action = mocksession.newaction(venv, "hello") @@ -780,7 +784,7 @@ def test_run_custom_install_command(newmocksession): install_command=easy_install {opts} {packages} """, ) - venv = mocksession.getenv("python") + venv = mocksession.getvenv("python") venv.just_created = True venv.envconfig.envdir.ensure(dir=1) action = mocksession.newaction(venv, "hello") @@ -799,7 +803,7 @@ def test_command_relative_issue36(newmocksession, tmpdir, monkeypatch): """, ) x = tmpdir.ensure("x") - venv = mocksession.getenv("python") + venv = mocksession.getvenv("python") x2 = venv.getcommandpath("./x", cwd=tmpdir) assert x == x2 mocksession.report.not_expect("warning", "*test command found but not*") @@ -822,7 +826,7 @@ def test_ignore_outcome_failing_cmd(newmocksession): """, ) - venv = mocksession.getenv("python") + venv = mocksession.getvenv("python") venv.test() assert venv.status == "ignored failed command" mocksession.report.expect("warning", "*command failed but result from testenv is ignored*") @@ -854,7 +858,7 @@ def tox_testenv_install_deps(self, action, venv): plugins=[Plugin()], ) - venv = mocksession.getenv("python") + venv = mocksession.getvenv("python") venv.update(action=mocksession.newaction(venv, "getenv")) assert log == [1, 2] @@ -880,10 +884,10 @@ def tox_runtest_post(self): plugins=[Plugin()], ) - venv = mocksession.getenv("python") + venv = mocksession.getvenv("python") venv.status = None assert log == [] - mocksession.runtestenv(venv) + runtestenv(venv, venv.envconfig.config) assert log == ["started", "finished"] diff --git a/tests/unit/test_z_cmdline.py b/tests/unit/test_z_cmdline.py index f1c359471..04faa2282 100644 --- a/tests/unit/test_z_cmdline.py +++ b/tests/unit/test_z_cmdline.py @@ -10,41 +10,12 @@ import pytest import tox -from tox._pytestplugin import ReportExpectMock from tox.config import parseconfig from tox.session import Session pytest_plugins = "pytester" -def test_report_protocol(newconfig): - config = newconfig( - [], - """ - [testenv:mypython] - deps=xy - """, - ) - - class Popen: - def __init__(self, *args, **kwargs): - pass - - def communicate(self): - return "", "" - - def wait(self): - pass - - session = Session(config, popen=Popen, Report=ReportExpectMock) - report = session.report - report.expect("using") - venv = session.getvenv("mypython") - action = session.newaction(venv, "update") - venv.update(action) - report.expect("logpopen") - - class TestSession: def test_log_pcall(self, mocksession): mocksession.config.logdir.ensure(dir=1) @@ -68,7 +39,7 @@ def test_summary_status(self, initproj, capfd): ) config = parseconfig([]) session = Session(config) - envs = session.venvlist + envs = list(session.venv_dict.values()) assert len(envs) == 2 env1, env2 = envs env1.status = "FAIL XYZ" @@ -621,7 +592,7 @@ def test_warning_emitted(cmd, initproj): """, }, ) - result = cmd() + cmd() result = cmd() assert "develop-inst-noop" in result.out assert "I am a warning" in result.err From 8b68a18b0bab8d6b63e0b9158626b03bdba980fe Mon Sep 17 00:00:00 2001 From: Bernat Gabor Date: Sun, 20 Jan 2019 19:13:47 +0000 Subject: [PATCH 02/12] more test fixes --- src/tox/_pytestplugin.py | 1 + tests/unit/config/test_config.py | 3 ++- tests/unit/package/test_package_parallel.py | 24 +++++++++--------- tests/unit/session/test_session.py | 27 +++++++++++---------- tests/unit/test_venv.py | 2 +- 5 files changed, 31 insertions(+), 26 deletions(-) diff --git a/src/tox/_pytestplugin.py b/src/tox/_pytestplugin.py index c53958282..f71bb388b 100644 --- a/src/tox/_pytestplugin.py +++ b/src/tox/_pytestplugin.py @@ -223,6 +223,7 @@ def popen(self, args, cwd, shell=None, stdout=None, stderr=None, env=None, **_): return process_call_mock def new_config(self, config): + update_default_reporter(config.option.quiet_level, config.option.verbose_level) self.config = config self.venv_dict.clear() self.existing_venvs.clear() diff --git a/tests/unit/config/test_config.py b/tests/unit/config/test_config.py index c56aad6a7..92dcafa8e 100644 --- a/tests/unit/config/test_config.py +++ b/tests/unit/config/test_config.py @@ -2664,7 +2664,8 @@ def test_no_tox_ini(self, cmd, initproj): result = cmd() assert result.ret msg = "ERROR: tox config file (either pyproject.toml, tox.ini, setup.cfg) not found\n" - assert result.out == msg + assert result.err == msg + assert not result.out def test_override_workdir(self, cmd, initproj): baddir = "badworkdir-123" diff --git a/tests/unit/package/test_package_parallel.py b/tests/unit/package/test_package_parallel.py index f8558eec4..7eeb162b5 100644 --- a/tests/unit/package/test_package_parallel.py +++ b/tests/unit/package/test_package_parallel.py @@ -3,6 +3,8 @@ import py +from tox.session.commands.run import sequential + def test_tox_parallel_build_safe(initproj, cmd, mock_venv, monkeypatch): initproj( @@ -45,20 +47,20 @@ def invoke_tox_in_thread(thread_name): with monkeypatch.context() as m: - def build_package(config, report, session): + def build_package(config, session): t1_build_started.set() - prev_run_test_env = tox.session.Session.runtestenv + t1_build_blocker.wait() + return prev_build_package(config, session) - def run_test_env(self, venv, redirect=False): - t2_build_finished.wait() - return prev_run_test_env(self, venv, redirect) + m.setattr(tox.package, "build_package", build_package) - session.runtestenv = partial(run_test_env, session) + prev_run_test_env = sequential.runtestenv - t1_build_blocker.wait() - return prev_build_package(config, report, session) + def run_test_env(venv, redirect=False): + t2_build_finished.wait() + return prev_run_test_env(venv, redirect) - m.setattr(tox.package, "build_package", build_package) + m.setattr(sequential, "runtestenv", run_test_env) t1 = threading.Thread(target=invoke_tox_in_thread, args=("t1",)) t1.start() @@ -66,10 +68,10 @@ def run_test_env(self, venv, redirect=False): with monkeypatch.context() as m: - def build_package(config, report, session): + def build_package(config, session): t2_build_started.set() try: - return prev_build_package(config, report, session) + return prev_build_package(config, session) finally: t2_build_finished.set() diff --git a/tests/unit/session/test_session.py b/tests/unit/session/test_session.py index 2ed7d9394..206d7af8d 100644 --- a/tests/unit/session/test_session.py +++ b/tests/unit/session/test_session.py @@ -8,60 +8,61 @@ import tox from tox.exception import MissingDependency, MissingDirectory +from tox.package import resolve_package -def test__resolve_pkg_missing_directory(tmpdir, mocksession): +def test_resolve_pkg_missing_directory(tmpdir, mocksession): distshare = tmpdir.join("distshare") spec = distshare.join("pkg123-*") with pytest.raises(MissingDirectory): - mocksession._resolve_package(spec) + resolve_package(spec) -def test__resolve_pkg_missing_directory_in_distshare(tmpdir, mocksession): +def test_resolve_pkg_missing_directory_in_distshare(tmpdir, mocksession): distshare = tmpdir.join("distshare") spec = distshare.join("pkg123-*") distshare.ensure(dir=1) with pytest.raises(MissingDependency): - mocksession._resolve_package(spec) + resolve_package(spec) -def test__resolve_pkg_multiple_valid_versions(tmpdir, mocksession): +def test_resolve_pkg_multiple_valid_versions(tmpdir, mocksession): distshare = tmpdir.join("distshare") distshare.ensure("pkg123-1.3.5.zip") p = distshare.ensure("pkg123-1.4.5.zip") - result = mocksession._resolve_package(distshare.join("pkg123-*")) + result = resolve_package(distshare.join("pkg123-*")) assert result == p mocksession.report.expect("info", "determin*pkg123*") -def test__resolve_pkg_with_invalid_version(tmpdir, mocksession): +def test_resolve_pkg_with_invalid_version(tmpdir, mocksession): distshare = tmpdir.join("distshare") distshare.ensure("pkg123-1.something_bad.zip") distshare.ensure("pkg123-1.3.5.zip") p = distshare.ensure("pkg123-1.4.5.zip") - result = mocksession._resolve_package(distshare.join("pkg123-*")) + result = resolve_package(distshare.join("pkg123-*")) mocksession.report.expect("warning", "*1.something_bad*") assert result == p -def test__resolve_pkg_with_alpha_version(tmpdir, mocksession): +def test_resolve_pkg_with_alpha_version(tmpdir, mocksession): distshare = tmpdir.join("distshare") distshare.ensure("pkg123-1.3.5.zip") distshare.ensure("pkg123-1.4.5a1.tar.gz") p = distshare.ensure("pkg123-1.4.5.zip") - result = mocksession._resolve_package(distshare.join("pkg123-*")) + result = resolve_package(distshare.join("pkg123-*")) assert result == p -def test__resolve_pkg_doubledash(tmpdir, mocksession): +def test_resolve_pkg_doubledash(tmpdir, mocksession): distshare = tmpdir.join("distshare") p = distshare.ensure("pkg-mine-1.3.0.zip") - res = mocksession._resolve_package(distshare.join("pkg-mine*")) + res = resolve_package(distshare.join("pkg-mine*")) assert res == p distshare.ensure("pkg-mine-1.3.0a1.zip") - res = mocksession._resolve_package(distshare.join("pkg-mine*")) + res = resolve_package(distshare.join("pkg-mine*")) assert res == p diff --git a/tests/unit/test_venv.py b/tests/unit/test_venv.py index 7ee6770e6..8a8d8fc4c 100644 --- a/tests/unit/test_venv.py +++ b/tests/unit/test_venv.py @@ -539,7 +539,7 @@ def test_matchingdependencies_latest(self, newconfig, mocksession): def test_python_recreation(self, tmpdir, newconfig, mocksession): pkg = tmpdir.ensure("package.tar.gz") - config = newconfig([], "") + config = newconfig(["-v"], "") mocksession.new_config(config) venv = mocksession.getvenv("python") create_config = venv._getliveconfig() From bb65a442bb0f6aab2c20f70b1cfc1a2d1564aa69 Mon Sep 17 00:00:00 2001 From: Bernat Gabor Date: Mon, 21 Jan 2019 11:35:53 +0000 Subject: [PATCH 03/12] more test fixes --- src/tox/_pytestplugin.py | 45 ++-- src/tox/action.py | 16 +- src/tox/constants.py | 10 + src/tox/helper/__init__.py | 0 src/tox/helper/build_isolated.py | 14 ++ src/tox/helper/build_requires.py | 13 + src/tox/helper/get_site_package_dir.py | 8 + src/tox/helper/get_version.py | 13 + src/tox/interpreters.py | 28 +-- src/tox/logs/env.py | 15 +- src/tox/package/builder/isolated.py | 44 ++-- src/tox/reporter.py | 10 +- src/tox/session/__init__.py | 9 +- src/tox/util/lock.py | 2 +- tests/unit/package/test_package_parallel.py | 3 +- tests/unit/session/test_session.py | 34 ++- tests/unit/test_venv.py | 258 ++++++++++---------- tests/unit/test_z_cmdline.py | 15 +- 18 files changed, 307 insertions(+), 230 deletions(-) create mode 100644 src/tox/helper/__init__.py create mode 100644 src/tox/helper/build_isolated.py create mode 100644 src/tox/helper/build_requires.py create mode 100644 src/tox/helper/get_site_package_dir.py create mode 100644 src/tox/helper/get_version.py diff --git a/src/tox/_pytestplugin.py b/src/tox/_pytestplugin.py index f71bb388b..ea59e3918 100644 --- a/src/tox/_pytestplugin.py +++ b/src/tox/_pytestplugin.py @@ -71,6 +71,7 @@ def cmd(request, monkeypatch): request.addfinalizer(py.path.local().chdir) def run(*argv): + reset_report() key = str("PYTHONPATH") python_paths = (i for i in (os.getcwd(), os.getenv(key)) if i) monkeypatch.setenv(key, os.pathsep.join(python_paths)) @@ -107,15 +108,17 @@ def __init__(self, args): def __enter__(self): self._start = time.time() + self._out_tell = sys.stdout.tell() + self._err_tell = sys.stderr.tell() return self def __exit__(self, exc_type, exc_val, exc_tb): self.duration = time.time() - self._start - self.out = self._read(sys.stdout) - self.err = self._read(sys.stderr) + self.out = self._read(sys.stdout, self._out_tell) + self.err = self._read(sys.stderr, self._err_tell) - def _read(self, out): - out.buffer.seek(0) + def _read(self, out, pos): + out.buffer.seek(pos) return out.buffer.read().decode(out.encoding, errors=out.errors) @property @@ -138,7 +141,10 @@ def __init__(self): def clear(self): self._index = -1 - self.instance.reported_lines.clear() + if six.PY3: + self.instance.reported_lines.clear() + else: + del self.instance.reported_lines[:] def getnext(self, cat): __tracebackhide__ = True @@ -208,13 +214,16 @@ def create_mocksession(request): class MockSession(Session): def __init__(self, config): - update_default_reporter(config.option.quiet_level, config.option.verbose_level) + self.logging_levels(config.option.quiet_level, config.option.verbose_level) super(MockSession, self).__init__(config, popen=self.popen) self._pcalls = [] self.report = ReportExpectMock() def _clearmocks(self): - self._pcalls.clear() + if six.PY3: + self._pcalls.clear() + else: + del self._pcalls[:] self.report.clear() def popen(self, args, cwd, shell=None, stdout=None, stderr=None, env=None, **_): @@ -223,11 +232,17 @@ def popen(self, args, cwd, shell=None, stdout=None, stderr=None, env=None, **_): return process_call_mock def new_config(self, config): - update_default_reporter(config.option.quiet_level, config.option.verbose_level) + self.logging_levels(config.option.quiet_level, config.option.verbose_level) self.config = config self.venv_dict.clear() self.existing_venvs.clear() + def logging_levels(self, quiet, verbose): + update_default_reporter(quiet, verbose) + if hasattr(self, "config"): + self.config.option.quiet_level = quiet + self.config.option.verbose_level = verbose + return MockSession(config) @@ -494,18 +509,18 @@ def current_tox_py(): def pytest_runtest_setup(item): - from tox.reporter import _INSTANCE - - _INSTANCE._reset() + reset_report() def pytest_runtest_teardown(item): - from tox.reporter import _INSTANCE - - _INSTANCE._reset() + reset_report() def pytest_pyfunc_call(pyfuncitem): + reset_report() + + +def reset_report(quiet=0, verbose=0): from tox.reporter import _INSTANCE - _INSTANCE._reset() + _INSTANCE._reset(quiet_level=quiet, verbose_level=verbose) diff --git a/src/tox/action.py b/src/tox/action.py index fe3434da7..bda9c3432 100644 --- a/src/tox/action.py +++ b/src/tox/action.py @@ -66,13 +66,15 @@ def popen( stream_getter = self._get_standard_streams( capture_err, cmd_args_shell, redirect, returnout ) + cwd = os.getcwd() if cwd is None else cwd with stream_getter as (fin, out_path, stderr, stdout): try: + args = self._rewrite_args(cwd, args) process = self.via_popen( - self._rewrite_args(cwd, args), + args, stdout=stdout, stderr=stderr, - cwd=cwd, + cwd=str(cwd), env=os.environ.copy() if env is None else env, universal_newlines=True, shell=False, @@ -88,20 +90,20 @@ def popen( output = self.feed_stdin(fin, process, redirect) exit_code = process.wait() if exit_code and not ignore_ret: - invoked = " ".join(map(str, process.args)) + invoked = " ".join(map(str, args)) if out_path: reporter.error( "invocation failed (exit code {:d}), logfile: {}".format(exit_code, out_path) ) output = out_path.read() reporter.error(output) - self.command_log.add_command(process.args, output, exit_code) + self.command_log.add_command(args, output, exit_code) raise InvocationError(invoked, exit_code, out_path) else: raise InvocationError(invoked, exit_code) if not output and out_path: output = out_path.read() - self.command_log.add_command(process.args, output, exit_code) + self.command_log.add_command(args, output, exit_code) return output def feed_stdin(self, fin, process, redirect): @@ -149,14 +151,14 @@ def _get_standard_streams(self, capture_err, cmd_args_shell, redirect, returnout stderr = subprocess.STDOUT if capture_err else None stdout_file = None if self.generate_tox_log or redirect: - stdout_file = open(self.get_log_path(self.name), "wt") + out_path = self.get_log_path(self.name) + stdout_file = out_path.open("wt") stdout_file.write( "actionid: {}\nmsg: {}\ncmdargs: {!r}\n\n".format( self.name, self.msg, cmd_args_shell ) ) stdout_file.flush() - out_path = py.path.local(stdout_file.name) fin = out_path.open("rb") fin.read() # read the header, so it won't be written to stdout stdout = stdout_file diff --git a/src/tox/constants.py b/src/tox/constants.py index ecf0c97d0..ac72058d9 100644 --- a/src/tox/constants.py +++ b/src/tox/constants.py @@ -2,9 +2,12 @@ They live in the tox namespace and can be accessed as tox.[NAMESPACE.]NAME """ +import os import re import sys +_THIS_FILE = os.path.realpath(os.path.abspath(__file__)) + def _construct_default_factors(cpython_versions, pypy_versions, other_interpreters): default_factors = {"py": sys.executable, "py2": "python2", "py3": "python3"} @@ -75,3 +78,10 @@ class PIP: ] INSTALL_SHORT_OPTIONS_ARGUMENT = ["-{}".format(option) for option in SHORT_OPTIONS] INSTALL_LONG_OPTIONS_ARGUMENT = ["--{}".format(option) for option in LONG_OPTIONS] + + +_HELP_DIR = os.path.join(os.path.dirname(_THIS_FILE), "helper") +VERSION_QUERY_SCRIPT = os.path.join(_HELP_DIR, "get_version.py") +SITE_PACKAGE_QUERY_SCRIPT = os.path.join(_HELP_DIR, "get_site_package_dir.py") +BUILD_REQUIRE_SCRIPT = os.path.join(_HELP_DIR, "build_requires.py") +BUILD_ISOLATED = os.path.join(_HELP_DIR, "build_isolated.py") diff --git a/src/tox/helper/__init__.py b/src/tox/helper/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/tox/helper/build_isolated.py b/src/tox/helper/build_isolated.py new file mode 100644 index 000000000..27e4e8ba9 --- /dev/null +++ b/src/tox/helper/build_isolated.py @@ -0,0 +1,14 @@ +import sys + +backend_spec = sys.argv[1] +backend_obj = sys.argv[2] + + +backend = __import__(backend_spec, fromlist=[None]) +if backend_obj: + backend = getattr(backend, backend_obj) + +dist_folder = sys.argv[3] + +basename = backend.build_sdist(dist_folder, {"--global-option": ["--formats=gztar"]}) +print(basename) diff --git a/src/tox/helper/build_requires.py b/src/tox/helper/build_requires.py new file mode 100644 index 000000000..3456c10f4 --- /dev/null +++ b/src/tox/helper/build_requires.py @@ -0,0 +1,13 @@ +import json +import sys + +backend_spec = sys.argv[1] +backend_obj = sys.argv[2] +backend = __import__(backend_spec, fromlist=[None]) +if backend_obj: + backend = getattr(backend, backend_obj) + + +for_build_requires = backend.get_requires_for_build_sdist(None) +output = json.dumps(for_build_requires) +print(output) diff --git a/src/tox/helper/get_site_package_dir.py b/src/tox/helper/get_site_package_dir.py new file mode 100644 index 000000000..584f5103e --- /dev/null +++ b/src/tox/helper/get_site_package_dir.py @@ -0,0 +1,8 @@ +from __future__ import unicode_literals + +import distutils.sysconfig +import json +import sys + +data = json.dumps({"dir": distutils.sysconfig.get_python_lib(prefix=sys.argv[1])}) +print(data) diff --git a/src/tox/helper/get_version.py b/src/tox/helper/get_version.py new file mode 100644 index 000000000..ef37a796f --- /dev/null +++ b/src/tox/helper/get_version.py @@ -0,0 +1,13 @@ +from __future__ import unicode_literals + +import json +import sys + +info = { + "executable": sys.executable, + "version_info": list(sys.version_info), + "version": sys.version, + "sysplatform": sys.platform, +} +info_as_dump = json.dumps(info) +print(info_as_dump) diff --git a/src/tox/interpreters.py b/src/tox/interpreters.py index 503b5065c..6c6cfcc33 100644 --- a/src/tox/interpreters.py +++ b/src/tox/interpreters.py @@ -1,3 +1,5 @@ +from __future__ import unicode_literals + import distutils.util import json import re @@ -7,6 +9,7 @@ import py import tox +from tox.constants import SITE_PACKAGE_QUERY_SCRIPT, VERSION_QUERY_SCRIPT class Interpreters: @@ -45,13 +48,7 @@ def get_sitepackagesdir(self, info, envdir): return "" envdir = str(envdir) try: - code = ( - "import distutils.sysconfig; import json;" - "print(json.dumps(" - "{{ 'dir': distutils.sysconfig.get_python_lib(prefix={!r})}}" - "))" - ) - res = exec_on_interpreter(str(info.executable), "-c", code.format(envdir)) + res = exec_on_interpreter(str(info.executable), SITE_PACKAGE_QUERY_SCRIPT, str(envdir)) except ExecFailed as e: print("execution failed: {} -- {}".format(e.out, e.err)) return "" @@ -62,18 +59,13 @@ def get_sitepackagesdir(self, info, envdir): def run_and_get_interpreter_info(name, executable): assert executable try: - result = exec_on_interpreter( - str(executable), - "-c", - "import sys; import json;" - 'print(json.dumps({"version_info": tuple(sys.version_info),' - ' "sysplatform": sys.platform}))', - ) + result = exec_on_interpreter(str(executable), VERSION_QUERY_SCRIPT) result["version_info"] = tuple(result["version_info"]) # fix json dump transformation + del result["version"] except ExecFailed as e: return NoInterpreterInfo(name, executable=e.executable, out=e.out, err=e.err) else: - return InterpreterInfo(name, executable, **result) + return InterpreterInfo(name, **result) def exec_on_interpreter(*args): @@ -168,12 +160,12 @@ def tox_get_python_executable(envconfig): def locate_via_py(*parts): ver = "-{}".format(".".join(parts)) - script = "import sys; print(sys.executable)" py_exe = distutils.spawn.find_executable("py") if py_exe: proc = subprocess.Popen( - (py_exe, ver, "-c", script), stdout=subprocess.PIPE, stderr=subprocess.PIPE + (py_exe, ver, VERSION_QUERY_SCRIPT), stdout=subprocess.PIPE, stderr=subprocess.PIPE ) out, _ = proc.communicate() + result = json.loads(out) if not proc.returncode: - return out.decode("UTF-8").strip() + return result["executable"] diff --git a/src/tox/logs/env.py b/src/tox/logs/env.py index f4ac20595..bbdc0be52 100644 --- a/src/tox/logs/env.py +++ b/src/tox/logs/env.py @@ -3,6 +3,8 @@ import json import subprocess +from tox.constants import VERSION_QUERY_SCRIPT + from .command import CommandLog @@ -15,17 +17,10 @@ def __init__(self, result_log, name, dict): self.dict = dict def set_python_info(self, python_executable): - cmd = [ - str(python_executable), - "-c", - "import sys; import json;" - "print(json.dumps({" - "'executable': sys.executable," - "'version_info': list(sys.version_info)," - "'version': sys.version}))", - ] + cmd = [str(python_executable), VERSION_QUERY_SCRIPT] result = subprocess.check_output(cmd, universal_newlines=True) - self.dict["python"] = json.loads(result) + answer = json.loads(result) + self.dict["python"] = answer def get_commandlog(self, name): """get the command log for a given group name""" diff --git a/src/tox/package/builder/isolated.py b/src/tox/package/builder/isolated.py index ea4d11260..8379fc526 100644 --- a/src/tox/package/builder/isolated.py +++ b/src/tox/package/builder/isolated.py @@ -1,5 +1,5 @@ +from __future__ import unicode_literals import json -import textwrap from collections import namedtuple import pkg_resources @@ -7,6 +7,7 @@ from tox import reporter from tox.config import DepConfig, get_py_project_toml +from tox.constants import BUILD_ISOLATED, BUILD_REQUIRE_SCRIPT BuildInfo = namedtuple("BuildInfo", ["requires", "backend_module", "backend_object"]) @@ -75,32 +76,28 @@ def abort(message): args = backend.split(":") module = args[0] - obj = "" if len(args) == 1 else ".{}".format(args[1]) + obj = args[1] if len(args) > 1 else "" - return BuildInfo(requires, module, "{}{}".format(module, obj)) + return BuildInfo(requires, module, obj) def perform_isolated_build(build_info, package_venv, dist_dir, setup_dir): with package_venv.new_action( "perform-isolated-build", package_venv.envconfig.envdir ) as action: - script = textwrap.dedent( - """ - import sys - import {} - basename = {}.build_{}({!r}, {{ "--global-option": ["--formats=gztar"]}}) - print(basename)""".format( - build_info.backend_module, build_info.backend_object, "sdist", str(dist_dir) - ) - ) - # need to start with an empty (but existing) source distribution folder if dist_dir.exists(): dist_dir.remove(rec=1, ignore_errors=True) dist_dir.ensure_dir() result = package_venv._pcall( - [package_venv.envconfig.envpython, "-c", script], + [ + package_venv.envconfig.envpython, + BUILD_ISOLATED, + build_info.backend_module, + build_info.backend_object, + str(dist_dir), + ], returnout=True, action=action, cwd=setup_dir, @@ -111,20 +108,13 @@ def perform_isolated_build(build_info, package_venv, dist_dir, setup_dir): def get_build_requires(build_info, package_venv, setup_dir): with package_venv.new_action("get-build-requires", package_venv.envconfig.envdir) as action: - script = textwrap.dedent( - """ - import {} - import json - - backend = {} - for_build_requires = backend.get_requires_for_build_{}(None) - print(json.dumps(for_build_requires)) - """.format( - build_info.backend_module, build_info.backend_object, "sdist" - ) - ).strip() result = package_venv._pcall( - [package_venv.envconfig.envpython, "-c", script], + [ + package_venv.envconfig.envpython, + BUILD_REQUIRE_SCRIPT, + build_info.backend_module, + build_info.backend_object, + ], returnout=True, action=action, cwd=setup_dir, diff --git a/src/tox/reporter.py b/src/tox/reporter.py index adab06b96..ed287957e 100644 --- a/src/tox/reporter.py +++ b/src/tox/reporter.py @@ -36,10 +36,10 @@ def verbosity(self): def log_popen(self, cwd, outpath, cmd_args_shell): """ log information about the action.popen() created process. """ + msg = " {}$ {} ".format(cwd, cmd_args_shell) if outpath: - self.verbosity1(" {}$ {} >{}".format(cwd, cmd_args_shell, outpath)) - else: - self.verbosity1(" {}$ {} ".format(cwd, cmd_args_shell)) + msg = "{} >{}".format(msg, outpath) + self.verbosity1(msg, of="logpopen") @property def messages(self): @@ -100,8 +100,8 @@ def skip(self, msg): def verbosity0(self, msg, **opts): self.logline_if(Verbosity.DEFAULT, "verbosity0", msg, **opts) - def verbosity1(self, msg, **opts): - self.logline_if(Verbosity.INFO, "verbosity1", msg, **opts) + def verbosity1(self, msg, of="verbosity1", **opts): + self.logline_if(Verbosity.INFO, of, msg, **opts) def verbosity2(self, msg, **opts): self.logline_if(Verbosity.DEBUG, "verbosity2", msg, **opts) diff --git a/src/tox/session/__init__.py b/src/tox/session/__init__.py index 2509f2b7c..a5ce663f4 100644 --- a/src/tox/session/__init__.py +++ b/src/tox/session/__init__.py @@ -85,7 +85,7 @@ def build_session(config): return Session(config) -class Session: +class Session(object): """The session object that ties together configuration, reporting, venv creation, testing.""" def __init__(self, config, popen=subprocess.Popen): @@ -149,9 +149,9 @@ def _evaluated_env_list(self): def hook(self): return self.config.pluginmanager.hook - def newaction(self, env, msg, *args): - new_action = Action( - env, + def newaction(self, name, msg, *args): + return Action( + name, msg, args, self.config.logdir, @@ -160,7 +160,6 @@ def newaction(self, env, msg, *args): self.popen, sys.executable, ) - return new_action def runcommand(self): reporter.using("tox-{} from {}".format(tox.__version__, tox.__file__)) diff --git a/src/tox/util/lock.py b/src/tox/util/lock.py index 7f7ad7ce1..0cff0137b 100644 --- a/src/tox/util/lock.py +++ b/src/tox/util/lock.py @@ -34,6 +34,6 @@ def get_unique_file(path, prefix, suffix, report): max_value = max(max_value, int(candidate.name[len(prefix) : -len(suffix)])) except ValueError: continue - winner = path.join("{}-{}.log".format(prefix, max_value + 1)) + winner = path.join("{}{}.log".format(prefix, max_value + 1)) winner.ensure(dir=0) return winner diff --git a/tests/unit/package/test_package_parallel.py b/tests/unit/package/test_package_parallel.py index 7eeb162b5..8d084cd8a 100644 --- a/tests/unit/package/test_package_parallel.py +++ b/tests/unit/package/test_package_parallel.py @@ -1,5 +1,4 @@ import traceback -from functools import partial import py @@ -15,7 +14,7 @@ def test_tox_parallel_build_safe(initproj, cmd, mock_venv, monkeypatch): envlist = py install_cmd = python -m -c 'print("ok")' -- {opts} {packages}' [testenv] - commands = python --version + commands = python -c 'import sys; print(sys.version)' """ }, ) diff --git a/tests/unit/session/test_session.py b/tests/unit/session/test_session.py index 206d7af8d..becb6a389 100644 --- a/tests/unit/session/test_session.py +++ b/tests/unit/session/test_session.py @@ -9,6 +9,7 @@ import tox from tox.exception import MissingDependency, MissingDirectory from tox.package import resolve_package +from tox.reporter import Verbosity def test_resolve_pkg_missing_directory(tmpdir, mocksession): @@ -27,6 +28,7 @@ def test_resolve_pkg_missing_directory_in_distshare(tmpdir, mocksession): def test_resolve_pkg_multiple_valid_versions(tmpdir, mocksession): + mocksession.logging_levels(quiet=Verbosity.DEFAULT, verbose=Verbosity.DEBUG) distshare = tmpdir.join("distshare") distshare.ensure("pkg123-1.3.5.zip") p = distshare.ensure("pkg123-1.4.5.zip") @@ -239,8 +241,8 @@ def build_session(config): monkeypatch.setattr(tox.session, "build_session", build_session) def popen(cmd, **kwargs): - activity_id = res.session._actions[-1].name - activity_name = res.session._actions[-1].activity + activity_id = _actions[-1].name + activity_name = _actions[-1].activity ret = "NOTSET" try: ret = res._popen(cmd, **kwargs) @@ -252,6 +254,25 @@ def popen(cmd, **kwargs): ) return ret + _actions = [] + from tox.action import Action + + _prev_enter = Action.__enter__ + + def enter(self): + _actions.append(self) + return _prev_enter(self) + + monkeypatch.setattr(Action, "__enter__", enter) + + _prev_exit = Action.__exit__ + + def exit_func(self, *args, **kwargs): + del _actions[_actions.index(self)] + _prev_exit(self, *args, **kwargs) + + monkeypatch.setattr(Action, "__exit__", exit_func) + res.result = cmd("-e", tox_env) res.cwd = os.getcwd() @@ -279,7 +300,7 @@ def assert_popen_env(res): assert res.result.ret == 0, res.result.out for tox_id, _, env, __, ___ in res.popens: assert env["TOX_WORK_DIR"] == os.path.join(res.cwd, ".tox") - if tox_id != "tox": + if tox_id != "GLOB": assert env["TOX_ENV_NAME"] == tox_id assert env["TOX_ENV_DIR"] == os.path.join(res.cwd, ".tox", tox_id) @@ -341,7 +362,7 @@ def test_command_prev_fail_command_skip_post_run(cmd, initproj, mock_venv): expected = textwrap.dedent( """ py run-test-pre: commands[0] | python -c 'raise SystemExit(2)' - ERROR: InvocationError for command '{} -c raise SystemExit(2)' (exited with code 2) + ERROR: InvocationError for command {} -c raise SystemExit(2) (exited with code 2) py run-test-post: commands[0] | python -c 'print("post")' post ___________________________________ summary ___________________________________{} @@ -350,5 +371,6 @@ def test_command_prev_fail_command_skip_post_run(cmd, initproj, mock_venv): sys.executable.replace("\\", "\\\\"), "_" if sys.platform != "win32" else "" ) ) - actual = result.out.replace(os.linesep, "\n") - assert expected in actual + have = result.out.replace(os.linesep, "\n") + actual = have[len(have) - len(expected) :] + assert actual == expected diff --git a/tests/unit/test_venv.py b/tests/unit/test_venv.py index 8a8d8fc4c..2b6b932b6 100644 --- a/tests/unit/test_venv.py +++ b/tests/unit/test_venv.py @@ -66,8 +66,8 @@ def test_create(mocksession, newconfig): venv = mocksession.getvenv("py123") assert venv.path == envconfig.envdir assert not venv.path.check() - action = mocksession.newaction(venv, "getenv") - tox_testenv_create(action=action, venv=venv) + with mocksession.newaction(venv, "getenv") as action: + tox_testenv_create(action=action, venv=venv) pcalls = mocksession._pcalls assert len(pcalls) >= 1 args = pcalls[0].args @@ -113,8 +113,8 @@ def test_create_sitepackages(mocksession, newconfig): ) mocksession.new_config(config) venv = mocksession.getvenv("site") - action = mocksession.newaction(venv, "getenv") - tox_testenv_create(action=action, venv=venv) + with mocksession.newaction(venv, "getenv") as action: + tox_testenv_create(action=action, venv=venv) pcalls = mocksession._pcalls assert len(pcalls) >= 1 args = pcalls[0].args @@ -122,8 +122,8 @@ def test_create_sitepackages(mocksession, newconfig): mocksession._clearmocks() venv = mocksession.getvenv("nosite") - action = mocksession.newaction(venv, "getenv") - tox_testenv_create(action=action, venv=venv) + with mocksession.newaction(venv, "getenv") as action: + tox_testenv_create(action=action, venv=venv) pcalls = mocksession._pcalls assert len(pcalls) >= 1 args = pcalls[0].args @@ -143,18 +143,18 @@ def test_install_deps_wildcard(newmocksession): """, ) venv = mocksession.getvenv("py123") - action = mocksession.newaction(venv, "getenv") - tox_testenv_create(action=action, venv=venv) - pcalls = mocksession._pcalls - assert len(pcalls) == 1 - distshare = venv.envconfig.config.distshare - distshare.ensure("dep1-1.0.zip") - distshare.ensure("dep1-1.1.zip") + with mocksession.newaction(venv, "getenv") as action: + tox_testenv_create(action=action, venv=venv) + pcalls = mocksession._pcalls + assert len(pcalls) == 1 + distshare = venv.envconfig.config.distshare + distshare.ensure("dep1-1.0.zip") + distshare.ensure("dep1-1.1.zip") - tox_testenv_install_deps(action=action, venv=venv) - assert len(pcalls) == 2 - args = pcalls[-1].args - assert pcalls[-1].cwd == venv.envconfig.config.toxinidir + tox_testenv_install_deps(action=action, venv=venv) + assert len(pcalls) == 2 + args = pcalls[-1].args + assert pcalls[-1].cwd == venv.envconfig.config.toxinidir assert py.path.local.sysfind("python") == args[0] assert ["-m", "pip"] == args[1:3] @@ -179,25 +179,25 @@ def test_install_deps_indexserver(newmocksession): """, ) venv = mocksession.getvenv("py123") - action = mocksession.newaction(venv, "getenv") - tox_testenv_create(action=action, venv=venv) - pcalls = mocksession._pcalls - assert len(pcalls) == 1 - pcalls[:] = [] + with mocksession.newaction(venv, "getenv") as action: + tox_testenv_create(action=action, venv=venv) + pcalls = mocksession._pcalls + assert len(pcalls) == 1 + pcalls[:] = [] - tox_testenv_install_deps(action=action, venv=venv) - # two different index servers, two calls - assert len(pcalls) == 3 - args = " ".join(pcalls[0].args) - assert "-i " not in args - assert "dep1" in args + tox_testenv_install_deps(action=action, venv=venv) + # two different index servers, two calls + assert len(pcalls) == 3 + args = " ".join(pcalls[0].args) + assert "-i " not in args + assert "dep1" in args - args = " ".join(pcalls[1].args) - assert "-i ABC" in args - assert "dep2" in args - args = " ".join(pcalls[2].args) - assert "-i ABC" in args - assert "dep3" in args + args = " ".join(pcalls[1].args) + assert "-i ABC" in args + assert "dep2" in args + args = " ".join(pcalls[2].args) + assert "-i ABC" in args + assert "dep3" in args def test_install_deps_pre(newmocksession): @@ -211,8 +211,8 @@ def test_install_deps_pre(newmocksession): """, ) venv = mocksession.getvenv("python") - action = mocksession.newaction(venv, "getenv") - tox_testenv_create(action=action, venv=venv) + with mocksession.newaction(venv, "getenv") as action: + tox_testenv_create(action=action, venv=venv) pcalls = mocksession._pcalls assert len(pcalls) == 1 pcalls[:] = [] @@ -254,12 +254,12 @@ def test_install_recreate(newmocksession, tmpdir): ) venv = mocksession.getvenv("python") - action = mocksession.newaction(venv, "update") - venv.update(action) - installpkg(venv, pkg) - mocksession.report.expect("verbosity0", "*create*") - venv.update(action) - mocksession.report.expect("verbosity0", "*recreate*") + with mocksession.newaction(venv, "update") as action: + venv.update(action) + installpkg(venv, pkg) + mocksession.report.expect("verbosity0", "*create*") + venv.update(action) + mocksession.report.expect("verbosity0", "*recreate*") def test_install_sdist_extras(newmocksession): @@ -272,8 +272,8 @@ def test_install_sdist_extras(newmocksession): """, ) venv = mocksession.getvenv("python") - action = mocksession.newaction(venv, "getenv") - tox_testenv_create(action=action, venv=venv) + with mocksession.newaction(venv, "getenv") as action: + tox_testenv_create(action=action, venv=venv) pcalls = mocksession._pcalls assert len(pcalls) == 1 pcalls[:] = [] @@ -292,8 +292,8 @@ def test_develop_extras(newmocksession, tmpdir): """, ) venv = mocksession.getvenv("python") - action = mocksession.newaction(venv, "getenv") - tox_testenv_create(action=action, venv=venv) + with mocksession.newaction(venv, "getenv") as action: + tox_testenv_create(action=action, venv=venv) pcalls = mocksession._pcalls assert len(pcalls) == 1 pcalls[:] = [] @@ -318,9 +318,8 @@ def test_env_variables_added_to_needs_reinstall(tmpdir, mocksession, newconfig, ) mocksession.new_config(config) venv = mocksession.getvenv("python") - action = mocksession.newaction(venv, "hello") - - venv._needs_reinstall(tmpdir, action) + with mocksession.newaction(venv, "hello") as action: + venv._needs_reinstall(tmpdir, action) pcalls = mocksession._pcalls assert len(pcalls) == 2 @@ -345,8 +344,8 @@ def test_test_hashseed_is_in_output(newmocksession, monkeypatch): monkeypatch.setattr("tox.config.make_hashseed", lambda: seed) mocksession = newmocksession([], "") venv = mocksession.getvenv("python") - action = mocksession.newaction(venv, "update") - venv.update(action) + with mocksession.newaction(venv, "update") as action: + venv.update(action) tox.venv.tox_runtest_pre(venv) mocksession.report.expect("verbosity0", "run-test-pre: PYTHONHASHSEED='{}'".format(seed)) @@ -360,8 +359,8 @@ def test_test_runtests_action_command_is_in_output(newmocksession): """, ) venv = mocksession.getvenv("python") - action = mocksession.newaction(venv, "update") - venv.update(action) + with mocksession.newaction(venv, "update") as action: + venv.update(action) venv.test() mocksession.report.expect("verbosity0", "*runtests*commands?0? | echo foo bar") @@ -444,16 +443,16 @@ def test_install_python3(newmocksession): """, ) venv = mocksession.getvenv("py123") - action = mocksession.newaction(venv, "getenv") - tox_testenv_create(action=action, venv=venv) - pcalls = mocksession._pcalls - assert len(pcalls) == 1 - args = pcalls[0].args - assert str(args[2]) == "virtualenv" - pcalls[:] = [] - action = mocksession.newaction(venv, "hello") - venv._install(["hello"], action=action) - assert len(pcalls) == 1 + with mocksession.newaction(venv, "getenv") as action: + tox_testenv_create(action=action, venv=venv) + pcalls = mocksession._pcalls + assert len(pcalls) == 1 + args = pcalls[0].args + assert str(args[2]) == "virtualenv" + pcalls[:] = [] + with mocksession.newaction(venv, "hello") as action: + venv._install(["hello"], action=action) + assert len(pcalls) == 1 args = pcalls[0].args assert py.path.local.sysfind("python") == args[0] assert ["-m", "pip"] == args[1:3] @@ -543,9 +542,9 @@ def test_python_recreation(self, tmpdir, newconfig, mocksession): mocksession.new_config(config) venv = mocksession.getvenv("python") create_config = venv._getliveconfig() - action = mocksession.newaction(venv, "update") - venv.update(action) - assert not venv.path_config.check() + with mocksession.newaction(venv, "update") as action: + venv.update(action) + assert not venv.path_config.check() installpkg(venv, pkg) assert venv.path_config.check() assert mocksession._pcalls @@ -554,43 +553,43 @@ def test_python_recreation(self, tmpdir, newconfig, mocksession): mocksession.report.expect("*", "*create*") # modify config and check that recreation happens mocksession._clearmocks() - action = mocksession.newaction(venv, "update") - venv.update(action) - mocksession.report.expect("*", "*reusing*") - mocksession._clearmocks() - action = mocksession.newaction(venv, "update") - create_config.base_resolved_python_path = py.path.local("balla") - create_config.writeconfig(venv.path_config) - venv.update(action) - mocksession.report.expect("verbosity0", "*recreate*") + with mocksession.newaction(venv, "update") as action: + venv.update(action) + mocksession.report.expect("*", "*reusing*") + mocksession._clearmocks() + with mocksession.newaction(venv, "update") as action: + create_config.base_resolved_python_path = py.path.local("balla") + create_config.writeconfig(venv.path_config) + venv.update(action) + mocksession.report.expect("verbosity0", "*recreate*") def test_dep_recreation(self, newconfig, mocksession): config = newconfig([], "") mocksession.new_config(config) venv = mocksession.getvenv("python") - action = mocksession.newaction(venv, "update") - venv.update(action) - cconfig = venv._getliveconfig() - cconfig.deps[:] = [("1" * 32, "xyz.zip")] - cconfig.writeconfig(venv.path_config) - mocksession._clearmocks() - action = mocksession.newaction(venv, "update") - venv.update(action) - mocksession.report.expect("*", "*recreate*") + with mocksession.newaction(venv, "update") as action: + venv.update(action) + cconfig = venv._getliveconfig() + cconfig.deps[:] = [("1" * 32, "xyz.zip")] + cconfig.writeconfig(venv.path_config) + mocksession._clearmocks() + with mocksession.newaction(venv, "update") as action: + venv.update(action) + mocksession.report.expect("*", "*recreate*") def test_develop_recreation(self, newconfig, mocksession): config = newconfig([], "") mocksession.new_config(config) venv = mocksession.getvenv("python") - action = mocksession.newaction(venv, "update") - venv.update(action) - cconfig = venv._getliveconfig() - cconfig.usedevelop = True - cconfig.writeconfig(venv.path_config) - mocksession._clearmocks() - action = mocksession.newaction(venv, "update") - venv.update(action) - mocksession.report.expect("verbosity0", "*recreate*") + with mocksession.newaction(venv, "update") as action: + venv.update(action) + cconfig = venv._getliveconfig() + cconfig.usedevelop = True + cconfig.writeconfig(venv.path_config) + mocksession._clearmocks() + with mocksession.newaction(venv, "update") as action: + venv.update(action) + mocksession.report.expect("verbosity0", "*recreate*") class TestVenvTest: @@ -604,33 +603,33 @@ def test_envbindir_path(self, newmocksession, monkeypatch): """, ) venv = mocksession.getvenv("python") - action = mocksession.newaction(venv, "getenv") - monkeypatch.setenv("PATH", "xyz") - sysfind_calls = [] - monkeypatch.setattr( - "py.path.local.sysfind", - classmethod(lambda *args, **kwargs: sysfind_calls.append(kwargs) or 0 / 0), - ) - - with pytest.raises(ZeroDivisionError): - venv._install(list("123"), action=action) - assert sysfind_calls.pop()["paths"] == [venv.envconfig.envbindir] - with pytest.raises(ZeroDivisionError): - venv.test(action) - assert sysfind_calls.pop()["paths"] == [venv.envconfig.envbindir] - with pytest.raises(ZeroDivisionError): - venv.run_install_command(["qwe"], action=action) - assert sysfind_calls.pop()["paths"] == [venv.envconfig.envbindir] - monkeypatch.setenv("PIP_RESPECT_VIRTUALENV", "1") - monkeypatch.setenv("PIP_REQUIRE_VIRTUALENV", "1") - monkeypatch.setenv("__PYVENV_LAUNCHER__", "1") - with pytest.raises(ZeroDivisionError): - venv.run_install_command(["qwe"], action=action) - assert "PIP_RESPECT_VIRTUALENV" not in os.environ - assert "PIP_REQUIRE_VIRTUALENV" not in os.environ - assert "__PYVENV_LAUNCHER__" not in os.environ - assert os.environ["PIP_USER"] == "0" - assert os.environ["PIP_NO_DEPS"] == "0" + with mocksession.newaction(venv, "getenv") as action: + monkeypatch.setenv("PATH", "xyz") + sysfind_calls = [] + monkeypatch.setattr( + "py.path.local.sysfind", + classmethod(lambda *args, **kwargs: sysfind_calls.append(kwargs) or 0 / 0), + ) + + with pytest.raises(ZeroDivisionError): + venv._install(list("123"), action=action) + assert sysfind_calls.pop()["paths"] == [venv.envconfig.envbindir] + with pytest.raises(ZeroDivisionError): + venv.test(action) + assert sysfind_calls.pop()["paths"] == [venv.envconfig.envbindir] + with pytest.raises(ZeroDivisionError): + venv.run_install_command(["qwe"], action=action) + assert sysfind_calls.pop()["paths"] == [venv.envconfig.envbindir] + monkeypatch.setenv("PIP_RESPECT_VIRTUALENV", "1") + monkeypatch.setenv("PIP_REQUIRE_VIRTUALENV", "1") + monkeypatch.setenv("__PYVENV_LAUNCHER__", "1") + with pytest.raises(ZeroDivisionError): + venv.run_install_command(["qwe"], action=action) + assert "PIP_RESPECT_VIRTUALENV" not in os.environ + assert "PIP_REQUIRE_VIRTUALENV" not in os.environ + assert "__PYVENV_LAUNCHER__" not in os.environ + assert os.environ["PIP_USER"] == "0" + assert os.environ["PIP_NO_DEPS"] == "0" def test_pythonpath_remove(self, newmocksession, monkeypatch, caplog): monkeypatch.setenv("PYTHONPATH", "/my/awesome/library") @@ -642,8 +641,8 @@ def test_pythonpath_remove(self, newmocksession, monkeypatch, caplog): """, ) venv = mocksession.getvenv("python") - action = mocksession.newaction(venv, "getenv") - venv.run_install_command(["qwe"], action=action) + with mocksession.newaction(venv, "getenv") as action: + venv.run_install_command(["qwe"], action=action) assert "PYTHONPATH" not in os.environ mocksession.report.expect("warning", "*Discarding $PYTHONPATH from environment*") @@ -663,8 +662,8 @@ def test_pythonpath_keep(self, newmocksession, monkeypatch, caplog): """, ) venv = mocksession.getvenv("python") - action = mocksession.newaction(venv, "getenv") - venv.run_install_command(["qwe"], action=action) + with mocksession.newaction(venv, "getenv") as action: + venv.run_install_command(["qwe"], action=action) mocksession.report.not_expect("warning", "*Discarding $PYTHONPATH from environment*") assert "PYTHONPATH" in os.environ @@ -764,8 +763,8 @@ def test_run_install_command(newmocksession): venv = mocksession.getvenv("python") venv.just_created = True venv.envconfig.envdir.ensure(dir=1) - action = mocksession.newaction(venv, "hello") - venv.run_install_command(packages=["whatever"], action=action) + with mocksession.newaction(venv, "hello") as action: + venv.run_install_command(packages=["whatever"], action=action) pcalls = mocksession._pcalls assert len(pcalls) == 1 args = pcalls[0].args @@ -787,8 +786,8 @@ def test_run_custom_install_command(newmocksession): venv = mocksession.getvenv("python") venv.just_created = True venv.envconfig.envdir.ensure(dir=1) - action = mocksession.newaction(venv, "hello") - venv.run_install_command(packages=["whatever"], action=action) + with mocksession.newaction(venv, "hello") as action: + venv.run_install_command(packages=["whatever"], action=action) pcalls = mocksession._pcalls assert len(pcalls) == 1 assert "easy_install" in pcalls[0].args[0] @@ -859,7 +858,8 @@ def tox_testenv_install_deps(self, action, venv): ) venv = mocksession.getvenv("python") - venv.update(action=mocksession.newaction(venv, "getenv")) + with mocksession.newaction(venv, "getenv") as action: + venv.update(action=action) assert log == [1, 2] diff --git a/tests/unit/test_z_cmdline.py b/tests/unit/test_z_cmdline.py index 04faa2282..2e29f1b73 100644 --- a/tests/unit/test_z_cmdline.py +++ b/tests/unit/test_z_cmdline.py @@ -11,6 +11,7 @@ import tox from tox.config import parseconfig +from tox.reporter import Verbosity from tox.session import Session pytest_plugins = "pytester" @@ -18,13 +19,16 @@ class TestSession: def test_log_pcall(self, mocksession): + mocksession.logging_levels(quiet=Verbosity.DEFAULT, verbose=Verbosity.INFO) mocksession.config.logdir.ensure(dir=1) assert not mocksession.config.logdir.listdir() - action = mocksession.newaction(None, "something") - action.popen(["echo"]) - match = mocksession.report.getnext("logpopen") - assert match[1].outpath.relto(mocksession.config.logdir) - assert match[1].shell is False + with mocksession.newaction("what", "something") as action: + action.popen(["echo"]) + match = mocksession.report.getnext("logpopen") + log_name = py.path.local(match[1].split(">")[-1].strip()).relto( + mocksession.config.logdir + ) + assert log_name == "what-0.log" def test_summary_status(self, initproj, capfd): initproj( @@ -136,6 +140,7 @@ def test_unknown_interpreter_and_env(cmd, initproj): basepython=xyz_unknown_interpreter [testenv] changedir=tests + skip_install = true """, }, ) From 7c756b92c131305715ce105190bd1a5d35f62d41 Mon Sep 17 00:00:00 2001 From: Bernat Gabor Date: Sun, 27 Jan 2019 19:22:55 +0000 Subject: [PATCH 04/12] ignore pip wheel metadata --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index addeafa7a..8e754eaaf 100644 --- a/.gitignore +++ b/.gitignore @@ -26,3 +26,5 @@ __pycache__ # release credentials.json + +pip-wheel-metadata From 426c8af63a7afb34325bd0d7c9788575e71577ba Mon Sep 17 00:00:00 2001 From: Bernat Gabor Date: Sun, 27 Jan 2019 19:27:33 +0000 Subject: [PATCH 05/12] avoid calling __init__ outside of constructor --- src/tox/_pytestplugin.py | 2 +- src/tox/package/builder/isolated.py | 1 + src/tox/session/__init__.py | 3 +++ 3 files changed, 5 insertions(+), 1 deletion(-) diff --git a/src/tox/_pytestplugin.py b/src/tox/_pytestplugin.py index ea59e3918..26b2de07d 100644 --- a/src/tox/_pytestplugin.py +++ b/src/tox/_pytestplugin.py @@ -250,7 +250,7 @@ def logging_levels(self, quiet, verbose): def newmocksession(mocksession, newconfig): def newmocksession_(args, source, plugins=()): config = newconfig(args, source, plugins=plugins) - mocksession.__init__(config) + mocksession._reset(config) return mocksession return newmocksession_ diff --git a/src/tox/package/builder/isolated.py b/src/tox/package/builder/isolated.py index 8379fc526..072da9c0b 100644 --- a/src/tox/package/builder/isolated.py +++ b/src/tox/package/builder/isolated.py @@ -1,4 +1,5 @@ from __future__ import unicode_literals + import json from collections import namedtuple diff --git a/src/tox/session/__init__.py b/src/tox/session/__init__.py index a5ce663f4..53241a0ca 100644 --- a/src/tox/session/__init__.py +++ b/src/tox/session/__init__.py @@ -89,6 +89,9 @@ class Session(object): """The session object that ties together configuration, reporting, venv creation, testing.""" def __init__(self, config, popen=subprocess.Popen): + self._reset(config, popen) + + def _reset(self, config, popen=subprocess.Popen): self.config = config self.popen = popen self.resultlog = ResultLog() From bab8d26258f8f1f462218ede01af7f1c252eca03 Mon Sep 17 00:00:00 2001 From: Bernat Gabor Date: Sun, 27 Jan 2019 19:28:05 +0000 Subject: [PATCH 06/12] use True over 1 in while --- src/tox/action.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tox/action.py b/src/tox/action.py index bda9c3432..ba491382e 100644 --- a/src/tox/action.py +++ b/src/tox/action.py @@ -116,7 +116,7 @@ def feed_stdin(self, fin, process, redirect): buf = getattr(sys.stdout, "buffer", sys.stdout) out = None last_time = time.time() - while 1: + while True: # we have to read one byte at a time, otherwise there # might be no output for a long time with slow tests data = fin.read(1) From 4a83815c83d12aed5e2aa200eb90a9d874a08203 Mon Sep 17 00:00:00 2001 From: Bernat Gabor Date: Sat, 2 Feb 2019 15:25:14 +0000 Subject: [PATCH 07/12] fix windows issues --- tests/unit/test_venv.py | 54 ++++++++++++++++++++--------------------- tox.ini | 2 +- 2 files changed, 28 insertions(+), 28 deletions(-) diff --git a/tests/unit/test_venv.py b/tests/unit/test_venv.py index 2b6b932b6..f79386d47 100644 --- a/tests/unit/test_venv.py +++ b/tests/unit/test_venv.py @@ -66,7 +66,7 @@ def test_create(mocksession, newconfig): venv = mocksession.getvenv("py123") assert venv.path == envconfig.envdir assert not venv.path.check() - with mocksession.newaction(venv, "getenv") as action: + with mocksession.newaction(venv.name, "getenv") as action: tox_testenv_create(action=action, venv=venv) pcalls = mocksession._pcalls assert len(pcalls) >= 1 @@ -113,7 +113,7 @@ def test_create_sitepackages(mocksession, newconfig): ) mocksession.new_config(config) venv = mocksession.getvenv("site") - with mocksession.newaction(venv, "getenv") as action: + with mocksession.newaction(venv.name, "getenv") as action: tox_testenv_create(action=action, venv=venv) pcalls = mocksession._pcalls assert len(pcalls) >= 1 @@ -122,7 +122,7 @@ def test_create_sitepackages(mocksession, newconfig): mocksession._clearmocks() venv = mocksession.getvenv("nosite") - with mocksession.newaction(venv, "getenv") as action: + with mocksession.newaction(venv.name, "getenv") as action: tox_testenv_create(action=action, venv=venv) pcalls = mocksession._pcalls assert len(pcalls) >= 1 @@ -143,7 +143,7 @@ def test_install_deps_wildcard(newmocksession): """, ) venv = mocksession.getvenv("py123") - with mocksession.newaction(venv, "getenv") as action: + with mocksession.newaction(venv.name, "getenv") as action: tox_testenv_create(action=action, venv=venv) pcalls = mocksession._pcalls assert len(pcalls) == 1 @@ -179,7 +179,7 @@ def test_install_deps_indexserver(newmocksession): """, ) venv = mocksession.getvenv("py123") - with mocksession.newaction(venv, "getenv") as action: + with mocksession.newaction(venv.name, "getenv") as action: tox_testenv_create(action=action, venv=venv) pcalls = mocksession._pcalls assert len(pcalls) == 1 @@ -211,7 +211,7 @@ def test_install_deps_pre(newmocksession): """, ) venv = mocksession.getvenv("python") - with mocksession.newaction(venv, "getenv") as action: + with mocksession.newaction(venv.name, "getenv") as action: tox_testenv_create(action=action, venv=venv) pcalls = mocksession._pcalls assert len(pcalls) == 1 @@ -254,7 +254,7 @@ def test_install_recreate(newmocksession, tmpdir): ) venv = mocksession.getvenv("python") - with mocksession.newaction(venv, "update") as action: + with mocksession.newaction(venv.name, "update") as action: venv.update(action) installpkg(venv, pkg) mocksession.report.expect("verbosity0", "*create*") @@ -272,7 +272,7 @@ def test_install_sdist_extras(newmocksession): """, ) venv = mocksession.getvenv("python") - with mocksession.newaction(venv, "getenv") as action: + with mocksession.newaction(venv.name, "getenv") as action: tox_testenv_create(action=action, venv=venv) pcalls = mocksession._pcalls assert len(pcalls) == 1 @@ -292,7 +292,7 @@ def test_develop_extras(newmocksession, tmpdir): """, ) venv = mocksession.getvenv("python") - with mocksession.newaction(venv, "getenv") as action: + with mocksession.newaction(venv.name, "getenv") as action: tox_testenv_create(action=action, venv=venv) pcalls = mocksession._pcalls assert len(pcalls) == 1 @@ -318,7 +318,7 @@ def test_env_variables_added_to_needs_reinstall(tmpdir, mocksession, newconfig, ) mocksession.new_config(config) venv = mocksession.getvenv("python") - with mocksession.newaction(venv, "hello") as action: + with mocksession.newaction(venv.name, "hello") as action: venv._needs_reinstall(tmpdir, action) pcalls = mocksession._pcalls @@ -344,7 +344,7 @@ def test_test_hashseed_is_in_output(newmocksession, monkeypatch): monkeypatch.setattr("tox.config.make_hashseed", lambda: seed) mocksession = newmocksession([], "") venv = mocksession.getvenv("python") - with mocksession.newaction(venv, "update") as action: + with mocksession.newaction(venv.name, "update") as action: venv.update(action) tox.venv.tox_runtest_pre(venv) mocksession.report.expect("verbosity0", "run-test-pre: PYTHONHASHSEED='{}'".format(seed)) @@ -359,7 +359,7 @@ def test_test_runtests_action_command_is_in_output(newmocksession): """, ) venv = mocksession.getvenv("python") - with mocksession.newaction(venv, "update") as action: + with mocksession.newaction(venv.name, "update") as action: venv.update(action) venv.test() mocksession.report.expect("verbosity0", "*runtests*commands?0? | echo foo bar") @@ -443,14 +443,14 @@ def test_install_python3(newmocksession): """, ) venv = mocksession.getvenv("py123") - with mocksession.newaction(venv, "getenv") as action: + with mocksession.newaction(venv.name, "getenv") as action: tox_testenv_create(action=action, venv=venv) pcalls = mocksession._pcalls assert len(pcalls) == 1 args = pcalls[0].args assert str(args[2]) == "virtualenv" pcalls[:] = [] - with mocksession.newaction(venv, "hello") as action: + with mocksession.newaction(venv.name, "hello") as action: venv._install(["hello"], action=action) assert len(pcalls) == 1 args = pcalls[0].args @@ -542,7 +542,7 @@ def test_python_recreation(self, tmpdir, newconfig, mocksession): mocksession.new_config(config) venv = mocksession.getvenv("python") create_config = venv._getliveconfig() - with mocksession.newaction(venv, "update") as action: + with mocksession.newaction(venv.name, "update") as action: venv.update(action) assert not venv.path_config.check() installpkg(venv, pkg) @@ -553,11 +553,11 @@ def test_python_recreation(self, tmpdir, newconfig, mocksession): mocksession.report.expect("*", "*create*") # modify config and check that recreation happens mocksession._clearmocks() - with mocksession.newaction(venv, "update") as action: + with mocksession.newaction(venv.name, "update") as action: venv.update(action) mocksession.report.expect("*", "*reusing*") mocksession._clearmocks() - with mocksession.newaction(venv, "update") as action: + with mocksession.newaction(venv.name, "update") as action: create_config.base_resolved_python_path = py.path.local("balla") create_config.writeconfig(venv.path_config) venv.update(action) @@ -567,13 +567,13 @@ def test_dep_recreation(self, newconfig, mocksession): config = newconfig([], "") mocksession.new_config(config) venv = mocksession.getvenv("python") - with mocksession.newaction(venv, "update") as action: + with mocksession.newaction(venv.name, "update") as action: venv.update(action) cconfig = venv._getliveconfig() cconfig.deps[:] = [("1" * 32, "xyz.zip")] cconfig.writeconfig(venv.path_config) mocksession._clearmocks() - with mocksession.newaction(venv, "update") as action: + with mocksession.newaction(venv.name, "update") as action: venv.update(action) mocksession.report.expect("*", "*recreate*") @@ -581,13 +581,13 @@ def test_develop_recreation(self, newconfig, mocksession): config = newconfig([], "") mocksession.new_config(config) venv = mocksession.getvenv("python") - with mocksession.newaction(venv, "update") as action: + with mocksession.newaction(venv.name, "update") as action: venv.update(action) cconfig = venv._getliveconfig() cconfig.usedevelop = True cconfig.writeconfig(venv.path_config) mocksession._clearmocks() - with mocksession.newaction(venv, "update") as action: + with mocksession.newaction(venv.name, "update") as action: venv.update(action) mocksession.report.expect("verbosity0", "*recreate*") @@ -603,7 +603,7 @@ def test_envbindir_path(self, newmocksession, monkeypatch): """, ) venv = mocksession.getvenv("python") - with mocksession.newaction(venv, "getenv") as action: + with mocksession.newaction(venv.name, "getenv") as action: monkeypatch.setenv("PATH", "xyz") sysfind_calls = [] monkeypatch.setattr( @@ -641,7 +641,7 @@ def test_pythonpath_remove(self, newmocksession, monkeypatch, caplog): """, ) venv = mocksession.getvenv("python") - with mocksession.newaction(venv, "getenv") as action: + with mocksession.newaction(venv.name, "getenv") as action: venv.run_install_command(["qwe"], action=action) assert "PYTHONPATH" not in os.environ mocksession.report.expect("warning", "*Discarding $PYTHONPATH from environment*") @@ -662,7 +662,7 @@ def test_pythonpath_keep(self, newmocksession, monkeypatch, caplog): """, ) venv = mocksession.getvenv("python") - with mocksession.newaction(venv, "getenv") as action: + with mocksession.newaction(venv.name, "getenv") as action: venv.run_install_command(["qwe"], action=action) mocksession.report.not_expect("warning", "*Discarding $PYTHONPATH from environment*") assert "PYTHONPATH" in os.environ @@ -763,7 +763,7 @@ def test_run_install_command(newmocksession): venv = mocksession.getvenv("python") venv.just_created = True venv.envconfig.envdir.ensure(dir=1) - with mocksession.newaction(venv, "hello") as action: + with mocksession.newaction(venv.name, "hello") as action: venv.run_install_command(packages=["whatever"], action=action) pcalls = mocksession._pcalls assert len(pcalls) == 1 @@ -786,7 +786,7 @@ def test_run_custom_install_command(newmocksession): venv = mocksession.getvenv("python") venv.just_created = True venv.envconfig.envdir.ensure(dir=1) - with mocksession.newaction(venv, "hello") as action: + with mocksession.newaction(venv.name, "hello") as action: venv.run_install_command(packages=["whatever"], action=action) pcalls = mocksession._pcalls assert len(pcalls) == 1 @@ -858,7 +858,7 @@ def tox_testenv_install_deps(self, action, venv): ) venv = mocksession.getvenv("python") - with mocksession.newaction(venv, "getenv") as action: + with mocksession.newaction(venv.name, "getenv") as action: venv.update(action=action) assert log == [1, 2] diff --git a/tox.ini b/tox.ini index 6b41d414c..9316f4b73 100644 --- a/tox.ini +++ b/tox.ini @@ -176,4 +176,4 @@ deps = {[testenv]deps} {[testenv:notify]deps} usedevelop = True commands = python -m pip list --format=columns - python -c "print('{envpython}')" + python -c "print(r'{envpython}')" From bfac373bdbc849ba6974c409cea16e3ddece1ddf Mon Sep 17 00:00:00 2001 From: Bernat Gabor Date: Sat, 2 Feb 2019 15:59:27 +0000 Subject: [PATCH 08/12] fix more --- src/tox/interpreters.py | 5 ++++- tests/unit/test_interpreters.py | 20 ++++++++++++-------- 2 files changed, 16 insertions(+), 9 deletions(-) diff --git a/src/tox/interpreters.py b/src/tox/interpreters.py index 6c6cfcc33..7e5ac6fe7 100644 --- a/src/tox/interpreters.py +++ b/src/tox/interpreters.py @@ -155,15 +155,18 @@ def tox_get_python_executable(envconfig): if m: return locate_via_py(*groups) + # Exceptions to the usual windows mapping win32map = {"python": sys.executable, "jython": r"c:\jython2.5.1\jython.bat"} + def locate_via_py(*parts): ver = "-{}".format(".".join(parts)) py_exe = distutils.spawn.find_executable("py") if py_exe: proc = subprocess.Popen( - (py_exe, ver, VERSION_QUERY_SCRIPT), stdout=subprocess.PIPE, stderr=subprocess.PIPE + (py_exe, ver, VERSION_QUERY_SCRIPT), stdout=subprocess.PIPE, stderr=subprocess.PIPE, + universal_newlines=True ) out, _ = proc.communicate() result = json.loads(out) diff --git a/tests/unit/test_interpreters.py b/tests/unit/test_interpreters.py index 9320a1080..0e6877ad2 100644 --- a/tests/unit/test_interpreters.py +++ b/tests/unit/test_interpreters.py @@ -1,4 +1,5 @@ import distutils.spawn +import inspect import os import subprocess import sys @@ -33,7 +34,9 @@ def fake_find_exe(exe): assert exe == "py" return "py" - def fake_popen(cmd, stdout, stderr): + from tox.helper import get_version + + def fake_popen(cmd, stdout, stderr, universal_newlines): fake_popen.last_call = cmd[:3] # need to pipe all stdout to collect the version information & need to @@ -42,22 +45,23 @@ def fake_popen(cmd, stdout, stderr): # requested Python interpreter not being installed on the system assert stdout is subprocess.PIPE assert stderr is subprocess.PIPE + assert universal_newlines is True class proc: returncode = 0 @staticmethod def communicate(): - return sys.executable.encode(), None + return get_version.info_as_dump, None return proc monkeypatch.setattr(distutils.spawn, "find_executable", fake_find_exe) monkeypatch.setattr(subprocess, "Popen", fake_popen) assert locate_via_py("3", "6") == sys.executable - assert fake_popen.last_call == ("py", "-3.6", "-c") + assert fake_popen.last_call == ("py", "-3.6", inspect.getsourcefile(get_version)) assert locate_via_py("3") == sys.executable - assert fake_popen.last_call == ("py", "-3", "-c") + assert fake_popen.last_call == ("py", "-3", inspect.getsourcefile(get_version)) def test_tox_get_python_executable(): @@ -172,10 +176,10 @@ def test_exec_failed(): class TestInterpreterInfo: @staticmethod def info( - name="my-name", - executable="my-executable", - version_info="my-version-info", - sysplatform="my-sys-platform", + name="my-name", + executable="my-executable", + version_info="my-version-info", + sysplatform="my-sys-platform", ): return InterpreterInfo(name, executable, version_info, sysplatform) From 053ae63075409cdd02a1e1a801d545aed47e6f8b Mon Sep 17 00:00:00 2001 From: Bernat Gabor Date: Sat, 2 Feb 2019 16:08:10 +0000 Subject: [PATCH 09/12] more fix --- src/tox/_pytestplugin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tox/_pytestplugin.py b/src/tox/_pytestplugin.py index 26b2de07d..fe2822697 100644 --- a/src/tox/_pytestplugin.py +++ b/src/tox/_pytestplugin.py @@ -250,7 +250,7 @@ def logging_levels(self, quiet, verbose): def newmocksession(mocksession, newconfig): def newmocksession_(args, source, plugins=()): config = newconfig(args, source, plugins=plugins) - mocksession._reset(config) + mocksession._reset(config, mocksession.popen) return mocksession return newmocksession_ From acdcdc9c5c2642b3c4856b503a87eef38e591fa1 Mon Sep 17 00:00:00 2001 From: Bernat Gabor Date: Sun, 3 Feb 2019 18:26:29 +0000 Subject: [PATCH 10/12] more fix --- src/tox/_pytestplugin.py | 3 ++- src/tox/interpreters.py | 8 ++++---- tests/unit/session/test_session.py | 2 +- tests/unit/test_interpreters.py | 8 ++++---- 4 files changed, 11 insertions(+), 10 deletions(-) diff --git a/src/tox/_pytestplugin.py b/src/tox/_pytestplugin.py index fe2822697..245d13971 100644 --- a/src/tox/_pytestplugin.py +++ b/src/tox/_pytestplugin.py @@ -16,7 +16,7 @@ from tox import venv from tox.config import parseconfig from tox.reporter import update_default_reporter -from tox.session import Session, main +from tox.session import Session, main, setup_reporter from tox.venv import CreationConfig, VirtualEnv, getdigest mark_dont_run_on_windows = pytest.mark.skipif(os.name == "nt", reason="non windows test") @@ -58,6 +58,7 @@ def create_new_config_file_(args, source=None, plugins=()): s = textwrap.dedent(source) p = tmpdir.join("tox.ini") p.write(s) + setup_reporter(args) with tmpdir.as_cwd(): return parseconfig(args, plugins=plugins) diff --git a/src/tox/interpreters.py b/src/tox/interpreters.py index 7e5ac6fe7..ead865b36 100644 --- a/src/tox/interpreters.py +++ b/src/tox/interpreters.py @@ -155,18 +155,18 @@ def tox_get_python_executable(envconfig): if m: return locate_via_py(*groups) - # Exceptions to the usual windows mapping win32map = {"python": sys.executable, "jython": r"c:\jython2.5.1\jython.bat"} - def locate_via_py(*parts): ver = "-{}".format(".".join(parts)) py_exe = distutils.spawn.find_executable("py") if py_exe: proc = subprocess.Popen( - (py_exe, ver, VERSION_QUERY_SCRIPT), stdout=subprocess.PIPE, stderr=subprocess.PIPE, - universal_newlines=True + (py_exe, ver, VERSION_QUERY_SCRIPT), + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True, ) out, _ = proc.communicate() result = json.loads(out) diff --git a/tests/unit/session/test_session.py b/tests/unit/session/test_session.py index becb6a389..ae7592823 100644 --- a/tests/unit/session/test_session.py +++ b/tests/unit/session/test_session.py @@ -368,7 +368,7 @@ def test_command_prev_fail_command_skip_post_run(cmd, initproj, mock_venv): ___________________________________ summary ___________________________________{} ERROR: py: commands failed """.format( - sys.executable.replace("\\", "\\\\"), "_" if sys.platform != "win32" else "" + sys.executable, "_" if sys.platform != "win32" else "" ) ) have = result.out.replace(os.linesep, "\n") diff --git a/tests/unit/test_interpreters.py b/tests/unit/test_interpreters.py index 0e6877ad2..3e6fb4bd4 100644 --- a/tests/unit/test_interpreters.py +++ b/tests/unit/test_interpreters.py @@ -176,10 +176,10 @@ def test_exec_failed(): class TestInterpreterInfo: @staticmethod def info( - name="my-name", - executable="my-executable", - version_info="my-version-info", - sysplatform="my-sys-platform", + name="my-name", + executable="my-executable", + version_info="my-version-info", + sysplatform="my-sys-platform", ): return InterpreterInfo(name, executable, version_info, sysplatform) From 060e694a021098dc29bc5d7e99699da1f0b0d223 Mon Sep 17 00:00:00 2001 From: Bernat Gabor Date: Sun, 3 Feb 2019 19:23:06 +0000 Subject: [PATCH 11/12] fix python 2 on Windows --- src/tox/helper/build_isolated.py | 8 +++----- src/tox/helper/build_requires.py | 4 ++-- src/tox/package/builder/isolated.py | 2 +- 3 files changed, 6 insertions(+), 8 deletions(-) diff --git a/src/tox/helper/build_isolated.py b/src/tox/helper/build_isolated.py index 27e4e8ba9..59680ad6d 100644 --- a/src/tox/helper/build_isolated.py +++ b/src/tox/helper/build_isolated.py @@ -1,14 +1,12 @@ import sys -backend_spec = sys.argv[1] -backend_obj = sys.argv[2] - +dist_folder = sys.argv[1] +backend_spec = sys.argv[2] +backend_obj = sys.argv[3] if len(sys.argv) >= 4 else None backend = __import__(backend_spec, fromlist=[None]) if backend_obj: backend = getattr(backend, backend_obj) -dist_folder = sys.argv[3] - basename = backend.build_sdist(dist_folder, {"--global-option": ["--formats=gztar"]}) print(basename) diff --git a/src/tox/helper/build_requires.py b/src/tox/helper/build_requires.py index 3456c10f4..08bf94126 100644 --- a/src/tox/helper/build_requires.py +++ b/src/tox/helper/build_requires.py @@ -2,12 +2,12 @@ import sys backend_spec = sys.argv[1] -backend_obj = sys.argv[2] +backend_obj = sys.argv[2] if len(sys.argv) >= 3 else None + backend = __import__(backend_spec, fromlist=[None]) if backend_obj: backend = getattr(backend, backend_obj) - for_build_requires = backend.get_requires_for_build_sdist(None) output = json.dumps(for_build_requires) print(output) diff --git a/src/tox/package/builder/isolated.py b/src/tox/package/builder/isolated.py index 072da9c0b..a932674cd 100644 --- a/src/tox/package/builder/isolated.py +++ b/src/tox/package/builder/isolated.py @@ -95,9 +95,9 @@ def perform_isolated_build(build_info, package_venv, dist_dir, setup_dir): [ package_venv.envconfig.envpython, BUILD_ISOLATED, + str(dist_dir), build_info.backend_module, build_info.backend_object, - str(dist_dir), ], returnout=True, action=action, From cb627e5c54fca52f97b6cf42d5e7479e7d367708 Mon Sep 17 00:00:00 2001 From: Bernat Gabor Date: Tue, 5 Feb 2019 07:37:17 +0000 Subject: [PATCH 12/12] use capfd --- src/tox/_pytestplugin.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/src/tox/_pytestplugin.py b/src/tox/_pytestplugin.py index 245d13971..651ba976c 100644 --- a/src/tox/_pytestplugin.py +++ b/src/tox/_pytestplugin.py @@ -66,7 +66,7 @@ def create_new_config_file_(args, source=None, plugins=()): @pytest.fixture -def cmd(request, monkeypatch): +def cmd(request, monkeypatch, capfd): if request.config.option.no_network: pytest.skip("--no-network was specified, test cannot run") request.addfinalizer(py.path.local().chdir) @@ -77,7 +77,7 @@ def run(*argv): python_paths = (i for i in (os.getcwd(), os.getenv(key)) if i) monkeypatch.setenv(key, os.pathsep.join(python_paths)) - with RunResult(argv) as result: + with RunResult(argv, capfd) as result: prev_run_command = Session.runcommand def run_command(self): @@ -99,24 +99,22 @@ def run_command(self): class RunResult: - def __init__(self, args): + def __init__(self, args, capfd): self.args = args self.ret = None self.duration = None self.out = None self.err = None self.session = None + self.capfd = capfd def __enter__(self): self._start = time.time() - self._out_tell = sys.stdout.tell() - self._err_tell = sys.stderr.tell() return self def __exit__(self, exc_type, exc_val, exc_tb): self.duration = time.time() - self._start - self.out = self._read(sys.stdout, self._out_tell) - self.err = self._read(sys.stderr, self._err_tell) + self.out, self.err = self.capfd.readouterr() def _read(self, out, pos): out.buffer.seek(pos)