implement tox environment provisioning

docs/changelog/998.feature.rst

@@ -0,0 +1,6 @@
+Auto provision host tox requirements. In case the host tox does not satisfy either the
+:conf:`minversion` or the :conf:`requires`, tox will now automatically create a virtual environment
+under :conf:`provision_tox_env` that satisfies those constraints and delegate all calls to this
+meta environment. This should allow automatically satisfying constraints on your tox environment,
+given you have at least version ``3.8.0`` of tox. Plugins no longer need to be manually satisfied
+by the users, increasing their ease of use.

docs/config.rst

@@ -37,7 +37,8 @@ Global settings are defined under the ``tox`` section as:
 .. conf:: minversion
 
    Define the minimal tox version required to run; if the host tox is less than this
-   the tool with exit with an error message indicating the user needs to upgrade tox.
+   the tool with create an environment and provision it with a tox that satisfies it
+   under :conf:`provision_tox_env`.
 
 .. conf:: toxworkdir ^ PATH ^ {toxinidir}/.tox
 
@@ -127,17 +128,22 @@ Global settings are defined under the ``tox`` section as:
     .. versionadded:: 3.2.0
 
     Specify python packages that need to exist alongside the tox installation for the tox build
-    to be able to start. Use this to specify plugin requirements and build dependencies.
+    to be able to start. Use this to specify plugin requirements and build dependencies. If
+    these dependencies are not specified tox will create :conf:`provision_tox_env` environment
+    so that they are satisfied and delegate all calls to that.
 
     .. code-block:: ini
 
         [tox]
         requires = tox-venv
                    setuptools >= 30.0.0
 
-    .. note:: tox does **not** install those required packages for you. tox only checks if the
-              requirements are satisfied and crashes early with an helpful error rather then later
-              in the process.
+.. conf:: provision_tox_env ^ string ^ .tox
+
+    .. versionadded:: 3.8.0
+
+    Name of the virtual environment used to provision a tox having all dependencies specified
+    inside :conf:`requires` and :conf:`minversion`.
 
 .. conf:: isolated_build ^ true|false ^ false
 

src/tox/action.py

@@ -140,7 +140,6 @@ def feed_stdin(self, fin, process, redirect):
             else:
                 out, err = process.communicate()
         except KeyboardInterrupt:
-            reporter.error("KEYBOARDINTERRUPT")
             process.wait()
             raise
         return out

src/tox/config/__init__.py

@@ -21,6 +21,7 @@
 import tox
 from tox.constants import INFO
 from tox.interpreters import Interpreters, NoInterpreterInfo
+from tox.reporter import update_default_reporter
 
 from .parallel import ENV_VAR_KEY as PARALLEL_ENV_VAR_KEY
 from .parallel import add_parallel_config, add_parallel_flags
@@ -227,6 +228,7 @@ def parseconfig(args, plugins=()):
     """
     pm = get_plugin_manager(plugins)
     config, option = parse_cli(args, pm)
+    update_default_reporter(config.option.quiet_level, config.option.verbose_level)
 
     for config_file in propose_configs(option.configfile):
         config_type = config_file.basename
@@ -572,7 +574,7 @@ def basepython_default(testenv_config, value):
 
     parser.add_testenv_attribute(
         name="basepython",
-        type="string",
+        type="basepython",
         default=None,
         postprocess=basepython_default,
         help="executable name or path of interpreter used to create a virtual test environment.",
@@ -977,25 +979,6 @@ def __init__(self, config, ini_path, ini_data):  # noqa
 
         reader.addsubstitutions(toxinidir=config.toxinidir, homedir=config.homedir)
 
-        # As older versions of tox may have bugs or incompatibilities that
-        # prevent parsing of tox.ini this must be the first thing checked.
-        config.minversion = reader.getstring("minversion", None)
-        if config.minversion:
-            # As older versions of tox may have bugs or incompatibilities that
-            # prevent parsing of tox.ini this must be the first thing checked.
-            config.minversion = reader.getstring("minversion", None)
-            if config.minversion:
-                tox_version = pkg_resources.parse_version(tox.__version__)
-                config_min_version = pkg_resources.parse_version(self.config.minversion)
-                if config_min_version > tox_version:
-                    raise tox.exception.MinVersionError(
-                        "tox version is {}, required is at least {}".format(
-                            tox.__version__, self.config.minversion
-                        )
-                    )
-
-        self.ensure_requires_satisfied(reader.getlist("requires"))
-
         if config.option.workdir is None:
             config.toxworkdir = reader.getpath("toxworkdir", "{toxinidir}/.tox")
         else:
@@ -1004,19 +987,30 @@ def __init__(self, config, ini_path, ini_data):  # noqa
         if os.path.exists(str(config.toxworkdir)):
             config.toxworkdir = config.toxworkdir.realpath()
 
-        if config.option.skip_missing_interpreters == "config":
-            val = reader.getbool("skip_missing_interpreters", False)
-            config.option.skip_missing_interpreters = "true" if val else "false"
-
+        reader.addsubstitutions(toxworkdir=config.toxworkdir)
         config.ignore_basepython_conflict = reader.getbool("ignore_basepython_conflict", False)
 
+        config.distdir = reader.getpath("distdir", "{toxworkdir}/dist")
+
+        reader.addsubstitutions(distdir=config.distdir)
+        config.distshare = reader.getpath("distshare", dist_share_default)
+        config.temp_dir = reader.getpath("temp_dir", "{toxworkdir}/.tmp")
+        reader.addsubstitutions(distshare=config.distshare)
+        config.sdistsrc = reader.getpath("sdistsrc", None)
+        config.setupdir = reader.getpath("setupdir", "{toxinidir}")
+        config.logdir = config.toxworkdir.join("log")
+
         # determine indexserver dictionary
         config.indexserver = {"default": IndexServerConfig("default")}
         prefix = "indexserver"
         for line in reader.getlist(prefix):
             name, url = map(lambda x: x.strip(), line.split("=", 1))
             config.indexserver[name] = IndexServerConfig(name, url)
 
+        if config.option.skip_missing_interpreters == "config":
+            val = reader.getbool("skip_missing_interpreters", False)
+            config.option.skip_missing_interpreters = "true" if val else "false"
+
         override = False
         if config.option.indexurl:
             for url_def in config.option.indexurl:
@@ -1037,16 +1031,7 @@ def __init__(self, config, ini_path, ini_data):  # noqa
             for name in config.indexserver:
                 config.indexserver[name] = IndexServerConfig(name, override)
 
-        reader.addsubstitutions(toxworkdir=config.toxworkdir)
-        config.distdir = reader.getpath("distdir", "{toxworkdir}/dist")
-
-        reader.addsubstitutions(distdir=config.distdir)
-        config.distshare = reader.getpath("distshare", dist_share_default)
-        config.temp_dir = reader.getpath("temp_dir", "{toxworkdir}/.tmp")
-        reader.addsubstitutions(distshare=config.distshare)
-        config.sdistsrc = reader.getpath("sdistsrc", None)
-        config.setupdir = reader.getpath("setupdir", "{toxinidir}")
-        config.logdir = config.toxworkdir.join("log")
+        self.handle_provision(config, reader)
 
         self.parse_build_isolation(config, reader)
         config.envlist, all_envs = self._getenvdata(reader, config)
@@ -1080,32 +1065,43 @@ def __init__(self, config, ini_path, ini_data):  # noqa
 
         config.skipsdist = reader.getbool("skipsdist", all_develop)
 
-    def parse_build_isolation(self, config, reader):
-        config.isolated_build = reader.getbool("isolated_build", False)
-        config.isolated_build_env = reader.getstring("isolated_build_env", ".package")
-        if config.isolated_build is True:
-            name = config.isolated_build_env
-            if name not in config.envconfigs:
-                config.envconfigs[name] = self.make_envconfig(
-                    name, "{}{}".format(testenvprefix, name), reader._subs, config
-                )
+    def handle_provision(self, config, reader):
+        requires_list = reader.getlist("requires")
+        config.minversion = reader.getstring("minversion", None)
+        requires_list.append("tox >= {}".format(config.minversion or tox.__version__))
+        config.provision_tox_env = name = reader.getstring("provision_tox_env", ".tox")
+        env_config = self.make_envconfig(
+            name, "{}{}".format(testenvprefix, name), reader._subs, config
+        )
+        env_config.deps = [DepConfig(r, None) for r in requires_list]
+        self.ensure_requires_satisfied(config, env_config)
 
     @staticmethod
-    def ensure_requires_satisfied(specified):
+    def ensure_requires_satisfied(config, env_config):
         missing_requirements = []
-        for s in specified:
+        deps = env_config.deps
+        for require in deps:
+            # noinspection PyBroadException
             try:
-                pkg_resources.get_distribution(s)
+                pkg_resources.get_distribution(require.name)
             except pkg_resources.RequirementParseError:
                 raise
             except Exception:
-                missing_requirements.append(str(pkg_resources.Requirement(s)))
+                missing_requirements.append(str(pkg_resources.Requirement(require.name)))
+        config.run_provision = bool(missing_requirements)
         if missing_requirements:
-            raise tox.exception.MissingRequirement(
-                "Packages {} need to be installed alongside tox in {}".format(
-                    ", ".join(missing_requirements), sys.executable
+            config.envconfigs[config.provision_tox_env] = env_config
+            raise tox.exception.MissingRequirement(config)
+
+    def parse_build_isolation(self, config, reader):
+        config.isolated_build = reader.getbool("isolated_build", False)
+        config.isolated_build_env = reader.getstring("isolated_build_env", ".package")
+        if config.isolated_build is True:
+            name = config.isolated_build_env
+            if name not in config.envconfigs:
+                config.envconfigs[name] = self.make_envconfig(
+                    name, "{}{}".format(testenvprefix, name), reader._subs, config
                 )
-            )
 
     def _list_section_factors(self, section):
         factors = set()
@@ -1132,6 +1128,11 @@ def make_envconfig(self, name, section, subs, config, replace=True):
                 if atype in ("bool", "path", "string", "dict", "dict_setenv", "argv", "argvlist"):
                     meth = getattr(reader, "get{}".format(atype))
                     res = meth(env_attr.name, env_attr.default, replace=replace)
+                elif atype == "basepython":
+                    no_fallback = name in (config.provision_tox_env,)
+                    res = reader.getstring(
+                        env_attr.name, env_attr.default, replace=replace, no_fallback=no_fallback
+                    )
                 elif atype == "space-separated-list":
                     res = reader.getlist(env_attr.name, sep=" ")
                 elif atype == "line-list":
@@ -1363,9 +1364,10 @@ def getargvlist(self, name, default="", replace=True):
     def getargv(self, name, default="", replace=True):
         return self.getargvlist(name, default, replace=replace)[0]
 
-    def getstring(self, name, default=None, replace=True, crossonly=False):
+    def getstring(self, name, default=None, replace=True, crossonly=False, no_fallback=False):
         x = None
-        for s in [self.section_name] + self.fallbacksections:
+        sections = [self.section_name] + ([] if no_fallback else self.fallbacksections)
+        for s in sections:
             try:
                 x = self._cfg[s][name]
                 break

src/tox/exception.py

@@ -1,4 +1,5 @@
 import os
+import pipes
 import signal
 
 
@@ -80,10 +81,8 @@ class MissingDependency(Error):
 class MissingRequirement(Error):
     """A requirement defined in :config:`require` is not met."""
 
+    def __init__(self, config):
+        self.config = config
 
-class MinVersionError(Error):
-    """The installed tox version is lower than requested minversion."""
-
-    def __init__(self, message):
-        self.message = message
-        super(MinVersionError, self).__init__(message)
+    def __str__(self):
+        return " ".join(pipes.quote(i) for i in self.config.requires)

src/tox/session/__init__.py

@@ -29,6 +29,7 @@
 
 from .commands.help import show_help
 from .commands.help_ini import show_help_ini
+from .commands.provision import provision_tox
 from .commands.run.parallel import run_parallel
 from .commands.run.sequential import run_sequential
 from .commands.show_config import show_config
@@ -55,7 +56,6 @@ def main(args):
     setup_reporter(args)
     try:
         config = load_config(args)
-        update_default_reporter(config.option.quiet_level, config.option.verbose_level)
         reporter.using("tox.ini: {}".format(config.toxinipath))
         config.logdir.ensure(dir=1)
         ensure_empty_dir(config.logdir)
@@ -66,19 +66,19 @@ def main(args):
         raise SystemExit(retcode)
     except KeyboardInterrupt:
         raise SystemExit(2)
-    except (tox.exception.MinVersionError, tox.exception.MissingRequirement) as exception:
-        reporter.error(str(exception))
-        raise SystemExit(1)
 
 
 def load_config(args):
-    config = parseconfig(args)
-    if config.option.help:
-        show_help(config)
-        raise SystemExit(0)
-    elif config.option.helpini:
-        show_help_ini(config)
-        raise SystemExit(0)
+    try:
+        config = parseconfig(args)
+        if config.option.help:
+            show_help(config)
+            raise SystemExit(0)
+        elif config.option.helpini:
+            show_help_ini(config)
+            raise SystemExit(0)
+    except tox.exception.MissingRequirement as exception:
+        config = exception.config
     return config
 
 
@@ -97,7 +97,7 @@ def _reset(self, config, popen=subprocess.Popen):
         self.popen = popen
         self.resultlog = ResultLog()
         self.existing_venvs = OrderedDict()
-        self.venv_dict = self._build_venvs()
+        self.venv_dict = {} if self.config.run_provision else self._build_venvs()
 
     def _build_venvs(self):
         try:
@@ -168,15 +168,19 @@ def newaction(self, name, msg, *args):
     def runcommand(self):
         reporter.using("tox-{} from {}".format(tox.__version__, tox.__file__))
         show_description = reporter.has_level(reporter.Verbosity.DEFAULT)
-        if self.config.option.showconfig:
-            self.showconfig()
-        elif self.config.option.listenvs:
-            self.showenvs(all_envs=False, description=show_description)
-        elif self.config.option.listenvs_all:
-            self.showenvs(all_envs=True, description=show_description)
+        if self.config.run_provision:
+            provision_tox_venv = self.getvenv(self.config.provision_tox_env)
+            provision_tox(provision_tox_venv, self.config.args)
         else:
-            with self.cleanup():
-                return self.subcommand_test()
+            if self.config.option.showconfig:
+                self.showconfig()
+            elif self.config.option.listenvs:
+                self.showenvs(all_envs=False, description=show_description)
+            elif self.config.option.listenvs_all:
+                self.showenvs(all_envs=True, description=show_description)
+            else:
+                with self.cleanup():
+                    return self.subcommand_test()
 
     @contextmanager
     def cleanup(self):

src/tox/session/commands/provision.py

@@ -0,0 +1,31 @@
+"""In case the tox environment is not correctly setup provision it and delegate execution"""
+import signal
+import subprocess
+
+
+def provision_tox(provision_venv, args):
+    ensure_meta_env_up_to_date(provision_venv)
+    process = start_meta_tox(args, provision_venv)
+    result_out = wait_for_meta_tox(process)
+    raise SystemExit(result_out)
+
+
+def ensure_meta_env_up_to_date(provision_venv):
+    if provision_venv.setupenv():
+        provision_venv.finishvenv()
+
+
+def start_meta_tox(args, provision_venv):
+    provision_args = [str(provision_venv.envconfig.envpython), "-m", "tox"] + args
+    process = subprocess.Popen(provision_args)
+    return process
+
+
+def wait_for_meta_tox(process):
+    try:
+        result_out = process.wait()
+    except KeyboardInterrupt:
+        # if we try to interrupt delegate interrupt to meta tox
+        process.send_signal(signal.SIGINT)
+        result_out = process.wait()
+    return result_out

src/tox/session/commands/show_env.py

@@ -6,7 +6,7 @@
 def show_envs(config, all_envs=False, description=False):
     env_conf = config.envconfigs  # this contains all environments
     default = config.envlist  # this only the defaults
-    ignore = {config.isolated_build_env}.union(default)
+    ignore = {config.isolated_build_env, config.provision_tox_env}.union(default)
     extra = [e for e in env_conf if e not in ignore] if all_envs else []
 
     if description: