From 7727a8d1c535f754c96e0c5099699cc0a751a2ef Mon Sep 17 00:00:00 2001 From: John Sirois Date: Fri, 13 Jul 2018 03:53:26 -0600 Subject: [PATCH] Hack around PEX not forwarding custom interpreters to PEXEnvironment. --- .../pants/contrib/mypy/tasks/mypy_task.py | 2 +- .../python/checks/tasks/python_eval.py | 2 +- .../tasks/python_execution_task_base.py | 2 +- .../pants/backend/python/tasks/wrapped_pex.py | 47 ++++++++++++------- 4 files changed, 32 insertions(+), 21 deletions(-) diff --git a/contrib/mypy/src/python/pants/contrib/mypy/tasks/mypy_task.py b/contrib/mypy/src/python/pants/contrib/mypy/tasks/mypy_task.py index 5222edd08114..9115e4e12fa0 100644 --- a/contrib/mypy/src/python/pants/contrib/mypy/tasks/mypy_task.py +++ b/contrib/mypy/src/python/pants/contrib/mypy/tasks/mypy_task.py @@ -97,7 +97,7 @@ def _run_mypy(self, py3_interpreter, mypy_args, **kwargs): path = os.path.realpath(os.path.join(self.workdir, str(py3_interpreter.identity), mypy_version)) if not os.path.isdir(path): self.merge_pexes(path, pex_info, py3_interpreter, [mypy_requirement_pex]) - pex = WrappedPEX(PEX(path, py3_interpreter), py3_interpreter) + pex = WrappedPEX(PEX(path, py3_interpreter)) return pex.run(mypy_args, **kwargs) def execute(self): diff --git a/contrib/python/src/python/pants/contrib/python/checks/tasks/python_eval.py b/contrib/python/src/python/pants/contrib/python/checks/tasks/python_eval.py index 904a46bf4919..bb70bd8e7a87 100644 --- a/contrib/python/src/python/pants/contrib/python/checks/tasks/python_eval.py +++ b/contrib/python/src/python/pants/contrib/python/checks/tasks/python_eval.py @@ -157,7 +157,7 @@ def _compile_target(self, vt): exec_pex = PEX(exec_pex_path, interpreter) extra_pex_paths = [pex.path() for pex in filter(None, [reqs_pex, srcs_pex])] - pex = WrappedPEX(exec_pex, interpreter, extra_pex_paths) + pex = WrappedPEX(exec_pex, extra_pex_paths) with self.context.new_workunit(name='eval', labels=[WorkUnitLabel.COMPILER, WorkUnitLabel.RUN, diff --git a/src/python/pants/backend/python/tasks/python_execution_task_base.py b/src/python/pants/backend/python/tasks/python_execution_task_base.py index 8ee154c1f45d..7d6f79d9daea 100644 --- a/src/python/pants/backend/python/tasks/python_execution_task_base.py +++ b/src/python/pants/backend/python/tasks/python_execution_task_base.py @@ -119,4 +119,4 @@ def create_pex(self, pex_info=None): extra_file.add_to(builder) builder.freeze() - return WrappedPEX(PEX(path, interpreter), interpreter) + return WrappedPEX(PEX(path, interpreter)) diff --git a/src/python/pants/backend/python/tasks/wrapped_pex.py b/src/python/pants/backend/python/tasks/wrapped_pex.py index 4a128f7baab2..92b13472154a 100644 --- a/src/python/pants/backend/python/tasks/wrapped_pex.py +++ b/src/python/pants/backend/python/tasks/wrapped_pex.py @@ -4,10 +4,14 @@ from __future__ import absolute_import, division, print_function, unicode_literals +import logging from builtins import object from copy import copy +logger = logging.getLogger(__name__) + + class WrappedPEX(object): """Wrapper around a PEX that exposes only its run() method. @@ -15,46 +19,53 @@ class WrappedPEX(object): """ _PEX_PATH_ENV_VAR_NAME = 'PEX_PATH' + _PEX_PYTHON_PATH_ENV_VAR_NAME = 'PEX_PYTHON_PATH' - # TODO(benjy): I think we can get rid of the interpreter argument. - # In all cases it appears to be set to pex.interpreter. - def __init__(self, pex, interpreter, extra_pex_paths=None): + def __init__(self, pex, extra_pex_paths=None): """ :param pex: The main pex we wrap. - :param interpreter: The interpreter the main pex will run on. :param extra_pex_paths: Other pexes, to "merge" in via the PEX_PATH mechanism. """ self._pex = pex - self._interpreter = interpreter self._extra_pex_paths = extra_pex_paths @property def interpreter(self): - return self._interpreter + return self._pex._interpreter def path(self): return self._pex.path() def cmdline(self, args=()): cmdline = ' '.join(self._pex.cmdline(args)) + + def render_env_var(key, value): + return '{key}={value}'.format(key=key, value=value) + + env_vars = [(self._PEX_PYTHON_PATH_ENV_VAR_NAME, self._pex._interpreter.binary)] + pex_path = self._pex_path() if pex_path: - return '{env_var_name}={pex_path} {cmdline}'.format(env_var_name=self._PEX_PATH_ENV_VAR_NAME, - pex_path=pex_path, - cmdline=cmdline) - else: - return cmdline + env_vars.append((self._PEX_PATH_ENV_VAR_NAME, pex_path)) + + return '{execution_control_env_vars} {cmdline}'.format( + execution_control_env_vars=' '.join(render_env_var(k, v) for k, v in env_vars), + cmdline=cmdline + ) def run(self, *args, **kwargs): + env = copy(kwargs.pop('env', {})) + + # Hack around bug in PEX where custom interpreters are not forwarded to PEXEnvironments. + # TODO(John Sirois): Remove when XXX is fixed. + env[self._PEX_PYTHON_PATH_ENV_VAR_NAME] = self._pex._interpreter.binary + pex_path = self._pex_path() if pex_path: - kwargs_copy = copy(kwargs) - env = copy(kwargs_copy.get('env')) if 'env' in kwargs_copy else {} - env[self._PEX_PATH_ENV_VAR_NAME] = self._pex_path() - kwargs_copy['env'] = env - return self._pex.run(*args, **kwargs_copy) - else: - return self._pex.run(*args, **kwargs) + env[self._PEX_PATH_ENV_VAR_NAME] = pex_path + + logger.debug('Executing WrappedPEX using: {}'.format(self.cmdline(args=tuple(*args)))) + return self._pex.run(*args, env=env, **kwargs) def _pex_path(self): if self._extra_pex_paths: