diff --git a/3rdparty/python/requirements.txt b/3rdparty/python/requirements.txt index 75583d00d7fb..f1a00a6a838c 100644 --- a/3rdparty/python/requirements.txt +++ b/3rdparty/python/requirements.txt @@ -17,7 +17,7 @@ more-itertools<6.0.0 ; python_version<'3' packaging==16.8 parameterized==0.6.1 pathspec==0.5.9 -pex==1.5.3 +pex==1.6.2 psutil==5.4.8 pycodestyle==2.4.0 pyflakes==2.0.0 @@ -32,7 +32,7 @@ requests[security]>=2.20.1 responses==0.10.4 scandir==1.2 setproctitle==1.1.10 -setuptools==40.4.3 +setuptools==40.6.3 six>=1.9.0,<2 subprocess32==3.2.7 ; python_version<'3' wheel==0.31.1 diff --git a/contrib/awslambda/python/src/python/pants/contrib/awslambda/python/subsystems/lambdex.py b/contrib/awslambda/python/src/python/pants/contrib/awslambda/python/subsystems/lambdex.py index e3136a9822d8..0313cb7b7783 100644 --- a/contrib/awslambda/python/src/python/pants/contrib/awslambda/python/subsystems/lambdex.py +++ b/contrib/awslambda/python/src/python/pants/contrib/awslambda/python/subsystems/lambdex.py @@ -9,5 +9,5 @@ class Lambdex(PythonToolBase): options_scope = 'lambdex' - default_requirements = ['lambdex==0.1.2'] + default_requirements = ['lambdex==0.1.3'] default_entry_point = 'lambdex.bin.lambdex' diff --git a/src/python/pants/backend/python/interpreter_cache.py b/src/python/pants/backend/python/interpreter_cache.py index 95908ddbd75b..b77c9bea9cea 100644 --- a/src/python/pants/backend/python/interpreter_cache.py +++ b/src/python/pants/backend/python/interpreter_cache.py @@ -7,14 +7,11 @@ import logging import os import shutil -from builtins import map, str +from builtins import str from collections import defaultdict from pex.interpreter import PythonInterpreter -from pex.package import EggPackage, Package, SourcePackage -from pex.resolver import resolve -from pants.backend.python.subsystems.python_repos import PythonRepos from pants.backend.python.subsystems.python_setup import PythonSetup from pants.backend.python.targets.python_target import PythonTarget from pants.base.exceptions import TaskError @@ -27,15 +24,6 @@ logger = logging.getLogger(__name__) -# TODO(wickman) Create a safer version of this and add to twitter.common.dirutil -def _safe_link(src, dst): - try: - os.unlink(dst) - except OSError: - pass - os.symlink(src, dst) - - # TODO: Move under subsystems/ . class PythonInterpreterCache(Subsystem): """Finds python interpreters on the local system.""" @@ -43,8 +31,7 @@ class PythonInterpreterCache(Subsystem): @classmethod def subsystem_dependencies(cls): - return (super(PythonInterpreterCache, cls).subsystem_dependencies() + - (PythonSetup, PythonRepos)) + return super(PythonInterpreterCache, cls).subsystem_dependencies() + (PythonSetup,) class UnsatisfiableInterpreterConstraintsError(TaskError): """Indicates a python interpreter matching given constraints could not be located.""" @@ -63,10 +50,6 @@ def _matching(cls, interpreters, filters=()): def _python_setup(self): return PythonSetup.global_instance() - @memoized_property - def _python_repos(self): - return PythonRepos.global_instance() - @memoized_property def _cache_dir(self): cache_dir = self._python_setup.interpreter_cache_dir @@ -124,9 +107,9 @@ def _interpreter_from_relpath(self, path, filters=()): return None except OSError: return None - interpreter = PythonInterpreter.from_binary(executable, include_site_extras=False) + interpreter = PythonInterpreter.from_binary(executable) if self._matches(interpreter, filters=filters): - return self._resolve(interpreter) + return interpreter return None def _setup_interpreter(self, interpreter, identity_str): @@ -134,7 +117,7 @@ def _setup_interpreter(self, interpreter, identity_str): with safe_concurrent_creation(cache_target_path) as safe_path: os.mkdir(safe_path) # Parent will already have been created by safe_concurrent_creation. os.symlink(interpreter.binary, os.path.join(safe_path, 'python')) - return self._resolve(interpreter, safe_path) + return interpreter def _setup_cached(self, filters=()): """Find all currently-cached interpreters.""" @@ -193,67 +176,6 @@ def unsatisfied_filters(): 'Initialized Python interpreter cache with {}'.format(', '.join([x.binary for x in matches]))) return matches - def _resolve(self, interpreter, interpreter_dir=None): - """Resolve and cache an interpreter with a setuptools and wheel capability.""" - interpreter = self._resolve_interpreter(interpreter, interpreter_dir, - self._python_setup.setuptools_requirement()) - if interpreter: - return self._resolve_interpreter(interpreter, interpreter_dir, - self._python_setup.wheel_requirement()) - - def _resolve_interpreter(self, interpreter, interpreter_dir, requirement): - """Given a :class:`PythonInterpreter` and a requirement, return an interpreter with the - capability of resolving that requirement or ``None`` if it's not possible to install a - suitable requirement. - - If interpreter_dir is unspecified, operates on the default location. - """ - if interpreter.satisfies([requirement]): - return interpreter - - if not interpreter_dir: - interpreter_dir = os.path.join(self._cache_dir, str(interpreter.identity)) - - target_link = os.path.join(interpreter_dir, requirement.key) - bdist = self._resolve_and_link(interpreter, requirement, target_link) - if bdist: - return interpreter.with_extra(bdist.name, bdist.raw_version, bdist.path) - else: - logger.debug('Failed to resolve requirement {} for {}'.format(requirement, interpreter)) - - def _resolve_and_link(self, interpreter, requirement, target_link): - # Short-circuit if there is a local copy. - if os.path.exists(target_link) and os.path.exists(os.path.realpath(target_link)): - bdist = Package.from_href(os.path.realpath(target_link)) - if bdist.satisfies(requirement): - return bdist - - # Since we're resolving to bootstrap a bare interpreter, we won't have wheel available. - # Explicitly set the precedence to avoid resolution of wheels or distillation of sdists into - # wheels. - precedence = (EggPackage, SourcePackage) - resolved_dists = resolve(requirements=[requirement], - fetchers=self._python_repos.get_fetchers(), - interpreter=interpreter, - # The local interpreter cache is, by definition, composed of - # interpreters for the 'current' platform. - platform='current', - context=self._python_repos.get_network_context(), - precedence=precedence) - if not resolved_dists: - return None - - assert len(resolved_dists) == 1, ('Expected exactly 1 distribution to be resolved for {}, ' - 'found:\n\t{}'.format(requirement, - '\n\t'.join(map(str, resolved_dists)))) - - dist_location = resolved_dists[0].distribution.location - target_location = os.path.join(os.path.dirname(target_link), os.path.basename(dist_location)) - shutil.move(dist_location, target_location) - _safe_link(target_location, target_link) - logger.debug(' installed {}'.format(target_location)) - return Package.from_href(target_location) - def _purge_interpreter(self, interpreter_dir): try: logger.info('Detected stale interpreter `{}` in the interpreter cache, purging.' @@ -262,5 +184,5 @@ def _purge_interpreter(self, interpreter_dir): except Exception as e: logger.warn( 'Caught exception {!r} during interpreter purge. Please run `./pants clean-all`!' - .format(e) + .format(e) ) diff --git a/src/python/pants/backend/python/subsystems/BUILD b/src/python/pants/backend/python/subsystems/BUILD index b392a006d827..14366c80b4da 100644 --- a/src/python/pants/backend/python/subsystems/BUILD +++ b/src/python/pants/backend/python/subsystems/BUILD @@ -5,7 +5,6 @@ python_library( dependencies = [ '3rdparty/python:future', '3rdparty/python:pex', - '3rdparty/python:setuptools', '3rdparty/python/twitter/commons:twitter.common.collections', 'src/python/pants/base:build_environment', 'src/python/pants/base:exceptions', diff --git a/src/python/pants/backend/python/subsystems/pex_build_util.py b/src/python/pants/backend/python/subsystems/pex_build_util.py index 4a64df0133bc..1a29bcbd572c 100644 --- a/src/python/pants/backend/python/subsystems/pex_build_util.py +++ b/src/python/pants/backend/python/subsystems/pex_build_util.py @@ -13,8 +13,10 @@ from pex.fetcher import Fetcher from pex.pex_builder import PEXBuilder from pex.resolver import resolve +from pex.util import DistributionHelper from twitter.common.collections import OrderedSet +from pants.backend.python.python_requirement import PythonRequirement from pants.backend.python.subsystems.python_repos import PythonRepos from pants.backend.python.subsystems.python_setup import PythonSetup from pants.backend.python.targets.python_binary import PythonBinary @@ -27,6 +29,7 @@ from pants.build_graph.files import Files from pants.subsystem.subsystem import Subsystem from pants.util.collections import assert_single_element +from pants.util.contextutil import temporary_file def is_python_target(tgt): @@ -110,6 +113,13 @@ class PexBuilderWrapper(object): class Factory(Subsystem): options_scope = 'pex-builder-wrapper' + @classmethod + def register_options(cls, register): + super(PexBuilderWrapper.Factory, cls).register_options(register) + register('--setuptools-version', advanced=True, default='40.6.3', + help='The setuptools version to include in the pex if namespace packages need to be ' + 'injected.') + @classmethod def subsystem_dependencies(cls): return super(PexBuilderWrapper.Factory, cls).subsystem_dependencies() + ( @@ -119,23 +129,38 @@ def subsystem_dependencies(cls): @classmethod def create(cls, builder, log=None): + options = cls.global_instance().get_options() + setuptools_requirement = 'setuptools=={}'.format(options.setuptools_version) + log = log or logging.getLogger(__name__) + return PexBuilderWrapper(builder=builder, python_repos_subsystem=PythonRepos.global_instance(), python_setup_subsystem=PythonSetup.global_instance(), + setuptools_requirement=PythonRequirement(setuptools_requirement), log=log) - def __init__(self, builder, python_repos_subsystem, python_setup_subsystem, log): + def __init__(self, + builder, + python_repos_subsystem, + python_setup_subsystem, + setuptools_requirement, + log): assert isinstance(builder, PEXBuilder) assert isinstance(python_repos_subsystem, PythonRepos) assert isinstance(python_setup_subsystem, PythonSetup) + assert isinstance(setuptools_requirement, PythonRequirement) assert log is not None self._builder = builder self._python_repos_subsystem = python_repos_subsystem self._python_setup_subsystem = python_setup_subsystem + self._setuptools_requirement = setuptools_requirement self._log = log + self._distributions = {} + self._frozen = False + def add_requirement_libs_from(self, req_libs, platforms=None): """Multi-platform dependency resolution for PEX files. @@ -180,7 +205,7 @@ def _resolve_distributions_by_platform(self, reqs, platforms): find_links = OrderedSet() for req in deduped_reqs: self._log.debug(' Dumping requirement: {}'.format(req)) - self._builder.add_requirement(req.requirement) + self._builder.add_requirement(str(req.requirement)) if req.repository: find_links.add(req.repository) @@ -205,7 +230,7 @@ def add_resolved_requirements(self, reqs, platforms=None): for dist in dists: if dist.location not in locations: self._log.debug(' Dumping distribution: .../{}'.format(os.path.basename(dist.location))) - self._builder.add_distribution(dist) + self.add_distribution(dist) locations.add(dist.location) def _resolve_multi(self, interpreter, requirements, platforms, find_links): @@ -233,7 +258,7 @@ def _resolve_multi(self, interpreter, requirements, platforms, find_links): requirements_cache_dir = os.path.join(python_setup.resolver_cache_dir, str(interpreter.identity)) resolved_dists = resolve( - requirements=[req.requirement for req in requirements], + requirements=[str(req.requirement) for req in requirements], interpreter=interpreter, fetchers=fetchers, platform=platform, @@ -267,13 +292,49 @@ def add_sources_from(self, tgt): raise TaskError('Old-style resources not supported for target {}. ' 'Depend on resources() targets instead.'.format(tgt.address.spec)) + def _prepare_inits(self): + chroot = self._builder.chroot() + sources = chroot.get('source') | chroot.get('resource') + + packages = set() + for source in sources: + if source.endswith('.py'): + pkg_dir = os.path.dirname(source) + if pkg_dir and pkg_dir not in packages: + package = '' + for component in pkg_dir.split(os.sep): + package = os.path.join(package, component) + packages.add(package) + + missing_pkg_files = [] + for package in packages: + pkg_file = os.path.join(package, '__init__.py') + if pkg_file not in sources: + missing_pkg_files.append(pkg_file) + + if missing_pkg_files: + with temporary_file() as ns_package: + ns_package.write(b'__import__("pkg_resources").declare_namespace(__name__)') + ns_package.flush() + for missing_pkg_file in missing_pkg_files: + self._builder.add_source(ns_package.name, missing_pkg_file) + + return missing_pkg_files + def freeze(self): - self._builder.freeze() + if not self._frozen: + if self._prepare_inits(): + dist = self._distributions.get('setuptools') + if not dist: + self.add_resolved_requirements([self._setuptools_requirement]) + self._builder.freeze() + self._frozen = True def set_entry_point(self, entry_point): self._builder.set_entry_point(entry_point) def build(self, safe_path): + self.freeze() self._builder.build(safe_path) def set_shebang(self, shebang): @@ -293,13 +354,19 @@ def add_interpreter_constraints_from(self, constraint_tgts): def add_direct_requirements(self, reqs): for req in reqs: - self._builder.add_requirement(req) + self._builder.add_requirement(str(req)) def add_distribution(self, dist): self._builder.add_distribution(dist) + self._register_distribution(dist) def add_dist_location(self, location): self._builder.add_dist_location(location) + dist = DistributionHelper.distribution_from_path(location) + self._register_distribution(dist) + + def _register_distribution(self, dist): + self._distributions[dist.key] = dist def set_script(self, script): self._builder.set_script(script) diff --git a/src/python/pants/backend/python/subsystems/python_native_code.py b/src/python/pants/backend/python/subsystems/python_native_code.py index 7c8bde33385a..0fcdd29001c9 100644 --- a/src/python/pants/backend/python/subsystems/python_native_code.py +++ b/src/python/pants/backend/python/subsystems/python_native_code.py @@ -125,18 +125,16 @@ class BuildSetupRequiresPex(ExecutablePexTool): options_scope = 'build-setup-requires-pex' @classmethod - def subsystem_dependencies(cls): - return super(BuildSetupRequiresPex, cls).subsystem_dependencies() + (PythonSetup,) - - @memoized_property - def python_setup(self): - return PythonSetup.global_instance() + def register_options(cls, register): + super(BuildSetupRequiresPex, cls).register_options(register) + register('--setuptools-version', advanced=True, fingerprint=True, default='40.6.3', + help='The setuptools version to use when executing `setup.py` scripts.') + register('--wheel-version', advanced=True, fingerprint=True, default='0.32.3', + help='The wheel version to use when executing `setup.py` scripts.') @property def base_requirements(self): - # TODO: would we ever want to configure these requirement versions separately from the global - # PythonSetup values? return [ - PythonRequirement('setuptools=={}'.format(self.python_setup.setuptools_version)), - PythonRequirement('wheel=={}'.format(self.python_setup.wheel_version)), + PythonRequirement('setuptools=={}'.format(self.get_options().setuptools_version)), + PythonRequirement('wheel=={}'.format(self.get_options().wheel_version)), ] diff --git a/src/python/pants/backend/python/subsystems/python_setup.py b/src/python/pants/backend/python/subsystems/python_setup.py index 88b609012a46..0fe6e9b24223 100644 --- a/src/python/pants/backend/python/subsystems/python_setup.py +++ b/src/python/pants/backend/python/subsystems/python_setup.py @@ -8,7 +8,6 @@ import os from pex.variables import Variables -from pkg_resources import Requirement from pants.option.custom_types import UnsetBool from pants.subsystem.subsystem import Subsystem @@ -34,9 +33,13 @@ def register_options(cls, register): "be ORed together. These constraints are applied in addition to any " "compatibilities required by the relevant targets.") register('--setuptools-version', advanced=True, default='40.4.3', - help='The setuptools version for this python environment.') + help='The setuptools version for this python environment.', + removal_version='1.17.0.dev2', + removal_hint='This option is now unused and can be removed from pants configuration.') register('--wheel-version', advanced=True, default='0.31.1', - help='The wheel version for this python environment.') + help='The wheel version for this python environment.', + removal_version='1.17.0.dev2', + removal_hint='This option is now unused and can be removed from pants configuration.') register('--platforms', advanced=True, type=list, metavar='', default=['current'], fingerprint=True, help='A list of platforms to be supported by this python environment. Each platform' @@ -78,14 +81,6 @@ def interpreter_constraints(self): def interpreter_search_paths(self): return self.expand_interpreter_search_paths(self.get_options().interpreter_search_paths) - @property - def setuptools_version(self): - return self.get_options().setuptools_version - - @property - def wheel_version(self): - return self.get_options().wheel_version - @property def platforms(self): return self.get_options().platforms @@ -136,24 +131,6 @@ def compatibility_or_constraints(self, target): return tuple(self.interpreter_constraints) return tuple(target.compatibility or self.interpreter_constraints) - def setuptools_requirement(self): - return self._failsafe_parse('setuptools=={0}'.format(self.setuptools_version)) - - def wheel_requirement(self): - return self._failsafe_parse('wheel=={0}'.format(self.wheel_version)) - - # This is a setuptools <1 and >1 compatible version of Requirement.parse. - # For setuptools <1, if you did Requirement.parse('setuptools'), it would - # return 'distribute' which of course is not desirable for us. So they - # added a replacement=False keyword arg. Sadly, they removed this keyword - # arg in setuptools >= 1 so we have to simply failover using TypeError as a - # catch for 'Invalid Keyword Argument'. - def _failsafe_parse(self, requirement): - try: - return Requirement.parse(requirement, replacement=False) - except TypeError: - return Requirement.parse(requirement) - @classmethod def expand_interpreter_search_paths(cls, interpreter_search_paths, pyenv_root_func=None): special_strings = { diff --git a/src/python/pants/backend/python/tasks/select_interpreter.py b/src/python/pants/backend/python/tasks/select_interpreter.py index 98a573069b4e..8dae0049127a 100644 --- a/src/python/pants/backend/python/tasks/select_interpreter.py +++ b/src/python/pants/backend/python/tasks/select_interpreter.py @@ -13,7 +13,6 @@ from pex.interpreter import PythonInterpreter from pants.backend.python.interpreter_cache import PythonInterpreterCache -from pants.backend.python.subsystems.python_setup import PythonSetup from pants.backend.python.targets.python_requirement_library import PythonRequirementLibrary from pants.backend.python.targets.python_target import PythonTarget from pants.base.exceptions import TaskError @@ -46,12 +45,11 @@ class SelectInterpreter(Task): def implementation_version(cls): # TODO(John Sirois): Fixup this task to use VTS results_dirs. Right now version bumps aren't # effective in dealing with workdir data format changes. - return super(SelectInterpreter, cls).implementation_version() + [('SelectInterpreter', 2)] + return super(SelectInterpreter, cls).implementation_version() + [('SelectInterpreter', 3)] @classmethod def subsystem_dependencies(cls): - return super(SelectInterpreter, cls).subsystem_dependencies() + ( - PythonSetup, PythonInterpreterCache) + return super(SelectInterpreter, cls).subsystem_dependencies() + (PythonInterpreterCache,) @classmethod def product_types(cls): @@ -95,12 +93,15 @@ def _create_interpreter_path_file(self, interpreter_path_file, targets): safe_mkdir_for(interpreter_path_file) with open(interpreter_path_file, 'w') as outfile: outfile.write('{}\n'.format(interpreter.binary)) - for dist, location in interpreter.extras.items(): - dist_name, dist_version = dist - outfile.write('{}\t{}\t{}\n'.format(dist_name, dist_version, location)) def _interpreter_path_file(self, target_set_id): - return os.path.join(self.workdir, target_set_id, 'interpreter.info') + # NB: The file name must be changed when its format changes. See the TODO in + # `implementation_version` above for more. + # + # The historical names to avoid: + # - interpreter.path + # - interpreter.info + return os.path.join(self.workdir, target_set_id, 'interpreter.binary') def _detect_and_purge_invalid_interpreter(self, interpreter_path_file): interpreter = self._get_interpreter(interpreter_path_file) @@ -114,13 +115,8 @@ def _detect_and_purge_invalid_interpreter(self, interpreter_path_file): @staticmethod def _get_interpreter(interpreter_path_file): with open(interpreter_path_file, 'r') as infile: - lines = infile.readlines() - binary = lines[0].strip() + binary = infile.read().strip() try: - interpreter = PythonInterpreter.from_binary(binary, include_site_extras=False) + return PythonInterpreter.from_binary(binary) except Executor.ExecutableNotFound: return None - for line in lines[1:]: - dist_name, dist_version, location = line.strip().split('\t') - interpreter = interpreter.with_extra(dist_name, dist_version, location) - return interpreter diff --git a/src/python/pants/backend/python/tasks/setup_py.py b/src/python/pants/backend/python/tasks/setup_py.py index 169663457a1a..62facfcd8f8f 100644 --- a/src/python/pants/backend/python/tasks/setup_py.py +++ b/src/python/pants/backend/python/tasks/setup_py.py @@ -14,7 +14,7 @@ from builtins import bytes, map, object, str, zip from collections import defaultdict -from pex.installer import InstallerBase, Packager +from pex.installer import Packager, WheelInstaller from pex.interpreter import PythonInterpreter from pex.pex import PEX from pex.pex_builder import PEXBuilder @@ -108,29 +108,11 @@ def _write_repr(o, indent=False, level=0): return output.getvalue() -class SetupPyRunner(InstallerBase): - _EXTRAS = ('setuptools', 'wheel') - +class SetupPyRunner(WheelInstaller): def __init__(self, source_dir, setup_command, **kw): self.__setup_command = setup_command super(SetupPyRunner, self).__init__(source_dir, **kw) - def mixins(self): - mixins = super(SetupPyRunner, self).mixins().copy() - extras = set(self._EXTRAS) - for (key, version) in self._interpreter.extras: - if key in extras: - mixins[key] = '{}=={}'.format(key, version) - extras.remove(key) - if not extras: - break - else: - # We know Pants sets up python interpreters with setuptools and wheel via the `PythonSetup` - # subsystem; so this should never happen - raise AssertionError("Expected interpreter {} to have the extras {}" - .format(self._interpreter, self._EXTRAS)) - return mixins - def _setup_command(self): return self.__setup_command diff --git a/tests/python/pants_test/backend/python/BUILD b/tests/python/pants_test/backend/python/BUILD index eee01488e6bc..234b59dfa365 100644 --- a/tests/python/pants_test/backend/python/BUILD +++ b/tests/python/pants_test/backend/python/BUILD @@ -30,19 +30,13 @@ python_tests( sources = ['test_interpreter_cache.py'], dependencies = [ '3rdparty/python:future', - '3rdparty/python:mock', - '3rdparty/python:pex', ':interpreter_selection_utils', 'src/python/pants/backend/python/subsystems', 'src/python/pants/backend/python:interpreter_cache', 'src/python/pants/util:contextutil', - 'tests/python/pants_test/testutils:git_util', 'tests/python/pants_test/testutils:pexrc_util', - 'tests/python/pants_test/testutils:py2_compat', - 'tests/python/pants_test:int-test', 'tests/python/pants_test:test_base', ], - timeout=1200 ) python_tests( diff --git a/tests/python/pants_test/backend/python/tasks/native/test_ctypes_integration.py b/tests/python/pants_test/backend/python/tasks/native/test_ctypes_integration.py index 219692587739..d30edc37866e 100644 --- a/tests/python/pants_test/backend/python/tasks/native/test_ctypes_integration.py +++ b/tests/python/pants_test/backend/python/tasks/native/test_ctypes_integration.py @@ -210,7 +210,7 @@ def test_pants_native_source_detection_for_local_ctypes_dists_for_current_platfo 'toolchain_variant': 'llvm', }, 'python-setup': { - 'platforms': ['current', 'this-platform-does_not-exist'] + 'platforms': ['current', 'this_platform_does_not_exist'] }, }) self.assert_success(pants_run) diff --git a/tests/python/pants_test/backend/python/tasks/test_setup_py.py b/tests/python/pants_test/backend/python/tasks/test_setup_py.py index 9f3cc34f3332..aa24268232b2 100644 --- a/tests/python/pants_test/backend/python/tasks/test_setup_py.py +++ b/tests/python/pants_test/backend/python/tasks/test_setup_py.py @@ -10,11 +10,9 @@ from textwrap import dedent from mock import Mock -from pex.package import Package from twitter.common.collections import OrderedSet from twitter.common.dirutil.chroot import Chroot -from pants.backend.python.subsystems.python_setup import PythonSetup from pants.backend.python.targets.python_library import PythonLibrary from pants.backend.python.tasks.select_interpreter import SelectInterpreter from pants.backend.python.tasks.setup_py import SetupPy, declares_namespace_package @@ -25,7 +23,7 @@ from pants.build_graph.target import Target from pants.fs.archive import TGZ from pants.util.collections_abc_backport import OrderedDict -from pants.util.contextutil import environment_as, temporary_dir, temporary_file +from pants.util.contextutil import temporary_dir, temporary_file from pants.util.dirutil import safe_mkdir from pants_test.backend.python.interpreter_selection_utils import skip_unless_python36_present from pants_test.backend.python.tasks.python_task_test_base import PythonTaskTestBase @@ -57,93 +55,6 @@ def run_execute(self, target, recursive=False): yield setup_py.context.products.get_data(SetupPy.PYTHON_DISTS_PRODUCT) -class TestSetupPyInterpreter(SetupPyTestBase): - class PythonPathInspectableSetupPy(SetupPy): - def _setup_boilerplate(self): - return dedent(""" - # DO NOT EDIT THIS FILE -- AUTOGENERATED BY PANTS - # Target: {setup_target} - - from setuptools import setup - - from foo.commands.print_sys_path import PrintSysPath - - setup( - cmdclass={{'print_sys_path': PrintSysPath}}, - **{setup_dict} - ) - """) - - @classmethod - def task_type(cls): - return cls.PythonPathInspectableSetupPy - - def test_setuptools_version(self): - self.create_file('src/python/foo/__init__.py') - self.create_python_library( - relpath='src/python/foo/commands', - name='commands', - source_contents_map={ - 'print_sys_path.py': dedent(""" - import os - import sys - from setuptools import Command - - - class PrintSysPath(Command): - user_options = [] - - def initialize_options(self): - pass - - def finalize_options(self): - pass - - def run(self): - with open(os.path.join(os.path.dirname(__file__), 'sys_path.txt'), 'w') as fp: - fp.write(os.linesep.join(sys.path)) - """) - }, - ) - foo = self.create_python_library( - relpath='src/python/foo', - name='foo', - dependencies=[ - 'src/python/foo/commands', - ], - provides=dedent(""" - setup_py( - name='foo', - version='0.0.0', - ) - """) - ) - self.set_options(run='print_sys_path') - - # Make sure setup.py can see our custom distutils Command 'print_sys_path'. - sdist_srcdir = os.path.join(self.distdir, 'foo-0.0.0', 'src') - with environment_as(PYTHONPATH=sdist_srcdir): - with self.run_execute(foo): - with open(os.path.join(sdist_srcdir, 'foo', 'commands', 'sys_path.txt'), 'r') as fp: - load_package = lambda: Package.from_href(fp.readline().strip()) - # We don't care about the ordering of `wheel` and `setuptools` on the `sys.path`, just - # that they are 1st as a group. - extras = {p.name: p for p in (load_package(), load_package())} - - def assert_extra(name, expected_version): - package = extras.get(name) - self.assertIsNotNone(package) - self.assertEqual(expected_version, package.raw_version) - - # The 1st two elements of the sys.path should be our custom SetupPyRunner Installer's - # setuptools and wheel mixins, which should match the setuptools and wheel versions - # specified by the PythonSetup subsystem. - init_subsystem(PythonSetup) - python_setup = PythonSetup.global_instance() - assert_extra('setuptools', python_setup.setuptools_version) - assert_extra('wheel', python_setup.wheel_version) - - class TestSetupPy(SetupPyTestBase): def setUp(self): diff --git a/tests/python/pants_test/backend/python/test_interpreter_cache.py b/tests/python/pants_test/backend/python/test_interpreter_cache.py index 997d4136a746..470a068ae5d9 100644 --- a/tests/python/pants_test/backend/python/test_interpreter_cache.py +++ b/tests/python/pants_test/backend/python/test_interpreter_cache.py @@ -10,10 +10,7 @@ from builtins import str from contextlib import contextmanager -import mock from future.utils import PY3 -from pex.package import EggPackage, Package, SourcePackage -from pex.resolver import Unsatisfiable, resolve from pants.backend.python.interpreter_cache import PythonInterpreter, PythonInterpreterCache from pants.subsystem.subsystem import Subsystem @@ -24,7 +21,6 @@ skip_unless_python27_and_python36_present) from pants_test.test_base import TestBase from pants_test.testutils.pexrc_util import setup_pexrc_with_pex_python_path -from pants_test.testutils.py2_compat import assertRegex class TestInterpreterCache(TestBase): @@ -43,11 +39,10 @@ def setUp(self): super(TestInterpreterCache, self).setUp() self._interpreter = PythonInterpreter.get() - def _create_interpreter_cache(self, setup_options=None, repos_options=None): + def _create_interpreter_cache(self, setup_options=None): Subsystem.reset(reset_options=True) self.context(for_subsystems=[PythonInterpreterCache], options={ 'python-setup': setup_options, - 'python-repos': repos_options }) return PythonInterpreterCache.global_instance() @@ -63,7 +58,7 @@ def _setup_cache_at(self, path, constraints=None, search_paths=None): setup_options.update(interpreter_constraints=constraints) if search_paths is not None: setup_options.update(interpreter_search_paths=search_paths) - return self._create_interpreter_cache(setup_options=setup_options, repos_options={}) + return self._create_interpreter_cache(setup_options=setup_options) def test_cache_setup_with_no_filters_uses_repo_default_excluded(self): bad_interpreter_requirement = self._make_bad_requirement(self._interpreter.identity.requirement) @@ -72,84 +67,15 @@ def test_cache_setup_with_no_filters_uses_repo_default_excluded(self): def test_cache_setup_with_no_filters_uses_repo_default(self): with self._setup_cache(constraints=[]) as (cache, _): - self.assertIn(self._interpreter, cache.setup()) + self.assertIn(self._interpreter.identity, [interp.identity for interp in cache.setup()]) def test_cache_setup_with_filter_overrides_repo_default(self): - bad_interpreter_requirement = self._make_bad_requirement(self._interpreter.identity.requirement) + repo_default_requirement = str(self._interpreter.identity.requirement) + bad_interpreter_requirement = self._make_bad_requirement(repo_default_requirement) with self._setup_cache(constraints=[bad_interpreter_requirement]) as (cache, _): - self.assertIn(self._interpreter, - cache.setup(filters=(str(self._interpreter.identity.requirement),))) - - def test_setup_using_eggs(self): - def link_egg(repo_root, requirement): - existing_dist_location = self._interpreter.get_location(requirement) - if existing_dist_location is not None: - existing_dist = Package.from_href(existing_dist_location) - requirement = '{}=={}'.format(existing_dist.name, existing_dist.raw_version) - - resolved_dists = resolve([requirement], - interpreter=self._interpreter, - precedence=(EggPackage, SourcePackage)) - self.assertEqual(1, len(resolved_dists)) - dist_location = resolved_dists[0].distribution.location - - assertRegex(self, dist_location, r'\.egg$') - os.symlink(dist_location, os.path.join(repo_root, os.path.basename(dist_location))) - - return Package.from_href(dist_location).raw_version - - with temporary_dir() as root: - egg_dir = os.path.join(root, 'eggs') - os.makedirs(egg_dir) - setuptools_version = link_egg(egg_dir, 'setuptools') - wheel_version = link_egg(egg_dir, 'wheel') - - interpreter_requirement = self._interpreter.identity.requirement - - cache = self._create_interpreter_cache( - setup_options={ - 'interpreter_cache_dir': None, - 'pants_workdir': os.path.join(root, 'workdir'), - 'constraints': [interpreter_requirement], - 'setuptools_version': setuptools_version, - 'wheel_version': wheel_version, - 'interpreter_search_paths': [os.path.dirname(self._interpreter.binary)] - }, - repos_options={ - 'indexes': [], - 'repos': [egg_dir], - } - ) - interpereters = cache.setup(filters=[str(interpreter_requirement)]) - self.assertGreater(len(interpereters), 0) - - def assert_egg_extra(interpreter, name, version): - location = interpreter.get_location('{}=={}'.format(name, version)) - self.assertIsNotNone(location) - self.assertIsInstance(Package.from_href(location), EggPackage) - - for interpreter in interpereters: - assert_egg_extra(interpreter, 'setuptools', setuptools_version) - assert_egg_extra(interpreter, 'wheel', wheel_version) - - def test_setup_resolve_failure_cleanup(self): - """Simulates a resolution failure during interpreter setup to avoid partial interpreter caching. - - See https://github.com/pantsbuild/pants/issues/2038 for more info. - """ - with mock.patch.object(PythonInterpreterCache, '_resolve') as mock_resolve, \ - self._setup_cache() as (cache, cache_path): - mock_resolve.side_effect = Unsatisfiable('nope') - - with self.assertRaises(Unsatisfiable): - cache._setup_interpreter(self._interpreter, os.path.join(cache_path, 'CPython-2.7.11')) - - # Before the bugfix, the above call would leave behind paths in the tmpdir that looked like: - # - # /tmp/tmpUrCSzk/CPython-2.7.11.tmp.a167fc50834a4f00aa280780c3e1ba21 - # - self.assertFalse('.tmp.' in ' '.join(os.listdir(cache_path)), - 'interpreter cache path contains tmp dirs!') + self.assertIn(self._interpreter.identity, + [interp.identity + for interp in cache.setup(filters=(repo_default_requirement,))]) @skip_unless_python27_and_python36_present def test_interpereter_cache_setup_using_pex_python_paths(self):