diff --git a/pex/bin/pex.py b/pex/bin/pex.py index 38a19eecd..c4ea84798 100755 --- a/pex/bin/pex.py +++ b/pex/bin/pex.py @@ -661,10 +661,11 @@ def walk_and_do(fn, src_dir): allow_prereleases=resolver_option_builder.prereleases_allowed, use_manylinux=options.use_manylinux) - for dist in resolveds: - log(' %s' % dist, v=options.verbosity) - pex_builder.add_distribution(dist) - pex_builder.add_requirement(dist.as_requirement()) + for resolved_dist in resolveds: + log(' %s -> %s' % (resolved_dist.requirement, resolved_dist.distribution), + v=options.verbosity) + pex_builder.add_distribution(resolved_dist.distribution) + pex_builder.add_requirement(resolved_dist.requirement) except Unsatisfiable as e: die(e) diff --git a/pex/environment.py b/pex/environment.py index 9f9c056c0..c1afbb4aa 100644 --- a/pex/environment.py +++ b/pex/environment.py @@ -161,6 +161,9 @@ def _resolve(self, working_set, reqs): # Resolve them one at a time so that we can figure out which ones we need to elide should # there be an interpreter incompatibility. for req in reqs: + if req.marker and not req.marker.evaluate(): + TRACER.log('Skipping activation of `%s` due to environment marker de-selection' % req) + continue with TRACER.timed('Resolving %s' % req, V=2): try: resolveds.update(working_set.resolve([req], env=self)) diff --git a/pex/resolver.py b/pex/resolver.py index 911735517..8db6ba5bd 100644 --- a/pex/resolver.py +++ b/pex/resolver.py @@ -159,6 +159,10 @@ def map_packages(resolved_packages): return _ResolvableSet([map_packages(rp) for rp in self.__tuples]) +class ResolvedDistribution(namedtuple('ResolvedDistribution', 'requirement distribution')): + """A requirement and the resolved distribution that satisfies it.""" + + class Resolver(object): """Interface for resolving resolvable entities into python packages.""" @@ -212,12 +216,10 @@ def expand_platform(): # platform. return expand_platform() - def __init__(self, allow_prereleases=None, interpreter=None, platform=None, - pkg_blacklist=None, use_manylinux=None): + def __init__(self, allow_prereleases=None, interpreter=None, platform=None, use_manylinux=None): self._interpreter = interpreter or PythonInterpreter.get() self._platform = self._maybe_expand_platform(self._interpreter, platform) self._allow_prereleases = allow_prereleases - self._blacklist = pkg_blacklist.copy() if pkg_blacklist else {} self._supported_tags = self._platform.supported_tags( self._interpreter, use_manylinux @@ -257,12 +259,6 @@ def build(self, package, options): 'Could not get distribution for %s on platform %s.' % (package, self._platform)) return dist - def _resolvable_is_blacklisted(self, resolvable_name): - return ( - resolvable_name in self._blacklist and - self._interpreter.identity.matches(self._blacklist[resolvable_name]) - ) - def resolve(self, resolvables, resolvable_set=None): resolvables = [(resolvable, None) for resolvable in resolvables] resolvable_set = resolvable_set or _ResolvableSet() @@ -277,10 +273,7 @@ def resolve(self, resolvables, resolvable_set=None): continue packages = self.package_iterator(resolvable, existing=resolvable_set.get(resolvable.name)) - # TODO: Remove blacklist strategy in favor of smart requirement handling - # https://github.com/pantsbuild/pex/issues/456 - if not self._resolvable_is_blacklisted(resolvable.name): - resolvable_set.merge(resolvable, packages, parent) + resolvable_set.merge(resolvable, packages, parent) processed_resolvables.add(resolvable) built_packages = {} @@ -327,7 +320,13 @@ def resolve(self, resolvables, resolvable_set=None): continue assert len(packages) > 0, 'ResolvableSet.packages(%s) should not be empty' % resolvable package = next(iter(packages)) - dists.append(distributions[package]) + distribution = distributions[package] + if isinstance(resolvable, ResolvableRequirement): + requirement = resolvable.requirement + else: + requirement = distribution.as_requirement() + dists.append(ResolvedDistribution(requirement=requirement, + distribution=distribution)) return dists @@ -404,7 +403,6 @@ def resolve(requirements, cache=None, cache_ttl=None, allow_prereleases=None, - pkg_blacklist=None, use_manylinux=None): """Produce all distributions needed to (recursively) meet `requirements` @@ -440,16 +438,8 @@ def resolve(requirements, ``context``. :keyword allow_prereleases: (optional) Include pre-release and development versions. If unspecified only stable versions will be resolved, unless explicitly included. - :keyword pkg_blacklist: (optional) A blacklist dict (str->str) that maps package name to - an interpreter constraint. If a package name is in the blacklist and its interpreter - constraint matches the target interpreter, skip the requirement. This is needed to ensure - that universal requirement resolves for a target interpreter version do not error out on - interpreter specific requirements such as backport libs like `functools32`. - For example, a valid blacklist is {'functools32': 'CPython>3'}. - NOTE: this keyword is a temporary fix and will be reverted in favor of a long term solution - tracked by: https://github.com/pantsbuild/pex/issues/456 :keyword use_manylinux: (optional) Whether or not to use manylinux for linux resolves. - :returns: List of :class:`pkg_resources.Distribution` instances meeting ``requirements``. + :returns: List of :class:`ResolvedDistribution` instances meeting ``requirements``. :raises Unsatisfiable: If ``requirements`` is not transitively satisfiable. :raises Untranslateable: If no compatible distributions could be acquired for a particular requirement. @@ -475,6 +465,10 @@ def resolve(requirements, .. versionchanged:: 1.0 ``resolver`` is now just a wrapper around the :class:`Resolver` and :class:`CachingResolver` classes. + + .. versionchanged:: 1.5.0 + The ``pkg_blacklist`` has been removed and the return type change to a list of + :class:`ResolvedDistribution`. """ builder = ResolverOptionsBuilder(fetchers=fetchers, @@ -489,14 +483,12 @@ def resolve(requirements, allow_prereleases=allow_prereleases, use_manylinux=use_manylinux, interpreter=interpreter, - platform=platform, - pkg_blacklist=pkg_blacklist) + platform=platform) else: resolver = Resolver(allow_prereleases=allow_prereleases, use_manylinux=use_manylinux, interpreter=interpreter, - platform=platform, - pkg_blacklist=pkg_blacklist) + platform=platform) return resolver.resolve(resolvables_from_iterable(requirements, builder)) @@ -510,7 +502,6 @@ def resolve_multi(requirements, cache=None, cache_ttl=None, allow_prereleases=None, - pkg_blacklist=None, use_manylinux=None): """A generator function that produces all distributions needed to meet `requirements` for multiple interpreters and/or platforms. @@ -542,15 +533,7 @@ def resolve_multi(requirements, ``context``. :keyword allow_prereleases: (optional) Include pre-release and development versions. If unspecified only stable versions will be resolved, unless explicitly included. - :keyword pkg_blacklist: (optional) A blacklist dict (str->str) that maps package name to - an interpreter constraint. If a package name is in the blacklist and its interpreter - constraint matches the target interpreter, skip the requirement. This is needed to ensure - that universal requirement resolves for a target interpreter version do not error out on - interpreter specific requirements such as backport libs like `functools32`. - For example, a valid blacklist is {'functools32': 'CPython>3'}. - NOTE: this keyword is a temporary fix and will be reverted in favor of a long term solution - tracked by: https://github.com/pantsbuild/pex/issues/456 - :yields: All :class:`pkg_resources.Distribution` instances meeting ``requirements``. + :yields: All :class:`ResolvedDistribution` instances meeting ``requirements``. :raises Unsatisfiable: If ``requirements`` is not transitively satisfiable. :raises Untranslateable: If no compatible distributions could be acquired for a particular requirement. @@ -571,7 +554,6 @@ def resolve_multi(requirements, cache, cache_ttl, allow_prereleases, - pkg_blacklist=pkg_blacklist, use_manylinux=use_manylinux): if resolvable not in seen: seen.add(resolvable) diff --git a/tests/test_environment.py b/tests/test_environment.py index 00479eda7..4cdefa3bc 100644 --- a/tests/test_environment.py +++ b/tests/test_environment.py @@ -127,20 +127,21 @@ def bad_interpreter(include_site_extras=True): # We need to run the bad interpreter with a modern, non-Apple-Extras setuptools in order to # successfully install psutil. for requirement in (SETUPTOOLS_REQUIREMENT, WHEEL_REQUIREMENT): - for dist in resolver.resolve([requirement], - cache=cache, - # We can't use wheels since we're bootstrapping them. - precedence=(SourcePackage, EggPackage), - interpreter=interpreter): + for resolved_dist in resolver.resolve([requirement], + cache=cache, + # We can't use wheels since we're bootstrapping them. + precedence=(SourcePackage, EggPackage), + interpreter=interpreter): + dist = resolved_dist.distribution interpreter = interpreter.with_extra(dist.key, dist.version, dist.location) with nested(yield_pex_builder(installer_impl=WheelInstaller, interpreter=interpreter), temporary_filename()) as (pb, pex_file): - for dist in resolver.resolve(['psutil==5.4.3'], - cache=cache, - precedence=(SourcePackage, WheelPackage), - interpreter=interpreter): - pb.add_dist_location(dist.location) + for resolved_dist in resolver.resolve(['psutil==5.4.3'], + cache=cache, + precedence=(SourcePackage, WheelPackage), + interpreter=interpreter): + pb.add_dist_location(resolved_dist.distribution.location) pb.build(pex_file) # NB: We want PEX to find the bare bad interpreter at runtime. diff --git a/tests/test_integration.py b/tests/test_integration.py index 6b2d86c96..78116aee4 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -12,7 +12,7 @@ import pytest from twitter.common.contextutil import environment_as, temporary_dir -from pex.compatibility import WINDOWS +from pex.compatibility import WINDOWS, to_bytes from pex.installer import EggInstaller from pex.pex_bootstrapper import get_pex_info from pex.testing import ( @@ -1107,3 +1107,39 @@ def test_setup_interpreter_constraint(): '-o', pex]) results.assert_success() subprocess.check_call([pex, '-c', 'import jsonschema']) + + +@pytest.mark.skipif(IS_PYPY, + reason='Our pyenv interpreter setup fails under pypy: ' + 'https://github.com/pantsbuild/pex/issues/477') +def test_setup_python_multiple(): + py27_interpreter = ensure_python_interpreter(PY27) + py36_interpreter = ensure_python_interpreter(PY36) + with temporary_dir() as out: + pex = os.path.join(out, 'pex.pex') + results = run_pex_command(['jsonschema==2.6.0', + '--disable-cache', + '--python-shebang=#!/usr/bin/env python', + '--python={}'.format(py27_interpreter), + '--python={}'.format(py36_interpreter), + '-o', pex]) + results.assert_success() + + pex_program = [pex, '-c'] + py2_only_program = pex_program + ['import functools32'] + both_program = pex_program + [ + 'import jsonschema, os, sys; print(os.path.realpath(sys.executable))' + ] + + with environment_as(PATH=os.path.dirname(py27_interpreter)): + subprocess.check_call(py2_only_program) + + stdout = subprocess.check_output(both_program) + assert to_bytes(os.path.realpath(py27_interpreter)) == stdout.strip() + + with environment_as(PATH=os.path.dirname(py36_interpreter)): + with pytest.raises(subprocess.CalledProcessError): + subprocess.check_call(py2_only_program) + + stdout = subprocess.check_output(both_program) + assert to_bytes(os.path.realpath(py36_interpreter)) == stdout.strip() diff --git a/tests/test_pex.py b/tests/test_pex.py index 20a926499..5ec6ac1f2 100644 --- a/tests/test_pex.py +++ b/tests/test_pex.py @@ -419,7 +419,8 @@ def test_execute_interpreter_file_program(): def test_pex_run_custom_setuptools_useable(): with temporary_dir() as resolve_cache: - dists = resolve(['setuptools==36.2.7'], cache=resolve_cache) + dists = [resolved_dist.distribution + for resolved_dist in resolve(['setuptools==36.2.7'], cache=resolve_cache)] with temporary_dir() as temp_dir: pex = write_simple_pex( temp_dir, @@ -440,11 +441,13 @@ def test_pex_run_conflicting_custom_setuptools_useable(): # > pkg_resources/py31compat.py # > pkg_resources/_vendor/appdirs.py with temporary_dir() as resolve_cache: - dists = resolve(['setuptools==20.3.1'], cache=resolve_cache) + dists = [resolved_dist.distribution + for resolved_dist in resolve(['setuptools==20.3.1'], cache=resolve_cache)] interpreter = PythonInterpreter.from_binary(sys.executable, path_extras=[dist.location for dist in dists], include_site_extras=False) - dists = resolve(['setuptools==40.4.3'], cache=resolve_cache) + dists = [resolved_dist.distribution + for resolved_dist in resolve(['setuptools==40.4.3'], cache=resolve_cache)] with temporary_dir() as temp_dir: pex = write_simple_pex( temp_dir, diff --git a/tests/test_resolver.py b/tests/test_resolver.py index ee35e041d..d6f3e3ba6 100644 --- a/tests/test_resolver.py +++ b/tests/test_resolver.py @@ -8,12 +8,11 @@ from twitter.common.contextutil import temporary_dir from pex.common import safe_copy -from pex.compatibility import PY2 from pex.crawler import Crawler from pex.fetcher import Fetcher from pex.package import EggPackage, SourcePackage from pex.resolvable import ResolvableRequirement -from pex.resolver import Resolver, Unsatisfiable, _ResolvableSet, resolve, resolve_multi +from pex.resolver import Resolver, Unsatisfiable, _ResolvableSet, resolve_multi from pex.resolver_options import ResolverOptionsBuilder from pex.testing import make_sdist @@ -33,8 +32,8 @@ def test_simple_local_resolve(): with temporary_dir() as td: safe_copy(project_sdist, os.path.join(td, os.path.basename(project_sdist))) fetchers = [Fetcher([td])] - dists = list(resolve_multi(['project'], fetchers=fetchers)) - assert len(dists) == 1 + resolved_dists = list(resolve_multi(['project'], fetchers=fetchers)) + assert len(resolved_dists) == 1 def test_diamond_local_resolve_cached(): @@ -47,10 +46,10 @@ def test_diamond_local_resolve_cached(): safe_copy(sdist, os.path.join(dd, os.path.basename(sdist))) fetchers = [Fetcher([dd])] with temporary_dir() as cd: - dists = list( + resolved_dists = list( resolve_multi(['project1', 'project2'], fetchers=fetchers, cache=cd, cache_ttl=1000) ) - assert len(dists) == 2 + assert len(resolved_dists) == 2 def test_cached_dependency_pinned_unpinned_resolution_multi_run(): @@ -64,39 +63,39 @@ def test_cached_dependency_pinned_unpinned_resolution_multi_run(): fetchers = [Fetcher([td])] with temporary_dir() as cd: # First run, pinning 1.0.0 in the cache - dists = list( + resolved_dists = list( resolve_multi(['project', 'project==1.0.0'], fetchers=fetchers, cache=cd, cache_ttl=1000) ) - assert len(dists) == 1 - assert dists[0].version == '1.0.0' + assert len(resolved_dists) == 1 + assert resolved_dists[0].distribution.version == '1.0.0' # This simulates separate invocations of pex but allows us to keep the same tmp cache dir Crawler.reset_cache() # Second, run, the unbounded 'project' req will find the 1.0.0 in the cache. But should also # return SourcePackages found in td - dists = list( + resolved_dists = list( resolve_multi(['project', 'project==1.1.0'], fetchers=fetchers, cache=cd, cache_ttl=1000) ) - assert len(dists) == 1 - assert dists[0].version == '1.1.0' + assert len(resolved_dists) == 1 + assert resolved_dists[0].distribution.version == '1.1.0' # Third run, if exact resolvable and inexact resolvable, and cache_ttl is expired, exact # resolvable should pull from pypi as well since inexact will and the resulting # resolvable_set.merge() would fail. Crawler.reset_cache() time.sleep(1) - dists = list( + resolved_dists = list( resolve_multi(['project', 'project==1.1.0'], fetchers=fetchers, cache=cd, cache_ttl=1) ) - assert len(dists) == 1 - assert dists[0].version == '1.1.0' + assert len(resolved_dists) == 1 + assert resolved_dists[0].distribution.version == '1.1.0' def test_ambiguous_transitive_resolvable(): @@ -111,14 +110,14 @@ def test_ambiguous_transitive_resolvable(): safe_copy(sdist, os.path.join(td, os.path.basename(sdist))) fetchers = [Fetcher([td])] with temporary_dir() as cd: - dists = list( + resolved_dists = list( resolve_multi(['foo', 'bar'], fetchers=fetchers, cache=cd, cache_ttl=1000) ) - assert len(dists) == 2 - assert dists[0].version == '1.0.0' + assert len(resolved_dists) == 2 + assert resolved_dists[0].distribution.version == '1.0.0' def test_resolve_prereleases(): @@ -131,12 +130,12 @@ def test_resolve_prereleases(): fetchers = [Fetcher([td])] def assert_resolve(expected_version, **resolve_kwargs): - dists = list( + resolved_dists = list( resolve_multi(['dep>=1,<4'], fetchers=fetchers, **resolve_kwargs) ) - assert 1 == len(dists) - dist = dists[0] - assert expected_version == dist.version + assert 1 == len(resolved_dists) + resolved_dist = resolved_dists[0] + assert expected_version == resolved_dist.distribution.version assert_resolve('2.0.0') assert_resolve('2.0.0', allow_prereleases=False) @@ -154,12 +153,12 @@ def test_resolve_prereleases_cached(): with temporary_dir() as cd: def assert_resolve(dep, expected_version, **resolve_kwargs): - dists = list( + resolved_dists = list( resolve_multi([dep], cache=cd, cache_ttl=1000, **resolve_kwargs) ) - assert 1 == len(dists) - dist = dists[0] - assert expected_version == dist.version + assert 1 == len(resolved_dists) + resolved_dist = resolved_dists[0] + assert expected_version == resolved_dist.distribution.version Crawler.reset_cache() @@ -186,12 +185,12 @@ def test_resolve_prereleases_and_no_version(): fetchers = [Fetcher([td])] def assert_resolve(deps, expected_version, **resolve_kwargs): - dists = list( + resolved_dists = list( resolve_multi(deps, fetchers=fetchers, **resolve_kwargs) ) - assert 1 == len(dists) - dist = dists[0] - assert expected_version == dist.version + assert 1 == len(resolved_dists) + resolved_dist = resolved_dists[0] + assert expected_version == resolved_dist.distribution.version # When allow_prereleases is specified, the requirement (from two dependencies) # for a specific pre-release version and no version specified, accepts the pre-release @@ -217,14 +216,14 @@ def test_resolve_prereleases_multiple_set(): fetchers = [Fetcher([td])] def assert_resolve(expected_version, **resolve_kwargs): - dists = list( + resolved_dists = list( resolve_multi(['dep>=3.0.0rc1', 'dep==3.0.0rc4'], fetchers=fetchers, **resolve_kwargs) ) - assert 1 == len(dists) - dist = dists[0] - assert expected_version == dist.version + assert 1 == len(resolved_dists) + resolved_dist = resolved_dists[0] + assert expected_version == resolved_dist.distribution.version # This should resolve with explicit prerelease being set or implicitly. assert_resolve('3.0.0rc4', allow_prereleases=True) @@ -350,26 +349,3 @@ def test_resolvable_set_built(): updated_rs.merge(rq, [binary_pkg]) assert updated_rs.get('foo') == set([binary_pkg]) assert updated_rs.packages() == [(rq, set([binary_pkg]), None, False)] - - -def test_resolver_blacklist(): - if PY2: - blacklist = {'project2': '<3'} - required_project = "project2;python_version>'3'" - else: - blacklist = {'project2': '>3'} - required_project = "project2;python_version<'3'" - - project1 = make_sdist(name='project1', version='1.0.0', install_reqs=[required_project]) - project2 = make_sdist(name='project2', version='1.1.0') - - with temporary_dir() as td: - safe_copy(project1, os.path.join(td, os.path.basename(project1))) - safe_copy(project2, os.path.join(td, os.path.basename(project2))) - fetchers = [Fetcher([td])] - - dists = resolve(['project1'], fetchers=fetchers) - assert len(dists) == 2 - - dists = resolve(['project1'], fetchers=fetchers, pkg_blacklist=blacklist) - assert len(dists) == 1