From f503f80cf4244d912e11d7a436146a5a4203e690 Mon Sep 17 00:00:00 2001 From: John Sirois Date: Thu, 28 Mar 2019 14:07:11 -0700 Subject: [PATCH] Use all compatible versions when calculating tags. Previously, we'd pass a single version to `get_supported` when we knew one which would lead to a single tag being generated for the abi3 case instead of one per previous minor version. Fixes #539 --- pex/platforms.py | 90 +++++++++++++++++++++++++++++++++------ scripts/style.sh | 8 +++- tests/test_environment.py | 4 +- tests/test_platform.py | 6 +-- tox.ini | 2 + 5 files changed, 91 insertions(+), 19 deletions(-) diff --git a/pex/platforms.py b/pex/platforms.py index 49ab56dc3..db3f22783 100644 --- a/pex/platforms.py +++ b/pex/platforms.py @@ -5,14 +5,82 @@ from collections import namedtuple -from pex.pep425tags import ( - get_abbr_impl, - get_abi_tag, - get_impl_ver, - get_platform, - get_supported, - get_supported_for_any_abi -) +from pex.orderedset import OrderedSet +from pex.pep425tags import get_abbr_impl, get_abi_tag, get_impl_ver, get_platform, get_supported + + +def _gen_all_compatible_versions(version): + # We select major and minor here in the context of implementation version strings. + # These are typically two digit characters; eg "27" or "36", but they can be three digit + # characters in the case of pypy; eg "271". In the typical case the 1st digit represents the + # python major version and the 2nd digit it's minor version. In the pypy case, the 1st digit still + # represents the (hosting) python major version, the 2nd the pypy major version and the 3rd the + # pypy minor version. In both cases the last digit is the minor version and the python in question + # guarantees backwards compatibility of minor version bumps within the major version as per + # semver. + # + # Concrete examples of what we want to return in each case: + # 1. typical case of cpython "36": ["36", "35", "34", "33", "32", "31", "30"] + # 2. pypy case of "271": ["271", "270"]. + # + # For more information on the pypy case see conversation here: + # https://github.com/pypa/pip/issues/2882 + # In particular https://github.com/pypa/pip/issues/2882#issuecomment-110925458 and + # https://github.com/pypa/pip/issues/2882#issuecomment-130404840. + # The fix work for pip handling of this is done here: https://github.com/pypa/pip/pull/3075 + + major, minor = version[:-1], version[-1] + + def iter_compatible_versions(): + # Support all previous minor Python versions. + for compatible_minor in range(int(minor), -1, -1): + yield '{major}{minor}'.format(major=major, minor=compatible_minor) + + return list(iter_compatible_versions()) + + +def _get_supported(version=None, platform=None, impl=None, abi=None, force_manylinux=False): + versions = _gen_all_compatible_versions(version) if version is not None else None + all_supported = get_supported( + versions=versions, + platform=platform, + impl=impl, + abi=abi + ) + + def iter_all_supported(): + for supported in all_supported: + yield supported + python_tag, abi_tag, platform_tag = supported + if platform_tag.startswith('linux') and force_manylinux: + yield python_tag, abi_tag, platform_tag.replace('linux', 'manylinux1') + + return list(OrderedSet(iter_all_supported())) + + +def _gen_all_abis(impl, version): + def tmpl_abi(impl, version, suffix): + return ''.join((impl, version, suffix)) + yield tmpl_abi(impl, version, 'd') + yield tmpl_abi(impl, version, 'dm') + yield tmpl_abi(impl, version, 'dmu') + yield tmpl_abi(impl, version, 'm') + yield tmpl_abi(impl, version, 'mu') + yield tmpl_abi(impl, version, 'u') + + +def _get_supported_for_any_abi(version=None, platform=None, impl=None, force_manylinux=False): + """Generates supported tags for unspecified ABI types to support more intuitive cross-platform + resolution.""" + unique_tags = { + tag for abi in _gen_all_abis(impl, version) + for tag in _get_supported(version=version, + platform=platform, + impl=impl, + abi=abi, + force_manylinux=force_manylinux) + } + return list(unique_tags) class Platform(namedtuple('Platform', ['platform', 'impl', 'version', 'abi'])): @@ -73,19 +141,17 @@ def supported_tags(self, interpreter=None, force_manylinux=True): # N.B. If we don't get an extended platform specifier, we generate # all possible ABI permutations to mimic earlier pex version # behavior and make cross-platform resolution more intuitive. - tags = get_supported_for_any_abi( + return _get_supported_for_any_abi( platform=self.platform, impl=interpreter.identity.abbr_impl, version=interpreter.identity.impl_ver, force_manylinux=force_manylinux ) else: - tags = get_supported( + return _get_supported( platform=self.platform, impl=self.impl, version=self.version, abi=self.abi, force_manylinux=force_manylinux ) - - return tags diff --git a/scripts/style.sh b/scripts/style.sh index eda83d36e..04e26267a 100755 --- a/scripts/style.sh +++ b/scripts/style.sh @@ -3,5 +3,9 @@ ROOT_DIR="$(git rev-parse --show-toplevel)" twitterstyle -n ImportOrder "${ROOT_DIR}/tests" $( - find "${ROOT_DIR}/pex" -path "${ROOT_DIR}/pex/vendor/_vendored" -prune , -name "*.py" -) \ No newline at end of file + find "${ROOT_DIR}/pex" -name "*.py" | \ + grep -v \ + -e "${ROOT_DIR}/pex/vendor/_vendored/" \ + -e "${ROOT_DIR}/pex/glibc.py" \ + -e "${ROOT_DIR}/pex/pep425tags.py" +) diff --git a/tests/test_environment.py b/tests/test_environment.py index 3617617fc..af6dee85e 100644 --- a/tests/test_environment.py +++ b/tests/test_environment.py @@ -325,7 +325,7 @@ def run(args, **env): def test_activate_extras_issue_615(): with yield_pex_builder() as pb: - for resolved_dist in resolver.resolve(['pex[requests]==1.5.1'], interpreter=pb.interpreter): + for resolved_dist in resolver.resolve(['pex[requests]==1.6.3'], interpreter=pb.interpreter): pb.add_requirement(resolved_dist.requirement) pb.add_dist_location(resolved_dist.distribution.location) pb.set_script('pex') @@ -339,4 +339,4 @@ def test_activate_extras_issue_615(): assert 0 == process.returncode, ( 'Process failed with exit code {} and output:\n{}'.format(process.returncode, stderr) ) - assert to_bytes('{} 1.5.1'.format(os.path.basename(pb.path()))) == stdout.strip() + assert to_bytes('{} 1.6.3'.format(os.path.basename(pb.path()))) == stdout.strip() diff --git a/tests/test_platform.py b/tests/test_platform.py index 50c8b81fe..4971be6a2 100644 --- a/tests/test_platform.py +++ b/tests/test_platform.py @@ -58,11 +58,11 @@ def test_platform_supported_tags_manylinux(): def test_platform_supported_tags_osx_minimal(): impl_tag = "{}{}".format(get_abbr_impl(), get_impl_ver()) assert_tags( - 'macosx-10.4-x86_64', + 'macosx-10.5-x86_64', [ (impl_tag, 'none', 'any'), ('py%s' % sys.version_info[0], 'none', 'any'), - (impl_tag, get_abi_tag(), 'macosx_10_4_x86_64') + (impl_tag, get_abi_tag(), 'macosx_10_5_x86_64') ] ) @@ -71,7 +71,7 @@ def test_platform_supported_tags_osx_full(): assert_tags( 'macosx-10.12-x86_64-cp-27-m', EXPECTED_BASE + [ - ('cp27', 'cp27m', 'macosx_10_4_x86_64'), + ('cp27', 'cp27m', 'macosx_10_4_intel'), ('cp27', 'cp27m', 'macosx_10_5_x86_64'), ('cp27', 'cp27m', 'macosx_10_6_x86_64'), ('cp27', 'cp27m', 'macosx_10_7_x86_64'), diff --git a/tox.ini b/tox.ini index 7eb909956..1e337992f 100644 --- a/tox.ini +++ b/tox.ini @@ -135,6 +135,8 @@ commands = --recursive \ --dont-skip __init__.py \ --skip-glob {toxinidir}/pex/vendor/_vendored/** \ + --skip {toxinidir}/pex/glibc.py \ + --skip {toxinidir}/pex/pep425tags.py \ {toxinidir}/pex {toxinidir}/tests [testenv:isort-check]