Use all compatible versions when calculating tags.

Previously, we'd pass a single version to `get_supported` when we knew
one which would lead to a single tag being generated for the abi3 case
instead of one per previous minor version.

Fixes pex-tool#539

pex/platforms.py

@@ -5,14 +5,82 @@
 
 from collections import namedtuple
 
-from pex.pep425tags import (
-    get_abbr_impl,
-    get_abi_tag,
-    get_impl_ver,
-    get_platform,
-    get_supported,
-    get_supported_for_any_abi
-)
+from pex.orderedset import OrderedSet
+from pex.pep425tags import get_abbr_impl, get_abi_tag, get_impl_ver, get_platform, get_supported
+
+
+def _gen_all_compatible_versions(version):
+  # We select major and minor here in the context of implementation version strings.
+  # These are typically two digit characters; eg "27" or "36", but they can be three digit
+  # characters in the case of pypy; eg "271". In the typical case the 1st digit represents the
+  # python major version and the 2nd digit it's minor version. In the pypy case, the 1st digit still
+  # represents the (hosting) python major version, the 2nd the pypy major version and the 3rd the
+  # pypy minor version. In both cases the last digit is the minor version and the python in question
+  # guarantees backwards compatibility of minor version bumps within the major version as per
+  # semver.
+  #
+  # Concrete examples of what we want to return in each case:
+  # 1. typical case of cpython "36": ["36", "35", "34", "33", "32", "31", "30"]
+  # 2. pypy case of "271": ["271", "270"].
+  #
+  # For more information on the pypy case see conversation here:
+  #   https://github.com/pypa/pip/issues/2882
+  # In particular https://github.com/pypa/pip/issues/2882#issuecomment-110925458 and
+  # https://github.com/pypa/pip/issues/2882#issuecomment-130404840.
+  # The fix work for pip handling of this is done here: https://github.com/pypa/pip/pull/3075
+
+  major, minor = version[:-1], version[-1]
+
+  def iter_compatible_versions():
+    # Support all previous minor Python versions.
+    for compatible_minor in range(int(minor), -1, -1):
+      yield '{major}{minor}'.format(major=major, minor=compatible_minor)
+
+  return list(iter_compatible_versions())
+
+
+def _get_supported(version=None, platform=None, impl=None, abi=None, force_manylinux=False):
+  versions = _gen_all_compatible_versions(version) if version is not None else None
+  all_supported = get_supported(
+    versions=versions,
+    platform=platform,
+    impl=impl,
+    abi=abi
+  )
+
+  def iter_all_supported():
+    for supported in all_supported:
+      yield supported
+      python_tag, abi_tag, platform_tag = supported
+      if platform_tag.startswith('linux') and force_manylinux:
+        yield python_tag, abi_tag, platform_tag.replace('linux', 'manylinux1')
+
+  return list(OrderedSet(iter_all_supported()))
+
+
+def _gen_all_abis(impl, version):
+  def tmpl_abi(impl, version, suffix):
+    return ''.join((impl, version, suffix))
+  yield tmpl_abi(impl, version, 'd')
+  yield tmpl_abi(impl, version, 'dm')
+  yield tmpl_abi(impl, version, 'dmu')
+  yield tmpl_abi(impl, version, 'm')
+  yield tmpl_abi(impl, version, 'mu')
+  yield tmpl_abi(impl, version, 'u')
+
+
+def _get_supported_for_any_abi(version=None, platform=None, impl=None, force_manylinux=False):
+  """Generates supported tags for unspecified ABI types to support more intuitive cross-platform
+     resolution."""
+  unique_tags = {
+    tag for abi in _gen_all_abis(impl, version)
+    for tag in _get_supported(version=version,
+                              platform=platform,
+                              impl=impl,
+                              abi=abi,
+                              force_manylinux=force_manylinux)
+  }
+  return list(unique_tags)
 
 
 class Platform(namedtuple('Platform', ['platform', 'impl', 'version', 'abi'])):
@@ -73,19 +141,17 @@ def supported_tags(self, interpreter=None, force_manylinux=True):
       # N.B. If we don't get an extended platform specifier, we generate
       # all possible ABI permutations to mimic earlier pex version
       # behavior and make cross-platform resolution more intuitive.
-      tags = get_supported_for_any_abi(
+      return _get_supported_for_any_abi(
         platform=self.platform,
         impl=interpreter.identity.abbr_impl,
         version=interpreter.identity.impl_ver,
         force_manylinux=force_manylinux
       )
     else:
-      tags = get_supported(
+      return _get_supported(
         platform=self.platform,
         impl=self.impl,
         version=self.version,
         abi=self.abi,
         force_manylinux=force_manylinux
       )
-
-    return tags

scripts/style.sh

@@ -3,5 +3,9 @@
 ROOT_DIR="$(git rev-parse --show-toplevel)"
 
 twitterstyle -n ImportOrder "${ROOT_DIR}/tests" $(
-  find "${ROOT_DIR}/pex" -path "${ROOT_DIR}/pex/vendor/_vendored" -prune , -name "*.py"
-)
+  find "${ROOT_DIR}/pex" -name "*.py" | \
+    grep -v \
+      -e "${ROOT_DIR}/pex/vendor/_vendored/" \
+      -e "${ROOT_DIR}/pex/glibc.py" \
+      -e "${ROOT_DIR}/pex/pep425tags.py"
+)

tests/test_environment.py

@@ -325,7 +325,7 @@ def run(args, **env):
 
 def test_activate_extras_issue_615():
   with yield_pex_builder() as pb:
-    for resolved_dist in resolver.resolve(['pex[requests]==1.5.1'], interpreter=pb.interpreter):
+    for resolved_dist in resolver.resolve(['pex[requests]==1.6.3'], interpreter=pb.interpreter):
       pb.add_requirement(resolved_dist.requirement)
       pb.add_dist_location(resolved_dist.distribution.location)
     pb.set_script('pex')
@@ -339,4 +339,4 @@ def test_activate_extras_issue_615():
     assert 0 == process.returncode, (
       'Process failed with exit code {} and output:\n{}'.format(process.returncode, stderr)
     )
-    assert to_bytes('{} 1.5.1'.format(os.path.basename(pb.path()))) == stdout.strip()
+    assert to_bytes('{} 1.6.3'.format(os.path.basename(pb.path()))) == stdout.strip()

tests/test_platform.py

@@ -58,11 +58,11 @@ def test_platform_supported_tags_manylinux():
 def test_platform_supported_tags_osx_minimal():
   impl_tag = "{}{}".format(get_abbr_impl(), get_impl_ver())
   assert_tags(
-    'macosx-10.4-x86_64',
+    'macosx-10.5-x86_64',
     [
       (impl_tag, 'none', 'any'),
       ('py%s' % sys.version_info[0], 'none', 'any'),
-      (impl_tag, get_abi_tag(), 'macosx_10_4_x86_64')
+      (impl_tag, get_abi_tag(), 'macosx_10_5_x86_64')
     ]
   )
 
@@ -71,7 +71,7 @@ def test_platform_supported_tags_osx_full():
   assert_tags(
     'macosx-10.12-x86_64-cp-27-m',
     EXPECTED_BASE + [
-      ('cp27', 'cp27m', 'macosx_10_4_x86_64'),
+      ('cp27', 'cp27m', 'macosx_10_4_intel'),
       ('cp27', 'cp27m', 'macosx_10_5_x86_64'),
       ('cp27', 'cp27m', 'macosx_10_6_x86_64'),
       ('cp27', 'cp27m', 'macosx_10_7_x86_64'),

tox.ini

@@ -135,6 +135,8 @@ commands =
         --recursive \
         --dont-skip __init__.py \
         --skip-glob {toxinidir}/pex/vendor/_vendored/** \
+        --skip {toxinidir}/pex/glibc.py \
+        --skip {toxinidir}/pex/pep425tags.py \
         {toxinidir}/pex {toxinidir}/tests
 
 [testenv:isort-check]