From 48ca8a741e3be53e5d12f1644bf4c7481d834bcc Mon Sep 17 00:00:00 2001 From: John Sirois Date: Thu, 28 Mar 2019 14:00:39 -0700 Subject: [PATCH 1/5] Pull in ~pristine copies of pep425tags/glibc from our vendored setuptools. --- pex/glibc.py | 99 +++---- pex/pep425tags.py | 547 +++++++++++++++++++-------------------- tests/test_pep425tags.py | 10 +- 3 files changed, 319 insertions(+), 337 deletions(-) diff --git a/pex/glibc.py b/pex/glibc.py index 1211cbba6..71dfa45b5 100644 --- a/pex/glibc.py +++ b/pex/glibc.py @@ -1,61 +1,64 @@ -# This file was copied from the pip project master branch on 2016/12/05 +# NB: Copied from our vendored setuptools' version at: +# pex/vendor/_vendored/setuptools/setuptools/glibc.py +# Modifications are marked with `# NB: Modified from ...` +# This file originally from pip: +# https://github.com/pypa/pip/blob/8f4f15a5a95d7d5b511ceaee9ed261176c181970/src/pip/_internal/utils/glibc.py from __future__ import absolute_import import ctypes -import platform import re - -from pex import pex_warnings +from pex import pex_warnings # NB: Modified from `import warnings` def glibc_version_string(): - "Returns glibc version string, or None if not using glibc." + "Returns glibc version string, or None if not using glibc." - # ctypes.CDLL(None) internally calls dlopen(NULL), and as the dlopen - # manpage says, "If filename is NULL, then the returned handle is for the - # main program". This way we can let the linker do the work to figure out - # which libc our process is actually using. - process_namespace = ctypes.CDLL(None) - try: - gnu_get_libc_version = process_namespace.gnu_get_libc_version - except AttributeError: - # Symbol doesn't exist -> therefore, we are not linked to - # glibc. - return None + # ctypes.CDLL(None) internally calls dlopen(NULL), and as the dlopen + # manpage says, "If filename is NULL, then the returned handle is for the + # main program". This way we can let the linker do the work to figure out + # which libc our process is actually using. + process_namespace = ctypes.CDLL(None) + try: + gnu_get_libc_version = process_namespace.gnu_get_libc_version + except AttributeError: + # Symbol doesn't exist -> therefore, we are not linked to + # glibc. + return None - # Call gnu_get_libc_version, which returns a string like "2.5" - gnu_get_libc_version.restype = ctypes.c_char_p - version_str = gnu_get_libc_version() - # py2 / py3 compatibility: - if not isinstance(version_str, str): - version_str = version_str.decode("ascii") + # Call gnu_get_libc_version, which returns a string like "2.5" + gnu_get_libc_version.restype = ctypes.c_char_p + version_str = gnu_get_libc_version() + # py2 / py3 compatibility: + if not isinstance(version_str, str): + version_str = version_str.decode("ascii") - return version_str + return version_str # Separated out from have_compatible_glibc for easier unit testing def check_glibc_version(version_str, required_major, minimum_minor): - # Parse string and check against requested version. - # - # We use a regexp instead of str.split because we want to discard any - # random junk that might come after the minor version -- this might happen - # in patched/forked versions of glibc (e.g. Linaro's version of glibc - # uses version strings like "2.20-2014.11"). See gh-3588. - m = re.match(r"(?P[0-9]+)\.(?P[0-9]+)", version_str) - if not m: - pex_warnings.warn("Expected glibc version with 2 components major.minor," - " got: %s" % version_str) - return False - return (int(m.group("major")) == required_major and - int(m.group("minor")) >= minimum_minor) + # Parse string and check against requested version. + # + # We use a regexp instead of str.split because we want to discard any + # random junk that might come after the minor version -- this might happen + # in patched/forked versions of glibc (e.g. Linaro's version of glibc + # uses version strings like "2.20-2014.11"). See gh-3588. + m = re.match(r"(?P[0-9]+)\.(?P[0-9]+)", version_str) + if not m: + # NB: Modified from `warnings.warn(..., RuntimeError)` + pex_warnings.warn("Expected glibc version with 2 components major.minor," + " got: %s" % version_str) + return False + return (int(m.group("major")) == required_major and + int(m.group("minor")) >= minimum_minor) def have_compatible_glibc(required_major, minimum_minor): - version_str = glibc_version_string() - if version_str is None: - return False - return check_glibc_version(version_str, required_major, minimum_minor) + version_str = glibc_version_string() + if version_str is None: + return False + return check_glibc_version(version_str, required_major, minimum_minor) # platform.libc_ver regularly returns completely nonsensical glibc @@ -76,9 +79,13 @@ def have_compatible_glibc(required_major, minimum_minor): # misleading. Solution: instead of using platform, use our code that actually # works. def libc_ver(): - glibc_version = glibc_version_string() - if glibc_version is None: - # For non-glibc platforms, fall back on platform.libc_ver - return platform.libc_ver() - else: - return ("glibc", glibc_version) + """Try to determine the glibc version + + Returns a tuple of strings (lib, version) which default to empty strings + in case the lookup fails. + """ + glibc_version = glibc_version_string() + if glibc_version is None: + return ("", "") + else: + return ("glibc", glibc_version) diff --git a/pex/pep425tags.py b/pex/pep425tags.py index e265fb6da..93c4a360b 100644 --- a/pex/pep425tags.py +++ b/pex/pep425tags.py @@ -1,6 +1,9 @@ -# This file was forked from the pip project master branch on 2016/12/05 -# TODO(John Sirois): Kill this file and use wheel.pep425tags from our vendored wheel instead. +# NB: Copied from our vendored setuptools' version at: +# pex/vendor/_vendored/setuptools/setuptools/pep425tags.py +# Modifications are marked with `# NB: Modified from ...` +# This file originally from pip: +# https://github.com/pypa/pip/blob/8f4f15a5a95d7d5b511ceaee9ed261176c181970/src/pip/_internal/pep425tags.py """Generate and work with PEP 425 Compatibility Tags. NB: Several functions here have their code grabbed by `pex.interpreter._generate_identity_source` @@ -12,345 +15,317 @@ from __future__ import absolute_import import distutils.util -import logging +from distutils import log import platform import re import sys import sysconfig +from collections import OrderedDict -from pex.glibc import have_compatible_glibc +from pex import glibc # NB: Modified from `from . import glibc` -logger = logging.getLogger(__name__) - -_OSX_ARCH_PAT = re.compile(r'(.+)_(\d+)_(\d+)_(.+)') +_osx_arch_pat = re.compile(r'(.+)_(\d+)_(\d+)_(.+)') # NB: Used in `pex.interpreter.ID_PY_TMPL` and should only rely on stdlib. If imports change, # please consult `pex.interpreter.ID_PY_TMPL` and adjust stdlib imports there as needed. def get_config_var(var): - try: - return sysconfig.get_config_var(var) - except IOError as e: # Issue #1074 - logger.warn(str(e)) - return None + try: + return sysconfig.get_config_var(var) + except IOError as e: # Issue #1074 + log.warn(str(e)) # NB: Modified from `warnings.warn("{}".format(e), RuntimeWarning)` + return None # NB: Used in `pex.interpreter.ID_PY_TMPL` and should only rely on stdlib. If imports change, # please consult `pex.interpreter.ID_PY_TMPL` and adjust stdlib imports there as needed. def get_abbr_impl(): - """Return abbreviated implementation name.""" - if hasattr(sys, 'pypy_version_info'): - pyimpl = 'pp' - elif sys.platform.startswith('java'): - pyimpl = 'jy' - elif sys.platform == 'cli': - pyimpl = 'ip' - else: - pyimpl = 'cp' - return pyimpl + """Return abbreviated implementation name.""" + if hasattr(sys, 'pypy_version_info'): + pyimpl = 'pp' + elif sys.platform.startswith('java'): + pyimpl = 'jy' + elif sys.platform == 'cli': + pyimpl = 'ip' + else: + pyimpl = 'cp' + return pyimpl # NB: Used in `pex.interpreter.ID_PY_TMPL` and should only rely on stdlib. If imports change, # please consult `pex.interpreter.ID_PY_TMPL` and adjust stdlib imports there as needed. def get_impl_ver(): - """Return implementation version.""" - impl_ver = get_config_var("py_version_nodot") - if not impl_ver or get_abbr_impl() == 'pp': - impl_ver = ''.join(map(str, get_impl_version_info())) - return impl_ver + """Return implementation version.""" + impl_ver = get_config_var("py_version_nodot") + if not impl_ver or get_abbr_impl() == 'pp': + impl_ver = ''.join(map(str, get_impl_version_info())) + return impl_ver # NB: Used in `pex.interpreter.ID_PY_TMPL` and should only rely on stdlib. If imports change, # please consult `pex.interpreter.ID_PY_TMPL` and adjust stdlib imports there as needed. def get_impl_version_info(): - """Return sys.version_info-like tuple for use in decrementing the minor - version.""" - if get_abbr_impl() == 'pp': - # as per https://github.com/pypa/pip/issues/2882 - return (sys.version_info[0], sys.pypy_version_info.major, - sys.pypy_version_info.minor) - else: - return sys.version_info[0], sys.version_info[1] + """Return sys.version_info-like tuple for use in decrementing the minor + version.""" + if get_abbr_impl() == 'pp': + # as per https://github.com/pypa/pip/issues/2882 + return (sys.version_info[0], sys.pypy_version_info.major, + sys.pypy_version_info.minor) + else: + return sys.version_info[0], sys.version_info[1] # NB: Used in `pex.interpreter.ID_PY_TMPL` and should only rely on stdlib. If imports change, # please consult `pex.interpreter.ID_PY_TMPL` and adjust stdlib imports there as needed. def get_flag(var, fallback, expected=True, warn=True): - """Use a fallback method for determining SOABI flags if the needed config - var is unset or unavailable.""" - val = get_config_var(var) - if val is None: - if warn: - logger.debug("Config variable '%s' is unset, Python ABI tag may " - "be incorrect", var) - return fallback() - return val == expected + """Use a fallback method for determining SOABI flags if the needed config + var is unset or unavailable.""" + val = get_config_var(var) + if val is None: + if warn: + log.debug("Config variable '%s' is unset, Python ABI tag may " + "be incorrect", var) + return fallback() + return val == expected # NB: Used in `pex.interpreter.ID_PY_TMPL` and should only rely on stdlib. If imports change, # please consult `pex.interpreter.ID_PY_TMPL` and adjust stdlib imports there as needed. def get_abi_tag(): - """Return the ABI tag based on SOABI (if available) or emulate SOABI - (CPython 2, PyPy).""" - soabi = get_config_var('SOABI') - impl = get_abbr_impl() - if not soabi and impl in ('cp', 'pp') and hasattr(sys, 'maxunicode'): - d = '' - m = '' - u = '' - if get_flag('Py_DEBUG', - lambda: hasattr(sys, 'gettotalrefcount'), - warn=(impl == 'cp')): - d = 'd' - if get_flag('WITH_PYMALLOC', - lambda: impl == 'cp', - warn=(impl == 'cp')): - m = 'm' - if (get_flag('Py_UNICODE_SIZE', - lambda: sys.maxunicode == 0x10ffff, - expected=4, - warn=(impl == 'cp' and sys.version_info < (3, 3))) and - sys.version_info < (3, 3)): - u = 'u' - abi = '%s%s%s%s%s' % (impl, get_impl_ver(), d, m, u) - elif soabi and soabi.startswith('cpython-'): - abi = 'cp' + soabi.split('-')[1] - elif soabi: - abi = soabi.replace('.', '_').replace('-', '_') - else: - abi = None - return abi + """Return the ABI tag based on SOABI (if available) or emulate SOABI + (CPython 2, PyPy).""" + soabi = get_config_var('SOABI') + impl = get_abbr_impl() + if not soabi and impl in {'cp', 'pp'} and hasattr(sys, 'maxunicode'): + d = '' + m = '' + u = '' + if get_flag('Py_DEBUG', + lambda: hasattr(sys, 'gettotalrefcount'), + warn=(impl == 'cp')): + d = 'd' + if get_flag('WITH_PYMALLOC', + lambda: impl == 'cp', + warn=(impl == 'cp')): + m = 'm' + + # NB: Modified from ~ `from .extern import six; six.PY2` + PY2 = sys.version_info[0] == 2 + if (get_flag('Py_UNICODE_SIZE', + lambda: sys.maxunicode == 0x10ffff, + expected=4, + warn=(impl == 'cp' and PY2)) and PY2): + u = 'u' + + abi = '%s%s%s%s%s' % (impl, get_impl_ver(), d, m, u) + elif soabi and soabi.startswith('cpython-'): + abi = 'cp' + soabi.split('-')[1] + elif soabi: + abi = soabi.replace('.', '_').replace('-', '_') + else: + abi = None + return abi def _is_running_32bit(): - return sys.maxsize == 2147483647 + return sys.maxsize == 2147483647 def get_platform(): - """Return our platform name 'win32', 'linux_x86_64'""" - if sys.platform == 'darwin': - # distutils.util.get_platform() returns the release based on the value - # of MACOSX_DEPLOYMENT_TARGET on which Python was built, which may - # be significantly older than the user's current machine. - release, _, machine = platform.mac_ver() - split_ver = release.split('.') + """Return our platform name 'win32', 'linux_x86_64'""" + if sys.platform == 'darwin': + # distutils.util.get_platform() returns the release based on the value + # of MACOSX_DEPLOYMENT_TARGET on which Python was built, which may + # be significantly older than the user's current machine. + release, _, machine = platform.mac_ver() + split_ver = release.split('.') - if machine == 'x86_64' and _is_running_32bit(): - machine = 'i386' - elif machine == 'ppc64' and _is_running_32bit(): - machine = 'ppc' + if machine == "x86_64" and _is_running_32bit(): + machine = "i386" + elif machine == "ppc64" and _is_running_32bit(): + machine = "ppc" - return 'macosx_{0}_{1}_{2}'.format(split_ver[0], split_ver[1], machine) + return 'macosx_{}_{}_{}'.format(split_ver[0], split_ver[1], machine) - # XXX remove distutils dependency - result = distutils.util.get_platform().replace('.', '_').replace('-', '_') - if result == 'linux_x86_64' and _is_running_32bit(): - # 32 bit Python program (running on a 64 bit Linux): pip should only - # install and run 32 bit compiled extensions in that case. - result = 'linux_i686' + # XXX remove distutils dependency + result = distutils.util.get_platform().replace('.', '_').replace('-', '_') + if result == "linux_x86_64" and _is_running_32bit(): + # 32 bit Python program (running on a 64 bit Linux): pip should only + # install and run 32 bit compiled extensions in that case. + result = "linux_i686" - return result + return result def is_manylinux1_compatible(): - # Only Linux, and only x86-64 / i686 - if get_platform() not in ('linux_x86_64', 'linux_i686'): - return False + # Only Linux, and only x86-64 / i686 + if get_platform() not in {"linux_x86_64", "linux_i686"}: + return False - # Check for presence of _manylinux module - try: - import _manylinux - return bool(_manylinux.manylinux1_compatible) - except (ImportError, AttributeError): - # Fall through to heuristic check below - pass + # Check for presence of _manylinux module + try: + import _manylinux + return bool(_manylinux.manylinux1_compatible) + except (ImportError, AttributeError): + # Fall through to heuristic check below + pass - # Check glibc version. CentOS 5 uses glibc 2.5. - return have_compatible_glibc(2, 5) + # Check glibc version. CentOS 5 uses glibc 2.5. + return glibc.have_compatible_glibc(2, 5) def get_darwin_arches(major, minor, machine): - """Return a list of supported arches (including group arches) for - the given major, minor and machine architecture of an macOS machine. - """ - arches = [] - - def _supports_arch(major, minor, arch): - # Looking at the application support for macOS versions in the chart - # provided by https://en.wikipedia.org/wiki/OS_X#Versions it appears - # our timeline looks roughly like: - # - # 10.0 - Introduces ppc support. - # 10.4 - Introduces ppc64, i386, and x86_64 support, however the ppc64 - # and x86_64 support is CLI only, and cannot be used for GUI - # applications. - # 10.5 - Extends ppc64 and x86_64 support to cover GUI applications. - # 10.6 - Drops support for ppc64 - # 10.7 - Drops support for ppc - # - # Note: The above information is taken from the "Application support" - # column in the chart not the "Processor support" since I believe - # that we care about what instruction sets an application can use - # not which processors the OS supports. - if arch == 'ppc': - return (major, minor) <= (10, 5) - if arch == 'ppc64': - return (major, minor) == (10, 5) - if arch == 'i386': - return (major, minor) >= (10, 4) - if arch == 'x86_64': - return (major, minor) >= (10, 4) - if arch in groups: - for garch in groups_dict[arch]: - if _supports_arch(major, minor, garch): - return True - return False - - groups = ('fat', 'intel', 'fat64', 'fat32') - groups_dict = {'fat': ('i386', 'ppc'), - 'intel': ('x86_64', 'i386'), - 'fat64': ('x86_64', 'ppc64'), - 'fat32': ('x86_64', 'i386', 'ppc')} - - if _supports_arch(major, minor, machine): - arches.append(machine) - - for garch in groups: - if machine in groups_dict[garch] and _supports_arch(major, minor, garch): - arches.append(garch) - - arches.append('universal') - - return arches - - -def _gen_all_abis(impl, version): - def tmpl_abi(impl, version, suffix): - return ''.join((impl, version, suffix)) - yield tmpl_abi(impl, version, 'd') - yield tmpl_abi(impl, version, 'dm') - yield tmpl_abi(impl, version, 'dmu') - yield tmpl_abi(impl, version, 'm') - yield tmpl_abi(impl, version, 'mu') - yield tmpl_abi(impl, version, 'u') - - -def get_supported_for_any_abi(version=None, noarch=False, platform=None, impl=None, - force_manylinux=False): - """Generates supported tags for unspecified ABI types to support more intuitive cross-platform - resolution.""" - unique_tags = { - tag for abi in _gen_all_abis(impl, version) - for tag in get_supported(version=version, - noarch=noarch, - platform=platform, - impl=impl, - abi=abi, - force_manylinux=force_manylinux) - } - return list(unique_tags) - - -def get_supported(version=None, noarch=False, platform=None, impl=None, abi=None, - force_manylinux=False): - """Return a list of supported tags for each version specified in - `version`. - - :param version: string version (e.g., "33", "32") or None. - If None, use local system Python version. - :param platform: specify the exact platform you want valid - tags for, or None. If None, use the local system platform. - :param impl: specify the exact implementation you want valid - tags for, or None. If None, use the local interpreter impl. - :param abi: specify the exact abi you want valid - tags for, or None. If None, use the local interpreter abi. - :param force_manylinux: Whether or not to force manylinux support. This is useful - when resolving for different target platform than current. - """ - supported = [] - - # Versions must be given with respect to the preference - if version is None: - versions = [] - version_info = get_impl_version_info() - major = version_info[:-1] - # Support all previous minor Python versions. - for minor in range(version_info[-1], -1, -1): - versions.append(''.join(map(str, major + (minor,)))) - else: - versions = [version] - - impl = impl or get_abbr_impl() - - abis = [] - - abi = abi or get_abi_tag() - if abi: - abis[0:0] = [abi] - - abi3s = set() - import imp - for suffix in imp.get_suffixes(): - if suffix[0].startswith('.abi'): - abi3s.add(suffix[0].split('.', 2)[1]) - - abis.extend(sorted(list(abi3s))) - - abis.append('none') - - if not noarch: - arch = platform or get_platform() - if arch.startswith('macosx'): - # support macosx-10.6-intel on macosx-10.9-x86_64 - match = _OSX_ARCH_PAT.match(arch) - if match: - name, major, minor, actual_arch = match.groups() - tpl = '{0}_{1}_%i_%s'.format(name, major) - arches = [] - for m in reversed(range(int(minor) + 1)): - for a in get_darwin_arches(int(major), m, actual_arch): - arches.append(tpl % (m, a)) - else: - # arch pattern didn't match (?!) - arches = [arch] - elif ( - (platform is None and is_manylinux1_compatible()) or - # N.B. Here we work around the fact that `is_manylinux1_compatible()` expects - # to be running on the target platform being built for with a feature flag approach. - (arch.startswith('linux') and force_manylinux) - ): - arches = [arch.replace('linux', 'manylinux1'), arch] - else: - arches = [arch] - - # Current version, current API (built specifically for our Python): - for abi in abis: - for arch in arches: - supported.append(('%s%s' % (impl, versions[0]), abi, arch)) - - # abi3 modules compatible with older version of Python - for version in versions[1:]: - # abi3 was introduced in Python 3.2 - if version in ('31', '30'): - break - for abi in abi3s: # empty set if not Python 3 + """Return a list of supported arches (including group arches) for + the given major, minor and machine architecture of an macOS machine. + """ + arches = [] + + def _supports_arch(major, minor, arch): + # Looking at the application support for macOS versions in the chart + # provided by https://en.wikipedia.org/wiki/OS_X#Versions it appears + # our timeline looks roughly like: + # + # 10.0 - Introduces ppc support. + # 10.4 - Introduces ppc64, i386, and x86_64 support, however the ppc64 + # and x86_64 support is CLI only, and cannot be used for GUI + # applications. + # 10.5 - Extends ppc64 and x86_64 support to cover GUI applications. + # 10.6 - Drops support for ppc64 + # 10.7 - Drops support for ppc + # + # Given that we do not know if we're installing a CLI or a GUI + # application, we must be conservative and assume it might be a GUI + # application and behave as if ppc64 and x86_64 support did not occur + # until 10.5. + # + # Note: The above information is taken from the "Application support" + # column in the chart not the "Processor support" since I believe + # that we care about what instruction sets an application can use + # not which processors the OS supports. + if arch == 'ppc': + return (major, minor) <= (10, 5) + if arch == 'ppc64': + return (major, minor) == (10, 5) + if arch == 'i386': + return (major, minor) >= (10, 4) + if arch == 'x86_64': + return (major, minor) >= (10, 5) + if arch in groups: + for garch in groups[arch]: + if _supports_arch(major, minor, garch): + return True + return False + + groups = OrderedDict([ + ("fat", ("i386", "ppc")), + ("intel", ("x86_64", "i386")), + ("fat64", ("x86_64", "ppc64")), + ("fat32", ("x86_64", "i386", "ppc")), + ]) + + if _supports_arch(major, minor, machine): + arches.append(machine) + + for garch in groups: + if machine in groups[garch] and _supports_arch(major, minor, garch): + arches.append(garch) + + arches.append('universal') + + return arches + + +def get_supported(versions=None, noarch=False, platform=None, + impl=None, abi=None): + """Return a list of supported tags for each version specified in + `versions`. + + :param versions: a list of string versions, of the form ["33", "32"], + or None. The first version will be assumed to support our ABI. + :param platform: specify the exact platform you want valid + tags for, or None. If None, use the local system platform. + :param impl: specify the exact implementation you want valid + tags for, or None. If None, use the local interpreter impl. + :param abi: specify the exact abi you want valid + tags for, or None. If None, use the local interpreter abi. + """ + supported = [] + + # Versions must be given with respect to the preference + if versions is None: + versions = [] + version_info = get_impl_version_info() + major = version_info[:-1] + # Support all previous minor Python versions. + for minor in range(version_info[-1], -1, -1): + versions.append(''.join(map(str, major + (minor,)))) + + impl = impl or get_abbr_impl() + + abis = [] + + abi = abi or get_abi_tag() + if abi: + abis[0:0] = [abi] + + abi3s = set() + import imp + for suffix in imp.get_suffixes(): + if suffix[0].startswith('.abi'): + abi3s.add(suffix[0].split('.', 2)[1]) + + abis.extend(sorted(list(abi3s))) + + abis.append('none') + + if not noarch: + arch = platform or get_platform() + if arch.startswith('macosx'): + # support macosx-10.6-intel on macosx-10.9-x86_64 + match = _osx_arch_pat.match(arch) + if match: + name, major, minor, actual_arch = match.groups() + tpl = '{}_{}_%i_%s'.format(name, major) + arches = [] + for m in reversed(range(int(minor) + 1)): + for a in get_darwin_arches(int(major), m, actual_arch): + arches.append(tpl % (m, a)) + else: + # arch pattern didn't match (?!) + arches = [arch] + elif platform is None and is_manylinux1_compatible(): + arches = [arch.replace('linux', 'manylinux1'), arch] + else: + arches = [arch] + + # Current version, current API (built specifically for our Python): + for abi in abis: + for arch in arches: + supported.append(('%s%s' % (impl, versions[0]), abi, arch)) + + # abi3 modules compatible with older version of Python + for version in versions[1:]: + # abi3 was introduced in Python 3.2 + if version in {'31', '30'}: + break + for abi in abi3s: # empty set if not Python 3 + for arch in arches: + supported.append(("%s%s" % (impl, version), abi, arch)) + + # Has binaries, does not use the Python API: for arch in arches: - supported.append(('%s%s' % (impl, version), abi, arch)) - - # Has binaries, does not use the Python API: - for arch in arches: - supported.append(('py%s' % (versions[0][0]), 'none', arch)) + supported.append(('py%s' % (versions[0][0]), 'none', arch)) - # No abi / arch, but requires our implementation: - supported.append(('%s%s' % (impl, versions[0]), 'none', 'any')) - # Tagged specifically as being cross-version compatible - # (with just the major version specified) - supported.append(('%s%s' % (impl, versions[0][0]), 'none', 'any')) + # No abi / arch, but requires our implementation: + supported.append(('%s%s' % (impl, versions[0]), 'none', 'any')) + # Tagged specifically as being cross-version compatible + # (with just the major version specified) + supported.append(('%s%s' % (impl, versions[0][0]), 'none', 'any')) - # No abi / arch, generic Python - for i, version in enumerate(versions): - supported.append(('py%s' % (version,), 'none', 'any')) - if i == 0: - supported.append(('py%s' % (version[0]), 'none', 'any')) + # No abi / arch, generic Python + for i, version in enumerate(versions): + supported.append(('py%s' % (version,), 'none', 'any')) + if i == 0: + supported.append(('py%s' % (version[0]), 'none', 'any')) - return supported + return supported diff --git a/tests/test_pep425tags.py b/tests/test_pep425tags.py index fc7b89e24..2c5886156 100644 --- a/tests/test_pep425tags.py +++ b/tests/test_pep425tags.py @@ -116,7 +116,7 @@ def test_manual_abi_dm_flags(self): class TestManylinux1Tags(object): @patch('pex.pep425tags.get_platform', lambda: 'linux_x86_64') - @patch('pex.pep425tags.have_compatible_glibc', lambda major, minor: True) + @patch('pex.glibc.have_compatible_glibc', lambda major, minor: True) def test_manylinux1_compatible_on_linux_x86_64(self): """ Test that manylinux1 is enabled on linux_x86_64 @@ -124,7 +124,7 @@ def test_manylinux1_compatible_on_linux_x86_64(self): assert pep425tags.is_manylinux1_compatible() @patch('pex.pep425tags.get_platform', lambda: 'linux_i686') - @patch('pex.pep425tags.have_compatible_glibc', lambda major, minor: True) + @patch('pex.glibc.have_compatible_glibc', lambda major, minor: True) def test_manylinux1_compatible_on_linux_i686(self): """ Test that manylinux1 is enabled on linux_i686 @@ -132,7 +132,7 @@ def test_manylinux1_compatible_on_linux_i686(self): assert pep425tags.is_manylinux1_compatible() @patch('pex.pep425tags.get_platform', lambda: 'linux_x86_64') - @patch('pex.pep425tags.have_compatible_glibc', lambda major, minor: False) + @patch('pex.glibc.have_compatible_glibc', lambda major, minor: False) def test_manylinux1_2(self): """ Test that manylinux1 is disabled with incompatible glibc @@ -140,7 +140,7 @@ def test_manylinux1_2(self): assert not pep425tags.is_manylinux1_compatible() @patch('pex.pep425tags.get_platform', lambda: 'arm6vl') - @patch('pex.pep425tags.have_compatible_glibc', lambda major, minor: True) + @patch('pex.glibc.have_compatible_glibc', lambda major, minor: True) def test_manylinux1_3(self): """ Test that manylinux1 is disabled on arm6vl @@ -148,7 +148,7 @@ def test_manylinux1_3(self): assert not pep425tags.is_manylinux1_compatible() @patch('pex.pep425tags.get_platform', lambda: 'linux_x86_64') - @patch('pex.pep425tags.have_compatible_glibc', lambda major, minor: True) + @patch('pex.glibc.have_compatible_glibc', lambda major, minor: True) @patch('sys.platform', 'linux2') def test_manylinux1_tag_is_first(self): """ From 7858f93d3be2d358a8b033b16078a3b7da5e477f Mon Sep 17 00:00:00 2001 From: John Sirois Date: Thu, 28 Mar 2019 14:07:11 -0700 Subject: [PATCH 2/5] Use all compatible versions when calculating tags. Previously, we'd pass a single version to `get_supported` when we knew one which would lead to a single tag being generated for the abi3 case instead of one per previous minor version. Fixes #539 --- pex/platforms.py | 90 +++++++++++++++++++++++++++++++++------ scripts/style.sh | 8 +++- tests/test_environment.py | 4 +- tests/test_platform.py | 6 +-- tox.ini | 2 + 5 files changed, 91 insertions(+), 19 deletions(-) diff --git a/pex/platforms.py b/pex/platforms.py index 49ab56dc3..db3f22783 100644 --- a/pex/platforms.py +++ b/pex/platforms.py @@ -5,14 +5,82 @@ from collections import namedtuple -from pex.pep425tags import ( - get_abbr_impl, - get_abi_tag, - get_impl_ver, - get_platform, - get_supported, - get_supported_for_any_abi -) +from pex.orderedset import OrderedSet +from pex.pep425tags import get_abbr_impl, get_abi_tag, get_impl_ver, get_platform, get_supported + + +def _gen_all_compatible_versions(version): + # We select major and minor here in the context of implementation version strings. + # These are typically two digit characters; eg "27" or "36", but they can be three digit + # characters in the case of pypy; eg "271". In the typical case the 1st digit represents the + # python major version and the 2nd digit it's minor version. In the pypy case, the 1st digit still + # represents the (hosting) python major version, the 2nd the pypy major version and the 3rd the + # pypy minor version. In both cases the last digit is the minor version and the python in question + # guarantees backwards compatibility of minor version bumps within the major version as per + # semver. + # + # Concrete examples of what we want to return in each case: + # 1. typical case of cpython "36": ["36", "35", "34", "33", "32", "31", "30"] + # 2. pypy case of "271": ["271", "270"]. + # + # For more information on the pypy case see conversation here: + # https://github.com/pypa/pip/issues/2882 + # In particular https://github.com/pypa/pip/issues/2882#issuecomment-110925458 and + # https://github.com/pypa/pip/issues/2882#issuecomment-130404840. + # The fix work for pip handling of this is done here: https://github.com/pypa/pip/pull/3075 + + major, minor = version[:-1], version[-1] + + def iter_compatible_versions(): + # Support all previous minor Python versions. + for compatible_minor in range(int(minor), -1, -1): + yield '{major}{minor}'.format(major=major, minor=compatible_minor) + + return list(iter_compatible_versions()) + + +def _get_supported(version=None, platform=None, impl=None, abi=None, force_manylinux=False): + versions = _gen_all_compatible_versions(version) if version is not None else None + all_supported = get_supported( + versions=versions, + platform=platform, + impl=impl, + abi=abi + ) + + def iter_all_supported(): + for supported in all_supported: + yield supported + python_tag, abi_tag, platform_tag = supported + if platform_tag.startswith('linux') and force_manylinux: + yield python_tag, abi_tag, platform_tag.replace('linux', 'manylinux1') + + return list(OrderedSet(iter_all_supported())) + + +def _gen_all_abis(impl, version): + def tmpl_abi(impl, version, suffix): + return ''.join((impl, version, suffix)) + yield tmpl_abi(impl, version, 'd') + yield tmpl_abi(impl, version, 'dm') + yield tmpl_abi(impl, version, 'dmu') + yield tmpl_abi(impl, version, 'm') + yield tmpl_abi(impl, version, 'mu') + yield tmpl_abi(impl, version, 'u') + + +def _get_supported_for_any_abi(version=None, platform=None, impl=None, force_manylinux=False): + """Generates supported tags for unspecified ABI types to support more intuitive cross-platform + resolution.""" + unique_tags = { + tag for abi in _gen_all_abis(impl, version) + for tag in _get_supported(version=version, + platform=platform, + impl=impl, + abi=abi, + force_manylinux=force_manylinux) + } + return list(unique_tags) class Platform(namedtuple('Platform', ['platform', 'impl', 'version', 'abi'])): @@ -73,19 +141,17 @@ def supported_tags(self, interpreter=None, force_manylinux=True): # N.B. If we don't get an extended platform specifier, we generate # all possible ABI permutations to mimic earlier pex version # behavior and make cross-platform resolution more intuitive. - tags = get_supported_for_any_abi( + return _get_supported_for_any_abi( platform=self.platform, impl=interpreter.identity.abbr_impl, version=interpreter.identity.impl_ver, force_manylinux=force_manylinux ) else: - tags = get_supported( + return _get_supported( platform=self.platform, impl=self.impl, version=self.version, abi=self.abi, force_manylinux=force_manylinux ) - - return tags diff --git a/scripts/style.sh b/scripts/style.sh index eda83d36e..04e26267a 100755 --- a/scripts/style.sh +++ b/scripts/style.sh @@ -3,5 +3,9 @@ ROOT_DIR="$(git rev-parse --show-toplevel)" twitterstyle -n ImportOrder "${ROOT_DIR}/tests" $( - find "${ROOT_DIR}/pex" -path "${ROOT_DIR}/pex/vendor/_vendored" -prune , -name "*.py" -) \ No newline at end of file + find "${ROOT_DIR}/pex" -name "*.py" | \ + grep -v \ + -e "${ROOT_DIR}/pex/vendor/_vendored/" \ + -e "${ROOT_DIR}/pex/glibc.py" \ + -e "${ROOT_DIR}/pex/pep425tags.py" +) diff --git a/tests/test_environment.py b/tests/test_environment.py index 3617617fc..af6dee85e 100644 --- a/tests/test_environment.py +++ b/tests/test_environment.py @@ -325,7 +325,7 @@ def run(args, **env): def test_activate_extras_issue_615(): with yield_pex_builder() as pb: - for resolved_dist in resolver.resolve(['pex[requests]==1.5.1'], interpreter=pb.interpreter): + for resolved_dist in resolver.resolve(['pex[requests]==1.6.3'], interpreter=pb.interpreter): pb.add_requirement(resolved_dist.requirement) pb.add_dist_location(resolved_dist.distribution.location) pb.set_script('pex') @@ -339,4 +339,4 @@ def test_activate_extras_issue_615(): assert 0 == process.returncode, ( 'Process failed with exit code {} and output:\n{}'.format(process.returncode, stderr) ) - assert to_bytes('{} 1.5.1'.format(os.path.basename(pb.path()))) == stdout.strip() + assert to_bytes('{} 1.6.3'.format(os.path.basename(pb.path()))) == stdout.strip() diff --git a/tests/test_platform.py b/tests/test_platform.py index 50c8b81fe..4971be6a2 100644 --- a/tests/test_platform.py +++ b/tests/test_platform.py @@ -58,11 +58,11 @@ def test_platform_supported_tags_manylinux(): def test_platform_supported_tags_osx_minimal(): impl_tag = "{}{}".format(get_abbr_impl(), get_impl_ver()) assert_tags( - 'macosx-10.4-x86_64', + 'macosx-10.5-x86_64', [ (impl_tag, 'none', 'any'), ('py%s' % sys.version_info[0], 'none', 'any'), - (impl_tag, get_abi_tag(), 'macosx_10_4_x86_64') + (impl_tag, get_abi_tag(), 'macosx_10_5_x86_64') ] ) @@ -71,7 +71,7 @@ def test_platform_supported_tags_osx_full(): assert_tags( 'macosx-10.12-x86_64-cp-27-m', EXPECTED_BASE + [ - ('cp27', 'cp27m', 'macosx_10_4_x86_64'), + ('cp27', 'cp27m', 'macosx_10_4_intel'), ('cp27', 'cp27m', 'macosx_10_5_x86_64'), ('cp27', 'cp27m', 'macosx_10_6_x86_64'), ('cp27', 'cp27m', 'macosx_10_7_x86_64'), diff --git a/tox.ini b/tox.ini index 7eb909956..1e337992f 100644 --- a/tox.ini +++ b/tox.ini @@ -135,6 +135,8 @@ commands = --recursive \ --dont-skip __init__.py \ --skip-glob {toxinidir}/pex/vendor/_vendored/** \ + --skip {toxinidir}/pex/glibc.py \ + --skip {toxinidir}/pex/pep425tags.py \ {toxinidir}/pex {toxinidir}/tests [testenv:isort-check] From 187580bba7ce1cd1a8513482e0798e3bdd9445aa Mon Sep 17 00:00:00 2001 From: John Sirois Date: Fri, 29 Mar 2019 10:47:04 -0700 Subject: [PATCH 3/5] Add tests. --- tests/test_integration.py | 29 ++++++++++++++++ tests/test_platform.py | 72 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 101 insertions(+) diff --git a/tests/test_integration.py b/tests/test_integration.py index 4c8bd5322..d40de0daa 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -14,6 +14,7 @@ from pex.compatibility import WINDOWS, nested, to_bytes from pex.installer import EggInstaller from pex.pex_info import PexInfo +from pex.resolver import resolve from pex.testing import ( IS_PYPY, NOT_CPYTHON27, @@ -1309,3 +1310,31 @@ def test_no_emit_warnings_emit_env_override(): def test_no_emit_warnings_verbose_override(): stderr = build_and_execute_pex_with_warnings('--no-emit-warnings', PEX_VERBOSE='1') assert stderr + + +@pytest.mark.skipif(IS_PYPY) +def test_issues_539_abi3_resolution(): + # The cryptography team releases the following relevant pre-built wheels for version 2.6.1: + # cryptography-2.6.1-cp27-cp27m-macosx_10_6_intel.whl + # cryptography-2.6.1-cp27-cp27m-manylinux1_x86_64.whl + # cryptography-2.6.1-cp27-cp27mu-manylinux1_x86_64.whl + # cryptography-2.6.1-cp34-abi3-macosx_10_6_intel.whl + # cryptography-2.6.1-cp34-abi3-manylinux1_x86_64.whl + # With pex in --no-build mode, we force a test that pex abi3 resolution works when this test is + # run under CPython>3.4,<4 on OSX and linux. + + with temporary_dir() as td: + # The dependency graph for cryptography-2.6.1 includes pycparser which is only released as an + # sdist. Since we want to test in --no-build, we pre-resolve/build the pycparser wheel here and + # add the resulting wheelhouse to the --no-build pex command. + resolve_cache = os.path.join(td, '.resolve_cache') + resolve(['pycparser'], cache=resolve_cache) + + cryptography_pex = os.path.join(td, 'cryptography.pex') + res = run_pex_command(['-f', resolve_cache, + '--no-build', + 'cryptography==2.6.1', + '-o', cryptography_pex]) + res.assert_success() + + subprocess.check_call([cryptography_pex, '-c', 'import cryptography']) diff --git a/tests/test_platform.py b/tests/test_platform.py index 4971be6a2..3de1289b2 100644 --- a/tests/test_platform.py +++ b/tests/test_platform.py @@ -6,6 +6,12 @@ from pex.pep425tags import get_abbr_impl, get_abi_tag, get_impl_ver from pex.platforms import Platform +try: + from mock import patch +except ImportError: + from unittest.mock import patch + + EXPECTED_BASE = [('py27', 'none', 'any'), ('py2', 'none', 'any')] @@ -91,3 +97,69 @@ def test_pypy_abi_prefix(): ('pp260', 'pypy_41', 'linux_x86_64'), ] ) + + +# NB: Having to patch here is a symptom of https://github.com/pantsbuild/pex/issues/694 +# Kill when the Platform API is fixed to not need to consult the local interpreter. +@patch('imp.get_suffixes', lambda: [('.abi3.so', 'rb', 3)]) +def test_platform_supported_tags_abi3(): + tags = Platform.create('linux-x86_64-cp-37-m').supported_tags() + expected_tags = [ + ('cp37', 'cp37m', 'linux_x86_64'), + ('cp37', 'cp37m', 'manylinux1_x86_64'), + ('cp37', 'abi3', 'linux_x86_64'), + ('cp37', 'abi3', 'manylinux1_x86_64'), + ('cp37', 'none', 'linux_x86_64'), + ('cp37', 'none', 'manylinux1_x86_64'), + ('cp36', 'abi3', 'linux_x86_64'), + ('cp36', 'abi3', 'manylinux1_x86_64'), + ('cp35', 'abi3', 'linux_x86_64'), + ('cp35', 'abi3', 'manylinux1_x86_64'), + ('cp34', 'abi3', 'linux_x86_64'), + ('cp34', 'abi3', 'manylinux1_x86_64'), + ('cp33', 'abi3', 'linux_x86_64'), + ('cp33', 'abi3', 'manylinux1_x86_64'), + ('cp32', 'abi3', 'linux_x86_64'), + ('cp32', 'abi3', 'manylinux1_x86_64'), + ('py3', 'none', 'linux_x86_64'), + ('py3', 'none', 'manylinux1_x86_64'), + ('cp37', 'none', 'any'), + ('cp3', 'none', 'any'), + ('py37', 'none', 'any'), + ('py3', 'none', 'any'), + ('py36', 'none', 'any'), + ('py35', 'none', 'any'), + ('py34', 'none', 'any'), + ('py33', 'none', 'any'), + ('py32', 'none', 'any'), + ('py31', 'none', 'any'), + ('py30', 'none', 'any'), + ] + assert expected_tags == tags + + +# NB: Having to patch here is a symptom of https://github.com/pantsbuild/pex/issues/694 +# Kill when the Platform API is fixed to not need to consult the local interpreter. +@patch('imp.get_suffixes', lambda: []) +def test_platform_supported_tags_no_abi3(): + tags = Platform.create('linux-x86_64-cp-37-m').supported_tags() + expected_tags = [ + ('cp37', 'cp37m', 'linux_x86_64'), + ('cp37', 'cp37m', 'manylinux1_x86_64'), + ('cp37', 'none', 'linux_x86_64'), + ('cp37', 'none', 'manylinux1_x86_64'), + ('py3', 'none', 'linux_x86_64'), + ('py3', 'none', 'manylinux1_x86_64'), + ('cp37', 'none', 'any'), + ('cp3', 'none', 'any'), + ('py37', 'none', 'any'), + ('py3', 'none', 'any'), + ('py36', 'none', 'any'), + ('py35', 'none', 'any'), + ('py34', 'none', 'any'), + ('py33', 'none', 'any'), + ('py32', 'none', 'any'), + ('py31', 'none', 'any'), + ('py30', 'none', 'any'), + ] + assert expected_tags == tags From 6dc4a87e432e0c66294543027cfaa6f34db84743 Mon Sep 17 00:00:00 2001 From: Eric Arellano Date: Fri, 29 Mar 2019 16:13:10 -0700 Subject: [PATCH 4/5] Update pex/platforms.py Co-Authored-By: jsirois --- pex/platforms.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pex/platforms.py b/pex/platforms.py index db3f22783..81c02842a 100644 --- a/pex/platforms.py +++ b/pex/platforms.py @@ -17,7 +17,7 @@ def _gen_all_compatible_versions(version): # represents the (hosting) python major version, the 2nd the pypy major version and the 3rd the # pypy minor version. In both cases the last digit is the minor version and the python in question # guarantees backwards compatibility of minor version bumps within the major version as per - # semver. + # semver and PEP 425 (https://www.python.org/dev/peps/pep-0425/#id1). # # Concrete examples of what we want to return in each case: # 1. typical case of cpython "36": ["36", "35", "34", "33", "32", "31", "30"] From 59048b7c12b9b8a2db15ae3675952b5682d02189 Mon Sep 17 00:00:00 2001 From: John Sirois Date: Fri, 29 Mar 2019 17:21:36 -0700 Subject: [PATCH 5/5] Clarify glibc/pep425tags copies a bit more. --- pex/glibc.py | 2 ++ pex/pep425tags.py | 2 ++ 2 files changed, 4 insertions(+) diff --git a/pex/glibc.py b/pex/glibc.py index 71dfa45b5..5900b65df 100644 --- a/pex/glibc.py +++ b/pex/glibc.py @@ -1,6 +1,8 @@ # NB: Copied from our vendored setuptools' version at: # pex/vendor/_vendored/setuptools/setuptools/glibc.py # Modifications are marked with `# NB: Modified from ...` +# TODO(John Sirois): Remove this file as part of https://github.com/pantsbuild/pex/issues/696 +# ------------------------------------------------------------------------------------------- # This file originally from pip: # https://github.com/pypa/pip/blob/8f4f15a5a95d7d5b511ceaee9ed261176c181970/src/pip/_internal/utils/glibc.py diff --git a/pex/pep425tags.py b/pex/pep425tags.py index 93c4a360b..1733f0034 100644 --- a/pex/pep425tags.py +++ b/pex/pep425tags.py @@ -1,6 +1,8 @@ # NB: Copied from our vendored setuptools' version at: # pex/vendor/_vendored/setuptools/setuptools/pep425tags.py # Modifications are marked with `# NB: Modified from ...` +# TODO(John Sirois): Remove this file as part of https://github.com/pantsbuild/pex/issues/696 +# ------------------------------------------------------------------------------------------- # This file originally from pip: # https://github.com/pypa/pip/blob/8f4f15a5a95d7d5b511ceaee9ed261176c181970/src/pip/_internal/pep425tags.py