diff --git a/distlib/__init__.py b/distlib/__init__.py index 31b741b..8cabdc6 100644 --- a/distlib/__init__.py +++ b/distlib/__init__.py @@ -8,16 +8,26 @@ __version__ = '0.3.8.dev0' + class DistlibException(Exception): pass + try: from logging import NullHandler -except ImportError: # pragma: no cover +except ImportError: # pragma: no cover + class NullHandler(logging.Handler): - def handle(self, record): pass - def emit(self, record): pass - def createLock(self): self.lock = None + + def handle(self, record): + pass + + def emit(self, record): + pass + + def createLock(self): + self.lock = None + logger = logging.getLogger(__name__) logger.addHandler(NullHandler()) diff --git a/distlib/compat.py b/distlib/compat.py index 1fe3d22..e93dc27 100644 --- a/distlib/compat.py +++ b/distlib/compat.py @@ -8,6 +8,7 @@ import os import re +import shutil import sys try: @@ -33,9 +34,8 @@ def quote(s): import urllib2 from urllib2 import (Request, urlopen, URLError, HTTPError, - HTTPBasicAuthHandler, HTTPPasswordMgr, - HTTPHandler, HTTPRedirectHandler, - build_opener) + HTTPBasicAuthHandler, HTTPPasswordMgr, HTTPHandler, + HTTPRedirectHandler, build_opener) if ssl: from urllib2 import HTTPSHandler import httplib @@ -50,15 +50,15 @@ def quote(s): # Leaving this around for now, in case it needs resurrecting in some way # _userprog = None # def splituser(host): - # """splituser('user[:passwd]@host[:port]') --> 'user[:passwd]', 'host[:port]'.""" - # global _userprog - # if _userprog is None: - # import re - # _userprog = re.compile('^(.*)@(.*)$') + # """splituser('user[:passwd]@host[:port]') --> 'user[:passwd]', 'host[:port]'.""" + # global _userprog + # if _userprog is None: + # import re + # _userprog = re.compile('^(.*)@(.*)$') - # match = _userprog.match(host) - # if match: return match.group(1, 2) - # return None, host + # match = _userprog.match(host) + # if match: return match.group(1, 2) + # return None, host else: # pragma: no cover from io import StringIO @@ -67,14 +67,12 @@ def quote(s): from io import TextIOWrapper as file_type import builtins import configparser - import shutil - from urllib.parse import (urlparse, urlunparse, urljoin, quote, - unquote, urlsplit, urlunsplit, splittype) + from urllib.parse import (urlparse, urlunparse, urljoin, quote, unquote, + urlsplit, urlunsplit, splittype) from urllib.request import (urlopen, urlretrieve, Request, url2pathname, - pathname2url, - HTTPBasicAuthHandler, HTTPPasswordMgr, - HTTPHandler, HTTPRedirectHandler, - build_opener) + pathname2url, HTTPBasicAuthHandler, + HTTPPasswordMgr, HTTPHandler, + HTTPRedirectHandler, build_opener) if ssl: from urllib.request import HTTPSHandler from urllib.error import HTTPError, URLError, ContentTooShortError @@ -88,14 +86,13 @@ def quote(s): from itertools import filterfalse filter = filter - try: from ssl import match_hostname, CertificateError -except ImportError: # pragma: no cover +except ImportError: # pragma: no cover + class CertificateError(ValueError): pass - def _dnsname_match(dn, hostname, max_wildcards=1): """Matching according to RFC 6125, section 6.4.3 @@ -145,7 +142,6 @@ def _dnsname_match(dn, hostname, max_wildcards=1): pat = re.compile(r'\A' + r'\.'.join(pats) + r'\Z', re.IGNORECASE) return pat.match(hostname) - def match_hostname(cert, hostname): """Verify that *cert* (in decoded format as returned by SSLSocket.getpeercert()) matches the *hostname*. RFC 2818 and RFC 6125 @@ -178,24 +174,26 @@ def match_hostname(cert, hostname): dnsnames.append(value) if len(dnsnames) > 1: raise CertificateError("hostname %r " - "doesn't match either of %s" - % (hostname, ', '.join(map(repr, dnsnames)))) + "doesn't match either of %s" % + (hostname, ', '.join(map(repr, dnsnames)))) elif len(dnsnames) == 1: raise CertificateError("hostname %r " - "doesn't match %r" - % (hostname, dnsnames[0])) + "doesn't match %r" % + (hostname, dnsnames[0])) else: raise CertificateError("no appropriate commonName or " - "subjectAltName fields were found") + "subjectAltName fields were found") try: from types import SimpleNamespace as Container except ImportError: # pragma: no cover + class Container(object): """ A generic container for when multiple values need to be returned """ + def __init__(self, **kwargs): self.__dict__.update(kwargs) @@ -214,6 +212,7 @@ def which(cmd, mode=os.F_OK | os.X_OK, path=None): path. """ + # Check that a given file can be accessed with the correct mode. # Additionally check that `file` is not a directory, as on Windows # directories pass the os.access check. @@ -237,7 +236,7 @@ def _access_check(fn, mode): if sys.platform == "win32": # The current directory takes precedence on Windows. - if not os.curdir in path: + if os.curdir not in path: path.insert(0, os.curdir) # PATHEXT is necessary to check on Windows. @@ -258,7 +257,7 @@ def _access_check(fn, mode): seen = set() for dir in path: normdir = os.path.normcase(dir) - if not normdir in seen: + if normdir not in seen: seen.add(normdir) for thefile in files: name = os.path.join(dir, thefile) @@ -277,6 +276,7 @@ def _access_check(fn, mode): from zipfile import ZipExtFile as BaseZipExtFile class ZipExtFile(BaseZipExtFile): + def __init__(self, base): self.__dict__.update(base.__dict__) @@ -288,6 +288,7 @@ def __exit__(self, *exc_info): # return None, so if an exception occurred, it will propagate class ZipFile(BaseZipFile): + def __enter__(self): return self @@ -299,9 +300,11 @@ def open(self, *args, **kwargs): base = BaseZipFile.open(self, *args, **kwargs) return ZipExtFile(base) + try: from platform import python_implementation -except ImportError: # pragma: no cover +except ImportError: # pragma: no cover + def python_implementation(): """Return a string identifying the Python implementation.""" if 'PyPy' in sys.version: @@ -312,12 +315,12 @@ def python_implementation(): return 'IronPython' return 'CPython' -import shutil + import sysconfig try: callable = callable -except NameError: # pragma: no cover +except NameError: # pragma: no cover from collections.abc import Callable def callable(obj): @@ -358,11 +361,11 @@ def fsdecode(filename): raise TypeError("expect bytes or str, not %s" % type(filename).__name__) + try: from tokenize import detect_encoding -except ImportError: # pragma: no cover +except ImportError: # pragma: no cover from codecs import BOM_UTF8, lookup - import re cookie_re = re.compile(r"coding[:=]\s*([-\w.]+)") @@ -401,6 +404,7 @@ def detect_encoding(readline): bom_found = False encoding = None default = 'utf-8' + def read_or_stop(): try: return readline() @@ -430,8 +434,8 @@ def find_cookie(line): if filename is None: msg = "unknown encoding: " + encoding else: - msg = "unknown encoding for {!r}: {}".format(filename, - encoding) + msg = "unknown encoding for {!r}: {}".format( + filename, encoding) raise SyntaxError(msg) if bom_found: @@ -440,7 +444,8 @@ def find_cookie(line): if filename is None: msg = 'encoding problem: utf-8' else: - msg = 'encoding problem for {!r}: utf-8'.format(filename) + msg = 'encoding problem for {!r}: utf-8'.format( + filename) raise SyntaxError(msg) encoding += '-sig' return encoding @@ -467,6 +472,7 @@ def find_cookie(line): return default, [first, second] + # For converting & <-> & etc. try: from html import escape @@ -479,12 +485,13 @@ def find_cookie(line): try: from collections import ChainMap -except ImportError: # pragma: no cover +except ImportError: # pragma: no cover from collections import MutableMapping try: from reprlib import recursive_repr as _recursive_repr except ImportError: + def _recursive_repr(fillvalue='...'): ''' Decorator to make a repr function return fillvalue for a recursive @@ -509,13 +516,15 @@ def wrapper(self): wrapper.__module__ = getattr(user_function, '__module__') wrapper.__doc__ = getattr(user_function, '__doc__') wrapper.__name__ = getattr(user_function, '__name__') - wrapper.__annotations__ = getattr(user_function, '__annotations__', {}) + wrapper.__annotations__ = getattr(user_function, + '__annotations__', {}) return wrapper return decorating_function class ChainMap(MutableMapping): - ''' A ChainMap groups multiple dicts (or other mappings) together + ''' + A ChainMap groups multiple dicts (or other mappings) together to create a single, updateable view. The underlying mappings are stored in a list. That list is public and can @@ -524,7 +533,6 @@ class ChainMap(MutableMapping): Lookups search the underlying mappings successively until a key is found. In contrast, writes, updates, and deletions only operate on the first mapping. - ''' def __init__(self, *maps): @@ -532,7 +540,7 @@ def __init__(self, *maps): If no mappings are provided, a single empty dictionary is used. ''' - self.maps = list(maps) or [{}] # always at least one map + self.maps = list(maps) or [{}] # always at least one map def __missing__(self, key): raise KeyError(key) @@ -540,16 +548,19 @@ def __missing__(self, key): def __getitem__(self, key): for mapping in self.maps: try: - return mapping[key] # can't use 'key in mapping' with defaultdict + return mapping[ + key] # can't use 'key in mapping' with defaultdict except KeyError: pass - return self.__missing__(key) # support subclasses that define __missing__ + return self.__missing__( + key) # support subclasses that define __missing__ def get(self, key, default=None): return self[key] if key in self else default def __len__(self): - return len(set().union(*self.maps)) # reuses stored hash values if possible + return len(set().union( + *self.maps)) # reuses stored hash values if possible def __iter__(self): return iter(set().union(*self.maps)) @@ -576,12 +587,12 @@ def copy(self): __copy__ = copy - def new_child(self): # like Django's Context.push() + def new_child(self): # like Django's Context.push() 'New ChainMap with a new dict followed by all previous maps.' return self.__class__({}, *self.maps) @property - def parents(self): # like Django's Context.pop() + def parents(self): # like Django's Context.pop() 'New ChainMap from maps[1:].' return self.__class__(*self.maps[1:]) @@ -592,7 +603,8 @@ def __delitem__(self, key): try: del self.maps[0][key] except KeyError: - raise KeyError('Key not found in the first mapping: {!r}'.format(key)) + raise KeyError( + 'Key not found in the first mapping: {!r}'.format(key)) def popitem(self): 'Remove and return an item pair from maps[0]. Raise KeyError is maps[0] is empty.' @@ -606,15 +618,18 @@ def pop(self, key, *args): try: return self.maps[0].pop(key, *args) except KeyError: - raise KeyError('Key not found in the first mapping: {!r}'.format(key)) + raise KeyError( + 'Key not found in the first mapping: {!r}'.format(key)) def clear(self): 'Clear maps[0], leaving maps[1:] intact.' self.maps[0].clear() + try: from importlib.util import cache_from_source # Python >= 3.4 except ImportError: # pragma: no cover + def cache_from_source(path, debug_override=None): assert path.endswith('.py') if debug_override is None: @@ -625,12 +640,13 @@ def cache_from_source(path, debug_override=None): suffix = 'o' return path + suffix + try: from collections import OrderedDict -except ImportError: # pragma: no cover -## {{{ http://code.activestate.com/recipes/576693/ (r9) -# Backport of OrderedDict() class that runs on Python 2.4, 2.5, 2.6, 2.7 and pypy. -# Passes Python2.7's test suite and incorporates all the latest updates. +except ImportError: # pragma: no cover + # {{{ http://code.activestate.com/recipes/576693/ (r9) + # Backport of OrderedDict() class that runs on Python 2.4, 2.5, 2.6, 2.7 and pypy. + # Passes Python2.7's test suite and incorporates all the latest updates. try: from thread import get_ident as _get_ident except ImportError: @@ -641,9 +657,9 @@ def cache_from_source(path, debug_override=None): except ImportError: pass - class OrderedDict(dict): 'Dictionary that remembers insertion order' + # An inherited dict maps keys to values. # The inherited dict provides __getitem__, __len__, __contains__, and get. # The remaining methods are order-aware. @@ -661,11 +677,12 @@ def __init__(self, *args, **kwds): ''' if len(args) > 1: - raise TypeError('expected at most 1 arguments, got %d' % len(args)) + raise TypeError('expected at most 1 arguments, got %d' % + len(args)) try: self.__root except AttributeError: - self.__root = root = [] # sentinel node + self.__root = root = [] # sentinel node root[:] = [root, root, None] self.__map = {} self.__update(*args, **kwds) @@ -779,7 +796,7 @@ def update(*args, **kwds): ''' if len(args) > 2: raise TypeError('update() takes at most 2 positional ' - 'arguments (%d given)' % (len(args),)) + 'arguments (%d given)' % (len(args), )) elif not args: raise TypeError('update() takes at least 1 argument (0 given)') self = args[0] @@ -825,14 +842,15 @@ def setdefault(self, key, default=None): def __repr__(self, _repr_running=None): 'od.__repr__() <==> repr(od)' - if not _repr_running: _repr_running = {} + if not _repr_running: + _repr_running = {} call_key = id(self), _get_ident() if call_key in _repr_running: return '...' _repr_running[call_key] = 1 try: if not self: - return '%s()' % (self.__class__.__name__,) + return '%s()' % (self.__class__.__name__, ) return '%s(%r)' % (self.__class__.__name__, self.items()) finally: del _repr_running[call_key] @@ -844,8 +862,8 @@ def __reduce__(self): for k in vars(OrderedDict()): inst_dict.pop(k, None) if inst_dict: - return (self.__class__, (items,), inst_dict) - return self.__class__, (items,) + return (self.__class__, (items, ), inst_dict) + return self.__class__, (items, ) def copy(self): 'od.copy() -> a shallow copy of od' @@ -868,7 +886,8 @@ def __eq__(self, other): ''' if isinstance(other, OrderedDict): - return len(self)==len(other) and self.items() == other.items() + return len(self) == len( + other) and self.items() == other.items() return dict.__eq__(self, other) def __ne__(self, other): @@ -888,19 +907,18 @@ def viewitems(self): "od.viewitems() -> a set-like object providing a view on od's items" return ItemsView(self) + try: from logging.config import BaseConfigurator, valid_ident -except ImportError: # pragma: no cover +except ImportError: # pragma: no cover IDENTIFIER = re.compile('^[a-z_][a-z0-9_]*$', re.I) - def valid_ident(s): m = IDENTIFIER.match(s) if not m: raise ValueError('Not a valid Python identifier: %r' % s) return True - # The ConvertingXXX classes are wrappers around standard Python containers, # and they serve to convert any suitable values in the container. The # conversion converts base dicts, lists and tuples to their wrapped @@ -916,7 +934,7 @@ class ConvertingDict(dict): def __getitem__(self, key): value = dict.__getitem__(self, key) result = self.configurator.convert(value) - #If the converted value is different, save for next time + # If the converted value is different, save for next time if value is not result: self[key] = result if type(result) in (ConvertingDict, ConvertingList, @@ -928,7 +946,7 @@ def __getitem__(self, key): def get(self, key, default=None): value = dict.get(self, key, default) result = self.configurator.convert(value) - #If the converted value is different, save for next time + # If the converted value is different, save for next time if value is not result: self[key] = result if type(result) in (ConvertingDict, ConvertingList, @@ -949,10 +967,11 @@ def pop(self, key, default=None): class ConvertingList(list): """A converting list wrapper.""" + def __getitem__(self, key): value = list.__getitem__(self, key) result = self.configurator.convert(value) - #If the converted value is different, save for next time + # If the converted value is different, save for next time if value is not result: self[key] = result if type(result) in (ConvertingDict, ConvertingList, @@ -972,6 +991,7 @@ def pop(self, idx=-1): class ConvertingTuple(tuple): """A converting tuple wrapper.""" + def __getitem__(self, key): value = tuple.__getitem__(self, key) result = self.configurator.convert(value) @@ -995,8 +1015,8 @@ class BaseConfigurator(object): DIGIT_PATTERN = re.compile(r'^\d+$') value_converters = { - 'ext' : 'ext_convert', - 'cfg' : 'cfg_convert', + 'ext': 'ext_convert', + 'cfg': 'cfg_convert', } # We might want to use a different one, e.g. importlib @@ -1042,7 +1062,6 @@ def cfg_convert(self, value): else: rest = rest[m.end():] d = self.config[m.groups()[0]] - #print d, rest while rest: m = self.DOT_PATTERN.match(rest) if m: @@ -1055,7 +1074,9 @@ def cfg_convert(self, value): d = d[idx] else: try: - n = int(idx) # try as number first (most likely) + n = int( + idx + ) # try as number first (most likely) d = d[n] except TypeError: d = d[idx] @@ -1064,7 +1085,7 @@ def cfg_convert(self, value): else: raise ValueError('Unable to convert ' '%r at %r' % (value, rest)) - #rest should be empty + # rest should be empty return d def convert(self, value): @@ -1073,14 +1094,15 @@ def convert(self, value): replaced by their converting alternatives. Strings are checked to see if they have a conversion format and are converted if they do. """ - if not isinstance(value, ConvertingDict) and isinstance(value, dict): + if not isinstance(value, ConvertingDict) and isinstance( + value, dict): value = ConvertingDict(value) value.configurator = self - elif not isinstance(value, ConvertingList) and isinstance(value, list): + elif not isinstance(value, ConvertingList) and isinstance( + value, list): value = ConvertingList(value) value.configurator = self - elif not isinstance(value, ConvertingTuple) and\ - isinstance(value, tuple): + elif not isinstance(value, ConvertingTuple) and isinstance(value, tuple): value = ConvertingTuple(value) value.configurator = self elif isinstance(value, string_types): diff --git a/distlib/database.py b/distlib/database.py index bc16e88..eb3765f 100644 --- a/distlib/database.py +++ b/distlib/database.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # -# Copyright (C) 2012-2017 The Python Software Foundation. +# Copyright (C) 2012-2023 The Python Software Foundation. # See LICENSE.txt and CONTRIBUTORS.txt. # """PEP 376 implementation.""" @@ -25,11 +25,10 @@ from .util import (parse_requirement, cached_property, parse_name_and_version, read_exports, write_exports, CSVReader, CSVWriter) - -__all__ = ['Distribution', 'BaseInstalledDistribution', - 'InstalledDistribution', 'EggInfoDistribution', - 'DistributionPath'] - +__all__ = [ + 'Distribution', 'BaseInstalledDistribution', 'InstalledDistribution', + 'EggInfoDistribution', 'DistributionPath' +] logger = logging.getLogger(__name__) @@ -46,6 +45,7 @@ class _Cache(object): """ A simple cache mapping names and .dist-info paths to distributions """ + def __init__(self): """ Initialise an instance. There is normally one for each DistributionPath. @@ -76,6 +76,7 @@ class DistributionPath(object): """ Represents a set of distributions installed on a path (typically sys.path). """ + def __init__(self, path=None, include_egg=False): """ Create an instance from a path, optionally including legacy (distutils/ @@ -111,7 +112,6 @@ def clear_cache(self): self._cache.clear() self._cache_egg.clear() - def _yield_distributions(self): """ Yield .dist-info and/or .egg(-info) distributions. @@ -134,11 +134,13 @@ def _yield_distributions(self): continue try: if self._include_dist and entry.endswith(DISTINFO_EXT): - possible_filenames = [METADATA_FILENAME, - WHEEL_METADATA_FILENAME, - LEGACY_METADATA_FILENAME] + possible_filenames = [ + METADATA_FILENAME, WHEEL_METADATA_FILENAME, + LEGACY_METADATA_FILENAME + ] for metadata_filename in possible_filenames: - metadata_path = posixpath.join(entry, metadata_filename) + metadata_path = posixpath.join( + entry, metadata_filename) pydist = finder.find(metadata_path) if pydist: break @@ -146,13 +148,15 @@ def _yield_distributions(self): continue with contextlib.closing(pydist.as_stream()) as stream: - metadata = Metadata(fileobj=stream, scheme='legacy') + metadata = Metadata(fileobj=stream, + scheme='legacy') logger.debug('Found %s', r.path) seen.add(r.path) - yield new_dist_class(r.path, metadata=metadata, + yield new_dist_class(r.path, + metadata=metadata, env=self) - elif self._include_egg and entry.endswith(('.egg-info', - '.egg')): + elif self._include_egg and entry.endswith( + ('.egg-info', '.egg')): logger.debug('Found %s', r.path) seen.add(r.path) yield old_dist_class(r.path, self) @@ -271,7 +275,7 @@ def provides_distribution(self, name, version=None): matcher = self._scheme.matcher('%s (%s)' % (name, version)) except ValueError: raise DistlibException('invalid name or version: %r, %r' % - (name, version)) + (name, version)) for dist in self.get_distributions(): # We hit a problem on Travis where enum34 was installed and doesn't @@ -346,12 +350,12 @@ def __init__(self, metadata): """ self.metadata = metadata self.name = metadata.name - self.key = self.name.lower() # for case-insensitive comparisons + self.key = self.name.lower() # for case-insensitive comparisons self.version = metadata.version self.locator = None self.digest = None - self.extras = None # additional features requested - self.context = None # environment marker overrides + self.extras = None # additional features requested + self.context = None # environment marker overrides self.download_urls = set() self.digests = {} @@ -362,7 +366,7 @@ def source_url(self): """ return self.metadata.source_url - download_url = source_url # Backward compatibility + download_url = source_url # Backward compatibility @property def name_and_version(self): @@ -386,10 +390,10 @@ def provides(self): def _get_requirements(self, req_attr): md = self.metadata reqts = getattr(md, req_attr) - logger.debug('%s: got requirements %r from metadata: %r', self.name, req_attr, - reqts) - return set(md.get_requirements(reqts, extras=self.extras, - env=self.context)) + logger.debug('%s: got requirements %r from metadata: %r', self.name, + req_attr, reqts) + return set( + md.get_requirements(reqts, extras=self.extras, env=self.context)) @property def run_requires(self): @@ -426,12 +430,11 @@ def matches_requirement(self, req): matcher = scheme.matcher(r.requirement) except UnsupportedVersionError: # XXX compat-mode if cannot read the version - logger.warning('could not read version %r - using name only', - req) + logger.warning('could not read version %r - using name only', req) name = req.split()[0] matcher = scheme.matcher(name) - name = matcher.key # case-insensitive + name = matcher.key # case-insensitive result = False for p in self.provides: @@ -466,9 +469,8 @@ def __eq__(self, other): if type(other) is not type(self): result = False else: - result = (self.name == other.name and - self.version == other.version and - self.source_url == other.source_url) + result = (self.name == other.name and self.version == other.version + and self.source_url == other.source_url) return result def __hash__(self): @@ -559,8 +561,8 @@ def __init__(self, path, metadata=None, env=None): if r is None: r = finder.find(LEGACY_METADATA_FILENAME) if r is None: - raise ValueError('no %s found in %s' % (METADATA_FILENAME, - path)) + raise ValueError('no %s found in %s' % + (METADATA_FILENAME, path)) with contextlib.closing(r.as_stream()) as stream: metadata = Metadata(fileobj=stream, scheme='legacy') @@ -571,7 +573,7 @@ def __init__(self, path, metadata=None, env=None): r = finder.find('REQUESTED') self.requested = r is not None - p = os.path.join(path, 'top_level.txt') + p = os.path.join(path, 'top_level.txt') if os.path.exists(p): with open(p, 'rb') as f: data = f.read().decode('utf-8') @@ -596,14 +598,14 @@ def _get_records(self): with contextlib.closing(r.as_stream()) as stream: with CSVReader(stream=stream) as record_reader: # Base location is parent dir of .dist-info dir - #base_location = os.path.dirname(self.path) - #base_location = os.path.abspath(base_location) + # base_location = os.path.dirname(self.path) + # base_location = os.path.abspath(base_location) for row in record_reader: missing = [None for i in range(len(row), 3)] path, checksum, size = row + missing - #if not os.path.isabs(path): - # path = path.replace('/', os.sep) - # path = os.path.join(base_location, path) + # if not os.path.isabs(path): + # path = path.replace('/', os.sep) + # path = os.path.join(base_location, path) results.append((path, checksum, size)) return results @@ -701,8 +703,8 @@ def write_installed_files(self, paths, prefix, dry_run=False): size = '%d' % os.path.getsize(path) with open(path, 'rb') as fp: hash_value = self.get_hash(fp.read()) - if path.startswith(base) or (base_under_prefix and - path.startswith(prefix)): + if path.startswith(base) or (base_under_prefix + and path.startswith(prefix)): path = os.path.relpath(path, base) writer.writerow((path, hash_value, size)) @@ -744,7 +746,8 @@ def check_installed_files(self): with open(path, 'rb') as f: actual_hash = self.get_hash(f.read(), hasher) if actual_hash != hash_value: - mismatches.append((path, 'hash', hash_value, actual_hash)) + mismatches.append( + (path, 'hash', hash_value, actual_hash)) return mismatches @cached_property @@ -791,7 +794,7 @@ def write_shared_locations(self, paths, dry_run=False): for key in ('prefix', 'lib', 'headers', 'scripts', 'data'): path = paths[key] if os.path.isdir(paths[key]): - lines.append('%s=%s' % (key, path)) + lines.append('%s=%s' % (key, path)) for ns in paths.get('namespace', ()): lines.append('namespace=%s' % ns) @@ -854,8 +857,8 @@ def list_distinfo_files(self): yield path def __eq__(self, other): - return (isinstance(other, InstalledDistribution) and - self.path == other.path) + return (isinstance(other, InstalledDistribution) + and self.path == other.path) # See http://docs.python.org/reference/datamodel#object.__hash__ __hash__ = object.__hash__ @@ -867,13 +870,14 @@ class EggInfoDistribution(BaseInstalledDistribution): if the given path happens to be a directory, the metadata is read from the file ``PKG-INFO`` under that directory.""" - requested = True # as we have no way of knowing, assume it was + requested = True # as we have no way of knowing, assume it was shared_locations = {} def __init__(self, path, env=None): + def set_name_and_version(s, n, v): s.name = n - s.key = n.lower() # for case-insensitive comparisons + s.key = n.lower() # for case-insensitive comparisons s.version = v self.path = path @@ -907,8 +911,8 @@ def parse_requires_data(data): if not line: # pragma: no cover continue if line.startswith('['): # pragma: no cover - logger.warning('Unexpected line: quitting requirement scan: %r', - line) + logger.warning( + 'Unexpected line: quitting requirement scan: %r', line) break r = parse_requirement(line) if not r: # pragma: no cover @@ -955,7 +959,8 @@ def parse_requires_path(req_path): metadata = Metadata(fileobj=fileobj, scheme='legacy') try: data = zipf.get_data('EGG-INFO/requires.txt') - tl_data = zipf.get_data('EGG-INFO/top_level.txt').decode('utf-8') + tl_data = zipf.get_data('EGG-INFO/top_level.txt').decode( + 'utf-8') requires = parse_requires_data(data.decode('utf-8')) except IOError: requires = None @@ -985,8 +990,8 @@ def parse_requires_path(req_path): return metadata def __repr__(self): - return '' % ( - self.name, self.version, self.path) + return '' % (self.name, self.version, + self.path) def __str__(self): return "%s %s" % (self.name, self.version) @@ -1042,7 +1047,7 @@ def _size(path): logger.warning('Non-existent file: %s', p) if p.endswith(('.pyc', '.pyo')): continue - #otherwise fall through and fail + # otherwise fall through and fail if not os.path.isdir(p): result.append((p, _md5(p), _size(p))) result.append((record_path, None, None)) @@ -1078,12 +1083,13 @@ def list_distinfo_files(self, absolute=False): yield line def __eq__(self, other): - return (isinstance(other, EggInfoDistribution) and - self.path == other.path) + return (isinstance(other, EggInfoDistribution) + and self.path == other.path) # See http://docs.python.org/reference/datamodel#object.__hash__ __hash__ = object.__hash__ + new_dist_class = InstalledDistribution old_dist_class = EggInfoDistribution @@ -1117,7 +1123,7 @@ def add_distribution(self, distribution): """ self.adjacency_list[distribution] = [] self.reverse_list[distribution] = [] - #self.missing[distribution] = [] + # self.missing[distribution] = [] def add_edge(self, x, y, label=None): """Add an edge from distribution *x* to distribution *y* with the given @@ -1177,7 +1183,7 @@ def to_dot(self, f, skip_disconnected=True): if len(adjs) == 0 and not skip_disconnected: disconnected.append(dist) for other, label in adjs: - if not label is None: + if label is not None: f.write('"%s" -> "%s" [label="%s"]\n' % (dist.name, other.name, label)) else: @@ -1255,8 +1261,8 @@ def make_graph(dists, scheme='default'): # now make the edges for dist in dists: - requires = (dist.run_requires | dist.meta_requires | - dist.build_requires | dist.dev_requires) + requires = (dist.run_requires | dist.meta_requires + | dist.build_requires | dist.dev_requires) for req in requires: try: matcher = scheme.matcher(req) @@ -1267,7 +1273,7 @@ def make_graph(dists, scheme='default'): name = req.split()[0] matcher = scheme.matcher(name) - name = matcher.key # case-insensitive + name = matcher.key # case-insensitive matched = False if name in provided: @@ -1327,7 +1333,7 @@ def get_required_dists(dists, dist): req = set() # required distributions todo = graph.adjacency_list[dist] # list of nodes we should inspect - seen = set(t[0] for t in todo) # already added to todo + seen = set(t[0] for t in todo) # already added to todo while todo: d = todo.pop()[0] diff --git a/distlib/index.py b/distlib/index.py index 9b6d129..56cd286 100644 --- a/distlib/index.py +++ b/distlib/index.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # -# Copyright (C) 2013 Vinay Sajip. +# Copyright (C) 2013-2023 Vinay Sajip. # Licensed to the Python Software Foundation under a contributor agreement. # See LICENSE.txt and CONTRIBUTORS.txt. # @@ -25,6 +25,7 @@ DEFAULT_INDEX = 'https://pypi.org/pypi' DEFAULT_REALM = 'pypi' + class PackageIndex(object): """ This class represents a package index compatible with PyPI, the Python @@ -119,7 +120,7 @@ def register(self, metadata): # pragma: no cover d = metadata.todict() d[':action'] = 'verify' request = self.encode_request(d.items(), []) - response = self.send_request(request) + self.send_request(request) d[':action'] = 'submit' request = self.encode_request(d.items(), []) return self.send_request(request) @@ -358,8 +359,7 @@ def verify_signature(self, signature_filename, data_filename, keystore) rc, stdout, stderr = self.run_command(cmd) if rc not in (0, 1): - raise DistlibException('verify command failed with error ' - 'code %s' % rc) + raise DistlibException('verify command failed with error code %s' % rc) return rc == 0 def download_file(self, url, destfile, digest=None, reporthook=None): diff --git a/distlib/locators.py b/distlib/locators.py index 966ebc0..f9f0788 100644 --- a/distlib/locators.py +++ b/distlib/locators.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # -# Copyright (C) 2012-2015 Vinay Sajip. +# Copyright (C) 2012-2023 Vinay Sajip. # Licensed to the Python Software Foundation under a contributor agreement. # See LICENSE.txt and CONTRIBUTORS.txt. # @@ -38,6 +38,7 @@ HTML_CONTENT_TYPE = re.compile('text/html|application/x(ht)?ml') DEFAULT_INDEX = 'https://pypi.org/pypi' + def get_all_distribution_names(url=None): """ Return all distribution names known by an index. @@ -52,6 +53,7 @@ def get_all_distribution_names(url=None): finally: client('close')() + class RedirectHandler(BaseRedirectHandler): """ A class to work around a bug in some Python 3.2.x releases. @@ -83,6 +85,7 @@ def http_error_302(self, req, fp, code, msg, headers): http_error_301 = http_error_303 = http_error_307 = http_error_302 + class Locator(object): """ A base class for locators - things that locate distributions. @@ -272,7 +275,7 @@ def same_project(name1, name2): 'python-version': ', '.join( ['.'.join(list(v[2:])) for v in wheel.pyver]), } - except Exception as e: # pragma: no cover + except Exception: # pragma: no cover logger.warning('invalid path for wheel: %s', path) elif not path.endswith(self.downloadable_extensions): # pragma: no cover logger.debug('Not downloadable: %s', path) @@ -293,7 +296,6 @@ def same_project(name1, name2): 'filename': filename, 'url': urlunparse((scheme, netloc, origpath, params, query, '')), - #'packagetype': 'sdist', } if pyver: # pragma: no cover result['python-version'] = pyver @@ -382,12 +384,9 @@ def locate(self, requirement, prereleases=False): else: if prereleases or not vcls(k).is_prerelease: slist.append(k) - # else: - # logger.debug('skipping pre-release ' - # 'version %s of %s', k, matcher.name) except Exception: # pragma: no cover logger.warning('error matching %s with %r', matcher, k) - pass # slist.append(k) + pass # slist.append(k) if len(slist) > 1: slist = sorted(slist, key=scheme.key) if slist: @@ -456,6 +455,7 @@ def _get_project(self, name): result['digests'][url] = digest return result + class PyPIJSONLocator(Locator): """ This locator uses PyPI's JSON interface. It's very limited in functionality @@ -476,7 +476,7 @@ def _get_project(self, name): url = urljoin(self.base_url, '%s/json' % quote(name)) try: resp = self.opener.open(url) - data = resp.read().decode() # for now + data = resp.read().decode() # for now d = json.loads(data) md = Metadata(scheme=self.scheme) data = d['info'] @@ -487,7 +487,7 @@ def _get_project(self, name): md.summary = data.get('summary') dist = Distribution(md) dist.locator = self - urls = d['urls'] + # urls = d['urls'] result[md.version] = dist for info in d['urls']: url = info['url'] @@ -745,7 +745,7 @@ def _fetch(self): try: self._seen.add(link) if (not self._process_download(link) and - self._should_queue(link, url, rel)): + self._should_queue(link, url, rel)): logger.debug('Queueing %s from %s', link, url) self._to_fetch.put(link) except MetadataInvalidError: # e.g. invalid versions @@ -756,7 +756,7 @@ def _fetch(self): # always do this, to avoid hangs :-) self._to_fetch.task_done() if not url: - #logger.debug('Sentinel seen, quitting.') + # logger.debug('Sentinel seen, quitting.') break def get_page(self, url): @@ -832,6 +832,7 @@ def get_distribution_names(self): result.add(match.group(1)) return result + class DirectoryLocator(Locator): """ This class locates distributions in a directory tree. @@ -897,6 +898,7 @@ def get_distribution_names(self): break return result + class JSONLocator(Locator): """ This locator uses special extended metadata (not available on PyPI) and is @@ -935,6 +937,7 @@ def _get_project(self, name): result['urls'].setdefault(dist.version, set()).add(info['url']) return result + class DistPathLocator(Locator): """ This locator finds installed distributions in a path. It can be useful for @@ -1245,7 +1248,7 @@ def find(self, requirement, meta_extras=None, prereleases=False): if name not in self.dists_by_name: self.add_distribution(dist) else: - #import pdb; pdb.set_trace() + # import pdb; pdb.set_trace() other = self.dists_by_name[name] if other != dist: self.try_to_replace(dist, other, problems) diff --git a/distlib/manifest.py b/distlib/manifest.py index 18beba3..420dcf1 100644 --- a/distlib/manifest.py +++ b/distlib/manifest.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # -# Copyright (C) 2012-2013 Python Software Foundation. +# Copyright (C) 2012-2023 Python Software Foundation. # See LICENSE.txt and CONTRIBUTORS.txt. # """ @@ -34,6 +34,7 @@ # _PYTHON_VERSION = sys.version_info[:2] + class Manifest(object): """ A list of files built by exploring the filesystem and filtered by applying various @@ -155,10 +156,7 @@ def process_directive(self, directive): elif action == 'exclude': for pattern in patterns: - found = self._exclude_pattern(pattern, anchor=True) - #if not found: - # logger.warning('no previously-included files ' - # 'found matching %r', pattern) + self._exclude_pattern(pattern, anchor=True) elif action == 'global-include': for pattern in patterns: @@ -168,11 +166,7 @@ def process_directive(self, directive): elif action == 'global-exclude': for pattern in patterns: - found = self._exclude_pattern(pattern, anchor=False) - #if not found: - # logger.warning('no previously-included files ' - # 'matching %r found anywhere in ' - # 'distribution', pattern) + self._exclude_pattern(pattern, anchor=False) elif action == 'recursive-include': for pattern in patterns: @@ -182,11 +176,7 @@ def process_directive(self, directive): elif action == 'recursive-exclude': for pattern in patterns: - found = self._exclude_pattern(pattern, prefix=thedir) - #if not found: - # logger.warning('no previously-included files ' - # 'matching %r found under directory %r', - # pattern, thedir) + self._exclude_pattern(pattern, prefix=thedir) elif action == 'graft': if not self._include_pattern(None, prefix=dirpattern): diff --git a/distlib/markers.py b/distlib/markers.py index 7ebe472..1514d46 100644 --- a/distlib/markers.py +++ b/distlib/markers.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # -# Copyright (C) 2012-2017 Vinay Sajip. +# Copyright (C) 2012-2023 Vinay Sajip. # Licensed to the Python Software Foundation under a contributor agreement. # See LICENSE.txt and CONTRIBUTORS.txt. # @@ -23,20 +23,25 @@ __all__ = ['interpret'] -_VERSION_PATTERN = re.compile(r'((\d+(\.\d+)*\w*)|\'(\d+(\.\d+)*\w*)\'|\"(\d+(\.\d+)*\w*)\")') +_VERSION_PATTERN = re.compile( + r'((\d+(\.\d+)*\w*)|\'(\d+(\.\d+)*\w*)\'|\"(\d+(\.\d+)*\w*)\")') _VERSION_MARKERS = {'python_version', 'python_full_version'} + def _is_version_marker(s): return isinstance(s, string_types) and s in _VERSION_MARKERS + def _is_literal(o): if not isinstance(o, string_types) or not o: return False return o[0] in '\'"' + def _get_versions(s): return {LV(m.groups()[0]) for m in _VERSION_PATTERN.finditer(s)} + class Evaluator(object): """ This class is used to evaluate marker expressions. @@ -47,10 +52,10 @@ class Evaluator(object): '===': lambda x, y: x == y, '~=': lambda x, y: x == y or x > y, '!=': lambda x, y: x != y, - '<': lambda x, y: x < y, - '<=': lambda x, y: x == y or x < y, - '>': lambda x, y: x > y, - '>=': lambda x, y: x == y or x > y, + '<': lambda x, y: x < y, + '<=': lambda x, y: x == y or x < y, + '>': lambda x, y: x > y, + '>=': lambda x, y: x == y or x > y, 'and': lambda x, y: x and y, 'or': lambda x, y: x or y, 'in': lambda x, y: x in y, @@ -77,12 +82,13 @@ def evaluate(self, expr, context): elhs = expr['lhs'] erhs = expr['rhs'] if _is_literal(expr['lhs']) and _is_literal(expr['rhs']): - raise SyntaxError('invalid comparison: %s %s %s' % (elhs, op, erhs)) + raise SyntaxError('invalid comparison: %s %s %s' % + (elhs, op, erhs)) lhs = self.evaluate(elhs, context) rhs = self.evaluate(erhs, context) - if ((_is_version_marker(elhs) or _is_version_marker(erhs)) and - op in ('<', '<=', '>', '>=', '===', '==', '!=', '~=')): + if ((_is_version_marker(elhs) or _is_version_marker(erhs)) + and op in ('<', '<=', '>', '>=', '===', '==', '!=', '~=')): lhs = LV(lhs) rhs = LV(rhs) elif _is_version_marker(elhs) and op in ('in', 'not in'): @@ -91,9 +97,12 @@ def evaluate(self, expr, context): result = self.operations[op](lhs, rhs) return result + _DIGITS = re.compile(r'\d+\.\d+') + def default_context(): + def format_full_version(info): version = '%s.%s.%s' % (info.major, info.minor, info.micro) kind = info.releaselevel @@ -102,7 +111,8 @@ def format_full_version(info): return version if hasattr(sys, 'implementation'): - implementation_version = format_full_version(sys.implementation.version) + implementation_version = format_full_version( + sys.implementation.version) implementation_name = sys.implementation.name else: implementation_version = '0' @@ -127,11 +137,13 @@ def format_full_version(info): } return result + DEFAULT_CONTEXT = default_context() del default_context evaluator = Evaluator() + def interpret(marker, execution_context=None): """ Interpret a marker and return a result depending on environment. @@ -144,9 +156,11 @@ def interpret(marker, execution_context=None): try: expr, rest = parse_marker(marker) except Exception as e: - raise SyntaxError('Unable to interpret marker syntax: %s: %s' % (marker, e)) + raise SyntaxError('Unable to interpret marker syntax: %s: %s' % + (marker, e)) if rest and rest[0] != '#': - raise SyntaxError('unexpected trailing data in marker: %s: %s' % (marker, rest)) + raise SyntaxError('unexpected trailing data in marker: %s: %s' % + (marker, rest)) context = dict(DEFAULT_CONTEXT) if execution_context: context.update(execution_context) diff --git a/distlib/scripts.py b/distlib/scripts.py index d8fdb3a..cfa45d2 100644 --- a/distlib/scripts.py +++ b/distlib/scripts.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # -# Copyright (C) 2013-2015 Vinay Sajip. +# Copyright (C) 2013-2023 Vinay Sajip. # Licensed to the Python Software Foundation under a contributor agreement. # See LICENSE.txt and CONTRIBUTORS.txt. # @@ -65,9 +65,11 @@ def enquote_executable(executable): executable = '"%s"' % executable return executable + # Keep the old name around (for now), as there is at least one project using it! _enquote_executable = enquote_executable + class ScriptMaker(object): """ A class to copy or create scripts from source scripts or callable @@ -77,21 +79,25 @@ class ScriptMaker(object): executable = None # for shebangs - def __init__(self, source_dir, target_dir, add_launchers=True, - dry_run=False, fileop=None): + def __init__(self, + source_dir, + target_dir, + add_launchers=True, + dry_run=False, + fileop=None): self.source_dir = source_dir self.target_dir = target_dir self.add_launchers = add_launchers self.force = False self.clobber = False # It only makes sense to set mode bits on POSIX. - self.set_mode = (os.name == 'posix') or (os.name == 'java' and - os._name == 'posix') + self.set_mode = (os.name == 'posix') or (os.name == 'java' + and os._name == 'posix') self.variants = set(('', 'X.Y')) self._fileop = fileop or FileOperator(dry_run) - self._is_nt = os.name == 'nt' or ( - os.name == 'java' and os._name == 'nt') + self._is_nt = os.name == 'nt' or (os.name == 'java' + and os._name == 'nt') self.version_info = sys.version_info def _get_alternate_executable(self, executable, options): @@ -102,6 +108,7 @@ def _get_alternate_executable(self, executable, options): return executable if sys.platform.startswith('java'): # pragma: no cover + def _is_shell(self, executable): """ Determine if the specified executable is a script @@ -146,8 +153,8 @@ def _build_shebang(self, executable, post_interp): max_shebang_length = 512 else: max_shebang_length = 127 - simple_shebang = ((b' ' not in executable) and - (shebang_length <= max_shebang_length)) + simple_shebang = ((b' ' not in executable) + and (shebang_length <= max_shebang_length)) if simple_shebang: result = b'#!' + executable + post_interp + b'\n' @@ -161,23 +168,25 @@ def _get_shebang(self, encoding, post_interp=b'', options=None): enquote = True if self.executable: executable = self.executable - enquote = False # assume this will be taken care of + enquote = False # assume this will be taken care of elif not sysconfig.is_python_build(): executable = get_executable() elif in_venv(): # pragma: no cover - executable = os.path.join(sysconfig.get_path('scripts'), - 'python%s' % sysconfig.get_config_var('EXE')) + executable = os.path.join( + sysconfig.get_path('scripts'), + 'python%s' % sysconfig.get_config_var('EXE')) else: # pragma: no cover if os.name == 'nt': # for Python builds from source on Windows, no Python executables with # a version suffix are created, so we use python.exe - executable = os.path.join(sysconfig.get_config_var('BINDIR'), - 'python%s' % (sysconfig.get_config_var('EXE'))) + executable = os.path.join( + sysconfig.get_config_var('BINDIR'), + 'python%s' % (sysconfig.get_config_var('EXE'))) else: executable = os.path.join( sysconfig.get_config_var('BINDIR'), - 'python%s%s' % (sysconfig.get_config_var('VERSION'), - sysconfig.get_config_var('EXE'))) + 'python%s%s' % (sysconfig.get_config_var('VERSION'), + sysconfig.get_config_var('EXE'))) if options: executable = self._get_alternate_executable(executable, options) @@ -202,7 +211,7 @@ def _get_shebang(self, encoding, post_interp=b'', options=None): executable = executable.encode('utf-8') # in case of IronPython, play safe and enable frames support if (sys.platform == 'cli' and '-X:Frames' not in post_interp - and '-X:FullFrames' not in post_interp): # pragma: no cover + and '-X:FullFrames' not in post_interp): # pragma: no cover post_interp += b' -X:Frames' shebang = self._build_shebang(executable, post_interp) # Python parser starts to read a script using UTF-8 until @@ -213,8 +222,8 @@ def _get_shebang(self, encoding, post_interp=b'', options=None): try: shebang.decode('utf-8') except UnicodeDecodeError: # pragma: no cover - raise ValueError( - 'The shebang (%r) is not decodable from utf-8' % shebang) + raise ValueError('The shebang (%r) is not decodable from utf-8' % + shebang) # If the script is encoded to a custom encoding (use a # #coding:xxx cookie), the shebang has to be decodable from # the script encoding too. @@ -222,15 +231,16 @@ def _get_shebang(self, encoding, post_interp=b'', options=None): try: shebang.decode(encoding) except UnicodeDecodeError: # pragma: no cover - raise ValueError( - 'The shebang (%r) is not decodable ' - 'from the script encoding (%r)' % (shebang, encoding)) + raise ValueError('The shebang (%r) is not decodable ' + 'from the script encoding (%r)' % + (shebang, encoding)) return shebang def _get_script_text(self, entry): - return self.script_template % dict(module=entry.prefix, - import_name=entry.suffix.split('.')[0], - func=entry.suffix) + return self.script_template % dict( + module=entry.prefix, + import_name=entry.suffix.split('.')[0], + func=entry.suffix) manifest = _DEFAULT_MANIFEST @@ -255,7 +265,8 @@ def _write_script(self, names, shebang, script_bytes, filenames, ext): source_date_epoch = os.environ.get('SOURCE_DATE_EPOCH') if source_date_epoch: date_time = time.gmtime(int(source_date_epoch))[:6] - zinfo = ZipInfo(filename='__main__.py', date_time=date_time) + zinfo = ZipInfo(filename='__main__.py', + date_time=date_time) zf.writestr(zinfo, script_bytes) else: zf.writestr('__main__.py', script_bytes) @@ -276,7 +287,7 @@ def _write_script(self, names, shebang, script_bytes, filenames, ext): 'use .deleteme logic') dfname = '%s.deleteme' % outname if os.path.exists(dfname): - os.remove(dfname) # Not allowed to fail here + os.remove(dfname) # Not allowed to fail here os.rename(outname, dfname) # nor here self._fileop.write_binary_file(outname, script_bytes) logger.debug('Able to replace executable using ' @@ -284,9 +295,10 @@ def _write_script(self, names, shebang, script_bytes, filenames, ext): try: os.remove(dfname) except Exception: - pass # still in use - ignore error + pass # still in use - ignore error else: - if self._is_nt and not outname.endswith('.' + ext): # pragma: no cover + if self._is_nt and not outname.endswith( + '.' + ext): # pragma: no cover outname = '%s.%s' % (outname, ext) if os.path.exists(outname) and not self.clobber: logger.warning('Skipping existing file %s', outname) @@ -305,8 +317,9 @@ def get_script_filenames(self, name): if 'X' in self.variants: result.add('%s%s' % (name, self.version_info[0])) if 'X.Y' in self.variants: - result.add('%s%s%s.%s' % (name, self.variant_separator, - self.version_info[0], self.version_info[1])) + result.add('%s%s%s.%s' % + (name, self.variant_separator, self.version_info[0], + self.version_info[1])) return result def _make_script(self, entry, filenames, options=None): @@ -384,12 +397,13 @@ def dry_run(self): def dry_run(self, value): self._fileop.dry_run = value - if os.name == 'nt' or (os.name == 'java' and os._name == 'nt'): # pragma: no cover + if os.name == 'nt' or (os.name == 'java' + and os._name == 'nt'): # pragma: no cover # Executable launcher support. # Launchers are from https://bitbucket.org/vinay.sajip/simple_launcher/ def _get_launcher(self, kind): - if struct.calcsize('P') == 8: # 64-bit + if struct.calcsize('P') == 8: # 64-bit bits = '64' else: bits = '32' @@ -400,8 +414,8 @@ def _get_launcher(self, kind): distlib_package = __name__.rsplit('.', 1)[0] resource = finder(distlib_package).find(name) if not resource: - msg = ('Unable to find resource %s in package %s' % (name, - distlib_package)) + msg = ('Unable to find resource %s in package %s' % + (name, distlib_package)) raise ValueError(msg) return resource.bytes diff --git a/distlib/util.py b/distlib/util.py index 82f518c..ba58858 100644 --- a/distlib/util.py +++ b/distlib/util.py @@ -33,7 +33,7 @@ from . import DistlibException from .compat import (string_types, text_type, shutil, raw_input, StringIO, cache_from_source, urlopen, urljoin, httplib, xmlrpclib, - splittype, HTTPHandler, BaseConfigurator, valid_ident, + HTTPHandler, BaseConfigurator, valid_ident, Container, configparser, URLError, ZipFile, fsdecode, unquote, urlparse) @@ -62,6 +62,7 @@ def parse_marker(marker_string): interpreted as a literal string, and a string not contained in quotes is a variable (such as os_name). """ + def marker_var(remaining): # either identifier, or literal string m = IDENTIFIER.match(remaining) @@ -87,7 +88,8 @@ def marker_var(remaining): else: m = STRING_CHUNK.match(remaining) if not m: - raise SyntaxError('error in string literal: %s' % remaining) + raise SyntaxError('error in string literal: %s' % + remaining) parts.append(m.groups()[0]) remaining = remaining[m.end():] else: @@ -95,7 +97,7 @@ def marker_var(remaining): raise SyntaxError('unterminated string: %s' % s) parts.append(q) result = ''.join(parts) - remaining = remaining[1:].lstrip() # skip past closing quote + remaining = remaining[1:].lstrip() # skip past closing quote return result, remaining def marker_expr(remaining): @@ -208,7 +210,8 @@ def get_versions(ver_remaining): ver_remaining = ver_remaining[m.end():] m = VERSION_IDENTIFIER.match(ver_remaining) if not m: - raise SyntaxError('invalid version: %s' % ver_remaining) + raise SyntaxError('invalid version: %s' % + ver_remaining) v = m.groups()[0] versions.append((op, v)) ver_remaining = ver_remaining[m.end():] @@ -221,7 +224,8 @@ def get_versions(ver_remaining): break m = COMPARE_OP.match(ver_remaining) if not m: - raise SyntaxError('invalid constraint: %s' % ver_remaining) + raise SyntaxError('invalid constraint: %s' % + ver_remaining) if not versions: versions = None return versions, ver_remaining @@ -231,7 +235,8 @@ def get_versions(ver_remaining): else: i = remaining.find(')', 1) if i < 0: - raise SyntaxError('unterminated parenthesis: %s' % remaining) + raise SyntaxError('unterminated parenthesis: %s' % + remaining) s = remaining[1:i] remaining = remaining[i + 1:].lstrip() # As a special diversion from PEP 508, allow a version number @@ -262,9 +267,14 @@ def get_versions(ver_remaining): if not versions: rs = distname else: - rs = '%s %s' % (distname, ', '.join(['%s %s' % con for con in versions])) - return Container(name=distname, extras=extras, constraints=versions, - marker=mark_expr, url=uri, requirement=rs) + rs = '%s %s' % (distname, ', '.join( + ['%s %s' % con for con in versions])) + return Container(name=distname, + extras=extras, + constraints=versions, + marker=mark_expr, + url=uri, + requirement=rs) def get_resources_dests(resources_root, rules): @@ -304,15 +314,15 @@ def in_venv(): def get_executable(): -# The __PYVENV_LAUNCHER__ dance is apparently no longer needed, as -# changes to the stub launcher mean that sys.executable always points -# to the stub on OS X -# if sys.platform == 'darwin' and ('__PYVENV_LAUNCHER__' -# in os.environ): -# result = os.environ['__PYVENV_LAUNCHER__'] -# else: -# result = sys.executable -# return result + # The __PYVENV_LAUNCHER__ dance is apparently no longer needed, as + # changes to the stub launcher mean that sys.executable always points + # to the stub on OS X + # if sys.platform == 'darwin' and ('__PYVENV_LAUNCHER__' + # in os.environ): + # result = os.environ['__PYVENV_LAUNCHER__'] + # else: + # result = sys.executable + # return result # Avoid normcasing: see issue #143 # result = os.path.normcase(sys.executable) result = sys.executable @@ -346,6 +356,7 @@ def extract_by_key(d, keys): result[key] = d[key] return result + def read_exports(stream): if sys.version_info[0] >= 3: # needs to be a text stream @@ -388,7 +399,7 @@ def read_stream(cp, stream): s = '%s = %s' % (name, value) entry = get_export_entry(s) assert entry is not None - #entry.dist = self + # entry.dist = self entries[name] = entry return result @@ -420,6 +431,7 @@ def tempdir(): finally: shutil.rmtree(td) + @contextlib.contextmanager def chdir(d): cwd = os.getcwd() @@ -441,19 +453,21 @@ def socket_timeout(seconds=15): class cached_property(object): + def __init__(self, func): self.func = func - #for attr in ('__name__', '__module__', '__doc__'): - # setattr(self, attr, getattr(func, attr, None)) + # for attr in ('__name__', '__module__', '__doc__'): + # setattr(self, attr, getattr(func, attr, None)) def __get__(self, obj, cls=None): if obj is None: return self value = self.func(obj) object.__setattr__(obj, self.func.__name__, value) - #obj.__dict__[self.func.__name__] = value = self.func(obj) + # obj.__dict__[self.func.__name__] = value = self.func(obj) return value + def convert_path(pathname): """Return 'pathname' as a name that will work on the native filesystem. @@ -482,6 +496,7 @@ def convert_path(pathname): class FileOperator(object): + def __init__(self, dry_run=False): self.dry_run = dry_run self.ensured = set() @@ -586,7 +601,12 @@ def ensure_dir(self, path): if self.record: self.dirs_created.add(path) - def byte_compile(self, path, optimize=False, force=False, prefix=None, hashed_invalidation=False): + def byte_compile(self, + path, + optimize=False, + force=False, + prefix=None, + hashed_invalidation=False): dpath = cache_from_source(path, not optimize) logger.info('Byte-compiling %s to %s', path, dpath) if not self.dry_run: @@ -597,9 +617,12 @@ def byte_compile(self, path, optimize=False, force=False, prefix=None, hashed_in assert path.startswith(prefix) diagpath = path[len(prefix):] compile_kwargs = {} - if hashed_invalidation and hasattr(py_compile, 'PycInvalidationMode'): - compile_kwargs['invalidation_mode'] = py_compile.PycInvalidationMode.CHECKED_HASH - py_compile.compile(path, dpath, diagpath, True, **compile_kwargs) # raise error + if hashed_invalidation and hasattr(py_compile, + 'PycInvalidationMode'): + compile_kwargs[ + 'invalidation_mode'] = py_compile.PycInvalidationMode.CHECKED_HASH + py_compile.compile(path, dpath, diagpath, True, + **compile_kwargs) # raise error self.record_as_written(dpath) return dpath @@ -661,9 +684,10 @@ def rollback(self): assert flist == ['__pycache__'] sd = os.path.join(d, flist[0]) os.rmdir(sd) - os.rmdir(d) # should fail if non-empty + os.rmdir(d) # should fail if non-empty self._init_record() + def resolve(module_name, dotted_path): if module_name in sys.modules: mod = sys.modules[module_name] @@ -680,6 +704,7 @@ def resolve(module_name, dotted_path): class ExportEntry(object): + def __init__(self, name, prefix, suffix, flags): self.name = name self.prefix = prefix @@ -698,20 +723,21 @@ def __eq__(self, other): if not isinstance(other, ExportEntry): result = False else: - result = (self.name == other.name and - self.prefix == other.prefix and - self.suffix == other.suffix and - self.flags == other.flags) + result = (self.name == other.name and self.prefix == other.prefix + and self.suffix == other.suffix + and self.flags == other.flags) return result __hash__ = object.__hash__ -ENTRY_RE = re.compile(r'''(?P([^\[]\S*)) +ENTRY_RE = re.compile( + r'''(?P([^\[]\S*)) \s*=\s*(?P(\w+)([:\.]\w+)*) \s*(\[\s*(?P[\w-]+(=\w+)?(,\s*\w+(=\w+)?)*)\s*\])? ''', re.VERBOSE) + def get_export_entry(specification): m = ENTRY_RE.search(specification) if not m: @@ -827,6 +853,7 @@ def get_process_umask(): os.umask(result) return result + def is_string_sequence(seq): result = True i = None @@ -837,8 +864,10 @@ def is_string_sequence(seq): assert i is not None return result -PROJECT_NAME_AND_VERSION = re.compile('([a-z0-9_]+([.-][a-z_][a-z0-9_]*)*)-' - '([a-z0-9_.+-]+)', re.I) + +PROJECT_NAME_AND_VERSION = re.compile( + '([a-z0-9_]+([.-][a-z_][a-z0-9_]*)*)-' + '([a-z0-9_.+-]+)', re.I) PYTHON_VERSION = re.compile(r'-py(\d\.?\d?)') @@ -866,10 +895,12 @@ def split_filename(filename, project_name=None): result = m.group(1), m.group(3), pyver return result + # Allow spaces in name because of legacy dists like "Twisted Core" NAME_VERSION_RE = re.compile(r'(?P[\w .-]+)\s*' r'\(\s*(?P[^\s)]+)\)$') + def parse_name_and_version(p): """ A utility method used to get name and version from a string. @@ -885,6 +916,7 @@ def parse_name_and_version(p): d = m.groupdict() return d['name'].strip().lower(), d['ver'] + def get_extras(requested, available): result = set() requested = set(requested or []) @@ -906,10 +938,13 @@ def get_extras(requested, available): logger.warning('undeclared extra: %s' % r) result.add(r) return result + + # # Extended metadata functionality # + def _get_external_data(url): result = {} try: @@ -923,21 +958,24 @@ def _get_external_data(url): logger.debug('Unexpected response for JSON request: %s', ct) else: reader = codecs.getreader('utf-8')(resp) - #data = reader.read().decode('utf-8') - #result = json.loads(data) + # data = reader.read().decode('utf-8') + # result = json.loads(data) result = json.load(reader) except Exception as e: logger.exception('Failed to get external data for %s: %s', url, e) return result + _external_data_base_url = 'https://www.red-dove.com/pypi/projects/' + def get_project_data(name): url = '%s/%s/project.json' % (name[0].upper(), name) url = urljoin(_external_data_base_url, url) result = _get_external_data(url) return result + def get_package_data(name, version): url = '%s/%s/package-%s.json' % (name[0].upper(), name, version) url = urljoin(_external_data_base_url, url) @@ -992,6 +1030,7 @@ class EventMixin(object): """ A very simple publish/subscribe system. """ + def __init__(self): self._subscribers = {} @@ -1053,18 +1092,20 @@ def publish(self, event, *args, **kwargs): logger.exception('Exception during event publication') value = None result.append(value) - logger.debug('publish %s: args = %s, kwargs = %s, result = %s', - event, args, kwargs, result) + logger.debug('publish %s: args = %s, kwargs = %s, result = %s', event, + args, kwargs, result) return result + # # Simple sequencing # class Sequencer(object): + def __init__(self): self._preds = {} self._succs = {} - self._nodes = set() # nodes with no preds/succs + self._nodes = set() # nodes with no preds/succs def add_node(self, node): self._nodes.add(node) @@ -1104,8 +1145,8 @@ def remove(self, pred, succ): raise ValueError('%r not a successor of %r' % (succ, pred)) def is_step(self, step): - return (step in self._preds or step in self._succs or - step in self._nodes) + return (step in self._preds or step in self._succs + or step in self._nodes) def get_steps(self, final): if not self.is_step(final): @@ -1134,7 +1175,7 @@ def get_steps(self, final): @property def strong_connections(self): - #http://en.wikipedia.org/wiki/Tarjan%27s_strongly_connected_components_algorithm + # http://en.wikipedia.org/wiki/Tarjan%27s_strongly_connected_components_algorithm index_counter = [0] stack = [] lowlinks = {} @@ -1159,11 +1200,11 @@ def strongconnect(node): if successor not in lowlinks: # Successor has not yet been visited strongconnect(successor) - lowlinks[node] = min(lowlinks[node],lowlinks[successor]) + lowlinks[node] = min(lowlinks[node], lowlinks[successor]) elif successor in stack: # the successor is in the stack and hence in the current # strongly connected component (SCC) - lowlinks[node] = min(lowlinks[node],index[successor]) + lowlinks[node] = min(lowlinks[node], index[successor]) # If `node` is a root node, pop the stack and generate an SCC if lowlinks[node] == index[node]: @@ -1172,7 +1213,8 @@ def strongconnect(node): while True: successor = stack.pop() connected_component.append(successor) - if successor == node: break + if successor == node: + break component = tuple(connected_component) # storing the result result.append(component) @@ -1195,12 +1237,14 @@ def dot(self): result.append('}') return '\n'.join(result) + # # Unarchiving functionality for zip, tar, tgz, tbz, whl # -ARCHIVE_EXTENSIONS = ('.tar.gz', '.tar.bz2', '.tar', '.zip', - '.tgz', '.tbz', '.whl') +ARCHIVE_EXTENSIONS = ('.tar.gz', '.tar.bz2', '.tar', '.zip', '.tgz', '.tbz', + '.whl') + def unarchive(archive_filename, dest_dir, format=None, check=True): @@ -1260,6 +1304,7 @@ def extraction_filter(member, path): return tarfile.tar_filter(member, path) except tarfile.FilterError as exc: raise ValueError(str(exc)) + archive.extraction_filter = extraction_filter archive.extractall(dest_dir) @@ -1282,11 +1327,12 @@ def zip_dir(directory): zf.write(full, dest) return result + # # Simple progress bar # -UNITS = ('', 'K', 'M', 'G','T','P') +UNITS = ('', 'K', 'M', 'G', 'T', 'P') class Progress(object): @@ -1341,8 +1387,8 @@ def percentage(self): def format_duration(self, duration): if (duration <= 0) and self.max is None or self.cur == self.min: result = '??:??:??' - #elif duration < 1: - # result = '--:--:--' + # elif duration < 1: + # result = '--:--:--' else: result = time.strftime('%H:%M:%S', time.gmtime(duration)) return result @@ -1352,7 +1398,7 @@ def ETA(self): if self.done: prefix = 'Done' t = self.elapsed - #import pdb; pdb.set_trace() + # import pdb; pdb.set_trace() else: prefix = 'ETA ' if self.max is None: @@ -1360,7 +1406,7 @@ def ETA(self): elif self.elapsed == 0 or (self.cur == self.min): t = 0 else: - #import pdb; pdb.set_trace() + # import pdb; pdb.set_trace() t = float(self.max - self.min) t /= self.cur - self.min t = (t - 1) * self.elapsed @@ -1378,6 +1424,7 @@ def speed(self): result /= 1000.0 return '%d %sB/s' % (result, unit) + # # Glob functionality # @@ -1425,22 +1472,23 @@ def _iglob(path_glob): for fn in _iglob(os.path.join(path, radical)): yield fn + if ssl: from .compat import (HTTPSHandler as BaseHTTPSHandler, match_hostname, CertificateError) - -# -# HTTPSConnection which verifies certificates/matches domains -# + # + # HTTPSConnection which verifies certificates/matches domains + # class HTTPSConnection(httplib.HTTPSConnection): - ca_certs = None # set this to the path to the certs file (.pem) - check_domain = True # only used if ca_certs is not None + ca_certs = None # set this to the path to the certs file (.pem) + check_domain = True # only used if ca_certs is not None # noinspection PyPropertyAccess def connect(self): - sock = socket.create_connection((self.host, self.port), self.timeout) + sock = socket.create_connection((self.host, self.port), + self.timeout) if getattr(self, '_tunnel_host', False): self.sock = sock self._tunnel() @@ -1468,6 +1516,7 @@ def connect(self): raise class HTTPSHandler(BaseHTTPSHandler): + def __init__(self, ca_certs, check_domain=True): BaseHTTPSHandler.__init__(self) self.ca_certs = ca_certs @@ -1494,8 +1543,9 @@ def https_open(self, req): return self.do_open(self._conn_maker, req) except URLError as e: if 'certificate verify failed' in str(e.reason): - raise CertificateError('Unable to verify server certificate ' - 'for %s' % req.host) + raise CertificateError( + 'Unable to verify server certificate ' + 'for %s' % req.host) else: raise @@ -1509,14 +1559,18 @@ def https_open(self, req): # handler for HTTP itself. # class HTTPSOnlyHandler(HTTPSHandler, HTTPHandler): + def http_open(self, req): - raise URLError('Unexpected HTTP request on what should be a secure ' - 'connection: %s' % req) + raise URLError( + 'Unexpected HTTP request on what should be a secure ' + 'connection: %s' % req) + # # XML-RPC with timeouts # class Transport(xmlrpclib.Transport): + def __init__(self, timeout, use_datetime=0): self.timeout = timeout xmlrpclib.Transport.__init__(self, use_datetime) @@ -1528,8 +1582,11 @@ def make_connection(self, host): self._connection = host, httplib.HTTPConnection(h) return self._connection[1] + if ssl: + class SafeTransport(xmlrpclib.SafeTransport): + def __init__(self, timeout, use_datetime=0): self.timeout = timeout xmlrpclib.SafeTransport.__init__(self, use_datetime) @@ -1541,12 +1598,13 @@ def make_connection(self, host): kwargs['timeout'] = self.timeout if not self._connection or host != self._connection[0]: self._extra_headers = eh - self._connection = host, httplib.HTTPSConnection(h, None, - **kwargs) + self._connection = host, httplib.HTTPSConnection( + h, None, **kwargs) return self._connection[1] class ServerProxy(xmlrpclib.ServerProxy): + def __init__(self, uri, **kwargs): self.timeout = timeout = kwargs.pop('timeout', None) # The above classes only come into play if a timeout @@ -1563,11 +1621,13 @@ def __init__(self, uri, **kwargs): self.transport = t xmlrpclib.ServerProxy.__init__(self, uri, **kwargs) + # # CSV functionality. This is provided because on 2.x, the csv module can't # handle Unicode. However, we need to deal with Unicode in e.g. RECORD files. # + def _csv_open(fn, mode, **kwargs): if sys.version_info[0] < 3: mode += 'b' @@ -1581,9 +1641,9 @@ def _csv_open(fn, mode, **kwargs): class CSVBase(object): defaults = { - 'delimiter': str(','), # The strs are used because we need native - 'quotechar': str('"'), # str in the csv API (2.x won't take - 'lineterminator': str('\n') # Unicode) + 'delimiter': str(','), # The strs are used because we need native + 'quotechar': str('"'), # str in the csv API (2.x won't take + 'lineterminator': str('\n') # Unicode) } def __enter__(self): @@ -1594,6 +1654,7 @@ def __exit__(self, *exc_info): class CSVReader(CSVBase): + def __init__(self, **kwargs): if 'stream' in kwargs: stream = kwargs['stream'] @@ -1618,7 +1679,9 @@ def next(self): __next__ = next + class CSVWriter(CSVBase): + def __init__(self, fn, **kwargs): self.stream = _csv_open(fn, 'w') self.writer = csv.writer(self.stream, **self.defaults) @@ -1633,10 +1696,12 @@ def writerow(self, row): row = r self.writer.writerow(row) + # # Configurator functionality # + class Configurator(BaseConfigurator): value_converters = dict(BaseConfigurator.value_converters) @@ -1647,6 +1712,7 @@ def __init__(self, config, base=None): self.base = base or os.getcwd() def configure_custom(self, config): + def convert(o): if isinstance(o, (list, tuple)): result = type(o)([convert(i) for i in o]) @@ -1696,6 +1762,7 @@ class SubprocessMixin(object): """ Mixin for running subprocesses and capturing their output """ + def __init__(self, verbose=False, progress=None): self.verbose = verbose self.progress = progress @@ -1722,8 +1789,10 @@ def reader(self, stream, context): stream.close() def run_command(self, cmd, **kwargs): - p = subprocess.Popen(cmd, stdout=subprocess.PIPE, - stderr=subprocess.PIPE, **kwargs) + p = subprocess.Popen(cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + **kwargs) t1 = threading.Thread(target=self.reader, args=(p.stdout, 'stdout')) t1.start() t2 = threading.Thread(target=self.reader, args=(p.stderr, 'stderr')) @@ -1743,15 +1812,17 @@ def normalize_name(name): # https://www.python.org/dev/peps/pep-0503/#normalized-names return re.sub('[-_.]+', '-', name).lower() + # def _get_pypirc_command(): - # """ - # Get the distutils command for interacting with PyPI configurations. - # :return: the command. - # """ - # from distutils.core import Distribution - # from distutils.config import PyPIRCCommand - # d = Distribution() - # return PyPIRCCommand(d) +# """ +# Get the distutils command for interacting with PyPI configurations. +# :return: the command. +# """ +# from distutils.core import Distribution +# from distutils.config import PyPIRCCommand +# d = Distribution() +# return PyPIRCCommand(d) + class PyPIRCFile(object): @@ -1776,9 +1847,10 @@ def read(self): if 'distutils' in sections: # let's get the list of servers index_servers = config.get('distutils', 'index-servers') - _servers = [server.strip() for server in - index_servers.split('\n') - if server.strip() != ''] + _servers = [ + server.strip() for server in index_servers.split('\n') + if server.strip() != '' + ] if _servers == []: # nothing set, let's try to get the default pypi if 'pypi' in sections: @@ -1789,7 +1861,8 @@ def read(self): result['username'] = config.get(server, 'username') # optional params - for key, default in (('repository', self.DEFAULT_REPOSITORY), + for key, default in (('repository', + self.DEFAULT_REPOSITORY), ('realm', self.DEFAULT_REALM), ('password', None)): if config.has_option(server, key): @@ -1800,11 +1873,11 @@ def read(self): # work around people having "repository" for the "pypi" # section of their config set to the HTTP (rather than # HTTPS) URL - if (server == 'pypi' and - repository in (self.DEFAULT_REPOSITORY, 'pypi')): + if (server == 'pypi' and repository + in (self.DEFAULT_REPOSITORY, 'pypi')): result['repository'] = self.DEFAULT_REPOSITORY - elif (result['server'] != repository and - result['repository'] != repository): + elif (result['server'] != repository + and result['repository'] != repository): result = {} elif 'server-login' in sections: # old format @@ -1834,20 +1907,24 @@ def update(self, username, password): with open(fn, 'w') as f: config.write(f) + def _load_pypirc(index): """ Read the PyPI access configuration as supported by distutils. """ return PyPIRCFile(url=index.url).read() + def _store_pypirc(index): PyPIRCFile().update(index.username, index.password) + # # get_platform()/get_host_platform() copied from Python 3.10.a0 source, with some minor # tweaks # + def get_host_platform(): """Return a string that identifies the current platform. This is used mainly to distinguish platform-specific build directories and platform-specific built @@ -1899,16 +1976,16 @@ def get_host_platform(): # At least on Linux/Intel, 'machine' is the processor -- # i386, etc. # XXX what about Alpha, SPARC, etc? - return "%s-%s" % (osname, machine) + return "%s-%s" % (osname, machine) elif osname[:5] == 'sunos': - if release[0] >= '5': # SunOS 5 == Solaris 2 + if release[0] >= '5': # SunOS 5 == Solaris 2 osname = 'solaris' release = '%d.%s' % (int(release[0]) - 3, release[2:]) # We can't use 'platform.architecture()[0]' because a # bootstrap problem. We use a dict to get an error # if some suspicious happens. - bitness = {2147483647:'32bit', 9223372036854775807:'64bit'} + bitness = {2147483647: '32bit', 9223372036854775807: '64bit'} machine += '.%s' % bitness[sys.maxsize] # fall through to standard osname-release-machine representation elif osname[:3] == 'aix': @@ -1916,7 +1993,7 @@ def get_host_platform(): return aix_platform() elif osname[:6] == 'cygwin': osname = 'cygwin' - rel_re = re.compile (r'[\d.]+', re.ASCII) + rel_re = re.compile(r'[\d.]+', re.ASCII) m = rel_re.match(release) if m: release = m.group() @@ -1927,16 +2004,15 @@ def get_host_platform(): except ImportError: import sysconfig osname, release, machine = _osx_support.get_platform_osx( - sysconfig.get_config_vars(), - osname, release, machine) + sysconfig.get_config_vars(), osname, release, machine) return '%s-%s-%s' % (osname, release, machine) _TARGET_TO_PLAT = { - 'x86' : 'win32', - 'x64' : 'win-amd64', - 'arm' : 'win-arm32', + 'x86': 'win32', + 'x64': 'win-amd64', + 'arm': 'win-arm32', } diff --git a/distlib/version.py b/distlib/version.py index ddb8db3..14171ac 100644 --- a/distlib/version.py +++ b/distlib/version.py @@ -247,7 +247,6 @@ def _pep_440_key(s): if not dev: dev = ('final',) - #print('%s -> %s' % (s, m.groups())) return epoch, nums, pre, post, dev, local @@ -387,6 +386,7 @@ def _match_compatible(self, version, constraint, prefix): pfx = '.'.join([str(i) for i in release_clause]) return _match_prefix(version, pfx) + _REPLACEMENTS = ( (re.compile('[.+-]$'), ''), # remove trailing puncts (re.compile(r'^[.](\d)'), r'0.\1'), # .N -> 0.N at start @@ -397,7 +397,7 @@ def _match_compatible(self, version, constraint, prefix): (re.compile('[.]{2,}'), '.'), # multiple runs of '.' (re.compile(r'\b(alfa|apha)\b'), 'alpha'), # misspelt alpha (re.compile(r'\b(pre-alpha|prealpha)\b'), - 'pre.alpha'), # standardise + 'pre.alpha'), # standardise (re.compile(r'\(beta\)$'), 'beta'), # remove parentheses ) @@ -425,7 +425,7 @@ def _suggest_semantic_version(s): # Now look for numeric prefix, and separate it out from # the rest. - #import pdb; pdb.set_trace() + # import pdb; pdb.set_trace() m = _NUMERIC_PREFIX.match(result) if not m: prefix = '0.0.0' @@ -443,7 +443,7 @@ def _suggest_semantic_version(s): prefix = '.'.join([str(i) for i in prefix]) suffix = suffix.strip() if suffix: - #import pdb; pdb.set_trace() + # import pdb; pdb.set_trace() # massage the suffix. for pat, repl in _SUFFIX_REPLACEMENTS: suffix = pat.sub(repl, suffix) @@ -513,7 +513,7 @@ def _suggest_normalized_version(s): rs = rs[1:] # Clean leading '0's on numbers. - #TODO: unintended side-effect on, e.g., "2003.05.09" + # TODO: unintended side-effect on, e.g., "2003.05.09" # PyPI stats: 77 (~2%) better rs = re.sub(r"\b0+(\d+)(?!\d)", r"\1", rs) @@ -572,6 +572,7 @@ def _suggest_normalized_version(s): # Legacy version processing (distribute-compatible) # + _VERSION_PART = re.compile(r'([a-z]+|\d+|[\.-])', re.I) _VERSION_REPLACE = { 'pre': 'c', @@ -619,7 +620,7 @@ def is_prerelease(self): result = False for x in self._parts: if (isinstance(x, string_types) and x.startswith('*') and - x < '*final'): + x < '*final'): result = True break return result @@ -650,6 +651,7 @@ def _match_compatible(self, version, constraint, prefix): # Semantic versioning # + _SEMVER_RE = re.compile(r'^(\d+)\.(\d+)\.(\d+)' r'(-[a-z0-9]+(\.[a-z0-9-]+)*)?' r'(\+[a-z0-9]+(\.[a-z0-9-]+)*)?$', re.I) @@ -731,6 +733,7 @@ def suggest(self, s): result = self.suggester(s) return result + _SCHEMES = { 'normalized': VersionScheme(_normalized_key, NormalizedMatcher, _suggest_normalized_version), diff --git a/distlib/wheel.py b/distlib/wheel.py index 028c2d9..4a5a30e 100644 --- a/distlib/wheel.py +++ b/distlib/wheel.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # -# Copyright (C) 2013-2020 Vinay Sajip. +# Copyright (C) 2013-2023 Vinay Sajip. # Licensed to the Python Software Foundation under a contributor agreement. # See LICENSE.txt and CONTRIBUTORS.txt. # @@ -24,8 +24,7 @@ from . import __version__, DistlibException from .compat import sysconfig, ZipFile, fsdecode, text_type, filter from .database import InstalledDistribution -from .metadata import (Metadata, METADATA_FILENAME, WHEEL_METADATA_FILENAME, - LEGACY_METADATA_FILENAME) +from .metadata import Metadata, WHEEL_METADATA_FILENAME, LEGACY_METADATA_FILENAME from .util import (FileOperator, convert_path, CSVReader, CSVWriter, Cache, cached_property, get_cache_base, read_exports, tempdir, get_platform) @@ -33,7 +32,7 @@ logger = logging.getLogger(__name__) -cache = None # created when needed +cache = None # created when needed if hasattr(sys, 'pypy_version_info'): # pragma: no cover IMP_PREFIX = 'pp' @@ -45,7 +44,7 @@ IMP_PREFIX = 'cp' VER_SUFFIX = sysconfig.get_config_var('py_version_nodot') -if not VER_SUFFIX: # pragma: no cover +if not VER_SUFFIX: # pragma: no cover VER_SUFFIX = '%s%s' % sys.version_info[:2] PYVER = 'py' + VER_SUFFIX IMPVER = IMP_PREFIX + VER_SUFFIX @@ -56,6 +55,7 @@ if ABI and ABI.startswith('cpython-'): ABI = ABI.replace('cpython-', 'cp').split('-')[0] else: + def _derive_abi(): parts = ['cp', VER_SUFFIX] if sysconfig.get_config_var('Py_DEBUG'): @@ -73,10 +73,12 @@ def _derive_abi(): if us == 4 or (us is None and sys.maxunicode == 0x10FFFF): parts.append('u') return ''.join(parts) + ABI = _derive_abi() del _derive_abi -FILENAME_RE = re.compile(r''' +FILENAME_RE = re.compile( + r''' (?P[^-]+) -(?P\d+[^-]*) (-(?P\d+[^-]*))? @@ -86,7 +88,8 @@ def _derive_abi(): \.whl$ ''', re.IGNORECASE | re.VERBOSE) -NAME_VERSION_RE = re.compile(r''' +NAME_VERSION_RE = re.compile( + r''' (?P[^-]+) -(?P\d+[^-]*) (-(?P\d+[^-]*))?$ @@ -109,12 +112,14 @@ def _derive_abi(): import importlib.machinery import importlib.util + def _get_suffixes(): if imp: return [s[0] for s in imp.get_suffixes()] else: return importlib.machinery.EXTENSION_SUFFIXES + def _load_dynamic(name, path): # https://docs.python.org/3/library/importlib.html#importing-a-source-file-directly if imp: @@ -126,7 +131,9 @@ def _load_dynamic(name, path): spec.loader.exec_module(module) return module + class Mounter(object): + def __init__(self): self.impure_wheels = {} self.libs = {} @@ -161,6 +168,7 @@ def load_module(self, fullname): result.__package__ = parts[0] return result + _hook = Mounter() @@ -227,8 +235,8 @@ def filename(self): arch = '.'.join(self.arch) # replace - with _ as a local version separator version = self.version.replace('-', '_') - return '%s-%s%s-%s-%s-%s.whl' % (self.name, version, buildver, - pyver, abi, arch) + return '%s-%s%s-%s-%s-%s.whl' % (self.name, version, buildver, pyver, + abi, arch) @property def exists(self): @@ -249,14 +257,14 @@ def metadata(self): info_dir = '%s.dist-info' % name_ver wrapper = codecs.getreader('utf-8') with ZipFile(pathname, 'r') as zf: - wheel_metadata = self.get_wheel_metadata(zf) - wv = wheel_metadata['Wheel-Version'].split('.', 1) - file_version = tuple([int(i) for i in wv]) + self.get_wheel_metadata(zf) + # wv = wheel_metadata['Wheel-Version'].split('.', 1) + # file_version = tuple([int(i) for i in wv]) # if file_version < (1, 1): - # fns = [WHEEL_METADATA_FILENAME, METADATA_FILENAME, - # LEGACY_METADATA_FILENAME] + # fns = [WHEEL_METADATA_FILENAME, METADATA_FILENAME, + # LEGACY_METADATA_FILENAME] # else: - # fns = [WHEEL_METADATA_FILENAME, METADATA_FILENAME] + # fns = [WHEEL_METADATA_FILENAME, METADATA_FILENAME] fns = [WHEEL_METADATA_FILENAME, LEGACY_METADATA_FILENAME] result = None for fn in fns: @@ -326,13 +334,14 @@ def get_hash(self, data, hash_kind=None): try: hasher = getattr(hashlib, hash_kind) except AttributeError: - raise DistlibException('Unsupported hash algorithm: %r' % hash_kind) + raise DistlibException('Unsupported hash algorithm: %r' % + hash_kind) result = hasher(data).digest() result = base64.urlsafe_b64encode(result).rstrip(b'=').decode('ascii') return hash_kind, result def write_record(self, records, record_path, archive_record_path): - records = list(records) # make a copy, as mutated + records = list(records) # make a copy, as mutated records.append((archive_record_path, '', '')) with CSVWriter(record_path) as writer: for row in records: @@ -341,7 +350,7 @@ def write_record(self, records, record_path, archive_record_path): def write_records(self, info, libdir, archive_paths): records = [] distinfo, info_dir = info - hasher = getattr(hashlib, self.hash_kind) + # hasher = getattr(hashlib, self.hash_kind) for ap, p in archive_paths: with open(p, 'rb') as f: data = f.read() @@ -466,6 +475,7 @@ def sorter(t): if '.dist-info' in ap: n += 10000 return (n, ap) + archive_paths = sorted(archive_paths, key=sorter) # Now, at last, RECORD. @@ -512,7 +522,8 @@ def install(self, paths, maker, **kwargs): dry_run = maker.dry_run warner = kwargs.get('warner') lib_only = kwargs.get('lib_only', False) - bc_hashed_invalidation = kwargs.get('bytecode_hashed_invalidation', False) + bc_hashed_invalidation = kwargs.get('bytecode_hashed_invalidation', + False) pathname = os.path.join(self.dirname, self.filename) name_ver = '%s-%s' % (self.name, self.version) @@ -553,11 +564,11 @@ def install(self, paths, maker, **kwargs): # make a new instance rather than a copy of maker's, # as we mutate it fileop = FileOperator(dry_run=dry_run) - fileop.record = True # so we can rollback if needed + fileop.record = True # so we can rollback if needed - bc = not sys.dont_write_bytecode # Double negatives. Lovely! + bc = not sys.dont_write_bytecode # Double negatives. Lovely! - outfiles = [] # for RECORD writing + outfiles = [] # for RECORD writing # for script copying/shebang processing workdir = tempfile.mkdtemp() @@ -611,7 +622,8 @@ def install(self, paths, maker, **kwargs): # So ... manually preserve permission bits as given in zinfo if os.name == 'posix': # just set the normal permission bits - os.chmod(outfile, (zinfo.external_attr >> 16) & 0x1FF) + os.chmod(outfile, + (zinfo.external_attr >> 16) & 0x1FF) outfiles.append(outfile) # Double check the digest of the written file if not dry_run and row[1]: @@ -624,8 +636,9 @@ def install(self, paths, maker, **kwargs): '%s' % outfile) if bc and outfile.endswith('.py'): try: - pyc = fileop.byte_compile(outfile, - hashed_invalidation=bc_hashed_invalidation) + pyc = fileop.byte_compile( + outfile, + hashed_invalidation=bc_hashed_invalidation) outfiles.append(pyc) except Exception: # Don't give up if byte-compilation fails, @@ -700,7 +713,7 @@ def install(self, paths, maker, **kwargs): fileop.set_executable_mode(filenames) if gui_scripts: - options = {'gui': True } + options = {'gui': True} for k, v in gui_scripts.items(): script = '%s = %s' % (k, v) filenames = maker.make(script, options) @@ -710,7 +723,7 @@ def install(self, paths, maker, **kwargs): dist = InstalledDistribution(p) # Write SHARED - paths = dict(paths) # don't change passed in dict + paths = dict(paths) # don't change passed in dict del paths['purelib'] del paths['platlib'] paths['lib'] = libdir @@ -761,7 +774,8 @@ def _get_extensions(self): extract = True else: file_time = os.stat(dest).st_mtime - file_time = datetime.datetime.fromtimestamp(file_time) + file_time = datetime.datetime.fromtimestamp( + file_time) info = zf.getinfo(relpath) wheel_time = datetime.datetime(*info.date_time) extract = wheel_time > file_time @@ -782,7 +796,7 @@ def is_mountable(self): """ Determine if a wheel is asserted as mountable by its metadata. """ - return True # for now - metadata details TBD + return True # for now - metadata details TBD def mount(self, append=False): pathname = os.path.abspath(os.path.join(self.dirname, self.filename)) @@ -820,10 +834,10 @@ def unmount(self): def verify(self): pathname = os.path.join(self.dirname, self.filename) name_ver = '%s-%s' % (self.name, self.version) - data_dir = '%s.data' % name_ver + # data_dir = '%s.data' % name_ver info_dir = '%s.dist-info' % name_ver - metadata_name = posixpath.join(info_dir, LEGACY_METADATA_FILENAME) + # metadata_name = posixpath.join(info_dir, LEGACY_METADATA_FILENAME) wheel_metadata_name = posixpath.join(info_dir, 'WHEEL') record_name = posixpath.join(info_dir, 'RECORD') @@ -832,9 +846,9 @@ def verify(self): with ZipFile(pathname, 'r') as zf: with zf.open(wheel_metadata_name) as bwf: wf = wrapper(bwf) - message = message_from_file(wf) - wv = message['Wheel-Version'].split('.', 1) - file_version = tuple([int(i) for i in wv]) + message_from_file(wf) + # wv = message['Wheel-Version'].split('.', 1) + # file_version = tuple([int(i) for i in wv]) # TODO version verification records = {} @@ -903,25 +917,25 @@ def get_version(path_map, info_dir): def update_version(version, path): updated = None try: - v = NormalizedVersion(version) + NormalizedVersion(version) i = version.find('-') if i < 0: updated = '%s+1' % version else: parts = [int(s) for s in version[i + 1:].split('.')] parts[-1] += 1 - updated = '%s+%s' % (version[:i], - '.'.join(str(i) for i in parts)) + updated = '%s+%s' % (version[:i], '.'.join( + str(i) for i in parts)) except UnsupportedVersionError: - logger.debug('Cannot update non-compliant (PEP-440) ' - 'version %r', version) + logger.debug( + 'Cannot update non-compliant (PEP-440) ' + 'version %r', version) if updated: md = Metadata(path=path) md.version = updated legacy = path.endswith(LEGACY_METADATA_FILENAME) md.write(path=path, legacy=legacy) - logger.debug('Version updated from %r to %r', version, - updated) + logger.debug('Version updated from %r to %r', version, updated) pathname = os.path.join(self.dirname, self.filename) name_ver = '%s-%s' % (self.name, self.version) @@ -963,7 +977,8 @@ def update_version(version, path): os.close(fd) else: if not os.path.isdir(dest_dir): - raise DistlibException('Not a directory: %r' % dest_dir) + raise DistlibException('Not a directory: %r' % + dest_dir) newpath = os.path.join(dest_dir, self.filename) archive_paths = list(path_map.items()) distinfo = os.path.join(workdir, info_dir) @@ -974,6 +989,7 @@ def update_version(version, path): shutil.copyfile(newpath, pathname) return modified + def _get_glibc_version(): import platform ver = platform.libc_ver() @@ -984,13 +1000,14 @@ def _get_glibc_version(): result = tuple(result) return result + def compatible_tags(): """ Return (pyver, abi, arch) tuples compatible with this Python. """ versions = [VER_SUFFIX] major = VER_SUFFIX[0] - for minor in range(sys.version_info[1] - 1, - 1, -1): + for minor in range(sys.version_info[1] - 1, -1, -1): versions.append(''.join([major, str(minor)])) abis = [] @@ -1023,7 +1040,7 @@ def compatible_tags(): while minor >= 0: for match in matches: s = '%s_%s_%s_%s' % (name, major, minor, match) - if s != ARCH: # already there + if s != ARCH: # already there arches.append(s) minor -= 1 @@ -1045,9 +1062,9 @@ def compatible_tags(): if parts >= (2, 17): result.append((''.join((IMP_PREFIX, versions[0])), abi, 'manylinux2014_%s' % arch)) - result.append((''.join((IMP_PREFIX, versions[0])), abi, - 'manylinux_%s_%s_%s' % (parts[0], parts[1], - arch))) + result.append( + (''.join((IMP_PREFIX, versions[0])), abi, + 'manylinux_%s_%s_%s' % (parts[0], parts[1], arch))) # where no ABI / arch dependency, but IMP_PREFIX dependency for i, version in enumerate(versions): @@ -1071,7 +1088,7 @@ def compatible_tags(): def is_compatible(wheel, tags=None): if not isinstance(wheel, Wheel): - wheel = Wheel(wheel) # assume it's a filename + wheel = Wheel(wheel) # assume it's a filename result = False if tags is None: tags = COMPATIBLE_TAGS