diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index a343d1a99a1..9346c531253 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -25,7 +25,6 @@ repos: ^src/pip/_internal/commands| ^src/pip/_internal/index| ^src/pip/_internal/models| - ^src/pip/_internal/network| ^src/pip/_internal/operations| ^src/pip/_internal/req| ^src/pip/_internal/vcs| diff --git a/src/pip/_internal/network/auth.py b/src/pip/_internal/network/auth.py index 370856645f8..74d225472f6 100644 --- a/src/pip/_internal/network/auth.py +++ b/src/pip/_internal/network/auth.py @@ -31,7 +31,8 @@ keyring = None except Exception as exc: logger.warning( - "Keyring is skipped due to an exception: %s", str(exc), + "Keyring is skipped due to an exception: %s", + str(exc), ) keyring = None @@ -62,14 +63,14 @@ def get_keyring_auth(url: Optional[str], username: Optional[str]) -> Optional[Au except Exception as exc: logger.warning( - "Keyring is skipped due to an exception: %s", str(exc), + "Keyring is skipped due to an exception: %s", + str(exc), ) keyring = None return None class MultiDomainBasicAuth(AuthBase): - def __init__( self, prompting: bool = True, index_urls: Optional[List[str]] = None ) -> None: @@ -105,8 +106,12 @@ def _get_index_url(self, url: str) -> Optional[str]: return u return None - def _get_new_credentials(self, original_url: str, allow_netrc: bool = True, - allow_keyring: bool = False) -> AuthInfo: + def _get_new_credentials( + self, + original_url: str, + allow_netrc: bool = True, + allow_keyring: bool = False, + ) -> AuthInfo: """Find and return credentials for the specified URL.""" # Split the credentials and netloc from the url. url, netloc, url_user_password = split_auth_netloc_from_url( @@ -145,10 +150,12 @@ def _get_new_credentials(self, original_url: str, allow_netrc: bool = True, # If we don't have a password and keyring is available, use it. if allow_keyring: # The index url is more specific than the netloc, so try it first + # fmt: off kr_auth = ( get_keyring_auth(index_url, username) or get_keyring_auth(netloc, username) ) + # fmt: on if kr_auth: logger.debug("Found credentials in keyring for %s", netloc) return kr_auth @@ -189,9 +196,9 @@ def _get_url_and_credentials( assert ( # Credentials were found - (username is not None and password is not None) or + (username is not None and password is not None) # Credentials were not found - (username is None and password is None) + or (username is None and password is None) ), f"Could not load credentials from url: {original_url}" return url, username, password @@ -244,9 +251,11 @@ def handle_401(self, resp: Response, **kwargs: Any) -> Response: parsed = urllib.parse.urlparse(resp.url) # Query the keyring for credentials: - username, password = self._get_new_credentials(resp.url, - allow_netrc=False, - allow_keyring=True) + username, password = self._get_new_credentials( + resp.url, + allow_netrc=False, + allow_keyring=True, + ) # Prompt the user for a new username and password save = False @@ -287,7 +296,8 @@ def warn_on_401(self, resp: Response, **kwargs: Any) -> None: """Response callback to warn about incorrect credentials.""" if resp.status_code == 401: logger.warning( - '401 Error, Credentials not correct for %s', resp.request.url, + "401 Error, Credentials not correct for %s", + resp.request.url, ) def save_credentials(self, resp: Response, **kwargs: Any) -> None: @@ -300,7 +310,7 @@ def save_credentials(self, resp: Response, **kwargs: Any) -> None: self._credentials_to_save = None if creds and resp.status_code < 400: try: - logger.info('Saving credentials to keyring') + logger.info("Saving credentials to keyring") keyring.set_password(*creds) except Exception: - logger.exception('Failed to save credentials') + logger.exception("Failed to save credentials") diff --git a/src/pip/_internal/network/cache.py b/src/pip/_internal/network/cache.py index 6902c913993..2d915e6fcec 100644 --- a/src/pip/_internal/network/cache.py +++ b/src/pip/_internal/network/cache.py @@ -50,7 +50,7 @@ def _get_cache_path(self, name: str) -> str: def get(self, key: str) -> Optional[bytes]: path = self._get_cache_path(key) with suppressed_cache_errors(): - with open(path, 'rb') as f: + with open(path, "rb") as f: return f.read() def set(self, key: str, value: bytes) -> None: diff --git a/src/pip/_internal/network/download.py b/src/pip/_internal/network/download.py index 9ec9ebf35e2..47af547d6fc 100644 --- a/src/pip/_internal/network/download.py +++ b/src/pip/_internal/network/download.py @@ -22,7 +22,7 @@ def _get_http_response_size(resp: Response) -> Optional[int]: try: - return int(resp.headers['content-length']) + return int(resp.headers["content-length"]) except (ValueError, KeyError, TypeError): return None @@ -30,7 +30,7 @@ def _get_http_response_size(resp: Response) -> Optional[int]: def _prepare_download( resp: Response, link: Link, - progress_bar: str + progress_bar: str, ) -> Iterable[bytes]: total_length = _get_http_response_size(resp) @@ -42,7 +42,7 @@ def _prepare_download( logged_url = redact_auth_from_url(url) if total_length: - logged_url = '{} ({})'.format(logged_url, format_size(total_length)) + logged_url = "{} ({})".format(logged_url, format_size(total_length)) if is_from_cache(resp): logger.info("Using cached %s", logged_url) @@ -65,9 +65,7 @@ def _prepare_download( if not show_progress: return chunks - return DownloadProgressProvider( - progress_bar, max=total_length - )(chunks) + return DownloadProgressProvider(progress_bar, max=total_length)(chunks) def sanitize_content_filename(filename: str) -> str: @@ -83,7 +81,7 @@ def parse_content_disposition(content_disposition: str, default_filename: str) - return the default filename if the result is empty. """ _type, params = cgi.parse_header(content_disposition) - filename = params.get('filename') + filename = params.get("filename") if filename: # We need to sanitize the filename to prevent directory traversal # in case the filename contains ".." path parts. @@ -97,14 +95,12 @@ def _get_http_response_filename(resp: Response, link: Link) -> str: """ filename = link.filename # fallback # Have a look at the Content-Disposition header for a better guess - content_disposition = resp.headers.get('content-disposition') + content_disposition = resp.headers.get("content-disposition") if content_disposition: filename = parse_content_disposition(content_disposition, filename) ext: Optional[str] = splitext(filename)[1] if not ext: - ext = mimetypes.guess_extension( - resp.headers.get('content-type', '') - ) + ext = mimetypes.guess_extension(resp.headers.get("content-type", "")) if ext: filename += ext if not ext and link.url != resp.url: @@ -115,7 +111,7 @@ def _get_http_response_filename(resp: Response, link: Link) -> str: def _http_get_download(session: PipSession, link: Link) -> Response: - target_url = link.url.split('#', 1)[0] + target_url = link.url.split("#", 1)[0] resp = session.get(target_url, headers=HEADERS, stream=True) raise_for_status(resp) return resp @@ -145,15 +141,14 @@ def __call__(self, link: Link, location: str) -> Tuple[str, str]: filepath = os.path.join(location, filename) chunks = _prepare_download(resp, link, self._progress_bar) - with open(filepath, 'wb') as content_file: + with open(filepath, "wb") as content_file: for chunk in chunks: content_file.write(chunk) - content_type = resp.headers.get('Content-Type', '') + content_type = resp.headers.get("Content-Type", "") return filepath, content_type class BatchDownloader: - def __init__( self, session: PipSession, @@ -173,7 +168,8 @@ def __call__( assert e.response is not None logger.critical( "HTTP error %s while getting %s", - e.response.status_code, link, + e.response.status_code, + link, ) raise @@ -181,8 +177,8 @@ def __call__( filepath = os.path.join(location, filename) chunks = _prepare_download(resp, link, self._progress_bar) - with open(filepath, 'wb') as content_file: + with open(filepath, "wb") as content_file: for chunk in chunks: content_file.write(chunk) - content_type = resp.headers.get('Content-Type', '') + content_type = resp.headers.get("Content-Type", "") yield link, (filepath, content_type) diff --git a/src/pip/_internal/network/lazy_wheel.py b/src/pip/_internal/network/lazy_wheel.py index 22818e22ce9..249bd058760 100644 --- a/src/pip/_internal/network/lazy_wheel.py +++ b/src/pip/_internal/network/lazy_wheel.py @@ -1,6 +1,6 @@ """Lazy ZIP over HTTP""" -__all__ = ['HTTPRangeRequestUnsupported', 'dist_from_wheel_url'] +__all__ = ["HTTPRangeRequestUnsupported", "dist_from_wheel_url"] from bisect import bisect_left, bisect_right from contextlib import contextmanager @@ -53,19 +53,19 @@ def __init__( raise_for_status(head) assert head.status_code == 200 self._session, self._url, self._chunk_size = session, url, chunk_size - self._length = int(head.headers['Content-Length']) + self._length = int(head.headers["Content-Length"]) self._file = NamedTemporaryFile() self.truncate(self._length) self._left: List[int] = [] self._right: List[int] = [] - if 'bytes' not in head.headers.get('Accept-Ranges', 'none'): - raise HTTPRangeRequestUnsupported('range request is not supported') + if "bytes" not in head.headers.get("Accept-Ranges", "none"): + raise HTTPRangeRequestUnsupported("range request is not supported") self._check_zip() @property def mode(self) -> str: """Opening mode, which is always rb.""" - return 'rb' + return "rb" @property def name(self) -> str: @@ -94,9 +94,9 @@ def read(self, size: int = -1) -> bytes: """ download_size = max(size, self._chunk_size) start, length = self.tell(), self._length - stop = length if size < 0 else min(start+download_size, length) - start = max(0, stop-download_size) - self._download(start, stop-1) + stop = length if size < 0 else min(start + download_size, length) + start = max(0, stop - download_size) + self._download(start, stop - 1) return self._file.read(size) def readable(self) -> bool: @@ -170,9 +170,9 @@ def _stream_response( ) -> Response: """Return HTTP response to a range request from start to end.""" headers = base_headers.copy() - headers['Range'] = f'bytes={start}-{end}' + headers["Range"] = f"bytes={start}-{end}" # TODO: Get range requests to be correctly cached - headers['Cache-Control'] = 'no-cache' + headers["Cache-Control"] = "no-cache" return self._session.get(self._url, headers=headers, stream=True) def _merge( @@ -187,11 +187,11 @@ def _merge( right (int): Index after last overlapping downloaded data """ lslice, rslice = self._left[left:right], self._right[left:right] - i = start = min([start]+lslice[:1]) - end = max([end]+rslice[-1:]) + i = start = min([start] + lslice[:1]) + end = max([end] + rslice[-1:]) for j, k in zip(lslice, rslice): if j > i: - yield i, j-1 + yield i, j - 1 i = k + 1 if i <= end: yield i, end diff --git a/src/pip/_internal/network/session.py b/src/pip/_internal/network/session.py index ebc26c81a55..faaae405921 100644 --- a/src/pip/_internal/network/session.py +++ b/src/pip/_internal/network/session.py @@ -77,13 +77,13 @@ # For more background, see: https://github.com/pypa/pip/issues/5499 CI_ENVIRONMENT_VARIABLES = ( # Azure Pipelines - 'BUILD_BUILDID', + "BUILD_BUILDID", # Jenkins - 'BUILD_ID', + "BUILD_ID", # AppVeyor, CircleCI, Codeship, Gitlab CI, Shippable, Travis CI - 'CI', + "CI", # Explicit environment variable. - 'PIP_IS_CI', + "PIP_IS_CI", ) @@ -109,19 +109,19 @@ def user_agent() -> str: }, } - if data["implementation"]["name"] == 'CPython': + if data["implementation"]["name"] == "CPython": data["implementation"]["version"] = platform.python_version() - elif data["implementation"]["name"] == 'PyPy': + elif data["implementation"]["name"] == "PyPy": pypy_version_info = sys.pypy_version_info # type: ignore - if pypy_version_info.releaselevel == 'final': + if pypy_version_info.releaselevel == "final": pypy_version_info = pypy_version_info[:3] data["implementation"]["version"] = ".".join( [str(x) for x in pypy_version_info] ) - elif data["implementation"]["name"] == 'Jython': + elif data["implementation"]["name"] == "Jython": # Complete Guess data["implementation"]["version"] = platform.python_version() - elif data["implementation"]["name"] == 'IronPython': + elif data["implementation"]["name"] == "IronPython": # Complete Guess data["implementation"]["version"] = platform.python_version() @@ -130,14 +130,18 @@ def user_agent() -> str: # https://github.com/nir0s/distro/pull/269 linux_distribution = distro.linux_distribution() # type: ignore - distro_infos = dict(filter( - lambda x: x[1], - zip(["name", "version", "id"], linux_distribution), - )) - libc = dict(filter( - lambda x: x[1], - zip(["lib", "version"], libc_ver()), - )) + distro_infos = dict( + filter( + lambda x: x[1], + zip(["name", "version", "id"], linux_distribution), + ) + ) + libc = dict( + filter( + lambda x: x[1], + zip(["lib", "version"], libc_ver()), + ) + ) if libc: distro_infos["libc"] = libc if distro_infos: @@ -157,6 +161,7 @@ def user_agent() -> str: if has_tls(): import _ssl as ssl + data["openssl_version"] = ssl.OPENSSL_VERSION setuptools_dist = get_default_environment().get_distribution("setuptools") @@ -167,7 +172,7 @@ def user_agent() -> str: # If for any reason `rustc --version` fails, silently ignore it try: rustc_output = subprocess.check_output( - ["rustc", "--version"], stderr=subprocess.STDOUT, timeout=.5 + ["rustc", "--version"], stderr=subprocess.STDOUT, timeout=0.5 ) except Exception: pass @@ -195,7 +200,6 @@ def user_agent() -> str: class LocalFSAdapter(BaseAdapter): - def send( self, request: PreparedRequest, @@ -219,11 +223,13 @@ def send( else: modified = email.utils.formatdate(stats.st_mtime, usegmt=True) content_type = mimetypes.guess_type(pathname)[0] or "text/plain" - resp.headers = CaseInsensitiveDict({ - "Content-Type": content_type, - "Content-Length": stats.st_size, - "Last-Modified": modified, - }) + resp.headers = CaseInsensitiveDict( + { + "Content-Type": content_type, + "Content-Length": stats.st_size, + "Last-Modified": modified, + } + ) resp.raw = open(pathname, "rb") resp.close = resp.raw.close @@ -235,7 +241,6 @@ def close(self) -> None: class InsecureHTTPAdapter(HTTPAdapter): - def cert_verify( self, conn: ConnectionPool, @@ -247,7 +252,6 @@ def cert_verify( class InsecureCacheControlAdapter(CacheControlAdapter): - def cert_verify( self, conn: ConnectionPool, @@ -293,7 +297,6 @@ def __init__( # Set the total number of retries that a particular request can # have. total=retries, - # A 503 error from PyPI typically means that the Fastly -> Origin # connection got interrupted in some way. A 503 error in general # is typically considered a transient error so we'll go ahead and @@ -301,7 +304,6 @@ def __init__( # A 500 may indicate transient error in Amazon S3 # A 520 or 527 - may indicate transient error in CloudFlare status_forcelist=[500, 503, 520, 527], - # Add a small amount of back off between failed requests in # order to prevent hammering the service. backoff_factor=0.25, @@ -358,43 +360,39 @@ def add_trusted_host( string came from. """ if not suppress_logging: - msg = f'adding trusted host: {host!r}' + msg = f"adding trusted host: {host!r}" if source is not None: - msg += f' (from {source})' + msg += f" (from {source})" logger.info(msg) host_port = parse_netloc(host) if host_port not in self.pip_trusted_origins: self.pip_trusted_origins.append(host_port) - self.mount( - build_url_from_netloc(host) + '/', - self._trusted_host_adapter - ) + self.mount(build_url_from_netloc(host) + "/", self._trusted_host_adapter) if not host_port[1]: # Mount wildcard ports for the same host. - self.mount( - build_url_from_netloc(host) + ':', - self._trusted_host_adapter - ) + self.mount(build_url_from_netloc(host) + ":", self._trusted_host_adapter) def iter_secure_origins(self) -> Iterator[SecureOrigin]: yield from SECURE_ORIGINS for host, port in self.pip_trusted_origins: - yield ('*', host, '*' if port is None else port) + yield ("*", host, "*" if port is None else port) def is_secure_origin(self, location: Link) -> bool: # Determine if this url used a secure transport mechanism parsed = urllib.parse.urlparse(str(location)) origin_protocol, origin_host, origin_port = ( - parsed.scheme, parsed.hostname, parsed.port, + parsed.scheme, + parsed.hostname, + parsed.port, ) # The protocol to use to see if the protocol matches. # Don't count the repository type as part of the protocol: in # cases such as "git+ssh", only use "ssh". (I.e., Only verify against # the last scheme.) - origin_protocol = origin_protocol.rsplit('+', 1)[-1] + origin_protocol = origin_protocol.rsplit("+", 1)[-1] # Determine if our origin is a secure origin by looking through our # hardcoded list of secure origins, as well as any additional ones @@ -411,9 +409,9 @@ def is_secure_origin(self, location: Link) -> bool: # We don't have both a valid address or a valid network, so # we'll check this origin against hostnames. if ( - origin_host and - origin_host.lower() != secure_host.lower() and - secure_host != "*" + origin_host + and origin_host.lower() != secure_host.lower() + and secure_host != "*" ): continue else: @@ -424,9 +422,9 @@ def is_secure_origin(self, location: Link) -> bool: # Check to see if the port matches. if ( - origin_port != secure_port and - secure_port != "*" and - secure_port is not None + origin_port != secure_port + and secure_port != "*" + and secure_port is not None ): continue diff --git a/src/pip/_internal/network/utils.py b/src/pip/_internal/network/utils.py index 298a32a38ae..094cf1b4a97 100644 --- a/src/pip/_internal/network/utils.py +++ b/src/pip/_internal/network/utils.py @@ -23,30 +23,32 @@ # you're not asking for a compressed file and will then decompress it # before sending because if that's the case I don't think it'll ever be # possible to make this work. -HEADERS: Dict[str, str] = {'Accept-Encoding': 'identity'} +HEADERS: Dict[str, str] = {"Accept-Encoding": "identity"} def raise_for_status(resp: Response) -> None: - http_error_msg = '' + http_error_msg = "" if isinstance(resp.reason, bytes): # We attempt to decode utf-8 first because some servers # choose to localize their reason strings. If the string # isn't utf-8, we fall back to iso-8859-1 for all other # encodings. try: - reason = resp.reason.decode('utf-8') + reason = resp.reason.decode("utf-8") except UnicodeDecodeError: - reason = resp.reason.decode('iso-8859-1') + reason = resp.reason.decode("iso-8859-1") else: reason = resp.reason if 400 <= resp.status_code < 500: http_error_msg = ( - f'{resp.status_code} Client Error: {reason} for url: {resp.url}') + f"{resp.status_code} Client Error: {reason} for url: {resp.url}" + ) elif 500 <= resp.status_code < 600: http_error_msg = ( - f'{resp.status_code} Server Error: {reason} for url: {resp.url}') + f"{resp.status_code} Server Error: {reason} for url: {resp.url}" + ) if http_error_msg: raise NetworkConnectionError(http_error_msg, response=resp) @@ -55,8 +57,7 @@ def raise_for_status(resp: Response) -> None: def response_chunks( response: Response, chunk_size: int = CONTENT_CHUNK_SIZE ) -> Iterator[bytes]: - """Given a requests Response, provide the data chunks. - """ + """Given a requests Response, provide the data chunks.""" try: # Special case for urllib3. for chunk in response.raw.stream( diff --git a/src/pip/_internal/network/xmlrpc.py b/src/pip/_internal/network/xmlrpc.py index 427902136da..4a7d55d0e50 100644 --- a/src/pip/_internal/network/xmlrpc.py +++ b/src/pip/_internal/network/xmlrpc.py @@ -40,9 +40,13 @@ def request( parts = (self._scheme, host, handler, None, None, None) url = urllib.parse.urlunparse(parts) try: - headers = {'Content-Type': 'text/xml'} - response = self._session.post(url, data=request_body, - headers=headers, stream=True) + headers = {"Content-Type": "text/xml"} + response = self._session.post( + url, + data=request_body, + headers=headers, + stream=True, + ) raise_for_status(response) self.verbose = verbose return self.parse_response(response.raw) @@ -50,6 +54,7 @@ def request( assert exc.response logger.critical( "HTTP error %s while getting %s", - exc.response.status_code, url, + exc.response.status_code, + url, ) raise