Adjust the Fallback logic for obtaining the hashes from private indexes

news/5866.bugfix.rst

@@ -0,0 +1 @@
+Fix regression of hash collection when downloading package from private indexes when the hash is not found in the index href url fragment.

pipenv/project.py

@@ -12,7 +12,7 @@
 from json.decoder import JSONDecodeError
 from pathlib import Path
 from urllib import parse
-from urllib.parse import unquote
+from urllib.parse import unquote, urljoin
 
 from pipenv.utils.constants import VCS_LIST
 
@@ -240,7 +240,9 @@ def get_requests_session_for_source(self, source):
  session = self.sessions[source["name"]]
  else:
  session = get_requests_session(
- self.s.PIPENV_MAX_RETRIES, source.get("verify_ssl", True)
+ self.s.PIPENV_MAX_RETRIES,
+ source.get("verify_ssl", True),
+ cache_dir=self.s.PIPENV_CACHE_DIR,
  )
  self.sessions[source["name"]] = session
  return session
@@ -315,10 +317,11 @@ def get_hashes_from_remote_index_urls(self, ireq, source):
  if params_dict.get(FAVORITE_HASH):
  collected_hashes.add(params_dict[FAVORITE_HASH][0])
  else: # Fallback to downloading the file to obtain hash
- if source["url"] not in package_url:
- package_url = f"{source['url']}{package_url}"
+ package_url = urljoin(source["url"], package_url)
  link = Link(package_url)
- collected_hashes.add(self.get_file_hash(session, link))
+ file_hash = self.get_file_hash(session, link)
+ if file_hash:
+ collected_hashes.add(file_hash)
  return self.prepend_hash_types(collected_hashes, FAVORITE_HASH)
  except (ValueError, KeyError, ConnectionError):
  if self.s.is_verbose():
@@ -334,6 +337,8 @@ def get_file_hash(self, session, link):
  h = hashlib.new(FAVORITE_HASH)
  err.print(f"Downloading file {link.filename} to obtain hash...")
  with open_file(link.url, session) as fp:
+ if fp is None:
+ return None
  for chunk in iter(lambda: fp.read(8096), b""):
  h.update(chunk)
  return f"{h.name}:{h.hexdigest()}"

pipenv/utils/fileutils.py

@@ -1,23 +1,20 @@
 """A collection for utilities for working with files and paths."""
 import atexit
+import io
 import os
 import sys
 import warnings
-from contextlib import closing, contextmanager
-from http.client import HTTPResponse as Urllib_HTTPResponse
+from contextlib import contextmanager
 from pathlib import Path
 from tempfile import TemporaryDirectory
-from typing import IO, Any, ContextManager, Optional, TypeVar, Union
+from typing import Any, Optional
 from urllib import parse as urllib_parse
 from urllib import request as urllib_request
 from urllib.parse import quote, urlparse
 
-from pipenv.patched.pip._vendor.requests import Session
-from pipenv.patched.pip._vendor.urllib3.response import (
- HTTPResponse as Urllib3_HTTPResponse,
-)
-
-_T = TypeVar("_T")
+from pipenv.patched.pip._internal.locations import USER_CACHE_DIR
+from pipenv.patched.pip._internal.network.download import PipSession
+from pipenv.utils import err
 
 
 def is_file_url(url: Any) -> bool:
@@ -118,14 +115,12 @@ def path_to_url(path):
 
 
 @contextmanager
-def open_file(
- link: Union[_T, str], session: Optional[Session] = None, stream: bool = True
-) -> ContextManager[Union[IO[bytes], Urllib3_HTTPResponse, Urllib_HTTPResponse]]:
+def open_file(link, session: Optional[PipSession] = None, stream: bool = False):
  """Open local or remote file for reading.
 
  :param pipenv.patched.pip._internal.index.Link link: A link object from resolving dependencies with
  pip, or else a URL.
- :param Optional[Session] session: A :class:`~requests.Session` instance
+ :param Optional[PipSession] session: A :class:`~PipSession` instance
  :param bool stream: Whether to stream the content if remote, default True
  :raises ValueError: If link points to a local directory.
  :return: a context manager to the opened file-like object
@@ -151,27 +146,15 @@ def open_file(
  # Remote URL
  headers = {"Accept-Encoding": "identity"}
  if not session:
- try:
- from pipenv.patched.pip._vendor.requests import Session # noqa
- except ImportError:
- session = None
- else:
- session = Session()
- if session is None:
- with closing(urllib_request.urlopen(link)) as f:
- yield f
+ session = PipSession(cache=USER_CACHE_DIR)
+ resp = session.get(link, headers=headers, stream=stream)
+ if resp.status_code != 200:
+ err.print(f"HTTP error {resp.status_code} while getting {link}")
+ yield None
  else:
- with session.get(link, headers=headers, stream=stream) as resp:
- try:
- raw = getattr(resp, "raw", None)
- result = raw if raw else resp
- yield result
- finally:
- if raw:
- conn = raw._connection
- if conn is not None:
- conn.close()
- result.close()
+ # Creating a buffer-like object
+ buffer = io.BytesIO(resp.content)
+ yield buffer
 
 
 @contextmanager
@@ -216,7 +199,7 @@ def create_tracked_tempdir(*args: Any, **kwargs: Any) -> str:
 
  This uses `TemporaryDirectory`, but does not remove the directory
  when the return value goes out of scope, instead registers a handler
- to cleanup on program exit. The return value is the path to the
+ to clean up on program exit. The return value is the path to the
  created directory.
  """
  tempdir = TemporaryDirectory(*args, **kwargs)

pipenv/utils/internet.py

@@ -3,19 +3,17 @@
 from html.parser import HTMLParser
 from urllib.parse import urlparse
 
-from pipenv.patched.pip._vendor import requests
-from pipenv.patched.pip._vendor.requests.adapters import HTTPAdapter
+from pipenv.patched.pip._internal.locations import USER_CACHE_DIR
+from pipenv.patched.pip._internal.network.download import PipSession
 from pipenv.patched.pip._vendor.urllib3 import util as urllib3_util
 
 
-def get_requests_session(max_retries=1, verify_ssl=True):
+def get_requests_session(max_retries=1, verify_ssl=True, cache_dir=USER_CACHE_DIR):
  """Load requests lazily."""
  pip_client_cert = os.environ.get("PIP_CLIENT_CERT")
- requests_session = requests.Session()
+ requests_session = PipSession(cache=cache_dir, retries=max_retries)
  if pip_client_cert:
  requests_session.cert = pip_client_cert
- adapter = HTTPAdapter(max_retries=max_retries)
- requests_session.mount("https://", adapter)
  if verify_ssl is False:
  requests_session.verify = False
  return requests_session

tasks/vendoring/__init__.py

@@ -795,4 +795,7 @@ def vendor_artifact(ctx, package, version=None):
  dest_file = dest_dir / dest_path
  with open(dest_file.as_posix(), "wb") as target_handle:
  with open_file(link) as fp:
+ if fp is None:
+ print(f"Error downloading {link}")
+ continue
  shutil.copyfileobj(fp, target_handle)