diff --git a/src/pip/_internal/cli/cmdoptions.py b/src/pip/_internal/cli/cmdoptions.py index f0950332115..478848ac238 100644 --- a/src/pip/_internal/cli/cmdoptions.py +++ b/src/pip/_internal/cli/cmdoptions.py @@ -1009,7 +1009,6 @@ def check_list_path_option(options: Values) -> None: default=[], choices=[ "2020-resolver", - "fast-deps", "truststore", "no-binary-enable-wheel-cache", ], diff --git a/src/pip/_internal/cli/req_command.py b/src/pip/_internal/cli/req_command.py index 1044809f040..550907671da 100644 --- a/src/pip/_internal/cli/req_command.py +++ b/src/pip/_internal/cli/req_command.py @@ -287,24 +287,6 @@ def make_requirement_preparer( temp_build_dir_path = temp_build_dir.path assert temp_build_dir_path is not None - resolver_variant = cls.determine_resolver_variant(options) - if resolver_variant == "2020-resolver": - lazy_wheel = "fast-deps" in options.features_enabled - if lazy_wheel: - logger.warning( - "pip is using lazily downloaded wheels using HTTP " - "range requests to obtain dependency information. " - "This experimental feature is enabled through " - "--use-feature=fast-deps and it is not ready for " - "production." - ) - else: - lazy_wheel = False - if "fast-deps" in options.features_enabled: - logger.warning( - "fast-deps has no effect when used with the legacy resolver." - ) - return RequirementPreparer( build_dir=temp_build_dir_path, src_dir=options.src_dir, @@ -317,7 +299,6 @@ def make_requirement_preparer( finder=finder, require_hashes=options.require_hashes, use_user_site=use_user_site, - lazy_wheel=lazy_wheel, verbosity=verbosity, ) diff --git a/src/pip/_internal/network/lazy_wheel.py b/src/pip/_internal/network/lazy_wheel.py deleted file mode 100644 index 854a6fa1fdc..00000000000 --- a/src/pip/_internal/network/lazy_wheel.py +++ /dev/null @@ -1,210 +0,0 @@ -"""Lazy ZIP over HTTP""" - -__all__ = ["HTTPRangeRequestUnsupported", "dist_from_wheel_url"] - -from bisect import bisect_left, bisect_right -from contextlib import contextmanager -from tempfile import NamedTemporaryFile -from typing import Any, Dict, Generator, List, Optional, Tuple -from zipfile import BadZipfile, ZipFile - -from pip._vendor.packaging.utils import canonicalize_name -from pip._vendor.requests.models import CONTENT_CHUNK_SIZE, Response - -from pip._internal.metadata import BaseDistribution, MemoryWheel, get_wheel_distribution -from pip._internal.network.session import PipSession -from pip._internal.network.utils import HEADERS, raise_for_status, response_chunks - - -class HTTPRangeRequestUnsupported(Exception): - pass - - -def dist_from_wheel_url(name: str, url: str, session: PipSession) -> BaseDistribution: - """Return a distribution object from the given wheel URL. - - This uses HTTP range requests to only fetch the portion of the wheel - containing metadata, just enough for the object to be constructed. - If such requests are not supported, HTTPRangeRequestUnsupported - is raised. - """ - with LazyZipOverHTTP(url, session) as zf: - # For read-only ZIP files, ZipFile only needs methods read, - # seek, seekable and tell, not the whole IO protocol. - wheel = MemoryWheel(zf.name, zf) # type: ignore - # After context manager exit, wheel.name - # is an invalid file by intention. - return get_wheel_distribution(wheel, canonicalize_name(name)) - - -class LazyZipOverHTTP: - """File-like object mapped to a ZIP file over HTTP. - - This uses HTTP range requests to lazily fetch the file's content, - which is supposed to be fed to ZipFile. If such requests are not - supported by the server, raise HTTPRangeRequestUnsupported - during initialization. - """ - - def __init__( - self, url: str, session: PipSession, chunk_size: int = CONTENT_CHUNK_SIZE - ) -> None: - head = session.head(url, headers=HEADERS) - raise_for_status(head) - assert head.status_code == 200 - self._session, self._url, self._chunk_size = session, url, chunk_size - self._length = int(head.headers["Content-Length"]) - self._file = NamedTemporaryFile() - self.truncate(self._length) - self._left: List[int] = [] - self._right: List[int] = [] - if "bytes" not in head.headers.get("Accept-Ranges", "none"): - raise HTTPRangeRequestUnsupported("range request is not supported") - self._check_zip() - - @property - def mode(self) -> str: - """Opening mode, which is always rb.""" - return "rb" - - @property - def name(self) -> str: - """Path to the underlying file.""" - return self._file.name - - def seekable(self) -> bool: - """Return whether random access is supported, which is True.""" - return True - - def close(self) -> None: - """Close the file.""" - self._file.close() - - @property - def closed(self) -> bool: - """Whether the file is closed.""" - return self._file.closed - - def read(self, size: int = -1) -> bytes: - """Read up to size bytes from the object and return them. - - As a convenience, if size is unspecified or -1, - all bytes until EOF are returned. Fewer than - size bytes may be returned if EOF is reached. - """ - download_size = max(size, self._chunk_size) - start, length = self.tell(), self._length - stop = length if size < 0 else min(start + download_size, length) - start = max(0, stop - download_size) - self._download(start, stop - 1) - return self._file.read(size) - - def readable(self) -> bool: - """Return whether the file is readable, which is True.""" - return True - - def seek(self, offset: int, whence: int = 0) -> int: - """Change stream position and return the new absolute position. - - Seek to offset relative position indicated by whence: - * 0: Start of stream (the default). pos should be >= 0; - * 1: Current position - pos may be negative; - * 2: End of stream - pos usually negative. - """ - return self._file.seek(offset, whence) - - def tell(self) -> int: - """Return the current position.""" - return self._file.tell() - - def truncate(self, size: Optional[int] = None) -> int: - """Resize the stream to the given size in bytes. - - If size is unspecified resize to the current position. - The current stream position isn't changed. - - Return the new file size. - """ - return self._file.truncate(size) - - def writable(self) -> bool: - """Return False.""" - return False - - def __enter__(self) -> "LazyZipOverHTTP": - self._file.__enter__() - return self - - def __exit__(self, *exc: Any) -> None: - self._file.__exit__(*exc) - - @contextmanager - def _stay(self) -> Generator[None, None, None]: - """Return a context manager keeping the position. - - At the end of the block, seek back to original position. - """ - pos = self.tell() - try: - yield - finally: - self.seek(pos) - - def _check_zip(self) -> None: - """Check and download until the file is a valid ZIP.""" - end = self._length - 1 - for start in reversed(range(0, end, self._chunk_size)): - self._download(start, end) - with self._stay(): - try: - # For read-only ZIP files, ZipFile only needs - # methods read, seek, seekable and tell. - ZipFile(self) # type: ignore - except BadZipfile: - pass - else: - break - - def _stream_response( - self, start: int, end: int, base_headers: Dict[str, str] = HEADERS - ) -> Response: - """Return HTTP response to a range request from start to end.""" - headers = base_headers.copy() - headers["Range"] = f"bytes={start}-{end}" - # TODO: Get range requests to be correctly cached - headers["Cache-Control"] = "no-cache" - return self._session.get(self._url, headers=headers, stream=True) - - def _merge( - self, start: int, end: int, left: int, right: int - ) -> Generator[Tuple[int, int], None, None]: - """Return a generator of intervals to be fetched. - - Args: - start (int): Start of needed interval - end (int): End of needed interval - left (int): Index of first overlapping downloaded data - right (int): Index after last overlapping downloaded data - """ - lslice, rslice = self._left[left:right], self._right[left:right] - i = start = min([start] + lslice[:1]) - end = max([end] + rslice[-1:]) - for j, k in zip(lslice, rslice): - if j > i: - yield i, j - 1 - i = k + 1 - if i <= end: - yield i, end - self._left[left:right], self._right[left:right] = [start], [end] - - def _download(self, start: int, end: int) -> None: - """Download bytes from start to end inclusively.""" - with self._stay(): - left = bisect_left(self._right, start) - right = bisect_right(self._left, end) - for start, end in self._merge(start, end, left, right): - response = self._stream_response(start, end) - response.raise_for_status() - self.seek(start) - for chunk in response_chunks(response, self._chunk_size): - self._file.write(chunk) diff --git a/src/pip/_internal/operations/prepare.py b/src/pip/_internal/operations/prepare.py index 4bf414cb005..b31391a500e 100644 --- a/src/pip/_internal/operations/prepare.py +++ b/src/pip/_internal/operations/prepare.py @@ -10,8 +10,6 @@ import shutil from typing import Dict, Iterable, List, Optional -from pip._vendor.packaging.utils import canonicalize_name - from pip._internal.distributions import make_distribution_for_install_requirement from pip._internal.distributions.installed import InstalledDistribution from pip._internal.exceptions import ( @@ -28,12 +26,7 @@ from pip._internal.metadata import BaseDistribution, get_metadata_distribution from pip._internal.models.direct_url import ArchiveInfo from pip._internal.models.link import Link -from pip._internal.models.wheel import Wheel from pip._internal.network.download import BatchDownloader, Downloader -from pip._internal.network.lazy_wheel import ( - HTTPRangeRequestUnsupported, - dist_from_wheel_url, -) from pip._internal.network.session import PipSession from pip._internal.operations.build.build_tracker import BuildTracker from pip._internal.req.req_install import InstallRequirement @@ -220,7 +213,6 @@ def __init__( finder: PackageFinder, require_hashes: bool, use_user_site: bool, - lazy_wheel: bool, verbosity: int, ) -> None: super().__init__() @@ -249,9 +241,6 @@ def __init__( # Should install in user site-packages? self.use_user_site = use_user_site - # Should wheels be downloaded lazily? - self.use_lazy_wheel = lazy_wheel - # How verbose should underlying tooling be? self.verbosity = verbosity @@ -356,10 +345,7 @@ def _fetch_metadata_only( "Metadata-only fetching is not used as hash checking is required", ) return None - # Try PEP 658 metadata first, then fall back to lazy wheel if unavailable. - return self._fetch_metadata_using_link_data_attr( - req - ) or self._fetch_metadata_using_lazy_wheel(req.link) + return self._fetch_metadata_using_link_data_attr(req) def _fetch_metadata_using_link_data_attr( self, @@ -402,35 +388,6 @@ def _fetch_metadata_using_link_data_attr( ) return metadata_dist - def _fetch_metadata_using_lazy_wheel( - self, - link: Link, - ) -> Optional[BaseDistribution]: - """Fetch metadata using lazy wheel, if possible.""" - # --use-feature=fast-deps must be provided. - if not self.use_lazy_wheel: - return None - if link.is_file or not link.is_wheel: - logger.debug( - "Lazy wheel is not used as %r does not point to a remote wheel", - link, - ) - return None - - wheel = Wheel(link.filename) - name = canonicalize_name(wheel.name) - logger.info( - "Obtaining dependency information from %s %s", - name, - wheel.version, - ) - url = link.url.split("#", 1)[0] - try: - return dist_from_wheel_url(name, url, self._session) - except HTTPRangeRequestUnsupported: - logger.debug("%s does not support range requests", url) - return None - def _complete_partial_requirements( self, partially_downloaded_reqs: Iterable[InstallRequirement], diff --git a/tests/data/src/simplewheel-1.0/build/lib/simplewheel/__init__.py b/tests/data/src/simplewheel-1.0/build/lib/simplewheel/__init__.py new file mode 100644 index 00000000000..4802e90f8ed --- /dev/null +++ b/tests/data/src/simplewheel-1.0/build/lib/simplewheel/__init__.py @@ -0,0 +1 @@ +__version__ = "1.0" diff --git a/tests/functional/test_fast_deps.py b/tests/functional/test_fast_deps.py deleted file mode 100644 index 0109db825b7..00000000000 --- a/tests/functional/test_fast_deps.py +++ /dev/null @@ -1,103 +0,0 @@ -import fnmatch -import json -import os -import pathlib -from os.path import basename -from typing import Iterable - -from pip._vendor.packaging.utils import canonicalize_name -from pytest import mark - -from tests.lib import PipTestEnvironment, TestData, TestPipResult - - -def pip(script: PipTestEnvironment, command: str, requirement: str) -> TestPipResult: - return script.pip( - command, - "--prefer-binary", - "--no-cache-dir", - "--use-feature=fast-deps", - requirement, - allow_stderr_warning=True, - ) - - -def assert_installed(script: PipTestEnvironment, names: str) -> None: - list_output = json.loads(script.pip("list", "--format=json").stdout) - installed = {canonicalize_name(item["name"]) for item in list_output} - assert installed.issuperset(map(canonicalize_name, names)) - - -@mark.network -@mark.parametrize( - ("requirement", "expected"), - ( - ("Paste==3.4.2", ("Paste", "six")), - ("Paste[flup]==3.4.2", ("Paste", "six", "flup")), - ), -) -def test_install_from_pypi( - requirement: str, expected: str, script: PipTestEnvironment -) -> None: - pip(script, "install", requirement) - assert_installed(script, expected) - - -@mark.network -@mark.parametrize( - ("requirement", "expected"), - ( - ("Paste==3.4.2", ("Paste-3.4.2-*.whl", "six-*.whl")), - ("Paste[flup]==3.4.2", ("Paste-3.4.2-*.whl", "six-*.whl", "flup-*")), - ), -) -def test_download_from_pypi( - requirement: str, expected: Iterable[str], script: PipTestEnvironment -) -> None: - result = pip(script, "download", requirement) - created = [basename(f) for f in result.files_created] - assert all(fnmatch.filter(created, f) for f in expected) - - -@mark.network -def test_build_wheel_with_deps(data: TestData, script: PipTestEnvironment) -> None: - result = pip(script, "wheel", os.fspath(data.packages / "requiresPaste")) - created = [basename(f) for f in result.files_created] - assert fnmatch.filter(created, "requirespaste-3.1.4-*.whl") - assert fnmatch.filter(created, "Paste-3.4.2-*.whl") - assert fnmatch.filter(created, "six-*.whl") - - -@mark.network -def test_require_hash(script: PipTestEnvironment, tmp_path: pathlib.Path) -> None: - reqs = tmp_path / "requirements.txt" - reqs.write_text( - "idna==2.10" - " --hash=sha256:" - "b97d804b1e9b523befed77c48dacec60e6dcb0b5391d57af6a65a312a90648c0" - " --hash=sha256:" - "b307872f855b18632ce0c21c5e45be78c0ea7ae4c15c828c20788b26921eb3f6" - ) - result = script.pip( - "download", - "--use-feature=fast-deps", - "-r", - str(reqs), - allow_stderr_warning=True, - ) - created = [basename(f) for f in result.files_created] - assert fnmatch.filter(created, "idna-2.10*") - - -@mark.network -def test_hash_mismatch(script: PipTestEnvironment, tmp_path: pathlib.Path) -> None: - reqs = tmp_path / "requirements.txt" - reqs.write_text("idna==2.10 --hash=sha256:irna") - result = script.pip( - "download", - "--use-feature=fast-deps", - "-r", - str(reqs), - expect_error=True, - ) - assert "DO NOT MATCH THE HASHES" in result.stderr diff --git a/tests/unit/test_network_lazy_wheel.py b/tests/unit/test_network_lazy_wheel.py deleted file mode 100644 index 79e86321793..00000000000 --- a/tests/unit/test_network_lazy_wheel.py +++ /dev/null @@ -1,66 +0,0 @@ -from typing import Iterator - -from pip._vendor.packaging.version import Version -from pytest import fixture, mark, raises - -from pip._internal.exceptions import InvalidWheel -from pip._internal.network.lazy_wheel import ( - HTTPRangeRequestUnsupported, - dist_from_wheel_url, -) -from pip._internal.network.session import PipSession -from tests.lib import TestData -from tests.lib.server import MockServer, file_response - -MYPY_0_782_WHL = ( - "https://files.pythonhosted.org/packages/9d/65/" - "b96e844150ce18b9892b155b780248955ded13a2581d31872e7daa90a503/" - "mypy-0.782-py3-none-any.whl" -) -MYPY_0_782_REQS = { - "typed-ast<1.5.0,>=1.4.0", - "typing-extensions>=3.7.4", - "mypy-extensions<0.5.0,>=0.4.3", - 'psutil>=4.0; extra == "dmypy"', -} - - -@fixture -def session() -> PipSession: - return PipSession() - - -@fixture -def mypy_whl_no_range(mock_server: MockServer, shared_data: TestData) -> Iterator[str]: - mypy_whl = shared_data.packages / "mypy-0.782-py3-none-any.whl" - mock_server.set_responses([file_response(mypy_whl)]) - mock_server.start() - base_address = f"http://{mock_server.host}:{mock_server.port}" - yield "{}/{}".format(base_address, "mypy-0.782-py3-none-any.whl") - mock_server.stop() - - -@mark.network -def test_dist_from_wheel_url(session: PipSession) -> None: - """Test if the acquired distribution contain correct information.""" - dist = dist_from_wheel_url("mypy", MYPY_0_782_WHL, session) - assert dist.canonical_name == "mypy" - assert dist.version == Version("0.782") - extras = list(dist.iter_provided_extras()) - assert extras == ["dmypy"] - assert {str(d) for d in dist.iter_dependencies(extras)} == MYPY_0_782_REQS - - -def test_dist_from_wheel_url_no_range( - session: PipSession, mypy_whl_no_range: str -) -> None: - """Test handling when HTTP range requests are not supported.""" - with raises(HTTPRangeRequestUnsupported): - dist_from_wheel_url("mypy", mypy_whl_no_range, session) - - -@mark.network -def test_dist_from_wheel_url_not_zip(session: PipSession) -> None: - """Test handling with the given URL does not point to a ZIP.""" - with raises(InvalidWheel): - dist_from_wheel_url("python", "https://www.python.org/", session) diff --git a/tests/unit/test_req.py b/tests/unit/test_req.py index bd828916593..4fe56408995 100644 --- a/tests/unit/test_req.py +++ b/tests/unit/test_req.py @@ -104,7 +104,6 @@ def _basic_resolver( finder=finder, require_hashes=require_hashes, use_user_site=False, - lazy_wheel=False, verbosity=0, ) yield Resolver(