From 3765b8dbe98ef5c688409792663bbebef48f07fe Mon Sep 17 00:00:00 2001 From: Danny McClanahan <1305167+cosmicexplorer@users.noreply.github.com> Date: Tue, 28 Dec 2021 10:58:35 -0500 Subject: [PATCH] move the resolution data printing to a new `resolve` command --- src/pip/_internal/commands/__init__.py | 5 + src/pip/_internal/commands/download.py | 92 +--------- src/pip/_internal/commands/resolve.py | 227 +++++++++++++++++++++++++ 3 files changed, 234 insertions(+), 90 deletions(-) create mode 100644 src/pip/_internal/commands/resolve.py diff --git a/src/pip/_internal/commands/__init__.py b/src/pip/_internal/commands/__init__.py index c72f24f30e2..6a30e4115b8 100644 --- a/src/pip/_internal/commands/__init__.py +++ b/src/pip/_internal/commands/__init__.py @@ -28,6 +28,11 @@ "DownloadCommand", "Download packages.", ), + "resolve": CommandInfo( + "pip._internal.commands.resolve", + "ResolveCommand", + "Resolve and print out package dependencies and metadata.", + ), "uninstall": CommandInfo( "pip._internal.commands.uninstall", "UninstallCommand", diff --git a/src/pip/_internal/commands/download.py b/src/pip/_internal/commands/download.py index e1035a85c5f..7de207f1365 100644 --- a/src/pip/_internal/commands/download.py +++ b/src/pip/_internal/commands/download.py @@ -1,39 +1,19 @@ -import json import logging import os -from dataclasses import dataclass, field from optparse import Values -from typing import Any, Dict, List - -from pip._vendor.packaging.requirements import Requirement +from typing import List from pip._internal.cli import cmdoptions from pip._internal.cli.cmdoptions import make_target_python from pip._internal.cli.req_command import RequirementCommand, with_cleanup from pip._internal.cli.status_codes import SUCCESS -from pip._internal.models.link import RequirementDownloadInfo from pip._internal.req.req_tracker import get_requirement_tracker -from pip._internal.resolution.base import RequirementSetWithCandidates from pip._internal.utils.misc import ensure_dir, normalize_path, write_output from pip._internal.utils.temp_dir import TempDirectory logger = logging.getLogger(__name__) -@dataclass -class DownloadInfos: - implicit_requirements: List[Requirement] = field(default_factory=list) - resolution: Dict[str, RequirementDownloadInfo] = field(default_factory=dict) - - def as_json(self) -> Dict[str, Any]: - return { - "implicit_requirements": [str(req) for req in self.implicit_requirements], - "resolution": { - name: info.as_json() for name, info in self.resolution.items() - }, - } - - class DownloadCommand(RequirementCommand): """ Download packages from: @@ -82,25 +62,6 @@ def add_options(self) -> None: help="Download packages into .", ) - self.cmd_opts.add_option( - "--print-download-urls", - dest="print_download_urls", - metavar="output-file", - default=None, - help=("Print URLs of any downloaded distributions to this file."), - ) - - self.cmd_opts.add_option( - "--avoid-wheel-downloads", - dest="avoid_wheel_downloads", - default=False, - action="store_true", - help=( - "Where possible, avoid downloading wheels. This is " - "currently only useful if --print-download-urls is set." - ), - ) - cmdoptions.add_target_python_options(self.cmd_opts) index_opts = cmdoptions.make_option_group( @@ -160,7 +121,6 @@ def run(self, options: Values, args: List[str]) -> int: options=options, ignore_requires_python=options.ignore_requires_python, py_version_info=options.python_version, - avoid_wheel_downloads=options.avoid_wheel_downloads, ) self.trace_basic_info(finder) @@ -169,59 +129,11 @@ def run(self, options: Values, args: List[str]) -> int: downloaded: List[str] = [] for req in requirement_set.requirements.values(): - # If this distribution was not already satisfied, that means we - # downloaded it. if req.satisfied_by is None: - preparer.save_linked_requirement(req) assert req.name is not None + preparer.save_linked_requirement(req) downloaded.append(req.name) - - download_infos = DownloadInfos() - if options.print_download_urls: - if isinstance(requirement_set, RequirementSetWithCandidates): - for candidate in requirement_set.candidates.mapping.values(): - # This will occur for the python version requirement, for example. - if candidate.name not in requirement_set.requirements: - assert ( - tuple(candidate.iter_dependencies(with_requires=True)) == () - ) - download_infos.implicit_requirements.append( - candidate.as_serializable_requirement() - ) - continue - req = requirement_set.requirements[candidate.name] - assert req.name is not None - assert req.link is not None - assert req.name not in download_infos.resolution - - dependencies: List[Requirement] = [] - for maybe_dep in candidate.iter_dependencies(with_requires=True): - if maybe_dep is None: - continue - maybe_req = maybe_dep.as_serializable_requirement() - if maybe_req is None: - continue - dependencies.append(maybe_req) - - download_infos.resolution[ - req.name - ] = RequirementDownloadInfo.from_req_and_link_and_deps( - req=candidate.as_serializable_requirement(), - dependencies=dependencies, - link=req.link, - ) - else: - logger.warning( - "--print-download-urls is being used with the legacy resolver. " - "The legacy resolver does not retain detailed dependency " - "information, so all the fields in the output JSON file " - "will be empty." - ) - if downloaded: write_output("Successfully downloaded %s", " ".join(downloaded)) - if options.print_download_urls: - with open(options.print_download_urls, "w") as f: - json.dump(download_infos.as_json(), f, indent=4) return SUCCESS diff --git a/src/pip/_internal/commands/resolve.py b/src/pip/_internal/commands/resolve.py new file mode 100644 index 00000000000..e6c6ac594a5 --- /dev/null +++ b/src/pip/_internal/commands/resolve.py @@ -0,0 +1,227 @@ +import json +import logging +import os +from dataclasses import dataclass, field +from optparse import Values +from typing import Any, Dict, List + +from pip._vendor.packaging.requirements import Requirement + +from pip._internal.cli import cmdoptions +from pip._internal.cli.cmdoptions import make_target_python +from pip._internal.cli.req_command import RequirementCommand, with_cleanup +from pip._internal.cli.status_codes import SUCCESS +from pip._internal.exceptions import CommandError +from pip._internal.models.link import RequirementDownloadInfo +from pip._internal.req.req_tracker import get_requirement_tracker +from pip._internal.resolution.base import RequirementSetWithCandidates +from pip._internal.utils.misc import ensure_dir, normalize_path, write_output +from pip._internal.utils.temp_dir import TempDirectory + +logger = logging.getLogger(__name__) + + +@dataclass +class DownloadInfos: + implicit_requirements: List[Requirement] = field(default_factory=list) + resolution: Dict[str, RequirementDownloadInfo] = field(default_factory=dict) + + def as_basic_log(self) -> str: + implicits = ", ".join(f"'{req}'" for req in self.implicit_requirements) + resolved = "\n".join( + f"{info.req}: {info.url}" for info in self.resolution.values() + ) + return '\n'.join([ + f"Implicit requirements: {implicits}", + "Resolution:", + f"{resolved}", + ]) + + def as_json(self) -> Dict[str, Any]: + return { + "implicit_requirements": [str(req) for req in self.implicit_requirements], + "resolution": { + name: info.as_json() for name, info in self.resolution.items() + }, + } + + +class ResolveCommand(RequirementCommand): + """ + Download packages from: + + - PyPI (and other indexes) using requirement specifiers. + - VCS project urls. + - Local project directories. + - Local or remote source archives. + + pip also supports downloading from "requirements files", which provide + an easy way to specify a whole environment to be downloaded. + """ + + usage = """ + %prog [options] [package-index-options] ... + %prog [options] -r [package-index-options] ... + %prog [options] ... + %prog [options] ... + %prog [options] ...""" + + def add_options(self) -> None: + self.cmd_opts.add_option(cmdoptions.constraints()) + self.cmd_opts.add_option(cmdoptions.requirements()) + self.cmd_opts.add_option(cmdoptions.no_deps()) + self.cmd_opts.add_option(cmdoptions.global_options()) + self.cmd_opts.add_option(cmdoptions.no_binary()) + self.cmd_opts.add_option(cmdoptions.only_binary()) + self.cmd_opts.add_option(cmdoptions.prefer_binary()) + self.cmd_opts.add_option(cmdoptions.src()) + self.cmd_opts.add_option(cmdoptions.pre()) + self.cmd_opts.add_option(cmdoptions.require_hashes()) + self.cmd_opts.add_option(cmdoptions.progress_bar()) + self.cmd_opts.add_option(cmdoptions.no_build_isolation()) + self.cmd_opts.add_option(cmdoptions.use_pep517()) + self.cmd_opts.add_option(cmdoptions.no_use_pep517()) + self.cmd_opts.add_option(cmdoptions.ignore_requires_python()) + + self.cmd_opts.add_option( + "-d", + "--dest", + "--destination-dir", + "--destination-directory", + dest="download_dir", + metavar="dir", + default=os.curdir, + help="Download packages into .", + ) + + self.cmd_opts.add_option( + "-o", + "--json", + "--json-output", + "--json-output-file", + dest="json_output_file", + metavar="file", + help="Print a JSON object representing the resolve into .", + ) + + cmdoptions.add_target_python_options(self.cmd_opts) + + index_opts = cmdoptions.make_option_group( + cmdoptions.index_group, + self.parser, + ) + + self.parser.insert_option_group(0, index_opts) + self.parser.insert_option_group(0, self.cmd_opts) + + @with_cleanup + def run(self, options: Values, args: List[str]) -> int: + + options.ignore_installed = True + # editable doesn't really make sense for `pip download`, but the bowels + # of the RequirementSet code require that property. + options.editables = [] + + cmdoptions.check_dist_restriction(options) + + options.download_dir = normalize_path(options.download_dir) + ensure_dir(options.download_dir) + + session = self.get_default_session(options) + + target_python = make_target_python(options) + finder = self._build_package_finder( + options=options, + session=session, + target_python=target_python, + ignore_requires_python=options.ignore_requires_python, + ) + + req_tracker = self.enter_context(get_requirement_tracker()) + + directory = TempDirectory( + delete=not options.no_clean, + kind="download", + globally_managed=True, + ) + + reqs = self.get_requirements(args, options, finder, session) + + preparer = self.make_requirement_preparer( + temp_build_dir=directory, + options=options, + req_tracker=req_tracker, + session=session, + finder=finder, + download_dir=options.download_dir, + use_user_site=False, + ) + + resolver = self.make_resolver( + preparer=preparer, + finder=finder, + options=options, + ignore_requires_python=options.ignore_requires_python, + py_version_info=options.python_version, + avoid_wheel_downloads=True, + ) + + self.trace_basic_info(finder) + + requirement_set = resolver.resolve(reqs, check_supported_wheels=True) + + downloaded: List[str] = [] + for req in requirement_set.requirements.values(): + # If this distribution was not already satisfied, that means we + # downloaded it while executing this command. + if req.satisfied_by is None: + preparer.save_linked_requirement(req) + assert req.name is not None + downloaded.append(req.name) + + download_infos = DownloadInfos() + if not isinstance(requirement_set, RequirementSetWithCandidates): + raise CommandError( + "The legacy resolver is being used via " + "--use-deprecated=legacy-resolver." + "The legacy resolver does not retain detailed dependency information, " + "so `pip resolve` cannot be used with it. " + ) + for candidate in requirement_set.candidates.mapping.values(): + # This will occur for the python version requirement, for example. + if candidate.name not in requirement_set.requirements: + assert tuple(candidate.iter_dependencies(with_requires=True)) == () + download_infos.implicit_requirements.append( + candidate.as_serializable_requirement() + ) + continue + req = requirement_set.requirements[candidate.name] + assert req.name is not None + assert req.link is not None + assert req.name not in download_infos.resolution + + dependencies: List[Requirement] = [] + for maybe_dep in candidate.iter_dependencies(with_requires=True): + if maybe_dep is None: + continue + maybe_req = maybe_dep.as_serializable_requirement() + if maybe_req is None: + continue + dependencies.append(maybe_req) + + download_infos.resolution[ + req.name + ] = RequirementDownloadInfo.from_req_and_link_and_deps( + req=candidate.as_serializable_requirement(), + dependencies=dependencies, + link=req.link, + ) + + if downloaded: + write_output("Successfully downloaded %s", " ".join(downloaded)) + write_output(download_infos.as_basic_log()) + if options.json_output_file: + with open(options.json_output_file, "w") as f: + json.dump(download_infos.as_json(), f, indent=4) + + return SUCCESS