From 77aa19b63c9e2aad410e6f5654c19f78b8101ddf Mon Sep 17 00:00:00 2001 From: Albert Tugushev Date: Fri, 12 Mar 2021 23:42:57 +0700 Subject: [PATCH] Add typings for the scripts.compile module (#1322) --- piptools/repositories/base.py | 21 +++++++++ piptools/repositories/local.py | 12 ++---- piptools/repositories/pypi.py | 38 ++++++++++++----- piptools/scripts/compile.py | 78 +++++++++++++++++----------------- piptools/utils.py | 2 +- piptools/writer.py | 6 +-- tests/conftest.py | 15 +++++++ 7 files changed, 111 insertions(+), 61 deletions(-) diff --git a/piptools/repositories/base.py b/piptools/repositories/base.py index d5c8a9dce..734ac6f57 100644 --- a/piptools/repositories/base.py +++ b/piptools/repositories/base.py @@ -1,11 +1,17 @@ +import optparse from abc import ABCMeta, abstractmethod from contextlib import contextmanager from typing import Iterator, Optional, Set +from pip._internal.index.package_finder import PackageFinder +from pip._internal.models.index import PyPI +from pip._internal.network.session import PipSession from pip._internal.req import InstallRequirement class BaseRepository(metaclass=ABCMeta): + DEFAULT_INDEX_URL = PyPI.simple_url + def clear_caches(self) -> None: """Should clear any caches used by the implementation.""" @@ -51,3 +57,18 @@ def copy_ireq_dependencies( it its name, we would lose track of those dependencies on combining that ireq with others. """ + + @property + @abstractmethod + def options(self) -> optparse.Values: + """Returns parsed pip options""" + + @property + @abstractmethod + def session(self) -> PipSession: + """Returns a session to make requests""" + + @property + @abstractmethod + def finder(self) -> PackageFinder: + """Returns a package finder to interact with simple repository API (PEP 503)""" diff --git a/piptools/repositories/local.py b/piptools/repositories/local.py index 9c2e136d1..40edca7ef 100644 --- a/piptools/repositories/local.py +++ b/piptools/repositories/local.py @@ -1,6 +1,6 @@ import optparse from contextlib import contextmanager -from typing import Dict, Iterator, List, Optional, Set, cast +from typing import Iterator, Mapping, Optional, Set, cast from pip._internal.index.package_finder import PackageFinder from pip._internal.models.candidate import InstallationCandidate @@ -41,7 +41,7 @@ class LocalRequirementsRepository(BaseRepository): def __init__( self, - existing_pins: Dict[str, InstallationCandidate], + existing_pins: Mapping[str, InstallationCandidate], proxied_repository: PyPIRepository, reuse_hashes: bool = True, ): @@ -50,8 +50,8 @@ def __init__( self.existing_pins = existing_pins @property - def options(self) -> List[optparse.Option]: - return cast(List[optparse.Option], self.repository.options) + def options(self) -> optparse.Values: + return self.repository.options @property def finder(self) -> PackageFinder: @@ -61,10 +61,6 @@ def finder(self) -> PackageFinder: def session(self) -> Session: return self.repository.session - @property - def DEFAULT_INDEX_URL(self) -> str: - return cast(str, self.repository.DEFAULT_INDEX_URL) - def clear_caches(self) -> None: self.repository.clear_caches() diff --git a/piptools/repositories/pypi.py b/piptools/repositories/pypi.py index 89afbbbec..732547775 100644 --- a/piptools/repositories/pypi.py +++ b/piptools/repositories/pypi.py @@ -2,6 +2,7 @@ import hashlib import itertools import logging +import optparse import os from contextlib import contextmanager from shutil import rmtree @@ -11,10 +12,13 @@ from pip._internal.cache import WheelCache from pip._internal.cli.progress_bars import BAR_TYPES from pip._internal.commands import create_command +from pip._internal.commands.install import InstallCommand +from pip._internal.index.package_finder import PackageFinder from pip._internal.models.candidate import InstallationCandidate -from pip._internal.models.index import PackageIndex, PyPI +from pip._internal.models.index import PackageIndex from pip._internal.models.link import Link from pip._internal.models.wheel import Wheel +from pip._internal.network.session import PipSession from pip._internal.req import InstallRequirement, RequirementSet from pip._internal.req.req_tracker import get_requirement_tracker from pip._internal.utils.hashes import FAVORITE_HASH @@ -43,7 +47,6 @@ class PyPIRepository(BaseRepository): - DEFAULT_INDEX_URL = PyPI.simple_url HASHABLE_PACKAGE_TYPES = {"bdist_wheel", "sdist"} """ @@ -57,18 +60,19 @@ def __init__(self, pip_args: List[str], cache_dir: str): # Use pip's parser for pip.conf management and defaults. # General options (find_links, index_url, extra_index_url, trusted_host, # and pre) are deferred to pip. - self.command = create_command("install") + self.command: InstallCommand = create_command("install") extra_pip_args = ["--use-deprecated", "legacy-resolver"] - self.options, _ = self.command.parse_args(pip_args + extra_pip_args) - if self.options.cache_dir: - self.options.cache_dir = normalize_path(self.options.cache_dir) - self.options.require_hashes = False - self.options.ignore_dependencies = False + options, _ = self.command.parse_args(pip_args + extra_pip_args) + if options.cache_dir: + options.cache_dir = normalize_path(options.cache_dir) + options.require_hashes = False + options.ignore_dependencies = False - self.session = self.command._build_session(self.options) - self.finder = self.command._build_package_finder( - options=self.options, session=self.session + self._options: optparse.Values = options + self._session = self.command._build_session(options) + self._finder = self.command._build_package_finder( + options=options, session=self.session ) # Caches @@ -91,6 +95,18 @@ def __init__(self, pip_args: List[str], cache_dir: str): def clear_caches(self) -> None: rmtree(self._download_dir, ignore_errors=True) + @property + def options(self) -> optparse.Values: + return self._options + + @property + def session(self) -> PipSession: + return self._session + + @property + def finder(self) -> PackageFinder: + return self._finder + def find_all_candidates(self, req_name: str) -> List[InstallationCandidate]: if req_name not in self._available_candidates_cache: candidates = self.finder.find_all_candidates(req_name) diff --git a/piptools/scripts/compile.py b/piptools/scripts/compile.py index 2b1fe2038..eaae76991 100755 --- a/piptools/scripts/compile.py +++ b/piptools/scripts/compile.py @@ -2,10 +2,10 @@ import shlex import sys import tempfile -from typing import Any +from typing import Any, BinaryIO, Optional, Tuple, cast import click -from click.utils import safecall +from click.utils import LazyFile, safecall from pip._internal.commands import create_command from pip._internal.req.constructors import install_req_from_line from pip._internal.utils.misc import redact_auth_from_url @@ -17,6 +17,7 @@ from ..locations import CACHE_DIR from ..logging import log from ..repositories import LocalRequirementsRepository, PyPIRepository +from ..repositories.base import BaseRepository from ..resolver import Resolver from ..utils import UNSAFE_PACKAGES, dedup, is_pinned_requirement, key_from_ireq from ..writer import OutputWriter @@ -186,7 +187,9 @@ def _get_default_option(option_name: str) -> Any: show_default=True, type=click.Path(file_okay=False, writable=True), ) -@click.option("--pip-args", help="Arguments to pass directly to the pip command.") +@click.option( + "--pip-args", "pip_args_str", help="Arguments to pass directly to the pip command." +) @click.option( "--emit-index-url/--no-emit-index-url", is_flag=True, @@ -194,35 +197,35 @@ def _get_default_option(option_name: str) -> Any: help="Add index URL to generated file", ) def cli( - ctx, - verbose, - quiet, - dry_run, - pre, - rebuild, - find_links, - index_url, - extra_index_url, - cert, - client_cert, - trusted_host, - header, - emit_trusted_host, - annotate, - upgrade, - upgrade_packages, - output_file, - allow_unsafe, - generate_hashes, - reuse_hashes, - src_files, - max_rounds, - build_isolation, - emit_find_links, - cache_dir, - pip_args, - emit_index_url, -): + ctx: click.Context, + verbose: int, + quiet: int, + dry_run: bool, + pre: bool, + rebuild: bool, + find_links: Tuple[str], + index_url: str, + extra_index_url: Tuple[str], + cert: Optional[str], + client_cert: Optional[str], + trusted_host: Tuple[str], + header: bool, + emit_trusted_host: bool, + annotate: bool, + upgrade: bool, + upgrade_packages: Tuple[str], + output_file: Optional[LazyFile], + allow_unsafe: bool, + generate_hashes: bool, + reuse_hashes: bool, + src_files: Tuple[str], + max_rounds: int, + build_isolation: bool, + emit_find_links: bool, + cache_dir: str, + pip_args_str: Optional[str], + emit_index_url: bool, +) -> None: """Compiles requirements.txt from requirements.in specs.""" log.verbosity = verbose - quiet @@ -261,13 +264,14 @@ def cli( output_file = click.open_file(file_name, "w+b", atomic=True, lazy=True) # Close the file at the end of the context execution + assert output_file is not None ctx.call_on_close(safecall(output_file.close_intelligently)) ### # Setup ### - right_args = shlex.split(pip_args or "") + right_args = shlex.split(pip_args_str or "") pip_args = [] for link in find_links: pip_args.extend(["-f", link]) @@ -288,6 +292,7 @@ def cli( pip_args.append("--no-build-isolation") pip_args.extend(right_args) + repository: BaseRepository repository = PyPIRepository(pip_args, cache_dir=cache_dir) # Parse all constraints coming from --upgrade-package/-P @@ -412,10 +417,7 @@ def cli( allow_unsafe=allow_unsafe, ) results = resolver.resolve(max_rounds=max_rounds) - if generate_hashes: - hashes = resolver.resolve_hashes(results) - else: - hashes = None + hashes = resolver.resolve_hashes(results) if generate_hashes else None except PipToolsError as e: log.error(str(e)) sys.exit(2) @@ -427,7 +429,7 @@ def cli( ## writer = OutputWriter( - output_file, + cast(BinaryIO, output_file), click_ctx=ctx, dry_run=dry_run, emit_header=header, diff --git a/piptools/utils.py b/piptools/utils.py index a5685ae2e..d58b5c067 100644 --- a/piptools/utils.py +++ b/piptools/utils.py @@ -297,7 +297,7 @@ def get_compile_command(click_ctx: click.Context) -> str: else: if isinstance(val, str) and is_url(val): val = redact_auth_from_url(val) - if option.name == "pip_args": + if option.name == "pip_args_str": # shlex.quote() would produce functional but noisily quoted results, # e.g. --pip-args='--cache-dir='"'"'/tmp/with spaces'"'"'' # Instead, we try to get more legible quoting via repr: diff --git a/piptools/writer.py b/piptools/writer.py index f497a1749..1d319f21a 100644 --- a/piptools/writer.py +++ b/piptools/writer.py @@ -1,7 +1,7 @@ import os import re from itertools import chain -from typing import BinaryIO, Dict, Iterator, List, Optional, Sequence, Set, Tuple +from typing import BinaryIO, Dict, Iterable, Iterator, List, Optional, Set, Tuple from click import unstyle from click.core import Context @@ -62,8 +62,8 @@ def __init__( annotate: bool, generate_hashes: bool, default_index_url: str, - index_urls: Sequence[str], - trusted_hosts: Sequence[str], + index_urls: Iterable[str], + trusted_hosts: Iterable[str], format_control: FormatControl, allow_unsafe: bool, find_links: List[str], diff --git a/tests/conftest.py b/tests/conftest.py index 13104d1f4..e824baf90 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,4 +1,5 @@ import json +import optparse import os import subprocess import sys @@ -8,7 +9,9 @@ import pytest from click.testing import CliRunner +from pip._internal.index.package_finder import PackageFinder from pip._internal.models.candidate import InstallationCandidate +from pip._internal.network.session import PipSession from pip._internal.req.constructors import ( install_req_from_editable, install_req_from_line, @@ -90,6 +93,18 @@ def copy_ireq_dependencies(self, source, dest): # No state to update. pass + @property + def options(self) -> optparse.Values: + """Not used""" + + @property + def session(self) -> PipSession: + """Not used""" + + @property + def finder(self) -> PackageFinder: + """Not used""" + class FakeInstalledDistribution: def __init__(self, line, deps=None):