From aa2d7a4c5c13f1afbedbf7684fae2c91790ac51c Mon Sep 17 00:00:00 2001 From: Wim Jeantine-Glenn Date: Tue, 22 Oct 2024 01:26:03 -0500 Subject: [PATCH 1/7] Remove pip subprocesses, replace with unearth --- .coveragerc | 8 - .github/dependabot.yml | 10 + .github/workflows/ci.yml | 19 +- johnnydep/__init__.py | 11 +- johnnydep/__main__.py | 4 - johnnydep/cli.py | 21 +- johnnydep/dot.py | 12 +- johnnydep/downloader.py | 41 +++ johnnydep/env_check.py | 44 +-- johnnydep/lib.py | 349 ++++++++++++++--------- johnnydep/logs.py | 8 - johnnydep/pipper.py | 262 ----------------- johnnydep/util.py | 43 ++- pyproject.toml | 16 +- tests/conftest.py | 141 ++++----- tests/test_cli.py | 179 +++++++----- tests/test_downloader.py | 80 ++++++ tests/test_gen.py | 33 +-- tests/test_lib.py | 134 ++++----- tests/test_pipper.py | 135 --------- tests/test_util.py | 23 +- tests/vanilla-0.1.2-py2.py3-none-any.whl | Bin 1070 -> 0 bytes 22 files changed, 675 insertions(+), 898 deletions(-) delete mode 100644 .coveragerc create mode 100644 .github/dependabot.yml delete mode 100644 johnnydep/__main__.py create mode 100644 johnnydep/downloader.py delete mode 100755 johnnydep/pipper.py create mode 100644 tests/test_downloader.py delete mode 100644 tests/test_pipper.py delete mode 100644 tests/vanilla-0.1.2-py2.py3-none-any.whl diff --git a/.coveragerc b/.coveragerc deleted file mode 100644 index 4d46ef4..0000000 --- a/.coveragerc +++ /dev/null @@ -1,8 +0,0 @@ -[run] -branch = true -parallel = true - -[report] -exclude_lines = - from johnnydep.cli import main - if __name__ == .__main__.: diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..baf6a61 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,10 @@ +version: 2 +updates: + - package-ecosystem: github-actions + directory: / + schedule: + interval: monthly + groups: + actions-infrastructure: + patterns: + - actions/* diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9c341a8..4b5c699 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2,9 +2,9 @@ name: CI on: push: - branches: ["main"] + branches: [main] pull_request: - branches: ["main"] + branches: [main] workflow_dispatch: jobs: @@ -13,6 +13,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: + fail-fast: false matrix: os: [ubuntu-latest, macos-latest, windows-latest] python-version: @@ -20,21 +21,21 @@ jobs: - "3.9" - "3.10" - "3.11" + - "3.12" + - "3.13" steps: - - uses: actions/checkout@v3 - - uses: actions/setup-python@v4 + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - name: Install dependencies - run: | - python -VV - python -m pip install -r requirements-dev.txt + run: pip install -r requirements-dev.txt - name: Run tests for ${{ matrix.python-version }} on ${{ matrix.os }} - run: python -m pytest + run: pytest - name: Upload coverage to Codecov - uses: codecov/codecov-action@v3 + uses: codecov/codecov-action@v4 with: token: ${{ secrets.CODECOV_TOKEN }} diff --git a/johnnydep/__init__.py b/johnnydep/__init__.py index bd1f865..43bb543 100644 --- a/johnnydep/__init__.py +++ b/johnnydep/__init__.py @@ -1,2 +1,11 @@ """Display dependency tree of Python distribution""" -from johnnydep.lib import * +from types import SimpleNamespace + + +config = SimpleNamespace( + env=None, + index_url=None, + extra_index_url=None, +) + +from .lib import * diff --git a/johnnydep/__main__.py b/johnnydep/__main__.py deleted file mode 100644 index 7a91da8..0000000 --- a/johnnydep/__main__.py +++ /dev/null @@ -1,4 +0,0 @@ -from johnnydep.cli import main - -if __name__ == "__main__": - main() diff --git a/johnnydep/cli.py b/johnnydep/cli.py index 539af4f..3f0f6ea 100644 --- a/johnnydep/cli.py +++ b/johnnydep/cli.py @@ -4,10 +4,11 @@ from importlib.metadata import version import johnnydep -from johnnydep.lib import has_error -from johnnydep.lib import JohnnyDist -from johnnydep.logs import configure_logging -from johnnydep.util import python_interpreter +from . import config +from .lib import has_error +from .lib import JohnnyDist +from .logs import configure_logging +from .util import python_interpreter FIELDS = { @@ -120,13 +121,11 @@ def main(argv=None, stdout=None): if "ALL" in args.fields: args.fields = list(FIELDS) configure_logging(verbosity=args.verbose) - dist = JohnnyDist( - args.req, - index_url=args.index_url, - env=args.env, - extra_index_url=args.extra_index_url, - ignore_errors=args.ignore_errors, - ) + config.index_url = args.index_url + config.extra_index_url = args.extra_index_url + if args.env is not None: + config.env = args.env + dist = JohnnyDist(args.req, ignore_errors=args.ignore_errors) rendered = dist.serialise( fields=args.fields, format=args.output_format, diff --git a/johnnydep/dot.py b/johnnydep/dot.py index 309f19c..142bbbb 100644 --- a/johnnydep/dot.py +++ b/johnnydep/dot.py @@ -1,8 +1,7 @@ from importlib.metadata import version -from anytree import LevelOrderIter - -from johnnydep.util import CircularMarker +from .util import _bfs +from .util import CircularMarker template = """\ @@ -26,13 +25,14 @@ def jd2dot(dist, comment=None): comment = "# " + comment title = dist.project_name.replace("-", "_") edges = [] - for node in LevelOrderIter(dist): + for node in _bfs(dist): if isinstance(node, CircularMarker): # todo - render cycles differently? continue - if node.parent is not None: + if node.parents: + [parent] = node.parents node_name = node._name_with_extras(attr="project_name") - parent_node_name = node.parent._name_with_extras(attr="project_name") + parent_node_name = parent._name_with_extras(attr="project_name") spec = node.req.specifier label = f' [label="{spec}"]' if spec else "" edge = f'"{parent_node_name}" -> "{node_name}"' diff --git a/johnnydep/downloader.py b/johnnydep/downloader.py new file mode 100644 index 0000000..1ad5f9f --- /dev/null +++ b/johnnydep/downloader.py @@ -0,0 +1,41 @@ +from urllib.parse import urlparse +from urllib.request import build_opener +from urllib.request import HTTPBasicAuthHandler +from urllib.request import HTTPPasswordMgrWithDefaultRealm + +from structlog import get_logger + + +log = get_logger(__name__) + + +def _urlretrieve(url, f, data=None, auth=None): + if auth is None: + opener = build_opener() + else: + # https://docs.python.org/3/howto/urllib2.html#id5 + password_mgr = HTTPPasswordMgrWithDefaultRealm() + username, password = auth + top_level_url = urlparse(url).netloc + password_mgr.add_password(None, top_level_url, username, password) + handler = HTTPBasicAuthHandler(password_mgr) + opener = build_opener(handler) + res = opener.open(url, data=data) + log.debug("resp info", url=url, headers=res.info()) + f.write(res.read()) + f.flush() + + +def download_dist(url, f, index_url, extra_index_url): + auth = None + if index_url: + parsed = urlparse(index_url) + if parsed.username and parsed.password and parsed.hostname == urlparse(url).hostname: + # handling private PyPI credentials in index_url + auth = (parsed.username, parsed.password) + if extra_index_url: + parsed = urlparse(extra_index_url) + if parsed.username and parsed.password and parsed.hostname == urlparse(url).hostname: + # handling private PyPI credentials in extra_index_url + auth = (parsed.username, parsed.password) + _urlretrieve(url, f, auth=auth) diff --git a/johnnydep/env_check.py b/johnnydep/env_check.py index 8f5db64..9c1bf03 100644 --- a/johnnydep/env_check.py +++ b/johnnydep/env_check.py @@ -1,42 +1,22 @@ import json -import os -import platform import sys - -def format_full_version(info): - version = "{0.major}.{0.minor}.{0.micro}".format(info) - kind = info.releaselevel - if kind != "final": - version += kind[0] + str(info.serial) - return version - - -# cribbed from packaging.markers to avoid a runtime dependency here -def default_environment(): - iver = format_full_version(sys.implementation.version) - implementation_name = sys.implementation.name - return { - "implementation_name": implementation_name, - "implementation_version": iver, - "os_name": os.name, - "platform_machine": platform.machine(), - "platform_release": platform.release(), - "platform_system": platform.system(), - "platform_version": platform.version(), - "python_full_version": platform.python_version(), - "platform_python_implementation": platform.python_implementation(), - "python_version": ".".join(platform.python_version_tuple()[:2]), - "sys_platform": sys.platform, - } +from packaging.markers import default_environment +from packaging.tags import interpreter_name +from unearth.pep425tags import get_supported def main(): - env = default_environment() + env = {} + env.update(default_environment()) env["python_executable"] = sys.executable - env = sorted(env.items()) - result = json.dumps(env, indent=2) - print(result) + env["py_ver"] = sys.version_info[0], sys.version_info[1] + env["impl"] = interpreter_name() + env["platforms"] = None + env["abis"] = None + env["supported_tags"] = ",".join(map(str, get_supported())) + txt = json.dumps(env, indent=2, sort_keys=True) + print(txt) if __name__ == "__main__": diff --git a/johnnydep/lib.py b/johnnydep/lib.py index edcf129..5a37d8c 100644 --- a/johnnydep/lib.py +++ b/johnnydep/lib.py @@ -1,35 +1,48 @@ +import hashlib +import io import json import os import re import subprocess +import sys from collections import defaultdict +from collections import deque from functools import cached_property +from functools import lru_cache from importlib.metadata import distribution from importlib.metadata import PackageNotFoundError from importlib.metadata import PathDistribution +from pathlib import Path from shutil import rmtree from tempfile import mkdtemp +from textwrap import dedent +from urllib.parse import urlparse from zipfile import Path as zipfile_path from zipfile import ZipFile -import anytree -import tabulate -import toml +import rich.box +import rich.markup +import tomli_w +import unearth import yaml -from cachetools.func import ttl_cache from packaging import requirements from packaging.markers import default_environment +from packaging.tags import parse_tag from packaging.utils import canonicalize_name from packaging.utils import canonicalize_version -from packaging.version import parse as parse_version +from packaging.version import Version +from rich.table import Table +from rich.tree import Tree from structlog import get_logger -from johnnydep import pipper -from johnnydep.dot import jd2dot -from johnnydep.util import CircularMarker - -__all__ = ["JohnnyDist", "gen_table", "flatten_deps", "has_error", "JohnnyError"] +from . import config +from .dot import jd2dot +from .downloader import download_dist +from .util import _bfs +from .util import _un_none +from .util import CircularMarker +__all__ = ["JohnnyDist", "gen_table", "gen_tree", "flatten_deps", "has_error", "JohnnyError"] logger = get_logger(__name__) @@ -38,44 +51,45 @@ class JohnnyError(Exception): pass -class JohnnyDist(anytree.NodeMixin): - def __init__(self, req_string, parent=None, index_url=None, env=None, extra_index_url=None, ignore_errors=False): +def get_or_create(req_string): + pass + + +class JohnnyDist: + def __init__(self, req_string, parent=None, ignore_errors=False): + if isinstance(req_string, Path): + req_string = str(req_string) log = self.log = logger.bind(dist=req_string) log.info("init johnnydist", parent=parent and str(parent.req)) - self.parent = parent - self.index_url = index_url - self.env = env - self.extra_index_url = extra_index_url + self._children = [] + self.parents = [] + if parent is not None: + self.parents.append(parent) self.ignore_errors = ignore_errors self.error = None self._recursed = False fname, sep, extras = req_string.partition("[") - if fname.endswith(".whl") and os.path.isfile(fname): + if fname.endswith(".whl") and Path(fname).is_file(): # crudely parse dist name and version from wheel filename # see https://peps.python.org/pep-0427/#file-name-convention - parts = os.path.basename(fname).split("-") - self.name = canonicalize_name(parts[0]) - self.specifier = "==" + canonicalize_version(parts[1]) + name, version, *rest = Path(fname).name.split("-") + self.name = canonicalize_name(name) + self.specifier = "==" + canonicalize_version(version) self.req = requirements.Requirement(self.name + sep + extras + self.specifier) self.import_names = _discover_import_names(fname) self.metadata = _extract_metadata(fname) self.entry_points = _discover_entry_points(fname) - self._from_fname = os.path.abspath(fname) + self._local_path = Path(fname).resolve() else: - self._from_fname = None + self._local_path = None self.req = requirements.Requirement(req_string) self.name = canonicalize_name(self.req.name) self.specifier = str(self.req.specifier) log.debug("fetching best wheel") try: - self.import_names, self.metadata, self.entry_points = _get_info( - dist_name=req_string, - index_url=index_url, - env=env, - extra_index_url=extra_index_url - ) - except subprocess.CalledProcessError as err: + self.import_names, self.metadata, self.entry_points = _get_info(dist_name=req_string) + except Exception as err: if not self.ignore_errors: raise self.import_names = None @@ -85,8 +99,6 @@ def __init__(self, req_string, parent=None, index_url=None, env=None, extra_inde self.extras_requested = sorted(self.req.extras) if parent is None: - if env: - log.debug("root node target env", **dict(env)) self.required_by = [] else: self.required_by = [str(parent.req)] @@ -98,10 +110,6 @@ def requires(self): if not all_requires: return [] result = [] - if self.env is None: - env_data = default_environment() - else: - env_data = dict(self.env) for req_str in all_requires: req = requirements.Requirement(req_str) req_short, _sep, _marker = str(req).partition(";") @@ -111,7 +119,7 @@ def requires(self): continue # conditional dependency - must be evaluated in environment context for extra in [None] + self.extras_requested: - if req.marker.evaluate(dict(env_data, extra=extra)): + if req.marker.evaluate(dict(config.env or default_environment(), extra=extra)): self.log.debug("included conditional dep", req=req_str) result.append(req_short) break @@ -124,25 +132,25 @@ def requires(self): def children(self): """my immediate deps, as a tuple of johnnydists""" if not self._recursed: - self.log.debug("populating dep tree") + assert not self._children + self.log.debug("populating dep graph") circular_deps = _detect_circular(self) if circular_deps: chain = " -> ".join([d._name_with_extras() for d in circular_deps]) summary = f"... " self.log.info("pruning circular dependency", chain=chain) _dep = CircularMarker(summary=summary, parent=self) + self._children = [_dep] else: for dep in self.requires: - JohnnyDist( + child = JohnnyDist( req_string=dep, parent=self, - index_url=self.index_url, - env=self.env, - extra_index_url=self.extra_index_url, ignore_errors=self.ignore_errors, ) + self._children.append(child) self._recursed = True - return super(JohnnyDist, self).children + return self._children @property def homepage(self): @@ -179,24 +187,21 @@ def license(self): @cached_property def versions_available(self): - result = pipper.get_versions( - self.project_name, - index_url=self.index_url, - env=self.env, - extra_index_url=self.extra_index_url, - ) - if self._from_fname is not None: - raw_version = os.path.basename(self._from_fname).split("-")[1] + finder = _get_package_finder() + matches = finder.find_matches(self.project_name, allow_prereleases=True) + versions = [p.version for p in matches][::-1] + if self._local_path is not None: + raw_version = self._local_path.name.split("-")[1] local_version = canonicalize_version(raw_version) - version_key = parse_version(local_version) - if local_version not in result: + version_key = Version(local_version) + if local_version not in versions: # when we're Python 3.10+ only, can use bisect.insort instead here i = 0 - for i, v in enumerate(result): - if version_key < parse_version(v): + for i, v in enumerate(versions): + if version_key < Version(v): break - result.insert(i, local_version) - return result + versions.insert(i, local_version) + return versions @cached_property def version_installed(self): @@ -254,28 +259,26 @@ def pinned(self): result = f"{self.project_name}{extras}=={version}" return result - @cached_property - def _best(self): - return pipper.get( - self.pinned, - index_url=self.index_url, - env=self.env, - extra_index_url=self.extra_index_url, - ignore_errors=True, - ) - @property def download_link(self): - if self._from_fname is not None: - return f"file://{self._from_fname}" - return self._best.get("url") + if self._local_path is not None: + return f"file://{self._local_path}" + package_finder = _get_package_finder() + return package_finder.find_best_match(self.req, allow_prereleases=True).best.link.url @property def checksum(self): - if self._from_fname is not None: - md5 = pipper.compute_checksum(self._from_fname, algorithm="md5") - return f"md5={md5}" - return self._best.get("checksum") + if self._local_path is not None: + return "md5=" + hashlib.md5(self._local_path.read_bytes()).hexdigest() + link = self.download_link + f = io.BytesIO() + download_dist( + url=link, + f=f, + index_url=config.index_url, + extra_index_url=config.extra_index_url, + ) + return "md5=" + hashlib.md5(f.getvalue()).hexdigest() def serialise(self, fields=("name", "summary"), recurse=True, format=None): if format == "pinned": @@ -285,11 +288,21 @@ def serialise(self, fields=("name", "summary"), recurse=True, format=None): return jd2dot(self) data = [{f: getattr(self, f, None) for f in fields}] if format == "human": - table = gen_table(self, extra_cols=fields) - if not recurse: - table = [next(table)] - tabulate.PRESERVE_WHITESPACE = True - return tabulate.tabulate(table, headers="keys") + cols = dict.fromkeys(fields) + cols.pop("name", None) + with_specifier = "specifier" not in cols + if recurse: + tree = gen_tree(self, with_specifier=with_specifier) + else: + tree = Tree(_to_str(self, with_specifier)) + tree.dist = self + table = gen_table(tree, cols=cols) + buf = io.StringIO() + rich.print(table, file=buf) + raw = buf.getvalue() + stripped = "\n".join([x.rstrip() for x in raw.splitlines() if x.strip()]) + result = dedent(stripped) + return result if recurse and self.requires: deps = flatten_deps(self) next(deps) # skip over root @@ -301,7 +314,13 @@ def serialise(self, fields=("name", "summary"), recurse=True, format=None): elif format == "yaml": result = yaml.safe_dump(data, sort_keys=False) elif format == "toml": - result = "\n".join([toml.dumps(d) for d in data]) + options = {} + can_indent = Version(tomli_w.__version__) >= Version("1.1.0") + if can_indent: + options["indent"] = 2 + result = "\n".join([tomli_w.dumps(_un_none(d), **options) for d in data]) + if not can_indent: + result = re.sub(r"^ ", " ", result, flags=re.MULTILINE) elif format == "pinned": result = "\n".join([d["pinned"] for d in data]) else: @@ -325,48 +344,85 @@ def _repr_pretty_(self, p, cycle): p.text(f"<{type(self).__name__} {fullname} at {hex(id(self))}>") -def gen_table(johnnydist, extra_cols=()): - extra_cols = {}.fromkeys(extra_cols) # de-dupe and preserve ordering - extra_cols.pop("name", None) # this is always included anyway, no need to ask for it - johnnydist.log.debug("generating table") - for prefix, _fill, dist in anytree.RenderTree(johnnydist): - row = {} - txt = str(dist.req) - if dist.error: - txt += " (FAILED)" - if "specifier" in extra_cols: - # can use https://docs.python.org/3/library/stdtypes.html#str.removesuffix - # after dropping support for Python-3.8 - suffix = str(dist.specifier) - if txt.endswith(suffix): - txt = txt[:len(txt) - len(suffix)] - row["name"] = prefix + txt - for col in extra_cols: - val = getattr(dist, col, "") - if isinstance(val, list): - val = ", ".join(val) - row[col] = val - yield row +def _to_str(dist, with_specifier=True): + txt = str(dist.req) + if dist.error: + txt += " (FAILED)" + if not with_specifier: + # can use https://docs.python.org/3/library/stdtypes.html#str.removesuffix + # after dropping support for Python-3.8 + suffix = str(dist.specifier) + if txt.endswith(suffix): + txt = txt[:len(txt) - len(suffix)] + return rich.markup.escape(txt) + + +def gen_tree(johnnydist, with_specifier=True): + johnnydist.log.debug("generating tree") + seen = set() + tree = Tree(_to_str(johnnydist, with_specifier)) + tree.dist = johnnydist + q = deque([tree]) + while q: + node = q.popleft() + jd = node.dist + pk = id(jd) + if pk in seen: + continue + seen.add(pk) + for child in jd.children: + tchild = node.add(_to_str(child, with_specifier)) + tchild.dist = child + q.append(tchild) + return tree + + +def gen_table(tree, cols): + table = Table(box=rich.box.SIMPLE_HEAVY, safe_box=False) + table.add_column("name", overflow="fold") + for col in cols: + table.add_column(col, overflow="fold") + rows = [] + stack = [tree] + while stack: + node = stack.pop() + rows.append(node) + stack += reversed(node.children) + buf = io.StringIO() + rich.print(tree, file=buf) + tree_lines = buf.getvalue().splitlines() + for row0, row in zip(tree_lines, rows): + data = [getattr(row.dist, c) for c in cols] + for i, d in enumerate(data): + if d is None: + data[i] = "" + elif not isinstance(d, str): + data[i] = ", ".join(map(str, d)) + escaped = [rich.markup.escape(x) for x in [row0, *data]] + table.add_row(*escaped) + return table def _detect_circular(dist): # detects a circular dependency when traversing from here to the root node, and returns # a chain of nodes in that case + # TODO: fix this for full DAG + dist0 = dist chain = [dist] - for ancestor in reversed(dist.ancestors): - chain.append(ancestor) - if ancestor.name == dist.name: - if ancestor.extras_requested == dist.extras_requested: - return chain[::-1] + while dist.parents: + [dist] = dist.parents + chain.append(dist) + if dist.name == dist0.name and dist.extras_requested == dist0.extras_requested: + return chain[::-1] def flatten_deps(johnnydist): - johnnydist.log.debug("resolving dep tree") + johnnydist.log.debug("resolving dep graph") dist_map = defaultdict(list) spec_map = defaultdict(str) extra_map = defaultdict(set) required_by_map = defaultdict(list) - for dep in anytree.iterators.LevelOrderIter(johnnydist): + for dep in _bfs(johnnydist): if dep.name == CircularMarker.glyph: continue dist_map[dep.name].append(dep) @@ -403,12 +459,7 @@ def flatten_deps(johnnydist): extra = f"[{','.join(sorted(extras))}]" else: extra = "" - dist = JohnnyDist( - req_string=f"{name}{extra}{spec}", - index_url=johnnydist.index_url, - env=johnnydist.env, - extra_index_url=johnnydist.extra_index_url, - ) + dist = JohnnyDist(req_string=f"{name}{extra}{spec}") dist.required_by = required_by yield dist # TODO: check if this new version causes any new reqs!! @@ -431,12 +482,9 @@ def _discover_import_names(whl_file): if len(parts) == 2 and parts[1] == "__init__.py": # found a top-level package public_names.append(parts[0]) - elif len(parts) == 1: - # TODO: find or make an exhaustive list of file extensions importable - name, ext = os.path.splitext(parts[0]) - if ext == ".py" or ext == ".so": - # found a top level module - public_names.append(name) + elif len(parts) == 1 and name.endswith((".py", ".so", ".pyd")): + # found a top level module + public_names.append(name.split(".")[0]) else: all_names = zf.read(top_level_fname).decode("utf-8").strip().splitlines() public_names = [n for n in all_names if not n.startswith("_")] @@ -445,7 +493,7 @@ def _discover_import_names(whl_file): def _path_dist(whl_file): - parts = os.path.basename(whl_file).split("-") + parts = Path(whl_file).name.split("-", maxsplit=2) metadata_path = "-".join(parts[:2]) + ".dist-info/" zf_path = zipfile_path(whl_file, metadata_path) return PathDistribution(zf_path) @@ -502,20 +550,55 @@ def has_error(dist): return any(has_error(n) for n in dist.children) -@ttl_cache(maxsize=512, ttl=60 * 5) -def _get_info(dist_name, index_url=None, env=None, extra_index_url=None): +def _get_package_finder(): + index_urls = [] + trusted_hosts = [] + if config.index_url: + index_urls.append(config.index_url) + trusted_hosts.append(urlparse(config.index_url).hostname) + if config.extra_index_url: + index_urls.append(config.extra_index_url) + trusted_hosts.append(urlparse(config.extra_index_url).hostname) + target_python = None + if config.env is not None: + target_python = unearth.TargetPython( + py_ver=config.env["py_ver"], + impl=config.env["impl"], + ) + valid_tags = [] + for tag in config.env["supported_tags"].split(","): + valid_tags.extend(parse_tag(tag)) + target_python._valid_tags = valid_tags + package_finder = unearth.PackageFinder( + index_urls=index_urls, + target_python=target_python, + trusted_hosts=trusted_hosts, + ) + return package_finder + + +@lru_cache(maxsize=None) +def _get_info(dist_name): log = logger.bind(dist_name=dist_name) tmpdir = mkdtemp() log.debug("created scratch", tmpdir=tmpdir) try: - data = pipper.get( - dist_name, - index_url=index_url, - env=env, - extra_index_url=extra_index_url, - tmpdir=tmpdir, - ) - dist_path = data["path"] + package_finder = _get_package_finder() + best = package_finder.find_best_match(dist_name, allow_prereleases=True).best + if best is None: + raise JohnnyError(f"Package not found {dist_name!r}") + dist_path = Path(tmpdir) / best.link.filename + with dist_path.open("wb") as f: + download_dist( + url=best.link.url, + f=f, + index_url=config.index_url, + extra_index_url=config.extra_index_url, + ) + if not dist_path.name.endswith("whl"): + args = [sys.executable, "-m", "uv", "build", "--wheel", str(dist_path)] + subprocess.run(args, capture_output=True, check=True) + [dist_path] = dist_path.parent.glob("*.whl") # extract any info we may need from downloaded dist right now, so the # downloaded file can be cleaned up immediately import_names = _discover_import_names(dist_path) @@ -525,7 +608,3 @@ def _get_info(dist_name, index_url=None, env=None, extra_index_url=None): log.debug("removing scratch", tmpdir=tmpdir) rmtree(tmpdir, ignore_errors=True) return import_names, metadata, entry_points - - -# TODO: multi-line progress bar? -# TODO: upload test dists to test PyPI index, document pip existing failure modes diff --git a/johnnydep/logs.py b/johnnydep/logs.py index b4fc42e..f4ac655 100644 --- a/johnnydep/logs.py +++ b/johnnydep/logs.py @@ -6,7 +6,6 @@ def configure_logging(verbosity=0): level = "DEBUG" if verbosity > 1 else "INFO" if verbosity == 1 else "WARNING" timestamper = structlog.processors.TimeStamper(fmt="%Y-%m-%d %H:%M:%S") - # Add the log level and a timestamp to the event_dict if the log entry is not from structlog pre_chain = [structlog.stdlib.add_log_level, timestamper] logging.config.dictConfig( { @@ -30,17 +29,10 @@ def configure_logging(verbosity=0): "class": "logging.StreamHandler", "formatter": "colored", }, - # "file": { - # "level": "DEBUG", - # "class": "logging.handlers.WatchedFileHandler", - # "filename": "johnnydep.log", - # "formatter": "plain", - # }, }, "loggers": { "": { "handlers": ["default"], - # "handlers": ["default", "file"], "level": "DEBUG", "propagate": True, } diff --git a/johnnydep/pipper.py b/johnnydep/pipper.py deleted file mode 100755 index 9e8e299..0000000 --- a/johnnydep/pipper.py +++ /dev/null @@ -1,262 +0,0 @@ -import hashlib -import json -import os -import sys -import tempfile -from argparse import ArgumentParser -from glob import glob -from subprocess import CalledProcessError -from subprocess import check_output -from subprocess import STDOUT -from urllib.parse import urlparse -from urllib.request import build_opener -from urllib.request import HTTPBasicAuthHandler -from urllib.request import HTTPPasswordMgrWithDefaultRealm - -from cachetools import cached -from cachetools import TTLCache -from cachetools.func import ttl_cache -from cachetools.keys import hashkey -from packaging import requirements -from structlog import get_logger - -from johnnydep.logs import configure_logging -from johnnydep.util import python_interpreter - -log = get_logger(__name__) - - -DEFAULT_INDEX = "https://pypi.org/simple/" - - -def urlretrieve(url, filename, data=None, auth=None): - if auth is None: - opener = build_opener() - else: - # https://docs.python.org/3/howto/urllib2.html#id5 - password_mgr = HTTPPasswordMgrWithDefaultRealm() - username, password = auth - top_level_url = urlparse(url).netloc - password_mgr.add_password(None, top_level_url, username, password) - handler = HTTPBasicAuthHandler(password_mgr) - opener = build_opener(handler) - res = opener.open(url, data=data) - headers = res.info() - with open(filename, "wb") as fp: - fp.write(res.read()) - return filename, headers - - -def compute_checksum(target, algorithm="sha256", blocksize=2 ** 13): - hashtype = getattr(hashlib, algorithm) - hash_ = hashtype() - log.debug("computing checksum", target=target, algorithm=algorithm) - with open(target, "rb") as f: - for chunk in iter(lambda: f.read(blocksize), b""): - hash_.update(chunk) - result = hash_.hexdigest() - log.debug("computed checksum", result=result) - return result - - -def _get_wheel_args(index_url, env, extra_index_url): - args = [ - sys.executable, - "-m", - "pip", - "wheel", - "-vvv", - "--no-deps", - "--no-cache-dir", - "--disable-pip-version-check", - "--progress-bar=off", - ] - if index_url is not None: - args += ["--index-url", index_url] - if index_url != DEFAULT_INDEX: - hostname = urlparse(index_url).hostname - if hostname: - args += ["--trusted-host", hostname] - if extra_index_url is not None: - args += ["--extra-index-url", extra_index_url, "--trusted-host", urlparse(extra_index_url).hostname] - if env is not None: - args[3:3] = ["--python", dict(env)["python_executable"]] - return args - - -def _download_dist(url, scratch_file, index_url, extra_index_url): - auth = None - if index_url: - parsed = urlparse(index_url) - if parsed.username and parsed.password and parsed.hostname == urlparse(url).hostname: - # handling private PyPI credentials in index_url - auth = (parsed.username, parsed.password) - if extra_index_url: - parsed = urlparse(extra_index_url) - if parsed.username and parsed.password and parsed.hostname == urlparse(url).hostname: - # handling private PyPI credentials in extra_index_url - auth = (parsed.username, parsed.password) - target, _headers = urlretrieve(url, scratch_file, auth=auth) - return target, _headers - - -@ttl_cache(maxsize=512, ttl=60 * 5) -def get_versions(dist_name, index_url=None, env=None, extra_index_url=None): - bare_name = requirements.Requirement(dist_name).name - log.debug("checking versions available", dist=bare_name) - args = _get_wheel_args(index_url, env, extra_index_url) + [dist_name + "==showmethemoney"] - try: - out = check_output(args, stderr=STDOUT) - except CalledProcessError as err: - # expected. we forced this by using a non-existing version number. - out = getattr(err, "output", b"") - else: - log.warning(out) - raise Exception("Unexpected success:" + " ".join(args)) - out = out.decode("utf-8") - lines = [] - msg = "Could not find a version that satisfies the requirement" - for line in out.splitlines(): - if msg in line: - lines.append(line) - try: - [line] = lines - except ValueError: - log.warning("failed to get versions", stdout=out) - raise - prefix = "(from versions: " - start = line.index(prefix) + len(prefix) - stop = line.rfind(")") - versions = line[start:stop] - if versions.lower() == "none": - return [] - versions = [v.strip() for v in versions.split(",") if v.strip()] - log.debug("found versions", dist=bare_name, versions=versions) - return versions - - -def _cache_key(dist_name, index_url=None, env=None, extra_index_url=None, tmpdir=None, ignore_errors=None): - return hashkey(dist_name, index_url, env, extra_index_url, ignore_errors) - - -# this decoration is a bit more complicated in order to avoid keying of tmpdir -# see https://github.com/tkem/cachetools/issues/146 -_get_cache = TTLCache(maxsize=512, ttl=60 * 5) - - -@cached(_get_cache, key=_cache_key) -def get(dist_name, index_url=None, env=None, extra_index_url=None, tmpdir=None, ignore_errors=False): - args = _get_wheel_args(index_url, env, extra_index_url) + [dist_name] - scratch_dir = tempfile.mkdtemp(dir=tmpdir) - log.debug("wheeling and dealing", scratch_dir=os.path.abspath(scratch_dir), args=" ".join(args)) - try: - out = check_output(args, stderr=STDOUT, cwd=scratch_dir).decode("utf-8") - except CalledProcessError as err: - out = getattr(err, "output", b"").decode("utf-8") - log.warning(out) - if not ignore_errors: - raise - log.debug("wheel command completed ok", dist_name=dist_name) - links = [] - local_links = [] - lines = out.splitlines() - for i, line in enumerate(lines): - line = line.strip() - if line.startswith("Downloading from URL "): - parts = line.split() - link = parts[3] - links.append(link) - elif line.startswith("Downloading link http"): - links.append(line.split()[2]) - elif line.startswith("Downloading "): - parts = line.split() - last = parts[-1] - if len(parts) == 3 and last.startswith("(") and last.endswith(")"): - link = parts[-2] - elif len(parts) == 4 and parts[-2].startswith("(") and last.endswith(")"): - link = parts[-3] - if not urlparse(link).scheme: - # newest pip versions have changed to not log the full url - # in the download event. it is becoming more and more annoying - # to preserve compatibility across a wide range of pip versions - next_line = lines[i + 1].strip() - if next_line.startswith("Added ") and " to build tracker" in next_line: - link = next_line.split(" to build tracker")[0].split()[-1] - else: - link = last - links.append(link) - elif line.startswith("Source in ") and "which satisfies requirement" in line: - link = line.split()[-1] - links.append(link) - elif line.startswith("Added ") and " from file://" in line: - [link] = [x for x in line.split() if x.startswith("file://")] - local_links.append(link) - if not links: - # prefer http scheme over file - links += local_links - links = list(dict.fromkeys(links)) # order-preserving dedupe - links = [link for link in links if "/" in link and not link.endswith(".metadata")] - if not links: - log.warning("could not find download link", out=out) - raise Exception("failed to collect dist") - if len(links) == 2: - # sometimes we collect the same link, once with a url fragment/checksum and once without - first, second = links - if first.startswith(second): - del links[1] - elif second.startswith(first): - del links[0] - if len(links) > 1: - log.debug("more than 1 link collected", out=out, links=links) - # Since PEP 517, maybe an sdist will also need to collect other distributions - # for the build system, even with --no-deps specified. pendulum==1.4.4 is one - # example, which uses poetry and doesn't publish any python37 wheel to PyPI. - # However, the dist itself should still be the first one downloaded. - link = links[0] - whls = glob(os.path.join(os.path.abspath(scratch_dir), "*.whl")) - try: - [whl] = whls - except ValueError: - if ignore_errors: - whl = "" - else: - raise - url, _sep, checksum = link.partition("#") - url = url.replace("/%2Bf/", "/+f/") # some versions of pip did not unquote this fragment in the log - if not checksum.startswith("md5=") and not checksum.startswith("sha256="): - # PyPI gives you the checksum in url fragment, as a convenience. But not all indices are so kind. - algorithm = "md5" - if os.path.basename(whl).lower() == url.rsplit("/", 1)[-1].lower(): - target = whl - else: - scratch_file = os.path.join(scratch_dir, os.path.basename(url)) - target, _headers = _download_dist(url, scratch_file, index_url, extra_index_url) - checksum = compute_checksum(target=target, algorithm=algorithm) - checksum = "=".join([algorithm, checksum]) - result = {"path": whl, "url": url, "checksum": checksum} - return result - - -def main(): - parser = ArgumentParser() - parser.add_argument("dist_name") - parser.add_argument("--index-url", "-i") - parser.add_argument("--extra-index-url") - parser.add_argument("--for-python", "-p", dest="env", type=python_interpreter) - parser.add_argument("--verbose", "-v", default=1, type=int, choices=range(3)) - debug = { - "sys.argv": sys.argv, - "sys.executable": sys.executable, - "sys.version": sys.version, - "sys.path": sys.path, - } - args = parser.parse_args() - configure_logging(verbosity=args.verbose) - log.debug("runtime info", **debug) - result = get(dist_name=args.dist_name, index_url=args.index_url, env=args.env, extra_index_url=args.extra_index_url) - text = json.dumps(result, indent=2, sort_keys=True, separators=(",", ": ")) - print(text) - - -if __name__ == "__main__": - main() diff --git a/johnnydep/util.py b/johnnydep/util.py index 0660142..ff06076 100644 --- a/johnnydep/util.py +++ b/johnnydep/util.py @@ -1,28 +1,39 @@ import json +import os from argparse import ArgumentTypeError +from collections import deque +from pathlib import Path from subprocess import CalledProcessError from subprocess import check_output -import anytree import structlog +import unearth -from johnnydep import env_check +from . import env_check + + +log = structlog.get_logger() def python_interpreter(path): + sub_env = os.environ.copy() + sub_env["PYTHONPATH"] = str(Path(unearth.__file__).parent.parent) + sub_env["PYTHONDONTWRITEBYTECODE"] = "1" try: - env_json = check_output([path, env_check.__file__]) + env_json = check_output( + [path, env_check.__file__], + env=sub_env, + ) except CalledProcessError: raise ArgumentTypeError("Invalid python env call") try: env = json.loads(env_json.decode()) except json.JSONDecodeError: raise ArgumentTypeError("Invalid python env output") - frozen = tuple(map(tuple, env)) - return frozen + return env -class CircularMarker(anytree.NodeMixin): +class CircularMarker: """ This is like a "fake" JohnnyDist instance which is used to render a node in circular dep trees like: @@ -41,9 +52,27 @@ def __init__(self, summary, parent): self.req = CircularMarker.glyph self.name = CircularMarker.glyph self.summary = summary - self.parent = parent + self.parents = [parent] + self.children = [] self.log = structlog.get_logger() def __getattr__(self, name): if name.startswith("_"): return super(CircularMarker, self).__getattribute__(name) + + +def _bfs(jdist): + seen = set() + q = deque([jdist]) + while q: + jd = q.popleft() + pk = id(jd) + if pk not in seen: + seen.add(pk) + yield jd + q += jd.children + + +def _un_none(d): + # toml can't serialize None + return {k: v for k, v in d.items() if v is not None} diff --git a/pyproject.toml b/pyproject.toml index 1d5c534..4b592b4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,16 +5,15 @@ description = "Display dependency tree of Python distribution" readme = "README.md" requires-python = ">=3.8" dependencies = [ - "anytree", - "cachetools", "colorama ; platform_system == 'Windows'", "importlib_metadata ; python_version < '3.10'", "packaging >= 17, != 22", - "pip >= 22.3", "PyYAML", + "rich", "structlog", - "tabulate", - "toml", + "tomli-w", + "unearth", + "uv", "wheel >= 0.32.0", ] @@ -30,7 +29,6 @@ homepage = "https://github.com/wimglenn/johnnydep" [project.scripts] johnnydep = "johnnydep.cli:main" -pipper = "johnnydep.pipper:main" [tool.pytest.ini_options] @@ -43,4 +41,10 @@ addopts = [ "--no-cov-on-fail", "--color=yes", "--disable-socket", + "--allow-unix-socket", ] + + +[tool.coverage.run] +branch = true +parallel = true diff --git a/tests/conftest.py b/tests/conftest.py index aabacf5..0a59b81 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,26 +1,21 @@ -import hashlib -import os.path -import subprocess -import sys +import os +import shutil +import tempfile from importlib.metadata import version +from pathlib import Path import pytest import whl +from unearth import PackageFinder from wimpy import working_directory from johnnydep import cli from johnnydep import dot from johnnydep import lib -from johnnydep import pipper - - -original_check_output = subprocess.check_output @pytest.fixture(autouse=True) def expire_caches(): - pipper.get_versions.cache_clear() - pipper._get_cache.clear() lib._get_info.cache_clear() @@ -61,7 +56,9 @@ def kill_env(): ) -def make_wheel(scratch_dir="/tmp/jdtest", callback=None, **extra): +def make_wheel(scratch_path=None, callback=None, **extra): + if scratch_path is None: + scratch_path = Path(tempfile.gettempdir()) kwargs = default_setup_kwargs.copy() kwargs.update(extra) name = kwargs["name"] @@ -79,8 +76,6 @@ def make_wheel(scratch_dir="/tmp/jdtest", callback=None, **extra): if not kwargs.get("requires_dist"): kwargs["requires_dist"] = [] for extra, reqs in extras.items(): - if isinstance(reqs, str): - reqs = [reqs] for req in reqs: if ";" in req: req, marker = req.split(";") @@ -92,10 +87,7 @@ def make_wheel(scratch_dir="/tmp/jdtest", callback=None, **extra): if "url" in kwargs: kwargs["home_page"] = kwargs.pop("url") if "platforms" in kwargs: - platforms = kwargs.pop("platforms") - if isinstance(platforms, str): - platforms = [p.strip() for p in platforms.split(",")] - kwargs["platform"] = platforms + kwargs["platform"] = kwargs.pop("platforms") if "classifiers" in kwargs: kwargs["classifier"] = kwargs.pop("classifiers") @@ -105,102 +97,65 @@ def make_wheel(scratch_dir="/tmp/jdtest", callback=None, **extra): if py_modules: kwargs["src"] = [] - with working_directory(scratch_dir): + with working_directory(scratch_path): for fname in py_modules: - if os.path.exists(fname): - raise Exception(f"already exists: {fname}") + assert not Path(fname).exists() with open(fname + ".py", "w"): pass - kwargs["src"].append(os.path.join(scratch_dir, fname + ".py")) - dist = whl.make_wheel(**kwargs) + kwargs["src"].append(f"{scratch_path / fname}" + ".py") + dist_path = Path(whl.make_wheel(**kwargs)).resolve() - dist_path = os.path.join(scratch_dir, f"{name}-{version}-py2.py3-none-any.whl") - with open(dist_path, mode="rb") as f: - md5 = hashlib.md5(f.read()).hexdigest() + fname = f"{name.replace('-', '_')}-{version}-py2.py3-none-any.whl" + assert dist_path == scratch_path.resolve() / fname if callback is not None: # contribute to test index - callback(name=name, path=dist_path, urlfragment="#md5=" + md5) + callback(dist_path) - return dist, dist_path, md5 + return dist_path @pytest.fixture -def add_to_index(): - index_data = {} +def add_to_index(mocker): + + find_links = set() + + def add_package(path): + find_links.add(path.parent) - def add_package(name, path, urlfragment=""): - index_data[path] = (name, urlfragment) + def mock_package_finder(index_urls, target_python, trusted_hosts): + return PackageFinder( + index_urls=[], + target_python=target_python, + find_links=list(find_links), + ) - add_package.index_data = index_data + mocker.patch("unearth.PackageFinder", mock_package_finder) yield add_package @pytest.fixture def make_dist(tmp_path, add_to_index): def f(**kwargs): - if "callback" not in kwargs: - kwargs["callback"] = add_to_index - if "scratch_dir" not in kwargs: - kwargs["scratch_dir"] = str(tmp_path) + kwargs.setdefault("callback", add_to_index) + kwargs.setdefault("scratch_path", tmp_path) return make_wheel(**kwargs) return f -@pytest.fixture(autouse=True) -def fake_subprocess(mocker, add_to_index): - - index_data = add_to_index.index_data - subprocess_check_output = subprocess.check_output - - def wheel_proc(args, stderr, cwd=None): - exe = args[0] - req = args[-1] - args = [ - exe, - "-m", - "pip", - "wheel", - "-vvv", - "--no-index", - "--no-deps", - "--no-cache-dir", - "--disable-pip-version-check", - ] - args.extend([f"--find-links={p}" for p in index_data]) - args.append(req) - output = subprocess_check_output(args, stderr=stderr, cwd=cwd) - lines = output.decode().splitlines() - for line in lines: - line = line.strip() - if line.startswith("Saved "): - fname = line.split("/")[-1].split("\\")[-1] - inject = "{0} Downloading from URL http://fakeindex/{1}{0}".format(os.linesep, fname) - output += inject.encode() - break - return output - - mocker.patch("johnnydep.pipper.check_output", wheel_proc) - - -@pytest.fixture -def fake_pip(mocker): - mocker.patch("johnnydep.pipper.check_output", original_check_output) - - def local_files_args(index_url, env, extra_index_url): - test_dir = os.path.abspath(os.path.join(__file__, os.pardir)) - canned = f"file://{os.path.join(test_dir, 'test_deps')}" - args = [ - sys.executable, - "-m", - "pip", - "wheel", - "-vvv", - "--no-index", - "--no-deps", - "--no-cache-dir", - "--disable-pip-version-check", - f"--find-links={canned}", - ] - return args - mocker.patch("johnnydep.pipper._get_wheel_args", local_files_args) +def pytest_assertrepr_compare(config, op, left, right): + # https://docs.pytest.org/en/latest/reference/reference.html#pytest.hookspec.pytest_assertrepr_compare + if isinstance(left, str) and isinstance(right, str) and op == "==": + left_lines = left.splitlines() + right_lines = right.splitlines() + if len(left_lines) > 1 or len(right_lines) > 1: + width, _ = shutil.get_terminal_size(fallback=(80, 24)) + width = max(width, 40) - 10 + lines = [ + "When comparing multiline strings:", + f" LEFT ({len(left)}) ".center(width, "="), + *left_lines, + f" RIGHT ({len(right)}) ".center(width, "="), + *right_lines, + ] + return lines diff --git a/tests/test_cli.py b/tests/test_cli.py index d953d73..f6391ca 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1,4 +1,6 @@ +import hashlib import sys +from pathlib import Path from textwrap import dedent import pytest @@ -6,19 +8,11 @@ from johnnydep.cli import main -def test_printed_table_on_stdout(mocker, capsys, make_dist): - make_dist() - mocker.patch("sys.argv", "johnnydep jdtest==0.1.2".split()) - main() - out, err = capsys.readouterr() - assert err == "" - assert out == dedent( - """\ - name summary - ------------- --------------------------------- - jdtest==0.1.2 default text for metadata summary - """ - ) +@pytest.fixture(scope="module", autouse=True) +def monkeymod(): + with pytest.MonkeyPatch.context() as mp: + mp.setenv("COLUMNS", "200") + yield mp def test_printed_table_on_stdout_with_specifier(make_dist, mocker, capsys): @@ -29,10 +23,10 @@ def test_printed_table_on_stdout_with_specifier(make_dist, mocker, capsys): assert err == "" assert out == dedent( """\ - name specifier - ------ ----------- - jdtest >=0.1 - """ + name specifier + ━━━━━━━━━━━━━━━━━━━━ + jdtest >=0.1 + """ ) @@ -40,19 +34,18 @@ def test_printed_tree_on_stdout(mocker, capsys, make_dist): make_dist(name="thing", extras_require={"xyz": ["spam>0.30.0"], "abc": ["eggs"]}) make_dist(name="spam", version="0.31") make_dist(name="eggs") - mocker.patch( - "sys.argv", "johnnydep thing[xyz] --fields extras_available extras_requested".split() - ) + argv = "johnnydep thing[xyz] --fields extras_available extras_requested".split() + mocker.patch("sys.argv", argv) main() out, err = capsys.readouterr() assert err == "" assert out == dedent( """\ - name extras_available extras_requested - --------------- ------------------ ------------------ - thing[xyz] abc, xyz xyz - └── spam>0.30.0 - """ + name extras_available extras_requested + ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + thing[xyz] abc, xyz xyz + └── spam>0.30.0 + """ ) @@ -77,14 +70,14 @@ def test_diamond_deptree(mocker, capsys, make_dist): assert err == "" assert out == dedent( """\ - name specifier requires required_by versions_available version_latest_in_spec - ------------------ ----------- -------------- ------------- -------------------- ------------------------ - distA distB1, distB2 0.1 0.1 - ├── distB1 distC[x,z]<0.3 distA 0.1 0.1 - │ └── distC[x,z] <0.3 distB1 0.1, 0.2, 0.3 0.2 - └── distB2 distC[y]!=0.2 distA 0.1 0.1 - └── distC[y] !=0.2 distB2 0.1, 0.2, 0.3 0.3 - """ + name specifier requires required_by versions_available version_latest_in_spec + ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + distA distB1, distB2 0.1 0.1 + ├── distB1 distC[x,z]<0.3 distA 0.1 0.1 + │ └── distC[x,z] <0.3 distB1 0.1, 0.2, 0.3 0.2 + └── distB2 distC[y]!=0.2 distA 0.1 0.1 + └── distC[y] !=0.2 distB2 0.1, 0.2, 0.3 0.3 + """ ) @@ -101,12 +94,12 @@ def test_unresolvable_deptree(mocker, capsys, make_dist): assert err == "" assert out == dedent( """\ - name requires required_by versions_available version_latest_in_spec - -------------- --------------------- ------------- -------------------- ------------------------ - distX distC<=0.1, distC>0.2 0.1 0.1 - ├── distC<=0.1 distX 0.1, 0.2, 0.3 0.1 - └── distC>0.2 distX 0.1, 0.2, 0.3 0.3 - """ + name requires required_by versions_available version_latest_in_spec + ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + distX distC<=0.1, distC>0.2 0.1 0.1 + ├── distC<=0.1 distX 0.1, 0.2, 0.3 0.1 + └── distC>0.2 distX 0.1, 0.2, 0.3 0.3 + """ ) @@ -127,70 +120,78 @@ def test_requirements_txt_output(mocker, capsys, make_dist): distB1==0.1 distB2==0.1 distC[x,y,z]==0.1 - """ + """ ) -def test_all_fields_toml_out(mocker, capsys, make_dist): - _dist, _dist_path, checksum = make_dist(name="wimpy", version="0.3", py_modules=["that"]) - mocker.patch("sys.argv", "johnnydep wimpy<0.4 --fields=ALL --output-format=toml".split()) +def test_all_fields_toml_out(mocker, capsys, make_dist, tmp_path): + dist_path = make_dist(name="example", version="0.3", py_modules=["that"]) + checksum = hashlib.md5(dist_path.read_bytes()).hexdigest() + mocker.patch("sys.argv", "johnnydep example<0.4 --fields=ALL --output-format=toml".split()) main() out, err = capsys.readouterr() assert err == "" assert out == dedent( - f"""\ - name = "wimpy" + f'''\ + name = "example" summary = "default text for metadata summary" specifier = "<0.4" requires = [] required_by = [] - import_names = [ "that",] + import_names = [ + "that", + ] console_scripts = [] homepage = "https://www.example.org/default" extras_available = [] extras_requested = [] - project_name = "wimpy" + project_name = "example" license = "MIT" - versions_available = [ "0.3",] - version_installed = "0.3" + versions_available = [ + "0.3", + ] version_latest = "0.3" version_latest_in_spec = "0.3" - download_link = "http://fakeindex/wimpy-0.3-py2.py3-none-any.whl" + download_link = "{tmp_path.as_uri()}/example-0.3-py2.py3-none-any.whl" checksum = "md5={checksum}" - """ + ''' ) -def test_ignore_errors_build_error(mocker, capsys, fake_pip, monkeypatch): +def test_ignore_errors_build_error(mocker, capsys, monkeypatch, add_to_index): monkeypatch.setenv("JDT3_FAIL", "1") + add_to_index(Path(__file__).parent / "test_deps" / "jdt1-0.1.tar.gz") mocker.patch("sys.argv", "johnnydep jdt1 --ignore-errors --fields name".split()) with pytest.raises(SystemExit(1)): main() out, err = capsys.readouterr() assert out == dedent( """\ - name - --------------------- - jdt1 - ├── jdt2 - │ ├── jdt3 (FAILED) - │ └── jdt4 - └── jdt5 - """) + name + ━━━━━━━━━━━━━━━━━━━━━━━ + jdt1 + ├── jdt2 + │ ├── jdt3 (FAILED) + │ └── jdt4 + └── jdt5 + """ + ) def test_root_has_error(mocker, capsys): mocker.patch("sys.argv", "johnnydep dist404 --ignore-errors --fields name".split()) + mocker.patch("unearth.PackageFinder.find_best_match").return_value.best = None with pytest.raises(SystemExit(1)): main() out, err = capsys.readouterr() assert out == dedent( """\ - name - ---------------- - dist404 (FAILED) - """) + name + ━━━━━━━━━━━━━━━━━━ + dist404 (FAILED) + """ + ) def test_no_deps(mocker, capsys, make_dist): @@ -200,10 +201,11 @@ def test_no_deps(mocker, capsys, make_dist): out, err = capsys.readouterr() assert out == dedent( """\ - name - ------ - distA - """) + name + ━━━━━━━ + distA + """ + ) def test_circular_deptree(mocker, capsys, make_dist): @@ -217,16 +219,17 @@ def test_circular_deptree(mocker, capsys, make_dist): out, err = capsys.readouterr() assert out == dedent( """\ - name summary - ----------------------- ----------------------------------------------------------------- - pkg0 default text for metadata summary - └── pkg1 default text for metadata summary - ├── pkg2 default text for metadata summary - │ └── pkg3 default text for metadata summary - │ └── pkg1 default text for metadata summary - │ └── ... ... pkg2 -> pkg3 -> pkg1> - └── quux default text for metadata summary - """) + name summary + ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + pkg0 default text for metadata summary + └── pkg1 default text for metadata summary + ├── pkg2 default text for metadata summary + │ └── pkg3 default text for metadata summary + │ └── pkg1 default text for metadata summary + │ └── ... ... pkg2 -> pkg3 -> pkg1> + └── quux default text for metadata summary + """ + ) def test_circular_deptree_resolve(mocker, capsys, make_dist): @@ -245,4 +248,22 @@ def test_circular_deptree_resolve(mocker, capsys, make_dist): pkg2==0.3 quux==0.1.2 pkg3==0.4 - """) + """ + ) + + +def test_explicit_env(mocker, make_dist, capsys): + make_dist() + argv = "johnnydep jdtest==0.1.2".split() + argv += ["-p", sys.executable] + mocker.patch("sys.argv", argv) + main() + out, err = capsys.readouterr() + assert err == "" + assert out == dedent( + """\ + name summary + ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + jdtest==0.1.2 default text for metadata summary + """ + ) diff --git a/tests/test_downloader.py b/tests/test_downloader.py new file mode 100644 index 0000000..d5d12fe --- /dev/null +++ b/tests/test_downloader.py @@ -0,0 +1,80 @@ +import pytest + +from johnnydep.downloader import download_dist + + +@pytest.mark.parametrize( + "url, index_url, extra_index_url, expected_auth, expected_top_level_url", + [ + ( + "https://pypi.example.com/packages", + "https://pypi.example.com/simple", + None, + None, + None, + ), + ( + "https://pypi.example.com/packages", + "https://user:pass@pypi.example.com/simple", + None, + ("user", "pass"), + "pypi.example.com", + ), + ( + "https://pypi.extra.com/packages", + "https://user:pass@pypi.example.com/simple", + "https://pypi.extra.com/simple", + None, + "pypi.example.com", + ), + ( + "https://pypi.extra.com/packages", + "https://user:pass@pypi.example.com/simple", + "https://user:extrapass@pypi.extra.com/simple", + ("user", "extrapass"), + "pypi.extra.com", + ), + ( + "https://pypi.extra.com/packages", + None, + "https://user:extrapass@pypi.extra.com/simple", + ("user", "extrapass"), + "pypi.extra.com", + ), + ], + ids=( + "index_url without auth", + "index_url with auth", + "extra_index_url without auth", + "extra_index_url with auth", + "extra_index_url with auth (no index_url)", + ), +) +def test_download_dist_auth(mocker, url, index_url, extra_index_url, expected_auth, expected_top_level_url, tmp_path): + mgr = mocker.patch("johnnydep.downloader.HTTPPasswordMgrWithDefaultRealm") + add_password_mock = mgr.return_value.add_password + + opener = mocker.patch("johnnydep.downloader.build_opener").return_value + mock_response = opener.open.return_value + mock_response.read.return_value = b"test body" + + scratch_path = tmp_path / "test-0.1.tar.gz" + with scratch_path.open("wb") as f: + download_dist( + url=url + "/test-0.1.tar.gz", + f=f, + index_url=index_url, + extra_index_url=extra_index_url, + ) + if expected_auth is None: + add_password_mock.assert_not_called() + else: + expected_realm = None + expected_username, expected_password = expected_auth + add_password_mock.assert_called_once_with( + expected_realm, + expected_top_level_url, + expected_username, + expected_password, + ) + assert scratch_path.read_bytes() == b"test body" diff --git a/tests/test_gen.py b/tests/test_gen.py index 75e8eaf..d96debc 100644 --- a/tests/test_gen.py +++ b/tests/test_gen.py @@ -1,4 +1,4 @@ -import os +from pathlib import Path import pytest @@ -6,7 +6,7 @@ from johnnydep.lib import JohnnyError -here = os.path.dirname(__file__) +here = Path(__file__).parent def test_generated_metadata_from_dist_name(make_dist): @@ -31,7 +31,7 @@ def test_generated_metadata_from_dist_name(make_dist): def test_generated_metadata_from_dist_path(make_dist): - _dist, dist_path, _checksum = make_dist() + dist_path = make_dist() jdist = JohnnyDist(dist_path) expected_metadata = { "author": "default author", @@ -50,9 +50,7 @@ def test_generated_metadata_from_dist_path(make_dist): def test_build_from_sdist(add_to_index): - sdist_fname = os.path.join(here, "copyingmock-0.2.tar.gz") - fragment = "#md5=9aa6ba13542d25e527fe358d53cdaf3b" - add_to_index(name="copyingmock", path=sdist_fname, urlfragment=fragment) + add_to_index(here / "copyingmock-0.2.tar.gz") dist = JohnnyDist("copyingmock") assert dist.name == "copyingmock" assert dist.summary == "A subclass of MagicMock that copies the arguments" @@ -69,9 +67,7 @@ def test_build_from_sdist(add_to_index): def test_plaintext_whl_metadata(add_to_index): # this dist uses an old-skool metadata version 1.2 - sdist_fname = os.path.join(here, "testpath-0.3.1-py2.py3-none-any.whl") - fragment = "#md5=12728181294cf6f815421081d620c494" - add_to_index(name="testpath", path=sdist_fname, urlfragment=fragment) + add_to_index(here / "testpath-0.3.1-py2.py3-none-any.whl") dist = JohnnyDist("testpath==0.3.1") assert dist.serialise(fields=["name", "summary", "import_names", "homepage"]) == [ { @@ -85,9 +81,7 @@ def test_plaintext_whl_metadata(add_to_index): def test_old_metadata_20(add_to_index): # the never-officially-supported-but-out-in-the-wild metadata 2.0 spec (generated by wheel v0.30.0) - whl_fname = os.path.join(here, "m20dist-0.1.2-py2.py3-none-any.whl") - fragment = "#md5=488652bac3e1705e5646ea6a51f4d441" - add_to_index(name="m20dist", path=whl_fname, urlfragment=fragment) + add_to_index(here / "m20dist-0.1.2-py2.py3-none-any.whl") jdist = JohnnyDist("m20dist") expected_metadata = { "author": "default author", @@ -105,15 +99,6 @@ def test_old_metadata_20(add_to_index): assert jdist.checksum == "md5=488652bac3e1705e5646ea6a51f4d441" -def test_index_file_without_checksum_in_urlfragment(add_to_index, mocker): - whl_fname = os.path.join(here, "vanilla-0.1.2-py2.py3-none-any.whl") - add_to_index(name="vanilla", path=whl_fname) - jdist = JohnnyDist("vanilla") - assert jdist.versions_available == ["0.1.2"] - mocker.patch("johnnydep.pipper.urlretrieve", return_value=(whl_fname, {})) - assert jdist.checksum == "md5=8a20520dcb1b7a729b827a2c4d75a0b6" - - def test_cant_pin(make_dist, mocker): make_dist(name="cantpin", version="0.6") jdist = JohnnyDist("cantpin") @@ -124,8 +109,8 @@ def test_cant_pin(make_dist, mocker): def test_whl_extras(): - whl_fname = os.path.join(here, "testwhlextra-1.0.0-py3-none-any.whl") - jdist = JohnnyDist(whl_fname + "[dev]") + whl_path = here / "testwhlextra-1.0.0-py3-none-any.whl" + jdist = JohnnyDist(f"{whl_path}[dev]") assert jdist.extras_requested == ["dev"], "should have found the extra dev deps" assert jdist.requires == ["black==22.1.0", "flake8==4.0.1", "xdoctest>=1.0.0"] - assert JohnnyDist(whl_fname).requires == ["xdoctest>=1.0.0"] + assert JohnnyDist(whl_path).requires == ["xdoctest>=1.0.0"] diff --git a/tests/test_lib.py b/tests/test_lib.py index 8409dd9..c992cb8 100644 --- a/tests/test_lib.py +++ b/tests/test_lib.py @@ -1,10 +1,10 @@ import json import os -from subprocess import CalledProcessError from textwrap import dedent import pytest +from johnnydep import lib from johnnydep.lib import flatten_deps from johnnydep.lib import JohnnyDist from johnnydep.lib import JohnnyError @@ -13,11 +13,8 @@ def test_version_nonexisting(make_dist): # v0.404 does not exist in index make_dist() - with pytest.raises(CalledProcessError) as cm: + with pytest.raises(JohnnyError("Package not found 'jdtest==0.404'")): JohnnyDist("jdtest==0.404") - msg = "DistributionNotFound: No matching distribution found for jdtest==0.404" + os.linesep - txt = cm.value.output.decode() - assert msg in txt def test_import_names_empty(make_dist): @@ -41,7 +38,7 @@ def test_version_installed(make_dist): def test_version_not_installed(make_dist): make_dist() jdist = JohnnyDist("jdtest") - assert jdist.version_installed is None + assert not jdist.version_installed def test_version_latest(make_dist): @@ -72,16 +69,6 @@ def test_version_latest_in_spec_prerelease_chosen(make_dist): assert jdist.version_latest_in_spec == "0.2a0" -def test_version_latest_in_spec_prerelease_out_of_spec(make_dist): - make_dist(version="0.1") - make_dist(version="0.2a0") - with pytest.raises(CalledProcessError) as cm: - JohnnyDist("jdtest>0.1") - msg = "DistributionNotFound: No matching distribution found for jdtest>0.1" + os.linesep - txt = cm.value.output.decode() - assert msg in txt - - def test_version_pinned_to_latest_in_spec(make_dist): make_dist(version="1.2.3") make_dist(version="2.3.4") @@ -102,11 +89,8 @@ def test_version_in_spec_not_avail(make_dist): make_dist(version="1.2.3") make_dist(version="2.3.4") make_dist(version="3.4.5") - with pytest.raises(CalledProcessError) as cm: + with pytest.raises(JohnnyError("Package not found 'jdtest>4'")): JohnnyDist("jdtest>4") - msg = "DistributionNotFound: No matching distribution found for jdtest>4" + os.linesep - txt = cm.value.output.decode() - assert msg in txt def test_project_name_different_from_canonical_name(make_dist): @@ -137,14 +121,7 @@ def test_no_homepage(make_dist): def test_dist_no_children(make_dist): make_dist() jdist = JohnnyDist("jdtest") - assert jdist.children == () - - -def test_download_link(make_dist): - make_dist() - jdist = JohnnyDist("jdtest") - # this link is coming from a faked package index set up in conftest.py - assert jdist.download_link == "http://fakeindex/jdtest-0.1.2-py2.py3-none-any.whl" + assert jdist.children == [] def test_checksum_md5(make_dist): @@ -159,14 +136,15 @@ def test_checksum_md5(make_dist): assert set(hashval) <= set("1234567890abcdef") -def test_scratch_dirs_are_being_cleaned_up(make_dist, mocker, tmp_path): +def test_scratch_dirs_are_being_cleaned_up(make_dist, mocker): make_dist() - scratch = str(tmp_path) - mkdtemp = mocker.patch("johnnydep.lib.mkdtemp", return_value=scratch) - rmtree = mocker.patch("johnnydep.lib.rmtree") + mkdtemp = mocker.spy(lib, "mkdtemp") + rmtree = mocker.spy(lib, "rmtree") JohnnyDist("jdtest") mkdtemp.assert_called_once_with() + [scratch] = mkdtemp.spy_return_list rmtree.assert_called_once_with(scratch, ignore_errors=True) + assert not os.path.exists(scratch) def test_extras_available_none(make_dist): @@ -260,7 +238,7 @@ def test_children(make_dist): jdist = JohnnyDist("parent") [child] = jdist.children assert isinstance(child, JohnnyDist) - assert isinstance(jdist.children, tuple) + assert isinstance(jdist.children, list) assert child.name == "child" @@ -288,10 +266,10 @@ def test_serialiser_toml(make_dist): make_dist() jdist = JohnnyDist("jdtest") assert jdist.serialise(format="toml") == dedent( - """\ + '''\ name = "jdtest" summary = "default text for metadata summary" - """ + ''' ) @@ -403,11 +381,8 @@ def test_resolve_unresolvable(make_dist): ] gen = flatten_deps(dist) assert next(gen) is dist - with pytest.raises(CalledProcessError) as cm: + with pytest.raises(JohnnyError("Package not found 'dist2<=0.1,>0.2'")): next(gen) - msg = "DistributionNotFound: No matching distribution found for dist2<=0.1,>0.2" + os.linesep - txt = cm.value.output.decode() - assert msg in txt def test_pprint(make_dist, mocker): @@ -430,36 +405,30 @@ def test_pprint(make_dist, mocker): def test_get_caching(make_dist, mocker): # this test is trying to make sure that distribution "c", a node which appears - # twice in the dependency tree, is only downloaded from the index once. - # i.e. check that the ttl caching on the downloader is working correctly - downloader = mocker.patch("johnnydep.lib.pipper.get") - _, c, _ = make_dist(name="c", description="leaf node") - _, b1, _ = make_dist(name="b1", install_requires=["c"], description="branch one") - _, b2, _ = make_dist(name="b2", install_requires=["c"], description="branch two") - _, a, _ = make_dist(name="a", install_requires=["b1", "b2"], description="root node") - downloader.side_effect = [ - {"path": a}, - {"path": b1}, - {"path": b2}, - {"path": c}, - ] + # twice in the dependency graph, is only downloaded from the index once. + # i.e. check that the caching on the downloader is working correctly. + make_dist(name="c", description="leaf node") + make_dist(name="b1", install_requires=["c"], description="branch one") + make_dist(name="b2", install_requires=["c"], description="branch two") + make_dist(name="a", install_requires=["b1", "b2"], description="root node") + spy = mocker.spy(lib, "download_dist") jdist = JohnnyDist("a") txt = jdist.serialise(format="human") - assert txt == dedent("""\ - name summary - --------- ---------- - a root node - ├── b1 branch one - │ └── c leaf node - └── b2 branch two - └── c leaf node""") - assert downloader.call_count == 4 - assert downloader.call_args_list == [ - mocker.call("a", env=None, extra_index_url=None, index_url=None, tmpdir=mocker.ANY), - mocker.call("b1", env=None, extra_index_url=None, index_url=None, tmpdir=mocker.ANY), - mocker.call("b2", env=None, extra_index_url=None, index_url=None, tmpdir=mocker.ANY), - mocker.call("c", env=None, extra_index_url=None, index_url=None, tmpdir=mocker.ANY), - ] + assert txt == dedent( + """\ + name summary + ━━━━━━━━━━━━━━━━━━━━━━━━ + a root node + ├── b1 branch one + │ └── c leaf node + └── b2 branch two + └── c leaf node""" + ) + assert spy.call_count == 4 + downloads = [call.kwargs["url"] for call in spy.call_args_list] + filenames = [download.split("/")[-1] for download in downloads] + distnames = [filename.split("-")[0] for filename in filenames] + assert distnames == ["a", "b1", "b2", "c"] def test_extras_parsing(make_dist): @@ -493,7 +462,7 @@ def test_ignore_errors(make_dist): assert len(dist.children) == 1 assert dist.children[0].name == "distb1" assert dist.children[0].error is not None - assert "No matching distribution found for distB1>=1.0" in dist.children[0].error.output.decode("utf-8") + assert "Package not found 'distB1>=1.0'" in str(dist.children[0].error) def test_flatten_failed(make_dist): @@ -504,25 +473,24 @@ def test_flatten_failed(make_dist): list(flatten_deps(dist)) -def test_local_whl_pinned(make_dist): +def test_local_whl_pinned(make_dist, mocker): # https://github.com/wimglenn/johnnydep/issues/105 - dist, dist_path, md5 = make_dist(name="loc", version="1.2.3", callback=None) + dist_path = make_dist(name="loc", version="1.2.3", callback=None) dist = JohnnyDist(dist_path) + mocker.patch("unearth.finder.PackageFinder.find_matches", return_value=[]) txt = dist.serialise(format="pinned").strip() assert txt == "loc==1.2.3" def test_local_whl_json(make_dist): make_dist(name="loc", version="0.1.1") - dist, dist_path, md5 = make_dist(name="loc", version="0.1.2", callback=None) + dist_path = make_dist(name="loc", version="0.1.2", callback=None) make_dist(name="loc", version="0.1.3") dist = JohnnyDist(dist_path) fields = ["download_link", "checksum", "versions_available"] txt = dist.serialise(format="json", fields=fields).strip() [result] = json.loads(txt) - algo, checksum = result["checksum"].split("=") - assert algo == "md5" - assert checksum == md5 + assert result["checksum"].startswith("md5=") link = result["download_link"] assert link.startswith("file://") assert link.endswith("loc-0.1.2-py2.py3-none-any.whl") @@ -539,3 +507,21 @@ def test_entry_points(make_dist): assert ep.group == "console_scripts" assert ep.value == "mypkg.mymod:foo" assert dist.console_scripts == ["my-script = mypkg.mymod:foo"] + + +def test_direct_path_version_insort(make_dist, tmp_path): + make_dist(name="foo", version="0.1.1") + make_dist(name="foo", version="0.1.3") + ext_path = tmp_path / "ext" + ext_path.mkdir() + path = make_dist(scratch_path=ext_path, name="foo", version="0.1.2", callback=None) + dist = JohnnyDist(path) + assert dist.specifier == "==0.1.2" + assert dist.versions_available == ["0.1.1", "0.1.2", "0.1.3"] + + +def test_ignore_errors_version_attrs(mocker): + mocker.patch("johnnydep.lib._get_info", side_effect=Exception) + mocker.patch("unearth.finder.PackageFinder.find_matches", return_value=[]) + dist = JohnnyDist("notexist", ignore_errors=True) + assert dist.version_latest is None diff --git a/tests/test_pipper.py b/tests/test_pipper.py deleted file mode 100644 index 94fb3c7..0000000 --- a/tests/test_pipper.py +++ /dev/null @@ -1,135 +0,0 @@ -import json -import os -import sys - -import pytest -from wimpy import working_directory - -import johnnydep.pipper - - -def test_pipper_main(mocker, capsys, make_dist, tmp_path): - make_dist(name="fakedist", version="1.2.3") - mocker.patch("sys.argv", "pipper.py fakedist".split()) - with working_directory(tmp_path): - johnnydep.pipper.main() - out, err = capsys.readouterr() - output = json.loads(out) - path = output.pop("path") - checksum = output.pop("checksum") - assert output == {"url": "http://fakeindex/fakedist-1.2.3-py2.py3-none-any.whl"} - assert os.path.isfile(path) - assert os.path.basename(path) == "fakedist-1.2.3-py2.py3-none-any.whl" - assert len(checksum) == 4 + 32 - assert checksum.startswith("md5=") - assert err == "" - - -def test_compute_checksum(tmp_path): - tmpfile = tmp_path.joinpath("fname") - fname = str(tmpfile) - tmpfile.write_text("spam and eggs") - md5 = johnnydep.pipper.compute_checksum(fname, algorithm="md5") - sha256 = johnnydep.pipper.compute_checksum(fname) - assert md5 == "b581660cff17e78c84c3a84ad02e6785" - assert sha256 == "7c788633adc75d113974372eec8c24776a581f095a747136e7ccf41b4a18b74e" - - -def test_get_wheel_args(): - fake_env = ("python_executable", "snek"), ("pip_version", "22.3") - url = "https://user:pass@example.org:8888/something" - args = johnnydep.pipper._get_wheel_args(index_url=url, env=fake_env, extra_index_url=None) - assert args == [ - sys.executable, - "-m", - "pip", - "--python", - "snek", - "wheel", - "-vvv", - "--no-deps", - "--no-cache-dir", - "--disable-pip-version-check", - "--progress-bar=off", - "--index-url", - "https://user:pass@example.org:8888/something", - "--trusted-host", - "example.org", - ] - - -@pytest.mark.parametrize( - "url, index_url, extra_index_url, expected_auth, expected_top_level_url", - [ - ( - "https://pypi.example.com/packages", - "https://pypi.example.com/simple", - None, - None, - None, - ), - ( - "https://pypi.example.com/packages", - "https://user:pass@pypi.example.com/simple", - None, - ("user", "pass"), - "pypi.example.com", - ), - ( - "https://pypi.extra.com/packages", - "https://user:pass@pypi.example.com/simple", - "https://pypi.extra.com/simple", - None, - "pypi.example.com", - ), - ( - "https://pypi.extra.com/packages", - "https://user:pass@pypi.example.com/simple", - "https://user:extrapass@pypi.extra.com/simple", - ("user", "extrapass"), - "pypi.extra.com", - ), - ( - "https://pypi.extra.com/packages", - None, - "https://user:extrapass@pypi.extra.com/simple", - ("user", "extrapass"), - "pypi.extra.com", - ), - ], - ids=( - "index_url without auth", - "index_url with auth", - "extra_index_url without auth", - "extra_index_url with auth", - "extra_index_url with auth (no index_url)", - ), -) -def test_download_dist_auth(mocker, url, index_url, extra_index_url, expected_auth, expected_top_level_url, tmp_path): - mgr = mocker.patch("johnnydep.pipper.HTTPPasswordMgrWithDefaultRealm") - add_password_mock = mgr.return_value.add_password - - opener = mocker.patch("johnnydep.pipper.build_opener").return_value - mock_response = opener.open.return_value - mock_response.read.return_value = b"test body" - - scratch_path = tmp_path / "test-0.1.tar.gz" - target, _headers = johnnydep.pipper._download_dist( - url=url + "/test-0.1.tar.gz", - scratch_file=str(scratch_path), - index_url=index_url, - extra_index_url=extra_index_url, - ) - if expected_auth is None: - add_password_mock.assert_not_called() - else: - expected_realm = None - expected_username, expected_password = expected_auth - add_password_mock.assert_called_once_with( - expected_realm, - expected_top_level_url, - expected_username, - expected_password, - ) - assert target == str(scratch_path) - assert scratch_path.read_bytes() == b"test body" diff --git a/tests/test_util.py b/tests/test_util.py index fd028c0..5472091 100644 --- a/tests/test_util.py +++ b/tests/test_util.py @@ -26,11 +26,10 @@ def test_bad_python_interpreter_output_triggers_argparse_error(mocker): def test_good_python_env(): data = python_interpreter(sys.executable) - assert isinstance(data, tuple) - data = dict(data) - for value in data.values(): - assert isinstance(value, str) + assert isinstance(data, dict) assert sorted(data) == [ + "abis", + "impl", "implementation_name", "implementation_version", "os_name", @@ -39,11 +38,19 @@ def test_good_python_env(): "platform_release", "platform_system", "platform_version", + "platforms", + "py_ver", "python_executable", "python_full_version", "python_version", + "supported_tags", "sys_platform", ] + assert data.pop("abis") is None + assert data.pop("platforms") is None + assert data.pop("py_ver") >= [3, 8] + for name, value in data.items(): + assert isinstance(value, str), name def test_placeholder_serializes(make_dist): @@ -53,3 +60,11 @@ def test_placeholder_serializes(make_dist): CircularMarker(summary=".", parent=dist) txt = dist.serialise(fields=FIELDS, format="human") assert txt + + +def test_placeholder_attr(): + cm = CircularMarker(summary=".", parent=None) + assert cm.blah is None + assert cm.__doc__ is not None + with pytest.raises(AttributeError): + cm._blah diff --git a/tests/vanilla-0.1.2-py2.py3-none-any.whl b/tests/vanilla-0.1.2-py2.py3-none-any.whl deleted file mode 100644 index 09b76f3f484e6346e014abd9b8790aa04ae9e9dc..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1070 zcmWIWW@Zs#U|`^2IJWDX&t$giM$AASBM_?saam$sW=>9`u7RGRo{?TkW^svbW?ovp zeo1~od`@avYK~q>Mag3(h5!^(ga!Vz#{+fe0kJF|Q^Gx5U46LEd-$Hc$m^}Eb?(gh z%|QlNj2{&FpY_)D(mAQWDaga?q>f&vKKshR3kDaA&8`?y?1U3fc>)hQ|GY_d<0WLf`^bO%sE-qmR@vAqAA`axYH}~dX)FWNr?y5vw9a4=vRDVe5-Nz zS#aah?J`$G_%?Yh$=hV{ZkJ!7|2DHVH5!itnYzQYAN8 z!|i9Y_;)HwPP=h3xS!>LcTf4#EtdPAXq!CYvb}IUq~^}La)t^f)YxFCE!DgL^s6q= z&vJPD9OUZkALLSc);GV&fT7{x?#};9HaH#ou*sR5b=eB$vkrR8B2pt*Hq|No`8C(3 zsQ&YwS=&l42Qbf?mN|Qk*rgS>qrY!cb$_y0nd#B=Tj!2HpK!23XWQj>&q}}VUNBE7 zPch`t>4wc3&WC?Gg!T#PXeF$F^VI450nLx!R`MUHndsVbWPSe8DZk`4@?ATr*eEPL zzwz`#lWlczbyIg0amkeKtg`50KHco0liaZgY0n z@nqesUrFy{Kb)*J_nYxIx6X-i&C^Hn0p5&EBFwl`I50rKU`Zp0A}YP38;PFWAqFxq zENOJZV Date: Wed, 23 Oct 2024 23:50:55 -0500 Subject: [PATCH 2/7] hash from link --- johnnydep/lib.py | 36 ++++++++++++++++++++++++++---------- tests/conftest.py | 2 ++ 2 files changed, 28 insertions(+), 10 deletions(-) diff --git a/johnnydep/lib.py b/johnnydep/lib.py index 5a37d8c..0128120 100644 --- a/johnnydep/lib.py +++ b/johnnydep/lib.py @@ -1,7 +1,6 @@ import hashlib import io import json -import os import re import subprocess import sys @@ -187,9 +186,7 @@ def license(self): @cached_property def versions_available(self): - finder = _get_package_finder() - matches = finder.find_matches(self.project_name, allow_prereleases=True) - versions = [p.version for p in matches][::-1] + versions = _get_versions(self.project_name) if self._local_path is not None: raw_version = self._local_path.name.split("-")[1] local_version = canonicalize_version(raw_version) @@ -263,17 +260,22 @@ def pinned(self): def download_link(self): if self._local_path is not None: return f"file://{self._local_path}" - package_finder = _get_package_finder() - return package_finder.find_best_match(self.req, allow_prereleases=True).best.link.url + best = _get_link(self.req) + if best is not None: + return best.link.url @property def checksum(self): if self._local_path is not None: return "md5=" + hashlib.md5(self._local_path.read_bytes()).hexdigest() - link = self.download_link + best = _get_link(self.req) + if best.link.hashes: + for hash in "md5", "sha256": + if hash in best.link.hashes: + return f"{hash}={best.link.hashes[hash]}" f = io.BytesIO() download_dist( - url=link, + url=best.link.url, f=f, index_url=config.index_url, extra_index_url=config.extra_index_url, @@ -577,14 +579,28 @@ def _get_package_finder(): return package_finder +@lru_cache(maxsize=None) +def _get_versions(req): + finder = _get_package_finder() + matches = finder.find_matches(req, allow_prereleases=True) + versions = [p.version for p in matches][::-1] + return versions + + +@lru_cache(maxsize=None) +def _get_link(req): + package_finder = _get_package_finder() + best = package_finder.find_best_match(req, allow_prereleases=True).best + return best + + @lru_cache(maxsize=None) def _get_info(dist_name): log = logger.bind(dist_name=dist_name) tmpdir = mkdtemp() log.debug("created scratch", tmpdir=tmpdir) try: - package_finder = _get_package_finder() - best = package_finder.find_best_match(dist_name, allow_prereleases=True).best + best = _get_link(dist_name) if best is None: raise JohnnyError(f"Package not found {dist_name!r}") dist_path = Path(tmpdir) / best.link.filename diff --git a/tests/conftest.py b/tests/conftest.py index 0a59b81..02ba28c 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -17,6 +17,8 @@ @pytest.fixture(autouse=True) def expire_caches(): lib._get_info.cache_clear() + lib._get_link.cache_clear() + lib._get_versions.cache_clear() @pytest.fixture(autouse=True) From cec07a25cca0ea815ba8e5b11f16ac0fb8a8d8b0 Mon Sep 17 00:00:00 2001 From: Wim Jeantine-Glenn Date: Thu, 24 Oct 2024 00:44:20 -0500 Subject: [PATCH 3/7] test hash shortcut --- tests/test_lib.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tests/test_lib.py b/tests/test_lib.py index c992cb8..9b5b33a 100644 --- a/tests/test_lib.py +++ b/tests/test_lib.py @@ -3,6 +3,7 @@ from textwrap import dedent import pytest +from packaging.requirements import Requirement from johnnydep import lib from johnnydep.lib import flatten_deps @@ -525,3 +526,12 @@ def test_ignore_errors_version_attrs(mocker): mocker.patch("unearth.finder.PackageFinder.find_matches", return_value=[]) dist = JohnnyDist("notexist", ignore_errors=True) assert dist.version_latest is None + + +def test_checksum_from_link(make_dist, mocker): + make_dist() + dist = JohnnyDist("jdtest") + get_link = mocker.patch("johnnydep.lib._get_link") + get_link.return_value.link.hashes = {"sha256": "cafef00d"} + assert dist.checksum == "sha256=cafef00d" + get_link.assert_called_once_with(Requirement("jdtest")) From 3f45af0f86fd21d6782b95d95c44955a3675f64f Mon Sep 17 00:00:00 2001 From: Wim Jeantine-Glenn Date: Sat, 26 Oct 2024 22:11:21 -0500 Subject: [PATCH 4/7] better caching --- johnnydep/lib.py | 105 +++++++++++++++++++++++++--------------------- tests/conftest.py | 3 +- tests/test_cli.py | 4 +- tests/test_gen.py | 4 +- tests/test_lib.py | 21 +++------- 5 files changed, 69 insertions(+), 68 deletions(-) diff --git a/johnnydep/lib.py b/johnnydep/lib.py index 0128120..677dac9 100644 --- a/johnnydep/lib.py +++ b/johnnydep/lib.py @@ -6,9 +6,11 @@ import sys from collections import defaultdict from collections import deque +from dataclasses import dataclass from functools import cached_property from functools import lru_cache from importlib.metadata import distribution +from importlib.metadata import EntryPoints from importlib.metadata import PackageNotFoundError from importlib.metadata import PathDistribution from pathlib import Path @@ -24,8 +26,8 @@ import tomli_w import unearth import yaml -from packaging import requirements from packaging.markers import default_environment +from packaging.requirements import Requirement from packaging.tags import parse_tag from packaging.utils import canonicalize_name from packaging.utils import canonicalize_version @@ -67,6 +69,10 @@ def __init__(self, req_string, parent=None, ignore_errors=False): self.ignore_errors = ignore_errors self.error = None self._recursed = False + self.checksum = None + self.import_names = None + self.metadata = {} + self.entry_points = None fname, sep, extras = req_string.partition("[") if fname.endswith(".whl") and Path(fname).is_file(): @@ -75,26 +81,29 @@ def __init__(self, req_string, parent=None, ignore_errors=False): name, version, *rest = Path(fname).name.split("-") self.name = canonicalize_name(name) self.specifier = "==" + canonicalize_version(version) - self.req = requirements.Requirement(self.name + sep + extras + self.specifier) + self.req = Requirement(self.name + sep + extras + self.specifier) self.import_names = _discover_import_names(fname) self.metadata = _extract_metadata(fname) self.entry_points = _discover_entry_points(fname) self._local_path = Path(fname).resolve() + self.checksum = "sha256=" + hashlib.sha256(self._local_path.read_bytes()).hexdigest() else: self._local_path = None - self.req = requirements.Requirement(req_string) + self.req = Requirement(req_string) self.name = canonicalize_name(self.req.name) self.specifier = str(self.req.specifier) log.debug("fetching best wheel") try: - self.import_names, self.metadata, self.entry_points = _get_info(dist_name=req_string) + info = _get_info(self.req) except Exception as err: if not self.ignore_errors: raise - self.import_names = None - self.metadata = {} - self.entry_points = None self.error = err + else: + self.import_names = info.import_names + self.metadata = info.metadata + self.entry_points = info.entry_points + self.checksum = "sha256=" + info.sha256 self.extras_requested = sorted(self.req.extras) if parent is None: @@ -110,7 +119,7 @@ def requires(self): return [] result = [] for req_str in all_requires: - req = requirements.Requirement(req_str) + req = Requirement(req_str) req_short, _sep, _marker = str(req).partition(";") if req.marker is None: # unconditional dependency @@ -186,7 +195,7 @@ def license(self): @cached_property def versions_available(self): - versions = _get_versions(self.project_name) + versions = _get_versions(self.req) if self._local_path is not None: raw_version = self._local_path.name.split("-")[1] local_version = canonicalize_version(raw_version) @@ -231,7 +240,7 @@ def version_latest_in_spec(self): def extras_available(self): extras = {x for x in self.metadata.get("provides_extra", []) if x} for req_str in self.metadata.get("requires_dist", []): - req = requirements.Requirement(req_str) + req = Requirement(req_str) extras |= set(re.findall(r"""extra == ['"](.*?)['"]""", str(req.marker))) return sorted(extras) @@ -260,27 +269,9 @@ def pinned(self): def download_link(self): if self._local_path is not None: return f"file://{self._local_path}" - best = _get_link(self.req) - if best is not None: - return best.link.url - - @property - def checksum(self): - if self._local_path is not None: - return "md5=" + hashlib.md5(self._local_path.read_bytes()).hexdigest() - best = _get_link(self.req) - if best.link.hashes: - for hash in "md5", "sha256": - if hash in best.link.hashes: - return f"{hash}={best.link.hashes[hash]}" - f = io.BytesIO() - download_dist( - url=best.link.url, - f=f, - index_url=config.index_url, - extra_index_url=config.extra_index_url, - ) - return "md5=" + hashlib.md5(f.getvalue()).hexdigest() + link = _get_link(self.req) + if link is not None: + return link.url def serialise(self, fields=("name", "summary"), recurse=True, format=None): if format == "pinned": @@ -580,37 +571,56 @@ def _get_package_finder(): @lru_cache(maxsize=None) -def _get_versions(req): +def _get_packages(project_name: str): finder = _get_package_finder() - matches = finder.find_matches(req, allow_prereleases=True) - versions = [p.version for p in matches][::-1] + seq = finder.find_all_packages(project_name, allow_yanked=True) + result = list(seq) + return result + + +def _get_versions(req: Requirement): + packages = _get_packages(req.name) + versions = {p.version for p in packages} + versions = sorted(versions, key=Version) return versions -@lru_cache(maxsize=None) -def _get_link(req): - package_finder = _get_package_finder() - best = package_finder.find_best_match(req, allow_prereleases=True).best - return best +def _get_link(req: Requirement): + packages = _get_packages(req.name) + ok = (p for p in packages if req.specifier.contains(p.version, prereleases=True)) + best = next(ok, None) + if best is not None: + return best.link + + +@dataclass +class _Info: + import_names: list + metadata: dict + entry_points: EntryPoints + sha256: str @lru_cache(maxsize=None) -def _get_info(dist_name): - log = logger.bind(dist_name=dist_name) +def _get_info(req: Requirement): + log = logger.bind(req=str(req)) + link = _get_link(req) + if link is None: + raise JohnnyError(f"Package not found {str(req)!r}") tmpdir = mkdtemp() log.debug("created scratch", tmpdir=tmpdir) try: - best = _get_link(dist_name) - if best is None: - raise JohnnyError(f"Package not found {dist_name!r}") - dist_path = Path(tmpdir) / best.link.filename + dist_path = Path(tmpdir) / link.filename with dist_path.open("wb") as f: download_dist( - url=best.link.url, + url=link.url, f=f, index_url=config.index_url, extra_index_url=config.extra_index_url, ) + sha256 = hashlib.sha256(dist_path.read_bytes()).hexdigest() + if link.hashes is not None and link.hashes.get("sha256", sha256) != sha256: + raise JohnnyError("checksum mismatch") if not dist_path.name.endswith("whl"): args = [sys.executable, "-m", "uv", "build", "--wheel", str(dist_path)] subprocess.run(args, capture_output=True, check=True) @@ -623,4 +633,5 @@ def _get_info(dist_name): finally: log.debug("removing scratch", tmpdir=tmpdir) rmtree(tmpdir, ignore_errors=True) - return import_names, metadata, entry_points + result = _Info(import_names, metadata, entry_points, sha256) + return result diff --git a/tests/conftest.py b/tests/conftest.py index 02ba28c..00c0161 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -16,9 +16,8 @@ @pytest.fixture(autouse=True) def expire_caches(): + lib._get_packages.cache_clear() lib._get_info.cache_clear() - lib._get_link.cache_clear() - lib._get_versions.cache_clear() @pytest.fixture(autouse=True) diff --git a/tests/test_cli.py b/tests/test_cli.py index f6391ca..c7e496c 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -126,7 +126,7 @@ def test_requirements_txt_output(mocker, capsys, make_dist): def test_all_fields_toml_out(mocker, capsys, make_dist, tmp_path): dist_path = make_dist(name="example", version="0.3", py_modules=["that"]) - checksum = hashlib.md5(dist_path.read_bytes()).hexdigest() + checksum = hashlib.sha256(dist_path.read_bytes()).hexdigest() mocker.patch("sys.argv", "johnnydep example<0.4 --fields=ALL --output-format=toml".split()) main() out, err = capsys.readouterr() @@ -153,7 +153,7 @@ def test_all_fields_toml_out(mocker, capsys, make_dist, tmp_path): version_latest = "0.3" version_latest_in_spec = "0.3" download_link = "{tmp_path.as_uri()}/example-0.3-py2.py3-none-any.whl" - checksum = "md5={checksum}" + checksum = "sha256={checksum}" ''' ) diff --git a/tests/test_gen.py b/tests/test_gen.py index d96debc..9eaf2be 100644 --- a/tests/test_gen.py +++ b/tests/test_gen.py @@ -62,7 +62,7 @@ def test_build_from_sdist(add_to_index): assert dist.project_name == "copyingmock" assert dist.download_link.startswith("file://") assert dist.download_link.endswith("copyingmock-0.2.tar.gz") - assert dist.checksum == "md5=9aa6ba13542d25e527fe358d53cdaf3b" + assert dist.checksum == "sha256=fa4c8aad336f6e74f7632f40ff5a271130be5def44ab3177af4578c4d4a66093" def test_plaintext_whl_metadata(add_to_index): @@ -96,7 +96,7 @@ def test_old_metadata_20(add_to_index): "version": "0.1.2", } assert jdist.metadata == expected_metadata - assert jdist.checksum == "md5=488652bac3e1705e5646ea6a51f4d441" + assert jdist.checksum == "sha256=bdcb144db3ba4beebbf5f8b249302560e8894bce6c3688dc79f587d6272ecea4" def test_cant_pin(make_dist, mocker): diff --git a/tests/test_lib.py b/tests/test_lib.py index 9b5b33a..ae79548 100644 --- a/tests/test_lib.py +++ b/tests/test_lib.py @@ -125,15 +125,15 @@ def test_dist_no_children(make_dist): assert jdist.children == [] -def test_checksum_md5(make_dist): +def test_checksum_sha256(make_dist): # the actual checksum value is not repeatable because of timestamps, file modes etc # so we just assert that we get a value which looks like a valid checkum make_dist() jdist = JohnnyDist("jdtest") hashtype, sep, hashval = jdist.checksum.partition("=") - assert hashtype == "md5" + assert hashtype == "sha256" assert sep == "=" - assert len(hashval) == 32 + assert len(hashval) == 64 assert set(hashval) <= set("1234567890abcdef") @@ -478,7 +478,7 @@ def test_local_whl_pinned(make_dist, mocker): # https://github.com/wimglenn/johnnydep/issues/105 dist_path = make_dist(name="loc", version="1.2.3", callback=None) dist = JohnnyDist(dist_path) - mocker.patch("unearth.finder.PackageFinder.find_matches", return_value=[]) + mocker.patch("unearth.finder.PackageFinder.find_all_packages", return_value=[]) txt = dist.serialise(format="pinned").strip() assert txt == "loc==1.2.3" @@ -491,7 +491,7 @@ def test_local_whl_json(make_dist): fields = ["download_link", "checksum", "versions_available"] txt = dist.serialise(format="json", fields=fields).strip() [result] = json.loads(txt) - assert result["checksum"].startswith("md5=") + assert result["checksum"].startswith("sha256=") link = result["download_link"] assert link.startswith("file://") assert link.endswith("loc-0.1.2-py2.py3-none-any.whl") @@ -523,15 +523,6 @@ def test_direct_path_version_insort(make_dist, tmp_path): def test_ignore_errors_version_attrs(mocker): mocker.patch("johnnydep.lib._get_info", side_effect=Exception) - mocker.patch("unearth.finder.PackageFinder.find_matches", return_value=[]) + mocker.patch("unearth.finder.PackageFinder.find_all_packages", return_value=[]) dist = JohnnyDist("notexist", ignore_errors=True) assert dist.version_latest is None - - -def test_checksum_from_link(make_dist, mocker): - make_dist() - dist = JohnnyDist("jdtest") - get_link = mocker.patch("johnnydep.lib._get_link") - get_link.return_value.link.hashes = {"sha256": "cafef00d"} - assert dist.checksum == "sha256=cafef00d" - get_link.assert_called_once_with(Requirement("jdtest")) From f5915336941344df47ef8ccb7e948032f30292ce Mon Sep 17 00:00:00 2001 From: Wim Jeantine-Glenn Date: Sat, 26 Oct 2024 22:20:07 -0500 Subject: [PATCH 5/7] compat --- johnnydep/lib.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/johnnydep/lib.py b/johnnydep/lib.py index 677dac9..6501aa3 100644 --- a/johnnydep/lib.py +++ b/johnnydep/lib.py @@ -10,7 +10,6 @@ from functools import cached_property from functools import lru_cache from importlib.metadata import distribution -from importlib.metadata import EntryPoints from importlib.metadata import PackageNotFoundError from importlib.metadata import PathDistribution from pathlib import Path @@ -597,7 +596,7 @@ def _get_link(req: Requirement): class _Info: import_names: list metadata: dict - entry_points: EntryPoints + entry_points: list sha256: str From 9839875ce83036054c27090efeb542a6e08b76e1 Mon Sep 17 00:00:00 2001 From: Wim Jeantine-Glenn Date: Sat, 26 Oct 2024 22:25:35 -0500 Subject: [PATCH 6/7] simple box --- johnnydep/lib.py | 2 +- tests/test_cli.py | 18 +++++++++--------- tests/test_lib.py | 2 +- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/johnnydep/lib.py b/johnnydep/lib.py index 6501aa3..b44eb44 100644 --- a/johnnydep/lib.py +++ b/johnnydep/lib.py @@ -370,7 +370,7 @@ def gen_tree(johnnydist, with_specifier=True): def gen_table(tree, cols): - table = Table(box=rich.box.SIMPLE_HEAVY, safe_box=False) + table = Table(box=rich.box.SIMPLE) table.add_column("name", overflow="fold") for col in cols: table.add_column(col, overflow="fold") diff --git a/tests/test_cli.py b/tests/test_cli.py index c7e496c..175248b 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -24,7 +24,7 @@ def test_printed_table_on_stdout_with_specifier(make_dist, mocker, capsys): assert out == dedent( """\ name specifier - ━━━━━━━━━━━━━━━━━━━━ + ──────────────────── jdtest >=0.1 """ ) @@ -42,7 +42,7 @@ def test_printed_tree_on_stdout(mocker, capsys, make_dist): assert out == dedent( """\ name extras_available extras_requested - ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + ─────────────────────────────────────────────────────── thing[xyz] abc, xyz xyz └── spam>0.30.0 """ @@ -71,7 +71,7 @@ def test_diamond_deptree(mocker, capsys, make_dist): assert out == dedent( """\ name specifier requires required_by versions_available version_latest_in_spec - ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + ───────────────────────────────────────────────────────────────────────────────────────────────────────────── distA distB1, distB2 0.1 0.1 ├── distB1 distC[x,z]<0.3 distA 0.1 0.1 │ └── distC[x,z] <0.3 distB1 0.1, 0.2, 0.3 0.2 @@ -95,7 +95,7 @@ def test_unresolvable_deptree(mocker, capsys, make_dist): assert out == dedent( """\ name requires required_by versions_available version_latest_in_spec - ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + ──────────────────────────────────────────────────────────────────────────────────────────────────── distX distC<=0.1, distC>0.2 0.1 0.1 ├── distC<=0.1 distX 0.1, 0.2, 0.3 0.1 └── distC>0.2 distX 0.1, 0.2, 0.3 0.3 @@ -169,7 +169,7 @@ def test_ignore_errors_build_error(mocker, capsys, monkeypatch, add_to_index): assert out == dedent( """\ name - ━━━━━━━━━━━━━━━━━━━━━━━ + ─────────────────────── jdt1 ├── jdt2 │ ├── jdt3 (FAILED) @@ -188,7 +188,7 @@ def test_root_has_error(mocker, capsys): assert out == dedent( """\ name - ━━━━━━━━━━━━━━━━━━ + ────────────────── dist404 (FAILED) """ ) @@ -202,7 +202,7 @@ def test_no_deps(mocker, capsys, make_dist): assert out == dedent( """\ name - ━━━━━━━ + ─────── distA """ ) @@ -220,7 +220,7 @@ def test_circular_deptree(mocker, capsys, make_dist): assert out == dedent( """\ name summary - ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + ───────────────────────────────────────────────────────────────────────────────────────────── pkg0 default text for metadata summary └── pkg1 default text for metadata summary ├── pkg2 default text for metadata summary @@ -263,7 +263,7 @@ def test_explicit_env(mocker, make_dist, capsys): assert out == dedent( """\ name summary - ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ + ─────────────────────────────────────────────────── jdtest==0.1.2 default text for metadata summary """ ) diff --git a/tests/test_lib.py b/tests/test_lib.py index ae79548..c59a5a9 100644 --- a/tests/test_lib.py +++ b/tests/test_lib.py @@ -418,7 +418,7 @@ def test_get_caching(make_dist, mocker): assert txt == dedent( """\ name summary - ━━━━━━━━━━━━━━━━━━━━━━━━ + ──────────────────────── a root node ├── b1 branch one │ └── c leaf node From f1c4bb8e5bfe921a497918078286ac3a5c2834e3 Mon Sep 17 00:00:00 2001 From: Wim Jeantine-Glenn Date: Thu, 31 Oct 2024 22:50:28 -0500 Subject: [PATCH 7/7] restore index urls to __init__ --- johnnydep/__init__.py | 9 ----- johnnydep/cli.py | 20 +++++++--- johnnydep/downloader.py | 17 +++----- johnnydep/lib.py | 84 ++++++++++++++++++++-------------------- johnnydep/util.py | 6 ++- tests/conftest.py | 2 +- tests/test_downloader.py | 27 ++++++------- tests/test_util.py | 8 ++-- 8 files changed, 87 insertions(+), 86 deletions(-) diff --git a/johnnydep/__init__.py b/johnnydep/__init__.py index 43bb543..eeef6c5 100644 --- a/johnnydep/__init__.py +++ b/johnnydep/__init__.py @@ -1,11 +1,2 @@ """Display dependency tree of Python distribution""" -from types import SimpleNamespace - - -config = SimpleNamespace( - env=None, - index_url=None, - extra_index_url=None, -) - from .lib import * diff --git a/johnnydep/cli.py b/johnnydep/cli.py index 3f0f6ea..c64f4d0 100644 --- a/johnnydep/cli.py +++ b/johnnydep/cli.py @@ -4,7 +4,6 @@ from importlib.metadata import version import johnnydep -from . import config from .lib import has_error from .lib import JohnnyDist from .logs import configure_logging @@ -121,11 +120,20 @@ def main(argv=None, stdout=None): if "ALL" in args.fields: args.fields = list(FIELDS) configure_logging(verbosity=args.verbose) - config.index_url = args.index_url - config.extra_index_url = args.extra_index_url - if args.env is not None: - config.env = args.env - dist = JohnnyDist(args.req, ignore_errors=args.ignore_errors) + if args.extra_index_url and not args.index_url: + index_urls = ("https://pypi.org/simple", args.extra_index_url) + else: + index_urls = () + if args.index_url: + index_urls += (args.index_url,) + if args.extra_index_url: + index_urls += (args.extra_index_url,) + dist = JohnnyDist( + args.req, + index_urls=index_urls, + env=args.env, + ignore_errors=args.ignore_errors, + ) rendered = dist.serialise( fields=args.fields, format=args.output_format, diff --git a/johnnydep/downloader.py b/johnnydep/downloader.py index 1ad5f9f..9630fd2 100644 --- a/johnnydep/downloader.py +++ b/johnnydep/downloader.py @@ -26,16 +26,11 @@ def _urlretrieve(url, f, data=None, auth=None): f.flush() -def download_dist(url, f, index_url, extra_index_url): +def download_dist(url, f, index_urls=()): auth = None - if index_url: - parsed = urlparse(index_url) - if parsed.username and parsed.password and parsed.hostname == urlparse(url).hostname: - # handling private PyPI credentials in index_url - auth = (parsed.username, parsed.password) - if extra_index_url: - parsed = urlparse(extra_index_url) - if parsed.username and parsed.password and parsed.hostname == urlparse(url).hostname: - # handling private PyPI credentials in extra_index_url - auth = (parsed.username, parsed.password) + for index_url in index_urls: + p = urlparse(index_url) + if p.username and p.password and p.hostname == urlparse(url).hostname: + # handling private PyPI credentials directly in index_url + auth = p.username, p.password _urlretrieve(url, f, auth=auth) diff --git a/johnnydep/lib.py b/johnnydep/lib.py index b44eb44..09a45a2 100644 --- a/johnnydep/lib.py +++ b/johnnydep/lib.py @@ -35,7 +35,6 @@ from rich.tree import Tree from structlog import get_logger -from . import config from .dot import jd2dot from .downloader import download_dist from .util import _bfs @@ -56,22 +55,23 @@ def get_or_create(req_string): class JohnnyDist: - def __init__(self, req_string, parent=None, ignore_errors=False): + def __init__(self, req_string, parent=None, index_urls=(), env=None, ignore_errors=False): if isinstance(req_string, Path): req_string = str(req_string) log = self.log = logger.bind(dist=req_string) log.info("init johnnydist", parent=parent and str(parent.req)) - self._children = [] + self._children = None self.parents = [] if parent is not None: self.parents.append(parent) - self.ignore_errors = ignore_errors + self._ignore_errors = ignore_errors self.error = None - self._recursed = False self.checksum = None self.import_names = None self.metadata = {} self.entry_points = None + self._index_urls = index_urls + self._env = env fname, sep, extras = req_string.partition("[") if fname.endswith(".whl") and Path(fname).is_file(): @@ -93,9 +93,9 @@ def __init__(self, req_string, parent=None, ignore_errors=False): self.specifier = str(self.req.specifier) log.debug("fetching best wheel") try: - info = _get_info(self.req) + info = _get_info(self.req, self._index_urls, self._env) except Exception as err: - if not self.ignore_errors: + if not self._ignore_errors: raise self.error = err else: @@ -126,7 +126,7 @@ def requires(self): continue # conditional dependency - must be evaluated in environment context for extra in [None] + self.extras_requested: - if req.marker.evaluate(dict(config.env or default_environment(), extra=extra)): + if req.marker.evaluate(dict(self._env or default_environment(), extra=extra)): self.log.debug("included conditional dep", req=req_str) result.append(req_short) break @@ -138,8 +138,8 @@ def requires(self): @property def children(self): """my immediate deps, as a tuple of johnnydists""" - if not self._recursed: - assert not self._children + if self._children is None: + self._children = [] self.log.debug("populating dep graph") circular_deps = _detect_circular(self) if circular_deps: @@ -153,10 +153,11 @@ def children(self): child = JohnnyDist( req_string=dep, parent=self, - ignore_errors=self.ignore_errors, + index_urls=self._index_urls, + env=self._env, + ignore_errors=self._ignore_errors, ) self._children.append(child) - self._recursed = True return self._children @property @@ -194,7 +195,7 @@ def license(self): @cached_property def versions_available(self): - versions = _get_versions(self.req) + versions = _get_versions(self.req, self._index_urls, self._env) if self._local_path is not None: raw_version = self._local_path.name.split("-")[1] local_version = canonicalize_version(raw_version) @@ -268,7 +269,7 @@ def pinned(self): def download_link(self): if self._local_path is not None: return f"file://{self._local_path}" - link = _get_link(self.req) + link = _get_link(self.req, self._index_urls, self._env) if link is not None: return link.url @@ -451,7 +452,13 @@ def flatten_deps(johnnydist): extra = f"[{','.join(sorted(extras))}]" else: extra = "" - dist = JohnnyDist(req_string=f"{name}{extra}{spec}") + dist = JohnnyDist( + req_string=f"{name}{extra}{spec}", + index_urls=johnnydist._index_urls, + env=johnnydist._env, + ignore_errors=johnnydist._ignore_errors, + ) + # TODO: set parents dist.required_by = required_by yield dist # TODO: check if this new version causes any new reqs!! @@ -542,23 +549,21 @@ def has_error(dist): return any(has_error(n) for n in dist.children) -def _get_package_finder(): - index_urls = [] - trusted_hosts = [] - if config.index_url: - index_urls.append(config.index_url) - trusted_hosts.append(urlparse(config.index_url).hostname) - if config.extra_index_url: - index_urls.append(config.extra_index_url) - trusted_hosts.append(urlparse(config.extra_index_url).hostname) +def _get_package_finder(index_urls, env): + trusted_hosts = () + for index_url in index_urls: + host = urlparse(index_url).hostname + if host != "pypi.org": + trusted_hosts += (host,) target_python = None - if config.env is not None: + if env is not None: + envd = dict(env) target_python = unearth.TargetPython( - py_ver=config.env["py_ver"], - impl=config.env["impl"], + py_ver=envd["py_ver"], + impl=envd["impl"], ) valid_tags = [] - for tag in config.env["supported_tags"].split(","): + for tag in envd["supported_tags"].split(","): valid_tags.extend(parse_tag(tag)) target_python._valid_tags = valid_tags package_finder = unearth.PackageFinder( @@ -570,22 +575,22 @@ def _get_package_finder(): @lru_cache(maxsize=None) -def _get_packages(project_name: str): - finder = _get_package_finder() +def _get_packages(project_name: str, index_urls: tuple, env: tuple): + finder = _get_package_finder(index_urls, env) seq = finder.find_all_packages(project_name, allow_yanked=True) result = list(seq) return result -def _get_versions(req: Requirement): - packages = _get_packages(req.name) +def _get_versions(req: Requirement, index_urls: tuple, env: tuple): + packages = _get_packages(req.name, index_urls, env) versions = {p.version for p in packages} versions = sorted(versions, key=Version) return versions -def _get_link(req: Requirement): - packages = _get_packages(req.name) +def _get_link(req: Requirement, index_urls: tuple, env: tuple): + packages = _get_packages(req.name, index_urls, env) ok = (p for p in packages if req.specifier.contains(p.version, prereleases=True)) best = next(ok, None) if best is not None: @@ -601,9 +606,9 @@ class _Info: @lru_cache(maxsize=None) -def _get_info(req: Requirement): +def _get_info(req: Requirement, index_urls: tuple, env: tuple): log = logger.bind(req=str(req)) - link = _get_link(req) + link = _get_link(req, index_urls, env) if link is None: raise JohnnyError(f"Package not found {str(req)!r}") tmpdir = mkdtemp() @@ -611,12 +616,7 @@ def _get_info(req: Requirement): try: dist_path = Path(tmpdir) / link.filename with dist_path.open("wb") as f: - download_dist( - url=link.url, - f=f, - index_url=config.index_url, - extra_index_url=config.extra_index_url, - ) + download_dist(url=link.url, f=f, index_urls=index_urls) sha256 = hashlib.sha256(dist_path.read_bytes()).hexdigest() if link.hashes is not None and link.hashes.get("sha256", sha256) != sha256: raise JohnnyError("checksum mismatch") diff --git a/johnnydep/util.py b/johnnydep/util.py index ff06076..ba7c30b 100644 --- a/johnnydep/util.py +++ b/johnnydep/util.py @@ -30,7 +30,11 @@ def python_interpreter(path): env = json.loads(env_json.decode()) except json.JSONDecodeError: raise ArgumentTypeError("Invalid python env output") - return env + for k, v in env.items(): + if isinstance(v, list): + # make result hashable + env[k] = tuple(v) + return tuple(env.items()) class CircularMarker: diff --git a/tests/conftest.py b/tests/conftest.py index 00c0161..162007b 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -125,7 +125,7 @@ def add_package(path): def mock_package_finder(index_urls, target_python, trusted_hosts): return PackageFinder( - index_urls=[], + index_urls=(), target_python=target_python, find_links=list(find_links), ) diff --git a/tests/test_downloader.py b/tests/test_downloader.py index d5d12fe..41c91d8 100644 --- a/tests/test_downloader.py +++ b/tests/test_downloader.py @@ -4,45 +4,47 @@ @pytest.mark.parametrize( - "url, index_url, extra_index_url, expected_auth, expected_top_level_url", + "url, index_urls, expected_auth, expected_top_level_url", [ ( "https://pypi.example.com/packages", - "https://pypi.example.com/simple", - None, + (), None, None, ), ( "https://pypi.example.com/packages", - "https://user:pass@pypi.example.com/simple", + ("https://pypi.example.com/simple",), + None, None, + ), + ( + "https://pypi.example.com/packages", + ("https://user:pass@pypi.example.com/simple",), ("user", "pass"), "pypi.example.com", ), ( "https://pypi.extra.com/packages", - "https://user:pass@pypi.example.com/simple", - "https://pypi.extra.com/simple", + ("https://user:pass@pypi.example.com/simple", "https://pypi.extra.com/simple"), None, "pypi.example.com", ), ( "https://pypi.extra.com/packages", - "https://user:pass@pypi.example.com/simple", - "https://user:extrapass@pypi.extra.com/simple", + ("https://user:pass@pypi.example.com/simple", "https://user:extrapass@pypi.extra.com/simple"), ("user", "extrapass"), "pypi.extra.com", ), ( "https://pypi.extra.com/packages", - None, - "https://user:extrapass@pypi.extra.com/simple", + ("https://user:extrapass@pypi.extra.com/simple",), ("user", "extrapass"), "pypi.extra.com", ), ], ids=( + "empty urls", "index_url without auth", "index_url with auth", "extra_index_url without auth", @@ -50,7 +52,7 @@ "extra_index_url with auth (no index_url)", ), ) -def test_download_dist_auth(mocker, url, index_url, extra_index_url, expected_auth, expected_top_level_url, tmp_path): +def test_download_dist_auth(mocker, url, index_urls, expected_auth, expected_top_level_url, tmp_path): mgr = mocker.patch("johnnydep.downloader.HTTPPasswordMgrWithDefaultRealm") add_password_mock = mgr.return_value.add_password @@ -63,8 +65,7 @@ def test_download_dist_auth(mocker, url, index_url, extra_index_url, expected_au download_dist( url=url + "/test-0.1.tar.gz", f=f, - index_url=index_url, - extra_index_url=extra_index_url, + index_urls=index_urls, ) if expected_auth is None: add_password_mock.assert_not_called() diff --git a/tests/test_util.py b/tests/test_util.py index 5472091..6b9eb00 100644 --- a/tests/test_util.py +++ b/tests/test_util.py @@ -26,8 +26,10 @@ def test_bad_python_interpreter_output_triggers_argparse_error(mocker): def test_good_python_env(): data = python_interpreter(sys.executable) - assert isinstance(data, dict) - assert sorted(data) == [ + assert isinstance(data, tuple) + data = dict(data) + keys = sorted(data) + assert keys == [ "abis", "impl", "implementation_name", @@ -48,7 +50,7 @@ def test_good_python_env(): ] assert data.pop("abis") is None assert data.pop("platforms") is None - assert data.pop("py_ver") >= [3, 8] + assert data.pop("py_ver") >= (3, 8) for name, value in data.items(): assert isinstance(value, str), name