From 73af2e7e2247ad6516ca343167cc5403ead323b6 Mon Sep 17 00:00:00 2001 From: Shubh Bapna Date: Thu, 3 Oct 2024 13:59:21 -0400 Subject: [PATCH] add a graph why command to explain why certain package appears in the graph Signed-off-by: Shubh Bapna --- src/fromager/clickext.py | 23 ++++++++ src/fromager/commands/graph.py | 96 +++++++++++++++++++++++++++---- src/fromager/dependency_graph.py | 3 +- src/fromager/requirements_file.py | 13 +++++ 4 files changed, 122 insertions(+), 13 deletions(-) diff --git a/src/fromager/clickext.py b/src/fromager/clickext.py index 907f4187..00a72f0a 100644 --- a/src/fromager/clickext.py +++ b/src/fromager/clickext.py @@ -4,6 +4,8 @@ import click from packaging.version import Version +from . import requirements_file + class ClickPath(click.Path): """ClickPath that returns pathlib.Path""" @@ -39,3 +41,24 @@ def convert( param, ctx, ) + + +class RequirementType(click.ParamType): + """RequirementType that returns a requirement type""" + + name = "requirement_type" + + def convert( + self, + value: str, + param: click.core.Parameter | None, + ctx: click.core.Context | None, + ) -> requirements_file.RequirementType: + try: + return requirements_file.RequirementType(value) + except Exception as e: + self.fail( + f"Invalid requirement type '{value}' ({e})", + param, + ctx, + ) diff --git a/src/fromager/commands/graph.py b/src/fromager/commands/graph.py index f7b94378..d8701c62 100644 --- a/src/fromager/commands/graph.py +++ b/src/fromager/commands/graph.py @@ -1,6 +1,7 @@ import itertools import json import logging +import pathlib import sys import typing @@ -9,8 +10,14 @@ from packaging.utils import canonicalize_name from packaging.version import Version -from fromager import clickext, dependency_graph, requirements_file +from fromager import clickext, context from fromager.commands import bootstrap +from fromager.dependency_graph import ( + ROOT, + DependencyGraph, + DependencyNode, +) +from fromager.requirements_file import RequirementType logger = logging.getLogger(__name__) @@ -32,9 +39,11 @@ def graph(): type=clickext.ClickPath(), ) @click.pass_obj -def to_constraints(wkctx, graph_file, output): +def to_constraints( + wkctx: context.WorkContext, graph_file: pathlib.Path, output: pathlib.Path +): "Convert a graph file to a constraints file." - graph = dependency_graph.DependencyGraph.from_file(graph_file) + graph = DependencyGraph.from_file(graph_file) if output: with open(output, "w") as f: bootstrap.write_constraints_file(graph, f) @@ -53,9 +62,9 @@ def to_constraints(wkctx, graph_file, output): type=clickext.ClickPath(), ) @click.pass_obj -def to_dot(wkctx, graph_file, output): +def to_dot(wkctx, graph_file: pathlib.Path, output: pathlib.Path): "Convert a graph file to a DOT file suitable to pass to graphviz." - graph = dependency_graph.DependencyGraph.from_file(graph_file) + graph = DependencyGraph.from_file(graph_file) if output: with open(output, "w") as f: write_dot(graph, f) @@ -63,7 +72,7 @@ def to_dot(wkctx, graph_file, output): write_dot(graph, sys.stdout) -def write_dot(graph: dependency_graph.DependencyGraph, output: typing.TextIO) -> None: +def write_dot(graph: DependencyGraph, output: typing.TextIO) -> None: install_constraints = set(node.key for node in graph.get_install_dependencies()) output.write("digraph {\n") @@ -110,7 +119,7 @@ def get_node_id(node): @click.pass_obj def explain_duplicates(wkctx, graph_file): "Report on duplicate installation requirements, and where they come from." - graph = dependency_graph.DependencyGraph.from_file(graph_file) + graph = DependencyGraph.from_file(graph_file) # Look for potential conflicts by tracking how many different versions of # each package are needed. @@ -153,6 +162,69 @@ def explain_duplicates(wkctx, graph_file): print(f" * No single version of {dep_name} meets all requirements") +@graph.command() +@click.option( + "--version", + type=clickext.PackageVersion(), + multiple=True, + help="filter by version for the given package", +) +@click.option( + "--recursive-depth", + type=int, + default=0, + help="recursively get why each package depends on each other. Set depth to -1 for full recursion till root", +) +@click.option( + "--req-type", + type=clickext.RequirementType(), + multiple=True, + help="filter by req type", +) +@click.argument( + "graph-file", + type=clickext.ClickPath(), +) +@click.argument("package-name", type=str) +@click.pass_obj +def why( + wkctx: context.WorkContext, + graph_file: pathlib.Path, + package_name: str, + version: list[Version], + recursive_depth: int, + req_type: list[RequirementType], +): + "Explain why a dependency shows up in the graph" + graph = DependencyGraph.from_file(graph_file) + package_nodes = graph.get_nodes_by_name(package_name) + if version: + package_nodes = [node for node in package_nodes if node.version in version] + for node in package_nodes: + print(f"\n{node.key}") + find_why(graph, node, recursive_depth, 1, req_type) + + +def find_why( + graph: DependencyGraph, + node: DependencyNode, + recursive_depth: int, + depth: int, + req_type: list[RequirementType], +): + for parent in node.parents: + if parent.destination_node.key == ROOT: + continue + if req_type and parent.req_type not in req_type: + continue + + print( + f"{' ' * depth} * is an {parent.req_type} dependency of {parent.destination_node.key} with req {parent.req}" + ) + if recursive_depth and (recursive_depth == -1 or depth <= recursive_depth): + find_why(graph, parent.destination_node, recursive_depth, depth + 1, []) + + @graph.command() @click.option( "-o", @@ -164,12 +236,14 @@ def explain_duplicates(wkctx, graph_file): type=clickext.ClickPath(), ) @click.pass_obj -def migrate_graph(wkctx, graph_file, output): +def migrate_graph( + wkctx: context.WorkContext, graph_file: pathlib.Path, output: pathlib.Path +): "Convert a old graph file into the the new format" - graph = dependency_graph.DependencyGraph() + graph = DependencyGraph() with open(graph_file, "r") as f: old_graph = json.load(f) - stack = [dependency_graph.ROOT] + stack = [ROOT] visited = set() while stack: curr_key = stack.pop() @@ -180,7 +254,7 @@ def migrate_graph(wkctx, graph_file, output): graph.add_dependency( parent_name=canonicalize_name(parent_name) if parent_name else None, parent_version=Version(parent_version) if parent_version else None, - req_type=requirements_file.RequirementType(req_type), + req_type=RequirementType(req_type), req_version=Version(req_version), req=Requirement(req), ) diff --git a/src/fromager/dependency_graph.py b/src/fromager/dependency_graph.py index 01b9a652..bf10e6e5 100644 --- a/src/fromager/dependency_graph.py +++ b/src/fromager/dependency_graph.py @@ -1,4 +1,3 @@ -import io import json import logging import pathlib @@ -178,7 +177,7 @@ def _to_dict(self): visited.add(node.key) return raw_graph - def serialize(self, file_handle: io.TextIOWrapper): + def serialize(self, file_handle: typing.TextIO): raw_graph = self._to_dict() json.dump(raw_graph, file_handle, indent=2, default=str) diff --git a/src/fromager/requirements_file.py b/src/fromager/requirements_file.py index f901f05b..92531f06 100644 --- a/src/fromager/requirements_file.py +++ b/src/fromager/requirements_file.py @@ -17,6 +17,19 @@ class RequirementType(StrEnum): BUILD_BACKEND = "build-backend" BUILD_SDIST = "build-sdist" + def __eq__(self, other: object) -> bool: + if isinstance(other, RequirementType): + if self.value == "build" or other.value == "build": + allowed_values = [ + "build-backend", + "build-system", + "build-sdist", + "build", + ] + return self.value in allowed_values and other.value in allowed_values + return self.value == other.value + return NotImplemented + def parse_requirements_file( req_file: pathlib.Path,