From 45507e26998854b6f5ee4b20176176d099318c0c Mon Sep 17 00:00:00 2001 From: Nick Howard Date: Tue, 7 Mar 2017 16:01:09 -0700 Subject: [PATCH] [engine] rm python graphmaker; create dot formatted display (#4295) ### Problem Printing rule graphs has not been converted to Rust. This can make it hard to visualize what the rule sets look like. Also, the Python rule graph construction is effectively dead code already. ### Solution Port rule graph visualization and visualization based testing to the rust side of the engine implementation. ### Result Now we're setup to use the RuleGraph from the Rust side for functions as described in #4303 and #4304. --- .../build_graph/build_file_address_mapper.py | 2 +- src/python/pants/engine/addressable.py | 6 + src/python/pants/engine/rules.py | 603 +---------------- src/python/pants/engine/scheduler.py | 36 +- src/python/pants/engine/subsystem/native.py | 4 + .../engine/subsystem/native_engine_version | 2 +- src/rust/engine/src/externs.rs | 4 + src/rust/engine/src/lib.rs | 70 +- src/rust/engine/src/rule_graph.rs | 232 +++++-- src/rust/engine/src/tasks.rs | 7 +- .../engine/legacy/test_filemap_integration.py | 2 +- tests/python/pants_test/engine/test_rules.py | 629 ++++++++++-------- 12 files changed, 650 insertions(+), 947 deletions(-) diff --git a/src/python/pants/build_graph/build_file_address_mapper.py b/src/python/pants/build_graph/build_file_address_mapper.py index ae9717e111a..19b920ee895 100644 --- a/src/python/pants/build_graph/build_file_address_mapper.py +++ b/src/python/pants/build_graph/build_file_address_mapper.py @@ -23,7 +23,7 @@ from pants.build_graph.address_lookup_error import AddressLookupError from pants.build_graph.address_mapper import AddressMapper from pants.build_graph.build_file_parser import BuildFileParser -from pants.util.dirutil import fast_relpath, longest_dir_prefix, join_specs +from pants.util.dirutil import fast_relpath, join_specs, longest_dir_prefix logger = logging.getLogger(__name__) diff --git a/src/python/pants/engine/addressable.py b/src/python/pants/engine/addressable.py index 5dc284b89b7..9614ea78a04 100644 --- a/src/python/pants/engine/addressable.py +++ b/src/python/pants/engine/addressable.py @@ -137,6 +137,12 @@ class Exactly(TypeConstraint): def satisfied_by_type(self, obj_type): return obj_type in self._types + def graph_str(self): + if len(self.types) == 1: + return self.types[0].__name__ + else: + return repr(self) + class SubclassesOf(TypeConstraint): """Objects of the exact type as well as any sub-types are allowed.""" diff --git a/src/python/pants/engine/rules.py b/src/python/pants/engine/rules.py index 11d3c871a8b..4f2f335fcbc 100644 --- a/src/python/pants/engine/rules.py +++ b/src/python/pants/engine/rules.py @@ -7,15 +7,12 @@ import logging from abc import abstractproperty -from collections import OrderedDict, defaultdict, deque -from textwrap import dedent +from collections import OrderedDict from twitter.common.collections import OrderedSet from pants.engine.addressable import Exactly -from pants.engine.isolated_process import SnapshottedProcess, SnapshottedProcessRequest -from pants.engine.selectors import (Select, SelectDependencies, SelectLiteral, SelectProjection, - SelectVariant, type_or_constraint_repr) +from pants.engine.selectors import type_or_constraint_repr from pants.util.meta import AbstractClass from pants.util.objects import datatype @@ -150,598 +147,4 @@ def add_task(product_type, rule): raise ValueError('singleton provided by {} has already been provided by: {}'.format( func.__name__, singletons[output_type])) singletons[output_type] = SingletonRule(output_type, func) - return cls(serializable_tasks, intrinsics, singletons) - - def all_rules(self): - """Returns a set containing all rules including instrinsics.""" - declared_rules = set(rule for rules_for_product in self.tasks.values() - for rule in rules_for_product) - declared_intrinsics = set(rule for rule in self.intrinsics.values()) - declared_singletons = set(rule for rule in self.singletons.values()) - return declared_rules.union(declared_intrinsics).union(declared_singletons) - - def all_produced_product_types(self, subject_type): - intrinsic_products = set(prod for subj, prod in self.intrinsics.keys() - if subj == subject_type) - singleton_products = self.singletons.keys() - task_products = self.tasks.keys() - # Unwrap Exactly's if they only contain one type. - # If we don't do this, then the wrapped and unwrapped products both end up in the graph. - # Heterogeneity of constraint types makes this tough. - task_products = set(t._types[0] if type(t) is Exactly and len(t._types) == 1 else t for t in task_products) - - return intrinsic_products.union(task_products).union(set(singleton_products)) - - def gen_rules(self, subject_type, product_type): - # Singletons or intrinsics that provide the requested product for the current subject type. - singleton_node_factory = self.singletons.get(product_type) - intrinsic_node_factory = self.intrinsics.get((subject_type, product_type)) - if singleton_node_factory: - yield singleton_node_factory - elif intrinsic_node_factory: - yield intrinsic_node_factory - else: - # Tasks that provide the requested product. - for node_factory in self._lookup_tasks(product_type): - yield node_factory - - def _lookup_tasks(self, product_type): - for entry in self.tasks.get(product_type, tuple()): - yield entry - - -class CanHaveDependencies(object): - """Marker class for graph entries that can have dependencies on other graph entries.""" - input_selectors = None - subject_type = None - - -class CanBeDependency(object): - """Marker class for graph entries that are leaves, and can be depended upon.""" - - -class RuleGraph(datatype('RuleGraph', - ['graph_maker', - 'root_subject_types', - 'root_rules', - 'rule_dependencies', - 'unfulfillable_rules'])): - """A graph containing rules mapping rules to their dependencies taking into account subject types. - - This is a graph of rules. It models dependencies between rules, along with the subject types for - those rules. This allows the resulting graph to include cases where a selector is fulfilled by the - subject of the graph. - - Because in - - `root_subject_types` the root subject types this graph was generated with. - `root_rules` A map from root rules, ie rules representing the expected selector / subject types - for requests, to the rules that can fulfill them. - `rule_dependencies` A map from rule entries to the rule entries they depend on. - The collections of dependencies are contained by RuleEdges objects. - Keys must be subclasses of CanHaveDependencies - values must be subclasses of CanBeDependency - `unfulfillable_rules` A map of rule entries to collections of Diagnostics - containing the reasons why they were eliminated from the graph. - - """ - - def dependency_edges_for_rule(self, rule, subject_type): - if type(rule) is RootRuleGraphEntry: - return self.root_rule_edges(RootRuleGraphEntry(rule.subject_type, rule.selector)) - else: - return self.rule_dependencies.get(RuleGraphEntry(subject_type, rule)) - - def is_unfulfillable(self, rule, subject_type): - return RuleGraphEntry(subject_type, rule) in self.unfulfillable_rules - - def root_rule_matching(self, subject_type, selector): - root_rule = RootRuleGraphEntry(subject_type, selector) - if root_rule in self.root_rules: - return root_rule - - def new_graph_with_root_for(self, subject_type, selector): - return self.graph_maker.new_graph_from_existing(subject_type, selector, self) - - def root_rule_edges(self, root_rule): - try: - return self.root_rules[root_rule] - except KeyError: - logger.error('missing root rule {}'.format(root_rule)) - raise - - def error_message(self): - """Returns a nice error message for errors in the rule graph.""" - collated_errors = defaultdict(lambda : defaultdict(set)) - for rule_entry, diagnostics in self.unfulfillable_rules.items(): - # don't include the root rules in the error - # message since they are not part of the task list. - # We could include them, but I think we'd want to have a different format, since they - # represent the execution requests. - if type(rule_entry) is RootRuleGraphEntry: - continue - for diagnostic in diagnostics: - collated_errors[rule_entry.rule][diagnostic.reason].add(diagnostic.subject_type) - - def subject_type_str(t): - if t is None: - return 'Any' - elif type(t) is type: - return t.__name__ - elif type(t) is tuple: - return ', '.join(x.__name__ for x in t) - else: - return str(t) - - def format_messages(rule, subject_types_by_reasons): - errors = '\n '.join(sorted('{} with subject types: {}' - .format(reason, ', '.join(sorted(subject_type_str(t) for t in subject_types))) - for reason, subject_types in subject_types_by_reasons.items())) - return '{}:\n {}'.format(rule, errors) - - used_rule_lookup = set(rule_entry.rule for rule_entry in self.rule_dependencies.keys()) - formatted_messages = sorted(format_messages(rule, subject_types_by_reasons) - for rule, subject_types_by_reasons in collated_errors.items() - if rule not in used_rule_lookup) - if not formatted_messages: - return None - return 'Rules with errors: {}\n {}'.format(len(formatted_messages), - '\n '.join(formatted_messages)) - - def __str__(self): - if not self.root_rules: - return '{empty graph}' - - root_subject_types_str = ', '.join(x.__name__ for x in self.root_subject_types) - return dedent(""" - {{ - root_subject_types: ({},) - root_rules: - {} - all_rules: - {} - }}""".format(root_subject_types_str, - '\n '.join(self._dependency_strs(self.root_rules)), - '\n '.join(self._dependency_strs(self.rule_dependencies)) - )).strip() - - def _dependency_strs(self, dependencies): - return sorted('{} => ({},)'.format(rule, ', '.join(str(d) for d in deps)) - for rule, deps in dependencies.items()) - - -class RuleGraphSubjectIsProduct(datatype('RuleGraphSubjectIsProduct', ['value']), CanBeDependency): - """Wrapper for when the dependency is the subject.""" - - @property - def output_product_type(self): - return self.value - - def __repr__(self): - return '{}({})'.format(type(self).__name__, self.value.__name__) - - def __str__(self): - return 'SubjectIsProduct({})'.format(self.value.__name__) - - -class RuleGraphLiteral(datatype('RuleGraphLiteral', ['value', 'product_type']), CanBeDependency): - """The dependency is the literal value held by SelectLiteral.""" - - @property - def output_product_type(self): - return self.product_type - - def __repr__(self): - return '{}({}, {})'.format(type(self).__name__, self.value, self.product_type.__name__) - - def __str__(self): - return 'Literal({}, {})'.format(self.value, self.product_type.__name__) - - -class RuleGraphEntry(datatype('RuleGraphEntry', ['subject_type', 'rule']), - CanBeDependency, - CanHaveDependencies): - """A synthetic rule with a specified subject type""" - - @property - def input_selectors(self): - return self.rule.input_selectors - - @property - def output_product_type(self): - return self.rule.output_product_type - - def __repr__(self): - return '{}({}, {})'.format(type(self).__name__, self.subject_type.__name__, self.rule) - - def __str__(self): - return '{} of {}'.format(self.rule, self.subject_type.__name__) - - -class RootRuleGraphEntry(datatype('RootRule', ['subject_type', 'selector']), CanHaveDependencies): - """A synthetic rule representing a root selector.""" - - @property - def input_selectors(self): - return (self.selector,) - - @property - def output_product_type(self): - return self.selector.product - - @property - def rule(self): - return self # might work - - def __str__(self): - return '{} for {}'.format(self.selector, self.subject_type.__name__) - - -class Diagnostic(datatype('Diagnostic', ['subject_type', 'reason'])): - """Holds on to error reasons for problems with the build graph.""" - - -class UnreachableRule(object): - """A rule entry that can't be reached.""" - - def __init__(self, rule): - self.rule = rule - - -class RuleEdges(object): - """Represents the edges from a rule to its dependencies via selectors.""" - # TODO add a highwater mark count to count how many branches are eliminated - - def __init__(self, dependencies=tuple(), selector_to_deps=None): - self._dependencies = dependencies - if selector_to_deps is None: - self._selector_to_deps = defaultdict(tuple) - else: - self._selector_to_deps = selector_to_deps - - def add_edges_via(self, selector, new_dependencies): - if selector is None and new_dependencies: - raise ValueError("Cannot specify a None selector with non-empty dependencies!") - tupled_other_rules = tuple(new_dependencies) - self._selector_to_deps[selector] += tupled_other_rules - self._dependencies += tupled_other_rules - - def has_edges_for(self, selector): - return selector in self._selector_to_deps - - def __contains__(self, rule): - return rule in self._dependencies - - def __iter__(self): - return self._dependencies.__iter__() - - def makes_unfulfillable(self, dep_to_eliminate): - """Returns true if removing dep_to_eliminate makes this set of edges unfulfillable.""" - if len(self._dependencies) == 1 and self._dependencies[0] == dep_to_eliminate: - return True - for selector, deps in self._selector_to_deps.items(): - if len(deps) == 1 and dep_to_eliminate == deps[0]: - return True - else: - return False - - def without_rule(self, dep_to_eliminate): - new_selector_to_deps = defaultdict(tuple) - for selector, deps in self._selector_to_deps.items(): - new_selector_to_deps[selector] = tuple(d for d in deps if d != dep_to_eliminate) - - return RuleEdges(tuple(d for d in self._dependencies if d != dep_to_eliminate), - new_selector_to_deps) - - -class GraphMaker(object): - - def __init__(self, rule_index, root_subject_types): - if not root_subject_types: - raise ValueError('root_subject_types must not be empty') - self.root_subject_types = root_subject_types - self.rule_index = rule_index - - def _root_selector(self, product): - return Select(product) - - def new_graph_from_existing(self, root_subject_type, root_selector, existing_graph): - root_rule = RootRuleGraphEntry(root_subject_type, root_selector) - root_rule_dependency_edges, edges, unfulfillable = self._construct_graph(root_rule, - root_rule_dependency_edges=existing_graph.root_rules, - rule_dependency_edges=existing_graph.rule_dependencies, - unfulfillable_rules=existing_graph.unfulfillable_rules - ) - root_rule_dependency_edges, edges = self._remove_unfulfillable_rules_and_dependents(root_rule_dependency_edges, - edges, unfulfillable) - return RuleGraph(self, - self.root_subject_types + [root_subject_type], - root_rule_dependency_edges, - edges, - unfulfillable) - - def generate_subgraph(self, root_subject, requested_product): - root_subject_type = type(root_subject) - root_rule = RootRuleGraphEntry(root_subject_type, self._root_selector(requested_product)) - root_rule_dependency_edges, edges, unfulfillable = self._construct_graph(root_rule) - root_rule_dependency_edges, edges = self._remove_unfulfillable_rules_and_dependents(root_rule_dependency_edges, - edges, unfulfillable) - return RuleGraph(self, - (root_subject_type,), - root_rule_dependency_edges, - edges, - unfulfillable) - - def full_graph(self): - """Produces a full graph based on the root subjects and all of the products produced by rules.""" - full_root_rule_dependency_edges = dict() - full_dependency_edges = {} - full_unfulfillable_rules = {} - for root_subject_type in self.root_subject_types: - for product in sorted(self.rule_index.all_produced_product_types(root_subject_type)): - beginning_root = RootRuleGraphEntry(root_subject_type, self._root_selector(product)) - root_dependencies, rule_dependency_edges, unfulfillable_rules = self._construct_graph( - beginning_root, - root_rule_dependency_edges=full_root_rule_dependency_edges, - rule_dependency_edges=full_dependency_edges, - unfulfillable_rules=full_unfulfillable_rules - ) - - full_root_rule_dependency_edges = dict(root_dependencies) - full_dependency_edges = rule_dependency_edges - full_unfulfillable_rules = unfulfillable_rules - - rules_in_graph = set(entry.rule for entry in full_dependency_edges.keys()) - rules_eliminated_during_construction = set(entry.rule - for entry in full_unfulfillable_rules.keys()) - - declared_rules = self.rule_index.all_rules() - unreachable_rules = declared_rules.difference(rules_in_graph, - rules_eliminated_during_construction, - # NB Singletons and intrinsics are ignored for - # purposes of reachability. - self.rule_index.singletons.values(), - self.rule_index.intrinsics.values()) - for rule in sorted(unreachable_rules): - full_unfulfillable_rules[UnreachableRule(rule)] = [Diagnostic(None, 'Unreachable')] - - full_root_rule_dependency_edges, full_dependency_edges = self._remove_unfulfillable_rules_and_dependents( - full_root_rule_dependency_edges, - full_dependency_edges, - full_unfulfillable_rules) - - return RuleGraph(self, - self.root_subject_types, - dict(full_root_rule_dependency_edges), - full_dependency_edges, - full_unfulfillable_rules) - - def _construct_graph(self, - beginning_rule, - root_rule_dependency_edges=None, - rule_dependency_edges=None, - unfulfillable_rules=None): - rules_to_traverse = deque([beginning_rule]) - root_rule_dependency_edges = dict() if root_rule_dependency_edges is None else root_rule_dependency_edges - rule_dependency_edges = dict() if rule_dependency_edges is None else rule_dependency_edges - unfulfillable_rules = dict() if unfulfillable_rules is None else unfulfillable_rules - - def _find_rhs_for_select(subject_type, selector): - if selector.type_constraint.satisfied_by_type(subject_type): - # NB a matching subject is always picked first - return (RuleGraphSubjectIsProduct(subject_type),) - else: - return tuple(RuleGraphEntry(subject_type, rule) - for rule in self.rule_index.gen_rules(subject_type, selector.product)) - - def mark_unfulfillable(entry, subject_type, reason): - if entry not in unfulfillable_rules: - unfulfillable_rules[entry] = [] - unfulfillable_rules[entry].append(Diagnostic(subject_type, reason)) - - def add_rules_to_graph(entry, selector_path, dep_rules): - unseen_dep_rules = [g for g in dep_rules - if g not in rule_dependency_edges and - g not in unfulfillable_rules and - g not in root_rule_dependency_edges] - rules_to_traverse.extend(unseen_dep_rules) - if type(entry) is RootRuleGraphEntry: - if entry in root_rule_dependency_edges: - root_rule_dependency_edges[entry].add_edges_via(selector_path, dep_rules) - else: - new_edges = RuleEdges() - new_edges.add_edges_via(selector_path, dep_rules) - root_rule_dependency_edges[entry] = new_edges - elif entry not in rule_dependency_edges: - new_edges = RuleEdges() - new_edges.add_edges_via(selector_path, dep_rules) - rule_dependency_edges[entry] = new_edges - else: - existing_deps = rule_dependency_edges[entry] - if existing_deps.has_edges_for(selector_path): - raise ValueError("rule {} already has dependencies set for selector {}" - .format(entry, selector_path)) - - existing_deps.add_edges_via(selector_path, dep_rules) - - while rules_to_traverse: - entry = rules_to_traverse.popleft() - if isinstance(entry, CanBeDependency) and not isinstance(entry, CanHaveDependencies): - continue - if not isinstance(entry, CanHaveDependencies): - raise TypeError("Cannot determine dependencies of entry not of type CanHaveDependencies: {}" - .format(entry)) - if entry in unfulfillable_rules: - continue - - if entry in rule_dependency_edges: - continue - - was_unfulfillable = False - - for selector in entry.input_selectors: - if type(selector) in (Select, SelectVariant): - # TODO, handle the Addresses / Variants case - rules_or_literals_for_selector = _find_rhs_for_select(entry.subject_type, selector) - if not rules_or_literals_for_selector: - mark_unfulfillable(entry, entry.subject_type, 'no matches for {}'.format(selector)) - was_unfulfillable = True - continue - add_rules_to_graph(entry, selector, rules_or_literals_for_selector) - elif type(selector) is SelectLiteral: - add_rules_to_graph(entry, - selector, - (RuleGraphLiteral(selector.subject, selector.product),)) - elif type(selector) is SelectDependencies: - initial_selector = selector.input_product_selector - initial_rules_or_literals = _find_rhs_for_select(entry.subject_type, initial_selector) - if not initial_rules_or_literals: - mark_unfulfillable(entry, - entry.subject_type, - 'no matches for {} when resolving {}' - .format(initial_selector, selector)) - was_unfulfillable = True - continue - - rules_for_dependencies = [] - for field_type in selector.field_types: - rules_for_field_subjects = _find_rhs_for_select(field_type, - selector.projected_product_selector) - rules_for_dependencies.extend(rules_for_field_subjects) - - if not rules_for_dependencies: - mark_unfulfillable(entry, - selector.field_types, - 'no matches for {} when resolving {}' - .format(selector.projected_product_selector, selector)) - was_unfulfillable = True - continue - - add_rules_to_graph(entry, - (selector, selector.input_product_selector), - initial_rules_or_literals) - add_rules_to_graph(entry, - (selector, selector.projected_product_selector), - tuple(rules_for_dependencies)) - elif type(selector) is SelectProjection: - # TODO, could validate that input product has fields - initial_rules_or_literals = _find_rhs_for_select(entry.subject_type, - selector.input_product_selector) - if not initial_rules_or_literals: - mark_unfulfillable(entry, - entry.subject_type, - 'no matches for {} when resolving {}' - .format(selector.input_product_selector, selector)) - was_unfulfillable = True - continue - - projected_rules = _find_rhs_for_select(selector.projected_subject, - selector.projected_product_selector) - if not projected_rules: - mark_unfulfillable(entry, - selector.projected_subject, - 'no matches for {} when resolving {}' - .format(selector.projected_product_selector, selector)) - was_unfulfillable = True - continue - - add_rules_to_graph(entry, - (selector, selector.input_product_selector), - initial_rules_or_literals) - add_rules_to_graph(entry, - (selector, selector.projected_product_selector), - projected_rules) - else: - raise TypeError('Unexpected type of selector: {}'.format(selector)) - - if type(entry.rule) is SnapshottedProcess: - # TODO, this is a copy of the SelectDependencies with some changes - # Need to come up with a better approach here, but this fixes things - # It's also not tested explicitly. - snapshot_selector = entry.rule.snapshot_selector - initial_selector = entry.rule.snapshot_selector.input_product_selector - initial_rules_or_literals = _find_rhs_for_select(SnapshottedProcessRequest, initial_selector) - if not initial_rules_or_literals: - mark_unfulfillable(entry, - entry.subject_type, - 'no matches for {} when resolving {}' - .format(initial_selector, snapshot_selector)) - was_unfulfillable = True - else: - - rules_for_dependencies = [] - for field_type in snapshot_selector.field_types: - rules_for_field_subjects = _find_rhs_for_select(field_type, - snapshot_selector.projected_product_selector) - rules_for_dependencies.extend(rules_for_field_subjects) - - if not rules_for_dependencies: - mark_unfulfillable(entry, - snapshot_selector.field_types, - 'no matches for {} when resolving {}' - .format(snapshot_selector.projected_product_selector, snapshot_selector)) - was_unfulfillable = True - else: - add_rules_to_graph(entry, - (snapshot_selector, snapshot_selector.input_product_selector), - initial_rules_or_literals) - add_rules_to_graph(entry, - (snapshot_selector, snapshot_selector.projected_product_selector), - tuple(rules_for_dependencies)) - - - if not was_unfulfillable: - # NB: In this case, there are no selectors. - add_rules_to_graph(entry, None, tuple()) - - return root_rule_dependency_edges, rule_dependency_edges, unfulfillable_rules - - def _remove_unfulfillable_rules_and_dependents(self, - root_rule_dependency_edges, - rule_dependency_edges, - unfulfillable_rules): - """Removes all unfulfillable rules transitively from the roots and the dependency edges. - - - Takes the current root rule set and dependency table and removes all rules that are not - transitively fulfillable. - - Deforestation. Leaping from tree to tree.""" - # could experiment with doing this for each rule added and deduping the traversal list - removal_traversal = deque(unfulfillable_rules.keys()) - while removal_traversal: - unfulfillable_entry = removal_traversal.popleft() - for current_entry, dependency_edges in tuple(rule_dependency_edges.items()): - if current_entry in unfulfillable_rules: - # NB: these are removed at the end - continue - - if dependency_edges.makes_unfulfillable(unfulfillable_entry): - unfulfillable_rules[current_entry] = [Diagnostic(current_entry.subject_type, - 'depends on unfulfillable {}'.format(unfulfillable_entry))] - removal_traversal.append(current_entry) - else: - rule_dependency_edges[current_entry] = dependency_edges.without_rule(unfulfillable_entry) - - for current_entry, dependency_edges in tuple(root_rule_dependency_edges.items()): - if current_entry in unfulfillable_rules: - # NB: these are removed at the end - continue - - if dependency_edges.makes_unfulfillable(unfulfillable_entry): - unfulfillable_rules[current_entry] = [Diagnostic(current_entry.subject_type, - 'depends on unfulfillable {}'.format(unfulfillable_entry))] - removal_traversal.append(current_entry) - else: - root_rule_dependency_edges[current_entry] = dependency_edges.without_rule(unfulfillable_entry) - - rule_dependency_edges = {k: v for k, v in rule_dependency_edges.items() - if k not in unfulfillable_rules} - root_rule_dependency_edges = {k: v for k, v in root_rule_dependency_edges.items() - if k not in unfulfillable_rules} - - for root_rule, deps in root_rule_dependency_edges.items(): - for d in deps: - if d not in rule_dependency_edges and isinstance(d, RuleGraphEntry): - raise ValueError('expected all referenced dependencies to have entries in the graph: {}'.format(d)) - return root_rule_dependency_edges, rule_dependency_edges + return cls(serializable_tasks, intrinsics, singletons) \ No newline at end of file diff --git a/src/python/pants/engine/scheduler.py b/src/python/pants/engine/scheduler.py index 7dc7125e28f..ca57c7101dc 100644 --- a/src/python/pants/engine/scheduler.py +++ b/src/python/pants/engine/scheduler.py @@ -17,7 +17,7 @@ from pants.build_graph.address import Address, BuildFileAddress from pants.engine.addressable import SubclassesOf from pants.engine.fs import FileContent, FilesContent, Path, PathGlobs, Snapshot -from pants.engine.isolated_process import create_snapshot_singletons, _Snapshots +from pants.engine.isolated_process import _Snapshots, create_snapshot_singletons from pants.engine.nodes import Return, Throw from pants.engine.rules import RuleIndex from pants.engine.selectors import (Select, SelectDependencies, SelectLiteral, SelectProjection, @@ -84,14 +84,17 @@ def graph_trace(self): yield line.rstrip() def assert_ruleset_valid(self): - listed = list(TypeId(self._to_id(t)) for t in self.root_subject_types) + root_type_ids = self._root_type_ids() - raw_value = self._native.lib.validator_run(self._scheduler, listed, len(listed)) + raw_value = self._native.lib.validator_run(self._scheduler, root_type_ids, len(root_type_ids)) value = self._from_value(raw_value) if isinstance(value, Exception): raise ValueError(str(value)) + def _root_type_ids(self): + return list(TypeId(self._to_id(t)) for t in sorted(self.root_subject_types)) + def _to_value(self, obj): return self._native.context.to_value(obj) @@ -198,6 +201,33 @@ def _register_tasks(self, tasks): def visualize_graph_to_file(self, filename): self._native.lib.graph_visualize(self._scheduler, bytes(filename)) + def rule_graph_visualization(self): + root_type_ids = self._root_type_ids() + + with temporary_file_path() as path: + self._native.lib.rule_graph_visualize( + self._scheduler, + root_type_ids, + len(root_type_ids), + bytes(path)) + with open(path) as fd: + for line in fd.readlines(): + yield line.rstrip() + + def rule_subgraph_visualization(self, root_subject_type, product_type): + root_type_id = TypeId(self._to_id(root_subject_type)) + + product_type_id = TypeConstraint(self._to_id(constraint_for(product_type))) + with temporary_file_path() as path: + self._native.lib.rule_subgraph_visualize( + self._scheduler, + root_type_id, + product_type_id, + bytes(path)) + with open(path) as fd: + for line in fd.readlines(): + yield line.rstrip() + def invalidate(self, filenames): filenames_buf = self._native.context.utf8_buf_buf(filenames) return self._native.lib.graph_invalidate(self._scheduler, filenames_buf) diff --git a/src/python/pants/engine/subsystem/native.py b/src/python/pants/engine/subsystem/native.py index edbd0e2f3fc..b3950f10e88 100644 --- a/src/python/pants/engine/subsystem/native.py +++ b/src/python/pants/engine/subsystem/native.py @@ -83,6 +83,7 @@ typedef void ExternContext; + // On the rust side the integration is defined in externs.rs typedef void (*extern_log)(ExternContext*, uint8_t, uint8_t*, uint64_t); typedef Key (*extern_key_for)(ExternContext*, Value*); typedef Value (*extern_val_for)(ExternContext*, Key*); @@ -196,6 +197,9 @@ Value validator_run(RawScheduler*, TypeId*, uint64_t); + void rule_graph_visualize(RawScheduler*, TypeId*, uint64_t, char*); + void rule_subgraph_visualize(RawScheduler*, TypeId, TypeConstraint, char*); + void nodes_destroy(RawNodes*); ''' ) diff --git a/src/python/pants/engine/subsystem/native_engine_version b/src/python/pants/engine/subsystem/native_engine_version index 76603f81c58..a4b9d538c96 100644 --- a/src/python/pants/engine/subsystem/native_engine_version +++ b/src/python/pants/engine/subsystem/native_engine_version @@ -1 +1 @@ -c99656dc56811ab5e99f1bbc312950d2c6a2432e +00bd3b7dfb8532b371806c6c38c52338eae0d82a diff --git a/src/rust/engine/src/externs.rs b/src/rust/engine/src/externs.rs index d73fb3ed47c..f1ce4b276e1 100644 --- a/src/rust/engine/src/externs.rs +++ b/src/rust/engine/src/externs.rs @@ -114,6 +114,10 @@ pub fn project_str(value: &Value, field: &str) -> String { val_to_str(&name_val) } +pub fn key_to_str(key: &Key) -> String { + val_to_str(&val_for(key)) +} + pub fn id_to_str(digest: Id) -> String { with_externs(|e| { (e.id_to_str)(e.context, digest).to_string().unwrap_or_else(|e| { diff --git a/src/rust/engine/src/lib.rs b/src/rust/engine/src/lib.rs index a4b6765f73a..408ba35f7ad 100644 --- a/src/rust/engine/src/lib.rs +++ b/src/rust/engine/src/lib.rs @@ -28,11 +28,14 @@ extern crate tar; extern crate tempdir; use std::ffi::CStr; +use std::fs::File; +use std::io; use std::mem; use std::os::raw; -use std::path::PathBuf; +use std::path::{Path, PathBuf}; use std::sync::Arc; + use context::Core; use core::{Function, Key, TypeConstraint, TypeId, Value}; use externs::{ @@ -60,7 +63,7 @@ use externs::{ with_vec, }; use nodes::Failure; -use rule_graph::{GraphMaker, RootSubjectTypes}; +use rule_graph::{GraphMaker, RuleGraph}; use scheduler::{RootResult, Scheduler, ExecutionStat}; use tasks::Tasks; use types::Types; @@ -495,7 +498,7 @@ pub extern fn validator_run( with_scheduler(scheduler_ptr, |raw| { with_vec(subject_types_ptr, subject_types_len as usize, |subject_types| { let graph_maker = GraphMaker::new(&raw.scheduler.core.tasks, - RootSubjectTypes { subject_types: subject_types.clone() }); + subject_types.clone()); let graph = graph_maker.full_graph(); match graph.validate() { @@ -510,6 +513,67 @@ pub extern fn validator_run( }) } +#[no_mangle] +pub extern fn rule_graph_visualize( + scheduler_ptr: *mut RawScheduler, + subject_types_ptr: *mut TypeId, + subject_types_len: u64, + path_ptr: *const raw::c_char +) { + with_scheduler(scheduler_ptr, |raw| { + with_vec(subject_types_ptr, subject_types_len as usize, |subject_types| { + let path_str = unsafe { CStr::from_ptr(path_ptr).to_string_lossy().into_owned() }; + let path = PathBuf::from(path_str); + + let graph = graph_full(raw, subject_types); + write_to_file(path.as_path(), &graph).unwrap_or_else(|e| { + println!("Failed to visualize to {}: {:?}", path.display(), e); + }); + }) + }) +} + +#[no_mangle] +pub extern fn rule_subgraph_visualize( + scheduler_ptr: *mut RawScheduler, + subject_type: TypeId, + product_type: TypeConstraint, + path_ptr: *const raw::c_char +) { + with_scheduler(scheduler_ptr, |raw| { + let path_str = unsafe { CStr::from_ptr(path_ptr).to_string_lossy().into_owned() }; + let path = PathBuf::from(path_str); + + let graph = graph_sub(raw, subject_type, product_type); + write_to_file(path.as_path(), &graph).unwrap_or_else(|e| { + println!("Failed to visualize to {}: {:?}", path.display(), e); + }); + }) +} + + +fn graph_full(raw: &mut RawScheduler, subject_types: &Vec) -> RuleGraph { + let graph_maker = GraphMaker::new(&raw.scheduler.core.tasks, + subject_types.clone()); + graph_maker.full_graph() +} + +fn graph_sub( + raw: &mut RawScheduler, + subject_type: TypeId, + product_type: TypeConstraint +) -> RuleGraph { + let graph_maker = GraphMaker::new(&raw.scheduler.core.tasks, + vec![subject_type.clone()]); + graph_maker.sub_graph(&subject_type, &product_type) +} + +fn write_to_file(path: &Path, graph: &RuleGraph) -> io::Result<()> { + let file = File::create(path)?; + let mut f = io::BufWriter::new(file); + graph.visualize(&mut f) +} + fn with_scheduler(scheduler_ptr: *mut RawScheduler, f: F) -> T where F: FnOnce(&mut RawScheduler)->T { let mut scheduler = unsafe { Box::from_raw(scheduler_ptr) }; diff --git a/src/rust/engine/src/rule_graph.rs b/src/rust/engine/src/rule_graph.rs index 2f43db88d27..54d46c7c0bd 100644 --- a/src/rust/engine/src/rule_graph.rs +++ b/src/rust/engine/src/rule_graph.rs @@ -4,6 +4,8 @@ use std::collections::{hash_map, HashMap, HashSet, VecDeque}; use std::hash::Hash; +use std::fmt; +use std::io; use core::{ANY_TYPE, Function, Id, Key, TypeConstraint, TypeId, Value}; use externs; @@ -133,11 +135,6 @@ type RuleDependencyEdges = HashMap; type RuleDiagnostics = Vec; type UnfulfillableRuleMap = HashMap; -#[derive(Debug)] -pub struct RootSubjectTypes { - pub subject_types: Vec -} - #[derive(Eq, Hash, PartialEq, Clone, Debug)] pub struct Diagnostic { subject_type: TypeId, @@ -147,16 +144,53 @@ pub struct Diagnostic { // Given the task index and the root subjects, it produces a rule graph that allows dependency nodes // to be found statically rather than dynamically. pub struct GraphMaker<'t> { - tasks: &'t Tasks, - root_subject_types: RootSubjectTypes + tasks: &'t Tasks, + root_subject_types: Vec } impl <'t> GraphMaker<'t> { - pub fn new(tasks: &'t Tasks, root_subject_types: RootSubjectTypes) -> GraphMaker<'t> { - GraphMaker { - tasks: tasks, - root_subject_types: root_subject_types - } + pub fn new(tasks: &'t Tasks, root_subject_types: Vec) -> GraphMaker<'t> { + GraphMaker { tasks: tasks, root_subject_types: root_subject_types } + } + + pub fn sub_graph(&self, subject_type: &TypeId, product_type: &TypeConstraint) -> RuleGraph { + let mut full_root_rule_dependency_edges: RootRuleDependencyEdges = HashMap::new(); + let mut full_dependency_edges: RuleDependencyEdges = HashMap::new(); + let mut full_unfulfillable_rules: UnfulfillableRuleMap = HashMap::new(); + + let beginning_root = if let Some(beginning_root) = self.gen_root_entry(subject_type, product_type) { + beginning_root + } else { + return RuleGraph { root_subject_types: vec![], + root_dependencies: full_root_rule_dependency_edges, + rule_dependency_edges: full_dependency_edges, + unfulfillable_rules: full_unfulfillable_rules, + } + }; + + let constructed_graph = self._construct_graph( + beginning_root, + full_root_rule_dependency_edges, + full_dependency_edges, + full_unfulfillable_rules + ); + + // less than ideal, the copying + full_root_rule_dependency_edges = constructed_graph.root_dependencies.clone(); + full_dependency_edges = constructed_graph.rule_dependency_edges.clone(); + full_unfulfillable_rules = constructed_graph.unfulfillable_rules.clone(); + + self.add_unreachable_rule_diagnostics(&full_dependency_edges, &mut full_unfulfillable_rules); + + let mut unfinished_graph = RuleGraph { + root_subject_types: self.root_subject_types.clone(), + root_dependencies: full_root_rule_dependency_edges, + rule_dependency_edges: full_dependency_edges, + unfulfillable_rules: full_unfulfillable_rules + }; + + self._remove_unfulfillable_rules_and_dependents(&mut unfinished_graph); + unfinished_graph } pub fn full_graph(&self) -> RuleGraph { @@ -178,6 +212,21 @@ impl <'t> GraphMaker<'t> { full_dependency_edges = constructed_graph.rule_dependency_edges.clone(); full_unfulfillable_rules = constructed_graph.unfulfillable_rules.clone(); } + + self.add_unreachable_rule_diagnostics(&full_dependency_edges, &mut full_unfulfillable_rules); + + let mut in_progress_graph = RuleGraph { + root_subject_types: self.root_subject_types.clone(), + root_dependencies: full_root_rule_dependency_edges, + rule_dependency_edges: full_dependency_edges, + unfulfillable_rules: full_unfulfillable_rules + }; + + self._remove_unfulfillable_rules_and_dependents(&mut in_progress_graph); + in_progress_graph + } + + fn add_unreachable_rule_diagnostics(&self, full_dependency_edges: &RuleDependencyEdges, full_unfulfillable_rules: &mut UnfulfillableRuleMap) { let rules_in_graph: HashSet<_> = full_dependency_edges.keys().map(|f| f.rule.clone()).collect(); let unfulfillable_discovered_during_construction: HashSet<_> = full_unfulfillable_rules.keys().map(|f| f.rule().clone()).collect(); let declared_rules = self.tasks.all_rules(); @@ -193,14 +242,6 @@ impl <'t> GraphMaker<'t> { let diagnostics = full_unfulfillable_rules.entry(Entry::new_unreachable(rule)).or_insert(vec![]); diagnostics.push(Diagnostic { subject_type: ANY_TYPE, reason: "Unreachable".to_string() }); } - - let unfinished_graph = RuleGraph { - root_dependencies: full_root_rule_dependency_edges, - rule_dependency_edges: full_dependency_edges, - unfulfillable_rules: full_unfulfillable_rules - }; - - self._remove_unfulfillable_rules_and_dependents(unfinished_graph) } fn _construct_graph(&self, @@ -380,11 +421,16 @@ impl <'t> GraphMaker<'t> { vec![]); } } - RuleGraph {root_dependencies: root_rule_dependency_edges, rule_dependency_edges: rule_dependency_edges, unfulfillable_rules: unfulfillable_rules} + RuleGraph { + root_subject_types: self.root_subject_types.clone(), + root_dependencies: root_rule_dependency_edges, + rule_dependency_edges: rule_dependency_edges, + unfulfillable_rules: unfulfillable_rules + } } fn _remove_unfulfillable_rules_and_dependents(&self, - mut rule_graph: RuleGraph) -> RuleGraph { + rule_graph: &mut RuleGraph) { // Removes all unfulfillable rules transitively from the roots and the dependency edges. // // Takes the current root rule set and dependency table and removes all rules that are not @@ -427,32 +473,56 @@ impl <'t> GraphMaker<'t> { } } } - rule_graph } - fn gen_root_entries(&self, product_types: &Vec) -> Vec { + fn gen_root_entries(&self, product_types: &HashSet) -> Vec { let mut result: Vec = Vec::new(); - for subj_type in &self.root_subject_types.subject_types { + for subj_type in &self.root_subject_types { for pt in product_types { - if let Some(tasks) = self.tasks.gen_tasks(subj_type, pt) { - if !tasks.is_empty() { - result.push(RootEntry { - subject_type: subj_type.clone(), - clause: vec![Selector::Select(Select { - product: pt.clone(), - variant_key: None - })] - }); - } + if let Some(entry) = self.gen_root_entry(subj_type, pt) { + result.push(entry); } } } result } + + fn gen_root_entry(&self, subject_type: &TypeId, product_type: &TypeConstraint) -> Option { + self.tasks.gen_tasks(subject_type, product_type) + .and_then(|tasks| if !tasks.is_empty() { Some(tasks) } else { None }) + .map(|_| { + RootEntry { + subject_type: subject_type.clone(), + clause: vec![ + Selector::Select(Select { + product: product_type.clone(), + variant_key: None + }) + ] + } + }) + } } + +/// +/// A graph containing rules mapping rules to their dependencies taking into account subject types. +/// +/// This is a graph of rules. It models dependencies between rules, along with the subject types for +/// those rules. This allows the resulting graph to include cases where a selector is fulfilled by the +/// subject of the graph. +/// +/// +/// `root_subject_types` the root subject types this graph was generated with. +/// `root_dependencies` A map from root rules, ie rules representing the expected selector / subject types +/// for requests, to the rules that can fulfill them. +/// `rule_dependency_edges` A map from rule entries to the rule entries they depend on. +/// The collections of dependencies are contained by RuleEdges objects. +/// `unfulfillable_rules` A map of rule entries to collections of Diagnostics +/// containing the reasons why they were eliminated from the graph. #[derive(Debug)] pub struct RuleGraph { + root_subject_types: Vec, root_dependencies: RootRuleDependencyEdges, rule_dependency_edges: RuleDependencyEdges, unfulfillable_rules: UnfulfillableRuleMap, @@ -460,7 +530,7 @@ pub struct RuleGraph { fn type_constraint_str(type_constraint: TypeConstraint) -> String { let val = to_val(type_constraint); - repr_of_val(&val) + call_on_val(&val, "graph_str") } fn to_val(type_constraint: TypeConstraint) -> Value { @@ -475,8 +545,8 @@ fn to_val_from_id(id: Id) -> Value { externs::val_for_id(id) } -fn repr_of_val(value: &Value) -> String { - let rpr_val = externs::project_ignoring_type(&value, "__repr__"); +fn call_on_val(value: &Value, method: &str) -> String { + let rpr_val = externs::project_ignoring_type(&value, method); let invoke_result = externs::invoke_runnable(&rpr_val, &[], false) .expect("string from calling repr"); @@ -509,21 +579,22 @@ fn selector_str(selector: &Selector) -> String { &Selector::SelectDependencies(ref s) => format!("SelectDependencies({}, {}, {}field_types=({},){})", type_constraint_str(s.product), type_constraint_str(s.dep_product), - if s.field == "dependencies" { "".to_string() } else {format!("{:?}, ", s.field)}, + if s.field == "dependencies" { "".to_string() } else {format!("'{}', ", s.field)}, s.field_types.iter() .map(|&f| type_str(f)) .collect::>() .join(", "), if s.transitive { ", transitive=True" } else { "" }.to_string() - ), - &Selector::SelectProjection(ref s) => format!("SelectProjection({}, {}, ({:?},), {})", + &Selector::SelectProjection(ref s) => format!("SelectProjection({}, {}, ('{}',), {})", type_constraint_str(s.product), type_str(s.projected_subject), s.field, type_constraint_str(s.input_product), ), - &Selector::SelectLiteral(ref s) => format!("SelectLiteral({:?})", s) + &Selector::SelectLiteral(ref s) => format!("SelectLiteral({}, {})", + externs::key_to_str(&s.subject), + type_constraint_str(s.product)) } } @@ -532,8 +603,19 @@ fn entry_str(entry: &Entry) -> String { &Entry::InnerEntry(ref inner) => { format!("{} of {}", task_display(&inner.rule), type_str(inner.subject_type)) } - other => { - format!("{:?}", other) + &Entry::Root(ref root) => { + format!("{} for {}", + root.clause.iter().map(|s| selector_str(s)).collect::>().join(", "), + type_str(root.subject_type)) + } + &Entry::SubjectIsProduct { subject_type } => { + format!("SubjectIsProduct({})", type_str(subject_type)) + } + &Entry::Literal { ref value, product } => { + format!("Literal({}, {})", externs::key_to_str(value), type_constraint_str(product)) + } + &Entry::Unreachable { ref rule, ref reason } => { + format!("Unreachable({}, {:?})", task_display(rule), reason) } } } @@ -600,6 +682,50 @@ impl RuleGraph { _ => false, }) } + + // TODO instead of this, make own fmt thing that accepts externs + pub fn visualize(&self, f: &mut io::Write) -> io::Result<()> { + if self.root_dependencies.is_empty() && self.rule_dependency_edges.is_empty() { + write!(f, "digraph {{\n")?; + write!(f, " // empty graph\n")?; + return write!(f, "}}"); + } + + + let mut root_subject_type_strs = self.root_subject_types.iter() + .map(|&t| type_str(t)) + .collect::>(); + root_subject_type_strs.sort(); + write!(f, "digraph {{\n")?; + write!(f, " // root subject types: {}\n", root_subject_type_strs.join(", "))?; + write!(f, " // root entries\n")?; + let mut root_rule_strs = self.root_dependencies.iter() + .map(|(k, deps)| { + let root_str = entry_str(&Entry::from(k.clone())); + format!(" \"{}\" [color=blue]\n \"{}\" -> {{{}}}", + root_str, + root_str, + deps.dependencies.iter() + .map(|d| format!("\"{}\"", entry_str(d))) + .collect::>() + .join(" ")) + }) + .collect::>(); + root_rule_strs.sort(); + write!(f, "{}\n", root_rule_strs.join("\n"))?; + + + write!(f, " // internal entries\n")?; + let mut internal_rule_strs = self.rule_dependency_edges.iter() + .map(|(k, deps)| format!(" \"{}\" -> {{{}}}", entry_str(&Entry::from(k.clone())), deps.dependencies.iter() + .map(|d| format!("\"{}\"", entry_str(d))) + .collect::>() + .join(" "))) + .collect::>(); + internal_rule_strs.sort(); + write!(f, "{}\n", internal_rule_strs.join("\n"))?; + write!(f, "}}") + } } #[derive(Eq, PartialEq, Clone, Debug)] @@ -623,8 +749,12 @@ impl RuleEdges { } let mut deps_for_selector = self.selector_to_dependencies.entry(selector_path).or_insert(vec![]); for d in new_dependencies { - deps_for_selector.push(d.clone()); - self.dependencies.push(d.clone()); + if !deps_for_selector.contains(d) { + deps_for_selector.push(d.clone()); + } + if !self.dependencies.contains(d) { + self.dependencies.push(d.clone()); + } } } @@ -652,7 +782,7 @@ impl RuleEdges { for (selector, deps) in &self.selector_to_dependencies { new_selector_deps.insert(selector.clone(), deps.iter().filter(|&d| d != dep).map(|d| d.clone()).collect()); } - RuleEdges { dependencies: new_deps, selector_to_dependencies: new_selector_deps} + RuleEdges { dependencies: new_deps, selector_to_dependencies: new_selector_deps } } } @@ -662,15 +792,17 @@ fn update_edges_based_on_unfulfillable_entry(edge_container: &mut HashMap, - K: Eq + Hash + Clone + K: Eq + Hash + Clone + fmt::Debug { let keys: Vec<_> = edge_container.keys() - .filter(|&c| !new_unfulfillable_rules.contains_key(&Entry::from(c.clone()))) .map(|c| c.clone()) .collect(); + for current_entry in keys { if let hash_map::Entry::Occupied(mut o) = edge_container.entry(current_entry) { - if o.get().makes_unfulfillable(&unfulfillable_entry) { + if new_unfulfillable_rules.contains_key(&Entry::from(o.key().clone())) { + o.remove(); + } else if o.get().makes_unfulfillable(&unfulfillable_entry) { let key_entry = Entry::from(o.key().clone()); let entry_subject = key_entry.subject_type(); diff --git a/src/rust/engine/src/tasks.rs b/src/rust/engine/src/tasks.rs index 47967de452e..64f108036e6 100644 --- a/src/rust/engine/src/tasks.rs +++ b/src/rust/engine/src/tasks.rs @@ -1,7 +1,7 @@ // Copyright 2017 Pants project contributors (see CONTRIBUTORS.md). // Licensed under the Apache License, Version 2.0 (see LICENSE). -use std::collections::HashMap; +use std::collections::{HashMap, HashSet}; use core::{Field, Function, FNV, Key, TypeConstraint, TypeId}; use selectors::{Selector, Select, SelectDependencies, SelectLiteral, SelectProjection}; @@ -51,8 +51,9 @@ impl Tasks { } } - pub fn all_product_types(&self) -> Vec { - self.all_rules().iter().map(|t| t.product).collect() + pub fn all_product_types(&self) -> HashSet { + self.all_rules().iter().map(|t| t.product) + .collect::>() } pub fn is_singleton_task(&self, sought_task: &Task) -> bool { diff --git a/tests/python/pants_test/engine/legacy/test_filemap_integration.py b/tests/python/pants_test/engine/legacy/test_filemap_integration.py index 6de0222bd1b..8361e6a7dde 100644 --- a/tests/python/pants_test/engine/legacy/test_filemap_integration.py +++ b/tests/python/pants_test/engine/legacy/test_filemap_integration.py @@ -110,4 +110,4 @@ def test_implicit_sources(self): test_out) test_out = self._extract_exclude_output('test_with_implicit_sources') - self.assertEquals({'test_a.py'}, test_out) \ No newline at end of file + self.assertEquals({'test_a.py'}, test_out) diff --git a/tests/python/pants_test/engine/test_rules.py b/tests/python/pants_test/engine/test_rules.py index 4c00d1ef3a0..3f0a3bf1ff9 100644 --- a/tests/python/pants_test/engine/test_rules.py +++ b/tests/python/pants_test/engine/test_rules.py @@ -17,7 +17,7 @@ from pants.engine.build_files import create_graph_tasks from pants.engine.fs import PathGlobs, create_fs_tasks from pants.engine.mapper import AddressMapper -from pants.engine.rules import GraphMaker, Rule, RuleIndex +from pants.engine.rules import Rule, RuleIndex from pants.engine.scheduler import WrappedNativeScheduler from pants.engine.selectors import Select, SelectDependencies, SelectLiteral, SelectProjection from pants.engine.subsystem.native import Native @@ -132,8 +132,8 @@ def test_ruleset_with_missing_product_type(self): self.assert_equal_with_printing(dedent(""" Rules with errors: 1 - (Exactly(A), (Select(Exactly(B)),), noop): - no matches for Select(Exactly(B)) with subject types: SubA + (A, (Select(B),), noop): + no matches for Select(B) with subject types: SubA """).strip(), str(cm.exception)) @@ -145,9 +145,9 @@ def test_ruleset_with_rule_with_two_missing_selects(self): self.assert_equal_with_printing(dedent(""" Rules with errors: 1 - (Exactly(A), (Select(Exactly(B)), Select(Exactly(C))), noop): - no matches for Select(Exactly(B)) with subject types: SubA - no matches for Select(Exactly(C)) with subject types: SubA + (A, (Select(B), Select(C)), noop): + no matches for Select(B) with subject types: SubA + no matches for Select(C) with subject types: SubA """).strip(), str(cm.exception)) @@ -168,10 +168,10 @@ def test_ruleset_with_superclass_of_selected_type_produced_fails(self): validator.assert_ruleset_valid() self.assert_equal_with_printing(dedent(""" Rules with errors: 2 - (Exactly(A), (Select(Exactly(B)),), noop): - depends on unfulfillable (Exactly(B), (Select(Exactly(SubA)),), noop) of C with subject types: C - (Exactly(B), (Select(Exactly(SubA)),), noop): - no matches for Select(Exactly(SubA)) with subject types: C + (A, (Select(B),), noop): + depends on unfulfillable (B, (Select(SubA),), noop) of C with subject types: C + (B, (Select(SubA),), noop): + no matches for Select(SubA) with subject types: C """).strip(), str(cm.exception)) @@ -214,8 +214,8 @@ def test_ruleset_with_failure_due_to_incompatible_subject_for_intrinsic(self): # This error message could note near matches like the intrinsic. self.assert_equal_with_printing(dedent(""" Rules with errors: 1 - (Exactly(D), (Select(Exactly(C)),), noop): - no matches for Select(Exactly(C)) with subject types: A + (D, (Select(C),), noop): + no matches for Select(C) with subject types: A """).strip(), str(cm.exception)) @@ -233,7 +233,7 @@ def test_ruleset_unreachable_due_to_product_of_select_dependencies(self): self.assert_equal_with_printing(dedent(""" Rules with errors: 1 - (Exactly(A), (SelectDependencies(Exactly(B), Exactly(SubA), field_types=(D,)),), noop): + (A, (SelectDependencies(B, SubA, field_types=(D,)),), noop): Unreachable with subject types: Any """).strip(), str(cm.exception)) @@ -253,10 +253,10 @@ def test_not_fulfillable_duplicated_dependency(self): self.assert_equal_with_printing(dedent(""" Rules with errors: 2 - (Exactly(B), (Select(Exactly(D)),), noop): - depends on unfulfillable (Exactly(D), (Select(Exactly(A)), SelectDependencies(Exactly(A), Exactly(SubA), field_types=(C,))), noop) of SubA with subject types: SubA - (Exactly(D), (Select(Exactly(A)), SelectDependencies(Exactly(A), Exactly(SubA), field_types=(C,))), noop): - depends on unfulfillable (Exactly(A), (Select(Exactly(SubA)),), noop) of C with subject types: SubA + (B, (Select(D),), noop): + depends on unfulfillable (D, (Select(A), SelectDependencies(A, SubA, field_types=(C,))), noop) of SubA with subject types: SubA + (D, (Select(A), SelectDependencies(A, SubA, field_types=(C,))), noop): + depends on unfulfillable (A, (Select(SubA),), noop) of C with subject types: SubA """).strip(), str(cm.exception)) @@ -271,8 +271,8 @@ def test_initial_select_projection_failure(self): self.assert_equal_with_printing(dedent(""" Rules with errors: 1 - (Exactly(A), (SelectProjection(Exactly(B), D, ("some",), Exactly(C)),), noop): - no matches for Select(Exactly(C)) when resolving SelectProjection(Exactly(B), D, ("some",), Exactly(C)) with subject types: SubA + (A, (SelectProjection(B, D, ('some',), C),), noop): + no matches for Select(C) when resolving SelectProjection(B, D, ('some',), C) with subject types: SubA """).strip(), str(cm.exception)) @@ -289,8 +289,8 @@ def test_secondary_select_projection_failure(self): self.assert_equal_with_printing(dedent(""" Rules with errors: 1 - (Exactly(A), (SelectProjection(Exactly(B), D, ("some",), Exactly(C)),), noop): - no matches for Select(Exactly(B)) when resolving SelectProjection(Exactly(B), D, ("some",), Exactly(C)) with subject types: D + (A, (SelectProjection(B, D, ('some',), C),), noop): + no matches for Select(B) when resolving SelectProjection(B, D, ('some',), C) with subject types: D """).strip(), str(cm.exception)) @@ -303,32 +303,21 @@ class RuleGraphMakerTest(unittest.TestCase): # TODO something with variants # TODO HasProducts? - def test_fails_if_root_subject_types_empty(self): - rules = [ - (A, (Select(B),), noop), - ] - with self.assertRaises(ValueError) as cm: - GraphMaker(RuleIndex.create(rules), tuple()) - self.assertEquals(dedent(""" - root_subject_types must not be empty - """).strip(), str(cm.exception)) - def test_smallest_full_test(self): rules = [ (Exactly(A), (Select(SubA),), noop) ] - - fullgraph = GraphMaker(RuleIndex.create(rules, tuple()), - root_subject_types={SubA}).full_graph() + fullgraph = self.create_full_graph({SubA}, RuleIndex.create(rules, tuple())) self.assert_equal_with_printing(dedent(""" - { - root_subject_types: (SubA,) - root_rules: - Select(A) for SubA => ((Exactly(A), (Select(SubA),), noop) of SubA,) - all_rules: - (Exactly(A), (Select(SubA),), noop) of SubA => (SubjectIsProduct(SubA),) - }""").strip(), fullgraph) + digraph { + // root subject types: SubA + // root entries + "Select(A) for SubA" [color=blue] + "Select(A) for SubA" -> {"(A, (Select(SubA),), noop) of SubA"} + // internal entries + "(A, (Select(SubA),), noop) of SubA" -> {"SubjectIsProduct(SubA)"} + }""").strip(), fullgraph) def test_full_graph_for_planner_example(self): symbol_table_cls = TargetTable @@ -336,71 +325,83 @@ def test_full_graph_for_planner_example(self): project_tree = 'Let us pretend that this is a ProjectTree!' tasks = create_graph_tasks(address_mapper, symbol_table_cls) + create_fs_tasks(project_tree) - rule_index = RuleIndex.create(tasks, intrinsic_entries=[]) - graphmaker = GraphMaker(rule_index, - root_subject_types={Address, + rule_index = RuleIndex.create(tasks, tuple()) + root_subject_types = {Address, PathGlobs, SingleAddress, SiblingAddresses, DescendantAddresses, - AscendantAddresses}) - fullgraph = graphmaker.full_graph() + AscendantAddresses} + fullgraph_str = self.create_full_graph(root_subject_types, rule_index) + print('---diagnostic------') - print(fullgraph.error_message()) + print(fullgraph_str) print('/---diagnostic------') - print(fullgraph) - - - # Assert that all of the rules specified the various task fns are present - declared_rules = rule_index.all_rules() - rules_remaining_in_graph_strs = set(str(r.rule) for r in fullgraph.rule_dependencies.keys()) - declared_rule_strings = set(str(r) for r in declared_rules) - self.assertEquals(declared_rule_strings, - rules_remaining_in_graph_strs - ) + in_root_rules = False + in_all_rules = False + all_rules = [] + root_rule_lines = [] + for line in fullgraph_str.splitlines(): + if line.startswith(' // root subject types:'): + pass + elif line.startswith(' // root entries'): + in_root_rules = True + elif line.startswith(' // internal entries'): + in_all_rules = True + elif in_all_rules: + all_rules.append(line) + elif in_root_rules: + root_rule_lines.append(line) + else: + pass + + self.assertEquals(31, len(all_rules)) + self.assertEquals(56, len(root_rule_lines)) # 2 lines per entry def test_smallest_full_test_multiple_root_subject_types(self): rules = [ (A, (Select(SubA),), noop), (B, (Select(A),), noop) ] - graphmaker = GraphMaker(RuleIndex.create(rules, tuple()), - root_subject_types=OrderedSet([SubA, A])) - fullgraph = graphmaker.full_graph() + fullgraph = self.create_full_graph(OrderedSet([SubA, A]), RuleIndex.create(rules, tuple())) self.assert_equal_with_printing(dedent(""" - { - root_subject_types: (SubA, A,) - root_rules: - Select(A) for A => (SubjectIsProduct(A),) - Select(A) for SubA => ((A, (Select(SubA),), noop) of SubA,) - Select(B) for A => ((B, (Select(A),), noop) of A,) - Select(B) for SubA => ((B, (Select(A),), noop) of SubA,) - all_rules: - (A, (Select(SubA),), noop) of SubA => (SubjectIsProduct(SubA),) - (B, (Select(A),), noop) of A => (SubjectIsProduct(A),) - (B, (Select(A),), noop) of SubA => ((A, (Select(SubA),), noop) of SubA,) - }""").strip(), - fullgraph) + digraph { + // root subject types: A, SubA + // root entries + "Select(A) for A" [color=blue] + "Select(A) for A" -> {"SubjectIsProduct(A)"} + "Select(A) for SubA" [color=blue] + "Select(A) for SubA" -> {"(A, (Select(SubA),), noop) of SubA"} + "Select(B) for A" [color=blue] + "Select(B) for A" -> {"(B, (Select(A),), noop) of A"} + "Select(B) for SubA" [color=blue] + "Select(B) for SubA" -> {"(B, (Select(A),), noop) of SubA"} + // internal entries + "(A, (Select(SubA),), noop) of SubA" -> {"SubjectIsProduct(SubA)"} + "(B, (Select(A),), noop) of A" -> {"SubjectIsProduct(A)"} + "(B, (Select(A),), noop) of SubA" -> {"(A, (Select(SubA),), noop) of SubA"} + }""").strip(), + fullgraph) def test_single_rule_depending_on_subject_selection(self): rules = [ (Exactly(A), (Select(SubA),), noop) ] - graphmaker = GraphMaker(RuleIndex.create(rules, tuple()), - root_subject_types=_suba_root_subject_types) - subgraph = graphmaker.generate_subgraph(SubA(), requested_product=A) + subgraph = self.create_subgraph(A, rules, SubA()) self.assert_equal_with_printing(dedent(""" - { - root_subject_types: (SubA,) - root_rules: - Select(A) for SubA => ((Exactly(A), (Select(SubA),), noop) of SubA,) - all_rules: - (Exactly(A), (Select(SubA),), noop) of SubA => (SubjectIsProduct(SubA),) - }""").strip(), subgraph) + digraph { + // root subject types: SubA + // root entries + "Select(A) for SubA" [color=blue] + "Select(A) for SubA" -> {"(A, (Select(SubA),), noop) of SubA"} + // internal entries + "(A, (Select(SubA),), noop) of SubA" -> {"SubjectIsProduct(SubA)"} + }""").strip(), + subgraph) def test_multiple_selects(self): rules = [ @@ -408,20 +409,19 @@ def test_multiple_selects(self): (B, tuple(), noop) ] - graphmaker = GraphMaker(RuleIndex.create(rules, tuple()), - root_subject_types=_suba_root_subject_types) - subgraph = graphmaker.generate_subgraph(SubA(), requested_product=A) + subgraph = self.create_subgraph(A, rules, SubA()) self.assert_equal_with_printing(dedent(""" - { - root_subject_types: (SubA,) - root_rules: - Select(A) for SubA => ((Exactly(A), (Select(SubA), Select(B)), noop) of SubA,) - all_rules: - (B, (), noop) of SubA => (,) - (Exactly(A), (Select(SubA), Select(B)), noop) of SubA => (SubjectIsProduct(SubA), (B, (), noop) of SubA,) - }""").strip(), - subgraph) + digraph { + // root subject types: SubA + // root entries + "Select(A) for SubA" [color=blue] + "Select(A) for SubA" -> {"(A, (Select(SubA), Select(B)), noop) of SubA"} + // internal entries + "(A, (Select(SubA), Select(B)), noop) of SubA" -> {"SubjectIsProduct(SubA)" "(B, (,), noop) of SubA"} + "(B, (,), noop) of SubA" -> {} + }""").strip(), + subgraph) def test_one_level_of_recursion(self): rules = [ @@ -429,19 +429,19 @@ def test_one_level_of_recursion(self): (B, (Select(SubA),), noop) ] - graphmaker = GraphMaker(RuleIndex.create(rules, tuple()), - root_subject_types=_suba_root_subject_types) - subgraph = graphmaker.generate_subgraph(SubA(), requested_product=A) + subgraph = self.create_subgraph(A, rules, SubA()) self.assert_equal_with_printing(dedent(""" - { - root_subject_types: (SubA,) - root_rules: - Select(A) for SubA => ((Exactly(A), (Select(B),), noop) of SubA,) - all_rules: - (B, (Select(SubA),), noop) of SubA => (SubjectIsProduct(SubA),) - (Exactly(A), (Select(B),), noop) of SubA => ((B, (Select(SubA),), noop) of SubA,) - }""").strip(), subgraph) + digraph { + // root subject types: SubA + // root entries + "Select(A) for SubA" [color=blue] + "Select(A) for SubA" -> {"(A, (Select(B),), noop) of SubA"} + // internal entries + "(A, (Select(B),), noop) of SubA" -> {"(B, (Select(SubA),), noop) of SubA"} + "(B, (Select(SubA),), noop) of SubA" -> {"SubjectIsProduct(SubA)"} + }""").strip(), + subgraph) def test_noop_removal_in_subgraph(self): rules = [ @@ -453,19 +453,22 @@ def test_noop_removal_in_subgraph(self): (B, C, noop), ] - graphmaker = GraphMaker(RuleIndex.create(rules, - intrinsics), - root_subject_types=_suba_root_subject_types) - subgraph = graphmaker.generate_subgraph(SubA(), requested_product=A) + subgraph = self.create_subgraph_with_intrinsics(intrinsics, + A, + rules, + SubA(), + _suba_root_subject_types) self.assert_equal_with_printing(dedent(""" - { - root_subject_types: (SubA,) - root_rules: - Select(A) for SubA => ((Exactly(A), (), noop) of SubA,) - all_rules: - (Exactly(A), (), noop) of SubA => (,) - }""").strip(), subgraph) + digraph { + // root subject types: SubA + // root entries + "Select(A) for SubA" [color=blue] + "Select(A) for SubA" -> {"(A, (,), noop) of SubA"} + // internal entries + "(A, (,), noop) of SubA" -> {} + }""").strip(), + subgraph) def test_noop_removal_full_single_subject_type(self): rules = [ @@ -477,18 +480,41 @@ def test_noop_removal_full_single_subject_type(self): (B, C, noop), ] - graphmaker = GraphMaker(RuleIndex.create(rules, intrinsics), - root_subject_types=_suba_root_subject_types) - fullgraph = graphmaker.full_graph() + fullgraph = self.create_full_graph(_suba_root_subject_types, + RuleIndex.create(rules, intrinsics)) + + self.assert_equal_with_printing(dedent(""" + digraph { + // root subject types: SubA + // root entries + "Select(A) for SubA" [color=blue] + "Select(A) for SubA" -> {"(A, (,), noop) of SubA"} + // internal entries + "(A, (,), noop) of SubA" -> {} + }""").strip(), + fullgraph) + + def test_root_tuple_removed_when_no_matches(self): + root_subjects = {C, D} + rules = [ + (Exactly(A), (Select(C),), noop), + (Exactly(B), (Select(D), Select(A)), noop), + ] + intrinsics = [] + + fullgraph = self.create_full_graph(root_subjects, + RuleIndex.create(rules, intrinsics)) self.assert_equal_with_printing(dedent(""" - { - root_subject_types: (SubA,) - root_rules: - Select(A) for SubA => ((Exactly(A), (), noop) of SubA,) - all_rules: - (Exactly(A), (), noop) of SubA => (,) - }""").strip(), fullgraph) + digraph { + // root subject types: C, D + // root entries + "Select(A) for C" [color=blue] + "Select(A) for C" -> {"(A, (Select(C),), noop) of C"} + // internal entries + "(A, (Select(C),), noop) of C" -> {"SubjectIsProduct(C)"} + }""").strip(), + fullgraph) def test_noop_removal_transitive(self): # If a noop-able rule has rules that depend on it, @@ -501,20 +527,22 @@ def test_noop_removal_transitive(self): intrinsics = [ (D, C, BoringRule(C)) ] - graphmaker = GraphMaker(RuleIndex.create(rules, intrinsics), - root_subject_types=_suba_root_subject_types, - - ) - subgraph = graphmaker.generate_subgraph(SubA(), requested_product=A) + subgraph = self.create_subgraph_with_intrinsics(intrinsics, + A, + rules, + SubA(), + _suba_root_subject_types) self.assert_equal_with_printing(dedent(""" - { - root_subject_types: (SubA,) - root_rules: - Select(A) for SubA => ((Exactly(A), (), noop) of SubA,) - all_rules: - (Exactly(A), (), noop) of SubA => (,) - }""").strip(), subgraph) + digraph { + // root subject types: SubA + // root entries + "Select(A) for SubA" [color=blue] + "Select(A) for SubA" -> {"(A, (,), noop) of SubA"} + // internal entries + "(A, (,), noop) of SubA" -> {} + }""").strip(), + subgraph) def test_select_dependencies_with_separate_types_for_subselectors(self): rules = [ @@ -523,21 +551,20 @@ def test_select_dependencies_with_separate_types_for_subselectors(self): (C, (Select(SubA),), noop) ] - graphmaker = GraphMaker(RuleIndex.create(rules, tuple()), - root_subject_types=_suba_root_subject_types) - subgraph = graphmaker.generate_subgraph(SubA(), requested_product=A) + subgraph = self.create_subgraph(A, rules, SubA()) self.assert_equal_with_printing(dedent(""" - { - root_subject_types: (SubA,) - root_rules: - Select(A) for SubA => ((Exactly(A), (SelectDependencies(B, C, field_types=(D,)),), noop) of SubA,) - all_rules: - (B, (Select(D),), noop) of D => (SubjectIsProduct(D),) - (C, (Select(SubA),), noop) of SubA => (SubjectIsProduct(SubA),) - (Exactly(A), (SelectDependencies(B, C, field_types=(D,)),), noop) of SubA => ((C, (Select(SubA),), noop) of SubA, (B, (Select(D),), noop) of D,) - }""").strip(), - subgraph) + digraph { + // root subject types: SubA + // root entries + "Select(A) for SubA" [color=blue] + "Select(A) for SubA" -> {"(A, (SelectDependencies(B, C, field_types=(D,)),), noop) of SubA"} + // internal entries + "(A, (SelectDependencies(B, C, field_types=(D,)),), noop) of SubA" -> {"(C, (Select(SubA),), noop) of SubA" "(B, (Select(D),), noop) of D"} + "(B, (Select(D),), noop) of D" -> {"SubjectIsProduct(D)"} + "(C, (Select(SubA),), noop) of SubA" -> {"SubjectIsProduct(SubA)"} + }""").strip(), + subgraph) def test_select_dependencies_with_subject_as_first_subselector(self): rules = [ @@ -545,20 +572,19 @@ def test_select_dependencies_with_subject_as_first_subselector(self): (B, (Select(D),), noop), ] - graphmaker = GraphMaker(RuleIndex.create(rules, tuple()), - root_subject_types=_suba_root_subject_types) - subgraph = graphmaker.generate_subgraph(SubA(), requested_product=A) + subgraph = self.create_subgraph(A, rules, SubA()) self.assert_equal_with_printing(dedent(""" - { - root_subject_types: (SubA,) - root_rules: - Select(A) for SubA => ((Exactly(A), (SelectDependencies(B, SubA, field_types=(D,)),), noop) of SubA,) - all_rules: - (B, (Select(D),), noop) of D => (SubjectIsProduct(D),) - (Exactly(A), (SelectDependencies(B, SubA, field_types=(D,)),), noop) of SubA => (SubjectIsProduct(SubA), (B, (Select(D),), noop) of D,) - }""").strip(), - subgraph) + digraph { + // root subject types: SubA + // root entries + "Select(A) for SubA" [color=blue] + "Select(A) for SubA" -> {"(A, (SelectDependencies(B, SubA, field_types=(D,)),), noop) of SubA"} + // internal entries + "(A, (SelectDependencies(B, SubA, field_types=(D,)),), noop) of SubA" -> {"SubjectIsProduct(SubA)" "(B, (Select(D),), noop) of D"} + "(B, (Select(D),), noop) of D" -> {"SubjectIsProduct(D)"} + }""").strip(), + subgraph) def test_select_dependencies_multiple_field_types_all_resolvable(self): rules = [ @@ -566,21 +592,20 @@ def test_select_dependencies_multiple_field_types_all_resolvable(self): (B, (Select(Exactly(C, D)),), noop), ] - graphmaker = GraphMaker(RuleIndex.create(rules, tuple()), - root_subject_types=_suba_root_subject_types) - subgraph = graphmaker.generate_subgraph(SubA(), requested_product=A) + subgraph = self.create_subgraph(A, rules, SubA()) self.assert_equal_with_printing(dedent(""" - { - root_subject_types: (SubA,) - root_rules: - Select(A) for SubA => ((Exactly(A), (SelectDependencies(B, SubA, field_types=(C, D,)),), noop) of SubA,) - all_rules: - (B, (Select(Exactly(C, D)),), noop) of C => (SubjectIsProduct(C),) - (B, (Select(Exactly(C, D)),), noop) of D => (SubjectIsProduct(D),) - (Exactly(A), (SelectDependencies(B, SubA, field_types=(C, D,)),), noop) of SubA => (SubjectIsProduct(SubA), (B, (Select(Exactly(C, D)),), noop) of C, (B, (Select(Exactly(C, D)),), noop) of D,) - }""").strip(), - subgraph) + digraph { + // root subject types: SubA + // root entries + "Select(A) for SubA" [color=blue] + "Select(A) for SubA" -> {"(A, (SelectDependencies(B, SubA, field_types=(C, D,)),), noop) of SubA"} + // internal entries + "(A, (SelectDependencies(B, SubA, field_types=(C, D,)),), noop) of SubA" -> {"SubjectIsProduct(SubA)" "(B, (Select(Exactly(C, D)),), noop) of C" "(B, (Select(Exactly(C, D)),), noop) of D"} + "(B, (Select(Exactly(C, D)),), noop) of C" -> {"SubjectIsProduct(C)"} + "(B, (Select(Exactly(C, D)),), noop) of D" -> {"SubjectIsProduct(D)"} + }""").strip(), + subgraph) def test_select_dependencies_multiple_field_types_all_resolvable_with_deps(self): rules = [ @@ -590,22 +615,21 @@ def test_select_dependencies_multiple_field_types_all_resolvable_with_deps(self) (C, (Select(D),), noop), ] - graphmaker = GraphMaker(RuleIndex.create(rules, tuple()), - root_subject_types=_suba_root_subject_types) - subgraph = graphmaker.generate_subgraph(SubA(), requested_product=A) + subgraph = self.create_subgraph(A, rules, SubA()) self.assert_equal_with_printing(dedent(""" - { - root_subject_types: (SubA,) - root_rules: - Select(A) for SubA => ((Exactly(A), (SelectDependencies(B, SubA, field_types=(C, D,)),), noop) of SubA,) - all_rules: - (B, (Select(C),), noop) of C => (SubjectIsProduct(C),) - (B, (Select(C),), noop) of D => ((C, (Select(D),), noop) of D,) - (C, (Select(D),), noop) of D => (SubjectIsProduct(D),) - (Exactly(A), (SelectDependencies(B, SubA, field_types=(C, D,)),), noop) of SubA => (SubjectIsProduct(SubA), (B, (Select(C),), noop) of C, (B, (Select(C),), noop) of D,) - }""").strip(), - subgraph) + digraph { + // root subject types: SubA + // root entries + "Select(A) for SubA" [color=blue] + "Select(A) for SubA" -> {"(A, (SelectDependencies(B, SubA, field_types=(C, D,)),), noop) of SubA"} + // internal entries + "(A, (SelectDependencies(B, SubA, field_types=(C, D,)),), noop) of SubA" -> {"SubjectIsProduct(SubA)" "(B, (Select(C),), noop) of C" "(B, (Select(C),), noop) of D"} + "(B, (Select(C),), noop) of C" -> {"SubjectIsProduct(C)"} + "(B, (Select(C),), noop) of D" -> {"(C, (Select(D),), noop) of D"} + "(C, (Select(D),), noop) of D" -> {"SubjectIsProduct(D)"} + }""").strip(), + subgraph) def test_select_dependencies_recurse_with_different_type(self): rules = [ @@ -615,25 +639,24 @@ def test_select_dependencies_recurse_with_different_type(self): (SubA, tuple(), noop) ] - graphmaker = GraphMaker(RuleIndex.create(rules, tuple()), - root_subject_types=_suba_root_subject_types) - subgraph = graphmaker.generate_subgraph(SubA(), requested_product=A) + subgraph = self.create_subgraph(A, rules, SubA()) self.assert_equal_with_printing(dedent(""" - { - root_subject_types: (SubA,) - root_rules: - Select(A) for SubA => ((Exactly(A), (SelectDependencies(B, SubA, field_types=(C, D,)),), noop) of SubA,) - all_rules: - (B, (Select(A),), noop) of C => ((Exactly(A), (SelectDependencies(B, SubA, field_types=(C, D,)),), noop) of C,) - (B, (Select(A),), noop) of D => ((Exactly(A), (SelectDependencies(B, SubA, field_types=(C, D,)),), noop) of D,) - (Exactly(A), (SelectDependencies(B, SubA, field_types=(C, D,)),), noop) of C => ((SubA, (), noop) of C, (B, (Select(A),), noop) of C, (B, (Select(A),), noop) of D,) - (Exactly(A), (SelectDependencies(B, SubA, field_types=(C, D,)),), noop) of D => ((SubA, (), noop) of D, (B, (Select(A),), noop) of C, (B, (Select(A),), noop) of D,) - (Exactly(A), (SelectDependencies(B, SubA, field_types=(C, D,)),), noop) of SubA => (SubjectIsProduct(SubA), (B, (Select(A),), noop) of C, (B, (Select(A),), noop) of D,) - (SubA, (), noop) of C => (,) - (SubA, (), noop) of D => (,) - }""").strip(), - subgraph) + digraph { + // root subject types: SubA + // root entries + "Select(A) for SubA" [color=blue] + "Select(A) for SubA" -> {"(A, (SelectDependencies(B, SubA, field_types=(C, D,)),), noop) of SubA"} + // internal entries + "(A, (SelectDependencies(B, SubA, field_types=(C, D,)),), noop) of C" -> {"(SubA, (,), noop) of C" "(B, (Select(A),), noop) of C" "(B, (Select(A),), noop) of D"} + "(A, (SelectDependencies(B, SubA, field_types=(C, D,)),), noop) of D" -> {"(SubA, (,), noop) of D" "(B, (Select(A),), noop) of C" "(B, (Select(A),), noop) of D"} + "(A, (SelectDependencies(B, SubA, field_types=(C, D,)),), noop) of SubA" -> {"SubjectIsProduct(SubA)" "(B, (Select(A),), noop) of C" "(B, (Select(A),), noop) of D"} + "(B, (Select(A),), noop) of C" -> {"(A, (SelectDependencies(B, SubA, field_types=(C, D,)),), noop) of C"} + "(B, (Select(A),), noop) of D" -> {"(A, (SelectDependencies(B, SubA, field_types=(C, D,)),), noop) of D"} + "(SubA, (,), noop) of C" -> {} + "(SubA, (,), noop) of D" -> {} + }""").strip(), + subgraph) def test_select_dependencies_non_matching_subselector_because_of_intrinsic(self): rules = [ @@ -642,16 +665,19 @@ def test_select_dependencies_non_matching_subselector_because_of_intrinsic(self) intrinsics = [ (C, B, noop), ] - graphmaker = GraphMaker(RuleIndex.create(rules, intrinsics), - root_subject_types=_suba_root_subject_types) - subgraph = graphmaker.generate_subgraph(SubA(), requested_product=A) + subgraph = self.create_subgraph_with_intrinsics(intrinsics, A, rules, SubA(), + _suba_root_subject_types) - self.assert_equal_with_printing('{empty graph}', subgraph) self.assert_equal_with_printing(dedent(""" - Rules with errors: 1 - (Exactly(A), (SelectDependencies(B, SubA, field_types=(D,)),), noop): - no matches for Select(B) when resolving SelectDependencies(B, SubA, field_types=(D,)) with subject types: D""").strip(), - subgraph.error_message()) + digraph { + // empty graph + }""").strip(), + subgraph) + #self.assert_equal_with_printing(dedent(""" + # Rules with errors: 1 + # (Exactly(A), (SelectDependencies(B, SubA, field_types=(D,)),), noop): + # no matches for Select(B) when resolving SelectDependencies(B, SubA, field_types=(D,)) with subject types: D""").strip(), + # subgraph.error_message()) def test_select_dependencies_with_matching_intrinsic(self): rules = [ @@ -661,20 +687,21 @@ def test_select_dependencies_with_matching_intrinsic(self): (B, C, noop), ] - graphmaker = GraphMaker(RuleIndex.create(rules, intrinsics), - root_subject_types=_suba_root_subject_types) - subgraph = graphmaker.generate_subgraph(SubA(), requested_product=A) + subgraph = self.create_subgraph_with_intrinsics(intrinsics, A, rules, SubA(), + _suba_root_subject_types) + #TODO perhaps intrinsics should be marked in the dot format somehow self.assert_equal_with_printing(dedent(""" - { - root_subject_types: (SubA,) - root_rules: - Select(A) for SubA => ((Exactly(A), (SelectDependencies(B, SubA, field_types=(C,)),), noop) of SubA,) - all_rules: - (Exactly(A), (SelectDependencies(B, SubA, field_types=(C,)),), noop) of SubA => (SubjectIsProduct(SubA), IntrinsicRule((C, B), noop) of C,) - IntrinsicRule((C, B), noop) of C => (,) - }""").strip(), - subgraph) + digraph { + // root subject types: SubA + // root entries + "Select(A) for SubA" [color=blue] + "Select(A) for SubA" -> {"(A, (SelectDependencies(B, SubA, field_types=(C,)),), noop) of SubA"} + // internal entries + "(A, (SelectDependencies(B, SubA, field_types=(C,)),), noop) of SubA" -> {"SubjectIsProduct(SubA)" "(B, (Select(C),), noop) of C"} + "(B, (Select(C),), noop) of C" -> {"SubjectIsProduct(C)"} + }""").strip(), + subgraph) def test_depends_on_multiple_one_noop(self): rules = [ @@ -683,19 +710,19 @@ def test_depends_on_multiple_one_noop(self): (A, (Select(SubA),), noop) ] - graphmaker = GraphMaker(RuleIndex.create(rules, tuple()), - root_subject_types=_suba_root_subject_types) - subgraph = graphmaker.generate_subgraph(SubA(), requested_product=B) + subgraph = self.create_subgraph(B, rules, SubA()) self.assert_equal_with_printing(dedent(""" - { - root_subject_types: (SubA,) - root_rules: - Select(B) for SubA => ((B, (Select(A),), noop) of SubA,) - all_rules: - (A, (Select(SubA),), noop) of SubA => (SubjectIsProduct(SubA),) - (B, (Select(A),), noop) of SubA => ((A, (Select(SubA),), noop) of SubA,) - }""").strip(), subgraph) + digraph { + // root subject types: SubA + // root entries + "Select(B) for SubA" [color=blue] + "Select(B) for SubA" -> {"(B, (Select(A),), noop) of SubA"} + // internal entries + "(A, (Select(SubA),), noop) of SubA" -> {"SubjectIsProduct(SubA)"} + "(B, (Select(A),), noop) of SubA" -> {"(A, (Select(SubA),), noop) of SubA"} + }""").strip(), + subgraph) def test_multiple_depend_on_same_rule(self): rules = [ @@ -704,22 +731,24 @@ def test_multiple_depend_on_same_rule(self): (A, (Select(SubA),), noop) ] - graphmaker = GraphMaker(RuleIndex.create(rules, tuple()), - root_subject_types=_suba_root_subject_types) - subgraph = graphmaker.full_graph() + subgraph = self.create_full_graph(_suba_root_subject_types, RuleIndex.create(rules, tuple())) self.assert_equal_with_printing(dedent(""" - { - root_subject_types: (SubA,) - root_rules: - Select(A) for SubA => ((A, (Select(SubA),), noop) of SubA,) - Select(B) for SubA => ((B, (Select(A),), noop) of SubA,) - Select(C) for SubA => ((C, (Select(A),), noop) of SubA,) - all_rules: - (A, (Select(SubA),), noop) of SubA => (SubjectIsProduct(SubA),) - (B, (Select(A),), noop) of SubA => ((A, (Select(SubA),), noop) of SubA,) - (C, (Select(A),), noop) of SubA => ((A, (Select(SubA),), noop) of SubA,) - }""").strip(), subgraph) + digraph { + // root subject types: SubA + // root entries + "Select(A) for SubA" [color=blue] + "Select(A) for SubA" -> {"(A, (Select(SubA),), noop) of SubA"} + "Select(B) for SubA" [color=blue] + "Select(B) for SubA" -> {"(B, (Select(A),), noop) of SubA"} + "Select(C) for SubA" [color=blue] + "Select(C) for SubA" -> {"(C, (Select(A),), noop) of SubA"} + // internal entries + "(A, (Select(SubA),), noop) of SubA" -> {"SubjectIsProduct(SubA)"} + "(B, (Select(A),), noop) of SubA" -> {"(A, (Select(SubA),), noop) of SubA"} + "(C, (Select(A),), noop) of SubA" -> {"(A, (Select(SubA),), noop) of SubA"} + }""").strip(), + subgraph) def test_select_literal(self): literally_a = A() @@ -727,18 +756,18 @@ def test_select_literal(self): (B, (SelectLiteral(literally_a, A),), noop) ] - graphmaker = GraphMaker(RuleIndex.create(rules, tuple()), - root_subject_types=_suba_root_subject_types) - subgraph = graphmaker.generate_subgraph(SubA(), requested_product=B) + subgraph = self.create_subgraph(B, rules, SubA()) self.assert_equal_with_printing(dedent(""" - { - root_subject_types: (SubA,) - root_rules: - Select(B) for SubA => ((B, (SelectLiteral(A(), A),), noop) of SubA,) - all_rules: - (B, (SelectLiteral(A(), A),), noop) of SubA => (Literal(A(), A),) - }""").strip(), subgraph) + digraph { + // root subject types: SubA + // root entries + "Select(B) for SubA" [color=blue] + "Select(B) for SubA" -> {"(B, (SelectLiteral(A(), A),), noop) of SubA"} + // internal entries + "(B, (SelectLiteral(A(), A),), noop) of SubA" -> {"Literal(A(), A)"} + }""").strip(), + subgraph) def test_select_projection_simple(self): rules = [ @@ -746,19 +775,18 @@ def test_select_projection_simple(self): (B, (Select(D),), noop), ] - graphmaker = GraphMaker(RuleIndex.create(rules, tuple()), - root_subject_types=_suba_root_subject_types) - subgraph = graphmaker.generate_subgraph(SubA(), requested_product=A) + subgraph = self.create_subgraph(A, rules, SubA()) self.assert_equal_with_printing(dedent(""" - { - root_subject_types: (SubA,) - root_rules: - Select(A) for SubA => ((Exactly(A), (SelectProjection(B, D, (u'some',), SubA),), noop) of SubA,) - all_rules: - (B, (Select(D),), noop) of D => (SubjectIsProduct(D),) - (Exactly(A), (SelectProjection(B, D, (u'some',), SubA),), noop) of SubA => (SubjectIsProduct(SubA), (B, (Select(D),), noop) of D,) - }""").strip(), + digraph { + // root subject types: SubA + // root entries + "Select(A) for SubA" [color=blue] + "Select(A) for SubA" -> {"(A, (SelectProjection(B, D, ('some',), SubA),), noop) of SubA"} + // internal entries + "(A, (SelectProjection(B, D, ('some',), SubA),), noop) of SubA" -> {"SubjectIsProduct(SubA)" "(B, (Select(D),), noop) of D"} + "(B, (Select(D),), noop) of D" -> {"SubjectIsProduct(D)"} + }""").strip(), subgraph) def test_successful_when_one_field_type_is_unfulfillable(self): @@ -768,19 +796,50 @@ def test_successful_when_one_field_type_is_unfulfillable(self): (D, (Select(Exactly(B)), SelectDependencies(B, SubA, field_types=(SubA, C))), noop) ] - graphmaker = GraphMaker(RuleIndex.create(rules, tuple()), - root_subject_types=_suba_root_subject_types) - subgraph = graphmaker.generate_subgraph(SubA(), requested_product=D) + subgraph = self.create_subgraph_with_intrinsics(tuple(), D, rules, SubA(), + _suba_root_subject_types) self.assert_equal_with_printing(dedent(""" - { - root_subject_types: (SubA,) - root_rules: - Select(D) for SubA => ((D, (Select(Exactly(B)), SelectDependencies(B, SubA, field_types=(SubA, C,))), noop) of SubA,) - all_rules: - (B, (Select(SubA),), noop) of SubA => (SubjectIsProduct(SubA),) - (D, (Select(Exactly(B)), SelectDependencies(B, SubA, field_types=(SubA, C,))), noop) of SubA => ((B, (Select(SubA),), noop) of SubA, SubjectIsProduct(SubA), (B, (Select(SubA),), noop) of SubA,) - }""").strip(), + digraph { + // root subject types: SubA + // root entries + "Select(D) for SubA" [color=blue] + "Select(D) for SubA" -> {"(D, (Select(B), SelectDependencies(B, SubA, field_types=(SubA, C,))), noop) of SubA"} + // internal entries + "(B, (Select(SubA),), noop) of SubA" -> {"SubjectIsProduct(SubA)"} + "(D, (Select(B), SelectDependencies(B, SubA, field_types=(SubA, C,))), noop) of SubA" -> {"(B, (Select(SubA),), noop) of SubA" "SubjectIsProduct(SubA)"} + }""").strip(), subgraph) + def create_scheduler(self, root_subject_types, rule_index): + native = Native.Factory.global_instance().create() + scheduler = WrappedNativeScheduler( + native=native, + build_root='/tmp', + ignore_patterns=tuple(), + rule_index=rule_index, + root_subject_types=root_subject_types) + return scheduler + + def create_full_graph(self, root_subject_types, rule_index): + scheduler = self.create_scheduler(root_subject_types, rule_index) + + return "\n".join(scheduler.rule_graph_visualization()) + + def create_real_subgraph(self, root_subject_types, rule_index, root_subject, product_type): + scheduler = self.create_scheduler(root_subject_types, rule_index) + + return "\n".join(scheduler.rule_subgraph_visualization(root_subject, product_type)) + + def create_subgraph(self, requested_product, rules, subject): + subgraph = self.create_subgraph_with_intrinsics(tuple(), requested_product, rules, subject, + _suba_root_subject_types) + return subgraph + + def create_subgraph_with_intrinsics(self, intrinsics, requested_product, rules, subject, + subject_types): + rule_index = RuleIndex.create(rules, intrinsics) + + return self.create_real_subgraph(subject_types, rule_index, type(subject), requested_product) + assert_equal_with_printing = assert_equal_with_printing