diff --git a/docs/_static/css/custom.css b/docs/_static/css/custom.css index 40b12e7c..4df980cd 100644 --- a/docs/_static/css/custom.css +++ b/docs/_static/css/custom.css @@ -1,8 +1,3 @@ -/* Remove execution count for notebook cells. */ -div.prompt { - display: none; -} - img.card-img-top { height: 52px; } diff --git a/docs/changes.rst b/docs/changes.rst index abe1014a..cbe0dbe4 100644 --- a/docs/changes.rst +++ b/docs/changes.rst @@ -6,6 +6,13 @@ chronological order. Releases follow `semantic versioning ` all releases are available on `PyPI `_ and `Anaconda.org `_. + +0.0.15 - 2021-xx-xx +------------------- + +- :gh:`80` replaces some remaining formatting using ``pprint`` with ``rich``. + + 0.0.14 - 2021-03-23 ------------------- diff --git a/docs/conf.py b/docs/conf.py index 596f99f2..cc62c2e4 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -7,18 +7,22 @@ # root, use os.path.abspath to make it absolute, like shown here. import os import sys +from datetime import datetime -import pytask import sphinx sys.path.insert(0, os.path.abspath("../src")) +import pytask # noqa: E402 + + # -- Project information --------------------------------------------------------------- project = "pytask" -copyright = "2020, Tobias Raabe" # noqa: A001 +year = datetime.now().year +copyright = f"2020-{year}, Tobias Raabe" # noqa: A001 author = "Tobias Raabe" # The full version, including alpha/beta/rc tags diff --git a/docs/rtd_environment.yml b/docs/rtd_environment.yml index e3a18157..9f00e37d 100644 --- a/docs/rtd_environment.yml +++ b/docs/rtd_environment.yml @@ -24,6 +24,3 @@ dependencies: - pony >=0.7.13 - pexpect - rich - - - pip: - - -e ../ diff --git a/src/_pytask/console.py b/src/_pytask/console.py index cbd2932c..26ab2fa2 100644 --- a/src/_pytask/console.py +++ b/src/_pytask/console.py @@ -1,9 +1,10 @@ """This module contains the code to format output on the command line.""" import os import sys +from typing import List from rich.console import Console - +from rich.tree import Tree _IS_WSL = "IS_WSL" in os.environ or "WSL_DISTRO_NAME" in os.environ _IS_WINDOWS_TERMINAL = "WT_SESSION" in os.environ @@ -26,3 +27,16 @@ console = Console(color_system=_COLOR_SYSTEM) + + +def format_strings_as_flat_tree(strings: List[str], title: str, icon: str) -> str: + """Format list of strings as flat tree.""" + tree = Tree(title) + for name in strings: + tree.add(icon + name) + + text = "".join( + [x.text for x in tree.__rich_console__(console, console.options)][:-1] + ) + + return text diff --git a/src/_pytask/dag.py b/src/_pytask/dag.py index 137bcd39..d2fca740 100644 --- a/src/_pytask/dag.py +++ b/src/_pytask/dag.py @@ -1,6 +1,6 @@ """Implement some capabilities to deal with the DAG.""" import itertools -import pprint +from pathlib import Path from typing import Dict from typing import Generator from typing import Iterable @@ -8,8 +8,11 @@ import attr import networkx as nx +from _pytask.console import format_strings_as_flat_tree +from _pytask.console import TASK_ICON from _pytask.mark import get_specific_markers_from_task from _pytask.nodes import MetaTask +from _pytask.nodes import reduce_node_name def descending_tasks(task_name: str, dag: nx.DiGraph) -> Generator[str, None, None]: @@ -52,14 +55,17 @@ class TopologicalSorter: _nodes_out = attr.ib(factory=set) @classmethod - def from_dag(cls, dag: nx.DiGraph) -> "TopologicalSorter": + def from_dag(cls, dag: nx.DiGraph, paths: List[Path] = None) -> "TopologicalSorter": + if paths is None: + paths = [] + if not dag.is_directed(): raise ValueError("Only directed graphs have a topological order.") tasks = [ dag.nodes[node]["task"] for node in dag.nodes if "task" in dag.nodes[node] ] - priorities = _extract_priorities_from_tasks(tasks) + priorities = _extract_priorities_from_tasks(tasks, paths) task_names = {task.name for task in tasks} task_dict = {name: nx.ancestors(dag, name) & task_names for name in task_names} @@ -118,7 +124,9 @@ def static_order(self) -> Generator[str, None, None]: self.done(new_task) -def _extract_priorities_from_tasks(tasks: List[MetaTask]) -> Dict[str, int]: +def _extract_priorities_from_tasks( + tasks: List[MetaTask], paths: List[Path] +) -> Dict[str, int]: """Extract priorities from tasks. Priorities are set via the ``pytask.mark.try_first`` and ``pytask.mark.try_last`` @@ -138,10 +146,19 @@ def _extract_priorities_from_tasks(tasks: List[MetaTask]) -> Dict[str, int]: tasks_w_mixed_priorities = [ name for name, p in priorities.items() if p["try_first"] and p["try_last"] ] + if tasks_w_mixed_priorities: + name_to_task = {task.name: task for task in tasks} + reduced_names = [ + reduce_node_name(name_to_task[name], paths) + for name in tasks_w_mixed_priorities + ] + text = format_strings_as_flat_tree( + reduced_names, "Tasks with mixed priorities", TASK_ICON + ) raise ValueError( "'try_first' and 'try_last' cannot be applied on the same task. See the " - f"following tasks for errors:\n\n{pprint.pformat(tasks_w_mixed_priorities)}" + f"following tasks for errors:\n\n{text}" ) # Recode to numeric values for sorting. diff --git a/src/_pytask/nodes.py b/src/_pytask/nodes.py index 9ab5a49b..7660cd11 100644 --- a/src/_pytask/nodes.py +++ b/src/_pytask/nodes.py @@ -366,3 +366,11 @@ def reduce_node_name(node, paths: List[Path]): raise ValueError(f"Unknown node {node} with type '{type(node)}'.") return name + + +def reduce_names_of_multiple_nodes(names, dag, paths): + """Reduce the names of multiple nodes in the DAG.""" + return [ + reduce_node_name(dag.nodes[n].get("node") or dag.nodes[n].get("task"), paths) + for n in names + ] diff --git a/src/_pytask/parametrize.py b/src/_pytask/parametrize.py index 4c323d68..830679f3 100644 --- a/src/_pytask/parametrize.py +++ b/src/_pytask/parametrize.py @@ -1,7 +1,6 @@ import copy import functools import itertools -import pprint import types from typing import Any from typing import Callable @@ -12,7 +11,8 @@ from typing import Union from _pytask.config import hookimpl -from _pytask.console import console +from _pytask.console import format_strings_as_flat_tree +from _pytask.console import TASK_ICON from _pytask.mark import Mark from _pytask.mark import MARK_GEN as mark # noqa: N811 from _pytask.shared import find_duplicates @@ -124,10 +124,12 @@ def pytask_parametrize_task(session, name, obj): names = [i[0] for i in names_and_functions] duplicates = find_duplicates(names) if duplicates: - formatted = pprint.pformat(duplicates, width=console.width) + text = format_strings_as_flat_tree( + duplicates, "Duplicated task ids", TASK_ICON + ) raise ValueError( "The following ids are duplicated while parametrizing task " - f"{obj.__name__}.\n\n{formatted}\n\nIt might be caused by " + f"'{obj.__name__}'.\n\n{text}\n\nIt might be caused by " "parametrizing the task with the same combination of arguments " "multiple times. Change the arguments or change the ids generated by " "the parametrization." diff --git a/src/_pytask/path.py b/src/_pytask/path.py index 0c49c65f..db7ff3b0 100644 --- a/src/_pytask/path.py +++ b/src/_pytask/path.py @@ -1,6 +1,6 @@ """This module contains code to handle paths.""" +import os from pathlib import Path -from pathlib import PurePath from typing import List from typing import Union @@ -83,19 +83,6 @@ def find_common_ancestor_of_nodes(*names: str) -> Path: def find_common_ancestor(*paths: Union[str, Path]) -> Path: """Find a common ancestor of many paths.""" - paths = [path if isinstance(path, PurePath) else Path(path) for path in paths] - - for path in paths: - if not path.is_absolute(): - raise ValueError( - f"Cannot find common ancestor for relative paths. {path} is relative." - ) - - common_parents = set.intersection(*[set(path.parents) for path in paths]) - - if len(common_parents) == 0: - raise ValueError("Paths have no common ancestor.") - else: - longest_parent = sorted(common_parents, key=lambda x: len(x.parts))[-1] - - return longest_parent + path = os.path.commonpath(paths) + path = Path(path) + return path diff --git a/src/_pytask/resolve_dependencies.py b/src/_pytask/resolve_dependencies.py index ce3538a5..538ae774 100644 --- a/src/_pytask/resolve_dependencies.py +++ b/src/_pytask/resolve_dependencies.py @@ -18,6 +18,7 @@ from _pytask.exceptions import NodeNotFoundError from _pytask.exceptions import ResolvingDependenciesError from _pytask.mark import Mark +from _pytask.nodes import reduce_names_of_multiple_nodes from _pytask.nodes import reduce_node_name from _pytask.path import find_common_ancestor_of_nodes from _pytask.report import ResolvingDependenciesReport @@ -180,7 +181,7 @@ def _check_if_root_nodes_are_available(dag): short_node_name = reduce_node_name( dag.nodes[node]["node"], [common_ancestor] ) - short_successors = _reduce_names_of_multiple_nodes( + short_successors = reduce_names_of_multiple_nodes( dag.successors(node), dag, [common_ancestor] ) dictionary[short_node_name] = short_successors @@ -231,7 +232,7 @@ def _check_if_tasks_have_the_same_products(dag): short_node_name = reduce_node_name( dag.nodes[node]["node"], [common_ancestor] ) - short_predecessors = _reduce_names_of_multiple_nodes( + short_predecessors = reduce_names_of_multiple_nodes( dag.predecessors(node), dag, [common_ancestor] ) dictionary[short_node_name] = short_predecessors @@ -257,11 +258,3 @@ def pytask_resolve_dependencies_log(report): console.print() console.rule(style=ColorCode.FAILED) - - -def _reduce_names_of_multiple_nodes(names, dag, paths): - """Reduce the names of multiple nodes in the DAG.""" - return [ - reduce_node_name(dag.nodes[n].get("node") or dag.nodes[n].get("task"), paths) - for n in names - ] diff --git a/tests/test_dag.py b/tests/test_dag.py index 0d96d555..5f45aad7 100644 --- a/tests/test_dag.py +++ b/tests/test_dag.py @@ -9,12 +9,21 @@ from _pytask.dag import task_and_descending_tasks from _pytask.dag import TopologicalSorter from _pytask.mark import Mark +from _pytask.nodes import MetaTask @attr.s -class _DummyTask: +class _DummyTask(MetaTask): name = attr.ib(type=str, converter=str) markers = attr.ib(factory=list) + path = attr.ib(default=None) + base_name = "" + + def execute(self): + pass + + def state(self): + pass @pytest.fixture() @@ -59,15 +68,32 @@ def test_node_and_neighbors(dag): @pytest.mark.parametrize( "tasks, expectation, expected", [ - ([_DummyTask("1", [Mark("try_last", (), {})])], does_not_raise(), {"1": -1}), - ([_DummyTask("1", [Mark("try_first", (), {})])], does_not_raise(), {"1": 1}), - ([_DummyTask("1", [])], does_not_raise(), {"1": 0}), - ( - [_DummyTask("1", [Mark("try_first", (), {}), Mark("try_last", (), {})])], + pytest.param( + [_DummyTask("1", [Mark("try_last", (), {})])], + does_not_raise(), + {"1": -1}, + id="test try_last", + ), + pytest.param( + [_DummyTask("1", [Mark("try_first", (), {})])], + does_not_raise(), + {"1": 1}, + id="test try_first", + ), + pytest.param( + [_DummyTask("1", [])], does_not_raise(), {"1": 0}, id="test no priority" + ), + pytest.param( + [ + _DummyTask( + "1", [Mark("try_first", (), {}), Mark("try_last", (), {})], "" + ) + ], pytest.raises(ValueError, match="'try_first' and 'try_last' cannot be"), {"1": 1}, + id="test mixed priorities", ), - ( + pytest.param( [ _DummyTask("1", [Mark("try_first", (), {})]), _DummyTask("2", []), @@ -80,7 +106,7 @@ def test_node_and_neighbors(dag): ) def test_extract_priorities_from_tasks(tasks, expectation, expected): with expectation: - result = _extract_priorities_from_tasks(tasks) + result = _extract_priorities_from_tasks(tasks, []) assert result == expected diff --git a/tests/test_execute.py b/tests/test_execute.py index 9361d3bb..bc733ba0 100644 --- a/tests/test_execute.py +++ b/tests/test_execute.py @@ -246,19 +246,13 @@ def test_scheduling_w_priorities(tmp_path): source = """ import pytask - @pytask.mark.try_first - def task_z(): - pass - - - def task_x(): - pass + def task_z(): pass + def task_x(): pass @pytask.mark.try_last - def task_y(): - pass + def task_y(): pass """ tmp_path.joinpath("task_dummy.py").write_text(textwrap.dedent(source)) @@ -268,3 +262,19 @@ def task_y(): assert session.execution_reports[0].task.name.endswith("task_z") assert session.execution_reports[1].task.name.endswith("task_x") assert session.execution_reports[2].task.name.endswith("task_y") + + +@pytest.mark.end_to_end +def test_scheduling_w_mixed_priorities(tmp_path): + source = """ + import pytask + + @pytask.mark.try_last + @pytask.mark.try_first + def task_mixed(): pass + """ + tmp_path.joinpath("task_dummy.py").write_text(textwrap.dedent(source)) + + session = main({"paths": tmp_path}) + + assert session.exit_code == 4 diff --git a/tests/test_path.py b/tests/test_path.py index 4f056e1b..6a9418c9 100644 --- a/tests/test_path.py +++ b/tests/test_path.py @@ -1,3 +1,4 @@ +import sys from contextlib import ExitStack as does_not_raise # noqa: N813 from pathlib import Path from pathlib import PurePosixPath @@ -47,26 +48,23 @@ def test_find_closest_ancestor(monkeypatch, path, potential_ancestors, expected) pytest.param( PurePosixPath("relative_1"), PurePosixPath("/home/relative_2"), - pytest.raises( - ValueError, match="Cannot find common ancestor for relative paths." - ), + pytest.raises(ValueError, match="Can't mix absolute and relative paths"), None, id="test path 1 is relative", ), pytest.param( PureWindowsPath("C:/home/relative_1"), PureWindowsPath("relative_2"), - pytest.raises( - ValueError, match="Cannot find common ancestor for relative paths." - ), + pytest.raises(ValueError, match="Can't mix absolute and relative paths"), None, id="test path 2 is relative", + marks=pytest.mark.skipif(sys.platform != "win32", reason="fails on UNIX."), ), pytest.param( PurePosixPath("/home/user/folder_a"), PurePosixPath("/home/user/folder_b/sub_folder"), does_not_raise(), - PurePosixPath("/home/user"), + Path("/home/user"), id="normal behavior with UNIX paths", ), pytest.param( @@ -75,13 +73,15 @@ def test_find_closest_ancestor(monkeypatch, path, potential_ancestors, expected) does_not_raise(), PureWindowsPath("C:\\home\\user"), id="normal behavior with Windows paths", + marks=pytest.mark.skipif(sys.platform != "win32", reason="fails on UNIX."), ), pytest.param( PureWindowsPath("C:\\home\\user\\folder_a"), PureWindowsPath("D:\\home\\user\\folder_b\\sub_folder"), - pytest.raises(ValueError, match="Paths have no common ancestor."), + pytest.raises(ValueError, match="Paths don't have the same drive"), None, id="no common ancestor", + marks=pytest.mark.skipif(sys.platform != "win32", reason="fails on UNIX."), ), ], )