From 0ea50500f7c3e8c5d68dcbc0fc3b95d4da44cc23 Mon Sep 17 00:00:00 2001 From: Tobias Raabe Date: Fri, 3 Oct 2025 12:15:56 +0200 Subject: [PATCH 01/11] Explain why tasks will be executed. --- src/_pytask/build.py | 10 + src/_pytask/database_utils.py | 38 ++++ src/_pytask/execute.py | 69 ++++++- src/_pytask/explain.py | 163 +++++++++++++++ src/_pytask/outcomes.py | 2 + src/_pytask/persist.py | 4 + src/_pytask/pluginmanager.py | 1 + src/_pytask/skipping.py | 10 + tests/test_explain.py | 363 ++++++++++++++++++++++++++++++++++ 9 files changed, 655 insertions(+), 5 deletions(-) create mode 100644 src/_pytask/explain.py create mode 100644 tests/test_explain.py diff --git a/src/_pytask/build.py b/src/_pytask/build.py index b5ccf0bf..78426cfd 100644 --- a/src/_pytask/build.py +++ b/src/_pytask/build.py @@ -76,6 +76,7 @@ def build( # noqa: C901, PLR0912, PLR0913 dry_run: bool = False, editor_url_scheme: Literal["no_link", "file", "vscode", "pycharm"] # noqa: PYI051 | str = "file", + explain: bool = False, expression: str = "", force: bool = False, ignore: Iterable[str] = (), @@ -125,6 +126,8 @@ def build( # noqa: C901, PLR0912, PLR0913 editor_url_scheme An url scheme that allows to click on task names, node names and filenames and jump right into you preferred editor to the right line. + explain + Explain why tasks need to be executed by showing what changed. expression Same as ``-k`` on the command line. Select tasks via expressions on task ids. force @@ -189,6 +192,7 @@ def build( # noqa: C901, PLR0912, PLR0913 "disable_warnings": disable_warnings, "dry_run": dry_run, "editor_url_scheme": editor_url_scheme, + "explain": explain, "expression": expression, "force": force, "ignore": ignore, @@ -324,6 +328,12 @@ def build( # noqa: C901, PLR0912, PLR0913 default=False, help="Execute a task even if it succeeded successfully before.", ) +@click.option( + "--explain", + is_flag=True, + default=False, + help="Explain why tasks need to be executed by showing what changed.", +) def build_command(**raw_config: Any) -> NoReturn: """Collect tasks, execute them and report the results.""" raw_config["command"] = "build" diff --git a/src/_pytask/database_utils.py b/src/_pytask/database_utils.py index dc262967..99590fff 100644 --- a/src/_pytask/database_utils.py +++ b/src/_pytask/database_utils.py @@ -22,6 +22,7 @@ "BaseTable", "DatabaseSession", "create_database", + "get_node_change_info", "update_states_in_database", ] @@ -83,3 +84,40 @@ def has_node_changed(task: PTask, node: PTask | PNode, state: str | None) -> boo return True return state != db_state.hash_ + + +def get_node_change_info( + task: PTask, node: PTask | PNode, state: str | None +) -> tuple[bool, str, dict[str, str]]: + """Get detailed information about why a node changed. + + Returns + ------- + tuple[bool, str, dict[str, str]] + A tuple of (has_changed, reason, details) where: + - has_changed: Whether the node has changed + - reason: The reason for the change ("missing", "not_in_db", "changed", + "unchanged") + - details: Additional details like old and new hash values + + """ + details: dict[str, str] = {} + + # If node does not exist, we receive None. + if state is None: + return True, "missing", details + + with DatabaseSession() as session: + db_state = session.get(State, (task.signature, node.signature)) + + # If the node is not in the database. + if db_state is None: + return True, "not_in_db", details + + # Check if state changed + if state != db_state.hash_: + details["old_hash"] = db_state.hash_ + details["new_hash"] = state + return True, "changed", details + + return False, "unchanged", details diff --git a/src/_pytask/execute.py b/src/_pytask/execute.py index d52b6fa0..857f951a 100644 --- a/src/_pytask/execute.py +++ b/src/_pytask/execute.py @@ -20,11 +20,15 @@ from _pytask.dag_utils import TopologicalSorter from _pytask.dag_utils import descending_tasks from _pytask.dag_utils import node_and_neighbors +from _pytask.database_utils import get_node_change_info from _pytask.database_utils import has_node_changed from _pytask.database_utils import update_states_in_database from _pytask.exceptions import ExecutionError from _pytask.exceptions import NodeLoadError from _pytask.exceptions import NodeNotFoundError +from _pytask.explain import ChangeReason +from _pytask.explain import TaskExplanation +from _pytask.explain import create_change_reason from _pytask.mark import Mark from _pytask.mark_utils import has_mark from _pytask.node_protocols import PNode @@ -99,6 +103,14 @@ def pytask_execute_build(session: Session) -> bool | None: @hookimpl def pytask_execute_task_protocol(session: Session, task: PTask) -> ExecutionReport: """Follow the protocol to execute each task.""" + # Initialize explanation for this task if in explain mode + if session.config.get("explain", False): + task._explanation = TaskExplanation( # type: ignore[attr-defined] + task_name=task.name, + would_execute=False, + reasons=[], + ) + session.hook.pytask_execute_task_log_start(session=session, task=task) try: session.hook.pytask_execute_task_setup(session=session, task=task) @@ -119,7 +131,7 @@ def pytask_execute_task_protocol(session: Session, task: PTask) -> ExecutionRepo @hookimpl(trylast=True) -def pytask_execute_task_setup(session: Session, task: PTask) -> None: # noqa: C901 +def pytask_execute_task_setup(session: Session, task: PTask) -> None: # noqa: C901, PLR0912 """Set up the execution of a task. 1. Check whether all dependencies of a task are available. @@ -130,11 +142,22 @@ def pytask_execute_task_setup(session: Session, task: PTask) -> None: # noqa: C raise WouldBeExecuted dag = session.dag + change_reasons = [] # Task generators are always executed since their states are not updated, but we # skip the checks as well. needs_to_be_executed = session.config["force"] or is_task_generator(task) + if session.config["force"] and session.config["explain"]: + change_reasons.append( + ChangeReason( + node_name="", + node_type="task", + reason="forced", + details={}, + ) + ) + if not needs_to_be_executed: predecessors = set(dag.predecessors(task.signature)) | {task.signature} for node_signature in node_and_neighbors(dag, task.signature): @@ -159,9 +182,39 @@ def pytask_execute_task_setup(session: Session, task: PTask) -> None: # noqa: C ) raise NodeNotFoundError(msg) - has_changed = has_node_changed(task=task, node=node, state=node_state) - if has_changed: - needs_to_be_executed = True + # Check if node changed and collect detailed info if in explain mode + if session.config["explain"]: + has_changed, reason, details = get_node_change_info( + task=task, node=node, state=node_state + ) + if has_changed: + needs_to_be_executed = True + # Determine node type + if node_signature == task.signature: + node_type = "source" + elif node_signature in predecessors: + node_type = "dependency" + else: + node_type = "product" + + change_reasons.append( + create_change_reason( + node=node, + node_type=node_type, + reason=reason, + old_hash=details.get("old_hash"), + new_hash=details.get("new_hash"), + ) + ) + else: + has_changed = has_node_changed(task=task, node=node, state=node_state) + if has_changed: + needs_to_be_executed = True + + # Update explanation on task if in explain mode + if session.config["explain"] and hasattr(task, "_explanation"): + task._explanation.would_execute = needs_to_be_executed # type: ignore[attr-defined] + task._explanation.reasons = change_reasons # type: ignore[attr-defined] if not needs_to_be_executed: collect_provisional_products(session, task) @@ -188,7 +241,7 @@ def _safe_load(node: PNode | PProvisionalNode, task: PTask, *, is_product: bool) @hookimpl(trylast=True) def pytask_execute_task(session: Session, task: PTask) -> bool: """Execute task.""" - if session.config["dry_run"]: + if session.config["dry_run"] or session.config["explain"]: raise WouldBeExecuted parameters = inspect.signature(task.function).parameters @@ -255,6 +308,8 @@ def pytask_execute_task_process_report( """ task = report.task + explain_mode = session.config.get("explain", False) + if report.outcome == TaskOutcome.SUCCESS: update_states_in_database(session, task.signature) elif report.exc_info and isinstance(report.exc_info[1], WouldBeExecuted): @@ -287,6 +342,10 @@ def pytask_execute_task_process_report( if report.exc_info and isinstance(report.exc_info[1], Exit): # pragma: no cover session.should_stop = True + # Update explanation with outcome if in explain mode + if explain_mode and hasattr(task, "_explanation"): + task._explanation.outcome = report.outcome # type: ignore[attr-defined] + return True diff --git a/src/_pytask/explain.py b/src/_pytask/explain.py new file mode 100644 index 00000000..31550bc0 --- /dev/null +++ b/src/_pytask/explain.py @@ -0,0 +1,163 @@ +"""Contains logic for explaining why tasks need to be re-executed.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING +from typing import Any + +from attrs import define +from attrs import field +from rich.text import Text + +from _pytask.console import console +from _pytask.outcomes import TaskOutcome +from _pytask.pluginmanager import hookimpl + +if TYPE_CHECKING: + from _pytask.node_protocols import PNode + from _pytask.node_protocols import PTask + from _pytask.reports import ExecutionReport + from _pytask.session import Session + + +@define +class ChangeReason: + """Represents a reason why a node changed.""" + + node_name: str + node_type: str # "source", "dependency", "product", "task" + reason: str # "changed", "missing", "not_in_db", "first_run" + details: dict[str, Any] = field(factory=dict) + + def format(self, verbose: int = 1) -> str: # noqa: PLR0911 + """Format the change reason as a string.""" + if self.reason == "missing": + return f" • {self.node_name}: Missing" + if self.reason == "not_in_db": + return ( + f" • {self.node_name}: Not in database (first run or database cleared)" + ) + if self.reason == "changed": + if verbose >= 2 and "old_hash" in self.details: # noqa: PLR2004 + return ( + f" • {self.node_name}: Changed\n" + f" Previous hash: {self.details['old_hash'][:8]}...\n" + f" Current hash: {self.details['new_hash'][:8]}..." + ) + return f" • {self.node_name}: Changed" + if self.reason == "first_run": + return " • First execution" + if self.reason == "forced": + return " • Forced execution (--force flag)" + return f" • {self.node_name}: {self.reason}" + + +@define +class TaskExplanation: + """Represents the explanation for why a task needs to be executed.""" + + task_name: str + would_execute: bool + outcome: TaskOutcome | None = None + reasons: list[ChangeReason] = field(factory=list) + + def format(self, verbose: int = 1) -> str: + """Format the task explanation as a string.""" + lines = [] + + if self.outcome == TaskOutcome.SKIP_UNCHANGED: + lines.append(f"{self.task_name}") + lines.append(" ✓ No changes detected") + elif self.outcome == TaskOutcome.PERSISTENCE: + lines.append(f"{self.task_name}") + lines.append(" • Persisted (products exist, changes ignored)") + elif self.outcome == TaskOutcome.SKIP: + lines.append(f"{self.task_name}") + lines.append(" • Skipped by marker") + elif not self.reasons: + lines.append(f"{self.task_name}") + lines.append(" ✓ No changes detected") + else: + lines.append(f"{self.task_name}") + lines.extend(reason.format(verbose) for reason in self.reasons) + + return "\n".join(lines) + + +def create_change_reason( + node: PNode | PTask, + node_type: str, + reason: str, + old_hash: str | None = None, + new_hash: str | None = None, +) -> ChangeReason: + """Create a ChangeReason object.""" + details = {} + if old_hash is not None: + details["old_hash"] = old_hash + if new_hash is not None: + details["new_hash"] = new_hash + + return ChangeReason( + node_name=node.name, + node_type=node_type, + reason=reason, + details=details, + ) + + +@hookimpl(tryfirst=True) +def pytask_execute_log_end(session: Session, reports: list[ExecutionReport]) -> None: + """Log explanations if --explain flag is set.""" + if not session.config.get("explain"): + return + + console.print() + console.rule("Explanation", style="bold blue") + console.print() + + # Collect all explanations + explanations = [ + report.task._explanation + for report in reports + if hasattr(report.task, "_explanation") + ] + + if not explanations: + console.print("No tasks require execution - everything is up to date.") + return + + # Group by outcome + would_execute = [e for e in explanations if e.would_execute] + skipped = [ + e + for e in explanations + if not e.would_execute and e.outcome != TaskOutcome.SKIP_UNCHANGED + ] + unchanged = [e for e in explanations if e.outcome == TaskOutcome.SKIP_UNCHANGED] + + verbose = session.config.get("verbose", 1) + + if would_execute: + console.print( + Text("Tasks that would be executed:", style="bold yellow"), + style="yellow", + ) + console.print() + for exp in would_execute: + console.print(exp.format(verbose)) + console.print() + + if skipped: + console.print(Text("Skipped tasks:", style="bold blue"), style="blue") + console.print() + for exp in skipped: + console.print(exp.format(verbose)) + console.print() + + if unchanged and verbose >= 2: # noqa: PLR2004 + console.print(Text("Tasks with no changes:", style="bold green"), style="green") + console.print() + for exp in unchanged: + console.print(exp.format(verbose)) + console.print() diff --git a/src/_pytask/outcomes.py b/src/_pytask/outcomes.py index acb46cbd..3dd4a2e1 100644 --- a/src/_pytask/outcomes.py +++ b/src/_pytask/outcomes.py @@ -95,6 +95,8 @@ class TaskOutcome(Enum): source files and products have not changed. SUCCESS Outcome for task which was executed successfully. + WOULD_BE_EXECUTED + Outcome for tasks which would be executed. """ diff --git a/src/_pytask/persist.py b/src/_pytask/persist.py index 59e06fab..8f2834ff 100644 --- a/src/_pytask/persist.py +++ b/src/_pytask/persist.py @@ -40,6 +40,8 @@ def pytask_execute_task_setup(session: Session, task: PTask) -> None: dependencies and before the same hook implementation for skipping tasks. """ + explain_mode = session.config.get("explain", False) + if has_mark(task, "persist"): all_states = [ ( @@ -63,6 +65,8 @@ def pytask_execute_task_setup(session: Session, task: PTask) -> None: ) if any_node_changed: collect_provisional_products(session, task) + if explain_mode and hasattr(task, "_explanation"): + task._explanation.outcome = TaskOutcome.PERSISTENCE # type: ignore[attr-defined] raise Persisted diff --git a/src/_pytask/pluginmanager.py b/src/_pytask/pluginmanager.py index dbe8b049..2a7ef4a6 100644 --- a/src/_pytask/pluginmanager.py +++ b/src/_pytask/pluginmanager.py @@ -49,6 +49,7 @@ def pytask_add_hooks(pm: PluginManager) -> None: "_pytask.dag_command", "_pytask.database", "_pytask.debugging", + "_pytask.explain", "_pytask.provisional", "_pytask.execute", "_pytask.live", diff --git a/src/_pytask/skipping.py b/src/_pytask/skipping.py index a7678154..86c77a99 100644 --- a/src/_pytask/skipping.py +++ b/src/_pytask/skipping.py @@ -49,15 +49,21 @@ def pytask_parse_config(config: dict[str, Any]) -> None: @hookimpl def pytask_execute_task_setup(session: Session, task: PTask) -> None: """Take a short-cut for skipped tasks during setup with an exception.""" + explain_mode = session.config.get("explain", False) + is_unchanged = has_mark(task, "skip_unchanged") and not has_mark( task, "would_be_executed" ) if is_unchanged and not session.config["force"]: collect_provisional_products(session, task) + if explain_mode and hasattr(task, "_explanation"): + task._explanation.outcome = TaskOutcome.SKIP_UNCHANGED # type: ignore[attr-defined] raise SkippedUnchanged is_skipped = has_mark(task, "skip") if is_skipped: + if explain_mode and hasattr(task, "_explanation"): + task._explanation.outcome = TaskOutcome.SKIP # type: ignore[attr-defined] raise Skipped skipif_marks = get_marks(task, "skipif") @@ -66,6 +72,8 @@ def pytask_execute_task_setup(session: Session, task: PTask) -> None: message = "\n".join(arg[1] for arg in marker_args if arg[0]) should_skip = any(arg[0] for arg in marker_args) if should_skip: + if explain_mode and hasattr(task, "_explanation"): + task._explanation.outcome = TaskOutcome.SKIP # type: ignore[attr-defined] raise Skipped(message) ancestor_failed_marks = get_marks(task, "skip_ancestor_failed") @@ -74,6 +82,8 @@ def pytask_execute_task_setup(session: Session, task: PTask) -> None: skip_ancestor_failed(*mark.args, **mark.kwargs) for mark in ancestor_failed_marks ) + if explain_mode and hasattr(task, "_explanation"): + task._explanation.outcome = TaskOutcome.SKIP_PREVIOUS_FAILED # type: ignore[attr-defined] raise SkippedAncestorFailed(message) diff --git a/tests/test_explain.py b/tests/test_explain.py new file mode 100644 index 00000000..4c9ae29c --- /dev/null +++ b/tests/test_explain.py @@ -0,0 +1,363 @@ +from __future__ import annotations + +import textwrap + +from pytask import ExitCode +from pytask import cli + + +def test_explain_source_file_changed(runner, tmp_path): + """Test --explain shows source file changed.""" + source = """ + from pathlib import Path + + def task_example(produces=Path("out.txt")): + produces.touch() + """ + tmp_path.joinpath("task_example.py").write_text(textwrap.dedent(source)) + + result = runner.invoke(cli, [tmp_path.as_posix()]) + assert result.exit_code == ExitCode.OK + assert "1 Succeeded" in result.output + + # Modify source file + tmp_path.joinpath("task_example.py").write_text(textwrap.dedent(source + "\n")) + + result = runner.invoke(cli, ["--explain", tmp_path.as_posix()]) + assert result.exit_code == ExitCode.OK + assert "Explanation" in result.output + assert "Changed" in result.output or "changed" in result.output + assert "task_example" in result.output + + +def test_explain_dependency_file_changed(runner, tmp_path): + """Test --explain detects changed dependency file.""" + source_first = """ + from pathlib import Path + + def task_first(produces=Path("out.txt")): + produces.write_text("hello") + """ + tmp_path.joinpath("task_first.py").write_text(textwrap.dedent(source_first)) + + source_second = """ + from pathlib import Path + + def task_second(path=Path("out.txt"), produces=Path("out2.txt")): + content = path.read_text() + produces.write_text(content + " world") + """ + tmp_path.joinpath("task_second.py").write_text(textwrap.dedent(source_second)) + + result = runner.invoke(cli, [tmp_path.as_posix()]) + assert result.exit_code == ExitCode.OK + assert "2 Succeeded" in result.output + + # Manually modify the intermediate file + tmp_path.joinpath("out.txt").write_text("modified") + + result = runner.invoke(cli, ["--explain", tmp_path.as_posix()]) + assert result.exit_code == ExitCode.OK + assert "out.txt" in result.output + assert "changed" in result.output.lower() + + +def test_explain_dependency_missing(runner, tmp_path): + """Test --explain detects missing dependency.""" + source_first = """ + from pathlib import Path + + def task_first(produces=Path("out.txt")): + produces.write_text("hello") + """ + tmp_path.joinpath("task_first.py").write_text(textwrap.dedent(source_first)) + + source_second = """ + from pathlib import Path + + def task_second(path=Path("out.txt"), produces=Path("out2.txt")): + produces.write_text(path.read_text()) + """ + tmp_path.joinpath("task_second.py").write_text(textwrap.dedent(source_second)) + + result = runner.invoke(cli, [tmp_path.as_posix()]) + assert result.exit_code == ExitCode.OK + + # Delete dependency + tmp_path.joinpath("out.txt").unlink() + + result = runner.invoke(cli, ["--explain", tmp_path.as_posix()]) + assert result.exit_code == ExitCode.OK + assert "out.txt" in result.output + assert "missing" in result.output.lower() or "not found" in result.output.lower() + + +def test_explain_product_missing(runner, tmp_path): + """Test --explain detects missing product.""" + source = """ + from pathlib import Path + + def task_example(produces=Path("out.txt")): + produces.write_text("hello") + """ + tmp_path.joinpath("task_example.py").write_text(textwrap.dedent(source)) + + result = runner.invoke(cli, [tmp_path.as_posix()]) + assert result.exit_code == ExitCode.OK + assert tmp_path.joinpath("out.txt").exists() + + # Delete product + tmp_path.joinpath("out.txt").unlink() + + result = runner.invoke(cli, ["--explain", tmp_path.as_posix()]) + assert result.exit_code == ExitCode.OK + assert "out.txt" in result.output + assert "missing" in result.output.lower() or "not found" in result.output.lower() + + +def test_explain_multiple_changes(runner, tmp_path): + """Test --explain shows multiple reasons when multiple things changed.""" + source = """ + from pathlib import Path + + def task_example(path=Path("input.txt"), produces=Path("out.txt")): + produces.write_text(path.read_text()) + """ + tmp_path.joinpath("input.txt").write_text("original") + tmp_path.joinpath("task_example.py").write_text(textwrap.dedent(source)) + + result = runner.invoke(cli, [tmp_path.as_posix()]) + assert result.exit_code == ExitCode.OK + + # Change both source and dependency + tmp_path.joinpath("task_example.py").write_text(textwrap.dedent(source + "\n")) + tmp_path.joinpath("input.txt").write_text("modified") + + result = runner.invoke(cli, ["--explain", tmp_path.as_posix()]) + assert result.exit_code == ExitCode.OK + # Should mention both changes + assert "task_example.py" in result.output or "source" in result.output.lower() + assert "input.txt" in result.output + + +def test_explain_with_persist_marker(runner, tmp_path): + """Test --explain respects @pytask.mark.persist.""" + source = """ + import pytask + from pathlib import Path + + @pytask.mark.persist + def task_example(path=Path("input.txt"), produces=Path("out.txt")): + produces.write_text(path.read_text()) + """ + tmp_path.joinpath("input.txt").write_text("original") + tmp_path.joinpath("task_example.py").write_text(textwrap.dedent(source)) + + result = runner.invoke(cli, [tmp_path.as_posix()]) + assert result.exit_code == ExitCode.OK + + # Change dependency + tmp_path.joinpath("input.txt").write_text("modified") + + result = runner.invoke(cli, ["--explain", tmp_path.as_posix()]) + assert result.exit_code == ExitCode.OK + # Should show as persisted, not as would be executed + assert "persist" in result.output.lower() or "unchanged" in result.output.lower() + + +def test_explain_cascade_execution(runner, tmp_path): + """Test --explain shows cascading task executions.""" + source_a = """ + from pathlib import Path + + def task_a(produces=Path("a.txt")): + produces.write_text("a") + """ + tmp_path.joinpath("task_a.py").write_text(textwrap.dedent(source_a)) + + source_b = """ + from pathlib import Path + + def task_b(path=Path("a.txt"), produces=Path("b.txt")): + produces.write_text(path.read_text() + "b") + """ + tmp_path.joinpath("task_b.py").write_text(textwrap.dedent(source_b)) + + source_c = """ + from pathlib import Path + + def task_c(path=Path("b.txt"), produces=Path("c.txt")): + produces.write_text(path.read_text() + "c") + """ + tmp_path.joinpath("task_c.py").write_text(textwrap.dedent(source_c)) + + result = runner.invoke(cli, [tmp_path.as_posix()]) + assert result.exit_code == ExitCode.OK + assert "3 Succeeded" in result.output + + # Change first task + tmp_path.joinpath("task_a.py").write_text(textwrap.dedent(source_a + "\n")) + + result = runner.invoke(cli, ["--explain", tmp_path.as_posix()]) + assert result.exit_code == ExitCode.OK + # All three tasks should be shown + assert "task_a" in result.output + assert "task_b" in result.output + assert "task_c" in result.output + + +def test_explain_first_run_no_database(runner, tmp_path): + """Test --explain on first run (no database).""" + source = """ + from pathlib import Path + + def task_example(produces=Path("out.txt")): + produces.touch() + """ + tmp_path.joinpath("task_example.py").write_text(textwrap.dedent(source)) + + result = runner.invoke(cli, ["--explain", tmp_path.as_posix()]) + assert result.exit_code == ExitCode.OK + assert "task_example" in result.output + # Should indicate first run or not in database + assert ( + "first" in result.output.lower() + or "not in database" in result.output.lower() + or "never executed" in result.output.lower() + ) + + +def test_explain_with_force_flag(runner, tmp_path): + """Test --explain with --force flag.""" + source = """ + from pathlib import Path + + def task_example(produces=Path("out.txt")): + produces.touch() + """ + tmp_path.joinpath("task_example.py").write_text(textwrap.dedent(source)) + + result = runner.invoke(cli, [tmp_path.as_posix()]) + assert result.exit_code == ExitCode.OK + + result = runner.invoke(cli, ["--force", "--explain", tmp_path.as_posix()]) + assert result.exit_code == ExitCode.OK + assert "force" in result.output.lower() or "task_example" in result.output + + +def test_explain_no_changes(runner, tmp_path): + """Test --explain when nothing changed.""" + source = """ + from pathlib import Path + + def task_example(produces=Path("out.txt")): + produces.touch() + """ + tmp_path.joinpath("task_example.py").write_text(textwrap.dedent(source)) + + result = runner.invoke(cli, [tmp_path.as_posix()]) + assert result.exit_code == ExitCode.OK + + result = runner.invoke(cli, ["--explain", tmp_path.as_posix()]) + assert result.exit_code == ExitCode.OK + # Should indicate nothing needs to be executed + assert ( + "no changes" in result.output.lower() + or "unchanged" in result.output.lower() + or "0 " in result.output + ) + + +def test_explain_with_dry_run(runner, tmp_path): + """Test --explain works with --dry-run.""" + source = """ + from pathlib import Path + + def task_example(produces=Path("out.txt")): + produces.touch() + """ + tmp_path.joinpath("task_example.py").write_text(textwrap.dedent(source)) + + result = runner.invoke(cli, [tmp_path.as_posix()]) + assert result.exit_code == ExitCode.OK + + # Modify source + tmp_path.joinpath("task_example.py").write_text(textwrap.dedent(source + "\n")) + + result = runner.invoke(cli, ["--dry-run", "--explain", tmp_path.as_posix()]) + assert result.exit_code == ExitCode.OK + assert "task_example" in result.output + # Should show explanation and not execute + assert ( + not tmp_path.joinpath("out.txt").read_text() + or tmp_path.joinpath("out.txt").stat().st_size == 0 + ) + + +def test_explain_skipped_tasks(runner, tmp_path): + """Test --explain handles skipped tasks.""" + source = """ + import pytask + from pathlib import Path + + @pytask.mark.skip + def task_skip(produces=Path("skip.txt")): + produces.touch() + + def task_example(produces=Path("out.txt")): + produces.touch() + """ + tmp_path.joinpath("task_example.py").write_text(textwrap.dedent(source)) + + result = runner.invoke(cli, ["--explain", tmp_path.as_posix()]) + assert result.exit_code == ExitCode.OK + assert "skip" in result.output.lower() + + +def test_explain_task_source_and_dependency_changed(runner, tmp_path): + """Test --explain when both task source and dependency change.""" + source = """ + from pathlib import Path + + def task_example(path=Path("data.txt"), produces=Path("out.txt")): + content = path.read_text() + produces.write_text(content) + """ + tmp_path.joinpath("data.txt").write_text("original") + tmp_path.joinpath("task_example.py").write_text(textwrap.dedent(source)) + + result = runner.invoke(cli, [tmp_path.as_posix()]) + assert result.exit_code == ExitCode.OK + + # Change both source and data + tmp_path.joinpath("task_example.py").write_text(textwrap.dedent(source + "\n")) + tmp_path.joinpath("data.txt").write_text("modified") + + result = runner.invoke(cli, ["--explain", tmp_path.as_posix()]) + assert result.exit_code == ExitCode.OK + assert "Explanation" in result.output + assert "task_example" in result.output + # Should show changes + assert "changed" in result.output.lower() or "Changed" in result.output + + +def test_explain_verbose_output(runner, tmp_path): + """Test --explain with -v shows more details.""" + source = """ + from pathlib import Path + + def task_example(produces=Path("out.txt")): + produces.touch() + """ + tmp_path.joinpath("task_example.py").write_text(textwrap.dedent(source)) + + result = runner.invoke(cli, [tmp_path.as_posix()]) + assert result.exit_code == ExitCode.OK + + # Modify source + tmp_path.joinpath("task_example.py").write_text(textwrap.dedent(source + "\n")) + + result = runner.invoke(cli, ["--explain", "-v", "1", tmp_path.as_posix()]) + assert result.exit_code == ExitCode.OK + # Verbose mode should show more details + assert "task_example" in result.output From 0b29ed03db383122f29f21bec1a45587008d2ce5 Mon Sep 17 00:00:00 2001 From: Tobias Raabe Date: Fri, 3 Oct 2025 12:26:12 +0200 Subject: [PATCH 02/11] Better colors but issues left. --- .pre-commit-config.yaml | 2 +- src/_pytask/explain.py | 19 +++++++++++++------ 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index b3b32647..e23e21a1 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -25,7 +25,7 @@ repos: - id: python-no-log-warn - id: text-unicode-replacement-char - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.13.2 + rev: v0.13.3 hooks: - id: ruff-format - id: ruff-check diff --git a/src/_pytask/explain.py b/src/_pytask/explain.py index 31550bc0..e965cf72 100644 --- a/src/_pytask/explain.py +++ b/src/_pytask/explain.py @@ -109,11 +109,11 @@ def create_change_reason( @hookimpl(tryfirst=True) def pytask_execute_log_end(session: Session, reports: list[ExecutionReport]) -> None: """Log explanations if --explain flag is set.""" - if not session.config.get("explain"): + if not session.config["explain"]: return console.print() - console.rule("Explanation", style="bold blue") + console.rule(Text("Explanation", style="bold blue"), style="bold blue") console.print() # Collect all explanations @@ -139,9 +139,12 @@ def pytask_execute_log_end(session: Session, reports: list[ExecutionReport]) -> verbose = session.config.get("verbose", 1) if would_execute: + # WOULD_BE_EXECUTED has style "success" in TaskOutcome console.print( - Text("Tasks that would be executed:", style="bold yellow"), - style="yellow", + Text( + "Tasks that would be executed:", + style=TaskOutcome.WOULD_BE_EXECUTED.style, + ), ) console.print() for exp in would_execute: @@ -149,14 +152,18 @@ def pytask_execute_log_end(session: Session, reports: list[ExecutionReport]) -> console.print() if skipped: - console.print(Text("Skipped tasks:", style="bold blue"), style="blue") + # SKIP has style "skipped" in TaskOutcome + console.print(Text("Skipped tasks:", style=TaskOutcome.SKIP.style)) console.print() for exp in skipped: console.print(exp.format(verbose)) console.print() if unchanged and verbose >= 2: # noqa: PLR2004 - console.print(Text("Tasks with no changes:", style="bold green"), style="green") + # SKIP_UNCHANGED has style "success" in TaskOutcome + console.print( + Text("Tasks with no changes:", style=TaskOutcome.SKIP_UNCHANGED.style) + ) console.print() for exp in unchanged: console.print(exp.format(verbose)) From 73dab52336a24bd9b1d2435772f2541eb8cac1c0 Mon Sep 17 00:00:00 2001 From: Tobias Raabe Date: Fri, 3 Oct 2025 12:55:51 +0200 Subject: [PATCH 03/11] Fix typing. --- src/_pytask/execute.py | 6 +++--- src/_pytask/persist.py | 2 +- src/_pytask/skipping.py | 8 ++++---- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/_pytask/execute.py b/src/_pytask/execute.py index 857f951a..ee21f2d7 100644 --- a/src/_pytask/execute.py +++ b/src/_pytask/execute.py @@ -213,8 +213,8 @@ def pytask_execute_task_setup(session: Session, task: PTask) -> None: # noqa: C # Update explanation on task if in explain mode if session.config["explain"] and hasattr(task, "_explanation"): - task._explanation.would_execute = needs_to_be_executed # type: ignore[attr-defined] - task._explanation.reasons = change_reasons # type: ignore[attr-defined] + task._explanation.would_execute = needs_to_be_executed + task._explanation.reasons = change_reasons if not needs_to_be_executed: collect_provisional_products(session, task) @@ -344,7 +344,7 @@ def pytask_execute_task_process_report( # Update explanation with outcome if in explain mode if explain_mode and hasattr(task, "_explanation"): - task._explanation.outcome = report.outcome # type: ignore[attr-defined] + task._explanation.outcome = report.outcome return True diff --git a/src/_pytask/persist.py b/src/_pytask/persist.py index 8f2834ff..58002687 100644 --- a/src/_pytask/persist.py +++ b/src/_pytask/persist.py @@ -66,7 +66,7 @@ def pytask_execute_task_setup(session: Session, task: PTask) -> None: if any_node_changed: collect_provisional_products(session, task) if explain_mode and hasattr(task, "_explanation"): - task._explanation.outcome = TaskOutcome.PERSISTENCE # type: ignore[attr-defined] + task._explanation.outcome = TaskOutcome.PERSISTENCE raise Persisted diff --git a/src/_pytask/skipping.py b/src/_pytask/skipping.py index 86c77a99..1a47a7f0 100644 --- a/src/_pytask/skipping.py +++ b/src/_pytask/skipping.py @@ -57,13 +57,13 @@ def pytask_execute_task_setup(session: Session, task: PTask) -> None: if is_unchanged and not session.config["force"]: collect_provisional_products(session, task) if explain_mode and hasattr(task, "_explanation"): - task._explanation.outcome = TaskOutcome.SKIP_UNCHANGED # type: ignore[attr-defined] + task._explanation.outcome = TaskOutcome.SKIP_UNCHANGED raise SkippedUnchanged is_skipped = has_mark(task, "skip") if is_skipped: if explain_mode and hasattr(task, "_explanation"): - task._explanation.outcome = TaskOutcome.SKIP # type: ignore[attr-defined] + task._explanation.outcome = TaskOutcome.SKIP raise Skipped skipif_marks = get_marks(task, "skipif") @@ -73,7 +73,7 @@ def pytask_execute_task_setup(session: Session, task: PTask) -> None: should_skip = any(arg[0] for arg in marker_args) if should_skip: if explain_mode and hasattr(task, "_explanation"): - task._explanation.outcome = TaskOutcome.SKIP # type: ignore[attr-defined] + task._explanation.outcome = TaskOutcome.SKIP raise Skipped(message) ancestor_failed_marks = get_marks(task, "skip_ancestor_failed") @@ -83,7 +83,7 @@ def pytask_execute_task_setup(session: Session, task: PTask) -> None: for mark in ancestor_failed_marks ) if explain_mode and hasattr(task, "_explanation"): - task._explanation.outcome = TaskOutcome.SKIP_PREVIOUS_FAILED # type: ignore[attr-defined] + task._explanation.outcome = TaskOutcome.SKIP_PREVIOUS_FAILED raise SkippedAncestorFailed(message) From e50d251d1141d00eeb255ca86ce6bfc22a9cfdd2 Mon Sep 17 00:00:00 2001 From: Tobias Raabe Date: Fri, 3 Oct 2025 16:54:24 +0200 Subject: [PATCH 04/11] refactor. --- src/_pytask/collect.py | 4 +- src/_pytask/execute.py | 16 ++------ src/_pytask/explain.py | 89 ++++++++++++++++++++++++----------------- src/_pytask/persist.py | 4 -- src/_pytask/skipping.py | 10 ----- tests/test_explain.py | 1 + 6 files changed, 59 insertions(+), 65 deletions(-) diff --git a/src/_pytask/collect.py b/src/_pytask/collect.py index 0df50442..7aad12a1 100644 --- a/src/_pytask/collect.py +++ b/src/_pytask/collect.py @@ -597,7 +597,9 @@ def pytask_collect_log( """Log collection.""" session.collection_end = time.time() - console.print(f"Collected {len(tasks)} task{'' if len(tasks) == 1 else 's'}.") + console.print( + f"Collected {len(tasks)} task{'' if len(tasks) == 1 else 's'}.", highlight=False + ) failed_reports = [r for r in reports if r.outcome == CollectionOutcome.FAIL] if failed_reports: diff --git a/src/_pytask/execute.py b/src/_pytask/execute.py index ee21f2d7..a70fb656 100644 --- a/src/_pytask/execute.py +++ b/src/_pytask/execute.py @@ -105,11 +105,7 @@ def pytask_execute_task_protocol(session: Session, task: PTask) -> ExecutionRepo """Follow the protocol to execute each task.""" # Initialize explanation for this task if in explain mode if session.config.get("explain", False): - task._explanation = TaskExplanation( # type: ignore[attr-defined] - task_name=task.name, - would_execute=False, - reasons=[], - ) + task.attributes["explanation"] = TaskExplanation(reasons=[]) session.hook.pytask_execute_task_log_start(session=session, task=task) try: @@ -212,9 +208,8 @@ def pytask_execute_task_setup(session: Session, task: PTask) -> None: # noqa: C needs_to_be_executed = True # Update explanation on task if in explain mode - if session.config["explain"] and hasattr(task, "_explanation"): - task._explanation.would_execute = needs_to_be_executed - task._explanation.reasons = change_reasons + if session.config["explain"] and "explanation" in task.attributes: + task.attributes["explanation"].reasons = change_reasons if not needs_to_be_executed: collect_provisional_products(session, task) @@ -308,7 +303,6 @@ def pytask_execute_task_process_report( """ task = report.task - explain_mode = session.config.get("explain", False) if report.outcome == TaskOutcome.SUCCESS: update_states_in_database(session, task.signature) @@ -342,10 +336,6 @@ def pytask_execute_task_process_report( if report.exc_info and isinstance(report.exc_info[1], Exit): # pragma: no cover session.should_stop = True - # Update explanation with outcome if in explain mode - if explain_mode and hasattr(task, "_explanation"): - task._explanation.outcome = report.outcome - return True diff --git a/src/_pytask/explain.py b/src/_pytask/explain.py index e965cf72..a10e1d7f 100644 --- a/src/_pytask/explain.py +++ b/src/_pytask/explain.py @@ -56,29 +56,26 @@ def format(self, verbose: int = 1) -> str: # noqa: PLR0911 class TaskExplanation: """Represents the explanation for why a task needs to be executed.""" - task_name: str - would_execute: bool - outcome: TaskOutcome | None = None reasons: list[ChangeReason] = field(factory=list) - def format(self, verbose: int = 1) -> str: + def format(self, task_name: str, outcome: TaskOutcome, verbose: int = 1) -> str: """Format the task explanation as a string.""" lines = [] - if self.outcome == TaskOutcome.SKIP_UNCHANGED: - lines.append(f"{self.task_name}") + if outcome == TaskOutcome.SKIP_UNCHANGED: + lines.append(f"{task_name}") lines.append(" ✓ No changes detected") - elif self.outcome == TaskOutcome.PERSISTENCE: - lines.append(f"{self.task_name}") + elif outcome == TaskOutcome.PERSISTENCE: + lines.append(f"{task_name}") lines.append(" • Persisted (products exist, changes ignored)") - elif self.outcome == TaskOutcome.SKIP: - lines.append(f"{self.task_name}") + elif outcome == TaskOutcome.SKIP: + lines.append(f"{task_name}") lines.append(" • Skipped by marker") elif not self.reasons: - lines.append(f"{self.task_name}") + lines.append(f"{task_name}") lines.append(" ✓ No changes detected") else: - lines.append(f"{self.task_name}") + lines.append(f"{task_name}") lines.extend(reason.format(verbose) for reason in self.reasons) return "\n".join(lines) @@ -116,55 +113,73 @@ def pytask_execute_log_end(session: Session, reports: list[ExecutionReport]) -> console.rule(Text("Explanation", style="bold blue"), style="bold blue") console.print() - # Collect all explanations - explanations = [ - report.task._explanation - for report in reports - if hasattr(report.task, "_explanation") + # Collect all reports with explanations + reports_with_explanations = [ + report for report in reports if "explanation" in report.task.attributes ] - if not explanations: + if not reports_with_explanations: console.print("No tasks require execution - everything is up to date.") return # Group by outcome - would_execute = [e for e in explanations if e.would_execute] + would_execute = [ + r + for r in reports_with_explanations + if r.outcome == TaskOutcome.WOULD_BE_EXECUTED + ] skipped = [ - e - for e in explanations - if not e.would_execute and e.outcome != TaskOutcome.SKIP_UNCHANGED + r + for r in reports_with_explanations + if r.outcome in (TaskOutcome.SKIP, TaskOutcome.SKIP_PREVIOUS_FAILED) + ] + unchanged = [ + r for r in reports_with_explanations if r.outcome == TaskOutcome.SKIP_UNCHANGED ] - unchanged = [e for e in explanations if e.outcome == TaskOutcome.SKIP_UNCHANGED] verbose = session.config.get("verbose", 1) if would_execute: - # WOULD_BE_EXECUTED has style "success" in TaskOutcome - console.print( + console.rule( Text( - "Tasks that would be executed:", + "─── Tasks that would be executed", style=TaskOutcome.WOULD_BE_EXECUTED.style, ), + align="left", + style=TaskOutcome.WOULD_BE_EXECUTED.style, ) console.print() - for exp in would_execute: - console.print(exp.format(verbose)) + for report in would_execute: + explanation = report.task.attributes["explanation"] + console.print(explanation.format(report.task.name, report.outcome, verbose)) console.print() if skipped: - # SKIP has style "skipped" in TaskOutcome - console.print(Text("Skipped tasks:", style=TaskOutcome.SKIP.style)) + console.rule( + Text("─── Skipped tasks", style=TaskOutcome.SKIP.style), + align="left", + style=TaskOutcome.SKIP.style, + ) console.print() - for exp in skipped: - console.print(exp.format(verbose)) + for report in skipped: + explanation = report.task.attributes["explanation"] + console.print(explanation.format(report.task.name, report.outcome, verbose)) console.print() if unchanged and verbose >= 2: # noqa: PLR2004 - # SKIP_UNCHANGED has style "success" in TaskOutcome - console.print( - Text("Tasks with no changes:", style=TaskOutcome.SKIP_UNCHANGED.style) + console.rule( + Text("─── Tasks with no changes", style=TaskOutcome.SKIP_UNCHANGED.style), + align="left", + style=TaskOutcome.SKIP_UNCHANGED.style, ) console.print() - for exp in unchanged: - console.print(exp.format(verbose)) + for report in unchanged: + explanation = report.task.attributes["explanation"] + console.print(explanation.format(report.task.name, report.outcome, verbose)) console.print() + + elif unchanged and verbose == 1: + console.print( + f"{len(unchanged)} task(s) with no changes (use -vv to show details)", + highlight=False, + ) diff --git a/src/_pytask/persist.py b/src/_pytask/persist.py index 58002687..59e06fab 100644 --- a/src/_pytask/persist.py +++ b/src/_pytask/persist.py @@ -40,8 +40,6 @@ def pytask_execute_task_setup(session: Session, task: PTask) -> None: dependencies and before the same hook implementation for skipping tasks. """ - explain_mode = session.config.get("explain", False) - if has_mark(task, "persist"): all_states = [ ( @@ -65,8 +63,6 @@ def pytask_execute_task_setup(session: Session, task: PTask) -> None: ) if any_node_changed: collect_provisional_products(session, task) - if explain_mode and hasattr(task, "_explanation"): - task._explanation.outcome = TaskOutcome.PERSISTENCE raise Persisted diff --git a/src/_pytask/skipping.py b/src/_pytask/skipping.py index 1a47a7f0..a7678154 100644 --- a/src/_pytask/skipping.py +++ b/src/_pytask/skipping.py @@ -49,21 +49,15 @@ def pytask_parse_config(config: dict[str, Any]) -> None: @hookimpl def pytask_execute_task_setup(session: Session, task: PTask) -> None: """Take a short-cut for skipped tasks during setup with an exception.""" - explain_mode = session.config.get("explain", False) - is_unchanged = has_mark(task, "skip_unchanged") and not has_mark( task, "would_be_executed" ) if is_unchanged and not session.config["force"]: collect_provisional_products(session, task) - if explain_mode and hasattr(task, "_explanation"): - task._explanation.outcome = TaskOutcome.SKIP_UNCHANGED raise SkippedUnchanged is_skipped = has_mark(task, "skip") if is_skipped: - if explain_mode and hasattr(task, "_explanation"): - task._explanation.outcome = TaskOutcome.SKIP raise Skipped skipif_marks = get_marks(task, "skipif") @@ -72,8 +66,6 @@ def pytask_execute_task_setup(session: Session, task: PTask) -> None: message = "\n".join(arg[1] for arg in marker_args if arg[0]) should_skip = any(arg[0] for arg in marker_args) if should_skip: - if explain_mode and hasattr(task, "_explanation"): - task._explanation.outcome = TaskOutcome.SKIP raise Skipped(message) ancestor_failed_marks = get_marks(task, "skip_ancestor_failed") @@ -82,8 +74,6 @@ def pytask_execute_task_setup(session: Session, task: PTask) -> None: skip_ancestor_failed(*mark.args, **mark.kwargs) for mark in ancestor_failed_marks ) - if explain_mode and hasattr(task, "_explanation"): - task._explanation.outcome = TaskOutcome.SKIP_PREVIOUS_FAILED raise SkippedAncestorFailed(message) diff --git a/tests/test_explain.py b/tests/test_explain.py index 4c9ae29c..d2aa944a 100644 --- a/tests/test_explain.py +++ b/tests/test_explain.py @@ -266,6 +266,7 @@ def task_example(produces=Path("out.txt")): or "unchanged" in result.output.lower() or "0 " in result.output ) + assert "1 task(s) with no changes (use -vv to show details)" in result.output def test_explain_with_dry_run(runner, tmp_path): From 483f0fc1ab42bf6e366546841238469183ad5d26 Mon Sep 17 00:00:00 2001 From: Tobias Raabe Date: Fri, 3 Oct 2025 18:43:28 +0200 Subject: [PATCH 05/11] Show persisted tasks.2 --- src/_pytask/execute.py | 18 +++++++++++++++++- src/_pytask/explain.py | 27 +++++++++++++++++++++++++-- tests/test_explain.py | 18 ++++++++++++++++-- 3 files changed, 58 insertions(+), 5 deletions(-) diff --git a/src/_pytask/execute.py b/src/_pytask/execute.py index a70fb656..75b11d28 100644 --- a/src/_pytask/execute.py +++ b/src/_pytask/execute.py @@ -135,6 +135,19 @@ def pytask_execute_task_setup(session: Session, task: PTask) -> None: # noqa: C """ if has_mark(task, "would_be_executed"): + # Add cascade reason if in explain mode + if session.config.get("explain", False) and "explanation" in task.attributes: + marks = [m for m in task.markers if m.name == "would_be_executed"] + if marks: + preceding_task_name = marks[0].kwargs.get("task_name", "unknown") + task.attributes["explanation"].reasons.append( + ChangeReason( + node_name=preceding_task_name, + node_type="task", + reason="cascade", + details={}, + ) + ) raise WouldBeExecuted dag = session.dag @@ -315,7 +328,10 @@ def pytask_execute_task_process_report( Mark( "would_be_executed", (), - {"reason": f"Previous task {task.name!r} would be executed."}, + { + "reason": f"Previous task {task.name!r} would be executed.", + "task_name": task.name, + }, ) ) else: diff --git a/src/_pytask/explain.py b/src/_pytask/explain.py index a10e1d7f..b3ffe979 100644 --- a/src/_pytask/explain.py +++ b/src/_pytask/explain.py @@ -49,6 +49,8 @@ def format(self, verbose: int = 1) -> str: # noqa: PLR0911 return " • First execution" if self.reason == "forced": return " • Forced execution (--force flag)" + if self.reason == "cascade": + return f" • Preceding task '{self.node_name}' would be executed" return f" • {self.node_name}: {self.reason}" @@ -104,7 +106,9 @@ def create_change_reason( @hookimpl(tryfirst=True) -def pytask_execute_log_end(session: Session, reports: list[ExecutionReport]) -> None: +def pytask_execute_log_end( # noqa: C901 + session: Session, reports: list[ExecutionReport] +) -> None: """Log explanations if --explain flag is set.""" if not session.config["explain"]: return @@ -136,6 +140,9 @@ def pytask_execute_log_end(session: Session, reports: list[ExecutionReport]) -> unchanged = [ r for r in reports_with_explanations if r.outcome == TaskOutcome.SKIP_UNCHANGED ] + persisted = [ + r for r in reports_with_explanations if r.outcome == TaskOutcome.PERSISTENCE + ] verbose = session.config.get("verbose", 1) @@ -166,6 +173,23 @@ def pytask_execute_log_end(session: Session, reports: list[ExecutionReport]) -> console.print(explanation.format(report.task.name, report.outcome, verbose)) console.print() + if persisted and verbose >= 2: # noqa: PLR2004 + console.rule( + Text("─── Persisted tasks", style=TaskOutcome.PERSISTENCE.style), + align="left", + style=TaskOutcome.PERSISTENCE.style, + ) + console.print() + for report in persisted: + explanation = report.task.attributes["explanation"] + console.print(explanation.format(report.task.name, report.outcome, verbose)) + console.print() + elif persisted and verbose == 1: + console.print( + f"{len(persisted)} persisted task(s) (use -vv to show details)", + highlight=False, + ) + if unchanged and verbose >= 2: # noqa: PLR2004 console.rule( Text("─── Tasks with no changes", style=TaskOutcome.SKIP_UNCHANGED.style), @@ -177,7 +201,6 @@ def pytask_execute_log_end(session: Session, reports: list[ExecutionReport]) -> explanation = report.task.attributes["explanation"] console.print(explanation.format(report.task.name, report.outcome, verbose)) console.print() - elif unchanged and verbose == 1: console.print( f"{len(unchanged)} task(s) with no changes (use -vv to show details)", diff --git a/tests/test_explain.py b/tests/test_explain.py index d2aa944a..a22ccb6f 100644 --- a/tests/test_explain.py +++ b/tests/test_explain.py @@ -159,10 +159,20 @@ def task_example(path=Path("input.txt"), produces=Path("out.txt")): # Change dependency tmp_path.joinpath("input.txt").write_text("modified") + # Test with default verbosity (should show summary) result = runner.invoke(cli, ["--explain", tmp_path.as_posix()]) assert result.exit_code == ExitCode.OK - # Should show as persisted, not as would be executed - assert "persist" in result.output.lower() or "unchanged" in result.output.lower() + assert "persisted task(s)" in result.output.lower() + assert "use -vv to show details" in result.output.lower() + + # Change dependency again for verbose test + tmp_path.joinpath("input.txt").write_text("modified again") + + # Test with verbose 2 (should show details) + result = runner.invoke(cli, ["--explain", "--verbose", "2", tmp_path.as_posix()]) + assert result.exit_code == ExitCode.OK + assert "Persisted tasks" in result.output + assert "Persisted (products exist, changes ignored)" in result.output def test_explain_cascade_execution(runner, tmp_path): @@ -204,6 +214,10 @@ def task_c(path=Path("b.txt"), produces=Path("c.txt")): assert "task_a" in result.output assert "task_b" in result.output assert "task_c" in result.output + # Check cascade explanations + assert "Preceding task" in result.output + assert "task_a" in result.output + assert "Changed" in result.output def test_explain_first_run_no_database(runner, tmp_path): From 20cf9738d90251a22cb9ffdb31b876beb17e0a53 Mon Sep 17 00:00:00 2001 From: Tobias Raabe Date: Fri, 3 Oct 2025 18:46:40 +0200 Subject: [PATCH 06/11] TO changes. --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 40dbabd9..ad84a7b3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ releases are available on [PyPI](https://pypi.org/project/pytask) and ## 0.5.6 - 2025-xx-xx - {pull}`703` fixes {issue}`701` by allowing `--capture tee-sys` again. +- {pull}`704` adds the `--explain` flag to show why tasks would be executed. Closes {issue}`466`. ## 0.5.5 - 2025-07-25 From 475bad57013c5fbe3742ca730ca610e96d8ec99a Mon Sep 17 00:00:00 2001 From: Tobias Raabe Date: Fri, 3 Oct 2025 18:56:45 +0200 Subject: [PATCH 07/11] Fix. --- src/_pytask/execute.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/_pytask/execute.py b/src/_pytask/execute.py index 75b11d28..c2c06429 100644 --- a/src/_pytask/execute.py +++ b/src/_pytask/execute.py @@ -104,7 +104,7 @@ def pytask_execute_build(session: Session) -> bool | None: def pytask_execute_task_protocol(session: Session, task: PTask) -> ExecutionReport: """Follow the protocol to execute each task.""" # Initialize explanation for this task if in explain mode - if session.config.get("explain", False): + if session.config["explain"]: task.attributes["explanation"] = TaskExplanation(reasons=[]) session.hook.pytask_execute_task_log_start(session=session, task=task) @@ -136,7 +136,7 @@ def pytask_execute_task_setup(session: Session, task: PTask) -> None: # noqa: C """ if has_mark(task, "would_be_executed"): # Add cascade reason if in explain mode - if session.config.get("explain", False) and "explanation" in task.attributes: + if session.config["explain"] and "explanation" in task.attributes: marks = [m for m in task.markers if m.name == "would_be_executed"] if marks: preceding_task_name = marks[0].kwargs.get("task_name", "unknown") From 399ac894246d13d3671a4322e766cdb18bbd1c90 Mon Sep 17 00:00:00 2001 From: Tobias Raabe Date: Fri, 3 Oct 2025 19:22:35 +0200 Subject: [PATCH 08/11] Add documentation. --- docs/source/_static/md/explain.md | 46 ++++++++++++++++++++++++ docs/source/tutorials/invoking_pytask.md | 16 +++++++++ src/_pytask/explain.py | 2 +- 3 files changed, 63 insertions(+), 1 deletion(-) create mode 100644 docs/source/_static/md/explain.md diff --git a/docs/source/_static/md/explain.md b/docs/source/_static/md/explain.md new file mode 100644 index 00000000..1d7d1877 --- /dev/null +++ b/docs/source/_static/md/explain.md @@ -0,0 +1,46 @@ +
+ +```console + +$ pytask --explain +────────────────────────── Start pytask session ───────────────────────── +Platform: darwin -- Python 3.12.0, pytask 0.5.6, pluggy 1.6.0 +Root: /Users/pytask-dev/git/my_project +Collected 3 tasks. + +┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━┓ +┃ Task ┃ Outcome ┃ +┡━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━┩ +│ task_data.py::task_create_data │ w │ +│ task_analysis.py::task_analyze │ w │ +│ task_plot.py::task_plot │ s │ +└───────────────────────────────────────────────────┴─────────┘ + +───────────────────────────────────────────────────────────────────────── +────────────────────────────── Explanation ────────────────────────────── + +─── Tasks that would be executed ──────────────────────────────────────── + +task_data.py::task_create_data + • task_data.py::task_create_data: Changed + +task_analysis.py::task_analyze + • Preceding task_data.py::task_create_data would be executed + +─── Skipped tasks ─────────────────────────────────────────────────────── + +task_plot.py::task_plot + • Skipped by marker + +1 persisted task(s) (use -vv to show details) + +───────────────────────────────────────────────────────────────────────── +╭─────────── Summary ──────────────╮ +│ 3 Collected tasks │ +│ 2 Would be executed (66.7%) │ +│ 1 Skipped (33.3%) │ +╰──────────────────────────────────╯ +─────────────────────── Succeeded in 0.02 seconds ─────────────────────── +``` + +
diff --git a/docs/source/tutorials/invoking_pytask.md b/docs/source/tutorials/invoking_pytask.md index f3a5396a..69ba37e5 100644 --- a/docs/source/tutorials/invoking_pytask.md +++ b/docs/source/tutorials/invoking_pytask.md @@ -83,6 +83,22 @@ Do a dry run to see which tasks will be executed without executing them. ```{include} ../_static/md/dry-run.md ``` +### Explaining why tasks are executed + +Use the `--explain` flag to understand why tasks need to be executed. This shows what +changed (source files, dependencies, products) and helps you understand pytask's +execution decisions. + +```{include} ../_static/md/explain.md +``` + +The explanation output respects the `--verbose` flag: + +- Default verbosity: Shows tasks that would be executed and skipped tasks +- `-v` or `--verbose 1`: Same as default, with summary for persisted and unchanged tasks +- `--verbose 2`: Shows detailed information including persisted and unchanged tasks with + change reasons + ## Functional interface pytask also has a functional interface that is explained in this diff --git a/src/_pytask/explain.py b/src/_pytask/explain.py index b3ffe979..fb592c55 100644 --- a/src/_pytask/explain.py +++ b/src/_pytask/explain.py @@ -50,7 +50,7 @@ def format(self, verbose: int = 1) -> str: # noqa: PLR0911 if self.reason == "forced": return " • Forced execution (--force flag)" if self.reason == "cascade": - return f" • Preceding task '{self.node_name}' would be executed" + return f" • Preceding {self.node_name} would be executed" return f" • {self.node_name}: {self.reason}" From a27b920517e9fe412f1a1663b9b468499f4bc46b Mon Sep 17 00:00:00 2001 From: Tobias Raabe Date: Mon, 6 Oct 2025 09:59:21 +0200 Subject: [PATCH 09/11] Use rich protocol but now it looks weird. --- src/_pytask/explain.py | 123 ++++++++++++++++++++++++++--------------- 1 file changed, 78 insertions(+), 45 deletions(-) diff --git a/src/_pytask/explain.py b/src/_pytask/explain.py index fb592c55..a4e408de 100644 --- a/src/_pytask/explain.py +++ b/src/_pytask/explain.py @@ -10,10 +10,17 @@ from rich.text import Text from _pytask.console import console +from _pytask.console import format_task_name from _pytask.outcomes import TaskOutcome from _pytask.pluginmanager import hookimpl if TYPE_CHECKING: + from collections.abc import Iterator + + from rich.console import Console + from rich.console import ConsoleOptions + from rich.console import RenderableType + from _pytask.node_protocols import PNode from _pytask.node_protocols import PTask from _pytask.reports import ExecutionReport @@ -28,30 +35,33 @@ class ChangeReason: node_type: str # "source", "dependency", "product", "task" reason: str # "changed", "missing", "not_in_db", "first_run" details: dict[str, Any] = field(factory=dict) + verbose: int = 1 - def format(self, verbose: int = 1) -> str: # noqa: PLR0911 - """Format the change reason as a string.""" + def __rich_console__( + self, console: Console, options: ConsoleOptions + ) -> Iterator[RenderableType]: + """Render the change reason using Rich protocol.""" if self.reason == "missing": - return f" • {self.node_name}: Missing" - if self.reason == "not_in_db": - return ( + yield Text(f" • {self.node_name}: Missing") + elif self.reason == "not_in_db": + yield Text( f" • {self.node_name}: Not in database (first run or database cleared)" ) - if self.reason == "changed": - if verbose >= 2 and "old_hash" in self.details: # noqa: PLR2004 - return ( - f" • {self.node_name}: Changed\n" - f" Previous hash: {self.details['old_hash'][:8]}...\n" - f" Current hash: {self.details['new_hash'][:8]}..." - ) - return f" • {self.node_name}: Changed" - if self.reason == "first_run": - return " • First execution" - if self.reason == "forced": - return " • Forced execution (--force flag)" - if self.reason == "cascade": - return f" • Preceding {self.node_name} would be executed" - return f" • {self.node_name}: {self.reason}" + elif self.reason == "changed": + if self.verbose >= 2 and "old_hash" in self.details: # noqa: PLR2004 + yield Text(f" • {self.node_name}: Changed") + yield Text(f" Previous hash: {self.details['old_hash'][:8]}...") + yield Text(f" Current hash: {self.details['new_hash'][:8]}...") + else: + yield Text(f" • {self.node_name}: Changed") + elif self.reason == "first_run": + yield Text(" • First execution") + elif self.reason == "forced": + yield Text(" • Forced execution (--force flag)") + elif self.reason == "cascade": + yield Text(f" • Preceding {self.node_name} would be executed") + else: + yield Text(f" • {self.node_name}: {self.reason}") @define @@ -59,28 +69,34 @@ class TaskExplanation: """Represents the explanation for why a task needs to be executed.""" reasons: list[ChangeReason] = field(factory=list) - - def format(self, task_name: str, outcome: TaskOutcome, verbose: int = 1) -> str: - """Format the task explanation as a string.""" - lines = [] - - if outcome == TaskOutcome.SKIP_UNCHANGED: - lines.append(f"{task_name}") - lines.append(" ✓ No changes detected") - elif outcome == TaskOutcome.PERSISTENCE: - lines.append(f"{task_name}") - lines.append(" • Persisted (products exist, changes ignored)") - elif outcome == TaskOutcome.SKIP: - lines.append(f"{task_name}") - lines.append(" • Skipped by marker") + task: PTask | None = None + outcome: TaskOutcome | None = None + verbose: int = 1 + editor_url_scheme: str = "" + + def __rich_console__( + self, console: Console, options: ConsoleOptions + ) -> Iterator[RenderableType]: + """Render the task explanation using Rich protocol.""" + # Format task name with proper styling and links + if self.task: + yield format_task_name(self.task, self.editor_url_scheme) + else: + yield Text("Unknown task") + + # Add explanation based on outcome + if self.outcome == TaskOutcome.SKIP_UNCHANGED: + yield Text(" ✓ No changes detected") + elif self.outcome == TaskOutcome.PERSISTENCE: + yield Text(" • Persisted (products exist, changes ignored)") + elif self.outcome == TaskOutcome.SKIP: + yield Text(" • Skipped by marker") elif not self.reasons: - lines.append(f"{task_name}") - lines.append(" ✓ No changes detected") + yield Text(" ✓ No changes detected") else: - lines.append(f"{task_name}") - lines.extend(reason.format(verbose) for reason in self.reasons) - - return "\n".join(lines) + for reason in self.reasons: + reason.verbose = self.verbose + yield reason def create_change_reason( @@ -106,7 +122,7 @@ def create_change_reason( @hookimpl(tryfirst=True) -def pytask_execute_log_end( # noqa: C901 +def pytask_execute_log_end( # noqa: C901, PLR0915 session: Session, reports: list[ExecutionReport] ) -> None: """Log explanations if --explain flag is set.""" @@ -145,6 +161,7 @@ def pytask_execute_log_end( # noqa: C901 ] verbose = session.config.get("verbose", 1) + editor_url_scheme = session.config.get("editor_url_scheme", "no_link") if would_execute: console.rule( @@ -158,7 +175,11 @@ def pytask_execute_log_end( # noqa: C901 console.print() for report in would_execute: explanation = report.task.attributes["explanation"] - console.print(explanation.format(report.task.name, report.outcome, verbose)) + explanation.task = report.task + explanation.outcome = report.outcome + explanation.verbose = verbose + explanation.editor_url_scheme = editor_url_scheme + console.print(explanation) console.print() if skipped: @@ -170,7 +191,11 @@ def pytask_execute_log_end( # noqa: C901 console.print() for report in skipped: explanation = report.task.attributes["explanation"] - console.print(explanation.format(report.task.name, report.outcome, verbose)) + explanation.task = report.task + explanation.outcome = report.outcome + explanation.verbose = verbose + explanation.editor_url_scheme = editor_url_scheme + console.print(explanation) console.print() if persisted and verbose >= 2: # noqa: PLR2004 @@ -182,7 +207,11 @@ def pytask_execute_log_end( # noqa: C901 console.print() for report in persisted: explanation = report.task.attributes["explanation"] - console.print(explanation.format(report.task.name, report.outcome, verbose)) + explanation.task = report.task + explanation.outcome = report.outcome + explanation.verbose = verbose + explanation.editor_url_scheme = editor_url_scheme + console.print(explanation) console.print() elif persisted and verbose == 1: console.print( @@ -199,7 +228,11 @@ def pytask_execute_log_end( # noqa: C901 console.print() for report in unchanged: explanation = report.task.attributes["explanation"] - console.print(explanation.format(report.task.name, report.outcome, verbose)) + explanation.task = report.task + explanation.outcome = report.outcome + explanation.verbose = verbose + explanation.editor_url_scheme = editor_url_scheme + console.print(explanation) console.print() elif unchanged and verbose == 1: console.print( From c653d45f0dc5e6d57b290244a9ea2d49bb44d072 Mon Sep 17 00:00:00 2001 From: Tobias Raabe Date: Sun, 12 Oct 2025 20:35:51 +0200 Subject: [PATCH 10/11] str become literal. rmeove hgihglighin. --- docs/source/_static/md/capture.md | 2 +- .../_static/md/clean-dry-run-directories.md | 2 +- docs/source/_static/md/clean-dry-run.md | 2 +- docs/source/_static/md/collect-nodes.md | 2 +- docs/source/_static/md/collect.md | 2 +- .../_static/md/defining-dependencies-products.md | 2 +- docs/source/_static/md/dry-run.md | 2 +- docs/source/_static/md/explain.md | 2 +- .../md/migrating-from-scripts-to-pytask.md | 2 +- docs/source/_static/md/pdb.md | 2 +- docs/source/_static/md/persist-executed.md | 2 +- docs/source/_static/md/persist-persisted.md | 2 +- docs/source/_static/md/persist-skipped.md | 2 +- docs/source/_static/md/profiling-tasks.md | 2 +- docs/source/_static/md/readme.md | 2 +- docs/source/_static/md/repeating-tasks.md | 2 +- docs/source/_static/md/show-locals.md | 2 +- docs/source/_static/md/trace.md | 2 +- docs/source/_static/md/try-first.md | 2 +- docs/source/_static/md/try-last.md | 2 +- docs/source/_static/md/warning.md | 2 +- docs/source/_static/md/write-a-task.md | 2 +- docs/source/tutorials/invoking_pytask.md | 4 ++-- src/_pytask/collect.py | 3 ++- src/_pytask/database_utils.py | 7 +++++-- src/_pytask/execute.py | 10 ++++++++-- src/_pytask/explain.py | 15 +++++++++++---- 27 files changed, 50 insertions(+), 33 deletions(-) diff --git a/docs/source/_static/md/capture.md b/docs/source/_static/md/capture.md index a14e22cb..0f41a9de 100644 --- a/docs/source/_static/md/capture.md +++ b/docs/source/_static/md/capture.md @@ -6,7 +6,7 @@ $ pytask ────────────────────────── Start pytask session ───────────────────────── Platform: win32 -- Python 3.12.0, pytask 0.5.3, pluggy 1.3.0 Root: C:\Users\pytask-dev\git\my_project -Collected 2 tasks. +Collected 2 tasks. ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━┓ ┃ Task ┃ Outcome ┃ diff --git a/docs/source/_static/md/clean-dry-run-directories.md b/docs/source/_static/md/clean-dry-run-directories.md index 8dff0403..0451985f 100644 --- a/docs/source/_static/md/clean-dry-run-directories.md +++ b/docs/source/_static/md/clean-dry-run-directories.md @@ -6,7 +6,7 @@ $ pytask clean --directories ────────────────────────── Start pytask session ───────────────────────── Platform: win32 -- Python 3.12.0, pytask 0.5.3, pluggy 1.3.0 Root: C:\Users\pytask-dev\git\my_project -Collected 1 task. +Collected 1 task. Files which can be removed: diff --git a/docs/source/_static/md/clean-dry-run.md b/docs/source/_static/md/clean-dry-run.md index 3baadb32..e5b094be 100644 --- a/docs/source/_static/md/clean-dry-run.md +++ b/docs/source/_static/md/clean-dry-run.md @@ -6,7 +6,7 @@ $ pytask clean ────────────────────────── Start pytask session ───────────────────────── Platform: win32 -- Python 3.12.0, pytask 0.5.3, pluggy 1.3.0 Root: C:\Users\pytask-dev\git\my_project -Collected 1 task. +Collected 1 task. Files which can be removed: diff --git a/docs/source/_static/md/collect-nodes.md b/docs/source/_static/md/collect-nodes.md index cb13848f..fac4a7cd 100644 --- a/docs/source/_static/md/collect-nodes.md +++ b/docs/source/_static/md/collect-nodes.md @@ -6,7 +6,7 @@ $ pytask ────────────────────────── Start pytask session ───────────────────────── Platform: win32 -- Python 3.12.0, pytask 0.5.3, pluggy 1.3.0 Root: C:\Users\pytask-dev\git\my_project -Collected 1 task. +Collected 1 task. Collected tasks: └── 🐍 <Module task_module.py> diff --git a/docs/source/_static/md/collect.md b/docs/source/_static/md/collect.md index 6fc0a420..5afdddf6 100644 --- a/docs/source/_static/md/collect.md +++ b/docs/source/_static/md/collect.md @@ -6,7 +6,7 @@ $ pytask ────────────────────────── Start pytask session ───────────────────────── Platform: win32 -- Python 3.12.0, pytask 0.5.3, pluggy 1.3.0 Root: C:\Users\pytask-dev\git\my_project -Collected 1 task. +Collected 1 task. Collected tasks: └── 🐍 <Module task_module.py> diff --git a/docs/source/_static/md/defining-dependencies-products.md b/docs/source/_static/md/defining-dependencies-products.md index 1112bfc0..161821e6 100644 --- a/docs/source/_static/md/defining-dependencies-products.md +++ b/docs/source/_static/md/defining-dependencies-products.md @@ -6,7 +6,7 @@ $ pytask ────────────────────────── Start pytask session ───────────────────────── Platform: win32 -- Python 3.12.0, pytask 0.5.3, pluggy 1.3.0 Root: C:\Users\pytask-dev\git\my_project -Collected 2 task. +Collected 2 task. ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━┓ ┃ Task ┃ Outcome ┃ diff --git a/docs/source/_static/md/dry-run.md b/docs/source/_static/md/dry-run.md index 3bb608e9..07c10b41 100644 --- a/docs/source/_static/md/dry-run.md +++ b/docs/source/_static/md/dry-run.md @@ -6,7 +6,7 @@ $ pytask --dry-run ────────────────────────── Start pytask session ───────────────────────── Platform: win32 -- Python 3.12.0, pytask 0.5.3, pluggy 1.3.0 Root: C:\Users\pytask-dev\git\my_project -Collected 1 task. +Collected 1 task. ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━┓ ┃ Task ┃ Outcome ┃ diff --git a/docs/source/_static/md/explain.md b/docs/source/_static/md/explain.md index 1d7d1877..f90fa425 100644 --- a/docs/source/_static/md/explain.md +++ b/docs/source/_static/md/explain.md @@ -6,7 +6,7 @@ $ pytask --explain ────────────────────────── Start pytask session ───────────────────────── Platform: darwin -- Python 3.12.0, pytask 0.5.6, pluggy 1.6.0 Root: /Users/pytask-dev/git/my_project -Collected 3 tasks. +Collected 3 tasks. ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━┓ ┃ Task ┃ Outcome ┃ diff --git a/docs/source/_static/md/migrating-from-scripts-to-pytask.md b/docs/source/_static/md/migrating-from-scripts-to-pytask.md index 75e0ea66..4f498035 100644 --- a/docs/source/_static/md/migrating-from-scripts-to-pytask.md +++ b/docs/source/_static/md/migrating-from-scripts-to-pytask.md @@ -6,7 +6,7 @@ $ pytask ────────────────────────── Start pytask session ───────────────────────── Platform: win32 -- Python 3.12.0, pytask 0.5.3, pluggy 1.3.0 Root: C:\Users\pytask-dev\git\my_project -Collected 1 task. +Collected 1 task. ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━┓ ┃ Task ┃ Outcome ┃ diff --git a/docs/source/_static/md/pdb.md b/docs/source/_static/md/pdb.md index 899e47eb..2131d5fb 100644 --- a/docs/source/_static/md/pdb.md +++ b/docs/source/_static/md/pdb.md @@ -6,7 +6,7 @@ $ pytask --pdb ────────────────────────── Start pytask session ───────────────────────── Platform: win32 -- Python 3.12.0, pytask 0.5.3, pluggy 1.3.0 Root: C:\Users\pytask-dev\git\my_project -Collected 1 task. +Collected 1 task. >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> Traceback >>>>>>>>>>>>>>>>>>>>>>>>>>>>>> ╭─────────────────── Traceback (most recent call last) ─────────────────╮ diff --git a/docs/source/_static/md/persist-executed.md b/docs/source/_static/md/persist-executed.md index b23d66f1..6a2f0b86 100644 --- a/docs/source/_static/md/persist-executed.md +++ b/docs/source/_static/md/persist-executed.md @@ -6,7 +6,7 @@ $ pytask ────────────────────────── Start pytask session ───────────────────────── Platform: win32 -- Python 3.12.0, pytask 0.5.3, pluggy 1.3.0 Root: C:\Users\pytask-dev\git\my_project -Collected 1 task. +Collected 1 task. ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━┓ ┃ Task ┃ Outcome ┃ diff --git a/docs/source/_static/md/persist-persisted.md b/docs/source/_static/md/persist-persisted.md index 88ffa1cc..5db81309 100644 --- a/docs/source/_static/md/persist-persisted.md +++ b/docs/source/_static/md/persist-persisted.md @@ -6,7 +6,7 @@ $ pytask ────────────────────────── Start pytask session ───────────────────────── Platform: win32 -- Python 3.12.0, pytask 0.5.3, pluggy 1.3.0 Root: C:\Users\pytask-dev\git\my_project -Collected 1 task. +Collected 1 task. ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━┓ ┃ Task ┃ Outcome ┃ diff --git a/docs/source/_static/md/persist-skipped.md b/docs/source/_static/md/persist-skipped.md index d31249d4..8c563282 100644 --- a/docs/source/_static/md/persist-skipped.md +++ b/docs/source/_static/md/persist-skipped.md @@ -6,7 +6,7 @@ $ pytask --verbose 2 ────────────────────────── Start pytask session ───────────────────────── Platform: win32 -- Python 3.12.0, pytask 0.5.3, pluggy 1.3.0 Root: C:\Users\pytask-dev\git\my_project -Collected 1 task. +Collected 1 task. ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━┓ ┃ Task ┃ Outcome ┃ diff --git a/docs/source/_static/md/profiling-tasks.md b/docs/source/_static/md/profiling-tasks.md index 215f62eb..f1212373 100644 --- a/docs/source/_static/md/profiling-tasks.md +++ b/docs/source/_static/md/profiling-tasks.md @@ -6,7 +6,7 @@ $ pytask profile ───────────────────────── Start pytask session ───────────────────────── Platform: win32 -- Python 3.12.0, pytask 0.5.3, pluggy 1.3.0 Root: C:\Users\pytask-dev\git\my_project -Collected 18 task. +Collected 18 task. ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━┓ ┃ Task ┃ Duration (in s) ┃ Size ┃ diff --git a/docs/source/_static/md/readme.md b/docs/source/_static/md/readme.md index 55d10b44..7bfcd368 100644 --- a/docs/source/_static/md/readme.md +++ b/docs/source/_static/md/readme.md @@ -6,7 +6,7 @@ $ pytask ──────────────────────────── Start pytask session ──────────────────────────── Platform: win32 -- Python 3.12.0, pytask 0.5.3, pluggy 1.3.0 Root: C:\Users\pytask-dev\git\my_project -Collected 1 task. +Collected 1 task. ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━┓ ┃ Task ┃ Outcome ┃ diff --git a/docs/source/_static/md/repeating-tasks.md b/docs/source/_static/md/repeating-tasks.md index 8dabb0b6..d4e357b9 100644 --- a/docs/source/_static/md/repeating-tasks.md +++ b/docs/source/_static/md/repeating-tasks.md @@ -6,7 +6,7 @@ $ pytask ────────────────────────── Start pytask session ───────────────────────── Platform: win32 -- Python 3.12.0, pytask 0.5.3, pluggy 1.3.0 Root: C:\Users\pytask-dev\git\my_project -Collected 10 task. +Collected 10 task. ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━┓ ┃ Task ┃ Outcome ┃ diff --git a/docs/source/_static/md/show-locals.md b/docs/source/_static/md/show-locals.md index 3cd41633..8352cb78 100644 --- a/docs/source/_static/md/show-locals.md +++ b/docs/source/_static/md/show-locals.md @@ -6,7 +6,7 @@ $ pytask --show-locals ──────────────────────────── Start pytask session ──────────────────────────── Platform: win32 -- Python 3.12.0, pytask 0.5.3, pluggy 1.3.0 Root: C:\Users\pytask-dev\git\my_project -Collected 1 task. +Collected 1 task. ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━┓ ┃ Task ┃ Outcome ┃ diff --git a/docs/source/_static/md/trace.md b/docs/source/_static/md/trace.md index 69ceb95f..0ab7edc2 100644 --- a/docs/source/_static/md/trace.md +++ b/docs/source/_static/md/trace.md @@ -6,7 +6,7 @@ $ pytask --trace ──────────────────────────── Start pytask session ──────────────────────────── Platform: win32 -- Python 3.12.0, pytask 0.5.3, pluggy 1.3.0 Root: C:\Users\pytask-dev\git\my_project -Collected 1 task. +Collected 1 task. >>>>>>>>>>>>>>>>>> PDB runcall (IO-capturing turned off) >>>>>>>>>>>>>>>>>> diff --git a/docs/source/_static/md/try-first.md b/docs/source/_static/md/try-first.md index 6c668ec8..bc71ba4b 100644 --- a/docs/source/_static/md/try-first.md +++ b/docs/source/_static/md/try-first.md @@ -6,7 +6,7 @@ $ pytask -s ────────────────────────── Start pytask session ───────────────────────── Platform: win32 -- Python 3.12.0, pytask 0.5.3, pluggy 1.3.0 Root: C:\Users\pytask-dev\git\my_project -Collected 2 task. +Collected 2 task. I'm first I'm second diff --git a/docs/source/_static/md/try-last.md b/docs/source/_static/md/try-last.md index 01d187f5..2abbe272 100644 --- a/docs/source/_static/md/try-last.md +++ b/docs/source/_static/md/try-last.md @@ -6,7 +6,7 @@ $ pytask -s ────────────────────────── Start pytask session ──────────────────────── Platform: win32 -- Python 3.12.0, pytask 0.5.3, pluggy 1.3.0 Root: C:\Users\pytask-dev\git\my_project -Collected 2 task. +Collected 2 task. I'm second I'm first diff --git a/docs/source/_static/md/warning.md b/docs/source/_static/md/warning.md index 49dbd3df..558af1ba 100644 --- a/docs/source/_static/md/warning.md +++ b/docs/source/_static/md/warning.md @@ -6,7 +6,7 @@ $ pytask ────────────────────────── Start pytask session ───────────────────────── Platform: win32 -- Python 3.12.0, pytask 0.5.3, pluggy 1.3.0 Root: C:\Users\pytask-dev\git\my_project -Collected 1 task. +Collected 1 task. ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━┓ ┃ Task ┃ Outcome ┃ diff --git a/docs/source/_static/md/write-a-task.md b/docs/source/_static/md/write-a-task.md index 7d335a61..9f12c4eb 100644 --- a/docs/source/_static/md/write-a-task.md +++ b/docs/source/_static/md/write-a-task.md @@ -6,7 +6,7 @@ $ pytask ────────────────────────── Start pytask session ───────────────────────── Platform: win32 -- Python 3.12.0, pytask 0.5.3, pluggy 1.3.0 Root: C:\Users\pytask-dev\git\my_project -Collected 1 task. +Collected 1 task. ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━┓ ┃ Task ┃ Outcome ┃ diff --git a/docs/source/tutorials/invoking_pytask.md b/docs/source/tutorials/invoking_pytask.md index 69ba37e5..7a86f960 100644 --- a/docs/source/tutorials/invoking_pytask.md +++ b/docs/source/tutorials/invoking_pytask.md @@ -86,8 +86,8 @@ Do a dry run to see which tasks will be executed without executing them. ### Explaining why tasks are executed Use the `--explain` flag to understand why tasks need to be executed. This shows what -changed (source files, dependencies, products) and helps you understand pytask's -execution decisions. +changed (source files, dependencies, products, previous tasks) and helps you understand +pytask's execution decisions. ```{include} ../_static/md/explain.md ``` diff --git a/src/_pytask/collect.py b/src/_pytask/collect.py index 7aad12a1..d1ed0c3a 100644 --- a/src/_pytask/collect.py +++ b/src/_pytask/collect.py @@ -598,7 +598,8 @@ def pytask_collect_log( session.collection_end = time.time() console.print( - f"Collected {len(tasks)} task{'' if len(tasks) == 1 else 's'}.", highlight=False + f"Collected {len(tasks)} task{'' if len(tasks) == 1 else 's'}.", + highlight=False, ) failed_reports = [r for r in reports if r.outcome == CollectionOutcome.FAIL] diff --git a/src/_pytask/database_utils.py b/src/_pytask/database_utils.py index 99590fff..77be06c5 100644 --- a/src/_pytask/database_utils.py +++ b/src/_pytask/database_utils.py @@ -3,6 +3,7 @@ from __future__ import annotations from typing import TYPE_CHECKING +from typing import Literal from sqlalchemy import create_engine from sqlalchemy.orm import DeclarativeBase @@ -88,12 +89,14 @@ def has_node_changed(task: PTask, node: PTask | PNode, state: str | None) -> boo def get_node_change_info( task: PTask, node: PTask | PNode, state: str | None -) -> tuple[bool, str, dict[str, str]]: +) -> tuple[ + bool, Literal["missing", "not_in_db", "changed", "unchanged"], dict[str, str] +]: """Get detailed information about why a node changed. Returns ------- - tuple[bool, str, dict[str, str]] + tuple[bool, Literal["missing", "not_in_db", "changed", "unchanged"], dict[str, str]] A tuple of (has_changed, reason, details) where: - has_changed: Whether the node has changed - reason: The reason for the change ("missing", "not_in_db", "changed", diff --git a/src/_pytask/execute.py b/src/_pytask/execute.py index c2c06429..64ba004e 100644 --- a/src/_pytask/execute.py +++ b/src/_pytask/execute.py @@ -27,6 +27,8 @@ from _pytask.exceptions import NodeLoadError from _pytask.exceptions import NodeNotFoundError from _pytask.explain import ChangeReason +from _pytask.explain import NodeType +from _pytask.explain import ReasonType from _pytask.explain import TaskExplanation from _pytask.explain import create_change_reason from _pytask.mark import Mark @@ -127,7 +129,7 @@ def pytask_execute_task_protocol(session: Session, task: PTask) -> ExecutionRepo @hookimpl(trylast=True) -def pytask_execute_task_setup(session: Session, task: PTask) -> None: # noqa: C901, PLR0912 +def pytask_execute_task_setup(session: Session, task: PTask) -> None: # noqa: C901, PLR0912, PLR0915 """Set up the execution of a task. 1. Check whether all dependencies of a task are available. @@ -199,6 +201,7 @@ def pytask_execute_task_setup(session: Session, task: PTask) -> None: # noqa: C if has_changed: needs_to_be_executed = True # Determine node type + node_type: NodeType if node_signature == task.signature: node_type = "source" elif node_signature in predecessors: @@ -206,11 +209,14 @@ def pytask_execute_task_setup(session: Session, task: PTask) -> None: # noqa: C else: node_type = "product" + # Cast reason since get_node_change_info can return "unchanged" + # but we only call this when has_changed is True + reason_typed: ReasonType = reason # type: ignore[assignment] change_reasons.append( create_change_reason( node=node, node_type=node_type, - reason=reason, + reason=reason_typed, old_hash=details.get("old_hash"), new_hash=details.get("new_hash"), ) diff --git a/src/_pytask/explain.py b/src/_pytask/explain.py index a4e408de..28dd59ee 100644 --- a/src/_pytask/explain.py +++ b/src/_pytask/explain.py @@ -4,6 +4,7 @@ from typing import TYPE_CHECKING from typing import Any +from typing import Literal from attrs import define from attrs import field @@ -27,13 +28,19 @@ from _pytask.session import Session +NodeType = Literal["source", "dependency", "product", "task"] +ReasonType = Literal[ + "changed", "missing", "not_in_db", "first_run", "forced", "cascade" +] + + @define class ChangeReason: """Represents a reason why a node changed.""" node_name: str - node_type: str # "source", "dependency", "product", "task" - reason: str # "changed", "missing", "not_in_db", "first_run" + node_type: NodeType + reason: ReasonType details: dict[str, Any] = field(factory=dict) verbose: int = 1 @@ -101,8 +108,8 @@ def __rich_console__( def create_change_reason( node: PNode | PTask, - node_type: str, - reason: str, + node_type: NodeType, + reason: ReasonType, old_hash: str | None = None, new_hash: str | None = None, ) -> ChangeReason: From 69694f44ec61814d8992291c2fffc7d6e2dac2c1 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sun, 12 Oct 2025 18:37:21 +0000 Subject: [PATCH 11/11] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/_pytask/console.py | 8 ++++---- tests/test_console.py | 12 ++++++------ tests/test_mark.py | 2 +- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/src/_pytask/console.py b/src/_pytask/console.py index 8a450172..fb8bd302 100644 --- a/src/_pytask/console.py +++ b/src/_pytask/console.py @@ -169,10 +169,10 @@ def format_strings_as_flat_tree( def create_url_style_for_task( - task_function: Callable[..., Any], edtior_url_scheme: str + task_function: Callable[..., Any], editor_url_scheme: str ) -> Style: """Create the style to add a link to a task id.""" - url_scheme = _EDITOR_URL_SCHEMES.get(edtior_url_scheme, edtior_url_scheme) + url_scheme = _EDITOR_URL_SCHEMES.get(editor_url_scheme, editor_url_scheme) if not url_scheme: return Style() @@ -190,9 +190,9 @@ def create_url_style_for_task( return style -def create_url_style_for_path(path: Path, edtior_url_scheme: str) -> Style: +def create_url_style_for_path(path: Path, editor_url_scheme: str) -> Style: """Create the style to add a link to a task id.""" - url_scheme = _EDITOR_URL_SCHEMES.get(edtior_url_scheme, edtior_url_scheme) + url_scheme = _EDITOR_URL_SCHEMES.get(editor_url_scheme, editor_url_scheme) return ( Style() if not url_scheme diff --git a/tests/test_console.py b/tests/test_console.py index 657ae790..ecaa3718 100644 --- a/tests/test_console.py +++ b/tests/test_console.py @@ -34,7 +34,7 @@ def task_func(): ... @pytest.mark.parametrize( - ("edtior_url_scheme", "expected"), + ("editor_url_scheme", "expected"), [ ("no_link", ""), ("file", "link file://{path}"), @@ -46,14 +46,14 @@ def task_func(): ... ), ], ) -def test_create_url_style_for_task(edtior_url_scheme, expected): +def test_create_url_style_for_task(editor_url_scheme, expected): path = Path(__file__) - style = create_url_style_for_task(task_func, edtior_url_scheme) + style = create_url_style_for_task(task_func, editor_url_scheme) assert style == Style.parse(expected.format(path=path)) @pytest.mark.parametrize( - ("edtior_url_scheme", "expected"), + ("editor_url_scheme", "expected"), [ ("no_link", ""), ("file", "link file://{path}"), @@ -65,9 +65,9 @@ def test_create_url_style_for_task(edtior_url_scheme, expected): ), ], ) -def test_create_url_style_for_path(edtior_url_scheme, expected): +def test_create_url_style_for_path(editor_url_scheme, expected): path = Path(__file__) - style = create_url_style_for_path(path, edtior_url_scheme) + style = create_url_style_for_path(path, editor_url_scheme) assert style == Style.parse(expected.format(path=path)) diff --git a/tests/test_mark.py b/tests/test_mark.py index 5ed81f0e..badf9792 100644 --- a/tests/test_mark.py +++ b/tests/test_mark.py @@ -356,7 +356,7 @@ def task_write_text(): ... @pytest.mark.parametrize("name", ["parametrize", "depends_on", "produces", "task"]) -def test_error_with_depreacated_markers(runner, tmp_path, name): +def test_error_with_deprecated_markers(runner, tmp_path, name): source = f""" from pytask import mark