diff --git a/.github/workflows/continuous-integration-workflow.yml b/.github/workflows/continuous-integration-workflow.yml index 806a9b55..124f32e0 100644 --- a/.github/workflows/continuous-integration-workflow.yml +++ b/.github/workflows/continuous-integration-workflow.yml @@ -72,7 +72,6 @@ jobs: - uses: conda-incubator/setup-miniconda@v2 with: auto-update-conda: true - python-version: 3.8 - name: Install core dependencies. shell: bash -l {0} diff --git a/docs/changes.rst b/docs/changes.rst index 307e35ca..04a61e03 100644 --- a/docs/changes.rst +++ b/docs/changes.rst @@ -7,11 +7,14 @@ all releases are available on `PyPI `_ and `Anaconda.org `_. -0.0.13 - 2021-xx-xx +0.0.13 - 2021-03-09 ------------------- - :gh:`72` adds conda-forge to the README and highlights importance of specifying dependencies and products. +- :gh:`62` implements the ``pytask.mark.skipif`` marker to conditionally skip tasks. + Many thanks to :ghuser:`roecla` for implementing this feature and a warm welcome since + she is the first pytask contributor! 0.0.12 - 2021-02-27 diff --git a/docs/reference_guides/marks.rst b/docs/reference_guides/marks.rst index feaf67d5..6baf1fd0 100644 --- a/docs/reference_guides/marks.rst +++ b/docs/reference_guides/marks.rst @@ -32,4 +32,35 @@ pytask.mark.try_first .. function:: try_first :noindex: - This + Indicate that the task should be executed as soon as possible. + + This indicator is a soft measure to influence the execution order of pytask. + + .. important:: + + This indicator is not intended for general use to influence the build order and + to overcome misspecification of task dependencies and products. + + It should only be applied to situations where it is hard to define all + dependencies and products and automatic inference may be incomplete like with + pytask-latex and latex-dependency-scanner. + + +pytask.mark.try_last +--------------------- + +.. function:: try_last + :noindex: + + Indicate that the task should be executed as late as possible. + + This indicator is a soft measure to influence the execution order of pytask. + + .. important:: + + This indicator is not intended for general use to influence the build order and + to overcome misspecification of task dependencies and products. + + It should only be applied to situations where it is hard to define all + dependencies and products and automatic inference may be incomplete like with + pytask-latex and latex-dependency-scanner. diff --git a/docs/tutorials/how_to_skip_tasks.rst b/docs/tutorials/how_to_skip_tasks.rst new file mode 100644 index 00000000..e574da45 --- /dev/null +++ b/docs/tutorials/how_to_skip_tasks.rst @@ -0,0 +1,69 @@ +How to skip tasks +================= + +Tasks are skipped automatically if neither their file nor any of their dependencies have +changed and all products exist. + +In addition, you may want pytask to skip tasks either generally or if certain conditions +are fulfilled. Skipping means the task itself and all tasks that depend on it will not +be executed, even if the task file or their dependencies have changed or products are +missing. + +This can be useful for example if you are working on a task that creates the dependency +of a long running task and you are not interested in the long running task's product for +the moment. In that case you can simply use ``@pytask.mark.skip`` in front of the long +running task to stop it from running: + +.. code-block:: python + + # Content of task_create_dependency.py + + + @pytask.mark.produces("dependency_of_long_running_task.md") + def task_you_are_working_on(produces): + ... + +.. code-block:: python + + # Content of task_long_running.py + + + @pytask.mark.skip + @pytask.mark.depends_on("dependency_of_long_running_task.md") + def task_that_takes_really_long_to_run(depends_on): + ... + + +In large projects, you may have many long running tasks that you only want to execute +sporadically, e.g. when you are not working locally but running the project on a server. + +In that case, we recommend using ``@pytask.mark.skipif`` which lets you supply a +condition and a reason as arguments: + + +.. code-block:: python + + # Content of a config.py + + NO_LONG_RUNNING_TASKS = True + +.. code-block:: python + + # Content of task_create_dependency.py + + + @pytask.mark.produces("run_always.md") + def task_always(produces): + ... + +.. code-block:: python + + # Content of task_long_running.py + + from config import NO_LONG_RUNNING_TASKS + + + @pytask.mark.skipif(NO_LONG_RUNNING_TASKS, "Skip long-running tasks.") + @pytask.mark.depends_on("dependency_of_long_running_task.md") + def task_that_takes_really_long_to_run(depends_on): + ... diff --git a/docs/tutorials/index.rst b/docs/tutorials/index.rst index 97ac9d17..10bf5f12 100644 --- a/docs/tutorials/index.rst +++ b/docs/tutorials/index.rst @@ -16,6 +16,7 @@ project. Start here if you are a new user. how_to_select_tasks how_to_clean how_to_collect + how_to_skip_tasks how_to_make_tasks_persist how_to_capture how_to_invoke_pytask diff --git a/src/_pytask/skipping.py b/src/_pytask/skipping.py index 125ce864..7f98aff3 100644 --- a/src/_pytask/skipping.py +++ b/src/_pytask/skipping.py @@ -16,6 +16,11 @@ def skip_ancestor_failed(reason: str = "No reason provided.") -> str: return reason +def skipif(condition: bool, *, reason: str) -> tuple: + """Function to parse information in ``@pytask.mark.skipif``.""" + return condition, reason + + @hookimpl def pytask_parse_config(config): markers = { @@ -24,6 +29,8 @@ def pytask_parse_config(config): "failed.", "skip_unchanged": "Internal decorator applied to tasks which have already been " "executed and have not been changed.", + "skipif": "Skip a task and all its subsequent tasks in case a condition is " + "fulfilled.", } config["markers"] = {**config["markers"], **markers} @@ -46,6 +53,14 @@ def pytask_execute_task_setup(task): if markers: raise Skipped + markers = get_specific_markers_from_task(task, "skipif") + if markers: + marker_args = [skipif(*marker.args, **marker.kwargs) for marker in markers] + message = "\n".join([arg[1] for arg in marker_args if arg[0]]) + should_skip = any(arg[0] for arg in marker_args) + if should_skip: + raise Skipped(message) + @hookimpl def pytask_execute_task_process_report(session, report): diff --git a/tests/test_skipping.py b/tests/test_skipping.py index 1f41ed18..432fd880 100644 --- a/tests/test_skipping.py +++ b/tests/test_skipping.py @@ -56,7 +56,7 @@ def task_dummy(depends_on, produces): @pytest.mark.end_to_end -def test_skip_if_ancestor_failed(tmp_path): +def test_skipif_ancestor_failed(tmp_path): source = """ import pytask @@ -102,6 +102,101 @@ def task_second(): assert isinstance(session.execution_reports[1].exc_info[1], Skipped) +@pytest.mark.end_to_end +def test_if_skipif_decorator_is_applied_skipping(tmp_path): + source = """ + import pytask + + @pytask.mark.skipif(condition=True, reason="bla") + @pytask.mark.produces("out.txt") + def task_first(): + assert False + + @pytask.mark.depends_on("out.txt") + def task_second(): + assert False + """ + tmp_path.joinpath("task_dummy.py").write_text(textwrap.dedent(source)) + + session = main({"paths": tmp_path}) + node = session.collection_reports[0].node + assert len(node.markers) == 1 + assert node.markers[0].name == "skipif" + assert node.markers[0].args == () + assert node.markers[0].kwargs == {"condition": True, "reason": "bla"} + + assert session.execution_reports[0].success + assert isinstance(session.execution_reports[0].exc_info[1], Skipped) + assert session.execution_reports[1].success + assert isinstance(session.execution_reports[1].exc_info[1], Skipped) + assert session.execution_reports[0].exc_info[1].args[0] == "bla" + + +@pytest.mark.end_to_end +def test_if_skipif_decorator_is_applied_execute(tmp_path): + source = """ + import pytask + + @pytask.mark.skipif(False, reason="bla") + @pytask.mark.produces("out.txt") + def task_first(produces): + with open(produces, "w") as f: + f.write("hello world.") + + @pytask.mark.depends_on("out.txt") + def task_second(): + pass + """ + tmp_path.joinpath("task_dummy.py").write_text(textwrap.dedent(source)) + + session = main({"paths": tmp_path}) + node = session.collection_reports[0].node + + assert len(node.markers) == 1 + assert node.markers[0].name == "skipif" + assert node.markers[0].args == (False,) + assert node.markers[0].kwargs == {"reason": "bla"} + assert session.execution_reports[0].success + assert session.execution_reports[0].exc_info is None + assert session.execution_reports[1].success + assert session.execution_reports[1].exc_info is None + + +@pytest.mark.end_to_end +def test_if_skipif_decorator_is_applied_any_condition_matches(tmp_path): + """Any condition of skipif has to be True and only their message is shown.""" + source = """ + import pytask + + @pytask.mark.skipif(condition=False, reason="I am fine") + @pytask.mark.skipif(condition=True, reason="No, I am not.") + @pytask.mark.produces("out.txt") + def task_first(): + assert False + + @pytask.mark.depends_on("out.txt") + def task_second(): + assert False + """ + tmp_path.joinpath("task_dummy.py").write_text(textwrap.dedent(source)) + + session = main({"paths": tmp_path}) + node = session.collection_reports[0].node + assert len(node.markers) == 2 + assert node.markers[0].name == "skipif" + assert node.markers[0].args == () + assert node.markers[0].kwargs == {"condition": True, "reason": "No, I am not."} + assert node.markers[1].name == "skipif" + assert node.markers[1].args == () + assert node.markers[1].kwargs == {"condition": False, "reason": "I am fine"} + + assert session.execution_reports[0].success + assert isinstance(session.execution_reports[0].exc_info[1], Skipped) + assert session.execution_reports[1].success + assert isinstance(session.execution_reports[1].exc_info[1], Skipped) + assert session.execution_reports[0].exc_info[1].args[0] == "No, I am not." + + @pytest.mark.unit @pytest.mark.parametrize( ("marker_name", "expectation"),