From df0336d9ca1a910c6251f64862481fc5687be573 Mon Sep 17 00:00:00 2001 From: Oliver Sanders Date: Mon, 2 Sep 2024 15:46:19 +0100 Subject: [PATCH] example: cycle on irregular intervals --- .../cycle-over-irregular-dates/.validate | 24 ++++ .../cycle-over-irregular-dates/index.rst | 132 ++++++++++++++++++ .../inter-dependent/flow.cylc | 51 +++++++ .../simple/flow.cylc | 40 ++++++ 4 files changed, 247 insertions(+) create mode 100755 cylc/flow/etc/examples/cycle-over-irregular-dates/.validate create mode 100644 cylc/flow/etc/examples/cycle-over-irregular-dates/index.rst create mode 100644 cylc/flow/etc/examples/cycle-over-irregular-dates/inter-dependent/flow.cylc create mode 100644 cylc/flow/etc/examples/cycle-over-irregular-dates/simple/flow.cylc diff --git a/cylc/flow/etc/examples/cycle-over-irregular-dates/.validate b/cylc/flow/etc/examples/cycle-over-irregular-dates/.validate new file mode 100755 index 00000000000..af06f1c5a7e --- /dev/null +++ b/cylc/flow/etc/examples/cycle-over-irregular-dates/.validate @@ -0,0 +1,24 @@ +#!/bin/bash +# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE. +# Copyright (C) NIWA & British Crown (Met Office) & Contributors. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +set -eux + +cylc lint ./simple +cylc validate --check-circular ./simple + +cylc lint ./inter-dependent +cylc validate --check-circular ./inter-dependent diff --git a/cylc/flow/etc/examples/cycle-over-irregular-dates/index.rst b/cylc/flow/etc/examples/cycle-over-irregular-dates/index.rst new file mode 100644 index 00000000000..181a57ebc22 --- /dev/null +++ b/cylc/flow/etc/examples/cycle-over-irregular-dates/index.rst @@ -0,0 +1,132 @@ +Irregular Cycling +----------------- + +We typically schedule tasks on regular intervals, e.g. ``P1D`` (every day) or +``PT1H`` (every hour), however, sometimes our intervals are irregular. + +:ref:`user_guide.scheduling.exclusions` can be used to "subtract" dates or +entire recurrences e.g: + +``PT1H!PT6H`` + Every hour, except every six hours. +``PT1H!(T00, T12)`` + Every hour, except at 00:00 and 12:00. + +However, sometimes we want to schedule tasks on completely irregular intervals +or at arbitrary dates. E.g, when working on case studies, you might have a list +or range of arbitrary dates to work with. + + +.. rubric:: Simple Example + +.. admonition:: Get a copy of this example + :class: hint + + .. code-block:: console + + $ cylc get-resources examples/cycle-over-irregular-dates/simple + +This example uses :ref:`Jinja` to define the list of dates and write out a +scheduling section for each. + +.. literalinclude:: simple/flow.cylc + :language: cylc + +.. tip:: + + You can see the result of this Jinja2 code by running the ``cylc view -p`` + command. + + +.. rubric:: Example with inter-cycle dependencies + +.. admonition:: Get a copy of this example + :class: hint + + .. code-block:: console + + $ cylc get-resources examples/cycle-over-irregular-dates/inter-dependent + +.. _Jinja2 loop variable: https://jinja.palletsprojects.com/en/3.0.x/templates/#list-of-control-structures + +If you have dependencies between the cycles, you can make this work by using the +`Jinja2 loop variable`_. + +For example, the previous iteration of the ``{% for date in DATES %}`` loop is +available as ``loop.previtem`` and the next as ``loop.nextitem``. + +If you need to make the tasks which cycle on *irregular* intervals dependent on +tasks which cycle on *regular* intervals, then you might find the +:py:func:`strftime ` function +helpful as a way of determining the nearest matching cycle. + +.. literalinclude:: inter-dependent/flow.cylc + :language: cylc + +You can see how the cycles are linked together using the ``cylc graph`` +command: + +.. NOTE: use "cylc graph . -o foo.dot --cycles 2000 2001" to generate this code + +.. digraph:: Example + :align: center + + size = "7,15" + + graph [fontname="sans" fontsize="25"] + node [fontname="sans"] + + subgraph "cluster_20000101T0000Z" { + label="20000101T0000Z" + style="dashed" + "20000101T0000Z/install" [label="install\n20000101T0000Z"] + "20000101T0000Z/prep" [label="prep\n20000101T0000Z"] + } + + subgraph "cluster_20000105T0600Z" { + label="20000105T0600Z" + style="dashed" + "20000105T0600Z/plot" [label="plot\n20000105T0600Z"] + "20000105T0600Z/run_model" [label="run_model\n20000105T0600Z"] + } + + subgraph "cluster_20000305T1200Z" { + label="20000305T1200Z" + style="dashed" + "20000305T1200Z/plot" [label="plot\n20000305T1200Z"] + "20000305T1200Z/run_model" [label="run_model\n20000305T1200Z"] + } + + subgraph "cluster_20000528T1336Z" { + label="20000528T1336Z" + style="dashed" + "20000528T1336Z/plot" [label="plot\n20000528T1336Z"] + "20000528T1336Z/run_model" [label="run_model\n20000528T1336Z"] + } + + subgraph "cluster_20010101T0000Z" { + label="20010101T0000Z" + style="dashed" + "20010101T0000Z/prep" [label="prep\n20010101T0000Z"] + } + + subgraph "cluster_20010105T2324Z" { + label="20010105T2324Z" + style="dashed" + "20010105T2324Z/plot" [label="plot\n20010105T2324Z"] + "20010105T2324Z/run_model" [label="run_model\n20010105T2324Z"] + } + + "20000101T0000Z/install" -> "20000101T0000Z/prep" + "20000101T0000Z/install" -> "20010101T0000Z/prep" + "20000101T0000Z/prep" -> "20000105T0600Z/run_model" + "20000101T0000Z/prep" -> "20000305T1200Z/run_model" + "20000101T0000Z/prep" -> "20000528T1336Z/run_model" + "20000105T0600Z/run_model" -> "20000105T0600Z/plot" + "20000105T0600Z/run_model" -> "20000305T1200Z/run_model" + "20000305T1200Z/run_model" -> "20000305T1200Z/plot" + "20000305T1200Z/run_model" -> "20000528T1336Z/run_model" + "20000528T1336Z/run_model" -> "20000528T1336Z/plot" + "20000528T1336Z/run_model" -> "20010105T2324Z/run_model" + "20010101T0000Z/prep" -> "20010105T2324Z/run_model" + "20010105T2324Z/run_model" -> "20010105T2324Z/plot" diff --git a/cylc/flow/etc/examples/cycle-over-irregular-dates/inter-dependent/flow.cylc b/cylc/flow/etc/examples/cycle-over-irregular-dates/inter-dependent/flow.cylc new file mode 100644 index 00000000000..f0c02a7d709 --- /dev/null +++ b/cylc/flow/etc/examples/cycle-over-irregular-dates/inter-dependent/flow.cylc @@ -0,0 +1,51 @@ +#!Jinja2 + +{% + set DATES = [ + '2000-01-01T00:00Z', + '2000-01-01T06:00Z', + '2000-03-05T12:00Z', + '2000-05-28T13:36Z', + '2001-01-05T23:24', + '2002-04-30T04:20', + ] +%} + +[meta] + title = Irregular cycling example + description = """ + A workflow that runs a group of tasks on arbitrary dates + with inter-cycle dependencies between those dates. + """ + +[scheduling] + initial cycle point = 2000 + [[graph]] + # define the tasks you want to run on startup + R1 = install + + # run this graph every year + P1Y = """ + install[^] => prep + """ + +{# loop over the list of dates #} +{% for date in DATES %} + # schedule the tasks to run at each date + R1/{{ date }} = """ + # make "run_model" depend on the "prep" task from the same year + prep[{{ date | strftime('%Y') }}] => run_model => plot + + {# include this for all but the first date #} + {% if not loop.first %} + # make the run_model task depend on its previous instance + run_model[ {{ loop.previtem }} ] => run_model + {% endif %} + """ +{% endfor %} + +[runtime] + [[install]] + [[prep]] + [[run_model]] + [[plot]] diff --git a/cylc/flow/etc/examples/cycle-over-irregular-dates/simple/flow.cylc b/cylc/flow/etc/examples/cycle-over-irregular-dates/simple/flow.cylc new file mode 100644 index 00000000000..be39f88f0c3 --- /dev/null +++ b/cylc/flow/etc/examples/cycle-over-irregular-dates/simple/flow.cylc @@ -0,0 +1,40 @@ +#!Jinja2 + +{% + set DATES = [ + '2000-01-01T00:00Z', + '2000-01-01T06:00Z', + '2000-03-05T12:00Z', + '2000-05-28T13:36Z', + '2001-01-05T23:24', + '2002-04-30T04:20', + ] +%} + +[meta] + title = Irregular cycling example + description = """ + A workflow that runs a group of tasks on arbitrary dates. + """ + +[scheduling] + # start cycling at the first date + initial cycle point = {{ DATES[0] }} + [[graph]] + # define the tasks you want to run on startup + R1 = install + +{# loop over the list of dates #} +{% for date in DATES %} + # schedule the tasks to run at each date + R1/{{ date }} = """ + # NOTE: install[^] references the task "install" in the first cycle + install[^] => prep => run_model => plot + """ +{% endfor %} + +[runtime] + [[install]] + [[prep]] + [[run_model]] + [[plot]]