Merge pull request #6349 from oliver-sanders/6336

example: cycle on irregular intervals

cylc/flow/etc/examples/cycle-over-irregular-dates/.validate

@@ -0,0 +1,24 @@
+#!/bin/bash
+# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE.
+# Copyright (C) NIWA & British Crown (Met Office) & Contributors.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+set -eux
+
+cylc lint ./simple
+cylc validate --check-circular ./simple
+
+cylc lint ./inter-dependent
+cylc validate --check-circular ./inter-dependent

cylc/flow/etc/examples/cycle-over-irregular-dates/index.rst

@@ -0,0 +1,132 @@
+Irregular Cycling
+-----------------
+
+We typically schedule tasks on regular intervals, e.g. ``P1D`` (every day) or
+``PT1H`` (every hour), however, sometimes our intervals are irregular.
+
+:ref:`user_guide.scheduling.exclusions` can be used to "subtract" dates or
+entire recurrences e.g:
+
+``PT1H!PT6H``
+   Every hour, except every six hours.
+``PT1H!(T00, T12)``
+   Every hour, except at 00:00 and 12:00.
+
+However, sometimes we want to schedule tasks on completely irregular intervals
+or at arbitrary dates. E.g, when working on case studies, you might have a list
+or range of arbitrary dates to work with.
+
+
+.. rubric:: Simple Example
+
+.. admonition:: Get a copy of this example
+   :class: hint
+
+   .. code-block:: console
+
+      $ cylc get-resources examples/cycle-over-irregular-dates/simple
+
+This example uses :ref:`Jinja` to define the list of dates and write out a
+scheduling section for each.
+
+.. literalinclude:: simple/flow.cylc
+   :language: cylc
+
+.. tip::
+
+   You can see the result of this Jinja2 code by running the ``cylc view -p``
+   command.
+
+
+.. rubric:: Example with inter-cycle dependencies
+
+.. admonition:: Get a copy of this example
+   :class: hint
+
+   .. code-block:: console
+
+      $ cylc get-resources examples/cycle-over-irregular-dates/inter-dependent
+
+.. _Jinja2 loop variable: https://jinja.palletsprojects.com/en/3.0.x/templates/#list-of-control-structures
+
+If you have dependencies between the cycles, you can make this work by using the
+`Jinja2 loop variable`_.
+
+For example, the previous iteration of the ``{% for date in DATES %}`` loop is
+available as ``loop.previtem`` and the next as ``loop.nextitem``.
+
+If you need to make the tasks which cycle on *irregular* intervals dependent on
+tasks which cycle on *regular* intervals, then you might find the
+:py:func:`strftime <cylc.flow.jinja.filters.strftime.strftime>` function
+helpful as a way of determining the nearest matching cycle.
+
+.. literalinclude:: inter-dependent/flow.cylc
+   :language: cylc
+
+You can see how the cycles are linked together using the ``cylc graph``
+command:
+
+.. NOTE: use "cylc graph . -o foo.dot --cycles 2000 2001" to generate this code
+
+.. digraph:: Example
+   :align: center
+
+   size = "7,15"
+
+   graph [fontname="sans" fontsize="25"]
+   node [fontname="sans"]
+
+   subgraph "cluster_20000101T0000Z" { 
+     label="20000101T0000Z"
+     style="dashed"
+     "20000101T0000Z/install" [label="install\n20000101T0000Z"]
+     "20000101T0000Z/prep" [label="prep\n20000101T0000Z"]
+   }
+
+   subgraph "cluster_20000105T0600Z" { 
+     label="20000105T0600Z"
+     style="dashed"
+     "20000105T0600Z/plot" [label="plot\n20000105T0600Z"]
+     "20000105T0600Z/run_model" [label="run_model\n20000105T0600Z"]
+   }
+
+   subgraph "cluster_20000305T1200Z" { 
+     label="20000305T1200Z"
+     style="dashed"
+     "20000305T1200Z/plot" [label="plot\n20000305T1200Z"]
+     "20000305T1200Z/run_model" [label="run_model\n20000305T1200Z"]
+   }
+
+   subgraph "cluster_20000528T1336Z" { 
+     label="20000528T1336Z"
+     style="dashed"
+     "20000528T1336Z/plot" [label="plot\n20000528T1336Z"]
+     "20000528T1336Z/run_model" [label="run_model\n20000528T1336Z"]
+   }
+
+   subgraph "cluster_20010101T0000Z" { 
+     label="20010101T0000Z"
+     style="dashed"
+     "20010101T0000Z/prep" [label="prep\n20010101T0000Z"]
+   }
+
+   subgraph "cluster_20010105T2324Z" { 
+     label="20010105T2324Z"
+     style="dashed"
+     "20010105T2324Z/plot" [label="plot\n20010105T2324Z"]
+     "20010105T2324Z/run_model" [label="run_model\n20010105T2324Z"]
+   }
+
+   "20000101T0000Z/install" -> "20000101T0000Z/prep"
+   "20000101T0000Z/install" -> "20010101T0000Z/prep"
+   "20000101T0000Z/prep" -> "20000105T0600Z/run_model"
+   "20000101T0000Z/prep" -> "20000305T1200Z/run_model"
+   "20000101T0000Z/prep" -> "20000528T1336Z/run_model"
+   "20000105T0600Z/run_model" -> "20000105T0600Z/plot"
+   "20000105T0600Z/run_model" -> "20000305T1200Z/run_model"
+   "20000305T1200Z/run_model" -> "20000305T1200Z/plot"
+   "20000305T1200Z/run_model" -> "20000528T1336Z/run_model"
+   "20000528T1336Z/run_model" -> "20000528T1336Z/plot"
+   "20000528T1336Z/run_model" -> "20010105T2324Z/run_model"
+   "20010101T0000Z/prep" -> "20010105T2324Z/run_model"
+   "20010105T2324Z/run_model" -> "20010105T2324Z/plot"

cylc/flow/etc/examples/cycle-over-irregular-dates/inter-dependent/flow.cylc

@@ -0,0 +1,51 @@
+#!Jinja2
+
+{%
+    set DATES = [
+        '2000-01-01T00:00Z',
+        '2000-01-01T06:00Z',
+        '2000-03-05T12:00Z',
+        '2000-05-28T13:36Z',
+        '2001-01-05T23:24',
+        '2002-04-30T04:20',
+    ]
+%}
+
+[meta]
+    title = Irregular cycling example
+    description = """
+        A workflow that runs a group of tasks on arbitrary dates
+        with inter-cycle dependencies between those dates.
+    """
+
+[scheduling]
+    initial cycle point = 2000
+    [[graph]]
+        # define the tasks you want to run on startup
+        R1 = install
+
+        # run this graph every year
+        P1Y = """
+            install[^] => prep
+        """
+
+{# loop over the list of dates #}
+{% for date in DATES %}
+        # schedule the tasks to run at each date
+        R1/{{ date }} = """
+            # make "run_model" depend on the "prep" task from the same year
+            prep[{{ date | strftime('%Y') }}] => run_model => plot
+
+    {# include this for all but the first date #}
+    {% if not loop.first %}
+            # make the run_model task depend on its previous instance
+            run_model[ {{ loop.previtem }} ] => run_model
+    {% endif %}
+        """
+{% endfor %}
+
+[runtime]
+    [[install]]
+    [[prep]]
+    [[run_model]]
+    [[plot]]

cylc/flow/etc/examples/cycle-over-irregular-dates/simple/flow.cylc

@@ -0,0 +1,40 @@
+#!Jinja2
+
+{%
+    set DATES = [
+        '2000-01-01T00:00Z',
+        '2000-01-01T06:00Z',
+        '2000-03-05T12:00Z',
+        '2000-05-28T13:36Z',
+        '2001-01-05T23:24',
+        '2002-04-30T04:20',
+    ]
+%}
+
+[meta]
+    title = Irregular cycling example
+    description = """
+        A workflow that runs a group of tasks on arbitrary dates.
+    """
+
+[scheduling]
+    # start cycling at the first date
+    initial cycle point = {{ DATES[0] }}
+    [[graph]]
+        # define the tasks you want to run on startup
+        R1 = install
+
+{# loop over the list of dates #}
+{% for date in DATES %}
+        # schedule the tasks to run at each date
+        R1/{{ date }} = """
+            # NOTE: install[^] references the task "install" in the first cycle
+            install[^] => prep => run_model => plot
+        """
+{% endfor %}
+
+[runtime]
+    [[install]]
+    [[prep]]
+    [[run_model]]
+    [[plot]]