Skip to content

Commit

Permalink
Merge branch 'master' into pr/wxtim/4471
Browse files Browse the repository at this point in the history
  • Loading branch information
@MetRonnie
MetRonnie committed Nov 1, 2021
2 parents 1cf2634 + 2f11d37 commit c3a5d14
Show file tree
Hide file tree
Showing 300 changed files with 2,440 additions and 1,638 deletions.
2 changes: 2 additions & 0 deletions .github/workflows/test_functional.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -134,6 +134,8 @@ jobs:
[platforms]
[[_local_at_indep_tcp]]
hosts = localhost
install target = localhost
job runner = at
__HERE__
cp "${PTH}/global.cylc" "${PTH}/global-tests.cylc"
Expand Down
6 changes: 6 additions & 0 deletions CHANGES.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -107,6 +107,9 @@ renamed to `CYLC_WORKFLOW_ID`. `CYLC_WORKFLOW_NAME` re-added as
[#4471](https://github.com/cylc/cylc-flow/pull/4471) - Users now get a different
error for a config item that isn't valid, to one that isn't set.

[#4457](https://github.com/cylc/cylc-flow/pull/4457) - Cylc 8
`cycle point time zone` now defaults to UTC, except in Cylc 7 compatibility mode.

### Fixes

[#4443](https://github.com/cylc/cylc-flow/pull/4443) - fix for slow polling
Expand Down Expand Up @@ -171,6 +174,9 @@ Third beta release of Cylc 8.
[#4286](https://github.com/cylc/cylc-flow/pull/4286) -
Add an option for displaying source workflows in `cylc scan`.

[#4300](https://github.com/cylc/cylc-flow/pull/4300) - Integer flow labels with
flow metadata, and improved task logging.

[#4291](https://github.com/cylc/cylc-flow/pull/4291) -
Remove obsolete `cylc edit` and `cylc search` commands.

Expand Down
17 changes: 5 additions & 12 deletions cylc/flow/cfgspec/globalcfg.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -471,7 +471,7 @@
Configuration of the Cylc Scheduler's main loop.
'''):
Conf('plugins', VDR.V_STRING_LIST,
['health check', 'prune flow labels', 'reset bad hosts'],
['health check', 'reset bad hosts'],
desc='''
Configure the default main loop plugins to use when
starting new workflows.
Expand All @@ -491,13 +491,6 @@
The interval with which this plugin is run.
''')

with Conf('prune flow labels', meta=MainLoopPlugin, desc='''
Prune redundant flow labels.
'''):
Conf('interval', VDR.V_INTERVAL, DurationFloat(600), desc='''
The interval with which this plugin is run.
''')

with Conf('reset bad hosts', meta=MainLoopPlugin, desc='''
Periodically clear the scheduler list of unreachable (bad)
hosts.
Expand Down Expand Up @@ -527,9 +520,9 @@
.. versionadded:: 8.0.0
'''):
Conf('source dirs', VDR.V_STRING_LIST, default=['~/cylc-src'], desc='''
A list of paths where ``cylc install <flow_name>`` will look for
a workflow of that name. All workflow source directories in these
locations will also show up in the GUI, ready for installation.
A list of paths for ``cylc install <name>`` to search for workflow
<name>. All workflow source directories in these locations will
also show up in the GUI, ready for installation.
.. note::
If workflow source directories of the same name exist in more
Expand Down Expand Up @@ -1145,7 +1138,7 @@ def load(self):
# Explicit config file override.
fname = os.path.join(conf_path_str, self.CONF_BASENAME)
self._load(fname, upgrader.USER_CONFIG)
elif conf_path_str is None:
else:
# Use default locations.
for conf_type, conf_dir in self.conf_dir_hierarchy:
fname = os.path.join(conf_dir, self.CONF_BASENAME)
Expand Down
148 changes: 80 additions & 68 deletions cylc/flow/cfgspec/workflow.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,8 @@
import contextlib
from itertools import product
import re
from typing import Any, Dict, Set
from textwrap import dedent
from typing import Any, Dict, Optional, Set

from metomi.isodatetime.data import Calendar

Expand All @@ -43,7 +44,6 @@
DEPRECATION_WARN = '''
.. deprecated:: 8.0.0
.. warning::
Deprecated section kept for compatibility with Cylc 7 workflow definitions.
Expand All @@ -53,25 +53,6 @@
Use :cylc:conf:`flow.cylc[runtime][<namespace>]platform` instead.
'''

SCRIPT_COMMON = '''
See also :ref:`JobScripts`.
User-defined script items:
* :cylc:conf:`[..]init-script`
* :cylc:conf:`[..]env-script`
* :cylc:conf:`[..]pre-script`
* :cylc:conf:`[..]script`
* :cylc:conf:`[..]post-script`
* :cylc:conf:`[..]err-script`
* :cylc:conf:`[..]exit-script`
Example::
{}
'''

DEPRECATED_IN_FAVOUR_OF_PLATFORMS = '''
.. deprecated:: 8.0.0
Expand All @@ -88,6 +69,29 @@
'''


def get_script_common_text(this: str, example: Optional[str] = None):
text = dedent('''
See also :ref:`JobScripts`.
Other user-defined script items:
''')
for item in [
'init-script', 'env-script', 'pre-script', 'script', 'post-script',
'err-script', 'exit-script'
]:
if item != this:
text += f"* :cylc:conf:`[..]{item}`\n"
text += dedent(f'''
Example::
{example if example else 'echo "Hello World"'}
''')
return text


with Conf(
'flow.cylc',
desc='''
Expand All @@ -111,12 +115,6 @@
Section for metadata items for this workflow. Cylc defines and uses
some terms (title, description, URL). Users can define more terms,
and use these in event handlers.
.. note::
A user could define "workflow-priority". An event handler
would then respond to failure events in a way set by
"workflow-priority".
'''):
Conf('description', VDR.V_STRING, '', desc='''
A multi-line description of the workflow. It can be retrieved at
Expand All @@ -141,7 +139,11 @@
Conf('<custom metadata>', VDR.V_STRING, '', desc='''
Any user-defined metadata item. These, like title, URL, etc. can be
passed to workflow event handlers to be interpreted according to
your needs. For example, "workflow-priority".
your needs.
For example, a user could define an item called
"workflow-priority". An event handler could then respond to
failure events in a way set by "workflow-priority".
''')
with Conf('scheduler', desc='''
.. versionchanged:: 8.0.0
Expand Down Expand Up @@ -263,11 +265,16 @@
represented with this string at the end.
If this isn't set (and :cylc:conf:`flow.cylc[scheduler]UTC mode`
is also not set), then it will default to the local time zone at
the time of running the workflow. This will persist over local time
zone changes (e.g. if the workflow is run during winter time, then
stopped, then restarted after summer time has begun, the cycle
points will remain in winter time).
is also not set), then it will default to:
- If your workflow is defined in a ``suite.rc`` file (Cylc 7
compatibility mode): local time zone when the workflow started.
- If your workflow is defined in a ``flow.cylc`` file: "Z" (UTC)
This will persist over local time zone changes (e.g. if the
workflow is run during winter time, then stopped, then restarted
after summer time has begun, the cycle points will remain
in winter time).
If this isn't set, and UTC mode is set to True, then this will
default to ``Z``. If you use a custom
Expand Down Expand Up @@ -727,7 +734,7 @@
See :ref:`task namespace rules. <namespace-names>`
legal values:
Example legal values:
- ``[foo]``
- ``[foo, bar, baz]``
Expand All @@ -740,70 +747,77 @@
all tasks in the workflow.
'''):
Conf('platform', VDR.V_STRING, desc='''
.. versionadded:: 8.0.0
The name of a compute resource defined in
:cylc:conf:`global.cylc[platforms]` or
:cylc:conf:`global.cylc[platform groups]`.
The platform specifies the host(s) that the task's jobs
will run on.
.. versionadded:: 8.0.0
''')
Conf('inherit', VDR.V_STRING_LIST, desc='''
A list of the immediate parent(s) of this namespace.
If no parents are listed default is ``root``.
''')
Conf('init-script', VDR.V_STRING, desc='''
Conf('init-script', VDR.V_STRING, desc=dedent('''
Custom script invoked by the task job script before the task
execution environment is configured - so it does not have
access to any workflow or task environment variables. It can be
an external command or script, or inlined scripting. The
original intention for this item was to allow remote tasks to
source login scripts to configure their access to cylc, but
this should no longer be necessary.
''' + SCRIPT_COMMON.format('echo "Hello World"'))
Conf('env-script', VDR.V_STRING, desc='''
''') + get_script_common_text(this='init-script'))
Conf('env-script', VDR.V_STRING, desc=dedent('''
Custom script invoked by the task job script between the
cylc-defined environment (workflow and task identity, etc.) and
the user-defined task runtime environment - so it has access
to the cylc environment (and the task environment has access
to variables defined by this scripting). It can be an
external command or script, or inlined scripting.
''' + SCRIPT_COMMON.format('echo "Hello World"'))
Conf('err-script', VDR.V_STRING, desc='''
''') + get_script_common_text(this='env-script'))
Conf('err-script', VDR.V_STRING, desc=dedent('''
Custom script to be invoked at the end of the error trap,
which is triggered due to failure of a command in the task job
script or trappable job kill. The output of this will always
be sent to STDERR and ``$1`` is set to the name of the signal
caught by the error trap. The script should be fast and use
very little system resource to ensure that the error trap can
return quickly. Companion of ``exit-script``, which is
executed on job success. It can be an external command or
script, or inlined scripting.
''' + SCRIPT_COMMON.format('echo "Hello World"'))
Conf('exit-script', VDR.V_STRING, desc='''
return quickly. Companion of :cylc:conf:`[..]exit-script`,
which is executed on job success. It can be an external
command or script, or inlined scripting.
''') + get_script_common_text(
this='err-script', example='echo "Uh oh, received ${1}"'
))
Conf('exit-script', VDR.V_STRING, desc=dedent('''
Custom script invoked at the very end of *successful* job
execution, just before the job script exits. It should
execute very quickly. Companion of ``err-script``, which is
executed on job failure. It can be an external command or
script, or inlined scripting.
''' + SCRIPT_COMMON.format('rm -f "$TMP_FILES"'))
Conf('pre-script', VDR.V_STRING, desc='''
execute very quickly. Companion of :cylc:conf:`[..]err-script`,
which is executed on job failure. It can be an external
command or script, or inlined scripting.
''') + get_script_common_text(
this='exit-script', example='rm -f "$TMP_FILES"'
))
Conf('pre-script', VDR.V_STRING, desc=dedent('''
Custom script invoked by the task job script immediately
before the ``script`` item (just below). It can be an
before :cylc:conf:`[..]script` (just below). It can be an
external command or script, or inlined scripting.
''' + SCRIPT_COMMON.format(
'echo "Hello from workflow ${CYLC_WORKFLOW_NAME}!"'))

Conf('script', VDR.V_STRING, desc='''
''') + get_script_common_text(
this='pre-script',
example='echo "Hello from workflow ${CYLC_WORKFLOW_ID}!"'
))
Conf('script', VDR.V_STRING, desc=dedent('''
The main custom script invoked from the task job script. It
can be an external command or script, or inlined scripting.
''' + SCRIPT_COMMON)
Conf('post-script', VDR.V_STRING, desc='''
''') + get_script_common_text(
this='script', example='my_script.sh'
))
Conf('post-script', VDR.V_STRING, desc=dedent('''
Custom script invoked by the task job script immediately
after the ``script`` item. It can be an external
after :cylc:conf:`[..]script` It can be an external
command or script, or inlined scripting.
''' + SCRIPT_COMMON)
''') + get_script_common_text(this='post-script'))

Conf('work sub-directory', VDR.V_STRING, desc='''
Task job scripts are executed from within *work directories*
Expand Down Expand Up @@ -1013,11 +1027,11 @@
excluded by omission from an ``include`` list.
''')

with Conf('job', desc='''
with Conf('job', desc=dedent('''
This section configures the means by which cylc submits task
job scripts to run.
''' + DEPRECATION_WARN):
''') + DEPRECATION_WARN):
Conf('batch system', VDR.V_STRING)
Conf('batch submit command template', VDR.V_STRING)

Expand Down Expand Up @@ -1074,17 +1088,17 @@
``%(finish_time)s``
Date-time when task job exits
``%(platform_name)s``
name of platform where the task job is submitted
Name of platform where the task job is submitted
``%(message)s``
Event message, if any
any task [meta] item, e.g.:
Any task [meta] item, e.g.:
``%(title)s``
Task title
``%(URL)s``
Task URL
``%(importance)s``
Example custom task metadata
any workflow ``[meta]`` item, prefixed with ``workflow_``
Any workflow ``[meta]`` item, prefixed with ``workflow_``
``%(workflow_title)s``
Workflow title
``%(workflow_URL)s``
Expand All @@ -1093,9 +1107,7 @@
Example custom workflow metadata.
Otherwise, the command line will be called with the following
default
Arguments:
default arguments:
.. code-block:: none
Expand Down
Loading

0 comments on commit c3a5d14

Please sign in to comment.