Skip to content

Commit

Permalink
Merge pull request #4724 from oliver-sanders/scheduler-event-suite-key
Browse files Browse the repository at this point in the history 
config: assorted fixes for python template variables supported in various configurations
  • Loading branch information
@hjoliver
hjoliver authored Mar 31, 2022
2 parents 60216da + db0d0c8 commit 9346019
Show file tree
Hide file tree
Showing 18 changed files with 970 additions and 322 deletions.
2 changes: 1 addition & 1 deletion cylc/flow/cfgspec/globalcfg.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -241,7 +241,7 @@
),
'abort on inactivity timeout': (
f'''
:Default For; :cylc:conf:`flow.cylc \
:Default For: :cylc:conf:`flow.cylc \
[scheduler][events]abort on inactivity timeout`.
Whether to abort if the inactivity timer times out.
Expand Down
137 changes: 57 additions & 80 deletions cylc/flow/cfgspec/workflow.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,8 @@
# Regex to check whether a string is a command
REC_COMMAND = re.compile(r'(`|\$\()\s*(.*)\s*([`)])$')

# Regex to strip `:Default For:` notices from docs imported from the global cfg
DEFAULT_FOR = re.compile(r'.*:[Dd]efault [Ff]or:.*')

# Cylc8 Deprecation note.
DEPRECATION_WARN = '''
Expand Down Expand Up @@ -137,17 +139,24 @@ def get_script_common_text(this: str, example: Optional[str] = None):
Conf('URL', VDR.V_STRING, '', desc='''
A web URL to workflow documentation.
It can be retrieved at run time with the ``cylc show`` command.
The template ``%(workflow_name)s`` will be replaced with the actual
workflow name.
The URL can be retrieved at run time with the ``cylc show``
command.
The template variable ``%(workflow)s`` will be replaced with the
actual workflow ID.
.. deprecated:: 8.0.0
The ``%(suite_name)s`` template variable is deprecated, please
use ``%(workflow)s``.
.. seealso::
:cylc:conf:`flow.cylc[runtime][<namespace>][meta]URL`.
Example:
``http://my-site.com/workflows/%(workflow_name)s/index.html``
``http://my-site.com/workflows/%(workflow)s/index.html``
''')
Conf('<custom metadata>', VDR.V_STRING, '', desc='''
Expand Down Expand Up @@ -362,6 +371,10 @@ def get_script_common_text(this: str, example: Optional[str] = None):
``$PATH`` (in the shell in which the scheduler runs).
They should require little resource to run and return
quickly.
Template variables can be used to configure handlers.
For a full list of supported variables see
:ref:`workflow_event_template_variables`.
''')
Conf('handler events', VDR.V_STRING_LIST, None, desc='''
Specify the events for which workflow event
Expand All @@ -373,8 +386,28 @@ def get_script_common_text(this: str, example: Optional[str] = None):
''')

for item, desc in EVENTS_DESCR.items():
# strip the `:Default For:` lines
desc = DEFAULT_FOR.sub('', dedent(desc))
if item.endswith("handlers"):
Conf(item, VDR.V_STRING_LIST, desc=desc)
Conf(item, VDR.V_STRING_LIST, desc=(
# add examples
desc + '\n' + dedent(rf'''
Examples:
.. code-block:: cylc
# configure a single event handler
{item} = echo foo
# provide context to the handler
{item} = echo %(workflow)s
# configure multiple event handlers
{item} = \
'echo %(workflow)s, %(event)s', \
'my_exe %(event)s %(message)s' \
'curl -X PUT -d event=%(event)s host:port'
''')))
elif item.startswith("abort on"):
Conf(item, VDR.V_BOOLEAN, desc=desc)
elif item.endswith("timeout"):
Expand All @@ -400,14 +433,9 @@ def get_script_common_text(this: str, example: Optional[str] = None):
{REPLACES} ``[cylc][events]mail footer``.
================ ======================
Syntax Description
================ ======================
``%(host)s`` Workflow host name.
``%(port)s`` Workflow port number.
``%(owner)s`` Workflow owner name.
``%(workflow)s`` Workflow name
================ ======================
Template variables may be used in the mail footer. For a list
of supported variables see
:ref:`_workflow_event_template_variables`.
Example:
Expand Down Expand Up @@ -1172,16 +1200,24 @@ def get_script_common_text(this: str, example: Optional[str] = None):
A URL link to task documentation for this task or task
family.
The templates ``%(workflow_name)s`` and
``%(task_name)s`` will be replaced with the actual
workflow and task names.
The templates ``%(workflow)s`` and
``%(task)s`` will be replaced with the actual
workflow ID and task name.
.. deprecated:: 8.0.0
The ``%(suite_name)s`` template variable is
deprecated, please use ``%(workflow)s``.
The ``%(task_name)s`` template variable is
deprecated, please use ``%(task)s``.
See also :cylc:conf:`[meta]URL <flow.cylc[meta]URL>`.
Example:
``http://my-site.com/workflows/%(workflow_name)s/'''
'%(task_name)s.html``')
``http://my-site.com/workflows/%(workflow)s/'''
'%(task)s.html``')
Conf('<custom metadata>', VDR.V_STRING, '', desc='''
Any user-defined metadata item.
Expand Down Expand Up @@ -1316,67 +1352,8 @@ def get_script_common_text(this: str, example: Optional[str] = None):
quickly.
Each task event handler can be specified as a list of command
lines or command line templates. They can contain any or all
of the following patterns, which will be substituted with
actual values:
``%(event)s``
Event name
``%(workflow)s``
Workflow name
``%(workflow_uuid)s``
Workflow UUID string
``%(point)s``
Cycle point
``%(name)s``
Task name
``%(submit_num)s``
Submit number
``%(try_num)s``
Try number
``%(id)s``
Task ID (i.e. %(name)s.%(point)s)
``%(job_runner_name)s``
Job runner name (previously ``%(batch_sys_name)s``)
``%(job_id)s``
Job ID in the job runner
(previously ``%(batch_sys_job_id)s``)
``%(submit_time)s``
Date-time when task job is submitted
``%(start_time)s``
Date-time when task job starts running
``%(finish_time)s``
Date-time when task job exits
``%(platform_name)s``
Name of platform where the task job is submitted
``%(message)s``
Event message, if any
Any task [meta] item, e.g.:
``%(title)s``
Task title
``%(URL)s``
Task URL
``%(importance)s``
Example custom task metadata
Any workflow ``[meta]`` item, prefixed with ``workflow_``
``%(workflow_title)s``
Workflow title
``%(workflow_URL)s``
Workflow URL.
``%(workflow_rating)s``
Example custom workflow metadata.
Otherwise, the command line will be called with the following
default arguments:
.. code-block:: none
<event-handler> %(event)s %(workflow)s %(id)s %(message)s
.. note::
Substitution patterns should not be quoted in the template
strings. This is done automatically where required.
lines or command line templates. For a full list of supported
template variables see :ref:`task_event_template_variables`.
For an explanation of the substitution syntax, see
`String Formatting Operations in the Python
Expand Down Expand Up @@ -1965,7 +1942,7 @@ def warn_about_depr_event_handler_tmpl(cfg):
if f'%({EventData.SuiteUUID.value})' in handler:
LOG.warning(
deprecation_msg.format(EventData.SuiteUUID.value,
EventData.WorkflowUUID.value)
EventData.UUID.value)
)


Expand Down
102 changes: 76 additions & 26 deletions cylc/flow/config.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -505,32 +505,7 @@ def __init__(
self.feet.append(foot)
self.feet.sort() # sort effects get_graph_raw output

# Replace workflow and task name in workflow and task URLs.
self.cfg['meta']['URL'] = self.cfg['meta']['URL'] % {
'workflow_name': self.workflow}
# BACK COMPAT: CYLC_WORKFLOW_NAME
# from:
# Cylc7
# to:
# Cylc8
# remove at:
# Cylc9
self.cfg['meta']['URL'] = RE_WORKFLOW_ID_VAR.sub(
self.workflow, self.cfg['meta']['URL'])
for name, cfg in self.cfg['runtime'].items():
cfg['meta']['URL'] = cfg['meta']['URL'] % {
'workflow_name': self.workflow, 'task_name': name}
# BACK COMPAT: CYLC_WORKFLOW_NAME, CYLC_TASK_NAME
# from:
# Cylc7
# to:
# Cylc8
# remove at:
# Cylc9
cfg['meta']['URL'] = RE_WORKFLOW_ID_VAR.sub(
self.workflow, cfg['meta']['URL'])
cfg['meta']['URL'] = RE_TASK_NAME_VAR.sub(
name, cfg['meta']['URL'])
self.process_metadata_urls()

if getattr(self.options, 'is_validate', False):
self.mem_log("config.py: before _check_circular()")
Expand Down Expand Up @@ -2263,3 +2238,78 @@ def get_validated_rsync_includes(self):
"Directories can only be from the top level, please "
"reconfigure:" + str(illegal_includes)[1:-1])
return includes

def process_metadata_urls(self):
"""Process [meta]URL items."""
# workflow metadata
url = self.cfg['meta']['URL']
try:
self.cfg['meta']['URL'] = url % {
'workflow': self.workflow,
}
except (KeyError, ValueError):
try:
# Replace workflow and task name in workflow and task URLs.
# BACK COMPAT: suite_name
# url:
# https://github.com/cylc/cylc-flow/pull/4724
# from:
# Cylc7
# to:
# Cylc8
# remove at:
# Cylc9
self.cfg['meta']['URL'] = url % {
# cylc 7
'suite_name': self.workflow,
# cylc 8
'workflow': self.workflow,
}
except (KeyError, ValueError):
raise UserInputError(f'Invalid template [meta]URL: {url}')
else:
LOG.warning(
'Detected deprecated template variables in [meta]URL.'
'\nSee the configuration documentation for details.'
)

# task metadata
self.cfg['meta']['URL'] = RE_WORKFLOW_ID_VAR.sub(
self.workflow, self.cfg['meta']['URL'])
for name, cfg in self.cfg['runtime'].items():
try:
cfg['meta']['URL'] = cfg['meta']['URL'] % {
'workflow': self.workflow,
'task': name,
}
except (KeyError, ValueError):
# BACK COMPAT: suite_name, task_name
# url:
# https://github.com/cylc/cylc-flow/pull/4724
# from:
# Cylc7
# to:
# Cylc8
# remove at:
# Cylc9
try:
cfg['meta']['URL'] = cfg['meta']['URL'] % {
# cylc 7
'suite_name': self.workflow,
'task_name': name,
# cylc 8
'workflow': self.workflow,
'task': name,
}
except (KeyError, ValueError):
raise UserInputError(f'Invalid template [meta]URL: {url}')
else:
LOG.warning(
'Detected deprecated template variables in'
f' [runtime][{name}][meta]URL.'
'\nSee the configuration documentation for details.'
)
cfg['meta']['URL'] = RE_WORKFLOW_ID_VAR.sub(
self.workflow, cfg['meta']['URL'])
cfg['meta']['URL'] = RE_TASK_NAME_VAR.sub(
name, cfg['meta']['URL'])
11 changes: 7 additions & 4 deletions cylc/flow/log_diagnosis.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,20 +18,23 @@
from difflib import unified_diff
import re
import os

from typing import TYPE_CHECKING

from cylc.flow import LOG
from cylc.flow.exceptions import WorkflowEventError
from cylc.flow.pathutil import get_workflow_test_log_name

if TYPE_CHECKING:
from cylc.flow.scheduler import Scheduler


RE_TRIG = re.compile(r'(.*? -triggered off \[.*\])$')


def run_reftest(config, ctx):
def run_reftest(schd: 'Scheduler') -> None:
"""Run reference test at shutdown."""
reffilename = config.get_ref_log_name()
curfilename = get_workflow_test_log_name(ctx.workflow)
reffilename = schd.config.get_ref_log_name()
curfilename = get_workflow_test_log_name(schd.workflow)
ref = _load_reflog(reffilename)
cur = _load_reflog(curfilename)
if not ref:
Expand Down
7 changes: 2 additions & 5 deletions cylc/flow/scheduler.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -97,8 +97,7 @@
from cylc.flow.resources import get_resources
from cylc.flow.subprocpool import SubProcPool
from cylc.flow.workflow_db_mgr import WorkflowDatabaseManager
from cylc.flow.workflow_events import (
WorkflowEventContext, WorkflowEventHandler)
from cylc.flow.workflow_events import WorkflowEventHandler
from cylc.flow.workflow_status import StopMode, AutoRestartMode
from cylc.flow import workflow_files
from cylc.flow.taskdef import TaskDef
Expand Down Expand Up @@ -1251,9 +1250,7 @@ def run_event_handlers(self, event, reason=""):
conf.run_mode('simulation', 'dummy')
):
return
self.workflow_event_handler.handle(conf, WorkflowEventContext(
event, str(reason), self.workflow, self.uuid_str, self.owner,
self.host, self.server.port))
self.workflow_event_handler.handle(self, event, str(reason))

def release_queued_tasks(self):
"""Release queued tasks, and submit task jobs.
Expand Down
Loading

0 comments on commit 9346019

Please sign in to comment.