-
Notifications
You must be signed in to change notification settings - Fork 94
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
main loop: plugins for periodic functions
- Loading branch information
1 parent
95371b8
commit 697b06e
Showing
6 changed files
with
190 additions
and
88 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,82 @@ | ||
| # THIS FILE IS PART OF THE CYLC SUITE ENGINE. | ||
| # Copyright (C) 2008-2019 NIWA & British Crown (Met Office) & Contributors. | ||
| # | ||
| # This program is free software: you can redistribute it and/or modify | ||
| # it under the terms of the GNU General Public License as published by | ||
| # the Free Software Foundation, either version 3 of the License, or | ||
| # (at your option) any later version. | ||
| # | ||
| # This program is distributed in the hope that it will be useful, | ||
| # but WITHOUT ANY WARRANTY; without even the implied warranty of | ||
| # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | ||
| # GNU General Public License for more details. | ||
| # | ||
| # You should have received a copy of the GNU General Public License | ||
| # along with this program. If not, see <http://www.gnu.org/licenses/>. | ||
| """Periodic functions for the main loop.""" | ||
|
|
||
| from collections import namedtuple | ||
| import pkg_resources | ||
| from time import time | ||
|
|
||
| from cylc.flow import LOG | ||
|
|
||
|
|
||
| MainLoopPlugin = namedtuple( | ||
| 'MainLoopPlugin', | ||
| ['before', 'on_change', 'after', 'interval', 'data'] | ||
| ) | ||
|
|
||
| TIMINGS = {} | ||
|
|
||
|
|
||
| def load_plugins(config): | ||
| plugins = {} | ||
| entry_points = pkg_resources.get_entry_map( | ||
| 'cylc-flow').get('main_loop', {}) | ||
| for name, module in entry_points.items(): | ||
| module = module.load() | ||
| conf = config.get(name.replace('_', ' '), {}) | ||
| plugins[name] = ( | ||
| MainLoopPlugin( | ||
| getattr(module, 'before', None), | ||
| getattr(module, 'on_change', None), | ||
| getattr(module, 'after', None), | ||
| conf.get('interval', None), | ||
| {} | ||
| ) | ||
| ) | ||
| return plugins | ||
|
|
||
|
|
||
| def before(plugins, scheduler): | ||
| TIMINGS[scheduler.uuid_str] = {} | ||
| for name, plugin in plugins.items(): | ||
| if plugin.before: | ||
| LOG.critical('main_loop:%s:before' % name) | ||
| ret = plugin.before(scheduler, plugin.data) | ||
| TIMINGS[scheduler.uuid_str][name] = 0 | ||
|
|
||
|
|
||
| def on_change(plugins, scheduler): | ||
| now = time() | ||
| for name in plugins: | ||
| plugin = plugins[name] | ||
| if ( | ||
| plugin.on_change | ||
| and ( | ||
| plugin.interval is None | ||
| or now - TIMINGS[scheduler.uuid_str][name] > plugin.interval | ||
| ) | ||
| ): | ||
| LOG.critical('main_loop:%s:on_change' % name) | ||
| plugin.on_change(scheduler, plugin.data) | ||
| TIMINGS[scheduler.uuid_str][name] = time() | ||
|
|
||
|
|
||
| def after(plugins): | ||
| for name in plugins: | ||
| plugin = plugins[name] | ||
| if plugin.after: | ||
| LOG.critical('main_loop:%s:after' % name) | ||
| plugin.after(plugin.data) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,73 @@ | ||
| import os | ||
|
|
||
| from cylc.flow import LOG | ||
| from cylc.flow import suite_files | ||
| from cylc.flow.cfgspec.glbl_cfg import glbl_cfg | ||
|
|
||
|
|
||
| def on_change(scheduler, data): | ||
| _check_if_condemned(scheduler) | ||
| _check_suite_run_dir(scheduler) | ||
| _check_contact_file(scheduler) | ||
|
|
||
|
|
||
| def _check_if_condemned(scheduler): | ||
| # 1.. check if suite host is condemned - if so auto restart. | ||
| if scheduler.stop_mode is None: | ||
| current_glbl_cfg = glbl_cfg(cached=False) | ||
| for host in current_glbl_cfg.get(['suite servers', | ||
| 'condemned hosts']): | ||
| if host.endswith('!'): | ||
| # host ends in an `!` -> force shutdown mode | ||
| mode = AutoRestartMode.FORCE_STOP | ||
| host = host[:-1] | ||
| else: | ||
| # normal mode (stop and restart the suite) | ||
| mode = AutoRestartMode.RESTART_NORMAL | ||
| if scheduler.auto_restart_time is not None: | ||
| # suite is already scheduled to stop-restart only | ||
| # AutoRestartMode.FORCE_STOP can override this. | ||
| continue | ||
|
|
||
| if get_fqdn_by_host(host) == scheduler.host: | ||
| # this host is condemned, take the appropriate action | ||
| LOG.info('The Cylc suite host will soon become ' | ||
| 'un-available.') | ||
| if mode == AutoRestartMode.FORCE_STOP: | ||
| # server is condemned in "force" mode -> stop | ||
| # the suite, don't attempt to restart | ||
| LOG.critical( | ||
| 'This suite will be shutdown as the suite ' | ||
| 'host is unable to continue running it.\n' | ||
| 'When another suite host becomes available ' | ||
| 'the suite can be restarted by:\n' | ||
| ' $ cylc restart %s', scheduler.suite) | ||
| if scheduler.set_auto_restart(mode=mode): | ||
| return # skip remaining health checks | ||
| elif (scheduler.set_auto_restart(current_glbl_cfg.get( | ||
| ['suite servers', 'auto restart delay']))): | ||
| # server is condemned -> configure the suite to | ||
| # auto stop-restart if possible, else, report the | ||
| # issue preventing this | ||
| return # skip remaining health checks | ||
| break | ||
|
|
||
| def _check_suite_run_dir(scheduler): | ||
| # 2. check if suite run dir still present - if not shutdown. | ||
| if not os.path.exists(scheduler.suite_run_dir): | ||
| raise OSError(ENOENT, os.strerror(ENOENT), scheduler.suite_run_dir) | ||
|
|
||
| def _check_contact_file(scheduler): | ||
| # 3. check if contact file consistent with current start - if not | ||
| # shutdown. | ||
| try: | ||
| contact_data = suite_files.load_contact_file( | ||
| scheduler.suite) | ||
| if contact_data != scheduler.contact_data: | ||
| raise AssertionError('contact file modified') | ||
| except (AssertionError, IOError, ValueError, | ||
| SuiteServiceFileError) as exc: | ||
| LOG.error( | ||
| "%s: contact file corrupted/modified and may be left", | ||
| suite_files.get_contact_file(scheduler.suite)) | ||
| raise exc |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters