From 00aa93fe769bc65066f742fa5a475a6c5912af16 Mon Sep 17 00:00:00 2001 From: "Bruno P. Kinoshita" Date: Fri, 22 Mar 2019 16:54:28 +1300 Subject: [PATCH 1/2] Fix scheduler#shutdown when reference-log option is used --- lib/cylc/scheduler.py | 2 +- tests/shutdown/19-log-reference.t | 36 +++++++++++++++++++++++++++++++ 2 files changed, 37 insertions(+), 1 deletion(-) create mode 100755 tests/shutdown/19-log-reference.t diff --git a/lib/cylc/scheduler.py b/lib/cylc/scheduler.py index 02a6ea3479f..38505665089 100644 --- a/lib/cylc/scheduler.py +++ b/lib/cylc/scheduler.py @@ -1716,7 +1716,7 @@ def shutdown(self, reason=None): self.suite, 'suite log') for line in open(logpath): if any(text in line for text in self.REF_LOG_TEXTS): - handle.write(line) + handle.write(line.encode('utf-8')) handle.close() except IOError as exc: LOG.exception(exc) diff --git a/tests/shutdown/19-log-reference.t b/tests/shutdown/19-log-reference.t new file mode 100755 index 00000000000..92a3986f86b --- /dev/null +++ b/tests/shutdown/19-log-reference.t @@ -0,0 +1,36 @@ +#!/bin/bash +# THIS FILE IS PART OF THE CYLC SUITE ENGINE. +# Copyright (C) 2008-2019 NIWA & British Crown (Met Office) & Contributors. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +#------------------------------------------------------------------------------- +# Test suite shuts down with reference log, specifically that there is no +# issue in the shutdown method when the --reference-log option is used. +. $(dirname $0)/test_header +#------------------------------------------------------------------------------- +set_test_number 1 +#------------------------------------------------------------------------------- +init_suite "${TEST_NAME_BASE}" <<'__SUITERC__' +[scheduling] + [[dependencies]] + graph = t1 +[runtime] + [[t1]] + script = true +__SUITERC__ +#------------------------------------------------------------------------------- +TEST_NAME=$TEST_NAME_BASE-run +suite_run_ok $TEST_NAME timeout 60s cylc run --debug --no-detach --reference-log $SUITE_NAME +#------------------------------------------------------------------------------- +purge_suite "${SUITE_NAME}" From 8fec490be00294087f95fc941a5f03d3b6202c2b Mon Sep 17 00:00:00 2001 From: "Bruno P. Kinoshita" Date: Mon, 1 Apr 2019 11:02:32 +1300 Subject: [PATCH 2/2] Use a logging.FileHandler with a filterer for the reference log --- lib/cylc/loggingutil.py | 28 ++++++++++++++++++++++++++++ lib/cylc/scheduler.py | 22 ++++------------------ 2 files changed, 32 insertions(+), 18 deletions(-) diff --git a/lib/cylc/loggingutil.py b/lib/cylc/loggingutil.py index ae6683126b3..9ba3c132fda 100644 --- a/lib/cylc/loggingutil.py +++ b/lib/cylc/loggingutil.py @@ -151,3 +151,31 @@ def do_rollover(self): header_record.args = header_record.args[0:-1] + ( header_record.__dict__[self.FILE_NUM],) logging.FileHandler.emit(self, header_record) + + +class ReferenceLogFileHandler(logging.FileHandler): + """A handler class which writes filtered reference logging records + to disk files. + """ + + REF_LOG_TEXTS = ( + 'triggered off', 'Initial point', 'Start point', 'Final point') + """List of texts used for filtering messages.""" + + def __init__(self, filename): + """Create the reference log file handler, specifying the file to + write the reference log lines.""" + super().__init__(filename) + self.formatter = CylcLogFormatter() + self.addFilter(self._filter) + + def _filter(self, record): + """Filter a logging record. From the base class Filterer (parent of + logging.Handler). + + Args: + record (logging.LogRecord): a log record. + Returns: + bool: True for message to be logged, False otherwise. + """ + return any(text in record.getMessage() for text in self.REF_LOG_TEXTS) diff --git a/lib/cylc/scheduler.py b/lib/cylc/scheduler.py index 38505665089..8d48dfc1022 100644 --- a/lib/cylc/scheduler.py +++ b/lib/cylc/scheduler.py @@ -43,7 +43,8 @@ import cylc.flags from cylc.host_appointer import HostAppointer, EmptyHostList from cylc.hostuserutil import get_host, get_user, get_fqdn_by_host -from cylc.loggingutil import TimestampRotatingFileHandler +from cylc.loggingutil import TimestampRotatingFileHandler,\ + ReferenceLogFileHandler from cylc.log_diagnosis import LogSpec from cylc.network.server import SuiteRuntimeServer from cylc.profiler import Profiler @@ -129,9 +130,6 @@ class Scheduler(object): 'reload_suite' ) - REF_LOG_TEXTS = ( - 'triggered off', 'Initial point', 'Start point', 'Final point') - def __init__(self, is_restart, options, args): self.options = options self.profiler = Profiler(self.options.profile_mode) @@ -1066,7 +1064,8 @@ def configure_reftest(self, recon=False): """Configure the reference test.""" if self.options.genref: self.config.cfg['cylc']['log resolved dependencies'] = True - + reference_log = os.path.join(self.config.fdir, 'reference.log') + LOG.addHandler(ReferenceLogFileHandler(reference_log)) elif self.options.reftest: rtc = self.config.cfg['cylc']['reference test'] req = rtc['required run mode'] @@ -1708,19 +1707,6 @@ def shutdown(self, reason=None): LOG.info(msg) - if self.options.genref: - try: - handle = open( - os.path.join(self.config.fdir, 'reference.log'), 'wb') - logpath = glbl_cfg().get_derived_host_item( - self.suite, 'suite log') - for line in open(logpath): - if any(text in line for text in self.REF_LOG_TEXTS): - handle.write(line.encode('utf-8')) - handle.close() - except IOError as exc: - LOG.exception(exc) - if self.proc_pool: if self.proc_pool.is_not_done(): # e.g. KeyboardInterrupt