Fix scheduler#shutdown when reference-log option is used

lib/cylc/loggingutil.py

@@ -151,3 +151,31 @@ def do_rollover(self):
                 header_record.args = header_record.args[0:-1] + (
                     header_record.__dict__[self.FILE_NUM],)
             logging.FileHandler.emit(self, header_record)
+
+
+class ReferenceLogFileHandler(logging.FileHandler):
+    """A handler class which writes filtered reference logging records
+    to disk files.
+    """
+
+    REF_LOG_TEXTS = (
+        'triggered off', 'Initial point', 'Start point', 'Final point')
+    """List of texts used for filtering messages."""
+
+    def __init__(self, filename):
+        """Create the reference log file handler, specifying the file to
+        write the reference log lines."""
+        super().__init__(filename)
+        self.formatter = CylcLogFormatter()
+        self.addFilter(self._filter)
+
+    def _filter(self, record):
+        """Filter a logging record. From the base class Filterer (parent of
+            logging.Handler).
+
+            Args:
+                record (logging.LogRecord): a log record.
+            Returns:
+                bool: True for message to be logged, False otherwise.
+        """
+        return any(text in record.getMessage() for text in self.REF_LOG_TEXTS)

lib/cylc/scheduler.py

@@ -43,7 +43,8 @@
 import cylc.flags
 from cylc.host_appointer import HostAppointer, EmptyHostList
 from cylc.hostuserutil import get_host, get_user, get_fqdn_by_host
-from cylc.loggingutil import TimestampRotatingFileHandler
+from cylc.loggingutil import TimestampRotatingFileHandler,\
+    ReferenceLogFileHandler
 from cylc.log_diagnosis import LogSpec
 from cylc.network.server import SuiteRuntimeServer
 from cylc.profiler import Profiler
@@ -129,9 +130,6 @@ class Scheduler(object):
         'reload_suite'
     )
 
-    REF_LOG_TEXTS = (
-        'triggered off', 'Initial point', 'Start point', 'Final point')
-
     def __init__(self, is_restart, options, args):
         self.options = options
         self.profiler = Profiler(self.options.profile_mode)
@@ -1066,7 +1064,8 @@ def configure_reftest(self, recon=False):
         """Configure the reference test."""
         if self.options.genref:
             self.config.cfg['cylc']['log resolved dependencies'] = True
-
+            reference_log = os.path.join(self.config.fdir, 'reference.log')
+            LOG.addHandler(ReferenceLogFileHandler(reference_log))
         elif self.options.reftest:
             rtc = self.config.cfg['cylc']['reference test']
             req = rtc['required run mode']
@@ -1708,19 +1707,6 @@ def shutdown(self, reason=None):
 
         LOG.info(msg)
 
-        if self.options.genref:
-            try:
-                handle = open(
-                    os.path.join(self.config.fdir, 'reference.log'), 'wb')
-                logpath = glbl_cfg().get_derived_host_item(
-                    self.suite, 'suite log')
-                for line in open(logpath):
-                    if any(text in line for text in self.REF_LOG_TEXTS):
-                        handle.write(line)
-                handle.close()
-            except IOError as exc:
-                LOG.exception(exc)
-
         if self.proc_pool:
             if self.proc_pool.is_not_done():
                 # e.g. KeyboardInterrupt

tests/shutdown/19-log-reference.t

@@ -0,0 +1,36 @@
+#!/bin/bash
+# THIS FILE IS PART OF THE CYLC SUITE ENGINE.
+# Copyright (C) 2008-2019 NIWA & British Crown (Met Office) & Contributors.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#-------------------------------------------------------------------------------
+# Test suite shuts down with reference log, specifically that there is no
+# issue in the shutdown method when the --reference-log option is used.
+. $(dirname $0)/test_header
+#-------------------------------------------------------------------------------
+set_test_number 1
+#-------------------------------------------------------------------------------
+init_suite "${TEST_NAME_BASE}" <<'__SUITERC__'
+[scheduling]
+    [[dependencies]]
+        graph = t1
+[runtime]
+    [[t1]]
+        script = true
+__SUITERC__
+#-------------------------------------------------------------------------------
+TEST_NAME=$TEST_NAME_BASE-run
+suite_run_ok $TEST_NAME timeout 60s cylc run --debug --no-detach --reference-log $SUITE_NAME
+#-------------------------------------------------------------------------------
+purge_suite "${SUITE_NAME}"