Global config changes: process pool, logging

cylc/flow/cfgspec/globalcfg.py

@@ -75,32 +75,31 @@
        Prior to Cylc 8, ``global.cylc`` was named ``global.rc``, but that name
        is no longer supported.
 ''') as SPEC:
-
-    # suite
-    Conf('process pool size', VDR.V_INTEGER, 4, desc='''
-        Maximum number of concurrent processes used to execute external job
-        submission, event handlers, and job poll and kill commands - see
-        :ref:`Managing External Command Execution`.
-    ''')
-    Conf('process pool timeout', VDR.V_INTERVAL, DurationFloat(600), desc='''
-        Interval after which long-running commands in the process pool will be
-        killed - see :ref:`Managing External Command Execution`.
-
-        .. note::
-           The default is set quite high to avoid killing important
-           processes when the system is under load.
-    ''')
-    Conf('run directory rolling archive length', VDR.V_INTEGER, -1, desc='''
-        The number of old run directory trees to retain at start-up.
-    ''')
-
     with Conf('scheduler', desc='''
         Default values for entries in :cylc:conf:`flow.cylc[scheduler]`
         section. This should not be confused with scheduling in the
         ``flow.cylc`` file.
     '''):
         Conf('UTC mode', VDR.V_BOOLEAN, False, desc='''
-                Default for :cylc:conf:`flow.cylc[scheduler]UTC mode`.
+            Default for :cylc:conf:`flow.cylc[scheduler]UTC mode`.
+        ''')
+        Conf('process pool size', VDR.V_INTEGER, 4, desc='''
+            Maximum number of concurrent processes used to execute external job
+            submission, event handlers, and job poll and kill commands - see
+            :ref:`Managing External Command Execution`.
+        ''')
+        Conf('process pool timeout', VDR.V_INTERVAL, DurationFloat(600),
+             desc='''
+            Interval after which long-running commands in the process pool
+            will be killed - see :ref:`Managing External Command Execution`.
+
+            .. note::
+               The default is set quite high to avoid killing important
+               processes when the system is under load.
+        ''')
+        Conf('run directory rolling archive length', VDR.V_INTEGER, -1,
+             desc='''
+            The number of old run directory trees to retain at start-up.
         ''')
 
         with Conf('events', desc='''
@@ -171,17 +170,18 @@
                     The interval with which this plugin is run.
                 ''')
 
-    with Conf('suite logging', desc='''
-        The suite event log, held under the suite run directory, is maintained
-        as a rolling archive. Logs are rolled over (backed up and started anew)
-        when they reach a configurable limit size.
-    '''):
-        Conf('rolling archive length', VDR.V_INTEGER, 5, desc='''
-            How many rolled logs to retain in the archive.
-        ''')
-        Conf('maximum size in bytes', VDR.V_INTEGER, 1000000, desc='''
-            Suite event logs are rolled over when they reach this file size.
-        ''')
+        with Conf('logging', desc='''
+            The workflow event log, held under the suite run directory, is
+            maintained as a rolling archive. Logs are rolled over (backed up
+            and started anew) when they reach a configurable limit size.
+        '''):
+            Conf('rolling archive length', VDR.V_INTEGER, 5, desc='''
+                How many rolled logs to retain in the archive.
+            ''')
+            Conf('maximum size in bytes', VDR.V_INTEGER, 1000000, desc='''
+                Suite event logs are rolled over when they reach this
+                file size.
+            ''')
 
     with Conf('editors', desc='''
         Choose your favourite text editor for editing suite configurations.

cylc/flow/loggingutil.py

@@ -114,7 +114,6 @@ class TimestampRotatingFileHandler(logging.FileHandler):
 
     FILE_HEADER_FLAG = 'cylc_log_file_header'
     FILE_NUM = 'cylc_log_num'
-    GLBL_KEY = 'suite logging'
     MIN_BYTES = 1024
 
     def __init__(self, log_file_path, no_detach=False, timestamp=True):
@@ -141,7 +140,8 @@ def should_rollover(self, record):
         """Should rollover?"""
         if self.stamp is None or self.stream is None:
             return True
-        max_bytes = glbl_cfg().get([self.GLBL_KEY, 'maximum size in bytes'])
+        max_bytes = glbl_cfg().get(
+            ['scheduler', 'logging', 'maximum size in bytes'])
         if max_bytes < self.MIN_BYTES:  # No silly value
             max_bytes = self.MIN_BYTES
         msg = "%s\n" % self.format(record)
@@ -168,7 +168,8 @@ def do_rollover(self):
             os.unlink(self.baseFilename)
         os.symlink(os.path.basename(filename), self.baseFilename)
         # Housekeep log files
-        arch_len = glbl_cfg().get([self.GLBL_KEY, 'rolling archive length'])
+        arch_len = glbl_cfg().get(
+            ['scheduler', 'logging', 'rolling archive length'])
         if arch_len:
             log_files = glob(self.baseFilename + '.*')
             log_files.sort()

cylc/flow/pathutil.py

@@ -112,7 +112,7 @@ def make_suite_run_tree(suite):
     """Create all top-level cylc-run output dirs on the suite host."""
     cfg = glbl_cfg().get()
     # Roll archive
-    archlen = cfg['run directory rolling archive length']
+    archlen = cfg['scheduler']['run directory rolling archive length']
     dir_ = os.path.expandvars(get_suite_run_dir(suite))
     for i in range(archlen, -1, -1):  # archlen...0
         if i > 0:

cylc/flow/subprocpool.py

@@ -120,8 +120,9 @@ class SubProcPool:
     RET_CODE_SUITE_STOPPING = 999
 
     def __init__(self):
-        self.size = glbl_cfg().get(['process pool size'])
-        self.proc_pool_timeout = glbl_cfg().get(['process pool timeout'])
+        self.size = glbl_cfg().get(['scheduler', 'process pool size'])
+        self.proc_pool_timeout = glbl_cfg().get(
+            ['scheduler', 'process pool timeout'])
         self.closed = False  # Close queue
         self.stopping = False  # No more job submit if True
         # .stopping may be set by an API command in a different thread

etc/bin/run-functional-tests

@@ -24,7 +24,7 @@ Run the Cylc test battery, in <CYLC_REPO_DIR>/tests.
 
 Options and arguments are appended to "prove -j \$NPROC -s -r \${@:-tests/f}".
 NPROC is the number of concurrent processes to run, which defaults to the
-global config "process pool size" setting.
+global config "[scheduler]process pool size" setting.
 
 The tests ignore normal site/user global config and instead use:
    ~/.cylc/flow/<cylc-version>/global-tests.cylc

tests/flakyfunctional/events/44-timeout.t

@@ -22,8 +22,10 @@
 
 set_test_number 4
 
-create_test_global_config "
-process pool timeout = PT10S" ""
+create_test_global_config "" "
+[scheduler]
+    process pool timeout = PT10S
+"
 
 install_suite "${TEST_NAME_BASE}" "${TEST_NAME_BASE}"
 

tests/flakyfunctional/job-submission/18-check-chunking.t

@@ -20,9 +20,10 @@
 . "$(dirname "$0")/test_header"
 set_test_number 3
 
-create_test_global_config '
-process pool size = 1
-' ''
+create_test_global_config '' '
+[scheduler]
+    process pool size = 1
+'
 
 init_suite "${TEST_NAME_BASE}" <<'__FLOW_CONFIG__'
 [scheduler]
@@ -39,12 +40,12 @@ init_suite "${TEST_NAME_BASE}" <<'__FLOW_CONFIG__'
     [[t1<p>]]
         # Reduce the load on many jobs sending the "started" message
         init-script = """
-sleep $((RANDOM % 10))
-"""
+            sleep $((RANDOM % 10))
+        """
         script = """
-wait
-sleep $((RANDOM % 5))
-"""
+            wait
+            sleep $((RANDOM % 5))
+        """
 __FLOW_CONFIG__
 
 

tests/flakyfunctional/job-submission/19-chatty.t

@@ -21,8 +21,9 @@ export REQUIRE_PLATFORM='batch:at'
 . "$(dirname "$0")/test_header"
 set_test_number 15
 
-create_test_global_config "
-process pool timeout = PT10S" "
+create_test_global_config "" "
+[scheduler]
+    process pool timeout = PT10S
 [platforms]
     [[$CYLC_TEST_PLATFORM]]
         batch submit command template = talkingnonsense %(job)s

tests/functional/events/47-long-output.t

@@ -24,8 +24,10 @@ fi
 
 set_test_number 10
 
-create_test_global_config "
-process pool timeout = PT10S" ""
+create_test_global_config "" "
+[scheduler]
+    process pool timeout = PT10S
+"
 
 # Long STDOUT output
 

tests/functional/job-submission/16-timeout.t

@@ -20,9 +20,9 @@ export REQUIRE_PLATFORM='batch:at comms:tcp'
 . "$(dirname "$0")/test_header"
 set_test_number 4
 
-create_test_global_config "
-process pool timeout = PT10S
-" "
+create_test_global_config "" "
+[scheduler]
+    process pool timeout = PT10S
 [platforms]
     [[$CYLC_TEST_PLATFORM]]
         batch submit command template = sleep 30

tests/functional/lib/bash/test_header

@@ -121,8 +121,8 @@
 #         Expect 2 OK tests.
 #     create_test_global_config [PRE [POST]]
 #         Create a new global config file $PWD/etc from global-tests.cylc
-#         with PRE and POST pre- and ap-pended (PRE for top level items with no
-#         section heading). PRE and POST are strings.
+#         with PRE and POST pre- and ap-pended (PRE for e.g. jinja2 shebang).
+#         PRE and POST are strings.
 #     localhost_fqdn
 #         Get the FQDN of the current host using the same mechanism Cylc uses.
 #     get_fqdn [TARGET]

tests/functional/logging/03-roll.t

@@ -35,9 +35,11 @@ init_suite "${TEST_NAME_BASE}" <<'__FLOW_CONFIG__'
 __FLOW_CONFIG__
 
 create_test_global_config '' '
-[suite logging]
-    rolling archive length = 8
-    maximum size in bytes = 2048'
+[scheduler]
+    [[logging]]
+        rolling archive length = 8
+        maximum size in bytes = 2048
+'
 run_ok "${TEST_NAME_BASE}-validate" \
     cylc validate "${SUITE_NAME}"
 suite_run_ok "${TEST_NAME_BASE}-run" \

tests/unit/test_pathutil.py

@@ -162,7 +162,8 @@ def test_get_suite_run_names(self, mocked_platform):
 )
 def test_make_suite_run_tree(caplog, tmpdir, mock_glbl_cfg, subdir):
     glbl_conf_str = f'''
-        run directory rolling archive length = 1
+        [scheduler]
+            run directory rolling archive length = 1
         [platforms]
             [[localhost]]
                 run directory = {tmpdir}