diff --git a/cylc/flow/cfgspec/globalcfg.py b/cylc/flow/cfgspec/globalcfg.py index db68cb496c4..6e40858b6c2 100644 --- a/cylc/flow/cfgspec/globalcfg.py +++ b/cylc/flow/cfgspec/globalcfg.py @@ -75,32 +75,31 @@ Prior to Cylc 8, ``global.cylc`` was named ``global.rc``, but that name is no longer supported. ''') as SPEC: - - # suite - Conf('process pool size', VDR.V_INTEGER, 4, desc=''' - Maximum number of concurrent processes used to execute external job - submission, event handlers, and job poll and kill commands - see - :ref:`Managing External Command Execution`. - ''') - Conf('process pool timeout', VDR.V_INTERVAL, DurationFloat(600), desc=''' - Interval after which long-running commands in the process pool will be - killed - see :ref:`Managing External Command Execution`. - - .. note:: - The default is set quite high to avoid killing important - processes when the system is under load. - ''') - Conf('run directory rolling archive length', VDR.V_INTEGER, -1, desc=''' - The number of old run directory trees to retain at start-up. - ''') - with Conf('scheduler', desc=''' Default values for entries in :cylc:conf:`flow.cylc[scheduler]` section. This should not be confused with scheduling in the ``flow.cylc`` file. '''): Conf('UTC mode', VDR.V_BOOLEAN, False, desc=''' - Default for :cylc:conf:`flow.cylc[scheduler]UTC mode`. + Default for :cylc:conf:`flow.cylc[scheduler]UTC mode`. + ''') + Conf('process pool size', VDR.V_INTEGER, 4, desc=''' + Maximum number of concurrent processes used to execute external job + submission, event handlers, and job poll and kill commands - see + :ref:`Managing External Command Execution`. + ''') + Conf('process pool timeout', VDR.V_INTERVAL, DurationFloat(600), + desc=''' + Interval after which long-running commands in the process pool + will be killed - see :ref:`Managing External Command Execution`. + + .. note:: + The default is set quite high to avoid killing important + processes when the system is under load. + ''') + Conf('run directory rolling archive length', VDR.V_INTEGER, -1, + desc=''' + The number of old run directory trees to retain at start-up. ''') with Conf('events', desc=''' @@ -171,17 +170,18 @@ The interval with which this plugin is run. ''') - with Conf('suite logging', desc=''' - The suite event log, held under the suite run directory, is maintained - as a rolling archive. Logs are rolled over (backed up and started anew) - when they reach a configurable limit size. - '''): - Conf('rolling archive length', VDR.V_INTEGER, 5, desc=''' - How many rolled logs to retain in the archive. - ''') - Conf('maximum size in bytes', VDR.V_INTEGER, 1000000, desc=''' - Suite event logs are rolled over when they reach this file size. - ''') + with Conf('logging', desc=''' + The workflow event log, held under the suite run directory, is + maintained as a rolling archive. Logs are rolled over (backed up + and started anew) when they reach a configurable limit size. + '''): + Conf('rolling archive length', VDR.V_INTEGER, 5, desc=''' + How many rolled logs to retain in the archive. + ''') + Conf('maximum size in bytes', VDR.V_INTEGER, 1000000, desc=''' + Suite event logs are rolled over when they reach this + file size. + ''') with Conf('editors', desc=''' Choose your favourite text editor for editing suite configurations. diff --git a/cylc/flow/loggingutil.py b/cylc/flow/loggingutil.py index def108a912e..8a3a0794cbf 100644 --- a/cylc/flow/loggingutil.py +++ b/cylc/flow/loggingutil.py @@ -114,7 +114,6 @@ class TimestampRotatingFileHandler(logging.FileHandler): FILE_HEADER_FLAG = 'cylc_log_file_header' FILE_NUM = 'cylc_log_num' - GLBL_KEY = 'suite logging' MIN_BYTES = 1024 def __init__(self, log_file_path, no_detach=False, timestamp=True): @@ -141,7 +140,8 @@ def should_rollover(self, record): """Should rollover?""" if self.stamp is None or self.stream is None: return True - max_bytes = glbl_cfg().get([self.GLBL_KEY, 'maximum size in bytes']) + max_bytes = glbl_cfg().get( + ['scheduler', 'logging', 'maximum size in bytes']) if max_bytes < self.MIN_BYTES: # No silly value max_bytes = self.MIN_BYTES msg = "%s\n" % self.format(record) @@ -168,7 +168,8 @@ def do_rollover(self): os.unlink(self.baseFilename) os.symlink(os.path.basename(filename), self.baseFilename) # Housekeep log files - arch_len = glbl_cfg().get([self.GLBL_KEY, 'rolling archive length']) + arch_len = glbl_cfg().get( + ['scheduler', 'logging', 'rolling archive length']) if arch_len: log_files = glob(self.baseFilename + '.*') log_files.sort() diff --git a/cylc/flow/pathutil.py b/cylc/flow/pathutil.py index dc336611fdf..45f38fe994c 100644 --- a/cylc/flow/pathutil.py +++ b/cylc/flow/pathutil.py @@ -112,7 +112,7 @@ def make_suite_run_tree(suite): """Create all top-level cylc-run output dirs on the suite host.""" cfg = glbl_cfg().get() # Roll archive - archlen = cfg['run directory rolling archive length'] + archlen = cfg['scheduler']['run directory rolling archive length'] dir_ = os.path.expandvars(get_suite_run_dir(suite)) for i in range(archlen, -1, -1): # archlen...0 if i > 0: diff --git a/cylc/flow/subprocpool.py b/cylc/flow/subprocpool.py index 8f987bc84da..845781cc8b8 100644 --- a/cylc/flow/subprocpool.py +++ b/cylc/flow/subprocpool.py @@ -120,8 +120,9 @@ class SubProcPool: RET_CODE_SUITE_STOPPING = 999 def __init__(self): - self.size = glbl_cfg().get(['process pool size']) - self.proc_pool_timeout = glbl_cfg().get(['process pool timeout']) + self.size = glbl_cfg().get(['scheduler', 'process pool size']) + self.proc_pool_timeout = glbl_cfg().get( + ['scheduler', 'process pool timeout']) self.closed = False # Close queue self.stopping = False # No more job submit if True # .stopping may be set by an API command in a different thread diff --git a/etc/bin/run-functional-tests b/etc/bin/run-functional-tests index 0a29db2e0e7..a66dc576b7a 100755 --- a/etc/bin/run-functional-tests +++ b/etc/bin/run-functional-tests @@ -24,7 +24,7 @@ Run the Cylc test battery, in /tests. Options and arguments are appended to "prove -j \$NPROC -s -r \${@:-tests/f}". NPROC is the number of concurrent processes to run, which defaults to the -global config "process pool size" setting. +global config "[scheduler]process pool size" setting. The tests ignore normal site/user global config and instead use: ~/.cylc/flow//global-tests.cylc diff --git a/tests/flakyfunctional/events/44-timeout.t b/tests/flakyfunctional/events/44-timeout.t index 92faf3d1115..b971f1102d0 100755 --- a/tests/flakyfunctional/events/44-timeout.t +++ b/tests/flakyfunctional/events/44-timeout.t @@ -22,8 +22,10 @@ set_test_number 4 -create_test_global_config " -process pool timeout = PT10S" "" +create_test_global_config "" " +[scheduler] + process pool timeout = PT10S +" install_suite "${TEST_NAME_BASE}" "${TEST_NAME_BASE}" diff --git a/tests/flakyfunctional/job-submission/18-check-chunking.t b/tests/flakyfunctional/job-submission/18-check-chunking.t index 92b04a2363c..bfe1ab89ac8 100644 --- a/tests/flakyfunctional/job-submission/18-check-chunking.t +++ b/tests/flakyfunctional/job-submission/18-check-chunking.t @@ -20,9 +20,10 @@ . "$(dirname "$0")/test_header" set_test_number 3 -create_test_global_config ' -process pool size = 1 -' '' +create_test_global_config '' ' +[scheduler] + process pool size = 1 +' init_suite "${TEST_NAME_BASE}" <<'__FLOW_CONFIG__' [scheduler] @@ -39,12 +40,12 @@ init_suite "${TEST_NAME_BASE}" <<'__FLOW_CONFIG__' [[t1

]] # Reduce the load on many jobs sending the "started" message init-script = """ -sleep $((RANDOM % 10)) -""" + sleep $((RANDOM % 10)) + """ script = """ -wait -sleep $((RANDOM % 5)) -""" + wait + sleep $((RANDOM % 5)) + """ __FLOW_CONFIG__ diff --git a/tests/flakyfunctional/job-submission/19-chatty.t b/tests/flakyfunctional/job-submission/19-chatty.t index d7d028bbd70..948113b8346 100755 --- a/tests/flakyfunctional/job-submission/19-chatty.t +++ b/tests/flakyfunctional/job-submission/19-chatty.t @@ -21,8 +21,9 @@ export REQUIRE_PLATFORM='batch:at' . "$(dirname "$0")/test_header" set_test_number 15 -create_test_global_config " -process pool timeout = PT10S" " +create_test_global_config "" " +[scheduler] + process pool timeout = PT10S [platforms] [[$CYLC_TEST_PLATFORM]] batch submit command template = talkingnonsense %(job)s diff --git a/tests/functional/events/47-long-output.t b/tests/functional/events/47-long-output.t index 8a078b51839..5d499005565 100755 --- a/tests/functional/events/47-long-output.t +++ b/tests/functional/events/47-long-output.t @@ -24,8 +24,10 @@ fi set_test_number 10 -create_test_global_config " -process pool timeout = PT10S" "" +create_test_global_config "" " +[scheduler] + process pool timeout = PT10S +" # Long STDOUT output diff --git a/tests/functional/job-submission/16-timeout.t b/tests/functional/job-submission/16-timeout.t index 9776e3e55cc..73e9c2743a8 100755 --- a/tests/functional/job-submission/16-timeout.t +++ b/tests/functional/job-submission/16-timeout.t @@ -20,9 +20,9 @@ export REQUIRE_PLATFORM='batch:at comms:tcp' . "$(dirname "$0")/test_header" set_test_number 4 -create_test_global_config " -process pool timeout = PT10S -" " +create_test_global_config "" " +[scheduler] + process pool timeout = PT10S [platforms] [[$CYLC_TEST_PLATFORM]] batch submit command template = sleep 30 diff --git a/tests/functional/lib/bash/test_header b/tests/functional/lib/bash/test_header index 527ad835cce..ebb4c606d1b 100644 --- a/tests/functional/lib/bash/test_header +++ b/tests/functional/lib/bash/test_header @@ -121,8 +121,8 @@ # Expect 2 OK tests. # create_test_global_config [PRE [POST]] # Create a new global config file $PWD/etc from global-tests.cylc -# with PRE and POST pre- and ap-pended (PRE for top level items with no -# section heading). PRE and POST are strings. +# with PRE and POST pre- and ap-pended (PRE for e.g. jinja2 shebang). +# PRE and POST are strings. # localhost_fqdn # Get the FQDN of the current host using the same mechanism Cylc uses. # get_fqdn [TARGET] diff --git a/tests/functional/logging/03-roll.t b/tests/functional/logging/03-roll.t index 9506cfebdef..f67eeb31504 100755 --- a/tests/functional/logging/03-roll.t +++ b/tests/functional/logging/03-roll.t @@ -35,9 +35,11 @@ init_suite "${TEST_NAME_BASE}" <<'__FLOW_CONFIG__' __FLOW_CONFIG__ create_test_global_config '' ' -[suite logging] - rolling archive length = 8 - maximum size in bytes = 2048' +[scheduler] + [[logging]] + rolling archive length = 8 + maximum size in bytes = 2048 +' run_ok "${TEST_NAME_BASE}-validate" \ cylc validate "${SUITE_NAME}" suite_run_ok "${TEST_NAME_BASE}-run" \ diff --git a/tests/unit/test_pathutil.py b/tests/unit/test_pathutil.py index 16c0211dcec..73c5214d007 100644 --- a/tests/unit/test_pathutil.py +++ b/tests/unit/test_pathutil.py @@ -162,7 +162,8 @@ def test_get_suite_run_names(self, mocked_platform): ) def test_make_suite_run_tree(caplog, tmpdir, mock_glbl_cfg, subdir): glbl_conf_str = f''' - run directory rolling archive length = 1 + [scheduler] + run directory rolling archive length = 1 [platforms] [[localhost]] run directory = {tmpdir}