From e34fcf860e5cf61ac347c9dcce0eaa9596d9c4d2 Mon Sep 17 00:00:00 2001 From: Hilary James Oliver Date: Mon, 11 May 2020 11:54:28 +1200 Subject: [PATCH 1/3] Always store point dump format in DB. --- cylc/flow/config.py | 8 +++++--- cylc/flow/cycling/integer.py | 6 ++++++ cylc/flow/cycling/iso8601.py | 5 +++++ cylc/flow/cycling/loader.py | 8 ++++++++ cylc/flow/scripts/cylc_validate.py | 1 - cylc/flow/suite_db_mgr.py | 4 ++-- 6 files changed, 26 insertions(+), 6 deletions(-) diff --git a/cylc/flow/config.py b/cylc/flow/config.py index 409baaa27b9..cc18109a133 100644 --- a/cylc/flow/config.py +++ b/cylc/flow/config.py @@ -50,7 +50,7 @@ from cylc.flow.cycling.loader import ( get_point, get_point_relative, get_interval, get_interval_cls, get_sequence, get_sequence_cls, init_cyclers, INTEGER_CYCLING_TYPE, - ISO8601_CYCLING_TYPE) + ISO8601_CYCLING_TYPE, get_dump_format) from cylc.flow.cycling.iso8601 import ingest_time import cylc.flow.flags from cylc.flow.graphnode import GraphNodeParser @@ -311,6 +311,8 @@ def __init__( # after the call to init_cyclers, we can start getting proper points. init_cyclers(self.cfg) + self.cycling_type = get_interval_cls().get_null().TYPE + self.cycle_point_dump_format = get_dump_format(self.cycling_type) # Running in UTC time? (else just use the system clock) if self.cfg['cylc']['UTC mode'] is None: @@ -374,7 +376,7 @@ def __init__( fcp_str = self.cfg['scheduling']['final cycle point'] if fcp_str is not None: # Is the final "point"(/interval) relative to initial? - if get_interval_cls().get_null().TYPE == INTEGER_CYCLING_TYPE: + if self.cycling_type == INTEGER_CYCLING_TYPE: if "P" in fcp_str: # Relative, integer cycling. self.final_point = get_point_relative( @@ -1050,7 +1052,7 @@ def compute_runahead_limits(self): if not limit: limit = None if (limit is not None and limit.isdigit() and - get_interval_cls().get_null().TYPE == ISO8601_CYCLING_TYPE): + self.cycling_type == ISO8601_CYCLING_TYPE): # Backwards-compatibility for raw number of hours. limit = "PT%sH" % limit diff --git a/cylc/flow/cycling/integer.py b/cylc/flow/cycling/integer.py index bb8bfe5274b..1a88f7aed10 100644 --- a/cylc/flow/cycling/integer.py +++ b/cylc/flow/cycling/integer.py @@ -595,6 +595,12 @@ def init_from_cfg(_): pass +def get_dump_format(cycling_type=None): + """Return cycle point string dump format.""" + # Not used for integer cycling. + return None + + def get_point_relative(point_expr, context_point): """Create a point from relative_string applied to base_point.""" if REC_RELATIVE_POINT.search(point_expr): diff --git a/cylc/flow/cycling/iso8601.py b/cylc/flow/cycling/iso8601.py index 54b67f98365..2c98048eff0 100644 --- a/cylc/flow/cycling/iso8601.py +++ b/cylc/flow/cycling/iso8601.py @@ -843,6 +843,11 @@ def init(num_expanded_year_digits=0, custom_dump_format=None, time_zone=None, ) +def get_dump_format(): + """Return cycle point string dump format.""" + return SuiteSpecifics.DUMP_FORMAT + + def get_point_relative(offset_string, base_point): """Create a point from offset_string applied to base_point.""" try: diff --git a/cylc/flow/cycling/loader.py b/cylc/flow/cycling/loader.py index f7c3e0e220b..7451e8e3762 100644 --- a/cylc/flow/cycling/loader.py +++ b/cylc/flow/cycling/loader.py @@ -36,6 +36,9 @@ POINTS = {INTEGER_CYCLING_TYPE: integer.IntegerPoint, ISO8601_CYCLING_TYPE: iso8601.ISO8601Point} +DUMP_FORMAT_GETTERS = {INTEGER_CYCLING_TYPE: integer.get_dump_format, + ISO8601_CYCLING_TYPE: iso8601.get_dump_format} + POINT_RELATIVE_GETTERS = { INTEGER_CYCLING_TYPE: integer.get_point_relative, ISO8601_CYCLING_TYPE: iso8601.get_point_relative @@ -73,6 +76,11 @@ def get_point_cls(cycling_type=None): return POINTS[cycling_type] +def get_dump_format(cycling_type=None): + """Return cycle point dump format, or None.""" + return DUMP_FORMAT_GETTERS[cycling_type]() + + def get_point_relative(*args, **kwargs): """Return a point from an offset expression and a base point.""" cycling_type = kwargs.pop("cycling_type", DefaultCycler.TYPE) diff --git a/cylc/flow/scripts/cylc_validate.py b/cylc/flow/scripts/cylc_validate.py index c1247ab65c6..1cd7e65f7ce 100755 --- a/cylc/flow/scripts/cylc_validate.py +++ b/cylc/flow/scripts/cylc_validate.py @@ -82,7 +82,6 @@ def parse_args(): @cli_function(parse_args) def main(_, options, reg): """cylc validate CLI.""" - # import pdb; pdb.set_trace() profiler = Profiler(options.profile_mode) profiler.start() diff --git a/cylc/flow/suite_db_mgr.py b/cylc/flow/suite_db_mgr.py index 2d55844efef..fa3ec70bc64 100644 --- a/cylc/flow/suite_db_mgr.py +++ b/cylc/flow/suite_db_mgr.py @@ -291,10 +291,10 @@ def put_suite_params(self, schd): {"key": "cylc_version", "value": CYLC_VERSION}, {"key": "UTC_mode", "value": get_utc_mode()}, ]) - if schd.config.cfg['cylc']['cycle point format']: + if schd.config.cycle_point_dump_format is not None: self.db_inserts_map[self.TABLE_SUITE_PARAMS].append({ "key": "cycle_point_format", - "value": schd.config.cfg['cylc']['cycle point format']}) + "value": schd.config.cycle_point_dump_format}) if schd.pool.is_held: self.db_inserts_map[self.TABLE_SUITE_PARAMS].append({ "key": self.KEY_HOLD, "value": 1}) From a6b6bb4c11265cabd7ab660648366c503d1bd7a3 Mon Sep 17 00:00:00 2001 From: Hilary James Oliver Date: Mon, 11 May 2020 13:50:07 +1200 Subject: [PATCH 2/3] Add functional test for DB cycle point format. --- tests/suite-state/06-format.t | 20 ++++++++--- tests/suite-state/06a-noformat.t | 56 +++++++++++++++++++++++++++++++ tests/suite-state/format/suite.rc | 13 ------- 3 files changed, 72 insertions(+), 17 deletions(-) create mode 100755 tests/suite-state/06a-noformat.t delete mode 100644 tests/suite-state/format/suite.rc diff --git a/tests/suite-state/06-format.t b/tests/suite-state/06-format.t index ad06f53b395..1649937e2ce 100755 --- a/tests/suite-state/06-format.t +++ b/tests/suite-state/06-format.t @@ -15,24 +15,36 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . #------------------------------------------------------------------------------- -# Test cylc suite-state "template" option +# Test "cylc suite-state" cycle point format conversion, when the target suite +# sets an explicit cycle point format, and the CLI does not. . "$(dirname "$0")/test_header" #------------------------------------------------------------------------------- set_test_number 5 #------------------------------------------------------------------------------- -install_suite "${TEST_NAME_BASE}" format +init_suite "${TEST_NAME_BASE}" <<'__SUITERC__' +[cylc] + UTC mode = True + cycle point format = CCYY-MM-DD +[scheduling] + initial cycle point = 20100101 +[[graph]] + R1 = foo +[runtime] + [[foo]] + script = true +__SUITERC__ #------------------------------------------------------------------------------- TEST_NAME="${TEST_NAME_BASE}-run" suite_run_ok "${TEST_NAME}" cylc run --debug --no-detach "${SUITE_NAME}" #------------------------------------------------------------------------------- -TEST_NAME=${TEST_NAME_BASE}-cli-template-poll +TEST_NAME=${TEST_NAME_BASE}-cli-poll run_ok "${TEST_NAME}" cylc suite-state "${SUITE_NAME}" -p 20100101T0000Z \ --task=foo --status=succeeded contains_ok "${TEST_NAME}.stdout" <<__OUT__ polling for 'succeeded': satisfied __OUT__ #------------------------------------------------------------------------------- -TEST_NAME=${TEST_NAME_BASE}-cli-template-dump +TEST_NAME=${TEST_NAME_BASE}-cli-dump run_ok "${TEST_NAME}" cylc suite-state "${SUITE_NAME}" -p 20100101T0000Z contains_ok "${TEST_NAME}.stdout" <<__OUT__ foo, 2010-01-01, succeeded diff --git a/tests/suite-state/06a-noformat.t b/tests/suite-state/06a-noformat.t new file mode 100755 index 00000000000..403f75b5e0d --- /dev/null +++ b/tests/suite-state/06a-noformat.t @@ -0,0 +1,56 @@ +#!/bin/bash +# THIS FILE IS PART OF THE CYLC SUITE ENGINE. +# Copyright (C) NIWA & British Crown (Met Office) & Contributors. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +#------------------------------------------------------------------------------- +# Test "cylc suite-state" cycle point format conversion, when the target suite +# sets no explicit cycle point format, and the CLI does (the reverse of 06.t). + +. "$(dirname "$0")/test_header" +#------------------------------------------------------------------------------- +set_test_number 5 +#------------------------------------------------------------------------------- +init_suite "${TEST_NAME_BASE}" <<'__SUITERC__' +[cylc] + UTC mode = True + # (Use default cycle point format) +[scheduling] + initial cycle point = 20100101T0000Z +[[graph]] + R1 = foo +[runtime] + [[foo]] + script = true +__SUITERC__ +#------------------------------------------------------------------------------- +TEST_NAME="${TEST_NAME_BASE}-run" +suite_run_ok "${TEST_NAME}" cylc run --debug --no-detach "${SUITE_NAME}" +#------------------------------------------------------------------------------- +TEST_NAME=${TEST_NAME_BASE}-cli-poll +run_ok "${TEST_NAME}" cylc suite-state "${SUITE_NAME}" -p 2010-01-01T00:00Z \ + --task=foo --status=succeeded +contains_ok "${TEST_NAME}.stdout" <<__OUT__ +polling for 'succeeded': satisfied +__OUT__ +#------------------------------------------------------------------------------- +TEST_NAME=${TEST_NAME_BASE}-cli-dump +run_ok "${TEST_NAME}" cylc suite-state "${SUITE_NAME}" -p 2010-01-01T00:00Z +contains_ok "${TEST_NAME}.stdout" <<__OUT__ +foo, 20100101T0000Z, succeeded +__OUT__ +#------------------------------------------------------------------------------- +purge_suite "${SUITE_NAME}" +#------------------------------------------------------------------------------- +exit 0 diff --git a/tests/suite-state/format/suite.rc b/tests/suite-state/format/suite.rc deleted file mode 100644 index 84ccd865335..00000000000 --- a/tests/suite-state/format/suite.rc +++ /dev/null @@ -1,13 +0,0 @@ -[cylc] - UTC mode = True - cycle point format = CCYY-MM-DD - -[scheduling] - initial cycle point = 20100101 - -[[graph]] - R1 = foo - -[runtime] - [[foo]] - script = true From 26e3db646030779a6d9e342cedb84862cb7f4b39 Mon Sep 17 00:00:00 2001 From: Hilary James Oliver Date: Mon, 11 May 2020 14:07:37 +1200 Subject: [PATCH 3/3] Update change log. --- CHANGES.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CHANGES.md b/CHANGES.md index ea9a0a05323..49008dcb7cd 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -38,6 +38,11 @@ namespace, under `cylc.jinja.filters`. [#3541](https://github.com/cylc/cylc-flow/pull/3541) - Don't warn that a task was already added to an internal queue, if the queue is the same. +[#3602](https://github.com/cylc/cylc-flow/pull/3602) - Fix a bug that prevented +cycle point format conversion by the `cylc suite-state` command and the +`suite_state` xtrigger function, if the target suite used the default format +but downstream command or suite did not. + ------------------------------------------------------------------------------- ## __cylc-8.0a2 (2019-Q4?)__