Test.sim mode changes on restart (#5789)

* Tests to ensure simulation mode is conserved across restarts of the scheduler.

* Test what happens to workflow run_mode upon restart.
Raise an exception if the user tries to change mode.

* default to live if run mode is none is db

* test mode = None on restart

changes.d/5789.fix.md

@@ -0,0 +1 @@
+Stop users changing run modes on restart.

cylc/flow/rundb.py

@@ -606,6 +606,19 @@ def select_workflow_params_restart_count(self):
         result = self.connect().execute(stmt).fetchone()
         return int(result[0]) if result else 0
 
+    def select_workflow_params_run_mode(self):
+        """Return original run_mode for workflow_params."""
+        stmt = rf"""
+            SELECT
+                value
+            FROM
+                {self.TABLE_WORKFLOW_PARAMS}
+            WHERE
+                key == 'run_mode'
+        """  # nosec (table name is code constant)
+        result = self.connect().execute(stmt).fetchone()
+        return result[0] if result else None
+
     def select_workflow_template_vars(self, callback):
         """Select from workflow_template_vars.
 

cylc/flow/scheduler.py

@@ -434,6 +434,18 @@ async def configure(self, params):
             # Mark this exc as expected (see docstring for .schd_expected):
             exc.schd_expected = True
             raise exc
+
+        # Prevent changing mode on restart.
+        if self.is_restart:
+            # check run mode against db
+            og_run_mode = self.workflow_db_mgr.get_pri_dao(
+            ).select_workflow_params_run_mode() or 'live'
+            run_mode = self.config.run_mode()
+            if run_mode != og_run_mode:
+                raise InputError(
+                    f'This workflow was originally run in {og_run_mode} mode:'
+                    f' Will not restart in {run_mode} mode.')
+
         self.profiler.log_memory("scheduler.py: after load_flow_file")
 
         self.workflow_db_mgr.on_workflow_start(self.is_restart)

tests/integration/test_mode_on_restart.py

@@ -0,0 +1,59 @@
+# THIS FILE IS PART OF THE CYLC WORKFLOW ENGINE.
+# Copyright (C) NIWA & British Crown (Met Office) & Contributors.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+"""What happens to the mode on restart?
+"""
+
+import pytest
+
+from cylc.flow.exceptions import InputError
+
+
+MODES = [('live'), ('simulation'), ('dummy')]
+
+
+@pytest.mark.parametrize('mode_before', MODES + [None])
+@pytest.mark.parametrize('mode_after', MODES)
+async def test_restart_mode(
+    flow, run, scheduler, start, one_conf,
+    mode_before, mode_after
+):
+    """Restarting a workflow in live mode leads to workflow in live mode.
+
+    N.B - we need use run becuase the check in question only happens
+    on start.
+    """
+    id_ = flow(one_conf)
+    schd = scheduler(id_, run_mode=mode_before)
+    async with start(schd):
+        if not mode_before:
+            mode_before = 'live'
+        assert schd.config.run_mode() == mode_before
+
+    schd = scheduler(id_, run_mode=mode_after)
+
+    if (
+        mode_before == mode_after
+        or not mode_before and mode_after != 'live'
+    ):
+        # Restarting in the same mode is fine.
+        async with run(schd):
+            assert schd.config.run_mode() == mode_before
+    else:
+        # Restarting in a new mode is not:
+        errormsg = f'^This.*{mode_before} mode: Will.*{mode_after} mode.$'
+        with pytest.raises(InputError, match=errormsg):
+            async with run(schd):
+                pass