Reduce Rose-Cylc interaction part 1

The `--host=HOST` option for Cylc commands should now be redundant.
Load suite contact file to determine whether a suite is running or not.
Find suite contact files to list running suites.
Modify `rose suite-gcontrol --all` to launch `cylc gscan`.
Modify `rose suite-scan` to launch `cylc scan` directly.
Rose suite automatic/custom environment variables now go to suite.rc.
New ROSE_SITE environment variable + site=SITE setting

bin/rose-suite-gcontrol

@@ -41,8 +41,6 @@
 # OPTIONS
 #     --all
 #         Open a suite control GUI for each running suite.
-#     --host=HOST
-#         Specify a host.
 #     --name=SUITE-NAME
 #         Specify the suite name.
 #     --quiet, -q

bin/rose-suite-scan

@@ -21,18 +21,9 @@
 #     rose suite-scan
 #
 # SYNOPSIS
-#     rose suite-scan [HOST ...]
+#     rose suite-scan [...]
 #
 # DESCRIPTION
-#     Scan for running suites.
-#
-#     If no HOST specified, use `[rose-suite-run] hosts` setting in site / user
-#     configuration. If no configuration defined, use `localhost`.
-#
-# OPTIONS
-#     --quiet, -q
-#         Decrement verbosity.
-#     --verbose, -v
-#         Increment verbosity.
+#     Run `cylc scan [...]`.
 #-------------------------------------------------------------------------------
-exec python -m rose.suite_scan "$@"
+exec cylc scan "$@"

bin/rose-suite-shutdown

@@ -42,8 +42,6 @@
 #     --all
 #         Shutdown all running suites. You will be prompted to confirm shutting
 #         down each affected suite.
-#     --host=HOST
-#         Specify a host.
 #     --name=SUITE-NAME
 #         Specify the suite name.
 #     --non-interactive, --yes, -y

etc/rose-bash-completion

@@ -318,24 +318,12 @@ _rose_suite_run__run() {
     COMPREPLY=( $(compgen -W "reload restart run" -- $1) )
 }
 
-_rose_suite_shutdown__host() {
-    local hosts
-    hosts=$(rose config rose-suite-run hosts 2>/dev/null)
-    hosts=$(__rose_get_expanded_hosts $hosts)
-    COMPREPLY=( $(compgen -W "$hosts" -- $1) )
-}
-
 _rose_suite_shutdown__name() {
     local names
     names=$(cylc print -xy 2>/dev/null)
     COMPREPLY=( $(compgen -W "$names" -- $1) )
 }
 
-_rose_suite_stop__host() {
-    _rose_suite_shutdown__host "$@"
-    return $?
-}
-
 _rose_suite_stop__name() {
     _rose_suite_shutdown__name "$@"
     return $?

etc/rose.conf.example

@@ -19,6 +19,8 @@ meta-path=DIR1[:DIR2[:...]]
 #
 # URL to Rose documentation.
 rose-doc=http://metomi.github.io/rose/
+# Site name, used by suite configuration for portability.
+site=SITE-NAME
 
 
 # Configuration of external commands.

etc/tutorial/cylc-forecasting-suite/.validate

@@ -1,3 +1,5 @@
 cylc validate "$TUT_DIR" -v -v --icp=2000
 rose tutorial "$(basename $TUT_DIR)" "${TEST_DIR}/cylc-forecasting-suite"
-cylc run "${TEST_DIR}/cylc-forecasting-suite" $(rose date --utc --offset -PT7H --format CCYY-MM-DDThh:00Z) --no-detach --until $(rose date --utc --offset -PT1H --format CCYY-MM-DDThh:00Z) 2>&1 >/dev/null | grep '\[post_process_exeter..*\] -(current:running)> succeeded at .*'
+cylc register "$(basename "${TEST_DIR}")/cylc-forecasting-suite" "${TEST_DIR}/cylc-forecasting-suite"
+cylc run "$(basename "${TEST_DIR}")/cylc-forecasting-suite" $(rose date --utc --offset -PT7H --format CCYY-MM-DDThh:00Z) --no-detach --until $(rose date --utc --offset -PT1H --format CCYY-MM-DDThh:00Z) 2>&1 >/dev/null | grep '\[post_process_exeter..*\] -(current:running)> succeeded at .*'
+rose suite-clean -y "$(basename "${TEST_DIR}")/cylc-forecasting-suite"

etc/tutorial/cylc-forecasting-suite/suite.rc

@@ -1,4 +1,5 @@
 [cylc]
+    abort if any task fails = True
     UTC mode = True
     [[parameters]]
         # A list of the weather stations we will be fetching observations from.

lib/python/rose/config_processor.py

@@ -79,17 +79,20 @@ def __init__(self, *args, **kwargs):
         if self.SCHEME is not None:
             self.PREFIX = self.SCHEME + ":"
 
-    def process(self, conf_tree, item, orig_keys=None, orig_value=None):
+    def process(self, conf_tree, item, orig_keys=None, orig_value=None,
+                **kwargs):
         """Sub-class should override this method.
 
-        conf_tree:
-            The relevant rose.config_tree.ConfigTree object with the full
-            configuration.
-        item: The current configuration item to process.
-        orig_keys:
-            The keys for locating the originating setting in conf_tree in a
-            recursive processing. None implies a top level call.
-        orig_value: The value of orig_keys in conf_tree.
+        Arguments:
+            conf_tree:
+                The relevant rose.config_tree.ConfigTree object with the full
+                configuration.
+            item: The current configuration item to process.
+            orig_keys:
+                The keys for locating the originating setting in conf_tree in a
+                recursive processing. None implies a top level call.
+            orig_value: The value of orig_keys in conf_tree.
+            **kwargs: Extra keyword arguments that may be useful.
         """
         pass
 

lib/python/rose/config_processors/jinja2.py

@@ -38,8 +38,22 @@ class ConfigProcessorForJinja2(ConfigProcessorBase):
     MSG_DONE = "Rose Configuration Insertion: Done"
     MSG_INIT = "Rose Configuration Insertion: Init"
 
-    def process(self, conf_tree, item, orig_keys=None, orig_value=None, **_):
-        """Process [jinja2:*] in "conf_tree.node"."""
+    def process(self, conf_tree, item, orig_keys=None, orig_value=None,
+                **kwargs):
+        """Process [jinja2:*] in "conf_tree.node".
+
+        Arguments:
+            conf_tree:
+                The relevant rose.config_tree.ConfigTree object with the full
+                configuration.
+            item: The current configuration item to process.
+            orig_keys:
+                The keys for locating the originating setting in conf_tree in a
+                recursive processing. None implies a top level call.
+            orig_value: The value of orig_keys in conf_tree.
+            **kwargs:
+                environ (dict): suite level environment variables.
+        """
         for s_key, s_node in sorted(conf_tree.node.value.items()):
             if (s_node.is_ignored() or
                     not s_key.startswith(self.PREFIX) or
@@ -63,6 +77,12 @@ def process(self, conf_tree, item, orig_keys=None, orig_value=None, **_):
                 except UnboundEnvironmentVariableError as exc:
                     raise ConfigProcessError([s_key, key], node.value, exc)
                 tmp_file.write(self.ASSIGN_TEMPL % (key, value))
+            environ = kwargs.get("environ")
+            if environ:
+                tmp_file.write('[cylc]\n')
+                tmp_file.write('    [[environment]]\n')
+                for key, value in sorted(environ.items()):
+                    tmp_file.write('        %s=%s\n' % (key, value))
             tmp_file.write(msg_done_ln)
             line_n = 0
             is_in_old_insert = False

lib/python/rose/popen.py

@@ -190,6 +190,33 @@ def run_bg(self, *args, **kwargs):
             raise RosePopenError(args, 1, "", str(exc))
         return proc
 
+    def run_nohup_gui(self, cmd):
+        """Launch+detach a GUI command with nohup.
+
+        Launch the GUI command with `nohup bash -c 'exec ...'` redirecting
+        standard input and outputs from and to `/dev/null`, and setting it to
+        become a process group leader. Equivalent to a double fork.
+
+        Arguments:
+            cmd (str): command string of the GUI.
+
+        Return (subprocess.Popen):
+            Return the process object for the "nohup" command.
+            Return None if no DISPLAY is set.
+        """
+        if 'DISPLAY' not in os.environ:
+            return
+        return self.run_bg(
+            r'nohup',
+            r'bash',
+            r'-c',
+            r'exec ' + cmd + r' <"/dev/null" >"/dev/null" 2>&1',
+            preexec_fn=os.setpgrp,
+            stdin=open(os.devnull),
+            stdout=open(os.devnull, "wb"),
+            stderr=open(os.devnull, "wb"),
+        )
+
     def run_ok(self, *args, **kwargs):
         """Same as RosePopener.run, but raise RosePopenError if ret_code != 1.
 

lib/python/rose/suite_control.py

@@ -17,85 +17,47 @@
 # You should have received a copy of the GNU General Public License
 # along with Rose. If not, see <http://www.gnu.org/licenses/>.
 # -----------------------------------------------------------------------------
-"""Launch suite engine's control commands from the correct suite host."""
+"""Launch suite engine's control commands."""
 
 import os
-import rose.config
 from rose.fs_util import FileSystemUtil
-from rose.host_select import HostSelector
 from rose.opt_parse import RoseOptionParser
-from rose.popen import RosePopener
 from rose.reporter import Reporter
 from rose.suite_engine_proc import SuiteEngineProcessor
-from rose.suite_scan import SuiteScanner
 import sys
 
 
 YES = "y"
 PROMPT = "Really %s %s at %s? [" + YES + " or n (default)] "
 
 
-class SuiteNotRunningError(Exception):
-
-    """An exception raised when a suite is not running."""
-
-    def __str__(self):
-        return "%s: does not appear to be running" % (self.args)
-
-
 class SuiteControl(object):
     """Launch suite engine's control commands from the correct suite host."""
 
-    def __init__(self, event_handler=None, popen=None, suite_engine_proc=None,
-                 host_selector=None):
+    def __init__(self, event_handler=None):
         self.event_handler = event_handler
-        if popen is None:
-            popen = RosePopener(event_handler)
-        self.popen = popen
-        if suite_engine_proc is None:
-            suite_engine_proc = SuiteEngineProcessor.get_processor(
-                event_handler=event_handler, popen=popen)
-        self.suite_engine_proc = suite_engine_proc
-        if host_selector is None:
-            host_selector = HostSelector(event_handler, popen)
-        self.host_selector = host_selector
-
-    def handle_event(self, *args, **kwargs):
-        """Call self.event_handler if it is callable."""
-        if callable(self.event_handler):
-            return self.event_handler(*args, **kwargs)
-
-    def gcontrol(self, suite_name, host=None, confirm=None, stderr=None,
+        self.suite_engine_proc = SuiteEngineProcessor.get_processor(
+            event_handler=event_handler)
+
+    def gcontrol(self, suite_name, confirm=None, stderr=None,
                  stdout=None, *args):
         """Launch suite engine's control GUI.
 
         suite_name: name of the suite.
-        host: a host where the suite is running.
         args: extra arguments for the suite engine's gcontrol command.
 
         N.B. "confirm", "stderr" and "stdout" are not used. They are included
         so that this method can have the same interface as the "shutdown"
         method.
 
         """
-        engine_version = self._get_engine_version(suite_name)
-        if host:
-            host_names = [host]
-        else:
-            host_names = self.suite_engine_proc.get_suite_run_hosts(
-                None, suite_name)
-        if not host_names:
-            raise SuiteNotRunningError(suite_name)
-        for host_name in host_names:
-            self.suite_engine_proc.gcontrol(
-                suite_name, host_name, engine_version, args)
-
-    def shutdown(self, suite_name, host=None, confirm=None, stderr=None,
+        self.suite_engine_proc.gcontrol(suite_name, args)
+
+    def shutdown(self, suite_name, confirm=None, stderr=None,
                  stdout=None, *args):
         """Shutdown the suite.
 
         suite_name: the name of the suite.
-        host: a host where the suite is running.
         confirm: If specified, must be a callable with the interface
                   b = confirm("shutdown", suite_name, host). This method will
                   only issue the shutdown command to suite_name at host if b is
@@ -105,28 +67,8 @@ def shutdown(self, suite_name, host=None, confirm=None, stderr=None,
         args: extra arguments for the suite engine's shutdown command.
 
         """
-        engine_version = self._get_engine_version(suite_name)
-        if host:
-            host_names = [host]
-        else:
-            host_names = self.suite_engine_proc.get_suite_run_hosts(
-                None, suite_name)
-        if not host_names:
-            raise SuiteNotRunningError(suite_name)
-        for host_name in host_names:
-            if confirm is None or confirm("shutdown", suite_name, host_name):
-                self.suite_engine_proc.shutdown(
-                    suite_name, host_name,
-                    engine_version, args, stderr, stdout)
-
-    def _get_engine_version(self, suite_name):
-        """Return the suite engine version for starting the suite."""
-        conf_path = self.suite_engine_proc.get_suite_dir(
-            suite_name, "log", "rose-suite-run.conf")
-        if os.access(conf_path, os.F_OK | os.R_OK):
-            conf = rose.config.load(conf_path)
-            key = self.suite_engine_proc.get_version_env_name()
-            return conf.get_value(["env", key])
+        if confirm is None or confirm("shutdown", suite_name, host_name):
+            self.suite_engine_proc.shutdown(suite_name, args, stderr, stdout)
 
 
 class SuiteNotFoundError(Exception):
@@ -167,7 +109,7 @@ def main():
     argv = sys.argv[1:]
     method_name = argv.pop(0)
     opt_parser = RoseOptionParser()
-    opt_parser.add_my_options("all", "host", "name", "non_interactive")
+    opt_parser.add_my_options("all", "name", "non_interactive")
     opts, args = opt_parser.parse_args(argv)
     event_handler = Reporter(opts.verbosity - opts.quietness)
     suite_control = SuiteControl(event_handler=event_handler)
@@ -176,10 +118,10 @@ def main():
     suite_names = []
     if not opts.non_interactive:
         confirm = prompt
-    if opts.all:
-        suite_scanner = SuiteScanner(event_handler=event_handler)
-        results = suite_scanner.scan()[0]
-        suite_names = [result.name for result in results]
+    if opts.all and method_name == 'gcontrol':
+        suite_control.suite_engine_proc.gscan(args)
+    elif opts.all:
+        suite_names = suite_control.suite_engine_proc.get_running_suites()
     else:
         if opts.name:
             suite_names.append(opts.name)
@@ -193,12 +135,11 @@ def main():
 
     if opts.debug_mode:
         for sname in suite_names:
-            method(sname, opts.host, confirm, sys.stderr, sys.stdout, *args)
+            method(sname, confirm, sys.stderr, sys.stdout, *args)
     else:
         for sname in suite_names:
             try:
-                method(sname, opts.host, confirm, sys.stderr, sys.stdout,
-                       *args)
+                method(sname, confirm, sys.stderr, sys.stdout, *args)
             except Exception as exc:
                 event_handler(exc)
                 sys.exit(1)