From b88a33c2d82d2a85e385d3eaddcfd38de04e84f9 Mon Sep 17 00:00:00 2001 From: Hilary James Oliver Date: Sun, 4 Nov 2018 23:44:47 +1300 Subject: [PATCH 1/7] Support non-standard remote run directories. --- lib/python/rose/opt_parse.py | 5 ++++ lib/python/rose/suite_engine_procs/cylc.py | 19 +++++++++++++ lib/python/rose/suite_run.py | 32 ++++++++++++++++------ 3 files changed, 48 insertions(+), 8 deletions(-) diff --git a/lib/python/rose/opt_parse.py b/lib/python/rose/opt_parse.py index f8028a3f68..5ff22a1d42 100644 --- a/lib/python/rose/opt_parse.py +++ b/lib/python/rose/opt_parse.py @@ -328,6 +328,11 @@ class RoseOptionParser(OptionParser): {"action": "store", "metavar": "NAME", "help": "Specify the suite name."}], + "run_dir": [ + ["--run-dir"], + {"action": "store", + "metavar": "DIR", + "help": "Suite run directory."}], "new_mode": [ ["--new", "-N"], {"action": "store_true", diff --git a/lib/python/rose/suite_engine_procs/cylc.py b/lib/python/rose/suite_engine_procs/cylc.py index 19501d36b6..94d388e514 100644 --- a/lib/python/rose/suite_engine_procs/cylc.py +++ b/lib/python/rose/suite_engine_procs/cylc.py @@ -67,6 +67,25 @@ def __init__(self, *args, **kwargs): self.host = None self.user = None + def get_suite_run_dir(self, suite_name, host, user): + """Get suite run directory on given host. + + Should be absolute (may contain remote $USER - known on suite host), or + relative to home (where home is the default ssh landing point) + """ + try: + out = self.popen( + "cylc", "get-global-config", "-i", + "[hosts][%s]run directory" % host)[0] + except RosePopenError: + out = self.SUITE_DIR_REL_ROOT + run_d = out.splitlines()[0] + # TODO - DOCUMENT ONLY '$USER' is replaced in 'run directory' + # (env vars would have to be evaluated on the remote host). + # (can be done for rose-suite-run remote, but not rsync!). + run_d = run_d.replace('$USER', user) + return "%s/%s" % (run_d, suite_name) + def check_global_conf_compat(self): """Raise exception on incompatible Cylc global configuration.""" expected = os.path.join("~", self.SUITE_DIR_REL_ROOT) diff --git a/lib/python/rose/suite_run.py b/lib/python/rose/suite_run.py index eb0b78f9a4..8d17aa6de3 100644 --- a/lib/python/rose/suite_run.py +++ b/lib/python/rose/suite_run.py @@ -84,6 +84,7 @@ class SuiteRunner(Runner): "log_keep", "log_name", "name", + "run_dir", "new_mode", "no_overwrite_mode", "opt_conf_keys", @@ -132,7 +133,6 @@ def run_impl(self, opts, args, uuid, work_files): if opts.remote: # opts.name always set for remote. return self._run_remote(opts, opts.name) - conf_tree = self.config_load(opts) self.fs_util.chdir(conf_tree.conf_dirs[0]) @@ -216,6 +216,7 @@ def run_impl(self, opts, args, uuid, work_files): suite_dir = os.path.join(temp_dir, suite_dir_rel) os.makedirs(suite_dir, 0o0700) else: + # TODO - LOCALHOST ONLY (leave as ~?) suite_dir = os.path.join( os.path.expanduser("~"), suite_dir_rel) @@ -358,7 +359,9 @@ def run_impl(self, opts, args, uuid, work_files): for auth in sorted(auths): host = auth if "@" in auth: - host = auth.split("@", 1)[1] + user, host = auth.split("@", 1) + else: + user = os.environ['USER'] # Remote shell command = self.popen.get_cmd("ssh", "-n", auth) # Provide ROSE_VERSION and CYLC_VERSION in the environment @@ -374,7 +377,9 @@ def run_impl(self, opts, args, uuid, work_files): "remote-rose-bin", host=host, conf_tree=conf_tree, default="rose") # Build remote "rose suite-run" command - shcommand += " %s suite-run -vv -n %s" % (rose_bin, suite_name) + shcommand += " %s suite-run -vv -n %s --run-dir=%s" % ( + rose_bin, suite_name, self.suite_engine_proc.get_suite_run_dir( + suite_name, host, user)) for key in ["new", "debug", "install-only"]: attr = key.replace("-", "_") + "_mode" if getattr(opts, attr, None) is not None: @@ -429,7 +434,14 @@ def run_impl(self, opts, args, uuid, work_files): filters = {"excludes": [], "includes": []} for name in ["", "log/", "share/", "share/cycle/", "work/"]: filters["excludes"].append(name + uuid) - target = auth + ":" + suite_dir_rel + host = auth + if "@" in auth: + user, host = auth.split("@", 1) + else: + user = os.environ['USER'] + target = (auth + ":" + + self.suite_engine_proc.get_suite_run_dir( + suite_name, host, user)) cmd = self._get_cmd_rsync(target, **filters) proc_queue.append( [self.popen.run_bg(*cmd), cmd, "rsync", auth]) @@ -496,15 +508,19 @@ def _run_conf( def _run_init_dir(self, opts, suite_name, conf_tree=None, r_opts=None, locs_conf=None): """Create the suite's directory.""" - suite_dir_rel = self._suite_dir_rel(suite_name) - home = os.path.expanduser("~") + if opts.run_dir: + suite_dir_home = opts.run_dir + else: + suite_dir_rel = self._suite_dir_rel(suite_name) + home = os.path.expanduser("~") + suite_dir_home = os.path.join(home, suite_dir_rel) + # TODO - CONSIDER root-dir etc. suite_dir_root = self._run_conf("root-dir", conf_tree=conf_tree, r_opts=r_opts) if suite_dir_root: if locs_conf is not None: locs_conf.set(["localhost", "root-dir"], suite_dir_root) suite_dir_root = env_var_process(suite_dir_root) - suite_dir_home = os.path.join(home, suite_dir_rel) if (suite_dir_root and os.path.realpath(home) != os.path.realpath(suite_dir_root)): suite_dir_real = os.path.join(suite_dir_root, suite_dir_rel) @@ -637,7 +653,7 @@ def _run_remote(self, opts, suite_name): self.fs_util.delete(suite_dir_rel) if opts.run_mode == "run" or not os.path.exists(suite_dir_rel): self._run_init_dir(opts, suite_name, r_opts=r_opts) - os.chdir(suite_dir_rel) + os.chdir(opts.run_dir) for name in ["share", "share/cycle", "work"]: uuid_file = os.path.join(name, r_opts["uuid"]) if os.path.exists(uuid_file): From edaddeba652acf5a6b3762d7dd3b51798a468371 Mon Sep 17 00:00:00 2001 From: Hilary James Oliver Date: Mon, 5 Nov 2018 22:02:51 +1300 Subject: [PATCH 2/7] Fix root-dir for non-standard run directory. --- lib/python/rose/suite_run.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/lib/python/rose/suite_run.py b/lib/python/rose/suite_run.py index 8d17aa6de3..601d73e0bf 100644 --- a/lib/python/rose/suite_run.py +++ b/lib/python/rose/suite_run.py @@ -514,7 +514,6 @@ def _run_init_dir(self, opts, suite_name, conf_tree=None, r_opts=None, suite_dir_rel = self._suite_dir_rel(suite_name) home = os.path.expanduser("~") suite_dir_home = os.path.join(home, suite_dir_rel) - # TODO - CONSIDER root-dir etc. suite_dir_root = self._run_conf("root-dir", conf_tree=conf_tree, r_opts=r_opts) if suite_dir_root: @@ -522,8 +521,12 @@ def _run_init_dir(self, opts, suite_name, conf_tree=None, r_opts=None, locs_conf.set(["localhost", "root-dir"], suite_dir_root) suite_dir_root = env_var_process(suite_dir_root) if (suite_dir_root and - os.path.realpath(home) != os.path.realpath(suite_dir_root)): - suite_dir_real = os.path.join(suite_dir_root, suite_dir_rel) + os.path.realpath(suite_dir_home) != + os.path.realpath(suite_dir_root)): + if opts.run_dir: + suite_dir_real = os.path.join(suite_dir_root, suite_name) + else: + suite_dir_real = os.path.join(suite_dir_root, suite_dir_rel) self.fs_util.makedirs(suite_dir_real) self.fs_util.symlink(suite_dir_real, suite_dir_home, opts.no_overwrite_mode) From 31df4b4298773fff12f349f649ab74a489cb9347 Mon Sep 17 00:00:00 2001 From: Hilary James Oliver Date: Mon, 5 Nov 2018 22:40:58 +1300 Subject: [PATCH 3/7] Support non-standard work directories. --- lib/python/rose/opt_parse.py | 5 +++++ lib/python/rose/suite_engine_procs/cylc.py | 19 +++++++++++++++++++ lib/python/rose/suite_run.py | 14 +++++++++++--- 3 files changed, 35 insertions(+), 3 deletions(-) diff --git a/lib/python/rose/opt_parse.py b/lib/python/rose/opt_parse.py index 5ff22a1d42..8d99ee5f2e 100644 --- a/lib/python/rose/opt_parse.py +++ b/lib/python/rose/opt_parse.py @@ -333,6 +333,11 @@ class RoseOptionParser(OptionParser): {"action": "store", "metavar": "DIR", "help": "Suite run directory."}], + "work_dir": [ + ["--work-dir"], + {"action": "store", + "metavar": "DIR", + "help": "Suite work and share directory."}], "new_mode": [ ["--new", "-N"], {"action": "store_true", diff --git a/lib/python/rose/suite_engine_procs/cylc.py b/lib/python/rose/suite_engine_procs/cylc.py index 94d388e514..25a7d4efe9 100644 --- a/lib/python/rose/suite_engine_procs/cylc.py +++ b/lib/python/rose/suite_engine_procs/cylc.py @@ -86,6 +86,25 @@ def get_suite_run_dir(self, suite_name, host, user): run_d = run_d.replace('$USER', user) return "%s/%s" % (run_d, suite_name) + def get_suite_work_dir(self, suite_name, host, user): + """Get suite run directory on given host. + + Should be absolute (may contain remote $USER - known on suite host), or + relative to home (where home is the default ssh landing point) + """ + try: + out = self.popen( + "cylc", "get-global-config", "-i", + "[hosts][%s]work directory" % host)[0] + except RosePopenError: + out = self.SUITE_DIR_REL_ROOT + wrk_d = out.splitlines()[0] + # TODO - DOCUMENT ONLY '$USER' is replaced in 'run directory' + # (env vars would have to be evaluated on the remote host). + # (can be done for rose-suite-run remote, but not rsync!). + wrk_d = wrk_d.replace('$USER', user) + return "%s/%s" % (wrk_d, suite_name) + def check_global_conf_compat(self): """Raise exception on incompatible Cylc global configuration.""" expected = os.path.join("~", self.SUITE_DIR_REL_ROOT) diff --git a/lib/python/rose/suite_run.py b/lib/python/rose/suite_run.py index 601d73e0bf..a0fbf84b7b 100644 --- a/lib/python/rose/suite_run.py +++ b/lib/python/rose/suite_run.py @@ -85,6 +85,7 @@ class SuiteRunner(Runner): "log_name", "name", "run_dir", + "work_dir", "new_mode", "no_overwrite_mode", "opt_conf_keys", @@ -377,8 +378,11 @@ def run_impl(self, opts, args, uuid, work_files): "remote-rose-bin", host=host, conf_tree=conf_tree, default="rose") # Build remote "rose suite-run" command - shcommand += " %s suite-run -vv -n %s --run-dir=%s" % ( - rose_bin, suite_name, self.suite_engine_proc.get_suite_run_dir( + shcommand += " %s suite-run -vv -n %s" % (rose_bin, suite_name) + shcommand += " --run-dir=%s --work-dir=%s" % ( + self.suite_engine_proc.get_suite_run_dir( + suite_name, host, user), + self.suite_engine_proc.get_suite_work_dir( suite_name, host, user)) for key in ["new", "debug", "install-only"]: attr = key.replace("-", "_") + "_mode" @@ -656,13 +660,17 @@ def _run_remote(self, opts, suite_name): self.fs_util.delete(suite_dir_rel) if opts.run_mode == "run" or not os.path.exists(suite_dir_rel): self._run_init_dir(opts, suite_name, r_opts=r_opts) - os.chdir(opts.run_dir) + # make work and share dirs relative to work_dir + self.fs_util.makedirs(opts.work_dir) + os.chdir(opts.work_dir) for name in ["share", "share/cycle", "work"]: uuid_file = os.path.join(name, r_opts["uuid"]) if os.path.exists(uuid_file): self.handle_event(name + "/" + r_opts["uuid"] + "\n", level=0) else: self._run_init_dir_work(opts, suite_name, name, r_opts=r_opts) + # make log dirs relative to run_dir + os.chdir(opts.run_dir) if not opts.install_only_mode: uuid_file = os.path.join("log", r_opts["uuid"]) if os.path.exists(uuid_file): From 5a78e3774450deb7d681f8192b815039420254db Mon Sep 17 00:00:00 2001 From: Hilary James Oliver Date: Tue, 6 Nov 2018 00:00:33 +1300 Subject: [PATCH 4/7] root-dir{share,work} for non-standard run dirs --- lib/python/rose/suite_run.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/lib/python/rose/suite_run.py b/lib/python/rose/suite_run.py index a0fbf84b7b..ebd8238163 100644 --- a/lib/python/rose/suite_run.py +++ b/lib/python/rose/suite_run.py @@ -618,7 +618,11 @@ def _run_init_dir_work(self, opts, suite_name, name, conf_tree=None, if locs_conf is not None: locs_conf.set(["localhost", key], item_root) item_root = env_var_process(item_root) - suite_dir_rel = self._suite_dir_rel(suite_name) + if (opts.work_dir and name == "work" or + opts.run_dir and name == "share"): + suite_dir_rel = suite_name + else: + suite_dir_rel = self._suite_dir_rel(suite_name) if os.path.isabs(item_root): item_path_source = os.path.join(item_root, suite_dir_rel, name) else: From 1e70873e98119c86742780661a4a31c7b83c25d3 Mon Sep 17 00:00:00 2001 From: Hilary James Oliver Date: Tue, 6 Nov 2018 00:20:08 +1300 Subject: [PATCH 5/7] Fix code duplication. --- lib/python/rose/suite_engine_procs/cylc.py | 35 ++++------------------ lib/python/rose/suite_run.py | 12 ++++---- 2 files changed, 12 insertions(+), 35 deletions(-) diff --git a/lib/python/rose/suite_engine_procs/cylc.py b/lib/python/rose/suite_engine_procs/cylc.py index 25a7d4efe9..a6259f7968 100644 --- a/lib/python/rose/suite_engine_procs/cylc.py +++ b/lib/python/rose/suite_engine_procs/cylc.py @@ -67,43 +67,20 @@ def __init__(self, *args, **kwargs): self.host = None self.user = None - def get_suite_run_dir(self, suite_name, host, user): - """Get suite run directory on given host. - - Should be absolute (may contain remote $USER - known on suite host), or - relative to home (where home is the default ssh landing point) - """ - try: - out = self.popen( - "cylc", "get-global-config", "-i", - "[hosts][%s]run directory" % host)[0] - except RosePopenError: - out = self.SUITE_DIR_REL_ROOT - run_d = out.splitlines()[0] - # TODO - DOCUMENT ONLY '$USER' is replaced in 'run directory' - # (env vars would have to be evaluated on the remote host). - # (can be done for rose-suite-run remote, but not rsync!). - run_d = run_d.replace('$USER', user) - return "%s/%s" % (run_d, suite_name) - - def get_suite_work_dir(self, suite_name, host, user): - """Get suite run directory on given host. - - Should be absolute (may contain remote $USER - known on suite host), or - relative to home (where home is the default ssh landing point) - """ + def get_suite_run_work_dir(self, item, suite_name, host, user): + """Get suite run or work (item) directory on given host.""" try: out = self.popen( "cylc", "get-global-config", "-i", - "[hosts][%s]work directory" % host)[0] + "[hosts][%s]%s directory" % (host, item))[0] except RosePopenError: out = self.SUITE_DIR_REL_ROOT - wrk_d = out.splitlines()[0] + the_d = out.splitlines()[0] # TODO - DOCUMENT ONLY '$USER' is replaced in 'run directory' # (env vars would have to be evaluated on the remote host). # (can be done for rose-suite-run remote, but not rsync!). - wrk_d = wrk_d.replace('$USER', user) - return "%s/%s" % (wrk_d, suite_name) + the_d = the_d.replace('$USER', user) + return "%s/%s" % (the_d, suite_name) def check_global_conf_compat(self): """Raise exception on incompatible Cylc global configuration.""" diff --git a/lib/python/rose/suite_run.py b/lib/python/rose/suite_run.py index ebd8238163..9ba96e9cb2 100644 --- a/lib/python/rose/suite_run.py +++ b/lib/python/rose/suite_run.py @@ -380,10 +380,10 @@ def run_impl(self, opts, args, uuid, work_files): # Build remote "rose suite-run" command shcommand += " %s suite-run -vv -n %s" % (rose_bin, suite_name) shcommand += " --run-dir=%s --work-dir=%s" % ( - self.suite_engine_proc.get_suite_run_dir( - suite_name, host, user), - self.suite_engine_proc.get_suite_work_dir( - suite_name, host, user)) + self.suite_engine_proc.get_suite_run_work_dir( + "run", suite_name, host, user), + self.suite_engine_proc.get_suite_run_work_dir( + "work", suite_name, host, user)) for key in ["new", "debug", "install-only"]: attr = key.replace("-", "_") + "_mode" if getattr(opts, attr, None) is not None: @@ -444,8 +444,8 @@ def run_impl(self, opts, args, uuid, work_files): else: user = os.environ['USER'] target = (auth + ":" + - self.suite_engine_proc.get_suite_run_dir( - suite_name, host, user)) + self.suite_engine_proc.get_suite_run_work_dir( + "run", suite_name, host, user)) cmd = self._get_cmd_rsync(target, **filters) proc_queue.append( [self.popen.run_bg(*cmd), cmd, "rsync", auth]) From 5e850767b22fbb68975a2f548a46d0805d323077 Mon Sep 17 00:00:00 2001 From: Hilary James Oliver Date: Tue, 6 Nov 2018 08:38:15 +1300 Subject: [PATCH 6/7] Fix style. --- lib/python/rose/suite_run.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/python/rose/suite_run.py b/lib/python/rose/suite_run.py index 9ba96e9cb2..a6e49fa86d 100644 --- a/lib/python/rose/suite_run.py +++ b/lib/python/rose/suite_run.py @@ -525,7 +525,7 @@ def _run_init_dir(self, opts, suite_name, conf_tree=None, r_opts=None, locs_conf.set(["localhost", "root-dir"], suite_dir_root) suite_dir_root = env_var_process(suite_dir_root) if (suite_dir_root and - os.path.realpath(suite_dir_home) != + os.path.realpath(suite_dir_home) != os.path.realpath(suite_dir_root)): if opts.run_dir: suite_dir_real = os.path.join(suite_dir_root, suite_name) @@ -618,10 +618,10 @@ def _run_init_dir_work(self, opts, suite_name, name, conf_tree=None, if locs_conf is not None: locs_conf.set(["localhost", key], item_root) item_root = env_var_process(item_root) - if (opts.work_dir and name == "work" or + if (opts.work_dir and name == "work" or opts.run_dir and name == "share"): suite_dir_rel = suite_name - else: + else: suite_dir_rel = self._suite_dir_rel(suite_name) if os.path.isabs(item_root): item_path_source = os.path.join(item_root, suite_dir_rel, name) From a8be1698ed2d66776eb3d0af362abfc07f8498a3 Mon Sep 17 00:00:00 2001 From: Hilary James Oliver Date: Tue, 6 Nov 2018 11:49:38 +1300 Subject: [PATCH 7/7] Fix code duplication (more). --- lib/python/rose/suite_run.py | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/lib/python/rose/suite_run.py b/lib/python/rose/suite_run.py index a6e49fa86d..67eb44554a 100644 --- a/lib/python/rose/suite_run.py +++ b/lib/python/rose/suite_run.py @@ -115,6 +115,16 @@ def __init__(self, *args, **kwargs): host_selector=self.host_selector, suite_engine_proc=self.suite_engine_proc) + @staticmethod + def _auth_split(auth): + """Return (user, host) from "[user@]host", defaulting to $USER.""" + host = auth + if "@" in auth: + user, host = auth.split("@", 1) + else: + user = os.environ['USER'] + return (user, host) + def run_impl(self, opts, args, uuid, work_files): # Log file, temporary if hasattr(self.event_handler, "contexts"): @@ -358,11 +368,7 @@ def run_impl(self, opts, args, uuid, work_files): auths = self.suite_engine_proc.get_tasks_auths(suite_name) proc_queue = [] # [[proc, command, "ssh"|"rsync", auth], ...] for auth in sorted(auths): - host = auth - if "@" in auth: - user, host = auth.split("@", 1) - else: - user = os.environ['USER'] + user, host = self._auth_split(auth) # Remote shell command = self.popen.get_cmd("ssh", "-n", auth) # Provide ROSE_VERSION and CYLC_VERSION in the environment @@ -438,11 +444,7 @@ def run_impl(self, opts, args, uuid, work_files): filters = {"excludes": [], "includes": []} for name in ["", "log/", "share/", "share/cycle/", "work/"]: filters["excludes"].append(name + uuid) - host = auth - if "@" in auth: - user, host = auth.split("@", 1) - else: - user = os.environ['USER'] + user, host = self._auth_split(auth) target = (auth + ":" + self.suite_engine_proc.get_suite_run_work_dir( "run", suite_name, host, user))