diff --git a/maestrowf/datastructures/core/study.py b/maestrowf/datastructures/core/study.py index b57bb1300..9dd9dbab5 100644 --- a/maestrowf/datastructures/core/study.py +++ b/maestrowf/datastructures/core/study.py @@ -510,7 +510,7 @@ def _stage_parameterized(self, dag): # Copy the step and set to not modified. self.step_combos[step].add(step) - workspace = make_safe_path(self._out_path, step) + workspace = make_safe_path(self._out_path, *[step]) self.workspaces[step] = workspace logger.debug("Workspace: %s", workspace) @@ -527,7 +527,7 @@ def _stage_parameterized(self, dag): # If we're looking at a parameter independent match # the workspace is the folder that contains all of # the outputs of all combinations for the step. - ws = make_safe_path(self._out_path, match) + ws = make_safe_path(self._out_path, *[match]) logger.info("Found funnel workspace -- %s", ws) else: ws = self.workspaces[match] @@ -584,10 +584,11 @@ def _stage_parameterized(self, dag): combo_str = combo.get_param_string(self.used_params[step]) if self._hash_ws: workspace = make_safe_path( - self._out_path, step, md5(combo_str).hexdigest()) + self._out_path, + *[step, md5(combo_str).hexdigest()]) else: workspace = \ - make_safe_path(self._out_path, step, combo_str) + make_safe_path(self._out_path, *[step, combo_str]) logger.debug("Workspace: %s", workspace) combo_str = "{}_{}".format(step, combo_str) self.workspaces[combo_str] = workspace @@ -613,7 +614,7 @@ def _stage_parameterized(self, dag): # If we're looking at a parameter independent match # the workspace is the folder that contains all of # the outputs of all combinations for the step. - ws = make_safe_path(self._out_path, match) + ws = make_safe_path(self._out_path, *[match]) logger.info("Found funnel workspace -- %s", ws) elif not self.used_params[match]: # If it's not a funneled dependency and the match @@ -695,7 +696,7 @@ def _stage_linear(self, dag): continue # Initialize management structures. - ws = make_safe_path(self._out_path, step) + ws = make_safe_path(self._out_path, *[step]) self.workspaces[step] = ws self.depends[step] = set() # Hub dependencies are not possible in linear studies. Empty set diff --git a/maestrowf/maestro.py b/maestrowf/maestro.py index 318ec9bdc..b91f1d6cb 100644 --- a/maestrowf/maestro.py +++ b/maestrowf/maestro.py @@ -135,7 +135,7 @@ def run_study(args): out_dir = environment.remove("OUTPUT_PATH") if args.out: # If out is specified in the args, ignore OUTPUT_PATH. - output_path = os.path.abspath(make_safe_path(args.out)) + output_path = os.path.abspath(args.out) # If we are automatically launching, just set the input as yes. if os.path.exists(output_path): @@ -167,7 +167,7 @@ def run_study(args): spec.name.replace(" ", "_"), time.strftime("%Y%m%d-%H%M%S") ) - output_path = make_safe_path(out_dir, out_name) + output_path = make_safe_path(out_dir, *[out_name]) environment.add(Variable("OUTPUT_PATH", output_path)) # Now that we know outpath, set up logging. @@ -236,8 +236,7 @@ def run_study(args): raise NotImplementedError("The 'dryrun' mode is in development.") # Pickle up the DAG - pkl_path = os.path.join(path, "{}.pkl".format( - study.name.replace(" ", "_").lower())) + pkl_path = make_safe_path(path, *["{}.pkl".format(study.name)]) exec_dag.pickle(pkl_path) # If we are automatically launching, just set the input as yes. @@ -259,12 +258,15 @@ def run_study(args): return completion_status.value else: # Launch manager with nohup + log_path = make_safe_path( + study.output_path, + *["{}.txt".format(exec_dag.name)]) + cmd = ["nohup", "conductor", "-t", str(args.sleeptime), "-d", str(args.debug_lvl), path, - "&>", "{}.txt".format(os.path.join( - study.output_path, exec_dag.name))] + "&>", log_path] LOGGER.debug(" ".join(cmd)) start_process(" ".join(cmd)) @@ -385,7 +387,7 @@ def setup_logging(args, path, name): logpath = args.logpath # Otherwise, we should just output to the OUTPUT_PATH. else: - logpath = make_safe_path(path, "logs") + logpath = make_safe_path(path, *["logs"]) loglevel = args.debug_lvl * 10 @@ -394,8 +396,8 @@ def setup_logging(args, path, name): formatter = logging.Formatter(LFORMAT) ROOTLOGGER.setLevel(loglevel) - logname = make_safe_path("{}.log".format(name)) - fh = logging.FileHandler(os.path.join(logpath, logname)) + log_path = make_safe_path(logpath, *["{}.log".format(name)]) + fh = logging.FileHandler(log_path) fh.setLevel(loglevel) fh.setFormatter(formatter) ROOTLOGGER.addHandler(fh) diff --git a/maestrowf/utils.py b/maestrowf/utils.py index b22d6457e..31c14051e 100644 --- a/maestrowf/utils.py +++ b/maestrowf/utils.py @@ -150,12 +150,21 @@ def csvtable_to_dict(fstream): return table -def make_safe_path(*args): +def make_safe_path(base_path, *args): + """ + Construct a subpath that is path safe. + + :params base_path: The base path to append args to. + :params *args: Path components to join into a path. + :returns: A joined subpath with invalid characters stripped. + """ valid = "-_.() {}{}".format(string.ascii_letters, string.digits) + path = [base_path] for arg in args: arg = "".join(c for c in arg if c in valid) arg = arg.replace(" ", "_") - return os.path.join(*args) + path.append(arg) + return os.path.join(*path) def start_process(cmd, cwd=None, env=None, shell=True):