diff --git a/pycbc/workflow/coincidence.py b/pycbc/workflow/coincidence.py index 424293093ae..fdaaa19859f 100644 --- a/pycbc/workflow/coincidence.py +++ b/pycbc/workflow/coincidence.py @@ -102,7 +102,11 @@ def create_node(self, trig_files, bank_file, stat_files, veto_file, node.add_input_opt('--template-bank', bank_file) node.add_input_list_opt('--trigger-files', trig_files) if len(stat_files) > 0: - node.add_input_list_opt('--statistic-files', stat_files) + node.add_input_list_opt( + '--statistic-files', + stat_files, + check_existing_options=False + ) if veto_file is not None: node.add_input_opt('--veto-files', veto_file) node.add_opt('--segment-name', veto_name) @@ -127,7 +131,11 @@ def create_node(self, trig_files, bank_file, stat_files, veto_file, node.add_input_opt('--template-bank', bank_file) node.add_input_list_opt('--trigger-files', trig_files) if len(stat_files) > 0: - node.add_input_list_opt('--statistic-files', stat_files) + node.add_input_list_opt( + '--statistic-files', + stat_files, + check_existing_options=False + ) if veto_file is not None: node.add_input_opt('--veto-files', veto_file) node.add_opt('--segment-name', veto_name) diff --git a/pycbc/workflow/configuration.py b/pycbc/workflow/configuration.py index a115e33dea3..6fef58acbb0 100644 --- a/pycbc/workflow/configuration.py +++ b/pycbc/workflow/configuration.py @@ -142,7 +142,8 @@ def resolve_url( import ciecplib # Make the scitokens logger a little quieter # (it is called through ciecpclib) - logging.getLogger('scitokens').setLevel(logger.level + 10) + curr_level = logging.getLogger().level + logging.getLogger('scitokens').setLevel(curr_level + 10) with ciecplib.Session() as s: if u.netloc in ("git.ligo.org", "code.pycbc.phy.syr.edu"): # authenticate with git.ligo.org using callback diff --git a/pycbc/workflow/minifollowups.py b/pycbc/workflow/minifollowups.py index d9543a20bc9..b216ad70098 100644 --- a/pycbc/workflow/minifollowups.py +++ b/pycbc/workflow/minifollowups.py @@ -210,7 +210,11 @@ def setup_single_det_minifollowups(workflow, single_trig_file, tmpltbank_file, node.add_input_opt('--foreground-censor-file', fg_file) node.add_opt('--foreground-segment-name', fg_name) if statfiles: - node.add_input_list_opt('--statistic-files', statfiles) + node.add_input_list_opt( + '--statistic-files', + statfiles, + check_existing_options=False, + ) if tags: node.add_list_opt('--tags', tags) node.new_output_file_opt(workflow.analysis_time, '.dax', '--dax-file') @@ -749,7 +753,11 @@ def make_sngl_ifo(workflow, sngl_file, bank_file, trigger_id, out_dir, ifo, node.add_opt('--trigger-id', str(trigger_id)) node.add_opt('--instrument', ifo) if statfiles is not None: - node.add_input_list_opt('--statistic-files', statfiles) + node.add_input_list_opt( + '--statistic-files', + statfiles, + check_existing_options=False, + ) if title is not None: node.add_opt('--title', f'"{title}"') node.new_output_file_opt(workflow.analysis_time, '.html', '--output-file') diff --git a/pycbc/workflow/pegasus_workflow.py b/pycbc/workflow/pegasus_workflow.py index f691bfee1a9..dd41ac21419 100644 --- a/pycbc/workflow/pegasus_workflow.py +++ b/pycbc/workflow/pegasus_workflow.py @@ -206,9 +206,18 @@ def add_raw_arg(self, arg): self._raw_options += [arg] - def add_opt(self, opt, value=None): - """ Add a option - """ + def add_opt(self, opt, value=None, check_existing_options=True, **kwargs): # pylint:disable=unused-argument + """ Add an option + """ + if check_existing_options and (opt in self._options + or opt in self._raw_options): + err_msg = ( + "Trying to set option %s with value %s, but it " + "has already been provided by the configuration file. " + "Usually this should not be given in the config file, " + "but contact developers to check" + ) % (opt, value) + raise ValueError(err_msg) if value is not None: if not isinstance(value, File): value = str(value) @@ -242,40 +251,57 @@ def add_output(self, inp): """ self._add_output(inp) - def add_input_opt(self, opt, inp): + def add_input_opt(self, opt, inp, **kwargs): """ Add an option that determines an input """ - self.add_opt(opt, inp._dax_repr()) + self.add_opt(opt, inp._dax_repr(), **kwargs) self._add_input(inp) - def add_output_opt(self, opt, out): + def add_output_opt(self, opt, out, **kwargs): """ Add an option that determines an output """ - self.add_opt(opt, out._dax_repr()) + self.add_opt(opt, out._dax_repr(), **kwargs) self._add_output(out) - def add_output_list_opt(self, opt, outputs): + def add_output_list_opt(self, opt, outputs, **kwargs): """ Add an option that determines a list of outputs """ - self.add_opt(opt) + self.add_opt(opt, **kwargs) + # Never check existing options for list option values + if 'check_existing_options' in kwargs: + kwargs['check_existing_options'] = False for out in outputs: - self.add_opt(out) + self.add_opt(out, **kwargs) self._add_output(out) - def add_input_list_opt(self, opt, inputs): + def add_input_list_opt(self, opt, inputs, **kwargs): """ Add an option that determines a list of inputs """ - self.add_opt(opt) + self.add_opt(opt, **kwargs) + # Never check existing options for list option values + if 'check_existing_options' in kwargs: + del kwargs['check_existing_options'] for inp in inputs: - self.add_opt(inp) + self.add_opt( + inp, + check_existing_options=False, + **kwargs + ) self._add_input(inp) - def add_list_opt(self, opt, values): + def add_list_opt(self, opt, values, **kwargs): """ Add an option with a list of non-file parameters. """ - self.add_opt(opt) + self.add_opt(opt, **kwargs) + # Never check existing options for list option values + if 'check_existing_options' in kwargs: + del kwargs['check_existing_options'] for val in values: - self.add_opt(val) + self.add_opt( + val, + check_existing_options=False, + **kwargs + ) def add_input_arg(self, inp): """ Add an input as an argument @@ -324,9 +350,10 @@ def __init__(self, name='my_workflow', directory=None, cache_file=None, # This sets the logger to one level less verbose than the root # (pycbc) logger + curr_level = logging.getLogger().level # Get the logger associated with the Pegasus workflow import pegasus_logger = logging.getLogger('Pegasus') - pegasus_logger.setLevel(logger.level + 10) + pegasus_logger.setLevel(curr_level + 10) self.name = name self._rc = dax.ReplicaCatalog() self._tc = dax.TransformationCatalog()