Workflow-supplied options to be checked if already given in the config (

#4773)

* Check whether an option has already been given in the config file when attempting to add options from the workflow generator

* Some CC fixes

* move add_opt checks into pegasus_workflow.py

* reimplement CC change

* missed underscores

* Fix to re-quieten some inherited logging

* CC

pycbc/workflow/coincidence.py

@@ -102,7 +102,11 @@ def create_node(self, trig_files, bank_file, stat_files, veto_file,
         node.add_input_opt('--template-bank', bank_file)
         node.add_input_list_opt('--trigger-files', trig_files)
         if len(stat_files) > 0:
-            node.add_input_list_opt('--statistic-files', stat_files)
+            node.add_input_list_opt(
+                '--statistic-files',
+                stat_files,
+                check_existing_options=False
+            )
         if veto_file is not None:
             node.add_input_opt('--veto-files', veto_file)
             node.add_opt('--segment-name', veto_name)
@@ -127,7 +131,11 @@ def create_node(self, trig_files, bank_file, stat_files, veto_file,
         node.add_input_opt('--template-bank', bank_file)
         node.add_input_list_opt('--trigger-files', trig_files)
         if len(stat_files) > 0:
-            node.add_input_list_opt('--statistic-files', stat_files)
+            node.add_input_list_opt(
+                '--statistic-files',
+                stat_files,
+                check_existing_options=False
+            )
         if veto_file is not None:
             node.add_input_opt('--veto-files', veto_file)
             node.add_opt('--segment-name', veto_name)

pycbc/workflow/configuration.py

@@ -142,7 +142,8 @@ def resolve_url(
         import ciecplib
         # Make the scitokens logger a little quieter
         # (it is called through ciecpclib)
-        logging.getLogger('scitokens').setLevel(logger.level + 10)
+        curr_level = logging.getLogger().level
+        logging.getLogger('scitokens').setLevel(curr_level + 10)
         with ciecplib.Session() as s:
             if u.netloc in ("git.ligo.org", "code.pycbc.phy.syr.edu"):
                 # authenticate with git.ligo.org using callback

pycbc/workflow/minifollowups.py

@@ -210,7 +210,11 @@ def setup_single_det_minifollowups(workflow, single_trig_file, tmpltbank_file,
         node.add_input_opt('--foreground-censor-file', fg_file)
         node.add_opt('--foreground-segment-name', fg_name)
     if statfiles:
-        node.add_input_list_opt('--statistic-files', statfiles)
+        node.add_input_list_opt(
+            '--statistic-files',
+            statfiles,
+            check_existing_options=False,
+        )
     if tags:
         node.add_list_opt('--tags', tags)
     node.new_output_file_opt(workflow.analysis_time, '.dax', '--dax-file')
@@ -749,7 +753,11 @@ def make_sngl_ifo(workflow, sngl_file, bank_file, trigger_id, out_dir, ifo,
     node.add_opt('--trigger-id', str(trigger_id))
     node.add_opt('--instrument', ifo)
     if statfiles is not None:
-        node.add_input_list_opt('--statistic-files', statfiles)
+        node.add_input_list_opt(
+            '--statistic-files',
+            statfiles,
+            check_existing_options=False,
+        )
     if title is not None:
         node.add_opt('--title', f'"{title}"')
     node.new_output_file_opt(workflow.analysis_time, '.html', '--output-file')

pycbc/workflow/pegasus_workflow.py

@@ -206,9 +206,18 @@ def add_raw_arg(self, arg):
 
         self._raw_options += [arg]
 
-    def add_opt(self, opt, value=None):
-        """ Add a option
-        """
+    def add_opt(self, opt, value=None, check_existing_options=True, **kwargs):  # pylint:disable=unused-argument
+        """ Add an option
+        """
+        if check_existing_options and (opt in self._options
+                                       or opt in self._raw_options):
+            err_msg = (
+                "Trying to set option %s with value %s, but it "
+                "has already been provided by the configuration file. "
+                "Usually this should not be given in the config file, "
+                "but contact developers to check"
+            ) % (opt, value)
+            raise ValueError(err_msg)
         if value is not None:
             if not isinstance(value, File):
                 value = str(value)
@@ -242,40 +251,57 @@ def add_output(self, inp):
         """
         self._add_output(inp)
 
-    def add_input_opt(self, opt, inp):
+    def add_input_opt(self, opt, inp, **kwargs):
         """ Add an option that determines an input
         """
-        self.add_opt(opt, inp._dax_repr())
+        self.add_opt(opt, inp._dax_repr(), **kwargs)
         self._add_input(inp)
 
-    def add_output_opt(self, opt, out):
+    def add_output_opt(self, opt, out, **kwargs):
         """ Add an option that determines an output
         """
-        self.add_opt(opt, out._dax_repr())
+        self.add_opt(opt, out._dax_repr(), **kwargs)
         self._add_output(out)
 
-    def add_output_list_opt(self, opt, outputs):
+    def add_output_list_opt(self, opt, outputs, **kwargs):
         """ Add an option that determines a list of outputs
         """
-        self.add_opt(opt)
+        self.add_opt(opt, **kwargs)
+        # Never check existing options for list option values
+        if 'check_existing_options' in kwargs:
+            kwargs['check_existing_options'] = False
         for out in outputs:
-            self.add_opt(out)
+            self.add_opt(out, **kwargs)
             self._add_output(out)
 
-    def add_input_list_opt(self, opt, inputs):
+    def add_input_list_opt(self, opt, inputs, **kwargs):
         """ Add an option that determines a list of inputs
         """
-        self.add_opt(opt)
+        self.add_opt(opt, **kwargs)
+        # Never check existing options for list option values
+        if 'check_existing_options' in kwargs:
+            del kwargs['check_existing_options']
         for inp in inputs:
-            self.add_opt(inp)
+            self.add_opt(
+                inp,
+                check_existing_options=False,
+                **kwargs
+            )
             self._add_input(inp)
 
-    def add_list_opt(self, opt, values):
+    def add_list_opt(self, opt, values, **kwargs):
         """ Add an option with a list of non-file parameters.
         """
-        self.add_opt(opt)
+        self.add_opt(opt, **kwargs)
+        # Never check existing options for list option values
+        if 'check_existing_options' in kwargs:
+            del kwargs['check_existing_options']
         for val in values:
-            self.add_opt(val)
+            self.add_opt(
+                val,
+                check_existing_options=False,
+                **kwargs
+            )
 
     def add_input_arg(self, inp):
         """ Add an input as an argument
@@ -324,9 +350,10 @@ def __init__(self, name='my_workflow', directory=None, cache_file=None,
         # This sets the logger to one level less verbose than the root
         # (pycbc) logger
 
+        curr_level = logging.getLogger().level
         # Get the logger associated with the Pegasus workflow import
         pegasus_logger = logging.getLogger('Pegasus')
-        pegasus_logger.setLevel(logger.level + 10)
+        pegasus_logger.setLevel(curr_level + 10)
         self.name = name
         self._rc = dax.ReplicaCatalog()
         self._tc = dax.TransformationCatalog()