Merge pull request #2479 from awgymer/2451-schema-build-update-defaults

2451 schema build update defaults

CHANGELOG.md

@@ -39,6 +39,7 @@
 
 ### General
 
+- Update `schema build` functionality to automatically update defaults which have changed in the `nextflow.config`([#2479](https://github.com/nf-core/tools/pull/2479))
 - Change testing framework for modules and subworkflows from pytest to nf-test ([#2490](https://github.com/nf-core/tools/pull/2490))
 - `bump_version` keeps now the indentation level of the updated version entries ([#2514](https://github.com/nf-core/tools/pull/2514))
 - Run tests with Python 3.12 ([#2522](https://github.com/nf-core/tools/pull/2522)).

nf_core/schema.py

@@ -35,7 +35,7 @@ def __init__(self):
         self.pipeline_dir = None
         self.schema_filename = None
         self.schema_defaults = {}
-        self.schema_params = []
+        self.schema_params = {}
         self.input_params = {}
         self.pipeline_params = {}
         self.invalid_nextflow_config_default_parameters = {}
@@ -110,7 +110,7 @@ def load_schema(self):
         with open(self.schema_filename, "r") as fh:
             self.schema = json.load(fh)
         self.schema_defaults = {}
-        self.schema_params = []
+        self.schema_params = {}
         log.debug(f"JSON file loaded: {self.schema_filename}")
 
     def sanitise_param_default(self, param):
@@ -141,6 +141,9 @@ def sanitise_param_default(self, param):
             param["default"] = float(param["default"])
             return param
 
+        if param["default"] is None:
+            return param
+
         # Strings
         param["default"] = str(param["default"])
         return param
@@ -154,18 +157,20 @@ def get_schema_defaults(self):
         """
         # Top level schema-properties (ungrouped)
         for p_key, param in self.schema.get("properties", {}).items():
-            self.schema_params.append(p_key)
+            self.schema_params[p_key] = ("properties", p_key)
             if "default" in param:
                 param = self.sanitise_param_default(param)
-                self.schema_defaults[p_key] = param["default"]
+                if param["default"] is not None:
+                    self.schema_defaults[p_key] = param["default"]
 
         # Grouped schema properties in subschema definitions
-        for _, definition in self.schema.get("definitions", {}).items():
+        for defn_name, definition in self.schema.get("definitions", {}).items():
             for p_key, param in definition.get("properties", {}).items():
-                self.schema_params.append(p_key)
+                self.schema_params[p_key] = ("definitions", defn_name, "properties", p_key)
                 if "default" in param:
                     param = self.sanitise_param_default(param)
-                    self.schema_defaults[p_key] = param["default"]
+                    if param["default"] is not None:
+                        self.schema_defaults[p_key] = param["default"]
 
     def save_schema(self, suppress_logging=False):
         """Save a pipeline schema to a file"""
@@ -239,9 +244,9 @@ def validate_default_params(self):
         except jsonschema.exceptions.ValidationError as e:
             raise AssertionError(f"Default parameters are invalid: {e.message}")
         for param, default in self.schema_defaults.items():
-            if default in ("null", "", None, "None"):
+            if default in ("null", "", None, "None") or default is False:
                 log.warning(
-                    f"[yellow][!] Default parameter '{param}' is empty or null. It is advisable to remove the default from the schema"
+                    f"[yellow][!] Default parameter '{param}' is empty, null, or False. It is advisable to remove the default from the schema"
                 )
         log.info("[green][✓] Default parameters match schema validation")
 
@@ -762,12 +767,15 @@ def prompt_remove_schema_notfound_config(self, p_key):
     def add_schema_found_configs(self):
         """
         Add anything that's found in the Nextflow params that's missing in the pipeline schema
+        Update defaults if they have changed
         """
         params_added = []
         params_ignore = self.pipeline_params.get("validationSchemaIgnoreParams", "").strip("\"'").split(",")
         params_ignore.append("validationSchemaIgnoreParams")
         for p_key, p_val in self.pipeline_params.items():
+            s_key = self.schema_params.get(p_key)
             # Check if key is in schema parameters
+            # Key is in pipeline but not in schema or ignored from schema
             if p_key not in self.schema_params and p_key not in params_ignore:
                 if (
                     self.no_prompts
@@ -782,7 +790,35 @@ def add_schema_found_configs(self):
                     self.schema["properties"][p_key] = self.build_schema_param(p_val)
                     log.debug(f"Adding '{p_key}' to pipeline schema")
                     params_added.append(p_key)
-
+            # Param has a default that does not match the schema
+            elif p_key in self.schema_defaults and (s_def := self.schema_defaults[p_key]) != (
+                p_def := self.build_schema_param(p_val).get("default")
+            ):
+                if self.no_prompts or Confirm.ask(
+                    f":sparkles: Default for [bold]'params.{p_key}'[/] in the pipeline config does not match schema. (schema: '{s_def}' | config: '{p_def}'). "
+                    "[blue]Update pipeline schema?"
+                ):
+                    s_key_def = s_key + ("default",)
+                    if p_def is None:
+                        nf_core.utils.nested_delitem(self.schema, s_key_def)
+                        log.debug(f"Removed '{p_key}' default from pipeline schema")
+                    else:
+                        nf_core.utils.nested_setitem(self.schema, s_key_def, p_def)
+                        log.debug(f"Updating '{p_key}' default to '{p_def}' in pipeline schema")
+            # There is no default in schema but now there is a default to write
+            elif (
+                s_key
+                and (p_key not in self.schema_defaults)
+                and (p_key not in params_ignore)
+                and (p_def := self.build_schema_param(p_val).get("default"))
+            ):
+                if self.no_prompts or Confirm.ask(
+                    f":sparkles: Default for [bold]'params.{p_key}'[/] is not in schema (def='{p_def}'). "
+                    "[blue]Update pipeline schema?"
+                ):
+                    s_key_def = s_key + ("default",)
+                    nf_core.utils.nested_setitem(self.schema, s_key_def, p_def)
+                    log.debug(f"Updating '{p_key}' default to '{p_def}' in pipeline schema")
         return params_added
 
     def build_schema_param(self, p_val):
@@ -806,13 +842,15 @@ def build_schema_param(self, p_val):
             p_val = None
 
         # Booleans
-        if p_val in ["True", "False"]:
-            p_val = p_val == "True"  # Convert to bool
+        if p_val in ["true", "false", "True", "False"]:
+            p_val = p_val in ["true", "True"]  # Convert to bool
             p_type = "boolean"
 
-        p_schema = {"type": p_type, "default": p_val}
+        # Don't return a default for anything false-y except 0
+        if not p_val and not (p_val == 0 and p_val is not False):
+            return {"type": p_type}
 
-        return p_schema
+        return {"type": p_type, "default": p_val}
 
     def launch_web_builder(self):
         """

nf_core/utils.py

@@ -1150,6 +1150,33 @@ def validate_file_md5(file_name, expected_md5hex):
     return True
 
 
+def nested_setitem(d, keys, value):
+    """Sets the value in a nested dict using a list of keys to traverse
+
+    Args:
+        d (dict): the nested dictionary to traverse
+        keys (list[Any]): A list of keys to iteratively traverse
+        value (Any): The value to be set for the last key in the chain
+    """
+    current = d
+    for k in keys[:-1]:
+        current = current[k]
+    current[keys[-1]] = value
+
+
+def nested_delitem(d, keys):
+    """Deletes a key from a nested dictionary
+
+    Args:
+        d (dict): the nested dictionary to traverse
+        keys (list[Any]): A list of keys to iteratively traverse, deleting the final one
+    """
+    current = d
+    for k in keys[:-1]:
+        current = current[k]
+    del current[keys[-1]]
+
+
 @contextmanager
 def set_wd(path: Path) -> Generator[None, None, None]:
     """Sets the working directory for this context.

tests/test_utils.py

@@ -209,6 +209,20 @@ def test_validate_file_md5():
         nf_core.utils.validate_file_md5(test_file, non_hex_string)
 
 
+def test_nested_setitem():
+    d = {"a": {"b": {"c": "value"}}}
+    nf_core.utils.nested_setitem(d, ["a", "b", "c"], "value new")
+    assert d["a"]["b"]["c"] == "value new"
+    assert d == {"a": {"b": {"c": "value new"}}}
+
+
+def test_nested_delitem():
+    d = {"a": {"b": {"c": "value"}}}
+    nf_core.utils.nested_delitem(d, ["a", "b", "c"])
+    assert "c" not in d["a"]["b"]
+    assert d == {"a": {"b": {}}}
+
+
 def test_set_wd():
     with tempfile.TemporaryDirectory() as tmpdirname:
         with nf_core.utils.set_wd(tmpdirname):